From 5ab53c916c306a4b9b49279d5252e649725a2f91 Mon Sep 17 00:00:00 2001 From: Andrew Brookins Date: Wed, 10 Nov 2021 11:31:02 -0800 Subject: [PATCH] Support both sync and asyncio uses --- .gitignore | 3 +- Makefile | 4 +-- aredis_om/model/__init__.py | 2 +- aredis_om/model/migrations/migrator.py | 6 ++-- aredis_om/model/model.py | 12 ++++--- make_sync.py | 20 +++++++---- pyproject.toml | 1 + setup.py | 47 -------------------------- tests/test_hash_model.py | 11 ++++-- tests/test_json_model.py | 11 ++++-- 10 files changed, 47 insertions(+), 70 deletions(-) delete mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 3c80a8e..4c754c2 100644 --- a/.gitignore +++ b/.gitignore @@ -133,4 +133,5 @@ data .install.stamp # Sync version of the library, via Unasync -redis_om/ \ No newline at end of file +redis_om/ +tests_sync/ \ No newline at end of file diff --git a/Makefile b/Makefile index e75faef..91c0fb9 100644 --- a/Makefile +++ b/Makefile @@ -62,14 +62,14 @@ format: $(INSTALL_STAMP) sync .PHONY: test test: $(INSTALL_STAMP) sync - $(POETRY) run pytest -n auto -s -vv ./tests/ --cov-report term-missing --cov $(NAME) $(SYNC_NAME) + $(POETRY) run pytest -n auto -vv ./tests/ ./tests_sync/ --cov-report term-missing --cov $(NAME) $(SYNC_NAME) .PHONY: test_oss test_oss: $(INSTALL_STAMP) sync # Specifically tests against a local OSS Redis instance via # docker-compose.yml. Do not use this for CI testing, where we should # instead have a matrix of Docker images. - REDIS_OM_URL="redis://localhost:6381" $(POETRY) run pytest -n auto -s -vv ./tests/ --cov-report term-missing --cov $(NAME) + REDIS_OM_URL="redis://localhost:6381" $(POETRY) run pytest -n auto -vv ./tests/ ./tests_sync/ --cov-report term-missing --cov $(NAME) .PHONY: shell diff --git a/aredis_om/model/__init__.py b/aredis_om/model/__init__.py index 7df1623..736d50f 100644 --- a/aredis_om/model/__init__.py +++ b/aredis_om/model/__init__.py @@ -1,2 +1,2 @@ from .migrations.migrator import MigrationError, Migrator -from .model import EmbeddedJsonModel, Field, HashModel, JsonModel, RedisModel +from .model import EmbeddedJsonModel, Field, HashModel, JsonModel, RedisModel, NotFoundError diff --git a/aredis_om/model/migrations/migrator.py b/aredis_om/model/migrations/migrator.py index c1c5d1f..664a08a 100644 --- a/aredis_om/model/migrations/migrator.py +++ b/aredis_om/model/migrations/migrator.py @@ -6,8 +6,6 @@ from typing import List, Optional from aioredis import Redis, ResponseError -from aredis_om.model.model import model_registry - log = logging.getLogger(__name__) @@ -96,6 +94,10 @@ class Migrator: if self.module: import_submodules(self.module) + # Import this at run-time to avoid triggering import-time side effects, + # e.g. checks for RedisJSON, etc. + from aredis_om.model.model import model_registry + for name, cls in model_registry.items(): hash_key = schema_hash_key(cls.Meta.index_name) try: diff --git a/aredis_om/model/model.py b/aredis_om/model/model.py index eb65fa3..3c7fa6b 100644 --- a/aredis_om/model/model.py +++ b/aredis_om/model/model.py @@ -10,7 +10,6 @@ from functools import reduce from typing import ( AbstractSet, Any, - AsyncGenerator, Callable, Dict, List, @@ -1295,7 +1294,7 @@ class HashModel(RedisModel, abc.ABC): return self @classmethod - async def all_pks(cls) -> AsyncGenerator[str, None]: # type: ignore + async def all_pks(cls): # type: ignore key_prefix = cls.make_key(cls._meta.primary_key_pattern.format(pk="")) # TODO: We assume the key ends with the default separator, ":" -- when # we make the separator configurable, we need to update this as well. @@ -1437,13 +1436,16 @@ class HashModel(RedisModel, abc.ABC): class JsonModel(RedisModel, abc.ABC): def __init_subclass__(cls, **kwargs): - if not has_redis_json(cls.db()): + # Generate the RediSearch schema once to validate fields. + cls.redisearch_schema() + + def __init__(self, *args, **kwargs): + if not has_redis_json(self.db()): log.error( "Your Redis instance does not have the RedisJson module " "loaded. JsonModel depends on RedisJson." ) - # Generate the RediSearch schema once to validate fields. - cls.redisearch_schema() + super().__init__(*args, **kwargs) async def save(self, pipeline: Optional[Pipeline] = None) -> "JsonModel": self.check() diff --git a/make_sync.py b/make_sync.py index 79909d4..a457882 100644 --- a/make_sync.py +++ b/make_sync.py @@ -3,23 +3,29 @@ from pathlib import Path import unasync +ADDITIONAL_REPLACEMENTS = { + "aredis_om": "redis_om", + "aioredis": "redis", + ":tests.": ":tests_sync.", +} + def main(): - additional_replacements = { - "aredis_om": "redis_om", - "aioredis": "redis" - } rules = [ unasync.Rule( fromdir="/aredis_om/", todir="/redis_om/", - additional_replacements=additional_replacements, + additional_replacements=ADDITIONAL_REPLACEMENTS, + ), + unasync.Rule( + fromdir="/tests/", + todir="/tests_sync/", + additional_replacements=ADDITIONAL_REPLACEMENTS, ), ] - filepaths = [] for root, _, filenames in os.walk( - Path(__file__).absolute().parent / "aredis_om" + Path(__file__).absolute().parent ): for filename in filenames: if filename.rpartition(".")[-1] in ("py", "pyi",): diff --git a/pyproject.toml b/pyproject.toml index 9cdbec6..7b96735 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ include=[ "docs/*", "images/*", + "redis_om/**/*", ] [tool.poetry.dependencies] diff --git a/setup.py b/setup.py deleted file mode 100644 index 8491fc6..0000000 --- a/setup.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -from setuptools import setup - -packages = \ -['aredis_om', - 'aredis_om.model', - 'aredis_om.model.cli', - 'aredis_om.model.migrations'] - -package_data = \ -{'': ['*']} - -install_requires = \ -['aioredis>=2.0.0,<3.0.0', - 'click>=8.0.1,<9.0.0', - 'pptree>=3.1,<4.0', - 'pydantic>=1.8.2,<2.0.0', - 'python-dotenv>=0.19.1,<0.20.0', - 'python-ulid>=1.0.3,<2.0.0', - 'redis>=3.5.3,<4.0.0', - 'six>=1.16.0,<2.0.0', - 'types-redis>=3.5.9,<4.0.0', - 'types-six>=1.16.1,<2.0.0'] - -entry_points = \ -{'console_scripts': ['migrate = redis_om.model.cli.migrate:migrate']} - -setup_kwargs = { - 'name': 'redis-om', - 'version': '0.0.11', - 'description': 'A high-level library containing useful Redis abstractions and tools, like an ORM and leaderboard.', - 'long_description': '
\n
\n
\n Redis OM\n
\n
\n
\n\n

\n

\n Object mapping, and more, for Redis and Python\n

\n

\n\n---\n\n[![Version][version-svg]][package-url]\n[![License][license-image]][license-url]\n[![Build Status][ci-svg]][ci-url]\n\n**Redis OM Python** makes it easy to model Redis data in your Python applications.\n\n**Redis OM Python** | [Redis OM Node.js][redis-om-js] | [Redis OM Spring][redis-om-spring] | [Redis OM .NET][redis-om-dotnet]\n\n
\n Table of contents\n\nspan\n\n\n\n- [💡 Why Redis OM?](#-why-redis-om)\n- [📇 Modeling Your Data](#-modeling-your-data)\n- [✓ Validating Data With Your Model](#-validating-data-with-your-model)\n- [🔎 Rich Queries and Embedded Models](#-rich-queries-and-embedded-models)\n- [💻 Installation](#-installation)\n- [📚 Documentation](#-documentation)\n- [⛏️ Troubleshooting](#-troubleshooting)\n- [✨ So, How Do You Get RediSearch and RedisJSON?](#-so-how-do-you-get-redisearch-and-redisjson)\n- [❤️ Contributing](#-contributing)\n- [📝 License](#-license)\n\n\n\n
\n\n## 💡 Why Redis OM?\n\nRedis OM provides high-level abstractions that make it easy to model and query data in Redis with modern Python applications.\n\nThis **preview** release contains the following features:\n\n* Declarative object mapping for Redis objects\n* Declarative secondary-index generation\n* Fluent APIs for querying Redis\n\n## 📇 Modeling Your Data\n\nRedis OM contains powerful declarative models that give you data validation, serialization, and persistence to Redis.\n\nCheck out this example of modeling customer data with Redis OM. First, we create a `Customer` model:\n\n```python\nimport datetime\nfrom typing import Optional\n\nfrom pydantic import EmailStr\n\nfrom aredis_om import HashModel\n\n\nclass Customer(HashModel):\n first_name: str\n last_name: str\n email: EmailStr\n join_date: datetime.date\n age: int\n bio: Optional[str]\n```\n\nNow that we have a `Customer` model, let\'s use it to save customer data to Redis.\n\n```python\nimport datetime\nfrom typing import Optional\n\nfrom pydantic import EmailStr\n\nfrom aredis_om import HashModel\n\n\nclass Customer(HashModel):\n first_name: str\n last_name: str\n email: EmailStr\n join_date: datetime.date\n age: int\n bio: Optional[str]\n\n\n# First, we create a new `Customer` object:\nandrew = Customer(\n first_name="Andrew",\n last_name="Brookins",\n email="andrew.brookins@example.com",\n join_date=datetime.date.today(),\n age=38,\n bio="Python developer, works at Redis, Inc."\n)\n\n# The model generates a globally unique primary key automatically\n# without needing to talk to Redis.\nprint(andrew.pk)\n# > \'01FJM6PH661HCNNRC884H6K30C\'\n\n# We can save the model to Redis by calling `save()`:\nandrew.save()\n\n# To retrieve this customer with its primary key, we use `Customer.get()`:\nassert Customer.get(andrew.pk) == andrew\n```\n\n**Ready to learn more?** Check out the [getting started](docs/getting_started.md) guide.\n\nOr, continue reading to see how Redis OM makes data validation a snap.\n\n## ✓ Validating Data With Your Model\n\nRedis OM uses [Pydantic][pydantic-url] to validate data based on the type annotations you assign to fields in a model class.\n\nThis validation ensures that fields like `first_name`, which the `Customer` model marked as a `str`, are always strings. **But every Redis OM model is also a Pydantic model**, so you can use Pydantic validators like `EmailStr`, `Pattern`, and many more for complex validations!\n\nFor example, because we used the `EmailStr` type for the `email` field, we\'ll get a validation error if we try to create a `Customer` with an invalid email address:\n\n```python\nimport datetime\nfrom typing import Optional\n\nfrom pydantic import EmailStr, ValidationError\n\nfrom aredis_om import HashModel\n\n\nclass Customer(HashModel):\n first_name: str\n last_name: str\n email: EmailStr\n join_date: datetime.date\n age: int\n bio: Optional[str]\n\n\ntry:\n Customer(\n first_name="Andrew",\n last_name="Brookins",\n email="Not an email address!",\n join_date=datetime.date.today(),\n age=38,\n bio="Python developer, works at Redis, Inc."\n )\nexcept ValidationError as e:\n print(e)\n """\n pydantic.error_wrappers.ValidationError: 1 validation error for Customer\n email\n value is not a valid email address (type=value_error.email)\n """\n```\n\n**Any existing Pydantic validator should work** as a drop-in type annotation with a Redis OM model. You can also write arbitrarily complex custom validations!\n\nTo learn more, see the [documentation on data validation](docs/validation.md).\n\n## 🔎 Rich Queries and Embedded Models\n\nData modeling, validation, and saving models to Redis all work regardless of how you run Redis.\n\nNext, we\'ll show you the **rich query expressions** and **embedded models** Redis OM provides when the [RediSearch][redisearch-url] and [RedisJSON][redis-json-url] modules are installed in your Redis deployment, or you\'re using [Redis Enterprise][redis-enterprise-url].\n\n**TIP**: *Wait, what\'s a Redis module?* If you aren\'t familiar with Redis modules, review the [So, How Do You Get RediSearch and RedisJSON?](#-so-how-do-you-get-redisearch-and-redisjson) section of this README.\n\n### Querying\n\nRedis OM comes with a rich query language that allows you to query Redis with Python expressions.\n\nTo show how this works, we\'ll make a small change to the `Customer` model we defined earlier. We\'ll add `Field(index=True)` to tell Redis OM that we want to index the `last_name` and `age` fields:\n\n```python\nimport datetime\nfrom typing import Optional\n\nfrom pydantic import EmailStr\n\nfrom aredis_om import (\n Field,\n HashModel,\n Migrator\n)\nfrom aredis_om import get_redis_connection\n\n\nclass Customer(HashModel):\n first_name: str\n last_name: str = Field(index=True)\n email: EmailStr\n join_date: datetime.date\n age: int = Field(index=True)\n bio: Optional[str]\n\n\n# Now, if we use this model with a Redis deployment that has the\n# RediSearch module installed, we can run queries like the following.\n\n# Before running queries, we need to run migrations to set up the\n# indexes that Redis OM will use. You can also use the `migrate`\n# CLI tool for this!\nredis = get_redis_connection()\nMigrator(redis).run()\n\n# Find all customers with the last name "Brookins"\nCustomer.find(Customer.last_name == "Brookins").all()\n\n# Find all customers that do NOT have the last name "Brookins"\nCustomer.find(Customer.last_name != "Brookins").all()\n\n# Find all customers whose last name is "Brookins" OR whose age is \n# 100 AND whose last name is "Smith"\nCustomer.find((Customer.last_name == "Brookins") | (\n Customer.age == 100\n) & (Customer.last_name == "Smith")).all()\n```\n\nThese queries -- and more! -- are possible because **Redis OM manages indexes for you automatically**.\n\nQuerying with this index features a rich expression syntax inspired by the Django ORM, SQLAlchemy, and Peewee. We think you\'ll enjoy it!\n\nTo learn more about how to query with Redis OM, see the [documentation on querying](docs/querying.md).\n****\n### Embedded Models\n\nRedis OM can store and query **nested models** like any document database, with the speed and power you get from Redis. Let\'s see how this works.\n\nIn the next example, we\'ll define a new `Address` model and embed it within the `Customer` model.\n\n```python\nimport datetime\nfrom typing import Optional\n\nfrom aredis_om import (\n EmbeddedJsonModel,\n JsonModel,\n Field,\n Migrator,\n)\nfrom aredis_om import get_redis_connection\n\n\nclass Address(EmbeddedJsonModel):\n address_line_1: str\n address_line_2: Optional[str]\n city: str = Field(index=True)\n state: str = Field(index=True)\n country: str\n postal_code: str = Field(index=True)\n\n\nclass Customer(JsonModel):\n first_name: str = Field(index=True)\n last_name: str = Field(index=True)\n email: str = Field(index=True)\n join_date: datetime.date\n age: int = Field(index=True)\n bio: Optional[str] = Field(index=True, full_text_search=True,\n default="")\n\n # Creates an embedded model.\n address: Address\n\n\n# With these two models and a Redis deployment with the RedisJSON \n# module installed, we can run queries like the following.\n\n# Before running queries, we need to run migrations to set up the\n# indexes that Redis OM will use. You can also use the `migrate`\n# CLI tool for this!\nredis = get_redis_connection()\nMigrator(redis).run()\n\n# Find all customers who live in San Antonio, TX\nCustomer.find(Customer.address.city == "San Antonio",\n Customer.address.state == "TX")\n```\n\nTo learn more, read the [documentation on embedded models](docs/embedded.md).\n\n## 💻 Installation\n\nInstallation is simple with `pip`, Poetry, or Pipenv.\n\n```sh\n# With pip\n$ pip install redis-om\n\n# Or, using Poetry\n$ poetry add redis-om\n```\n\n## 📚 Documentation\n\nThe Redis OM documentation is available [here](docs/index.md).\n\n## ⛏️ Troubleshooting\n\nIf you run into trouble or have any questions, we\'re here to help!\n\nFirst, check the [FAQ](docs/faq.md). If you don\'t find the answer there,\nhit us up on the [Redis Discord Server](http://discord.gg/redis).\n\n## ✨ So How Do You Get RediSearch and RedisJSON?\n\nSome advanced features of Redis OM rely on core features from two source available Redis modules: [RediSearch][redisearch-url] and [RedisJSON][redis-json-url].\n\nYou can run these modules in your self-hosted Redis deployment, or you can use [Redis Enterprise][redis-enterprise-url], which includes both modules.\n\nTo learn more, read [our documentation](docs/redis_modules.md).\n\n## ❤️ Contributing\n\nWe\'d love your contributions!\n\n**Bug reports** are especially helpful at this stage of the project. [You can open a bug report on GitHub](https://github.com/redis-om/redis-om-python/issues/new).\n\nYou can also **contribute documentation** -- or just let us know if something needs more detail. [Open an issue on GitHub](https://github.com/redis-om/redis-om-python/issues/new) to get started.\n\n## 📝 License\n\nRedis OM uses the [BSD 3-Clause license][license-url].\n\n\n\n[version-svg]: https://img.shields.io/pypi/v/redis-om?style=flat-square\n[package-url]: https://pypi.org/project/redis-om/\n[ci-svg]: https://img.shields.io/github/workflow/status/redis-om/redis-om-python/python?style=flat-square\n[ci-url]: https://github.com/redis-om/redis-om-python/actions/workflows/build.yml\n[license-image]: http://img.shields.io/badge/license-MIT-green.svg?style=flat-square\n[license-url]: LICENSE\n\n\n[redis-om-website]: https://developer.redis.com\n[redis-om-js]: https://github.com/redis-om/redis-om-js\n[redis-om-dotnet]: https://github.com/redis-om/redis-om-dotnet\n[redis-om-spring]: https://github.com/redis-om/redis-om-spring\n[redisearch-url]: https://oss.redis.com/redisearch/\n[redis-json-url]: https://oss.redis.com/redisjson/\n[pydantic-url]: https://github.com/samuelcolvin/pydantic\n[ulid-url]: https://github.com/ulid/spec\n[redis-enterprise-url]: https://redis.com/try-free/\n', - 'author': 'Andrew Brookins', - 'author_email': 'andrew.brookins@redis.com', - 'maintainer': 'Andrew Brookins', - 'maintainer_email': 'andrew.brookins@redis.com', - 'url': 'https://github.com/redis-developer/redis-om-python', - 'packages': packages, - 'package_data': package_data, - 'install_requires': install_requires, - 'entry_points': entry_points, - 'python_requires': '>=3.7,<4.0', -} -from build import * -build(setup_kwargs) - -setup(**setup_kwargs) diff --git a/tests/test_hash_model.py b/tests/test_hash_model.py index f3da1d3..4a3bbb0 100644 --- a/tests/test_hash_model.py +++ b/tests/test_hash_model.py @@ -14,9 +14,12 @@ from aredis_om import ( Migrator, QueryNotSupportedError, RedisModelError, - has_redisearch, ) +# We need to run this check as sync code (during tests) even in async mode +# because we call it in the top-level module scope. +from redis_om import has_redisearch + if not has_redisearch(): pytestmark = pytest.mark.skip @@ -438,7 +441,11 @@ def test_schema(m, key_prefix): another_integer: int another_float: float + # We need to build the key prefix because it will differ based on whether + # these tests were copied into the tests_sync folder and unasynce'd. + key_prefix = Address.make_key(Address._meta.primary_key_pattern.format(pk="")) + assert ( Address.redisearch_schema() - == f"ON HASH PREFIX 1 {key_prefix}:tests.test_hash_model.Address: SCHEMA pk TAG SEPARATOR | a_string TAG SEPARATOR | a_full_text_string TAG SEPARATOR | a_full_text_string_fts TEXT an_integer NUMERIC SORTABLE a_float NUMERIC" + == f"ON HASH PREFIX 1 {key_prefix} SCHEMA pk TAG SEPARATOR | a_string TAG SEPARATOR | a_full_text_string TAG SEPARATOR | a_full_text_string_fts TEXT an_integer NUMERIC SORTABLE a_float NUMERIC" ) diff --git a/tests/test_json_model.py b/tests/test_json_model.py index 91fc918..5d4b990 100644 --- a/tests/test_json_model.py +++ b/tests/test_json_model.py @@ -16,9 +16,12 @@ from aredis_om import ( NotFoundError, QueryNotSupportedError, RedisModelError, - has_redis_json, ) +# We need to run this check as sync code (during tests) even in async mode +# because we call it in the top-level module scope. +from redis_om import has_redis_json + if not has_redis_json(): pytestmark = pytest.mark.skip @@ -148,7 +151,6 @@ async def test_validates_field(address, m): ) -# Passes validation @pytest.mark.asyncio async def test_validation_passes(address, m): member = m.Member( @@ -658,7 +660,10 @@ async def test_list_field_limitations(m, redis): @pytest.mark.asyncio async def test_schema(m, key_prefix): + # We need to build the key prefix because it will differ based on whether + # these tests were copied into the tests_sync folder and unasynce'd. + key_prefix = m.Member.make_key(m.Member._meta.primary_key_pattern.format(pk="")) assert ( m.Member.redisearch_schema() - == f"ON JSON PREFIX 1 {key_prefix}:tests.test_json_model.Member: SCHEMA $.pk AS pk TAG SEPARATOR | $.first_name AS first_name TAG SEPARATOR | $.last_name AS last_name TAG SEPARATOR | $.email AS email TAG SEPARATOR | $.age AS age NUMERIC $.bio AS bio TAG SEPARATOR | $.bio AS bio_fts TEXT $.address.pk AS address_pk TAG SEPARATOR | $.address.city AS address_city TAG SEPARATOR | $.address.postal_code AS address_postal_code TAG SEPARATOR | $.address.note.pk AS address_note_pk TAG SEPARATOR | $.address.note.description AS address_note_description TAG SEPARATOR | $.orders[*].pk AS orders_pk TAG SEPARATOR | $.orders[*].items[*].pk AS orders_items_pk TAG SEPARATOR | $.orders[*].items[*].name AS orders_items_name TAG SEPARATOR |" + == f"ON JSON PREFIX 1 {key_prefix} SCHEMA $.pk AS pk TAG SEPARATOR | $.first_name AS first_name TAG SEPARATOR | $.last_name AS last_name TAG SEPARATOR | $.email AS email TAG SEPARATOR | $.age AS age NUMERIC $.bio AS bio TAG SEPARATOR | $.bio AS bio_fts TEXT $.address.pk AS address_pk TAG SEPARATOR | $.address.city AS address_city TAG SEPARATOR | $.address.postal_code AS address_postal_code TAG SEPARATOR | $.address.note.pk AS address_note_pk TAG SEPARATOR | $.address.note.description AS address_note_description TAG SEPARATOR | $.orders[*].pk AS orders_pk TAG SEPARATOR | $.orders[*].items[*].pk AS orders_items_pk TAG SEPARATOR | $.orders[*].items[*].name AS orders_items_name TAG SEPARATOR |" )