From 7c0bea751b029a699ec9f5d20f8a64f240f7b7d7 Mon Sep 17 00:00:00 2001 From: Andrew Brookins Date: Fri, 22 Oct 2021 08:31:08 -0700 Subject: [PATCH] WIP on README --- README.md | 156 ++++++++++++++++++++-------- demo.py | 72 +++++++++++++ docs/redis_modules.md | 30 ++++++ poetry.lock | 37 ++++++- pyproject.toml | 1 + redis_om/model/model.py | 22 ++-- tests/test_pydantic_integrations.py | 50 +++++++++ 7 files changed, 319 insertions(+), 49 deletions(-) create mode 100644 demo.py create mode 100644 docs/redis_modules.md create mode 100644 tests/test_pydantic_integrations.py diff --git a/README.md b/README.md index 950ad06..63ae933 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,118 @@ This *preview release* includes our first major component: a **declarative model ### Object Mapping -With Redis OM, you get powerful data modeling, validation, and query expressions with a small amount of code. +With Redis OM, you get powerful data modeling, extensible data validation with [Pydantic](pydantic-url), and rich query expressions with a small amount of code. + +Check out this example of data modeling and validation. First, we're going to create a `Customer` model that we can use to save data to Redis. + +```python +import datetime +from typing import Optional + +from pydantic import EmailStr + +from redis_om.model import ( + HashModel, +) + + +class Customer(HashModel): + first_name: str + last_name: str + email: EmailStr + join_date: datetime.date + age: int + bio: Optional[str] +``` + +Here, we've defined a `Customer` model with the `HashModel` class from redis-om. This model will save data in Redis as a [Redis Hash](https://redis.io/topics/data-types). + +Next, let's see how Redis OM makes it easy to save and retrieve `Customer` data in Redis. + +```python +# We can create a new Customer object: +andrew = Customer( + first_name="Andrew", + last_name="Brookins", + email="andrew.brookins@example.com", + join_date=datetime.date.today(), + age=38, + bio="Python developer, works at Redis, Inc." +) + +# The model generates a globally unique primary key automatically without +# needing to talk to Redis. +print(andrew.pk) +# '01FJM6PH661HCNNRC884H6K30C' + +# We can save the model to Redis. +andrew.save() + +# Now, we can retrieve this customer with its primary key: +other_andrew = Customer.get('01FJM6PH661HCNNRC884H6K30C') + +# The original model and this one pass an equality check. +assert other_andrew == andrew +``` + +Now, let's talk about **validation**. Did you notice the type annotation for the `email` field was `EmailStr`? + +`EmailStr` is a [Pydantic field validator](https://pydantic-docs.helpmanual.io/usage/types/). Because every Redis OM model is also a Pydantic model, you can use Pydantic validators like `EmailStr`, `Pattern`, and many more! + +Let's see what happens if we try to instantiate our `Customer` class with an invalid email address. + +```python +# We'll get a validation error if we try to use an invalid email address! +Customer( + first_name="Andrew", + last_name="Brookins", + email="Not an email address!", + join_date=datetime.date.today(), + age=38, + bio="Python developer, works at Redis, Inc." +) +# Traceback: +# pydantic.error_wrappers.ValidationError: 1 validation error for Customer +# email +# value is not a valid email address (type=value_error.email) + +# We'll also get a validation error if we try to save a model +# instance with an invalid email. +andrew = Customer( + first_name="Andrew", + last_name="Brookins", + email="andrew.brookins@example.com", + join_date=datetime.date.today(), + age=38, + bio="Python developer, works at Redis, Inc." +) + +# Sometime later... +andrew.email = "Not valid" +andrew.save() + +# Traceback: +# pydantic.error_wrappers.ValidationError: 1 validation error for Customer +# email +# value is not a valid email address (type=value_error.email) +``` + +Data modeling, validation, and persistent to Redis all work regardless of where you run Redis. But can we do more? + +Yes, we can! Next, we'll talk about the **rich query expressions** and **embedded models** that Redis OM gives you when you're using the RediSearch and RedisJSON Redis modules. + +### Querying +Querying uses a rich expression syntax inspired by the Django ORM, SQLAlchemy, and Peewee. + +The example code defines `Address` and `Customer` models for use with a Redis database with the [RedisJSON](redis-json-url) module installed. + +With these two classes defined, you can now: + +* Validate data based on the model's type annotations using Pydantic +* Persist model instances to Redis as JSON +* Instantiate model instances from Redis by primary key (a client-generated [ULID](ulid-url)) +* Query on any indexed fields in the models -Check out this example: ```python import datetime @@ -80,18 +189,6 @@ class Customer(JsonModel): address: Address ``` -The example code defines `Address` and `Customer` models for use with a Redis database with the [RedisJSON](redis-json-url) module installed. - -With these two classes defined, you can now: - -* Validate data based on the model's type annotations using [Pydantic](pydantic-url) -* Persist model instances to Redis as JSON -* Instantiate model instances from Redis by primary key (a client-generated [ULID](ulid-url)) -* Query on any indexed fields in the models - -### Querying -Querying uses a rich expression syntax inspired by the Django ORM, SQLAlchemy, and Peewee. - Here are a few example queries that use the models we defined earlier: ```python @@ -139,34 +236,9 @@ hit us up on the [Redis Discord Server](http://discord.gg/redis). ## ✨ RediSearch and RedisJSON -Redis OM relies on core features from two source available Redis modules: **RediSearch** and **RedisJSON**. +Some advanced features of Redis OM rely on core features from two source available Redis modules: **RediSearch** and **RedisJSON**. -These modules are the "magic" behind the scenes: - -* RediSearch adds querying, indexing, and full-text search to Redis -* RedisJSON adds the JSON data type to Redis - -### Why this is important - -Without RediSearch or RedisJSON installed, you can still use Redis OM to create declarative models backed by Redis. - -We'll store your model data in Redis as Hashes, and you can retrieve models using their primary keys. You'll also get all the validation features from Pydantic. - -So, what won't work without these modules? - -1. Without RedisJSON, you won't be able to nest models inside each other, like we did with the example model of a `Customer` model that has an `Address` embedded inside it. -2. Without RediSearch, you won't be able to use our expressive queries to find models -- just primary keys. - -### So how do you get RediSearch and RedisJSON? - -You can use RediSearch and RedisJSON with your self-hosted Redis deployment. Just follow the instructions on installing the binary versions of the modules in their Quick Start Guides: - -- [RedisJSON Quick Start - Running Binaries](https://oss.redis.com/redisjson/#download-and-running-binaries) -- [RediSearch Quick Start - Running Binaries](https://oss.redis.com/redisearch/Quick_Start/#download_and_running_binaries) - -**NOTE**: Both Quick Start Guides also have instructions on how to run these modules in Redis with Docker. - -Don't want to run Redis yourself? RediSearch and RedisJSON are also available on Redis Cloud. [Get started here.](https://redis.com/try-free/) +To learn more, read [our documentation](docs/redis_modules.md). ## ❤️ Contributing @@ -178,7 +250,7 @@ You can also **contribute documentation** -- or just let us know if something ne ## License -Redis OM is [MIT licensed][license-url]. +Redis OM uses the [BSD 3-Clause license][license-url]. diff --git a/demo.py b/demo.py new file mode 100644 index 0000000..b3cef61 --- /dev/null +++ b/demo.py @@ -0,0 +1,72 @@ +import datetime +from typing import Optional + +from pydantic import EmailStr + +from redis_om.model import ( + HashModel +) + + +class Customer(HashModel): + first_name: str + last_name: str + email: EmailStr + join_date: datetime.date + age: int + bio: Optional[str] + + +# Now we can create new Customer objects: +andrew = Customer( + first_name="Andrew", + last_name="Brookins", + email="andrew.brookins@example.com", + join_date=datetime.date.today(), + age=38, + bio="Python developer, works at Redis, Inc." +) + +# The model generates a globally unique primary key automatically. +print(andrew.pk) +# '01FJM6PH661HCNNRC884H6K30C' + +# You can save the model to Redis. +andrew.save() + +# Later, you can retrieve this customer with its primary key: +other_andrew = Customer.get('01FJM6PH661HCNNRC884H6K30C') + +# The original model and this one pass an equality check. +assert other_andrew == andrew + + +# We'll get a validation error if we try to use an invalid email address! +Customer( + first_name="Andrew", + last_name="Brookins", + email="Not an email address!", + join_date=datetime.date.today(), + age=38, + bio="Python developer, works at Redis, Inc." +) + +# We'll also get a validation error if we try to save a model +# instance with an invalid email. +andrew = Customer( + first_name="Andrew", + last_name="Brookins", + email="andrew.brookins@example.com", + join_date=datetime.date.today(), + age=38, + bio="Python developer, works at Redis, Inc." +) + +# Sometime later... +andrew.email = "Not valid" +andrew.save() + +# Traceback: +# pydantic.error_wrappers.ValidationError: 1 validation error for Customer +# email +# value is not a valid email address (type=value_error.email) diff --git a/docs/redis_modules.md b/docs/redis_modules.md new file mode 100644 index 0000000..cd110dc --- /dev/null +++ b/docs/redis_modules.md @@ -0,0 +1,30 @@ +# Redis Modules + +Some advanced features of Redis OM, like rich query expressions and saving data as JSON, rely on core features from two source available Redis modules: **RediSearch** and **RedisJSON**. + +These modules are the "magic" behind the scenes: + +* RediSearch adds querying, indexing, and full-text search to Redis +* RedisJSON adds the JSON data type to Redis + +## Why this is important + +Without RediSearch or RedisJSON installed, you can still use Redis OM to create declarative models backed by Redis. + +We'll store your model data in Redis as Hashes, and you can retrieve models using their primary keys. You'll also get all the validation features from Pydantic. + +So, what won't work without these modules? + +1. Without RedisJSON, you won't be able to nest models inside each other, like we did with the example model of a `Customer` model that has an `Address` embedded inside it. +2. Without RediSearch, you won't be able to use our expressive queries to find models -- just primary keys. + +## So how do you get RediSearch and RedisJSON? + +You can use RediSearch and RedisJSON with your self-hosted Redis deployment. Just follow the instructions on installing the binary versions of the modules in their Quick Start Guides: + +- [RedisJSON Quick Start - Running Binaries](https://oss.redis.com/redisjson/#download-and-running-binaries) +- [RediSearch Quick Start - Running Binaries](https://oss.redis.com/redisearch/Quick_Start/#download_and_running_binaries) + +**NOTE**: Both Quick Start Guides also have instructions on how to run these modules in Redis with Docker. + +Don't want to run Redis yourself? RediSearch and RedisJSON are also available on Redis Cloud. [Get started here.](https://redis.com/try-free/) \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index c6d54c5..0ef59fd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -217,6 +217,21 @@ category = "dev" optional = false python-versions = ">=3.5" +[[package]] +name = "dnspython" +version = "2.1.0" +description = "DNS toolkit" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +dnssec = ["cryptography (>=2.6)"] +doh = ["requests", "requests-toolbelt"] +idna = ["idna (>=2.1)"] +curio = ["curio (>=1.2)", "sniffio (>=1.1)"] +trio = ["trio (>=0.14.0)", "sniffio (>=1.1)"] + [[package]] name = "docutils" version = "0.17.1" @@ -225,6 +240,18 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "email-validator" +version = "1.1.3" +description = "A robust email syntax and deliverability validation library for Python 2.x/3.x." +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.dependencies] +dnspython = ">=1.15.0" +idna = ">=2.0.0" + [[package]] name = "execnet" version = "1.9.0" @@ -1013,7 +1040,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "6b31dc9e814263cba72d68f76b04bc3af635c685c6281a29e962d53cb05b2d0f" +content-hash = "de30b2382aaeb2fe0675658bce5a3e5bc21a14e85c66d94a90054bc73a7831cd" [metadata.files] aioredis = [ @@ -1185,10 +1212,18 @@ decorator = [ {file = "decorator-5.1.0-py3-none-any.whl", hash = "sha256:7b12e7c3c6ab203a29e157335e9122cb03de9ab7264b137594103fd4a683b374"}, {file = "decorator-5.1.0.tar.gz", hash = "sha256:e59913af105b9860aa2c8d3272d9de5a56a4e608db9a2f167a8480b323d529a7"}, ] +dnspython = [ + {file = "dnspython-2.1.0-py3-none-any.whl", hash = "sha256:95d12f6ef0317118d2a1a6fc49aac65ffec7eb8087474158f42f26a639135216"}, + {file = "dnspython-2.1.0.zip", hash = "sha256:e4a87f0b573201a0f3727fa18a516b055fd1107e0e5477cded4a2de497df1dd4"}, +] docutils = [ {file = "docutils-0.17.1-py2.py3-none-any.whl", hash = "sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61"}, {file = "docutils-0.17.1.tar.gz", hash = "sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125"}, ] +email-validator = [ + {file = "email_validator-1.1.3-py2.py3-none-any.whl", hash = "sha256:5675c8ceb7106a37e40e2698a57c056756bf3f272cfa8682a4f87ebd95d8440b"}, + {file = "email_validator-1.1.3.tar.gz", hash = "sha256:aa237a65f6f4da067119b7df3f13e89c25c051327b2b5b66dc075f33d62480d7"}, +] execnet = [ {file = "execnet-1.9.0-py2.py3-none-any.whl", hash = "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"}, {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, diff --git a/pyproject.toml b/pyproject.toml index 477edad..8859537 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ coverage = "^6.0.2" pytest-cov = "^3.0.0" pytest-xdist = "^2.4.0" twine = "^3.4.2" +email-validator = "^1.1.3" [tool.poetry.scripts] migrate = "redis_om.model.cli.migrate:migrate" diff --git a/redis_om/model/model.py b/redis_om/model/model.py index 84f6c9c..3031bf9 100644 --- a/redis_om/model/model.py +++ b/redis_om/model/model.py @@ -31,7 +31,7 @@ import redis from pydantic import BaseModel, validator from pydantic.fields import FieldInfo as PydanticFieldInfo from pydantic.fields import ModelField, Undefined, UndefinedType -from pydantic.main import ModelMetaclass +from pydantic.main import ModelMetaclass, validate_model from pydantic.typing import NoArgAnyCallable from pydantic.utils import Representation from redis.client import Pipeline @@ -1162,15 +1162,16 @@ class RedisModel(BaseModel, abc.ABC, metaclass=ModelMeta): # TODO: Add transaction support return [model.save() for model in models] - @classmethod - def values(cls): - """Return raw values from Redis instead of model instances.""" - raise NotImplementedError - @classmethod def redisearch_schema(cls): raise NotImplementedError + def check(self): + """Run all validations.""" + *_, validation_error = validate_model(self.__class__, self.__dict__) + if validation_error: + raise validation_error + class HashModel(RedisModel, abc.ABC): def __init_subclass__(cls, **kwargs): @@ -1190,6 +1191,7 @@ class HashModel(RedisModel, abc.ABC): ) def save(self, pipeline: Optional[Pipeline] = None) -> "HashModel": + self.check() if pipeline is None: db = self.db() else: @@ -1226,6 +1228,12 @@ class HashModel(RedisModel, abc.ABC): schema_parts = [schema_prefix] + cls.schema_for_fields() return " ".join(schema_parts) + def update(self, **field_values): + validate_model_fields(self.__class__, field_values) + for field, value in field_values.items(): + setattr(self, field, value) + self.save() + @classmethod def schema_for_fields(cls): schema_parts = [] @@ -1312,6 +1320,7 @@ class JsonModel(RedisModel, abc.ABC): cls.redisearch_schema() def save(self, pipeline: Optional[Pipeline] = None) -> "JsonModel": + self.check() if pipeline is None: db = self.db() else: @@ -1320,6 +1329,7 @@ class JsonModel(RedisModel, abc.ABC): return self def update(self, **field_values): + # TODO: Better support for embedded field models. validate_model_fields(self.__class__, field_values) for field, value in field_values.items(): setattr(self, field, value) diff --git a/tests/test_pydantic_integrations.py b/tests/test_pydantic_integrations.py new file mode 100644 index 0000000..d2cf647 --- /dev/null +++ b/tests/test_pydantic_integrations.py @@ -0,0 +1,50 @@ +import abc +import datetime +from collections import namedtuple + +import pytest +from pydantic import EmailStr, ValidationError + +from redis_om.model import HashModel, Field +from redis_om.model.migrations.migrator import Migrator + + +today = datetime.date.today() + + +@pytest.fixture +def m(key_prefix): + class BaseHashModel(HashModel, abc.ABC): + class Meta: + global_key_prefix = key_prefix + + class Member(BaseHashModel): + first_name: str + last_name: str + email: EmailStr = Field(index=True) + join_date: datetime.date + age: int + + Migrator().run() + + return namedtuple("Models", ["Member"])(Member) + + +def test_email_str(m): + with pytest.raises(ValidationError): + m.Member( + first_name="Andrew", + last_name="Brookins", + email="not an email!", + age=38, + join_date=today, + ) + + with pytest.raises(ValidationError): + m.Member( + first_name="Andrew", + last_name="Brookins", + email="andrew@bad-domain", + age=38, + join_date=today, + ) \ No newline at end of file