From ccad3de32df8eee9b8c365c6150b132f9f2d4ffa Mon Sep 17 00:00:00 2001 From: Andrew Brookins Date: Mon, 30 Aug 2021 18:08:07 -0700 Subject: [PATCH] WIP on basic non-relational model functionality --- .gitignore | 130 ++++++++ README.md | 206 ++++++++++++ poetry.lock | 495 +++++++++++++++++++++++++++++ pyproject.toml | 20 ++ redis_developer/__init__.py | 0 redis_developer/orm/__init__.py | 5 + redis_developer/orm/connections.py | 5 + redis_developer/orm/encoders.py | 179 +++++++++++ redis_developer/orm/model.py | 390 +++++++++++++++++++++++ redis_developer/orm/util.py | 71 +++++ test.py | 165 ++++++++++ 11 files changed, 1666 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 poetry.lock create mode 100644 pyproject.toml create mode 100644 redis_developer/__init__.py create mode 100644 redis_developer/orm/__init__.py create mode 100644 redis_developer/orm/connections.py create mode 100644 redis_developer/orm/encoders.py create mode 100644 redis_developer/orm/model.py create mode 100644 redis_developer/orm/util.py create mode 100644 test.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1d4e9fe --- /dev/null +++ b/.gitignore @@ -0,0 +1,130 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +data diff --git a/README.md b/README.md new file mode 100644 index 0000000..9d9fafc --- /dev/null +++ b/README.md @@ -0,0 +1,206 @@ +# Redis Developer Python + +redis-developer-python is a high-level library containing useful Redis +abstractions and tools, like an ORM and leaderboard. + + +## ORM/ODM + +redis-developer-python includes an ORM/ODM. + + +### Declarative model classes + +```pyhon +import decimal +import datetime +from typing import Optional + +from redis import Redis + +from redis_developer.orm import ( + RedisModel, + Field, + Relationship +) + +db = Redis() + + +# Declarative model classes +class BaseModel(RedisModel): + config: + database = db + + +class Address(BaseModel): + address_line_1: str + address_line_2: str + city: str + country: str + postal_code: str + + +class Order(BaseModel): + total: decimal.Decimal + currency: str + created_on: datetime.datetime + + +class Member(BaseModel): + # An auto-incrementing primary key is added by default if no primary key + # is specified. + id: Optional[int] = Field(default=None, primary_key=True) + first_name: str + last_name: str + email: str = Field(unique=True, index=True) + zipcode: Optional[int] + join_date: datetime.date + + # Creates an embedded document: stored as hash fields or JSON document. + address: Address + + # Creates a relationship to data in separate Hash or JSON documents. + orders: Relationship(Order, backref='recommended', + field_name='recommended_by') + + # Creates a self-relationship. + recommended_by: Relationship('Member', backref='recommended', + field_name='recommended_by') + + class Meta: + key_pattern = "member:{id}" + + +# Validation + +# Raises ValidationError: last_name is required +Member( + first_name="Andrew", + zipcode="97086", + join_date=datetime.date.today() +) + +# Passes validation +Member( + first_name="Andrew", + last_name="Brookins", + zipcode="97086", + join_date=datetime.date.today() +) + +# Raises ValidationError: zipcode is not a number +Member( + first_name="Andrew", + last_name="Brookins", + zipcode="not a number", + join_date=datetime.date.today() +) + +# Persist a model instance to Redis +member = Member( + first_name="Andrew", + last_name="Brookins", + zipcode="97086", + join_date=datetime.date.today() +) +# Assign the return value to get any auto-fields filled in, +# like the primary key (if an auto-incrementing integer). +member = member.save() + +# Hydrate a model instance from Redis using the primary key. +member = Member.get(d=1) + +# Hydrate a model instance from Redis using a secondary index on a unique field. +member = Member.get(email="a.m.brookins@gmail.com") + +# What if the field wasn't unique and there were two "a.m.brookins@gmail.com" +# entries? +# This would raise a MultipleObjectsReturned error: +member = Member.get(Member.email == "a.m.brookins@gmail.com") + +# What if you know there might be multiple results? Use filter(): +members = Member.filter(Member.last_name == "Brookins") + +# What if you want to only return values that don't match a query? +members = Member.exclude(last_name="Brookins") + +# You can combine filer() and exclude(): +members = Member.filter(last_name="Brookins").exclude(first_name="Andrew") +``` + + +### Serialization and validation based on model classes +### Save a model instance to Redis +### Get a single model instance from Redis +### Update a model instance in Redis +### Batch/bulk insert and updates +### Declarative index creation and automatic index management +### Declarative “primary key” +### Declarative relationships (via Sorted Sets) or Embedded documents (JSON) +### Exact-value queries on indexed fields +### Ad-hoc numeric range and full-text queries (RediSearch) +### Aggregations (RediSearch) + +### Unanswered Questions + +What's the difference between these two forms? + +```python +from redis_developer.orm import ( + RedisModel, + indexed, + unique +) + + +class Member(RedisModel): + email: unique(str) + email: indexed(str) + + # email: Indexed[str] <- Probably not possible? + # email: IndexedStr <- This is how constrained types work in Pydantic + + class Meta: + primary_key = "id" + indexes = ["email"] # <- How about this? +``` + +It appears that Pydantic uses functions when declaring the type requires some +kind of parameter. E.g., the max and min values for a constrained numeric +field. + +Indexing probably requires, in some cases, parameters... so it should be a +function, probably. And in general, function vs. class appears to be only a case +of whether parameters are required. + +1. unique() and indexed() require lots of work. +2. IndexedStr - what does that even mean exactly? +3. indexes = [] - Here, we could hook into class-level validation and add logic + to make sure that any indexed values were unique. Right? + +### Unique checking + +When is the right time to check if e.g. an email field is unique in Redis? + +If we check on instantiation of the model, we'll still need to check again when +we save the model. + + +### Field() vs constrained int, etc. + +Pydantic includes field helpers like constr, etc. that apply a schema to values. +On top of that, we'll have a Field() helper that includes options common to all +data types possible for a field. + +This is where we'll track if we should index a field, verify uniqueness, etc. +But for facts like numeric constraints, we'll rely on Pydantic. + + +### Automatic fields + +Redis doesn't have server-side automatic values, dates, etc. So we don't need to +worry about refreshing from the server to get the automatically-created values. + +As soon as someone saves a model, we, the ORM, will have created the automatic +values, so we can just set them in the model instance. diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..04271f7 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,495 @@ +[[package]] +name = "aioredis" +version = "2.0.0" +description = "asyncio (PEP 3156) Redis support" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +async-timeout = "*" +typing-extensions = "*" + +[package.extras] +hiredis = ["hiredis (>=1.0)"] + +[[package]] +name = "appnope" +version = "0.1.2" +description = "Disable App Nap on macOS >= 10.9" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "async-timeout" +version = "3.0.1" +description = "Timeout context manager for asyncio programs" +category = "main" +optional = false +python-versions = ">=3.5.3" + +[[package]] +name = "atomicwrites" +version = "1.4.0" +description = "Atomic file writes." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "attrs" +version = "21.2.0" +description = "Classes Without Boilerplate" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.extras] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] +docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] + +[[package]] +name = "backcall" +version = "0.2.0" +description = "Specifications for callback functions passed in to an API" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "colorama" +version = "0.4.4" +description = "Cross-platform colored terminal text." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "decorator" +version = "5.0.9" +description = "Decorators for Humans" +category = "dev" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "ipdb" +version = "0.13.9" +description = "IPython-enabled pdb" +category = "dev" +optional = false +python-versions = ">=2.7" + +[package.dependencies] +decorator = {version = "*", markers = "python_version > \"3.6\""} +ipython = {version = ">=7.17.0", markers = "python_version > \"3.6\""} +toml = {version = ">=0.10.2", markers = "python_version > \"3.6\""} + +[[package]] +name = "ipython" +version = "7.26.0" +description = "IPython: Productive Interactive Computing" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +appnope = {version = "*", markers = "sys_platform == \"darwin\""} +backcall = "*" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +decorator = "*" +jedi = ">=0.16" +matplotlib-inline = "*" +pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} +pickleshare = "*" +prompt-toolkit = ">=2.0.0,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.1.0" +pygments = "*" +traitlets = ">=4.2" + +[package.extras] +all = ["Sphinx (>=1.3)", "ipykernel", "ipyparallel", "ipywidgets", "nbconvert", "nbformat", "nose (>=0.10.1)", "notebook", "numpy (>=1.17)", "pygments", "qtconsole", "requests", "testpath"] +doc = ["Sphinx (>=1.3)"] +kernel = ["ipykernel"] +nbconvert = ["nbconvert"] +nbformat = ["nbformat"] +notebook = ["notebook", "ipywidgets"] +parallel = ["ipyparallel"] +qtconsole = ["qtconsole"] +test = ["nose (>=0.10.1)", "requests", "testpath", "pygments", "nbformat", "ipykernel", "numpy (>=1.17)"] + +[[package]] +name = "ipython-genutils" +version = "0.2.0" +description = "Vestigial utilities from IPython" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "jedi" +version = "0.18.0" +description = "An autocompletion tool for Python that can be used for text editors." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +parso = ">=0.8.0,<0.9.0" + +[package.extras] +qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] +testing = ["Django (<3.1)", "colorama", "docopt", "pytest (<6.0.0)"] + +[[package]] +name = "matplotlib-inline" +version = "0.1.2" +description = "Inline Matplotlib backend for Jupyter" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +traitlets = "*" + +[[package]] +name = "packaging" +version = "21.0" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pyparsing = ">=2.0.2" + +[[package]] +name = "parso" +version = "0.8.2" +description = "A Python Parser" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] +testing = ["docopt", "pytest (<6.0.0)"] + +[[package]] +name = "pexpect" +version = "4.8.0" +description = "Pexpect allows easy control of interactive console applications." +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +ptyprocess = ">=0.5" + +[[package]] +name = "pickleshare" +version = "0.7.5" +description = "Tiny 'shelve'-like database with concurrency support" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "pluggy" +version = "0.13.1" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +dev = ["pre-commit", "tox"] + +[[package]] +name = "prompt-toolkit" +version = "3.0.20" +description = "Library for building powerful interactive command lines in Python" +category = "dev" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +wcwidth = "*" + +[[package]] +name = "ptyprocess" +version = "0.7.0" +description = "Run a subprocess in a pseudo terminal" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "py" +version = "1.10.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pydantic" +version = "1.8.2" +description = "Data validation and settings management using python 3.6 type hinting" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +typing-extensions = ">=3.7.4.3" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pygments" +version = "2.10.0" +description = "Pygments is a syntax highlighting package written in Python." +category = "dev" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "pyparsing" +version = "2.4.7" +description = "Python parsing module" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "pytest" +version = "6.2.4" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<1.0.0a1" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "redis" +version = "3.5.3" +description = "Python client for Redis key-value store" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.extras] +hiredis = ["hiredis (>=0.1.3)"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "traitlets" +version = "5.0.5" +description = "Traitlets Python configuration system" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +ipython-genutils = "*" + +[package.extras] +test = ["pytest"] + +[[package]] +name = "typing-extensions" +version = "3.10.0.0" +description = "Backported and Experimental Type Hints for Python 3.5+" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "wcwidth" +version = "0.2.5" +description = "Measures the displayed width of unicode strings in a terminal" +category = "dev" +optional = false +python-versions = "*" + +[metadata] +lock-version = "1.1" +python-versions = "^3.9" +content-hash = "793053fcea2b6bd98faab2a15202160db5bffa41f7a6646ba7801dae5149c31d" + +[metadata.files] +aioredis = [ + {file = "aioredis-2.0.0-py3-none-any.whl", hash = "sha256:9921d68a3df5c5cdb0d5b49ad4fc88a4cfdd60c108325df4f0066e8410c55ffb"}, + {file = "aioredis-2.0.0.tar.gz", hash = "sha256:3a2de4b614e6a5f8e104238924294dc4e811aefbe17ddf52c04a93cbf06e67db"}, +] +appnope = [ + {file = "appnope-0.1.2-py2.py3-none-any.whl", hash = "sha256:93aa393e9d6c54c5cd570ccadd8edad61ea0c4b9ea7a01409020c9aa019eb442"}, + {file = "appnope-0.1.2.tar.gz", hash = "sha256:dd83cd4b5b460958838f6eb3000c660b1f9caf2a5b1de4264e941512f603258a"}, +] +async-timeout = [ + {file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"}, + {file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"}, +] +atomicwrites = [ + {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, + {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, +] +attrs = [ + {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, + {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, +] +backcall = [ + {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, + {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, +] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +decorator = [ + {file = "decorator-5.0.9-py3-none-any.whl", hash = "sha256:6e5c199c16f7a9f0e3a61a4a54b3d27e7dad0dbdde92b944426cb20914376323"}, + {file = "decorator-5.0.9.tar.gz", hash = "sha256:72ecfba4320a893c53f9706bebb2d55c270c1e51a28789361aa93e4a21319ed5"}, +] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] +ipdb = [ + {file = "ipdb-0.13.9.tar.gz", hash = "sha256:951bd9a64731c444fd907a5ce268543020086a697f6be08f7cc2c9a752a278c5"}, +] +ipython = [ + {file = "ipython-7.26.0-py3-none-any.whl", hash = "sha256:892743b65c21ed72b806a3a602cca408520b3200b89d1924f4b3d2cdb3692362"}, + {file = "ipython-7.26.0.tar.gz", hash = "sha256:0cff04bb042800129348701f7bd68a430a844e8fb193979c08f6c99f28bb735e"}, +] +ipython-genutils = [ + {file = "ipython_genutils-0.2.0-py2.py3-none-any.whl", hash = "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8"}, + {file = "ipython_genutils-0.2.0.tar.gz", hash = "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8"}, +] +jedi = [ + {file = "jedi-0.18.0-py2.py3-none-any.whl", hash = "sha256:18456d83f65f400ab0c2d3319e48520420ef43b23a086fdc05dff34132f0fb93"}, + {file = "jedi-0.18.0.tar.gz", hash = "sha256:92550a404bad8afed881a137ec9a461fed49eca661414be45059329614ed0707"}, +] +matplotlib-inline = [ + {file = "matplotlib-inline-0.1.2.tar.gz", hash = "sha256:f41d5ff73c9f5385775d5c0bc13b424535c8402fe70ea8210f93e11f3683993e"}, + {file = "matplotlib_inline-0.1.2-py3-none-any.whl", hash = "sha256:5cf1176f554abb4fa98cb362aa2b55c500147e4bdbb07e3fda359143e1da0811"}, +] +packaging = [ + {file = "packaging-21.0-py3-none-any.whl", hash = "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"}, + {file = "packaging-21.0.tar.gz", hash = "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7"}, +] +parso = [ + {file = "parso-0.8.2-py2.py3-none-any.whl", hash = "sha256:a8c4922db71e4fdb90e0d0bc6e50f9b273d3397925e5e60a717e719201778d22"}, + {file = "parso-0.8.2.tar.gz", hash = "sha256:12b83492c6239ce32ff5eed6d3639d6a536170723c6f3f1506869f1ace413398"}, +] +pexpect = [ + {file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"}, + {file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"}, +] +pickleshare = [ + {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"}, + {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, +] +pluggy = [ + {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, + {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, +] +prompt-toolkit = [ + {file = "prompt_toolkit-3.0.20-py3-none-any.whl", hash = "sha256:6076e46efae19b1e0ca1ec003ed37a933dc94b4d20f486235d436e64771dcd5c"}, + {file = "prompt_toolkit-3.0.20.tar.gz", hash = "sha256:eb71d5a6b72ce6db177af4a7d4d7085b99756bf656d98ffcc4fecd36850eea6c"}, +] +ptyprocess = [ + {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, + {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, +] +py = [ + {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, + {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, +] +pydantic = [ + {file = "pydantic-1.8.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:05ddfd37c1720c392f4e0d43c484217b7521558302e7069ce8d318438d297739"}, + {file = "pydantic-1.8.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a7c6002203fe2c5a1b5cbb141bb85060cbff88c2d78eccbc72d97eb7022c43e4"}, + {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:589eb6cd6361e8ac341db97602eb7f354551482368a37f4fd086c0733548308e"}, + {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:10e5622224245941efc193ad1d159887872776df7a8fd592ed746aa25d071840"}, + {file = "pydantic-1.8.2-cp36-cp36m-win_amd64.whl", hash = "sha256:99a9fc39470010c45c161a1dc584997f1feb13f689ecf645f59bb4ba623e586b"}, + {file = "pydantic-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a83db7205f60c6a86f2c44a61791d993dff4b73135df1973ecd9eed5ea0bda20"}, + {file = "pydantic-1.8.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:41b542c0b3c42dc17da70554bc6f38cbc30d7066d2c2815a94499b5684582ecb"}, + {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:ea5cb40a3b23b3265f6325727ddfc45141b08ed665458be8c6285e7b85bd73a1"}, + {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:18b5ea242dd3e62dbf89b2b0ec9ba6c7b5abaf6af85b95a97b00279f65845a23"}, + {file = "pydantic-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:234a6c19f1c14e25e362cb05c68afb7f183eb931dd3cd4605eafff055ebbf287"}, + {file = "pydantic-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:021ea0e4133e8c824775a0cfe098677acf6fa5a3cbf9206a376eed3fc09302cd"}, + {file = "pydantic-1.8.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e710876437bc07bd414ff453ac8ec63d219e7690128d925c6e82889d674bb505"}, + {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:ac8eed4ca3bd3aadc58a13c2aa93cd8a884bcf21cb019f8cfecaae3b6ce3746e"}, + {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:4a03cbbe743e9c7247ceae6f0d8898f7a64bb65800a45cbdc52d65e370570820"}, + {file = "pydantic-1.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:8621559dcf5afacf0069ed194278f35c255dc1a1385c28b32dd6c110fd6531b3"}, + {file = "pydantic-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8b223557f9510cf0bfd8b01316bf6dd281cf41826607eada99662f5e4963f316"}, + {file = "pydantic-1.8.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:244ad78eeb388a43b0c927e74d3af78008e944074b7d0f4f696ddd5b2af43c62"}, + {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:05ef5246a7ffd2ce12a619cbb29f3307b7c4509307b1b49f456657b43529dc6f"}, + {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:54cd5121383f4a461ff7644c7ca20c0419d58052db70d8791eacbbe31528916b"}, + {file = "pydantic-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:4be75bebf676a5f0f87937c6ddb061fa39cbea067240d98e298508c1bda6f3f3"}, + {file = "pydantic-1.8.2-py3-none-any.whl", hash = "sha256:fec866a0b59f372b7e776f2d7308511784dace622e0992a0b59ea3ccee0ae833"}, + {file = "pydantic-1.8.2.tar.gz", hash = "sha256:26464e57ccaafe72b7ad156fdaa4e9b9ef051f69e175dbbb463283000c05ab7b"}, +] +pygments = [ + {file = "Pygments-2.10.0-py3-none-any.whl", hash = "sha256:b8e67fe6af78f492b3c4b3e2970c0624cbf08beb1e493b2c99b9fa1b67a20380"}, + {file = "Pygments-2.10.0.tar.gz", hash = "sha256:f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6"}, +] +pyparsing = [ + {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, + {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, +] +pytest = [ + {file = "pytest-6.2.4-py3-none-any.whl", hash = "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"}, + {file = "pytest-6.2.4.tar.gz", hash = "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b"}, +] +redis = [ + {file = "redis-3.5.3-py2.py3-none-any.whl", hash = "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24"}, + {file = "redis-3.5.3.tar.gz", hash = "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2"}, +] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +traitlets = [ + {file = "traitlets-5.0.5-py3-none-any.whl", hash = "sha256:69ff3f9d5351f31a7ad80443c2674b7099df13cc41fc5fa6e2f6d3b0330b0426"}, + {file = "traitlets-5.0.5.tar.gz", hash = "sha256:178f4ce988f69189f7e523337a3e11d91c786ded9360174a3d9ca83e79bc5396"}, +] +typing-extensions = [ + {file = "typing_extensions-3.10.0.0-py2-none-any.whl", hash = "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497"}, + {file = "typing_extensions-3.10.0.0-py3-none-any.whl", hash = "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"}, + {file = "typing_extensions-3.10.0.0.tar.gz", hash = "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342"}, +] +wcwidth = [ + {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, + {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c8569a0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,20 @@ +[tool.poetry] +name = "redis-developer-python" +version = "0.1.0" +description = "A high-level library containing useful Redis abstractions and tools, like an ORM and leaderboard." +authors = ["Andrew Brookins "] +license = "MIT" + +[tool.poetry.dependencies] +python = "^3.9" +redis = "^3.5.3" +aioredis = "^2.0.0" +pydantic = "^1.8.2" + +[tool.poetry.dev-dependencies] +pytest = "^6.2.4" +ipdb = "^0.13.9" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/redis_developer/__init__.py b/redis_developer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/redis_developer/orm/__init__.py b/redis_developer/orm/__init__.py new file mode 100644 index 0000000..34a5671 --- /dev/null +++ b/redis_developer/orm/__init__.py @@ -0,0 +1,5 @@ +from .model import ( + RedisModel, + Relationship, + Field +) diff --git a/redis_developer/orm/connections.py b/redis_developer/orm/connections.py new file mode 100644 index 0000000..0124d33 --- /dev/null +++ b/redis_developer/orm/connections.py @@ -0,0 +1,5 @@ +import redis + + +def get_redis_connection() -> redis.Redis: + return redis.Redis() diff --git a/redis_developer/orm/encoders.py b/redis_developer/orm/encoders.py new file mode 100644 index 0000000..8b2fbd2 --- /dev/null +++ b/redis_developer/orm/encoders.py @@ -0,0 +1,179 @@ +""" +This file adapted from FastAPI's encoders. + +Licensed under the MIT License (MIT). + +Copyright (c) 2018 Sebastián Ramírez + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import dataclasses +from collections import defaultdict +from enum import Enum +from pathlib import PurePath +from types import GeneratorType +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union + +from pydantic import BaseModel +from pydantic.json import ENCODERS_BY_TYPE + +SetIntStr = Set[Union[int, str]] +DictIntStrAny = Dict[Union[int, str], Any] + + +def generate_encoders_by_class_tuples( + type_encoder_map: Dict[Any, Callable[[Any], Any]] +) -> Dict[Callable[[Any], Any], Tuple[Any, ...]]: + encoders_by_class_tuples: Dict[Callable[[Any], Any], Tuple[Any, ...]] = defaultdict( + tuple + ) + for type_, encoder in type_encoder_map.items(): + encoders_by_class_tuples[encoder] += (type_,) + return encoders_by_class_tuples + + +encoders_by_class_tuples = generate_encoders_by_class_tuples(ENCODERS_BY_TYPE) + + +def jsonable_encoder( + obj: Any, + include: Optional[Union[SetIntStr, DictIntStrAny]] = None, + exclude: Optional[Union[SetIntStr, DictIntStrAny]] = None, + by_alias: bool = True, + exclude_unset: bool = False, + exclude_defaults: bool = False, + exclude_none: bool = False, + custom_encoder: Dict[Any, Callable[[Any], Any]] = {}, + sqlalchemy_safe: bool = True, +) -> Any: + if include is not None and not isinstance(include, (set, dict)): + include = set(include) + if exclude is not None and not isinstance(exclude, (set, dict)): + exclude = set(exclude) + if isinstance(obj, BaseModel): + encoder = getattr(obj.__config__, "json_encoders", {}) + if custom_encoder: + encoder.update(custom_encoder) + obj_dict = obj.dict( + include=include, # type: ignore # in Pydantic + exclude=exclude, # type: ignore # in Pydantic + by_alias=by_alias, + exclude_unset=exclude_unset, + exclude_none=exclude_none, + exclude_defaults=exclude_defaults, + ) + if "__root__" in obj_dict: + obj_dict = obj_dict["__root__"] + return jsonable_encoder( + obj_dict, + exclude_none=exclude_none, + exclude_defaults=exclude_defaults, + custom_encoder=encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) + if dataclasses.is_dataclass(obj): + return dataclasses.asdict(obj) + if isinstance(obj, Enum): + return obj.value + if isinstance(obj, PurePath): + return str(obj) + if isinstance(obj, (str, int, float, type(None))): + return obj + if isinstance(obj, dict): + encoded_dict = {} + for key, value in obj.items(): + if ( + ( + not sqlalchemy_safe + or (not isinstance(key, str)) + or (not key.startswith("_sa")) + ) + and (value is not None or not exclude_none) + and ((include and key in include) or not exclude or key not in exclude) + ): + encoded_key = jsonable_encoder( + key, + by_alias=by_alias, + exclude_unset=exclude_unset, + exclude_none=exclude_none, + custom_encoder=custom_encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) + encoded_value = jsonable_encoder( + value, + by_alias=by_alias, + exclude_unset=exclude_unset, + exclude_none=exclude_none, + custom_encoder=custom_encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) + encoded_dict[encoded_key] = encoded_value + return encoded_dict + if isinstance(obj, (list, set, frozenset, GeneratorType, tuple)): + encoded_list = [] + for item in obj: + encoded_list.append( + jsonable_encoder( + item, + include=include, + exclude=exclude, + by_alias=by_alias, + exclude_unset=exclude_unset, + exclude_defaults=exclude_defaults, + exclude_none=exclude_none, + custom_encoder=custom_encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) + ) + return encoded_list + + if custom_encoder: + if type(obj) in custom_encoder: + return custom_encoder[type(obj)](obj) + else: + for encoder_type, encoder in custom_encoder.items(): + if isinstance(obj, encoder_type): + return encoder(obj) + + if type(obj) in ENCODERS_BY_TYPE: + return ENCODERS_BY_TYPE[type(obj)](obj) + for encoder, classes_tuple in encoders_by_class_tuples.items(): + if isinstance(obj, classes_tuple): + return encoder(obj) + + errors: List[Exception] = [] + try: + data = dict(obj) + except Exception as e: + errors.append(e) + try: + data = vars(obj) + except Exception as e: + errors.append(e) + raise ValueError(errors) + return jsonable_encoder( + data, + by_alias=by_alias, + exclude_unset=exclude_unset, + exclude_defaults=exclude_defaults, + exclude_none=exclude_none, + custom_encoder=custom_encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) diff --git a/redis_developer/orm/model.py b/redis_developer/orm/model.py new file mode 100644 index 0000000..8a72309 --- /dev/null +++ b/redis_developer/orm/model.py @@ -0,0 +1,390 @@ +import datetime +from dataclasses import dataclass +from enum import Enum +from typing import ( + AbstractSet, + Any, + Callable, + Dict, + Mapping, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, + Sequence, ClassVar, TYPE_CHECKING, no_type_check, +) + +import redis +from pydantic import BaseModel +from pydantic.fields import FieldInfo as PydanticFieldInfo +from pydantic.fields import ModelField, Undefined, UndefinedType +from pydantic.main import BaseConfig, ModelMetaclass, validate_model +from pydantic.typing import NoArgAnyCallable, resolve_annotations +from pydantic.utils import Representation + +from .encoders import jsonable_encoder +from .util import uuid_from_time + +_T = TypeVar("_T") + + +class RedisModelError(Exception): + pass + + +class NotFoundError(Exception): + pass + + +class Operations(Enum): + EQ = 1 + LT = 2 + GT = 3 + + +@dataclass +class Expression: + field: ModelField + op: Operations + right_value: Any + + +class ExpressionProxy: + def __init__(self, field: ModelField): + self.field = field + + def __eq__(self, other: Any) -> Expression: + return Expression(field=self.field, op=Operations.EQ, right_value=other) + + def __lt__(self, other: Any) -> Expression: + return Expression(field=self.field, op=Operations.LT, right_value=other) + + def __gt__(self, other: Any) -> Expression: + return Expression(field=self.field, op=Operations.GT, right_value=other) + + +def __dataclass_transform__( + *, + eq_default: bool = True, + order_default: bool = False, + kw_only_default: bool = False, + field_descriptors: Tuple[Union[type, Callable[..., Any]], ...] = (()), +) -> Callable[[_T], _T]: + return lambda a: a + + +class FieldInfo(PydanticFieldInfo): + def __init__(self, default: Any = Undefined, **kwargs: Any) -> None: + primary_key = kwargs.pop("primary_key", False) + nullable = kwargs.pop("nullable", Undefined) + foreign_key = kwargs.pop("foreign_key", Undefined) + index = kwargs.pop("index", Undefined) + unique = kwargs.pop("unique", Undefined) + super().__init__(default=default, **kwargs) + self.primary_key = primary_key + self.nullable = nullable + self.foreign_key = foreign_key + self.index = index + self.unique = unique + + +class RelationshipInfo(Representation): + def __init__( + self, + *, + back_populates: Optional[str] = None, + link_model: Optional[Any] = None, + ) -> None: + self.back_populates = back_populates + self.link_model = link_model + + +def Field( + default: Any = Undefined, + *, + default_factory: Optional[NoArgAnyCallable] = None, + alias: str = None, + title: str = None, + description: str = None, + exclude: Union[ + AbstractSet[Union[int, str]], Mapping[Union[int, str], Any], Any + ] = None, + include: Union[ + AbstractSet[Union[int, str]], Mapping[Union[int, str], Any], Any + ] = None, + const: bool = None, + gt: float = None, + ge: float = None, + lt: float = None, + le: float = None, + multiple_of: float = None, + min_items: int = None, + max_items: int = None, + min_length: int = None, + max_length: int = None, + allow_mutation: bool = True, + regex: str = None, + primary_key: bool = False, + unique: bool = False, + foreign_key: Optional[Any] = None, + nullable: Union[bool, UndefinedType] = Undefined, + index: Union[bool, UndefinedType] = Undefined, + schema_extra: Optional[Dict[str, Any]] = None, +) -> Any: + current_schema_extra = schema_extra or {} + field_info = FieldInfo( + default, + default_factory=default_factory, + alias=alias, + title=title, + description=description, + exclude=exclude, + include=include, + const=const, + gt=gt, + ge=ge, + lt=lt, + le=le, + multiple_of=multiple_of, + min_items=min_items, + max_items=max_items, + min_length=min_length, + max_length=max_length, + allow_mutation=allow_mutation, + regex=regex, + primary_key=primary_key, + unique=unique, + foreign_key=foreign_key, + nullable=nullable, + index=index, + **current_schema_extra, + ) + field_info._validate() + return field_info + + +def Relationship( + *, + back_populates: Optional[str] = None, + link_model: Optional[Any] = None +) -> Any: + relationship_info = RelationshipInfo( + back_populates=back_populates, + link_model=link_model, + ) + return relationship_info + + +@__dataclass_transform__(kw_only_default=True, field_descriptors=(Field, FieldInfo)) +class RedisModelMetaclass(ModelMetaclass): + __redismodel_relationships__: Dict[str, RelationshipInfo] + __config__: Type[BaseConfig] + __fields__: Dict[str, ModelField] + + # From Pydantic + def __new__(cls, name, bases, class_dict: dict, **kwargs) -> Any: + relationships: Dict[str, RelationshipInfo] = {} + dict_for_pydantic = {} + original_annotations = resolve_annotations( + class_dict.get("__annotations__", {}), class_dict.get("__module__", None) + ) + pydantic_annotations = {} + relationship_annotations = {} + for k, v in class_dict.items(): + if isinstance(v, RelationshipInfo): + relationships[k] = v + else: + dict_for_pydantic[k] = v + for k, v in original_annotations.items(): + if k in relationships: + relationship_annotations[k] = v + else: + pydantic_annotations[k] = v + dict_used = { + **dict_for_pydantic, + "__weakref__": None, + "__redismodel_relationships__": relationships, + "__annotations__": pydantic_annotations, + } + # Duplicate logic from Pydantic to filter config kwargs because if they are + # passed directly including the registry Pydantic will pass them over to the + # superclass causing an error + allowed_config_kwargs: Set[str] = { + key + for key in dir(BaseConfig) + if not ( + key.startswith("__") and key.endswith("__") + ) # skip dunder methods and attributes + } + pydantic_kwargs = kwargs.copy() + config_kwargs = { + key: pydantic_kwargs.pop(key) + for key in pydantic_kwargs.keys() & allowed_config_kwargs + } + new_cls = super().__new__(cls, name, bases, dict_used, **config_kwargs) + new_cls.__annotations__ = { + **relationship_annotations, + **pydantic_annotations, + **new_cls.__annotations__, + } + return new_cls + + +@dataclass +class PrimaryKey: + name: str + field: ModelField + + +class DefaultMeta: + global_key_prefix: Optional[str] = None + model_key_prefix: Optional[str] = None + primary_key_pattern: Optional[str] = None + database: Optional[redis.Redis] = None + primary_key: Optional[PrimaryKey] = None + + +class RedisModel(BaseModel, metaclass=RedisModelMetaclass): + """ + TODO: Convert expressions to Redis commands, execute + TODO: Key prefix vs. "key pattern" (that's actually the primary key pattern) + TODO: Default key prefix is model name lowercase + TODO: Build primary key pattern from PK field name, model prefix + TODO: Default PK pattern is model name:pk field + """ + pk: Optional[str] = Field(default=None, primary_key=True) + + class Config: + orm_mode = True + arbitrary_types_allowed = True + extra = 'allow' + + Meta = DefaultMeta + + def __init_subclass__(cls, **kwargs): + # Create proxies for each model field so that we can use the field + # in queries, like Model.get(Model.field_name == 1) + super().__init_subclass__(**kwargs) + + for name, field in cls.__fields__.items(): + setattr(cls, name, ExpressionProxy(field)) + # Check if this is our FieldInfo version with extended ORM metadata. + if isinstance(field.field_info, FieldInfo): + if field.field_info.primary_key: + cls.Meta.primary_key = PrimaryKey(name=name, field=field) + if not hasattr(cls.Meta, 'primary_key_pattern'): + cls.Meta.primary_key_pattern = f"{cls.Meta.primary_key.name}:{{pk}}" + + def __init__(__pydantic_self__, **data: Any) -> None: + # Uses something other than `self` the first arg to allow "self" as a + # settable attribute + if TYPE_CHECKING: + __pydantic_self__.__dict__: Dict[str, Any] = {} + __pydantic_self__.__fields_set__: Set[str] = set() + + values, fields_set, validation_error = validate_model( + __pydantic_self__.__class__, data + ) + + if validation_error: + raise validation_error + + __pydantic_self__.validate_primary_key() + + object.__setattr__(__pydantic_self__, '__dict__', values) + + @classmethod + @no_type_check + def _get_value(cls, *args, **kwargs) -> Any: + """ + Always send None as an empty string. + + TODO: How broken is this? + """ + val = super()._get_value(*args, **kwargs) + if val is None: + return "" + return val + + @classmethod + def validate_primary_key(cls): + """Check for a primary key. We need one (and only one).""" + primary_keys = 0 + for name, field in cls.__fields__.items(): + if field.field_info.primary_key: + primary_keys += 1 + + # TODO: Automatically create a primary key field instead? + if primary_keys == 0: + raise RedisModelError("You must define a primary key for the model") + elif primary_keys > 1: + raise RedisModelError("You must define only one primary key for a model") + + @classmethod + def key(cls, part: str): + global_prefix = getattr(cls.Meta, 'global_key_prefix', '') + model_prefix = getattr(cls.Meta, 'model_key_prefix', '') + return f"{global_prefix}{model_prefix}{part}" + + @classmethod + def get(cls, pk: Any): + # TODO: Getting related objects + pk_pattern = cls.Meta.primary_key_pattern.format(pk=str(pk)) + print("GET ", cls.key(pk_pattern)) + document = cls.db().hgetall(cls.key(pk_pattern)) + if not document: + raise NotFoundError + return cls.parse_obj(document) + + def delete(self): + # TODO: deleting relationships + pk = self.__fields__[self.Meta.primary_key.field.name] + pk_pattern = self.Meta.primary_key_pattern.format(pk=pk) + return self.db().delete(self.key(pk_pattern)) + + @classmethod + def db(cls): + return cls.Meta.database + + @classmethod + def filter(cls, *expressions: Sequence[Expression]): + return cls + + @classmethod + def exclude(cls, *expressions: Sequence[Expression]): + return cls + + @classmethod + def add(cls, models: Sequence['RedisModel']) -> Sequence['RedisModel']: + return [model.save() for model in models] + + @classmethod + def update(cls, **field_values): + return cls + + @classmethod + def values(cls): + """Return raw values from Redis instead of model instances.""" + return cls + + def save(self) -> 'RedisModel': + pk_field = self.Meta.primary_key.field + document = jsonable_encoder(self.dict()) + pk = document[pk_field.name] + + if not pk: + pk = str(uuid_from_time(datetime.datetime.now())) + setattr(self, pk_field.name, pk) + document[pk_field.name] = pk + + pk_pattern = self.Meta.primary_key_pattern.format(pk=pk) + success = self.db().hset(self.key(pk_pattern), mapping=document) + return success + + Meta = DefaultMeta + + def __init__(self, **data: Any) -> None: + """Validate that a model instance has a primary key.""" + super().__init__(**data) diff --git a/redis_developer/orm/util.py b/redis_developer/orm/util.py new file mode 100644 index 0000000..7e44959 --- /dev/null +++ b/redis_developer/orm/util.py @@ -0,0 +1,71 @@ +# Adapted from the Cassandra Python driver. +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import calendar +import random +import uuid + + +def uuid_from_time(time_arg, node=None, clock_seq=None): + """ + Converts a datetime or timestamp to a type 1 :class:`uuid.UUID`. + + :param time_arg: + The time to use for the timestamp portion of the UUID. + This can either be a :class:`datetime` object or a timestamp + in seconds (as returned from :meth:`time.time()`). + :type datetime: :class:`datetime` or timestamp + + :param node: + None integer for the UUID (up to 48 bits). If not specified, this + field is randomized. + :type node: long + + :param clock_seq: + Clock sequence field for the UUID (up to 14 bits). If not specified, + a random sequence is generated. + :type clock_seq: int + + :rtype: :class:`uuid.UUID` + + """ + if hasattr(time_arg, 'utctimetuple'): + seconds = int(calendar.timegm(time_arg.utctimetuple())) + microseconds = (seconds * 1e6) + time_arg.time().microsecond + else: + microseconds = int(time_arg * 1e6) + + # 0x01b21dd213814000 is the number of 100-ns intervals between the + # UUID epoch 1582-10-15 00:00:00 and the Unix epoch 1970-01-01 00:00:00. + intervals = int(microseconds * 10) + 0x01b21dd213814000 + + time_low = intervals & 0xffffffff + time_mid = (intervals >> 32) & 0xffff + time_hi_version = (intervals >> 48) & 0x0fff + + if clock_seq is None: + clock_seq = random.getrandbits(14) + else: + if clock_seq > 0x3fff: + raise ValueError('clock_seq is out of range (need a 14-bit value)') + + clock_seq_low = clock_seq & 0xff + clock_seq_hi_variant = 0x80 | ((clock_seq >> 8) & 0x3f) + + if node is None: + node = random.getrandbits(48) + + return uuid.UUID(fields=(time_low, time_mid, time_hi_version, + clock_seq_hi_variant, clock_seq_low, node), version=1) diff --git a/test.py b/test.py new file mode 100644 index 0000000..57b1a50 --- /dev/null +++ b/test.py @@ -0,0 +1,165 @@ +import decimal +import datetime +from typing import Optional, List + +import redis +from pydantic import ValidationError + +from redis_developer.orm import ( + RedisModel, + Field, + Relationship, +) + + +# Declarative model classes + +class BaseRedisModel(RedisModel): + class Meta: + database = redis.Redis(password="my-password", decode_responses=True) + model_key_prefix = "redis-developer:" + + +class Address(BaseRedisModel): + address_line_1: str + address_line_2: Optional[str] + city: str + country: str + postal_code: str + + +class Order(BaseRedisModel): + total: decimal.Decimal + currency: str + created_on: datetime.datetime + + +class Member(BaseRedisModel): + first_name: str + last_name: str + email: str = Field(unique=True, index=True) + join_date: datetime.date + + # Creates an embedded document: stored as hash fields or JSON document. + address: Address + + # Creates a relationship to data in separate Hash or JSON documents. + orders: Optional[List[Order]] = Relationship(back_populates='member') + + # Creates a self-relationship. + recommended_by: Optional['Member'] = Relationship(back_populates='recommended') + + class Meta(BaseRedisModel.Meta): + model_key_prefix = "member" + primary_key_pattern = "" + + +# Validation + +address = Address( + address_line_1="1 Main St.", + city="Happy Town", + state="WY", + postal_code=11111, + country="USA" +) + +# Raises ValidationError: last_name, address are required +try: + Member( + first_name="Andrew", + zipcode="97086", + join_date=datetime.date.today() + ) +except ValidationError as e: + pass + + +# Raises ValidationError: join_date is not a date +try: + Member( + first_name="Andrew", + last_name="Brookins", + join_date="yesterday" + ) +except ValidationError as e: + pass + + +# Passes validation +member = Member( + first_name="Andrew", + last_name="Brookins", + email="a@example.com", + address=address, + join_date=datetime.date.today() +) + +exit() + +# Save a model instance to Redis + +address.save() + +address2 = Address.get(address.pk) +assert address2 == address + + +exit() + +# Save a model with relationships (TODO!) + +member.save() + + +# Save many model instances to Redis +today = datetime.date.today() +members = [ + Member( + first_name="Andrew", + last_name="Brookins", + email="a@example.com", + address=address, + join_date=today + ), + Member( + first_name="Kim", + last_name="Brookins", + email="k@example.com", + address=address, + join_date=today + ) +] +Member.add(members) + +# Get a model instance from Redis using the primary key. +member = Member.get(1) + + +# Update a model instance in Redis +member.first_name = "Brian" +member.last_name = "Sam-Bodden" +member.save() + +# Or, with an implicit save: +member.update(first_name="Brian", last_name="Sam-Bodden") + +# Or, affecting multiple model instances with an implicit save: +Member.filter(Member.last_name == "Brookins").update(last_name="Sam-Bodden") + + +# Exact-value queries on indexed fields + +# What if the field wasn't unique and there were two "a@example.com" +# entries? This would raise a MultipleObjectsReturned error: +member = Member.get(Member.email == "a.m.brookins@gmail.com") + +# What if you know there might be multiple results? Use filter(): +members = Member.filter(Member.last_name == "Brookins") + +# What if you want to only return values that don't match a query? +members = Member.exclude(Member.last_name == "Brookins") + +# You can combine filer() and exclude(): +members = Member.filter(Member.last_name == "Brookins").exclude( + Member.first_name == "Andrew")