From 11ed5fd322c21a3895b38f699eba350da094886a Mon Sep 17 00:00:00 2001 From: collerek Date: Wed, 28 Apr 2021 17:04:29 +0200 Subject: [PATCH 1/6] add large binary field, tests and docs --- README.md | 13 +++++- docs/fields/field-types.md | 11 +++++ docs/index.md | 13 +++++- docs/releases.md | 11 +++++ examples/db.sqlite | Bin 12288 -> 12288 bytes examples/script_from_readme.py | 12 +++++- ormar/__init__.py | 2 + ormar/fields/__init__.py | 2 + ormar/fields/model_fields.py | 47 +++++++++++++++++++++ ormar/models/helpers/validation.py | 2 + tests/test_fastapi/test_choices_schema.py | 7 +++ tests/test_model_definition/test_models.py | 40 +++++++++++++----- tests/test_types.py | 2 +- 13 files changed, 148 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 582f2f5..0ae02b4 100644 --- a/README.md +++ b/README.md @@ -451,6 +451,16 @@ async def aggregations(): # to read more about aggregated functions # visit: https://collerek.github.io/ormar/queries/aggregations/ + +async def with_connect(function): + # note that for any other backend than sqlite you actually need to + # connect to the database to perform db operations + async with database: + await function() + + # note that if you use framework like `fastapi` you shouldn't connect + # in your endpoints but have a global connection pool + # check https://collerek.github.io/ormar/fastapi/ and section with db connection # gather and execute all functions # note - normally import should be at the beginning of the file @@ -462,7 +472,7 @@ for func in [create, read, update, delete, joins, filter_and_sort, subset_of_columns, pagination, aggregations]: print(f"Executing: {func.__name__}") - asyncio.run(func()) + asyncio.run(with_connect(func)) # drop the database tables metadata.drop_all(engine) @@ -521,6 +531,7 @@ Available Model Fields (with required args - optional ones in docs): * `BigInteger()` * `Decimal(scale, precision)` * `UUID()` +* `LargeBinary(max_length)` * `EnumField` - by passing `choices` to any other Field type * `EncryptedString` - by passing `encrypt_secret` and `encrypt_backend` * `ForeignKey(to)` diff --git a/docs/fields/field-types.md b/docs/fields/field-types.md index 8a5d6a8..d11622b 100644 --- a/docs/fields/field-types.md +++ b/docs/fields/field-types.md @@ -127,6 +127,17 @@ You can use either `length` and `precision` parameters or `max_digits` and `deci * Sqlalchemy column: `sqlalchemy.JSON` * Type (used for pydantic): `pydantic.Json` +### LargeBinary + +`LargeBinary(max_length)` has a required `max_length` parameter. + +* Sqlalchemy column: `sqlalchemy.LargeBinary` +* Type (used for pydantic): `bytes` + +LargeBinary length is used in some backend (i.e. mysql) to determine the size of the field, +in other backends it's simply ignored yet in ormar it's always required. It should be max +size of the file/bytes in bytes. + ### UUID `UUID(uuid_format: str = 'hex')` has no required parameters. diff --git a/docs/index.md b/docs/index.md index 582f2f5..0ae02b4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -451,6 +451,16 @@ async def aggregations(): # to read more about aggregated functions # visit: https://collerek.github.io/ormar/queries/aggregations/ + +async def with_connect(function): + # note that for any other backend than sqlite you actually need to + # connect to the database to perform db operations + async with database: + await function() + + # note that if you use framework like `fastapi` you shouldn't connect + # in your endpoints but have a global connection pool + # check https://collerek.github.io/ormar/fastapi/ and section with db connection # gather and execute all functions # note - normally import should be at the beginning of the file @@ -462,7 +472,7 @@ for func in [create, read, update, delete, joins, filter_and_sort, subset_of_columns, pagination, aggregations]: print(f"Executing: {func.__name__}") - asyncio.run(func()) + asyncio.run(with_connect(func)) # drop the database tables metadata.drop_all(engine) @@ -521,6 +531,7 @@ Available Model Fields (with required args - optional ones in docs): * `BigInteger()` * `Decimal(scale, precision)` * `UUID()` +* `LargeBinary(max_length)` * `EnumField` - by passing `choices` to any other Field type * `EncryptedString` - by passing `encrypt_secret` and `encrypt_backend` * `ForeignKey(to)` diff --git a/docs/releases.md b/docs/releases.md index 192c932..bc6a148 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,3 +1,14 @@ +# 0.10.6 + +## ✨ Features + +* Add `LargeBinary(max_length)` field type [#166](https://github.com/collerek/ormar/issues/166) + +## 💬 Other + +* Add connecting the database in quickstart in readme [#180](https://github.com/collerek/ormar/issues/180) + + # 0.10.5 ## 🐛 Fixes diff --git a/examples/db.sqlite b/examples/db.sqlite index e9f6c4221cd1530b58dbcd885cfb6d3cb358e8b0..95eab2d25c3999d8c3b39d4937fe940eb9a5589e 100644 GIT binary patch delta 33 hcmZojXh@hKEhxgkz`zW}OhB4rqK+}6$i{>P`T$(_1%?0s delta 33 icmZojXh@hKEx3Szfq@x{nSiv$L>*(s1sf9<=mP+6D+ay* diff --git a/examples/script_from_readme.py b/examples/script_from_readme.py index ce4ee92..782841b 100644 --- a/examples/script_from_readme.py +++ b/examples/script_from_readme.py @@ -319,6 +319,16 @@ async def aggregations(): # visit: https://collerek.github.io/ormar/queries/aggregations/ +async def with_connect(function): + # note that for any other backend than sqlite you actually need to + # connect to the database to perform db operations + async with database: + await function() + + # note that if you use framework like `fastapi` you shouldn't connect + # in your endpoints but have a global connection pool + # check https://collerek.github.io/ormar/fastapi/ and section with db connection + # gather and execute all functions # note - normally import should be at the beginning of the file import asyncio @@ -329,7 +339,7 @@ for func in [create, read, update, delete, joins, filter_and_sort, subset_of_columns, pagination, aggregations]: print(f"Executing: {func.__name__}") - asyncio.run(func()) + asyncio.run(with_connect(func)) # drop the database tables metadata.drop_all(engine) \ No newline at end of file diff --git a/ormar/__init__.py b/ormar/__init__.py index 8b4f371..1c3d104 100644 --- a/ormar/__init__.py +++ b/ormar/__init__.py @@ -53,6 +53,7 @@ from ormar.fields import ( ForeignKeyField, Integer, JSON, + LargeBinary, ManyToMany, ManyToManyField, String, @@ -124,4 +125,5 @@ __all__ = [ "EncryptBackends", "ENCODERS_MAP", "DECODERS_MAP", + "LargeBinary", ] diff --git a/ormar/fields/__init__.py b/ormar/fields/__init__.py index e0cb3b0..bc679c4 100644 --- a/ormar/fields/__init__.py +++ b/ormar/fields/__init__.py @@ -16,6 +16,7 @@ from ormar.fields.model_fields import ( Float, Integer, JSON, + LargeBinary, String, Text, Time, @@ -50,4 +51,5 @@ __all__ = [ "EncryptBackend", "DECODERS_MAP", "ENCODERS_MAP", + "LargeBinary", ] diff --git a/ormar/fields/model_fields.py b/ormar/fields/model_fields.py index 2236e71..3f4417c 100644 --- a/ormar/fields/model_fields.py +++ b/ormar/fields/model_fields.py @@ -415,6 +415,53 @@ class JSON(ModelFieldFactory, pydantic.Json): return sqlalchemy.JSON() +class LargeBinary(ModelFieldFactory, bytes): + """ + LargeBinary field factory that construct Field classes and populated their values. + """ + + _type = bytes + + def __new__( # type: ignore # noqa CFQ002 + cls, *, max_length: int = None, **kwargs: Any + ) -> BaseField: # type: ignore + kwargs = { + **kwargs, + **{ + k: v + for k, v in locals().items() + if k not in ["cls", "__class__", "kwargs"] + }, + } + return super().__new__(cls, **kwargs) + + @classmethod + def get_column_type(cls, **kwargs: Any) -> Any: + """ + Return proper type of db column for given field type. + Accepts required and optional parameters that each column type accepts. + + :param kwargs: key, value pairs of sqlalchemy options + :type kwargs: Any + :return: initialized column with proper options + :rtype: sqlalchemy Column + """ + return sqlalchemy.LargeBinary(length=kwargs.get("max_length")) + + @classmethod + def validate(cls, **kwargs: Any) -> None: + """ + Used to validate if all required parameters on a given field type are set. + :param kwargs: all params passed during construction + :type kwargs: Any + """ + max_length = kwargs.get("max_length", None) + if max_length is None or max_length <= 0: + raise ModelDefinitionError( + "Parameter max_length is required for field LargeBinary" + ) + + class BigInteger(Integer, int): """ BigInteger field factory that construct Field classes and populated their values. diff --git a/ormar/models/helpers/validation.py b/ormar/models/helpers/validation.py index a535f65..e45f3a8 100644 --- a/ormar/models/helpers/validation.py +++ b/ormar/models/helpers/validation.py @@ -73,6 +73,8 @@ def convert_choices_if_needed( # noqa: CCR001 else value ) choices = [round(float(o), precision) for o in choices] + elif field.__type__ == bytes: + value = value if isinstance(value, bytes) else value.encode("utf-8") return value, choices diff --git a/tests/test_fastapi/test_choices_schema.py b/tests/test_fastapi/test_choices_schema.py index 978c3f6..6d87fa4 100644 --- a/tests/test_fastapi/test_choices_schema.py +++ b/tests/test_fastapi/test_choices_schema.py @@ -1,6 +1,7 @@ import datetime import decimal import uuid +from base64 import b64encode from enum import Enum import databases @@ -22,6 +23,10 @@ uuid1 = uuid.uuid4() uuid2 = uuid.uuid4() +blob = b"test" +blob2 = b"test2icac89uc98" + + class EnumTest(Enum): val1 = "Val1" val2 = "Val2" @@ -57,6 +62,7 @@ class Organisation(ormar.Model): random_json: pydantic.Json = ormar.JSON(choices=["aa", '{"aa":"bb"}']) random_uuid: uuid.UUID = ormar.UUID(choices=[uuid1, uuid2]) enum_string: str = ormar.String(max_length=100, choices=list(EnumTest)) + blob_col: bytes = ormar.LargeBinary(max_length=100000, choices=[blob, blob2]) @app.on_event("startup") @@ -111,6 +117,7 @@ def test_all_endpoints(): "random_json": '{"aa":"bb"}', "random_uuid": str(uuid1), "enum_string": EnumTest.val1.value, + "blob_col": blob.decode("utf-8"), }, ) diff --git a/tests/test_model_definition/test_models.py b/tests/test_model_definition/test_models.py index 3459768..46d8ba0 100644 --- a/tests/test_model_definition/test_models.py +++ b/tests/test_model_definition/test_models.py @@ -1,6 +1,6 @@ import asyncio -import uuid import datetime +import uuid from typing import List import databases @@ -9,7 +9,7 @@ import pytest import sqlalchemy import ormar -from ormar.exceptions import QueryDefinitionError, NoMatch, ModelError +from ormar.exceptions import ModelError, NoMatch, QueryDefinitionError from tests.settings import DATABASE_URL database = databases.Database(DATABASE_URL, force_rollback=True) @@ -26,6 +26,20 @@ class JsonSample(ormar.Model): test_json = ormar.JSON(nullable=True) +blob = b"test" +blob2 = b"test2icac89uc98" + + +class LargeBinarySample(ormar.Model): + class Meta: + tablename = "my_bolbs" + metadata = metadata + database = database + + id: int = ormar.Integer(primary_key=True) + test_binary = ormar.LargeBinary(max_length=100000, choices=[blob, blob2]) + + class UUIDSample(ormar.Model): class Meta: tablename = "uuids" @@ -102,15 +116,8 @@ class Country(ormar.Model): ) -@pytest.fixture(scope="module") -def event_loop(): - loop = asyncio.get_event_loop() - yield loop - loop.close() - - @pytest.fixture(autouse=True, scope="module") -async def create_test_database(): +def create_test_database(): engine = sqlalchemy.create_engine(DATABASE_URL) metadata.drop_all(engine) metadata.create_all(engine) @@ -151,6 +158,19 @@ async def test_json_column(): assert items[1].test_json == dict(aa=12) +@pytest.mark.asyncio +async def test_binary_column(): + async with database: + async with database.transaction(force_rollback=True): + await LargeBinarySample.objects.create(test_binary=blob) + await LargeBinarySample.objects.create(test_binary=blob2) + + items = await LargeBinarySample.objects.all() + assert len(items) == 2 + assert items[0].test_binary == blob + assert items[1].test_binary == blob2 + + @pytest.mark.asyncio async def test_uuid_column(): async with database: diff --git a/tests/test_types.py b/tests/test_types.py index ba819ed..d4f8dad 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -57,7 +57,7 @@ def create_test_database(): def assert_type(book: Book): - print(book) + _ = str(book) @pytest.mark.asyncio From d2921167bd674e0294b6440e8cf334d8d82216bd Mon Sep 17 00:00:00 2001 From: collerek Date: Wed, 28 Apr 2021 17:16:30 +0200 Subject: [PATCH 2/6] fix coverage --- Makefile | 8 +++++++- .../test_model_definition/test_model_definition.py | 13 +++++++++++++ tests/test_model_definition/test_pydantic_fields.py | 6 +++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 721821b..8dfb73b 100644 --- a/Makefile +++ b/Makefile @@ -16,4 +16,10 @@ test_mysql: docker-compose -f scripts/docker-compose.yml stop mysql test_sqlite: - bash scripts/test.sh -svv \ No newline at end of file + bash scripts/test.sh -svv + +test: + pytest + +coverage: + pytest --cov=ormar --cov=tests --cov-fail-under=100 --cov-report=term-missing \ No newline at end of file diff --git a/tests/test_model_definition/test_model_definition.py b/tests/test_model_definition/test_model_definition.py index f3cf30f..3585949 100644 --- a/tests/test_model_definition/test_model_definition.py +++ b/tests/test_model_definition/test_model_definition.py @@ -218,6 +218,19 @@ def test_decimal_error_in_model_definition(): test: decimal.Decimal = ormar.Decimal(primary_key=True) +@typing.no_type_check +def test_binary_error_without_length_model_definition(): + with pytest.raises(ModelDefinitionError): + + class ExampleModel2(Model): + class Meta: + tablename = "example6" + database = database + metadata = metadata + + test: bytes = ormar.LargeBinary(primary_key=True) + + @typing.no_type_check def test_string_error_in_model_definition(): with pytest.raises(ModelDefinitionError): diff --git a/tests/test_model_definition/test_pydantic_fields.py b/tests/test_model_definition/test_pydantic_fields.py index b4be0ae..4aae511 100644 --- a/tests/test_model_definition/test_pydantic_fields.py +++ b/tests/test_model_definition/test_pydantic_fields.py @@ -17,7 +17,7 @@ class BaseMeta(ormar.ModelMeta): database = database -class Test(ormar.Model): +class ModelTest(ormar.Model): class Meta(BaseMeta): pass @@ -44,7 +44,7 @@ def create_test_database(): @pytest.mark.asyncio async def test_working_with_pydantic_fields(): async with database: - test = Test(name="Test") + test = ModelTest(name="Test") assert test.name == "Test" assert test.url == "www.example.com" @@ -52,7 +52,7 @@ async def test_working_with_pydantic_fields(): assert test.url == "www.sdta.ada.pt" await test.save() - test_check = await Test.objects.get() + test_check = await ModelTest.objects.get() assert test_check.name == "Test" assert test_check.url == "www.example.com" From 734c33920bdb84f6687deb0aa6f5aed2773dd3af Mon Sep 17 00:00:00 2001 From: collerek Date: Wed, 28 Apr 2021 19:17:59 +0200 Subject: [PATCH 3/6] add support for normal pydantic fields --- Makefile | 8 +- docs/fields/common-parameters.md | 6 +- docs/fields/pydantic-fields.md | 195 ++++++++++++++++++ docs/releases.md | 21 +- mkdocs.yml | 1 + ormar/fields/base.py | 9 + ormar/models/helpers/pydantic.py | 1 + ormar/models/newbasemodel.py | 10 +- .../test_pydantic_fields.py | 106 ++++++++-- 9 files changed, 340 insertions(+), 17 deletions(-) create mode 100644 docs/fields/pydantic-fields.md diff --git a/Makefile b/Makefile index 8dfb73b..d560198 100644 --- a/Makefile +++ b/Makefile @@ -22,4 +22,10 @@ test: pytest coverage: - pytest --cov=ormar --cov=tests --cov-fail-under=100 --cov-report=term-missing \ No newline at end of file + pytest --cov=ormar --cov=tests --cov-fail-under=100 --cov-report=term-missing + +black: + black ormar tests + +mypy: + mypy ormar tests \ No newline at end of file diff --git a/docs/fields/common-parameters.md b/docs/fields/common-parameters.md index 6cc127d..edcfb48 100644 --- a/docs/fields/common-parameters.md +++ b/docs/fields/common-parameters.md @@ -98,7 +98,11 @@ Sets the unique constraint on a table's column. Used in sql only. -## pydantic_only +## pydantic_only (**DEPRECATED**) + +**This parameter is deprecated and will be removed in one of next releases!** + +**To check how to declare pydantic only fields that are not saved into database see [pydantic fields section](pydantic-fields.md)** `pydantic_only`: `bool` = `False` diff --git a/docs/fields/pydantic-fields.md b/docs/fields/pydantic-fields.md new file mode 100644 index 0000000..042bd36 --- /dev/null +++ b/docs/fields/pydantic-fields.md @@ -0,0 +1,195 @@ +# Pydantic only fields + +Ormar allows you to declare normal `pydantic` fields in its model, so you have access to +all basic and custom pydantic fields like `str`, `int`, `HttpUrl`, `PaymentCardNumber` etc. + +You can even declare fields leading to nested pydantic only Models, not only single fields. + +Since those fields are not stored in database (that's the whole point of those fields), +you have to provide a meaningful value for them, either by setting a default one or +providing one during model initialization. + +If `ormar` cannot resolve the value for pydantic field it will fail during loading data from the database, +with missing required value for declared pydantic field. + +Options to provide a value are described below. + +Of course you can combine few or all of them in one model. + +## Optional field + +If you set a field as `Optional`, it defaults to `None` if not provided and that's +exactly what's going to happen during loading from database. + +```python +database = databases.Database(DATABASE_URL) +metadata = sqlalchemy.MetaData() + + +class BaseMeta(ormar.ModelMeta): + metadata = metadata + database = database + +class ModelTest(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=200) + number: Optional[PaymentCardNumber] + +test = ModelTest(name="Test") +assert test.name == "Test" +assert test.number is None +test.number = "123456789015" + +await test.save() +test_check = await ModelTest.objects.get() + +assert test_check.name == "Test" +# after load it's back to None +assert test_check.number is None +``` + +## Field with default value + +By setting a default value, this value will be set on initialization and database load. +Note that setting a default to `None` is the same as setting the field to `Optional`. + +```python +database = databases.Database(DATABASE_URL) +metadata = sqlalchemy.MetaData() + + +class BaseMeta(ormar.ModelMeta): + metadata = metadata + database = database + +class ModelTest(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=200) + url: HttpUrl = "https://www.example.com" + +test = ModelTest(name="Test") +assert test.name == "Test" +assert test.url == "https://www.example.com" + +test.url = "https://www.sdta.ada.pt" +assert test.url == "https://www.sdta.ada.pt" + +await test.save() +test_check = await ModelTest.objects.get() + +assert test_check.name == "Test" +# after load it's back to default +assert test_check.url == "https://www.example.com" +``` + +## Default factory function + +By setting a `default_factory` function, this result of the function call will be set +on initialization and each database load. + +```python +from pydantic import Field, PaymentCardNumber +# ... + +database = databases.Database(DATABASE_URL) +metadata = sqlalchemy.MetaData() + + +class BaseMeta(ormar.ModelMeta): + metadata = metadata + database = database + +CARD_NUMBERS = [ + "123456789007", + "123456789015", + "123456789023", + "123456789031", + "123456789049", +] + + +def get_number(): + return random.choice(CARD_NUMBERS) + + +class ModelTest2(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=200) + # note that you do not call the function, just pass reference + number: PaymentCardNumber = Field(default_factory=get_number) + +# note that you still CAN provide a value +test = ModelTest2(name="Test2", number="4000000000000002") +assert test.name == "Test2" +assert test.number == "4000000000000002" + +await test.save() +test_check = await ModelTest2.objects.get() + +assert test_check.name == "Test2" +# after load value is set to be one of the CARD_NUMBERS +assert test_check.number in CARD_NUMBERS +assert test_check.number != test.number +``` + +## Custom setup in `__init__` + +You can provide a value for the field in your `__init__()` method before calling a `super()` init method. + +```python +from pydantic import BaseModel +# ... + +database = databases.Database(DATABASE_URL) +metadata = sqlalchemy.MetaData() + + +class BaseMeta(ormar.ModelMeta): + metadata = metadata + database = database + +class PydanticTest(BaseModel): + aa: str + bb: int + + +class ModelTest3(ormar.Model): + class Meta(BaseMeta): + pass + + # provide your custom init function + def __init__(self, **kwargs): + # add value for required field without default value + kwargs["pydantic_test"] = PydanticTest(aa="random", bb=42) + # remember to call ormar.Model init! + super().__init__(**kwargs) + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=200) + pydantic_test: PydanticTest + +test = ModelTest3(name="Test3") +assert test.name == "Test3" +assert test.pydantic_test.bb == 42 +test.pydantic.aa = "new value" +assert test.pydantic.aa == "new value" + +await test.save() +test_check = await ModelTest3.objects.get() + +assert test_check.name == "Test3" +# after load it's back to value provided in init +assert test_check.pydantic_test.aa == "random" +``` + +!!!warning + If you do not provide a value in one of the above ways `ValidationError` will be raised on load from database. \ No newline at end of file diff --git a/docs/releases.md b/docs/releases.md index bc6a148..b82bf33 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -3,10 +3,29 @@ ## ✨ Features * Add `LargeBinary(max_length)` field type [#166](https://github.com/collerek/ormar/issues/166) +* Add support for normal pydantic fields (including Models) instead of `pydantic_only` + attribute which is now deprecated [#160](https://github.com/collerek/ormar/issues/160). + Pydantic fields should be declared normally as in pydantic model next to ormar fields, + note that (obviously) `ormar` does not save and load the value for this field in + database that mean that **ONE** of the following has to be true: + + * pydantic field declared on ormar model has to be `Optional` (defaults to None) + * pydantic field has to have a default value set + * pydantic field has `default_factory` function set + * ormar.Model with pydantic field has to overwrite `__init__()` and provide the value there + + If none of the above `ormar` (or rather pydantic) will fail during loading data from the database, + with missing required value for declared pydantic field. + +## 🐛 Fixes + +* By default `pydantic` is not validating fields during assignment, + which is not a desirable setting for an ORM, now all `ormar.Models` + have validation turned-on during assignment (like `model.column = 'value'`) ## 💬 Other -* Add connecting the database in quickstart in readme [#180](https://github.com/collerek/ormar/issues/180) +* Add connecting to the database in QuickStart in readme [#180](https://github.com/collerek/ormar/issues/180) # 0.10.5 diff --git a/mkdocs.yml b/mkdocs.yml index c32dbcc..1b6020c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,6 +12,7 @@ nav: - Fields: - Common parameters: fields/common-parameters.md - Fields types: fields/field-types.md + - Pydantic only fields: fields/pydantic-fields.md - Fields encryption: fields/encryption.md - Relations: - relations/index.md diff --git a/ormar/fields/base.py b/ormar/fields/base.py index c435ac6..271e9bf 100644 --- a/ormar/fields/base.py +++ b/ormar/fields/base.py @@ -1,3 +1,4 @@ +import warnings from typing import Any, Dict, List, Optional, TYPE_CHECKING, Type, Union import sqlalchemy @@ -43,6 +44,14 @@ class BaseField(FieldInfo): self.index: bool = kwargs.pop("index", False) self.unique: bool = kwargs.pop("unique", False) self.pydantic_only: bool = kwargs.pop("pydantic_only", False) + if self.pydantic_only: + warnings.warn( + "Parameter `pydantic_only` is deprecated and will " + "be removed in one of the next releases.\n You can declare " + "pydantic fields in a normal way. \n Check documentation: " + "https://collerek.github.io/ormar/fields/pydantic-fields", + DeprecationWarning, + ) self.choices: typing.Sequence = kwargs.pop("choices", False) self.virtual: bool = kwargs.pop( diff --git a/ormar/models/helpers/pydantic.py b/ormar/models/helpers/pydantic.py index 8c9e498..e844246 100644 --- a/ormar/models/helpers/pydantic.py +++ b/ormar/models/helpers/pydantic.py @@ -98,6 +98,7 @@ def get_pydantic_base_orm_config() -> Type[pydantic.BaseConfig]: class Config(pydantic.BaseConfig): orm_mode = True + validate_assignment = True return Config diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index 772fd4b..cd522ba 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -12,6 +12,7 @@ from typing import ( Sequence, Set, TYPE_CHECKING, + Tuple, Type, Union, cast, @@ -150,6 +151,13 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass k, self.Meta.model_fields[k].expand_relationship( v, self, to_register=False, + ) + if k in self.Meta.model_fields + else ( + v + if k in self.__fields__ + # some random key will raise KeyError + else self.__fields__["_Q*DHPQ(JAS*((JA)###*(&"] ), "dumps", ) @@ -243,7 +251,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass else: if name in object.__getattribute__(self, "_choices_fields"): validate_choices(field=self.Meta.model_fields[name], value=value) - super().__setattr__(name, value) + super().__setattr__(name, self._convert_json(name, value, op="dumps")) self.set_save_status(False) def __getattribute__(self, item: str) -> Any: # noqa: CCR001 diff --git a/tests/test_model_definition/test_pydantic_fields.py b/tests/test_model_definition/test_pydantic_fields.py index 4aae511..de5fb24 100644 --- a/tests/test_model_definition/test_pydantic_fields.py +++ b/tests/test_model_definition/test_pydantic_fields.py @@ -1,9 +1,10 @@ +import random from typing import Optional import databases import pytest import sqlalchemy -from pydantic import HttpUrl +from pydantic import BaseModel, Field, HttpUrl, PaymentCardNumber import ormar from tests.settings import DATABASE_URL @@ -21,15 +22,54 @@ class ModelTest(ormar.Model): class Meta(BaseMeta): pass - def __init__(self, **kwargs): - # you need to pop non - db fields as ormar will complain that it's unknown field - url = kwargs.pop("url", self.__fields__["url"].get_default()) - super().__init__(**kwargs) - self.url = url + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=200) + url: HttpUrl = "https://www.example.com" + number: Optional[PaymentCardNumber] + + +CARD_NUMBERS = [ + "123456789007", + "123456789015", + "123456789023", + "123456789031", + "123456789049", +] + + +def get_number(): + return random.choice(CARD_NUMBERS) + + +class ModelTest2(ormar.Model): + class Meta(BaseMeta): + pass id: int = ormar.Integer(primary_key=True) name: str = ormar.String(max_length=200) - url: HttpUrl = "www.example.com" # field with default + url: HttpUrl = "https://www.example2.com" + number: PaymentCardNumber = Field(default_factory=get_number) + + +class PydanticTest(BaseModel): + aa: str + bb: int + + +class ModelTest3(ormar.Model): + class Meta(BaseMeta): + pass + + def __init__(self, **kwargs): + kwargs["number"] = get_number() + kwargs["pydantic_test"] = PydanticTest(aa="random", bb=42) + super().__init__(**kwargs) + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=200) + url: HttpUrl = "https://www.example3.com" + number: PaymentCardNumber + pydantic_test: PydanticTest @pytest.fixture(autouse=True, scope="module") @@ -46,16 +86,56 @@ async def test_working_with_pydantic_fields(): async with database: test = ModelTest(name="Test") assert test.name == "Test" - assert test.url == "www.example.com" + assert test.url == "https://www.example.com" + assert test.number is None + test.number = "123456789015" - test.url = "www.sdta.ada.pt" - assert test.url == "www.sdta.ada.pt" + test.url = "https://www.sdta.ada.pt" + assert test.url == "https://www.sdta.ada.pt" await test.save() test_check = await ModelTest.objects.get() assert test_check.name == "Test" - assert test_check.url == "www.example.com" + assert test_check.url == "https://www.example.com" + assert test_check.number is None - # TODO add validate assignment to pydantic config - # test_check.email = 1 + +@pytest.mark.asyncio +async def test_default_factory_for_pydantic_fields(): + async with database: + test = ModelTest2(name="Test2", number="4000000000000002") + assert test.name == "Test2" + assert test.url == "https://www.example2.com" + assert test.number == "4000000000000002" + + test.url = "http://www.sdta.ada.pt" + assert test.url == "http://www.sdta.ada.pt" + + await test.save() + test_check = await ModelTest2.objects.get() + + assert test_check.name == "Test2" + assert test_check.url == "https://www.example2.com" + assert test_check.number in CARD_NUMBERS + assert test_check.number != test.number + + +@pytest.mark.asyncio +async def test_init_setting_for_pydantic_fields(): + async with database: + test = ModelTest3(name="Test3") + assert test.name == "Test3" + assert test.url == "https://www.example3.com" + assert test.pydantic_test.bb == 42 + + test.url = "http://www.sdta.ada.pt" + assert test.url == "http://www.sdta.ada.pt" + + await test.save() + test_check = await ModelTest3.objects.get() + + assert test_check.name == "Test3" + assert test_check.url == "https://www.example3.com" + assert test_check.number in CARD_NUMBERS + assert test_check.pydantic_test.aa == "random" From 12c002776b0b38a566254475bb957b1416f756b0 Mon Sep 17 00:00:00 2001 From: collerek Date: Fri, 30 Apr 2021 16:46:41 +0200 Subject: [PATCH 4/6] add examples to openapi schema, some perf improvements --- docs/releases.md | 7 +- ormar/__init__.py | 2 +- ormar/fields/base.py | 1 + ormar/fields/foreign_key.py | 2 +- ormar/fields/model_fields.py | 15 +++ ormar/models/helpers/models.py | 7 ++ ormar/models/helpers/validation.py | 58 +++++++++ ormar/models/metaclass.py | 1 + ormar/models/mixins/relation_mixin.py | 6 +- ormar/models/newbasemodel.py | 160 +++++++++++++----------- tests/test_fastapi/test_fastapi_docs.py | 13 ++ 11 files changed, 194 insertions(+), 78 deletions(-) diff --git a/docs/releases.md b/docs/releases.md index b82bf33..bb8787e 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -16,6 +16,9 @@ If none of the above `ormar` (or rather pydantic) will fail during loading data from the database, with missing required value for declared pydantic field. +* Ormar provides now a meaningful examples in openapi schema, including nested models. + The same algorithm is used to iterate related models without looks + as with `dict()` and `select/load_all`. Examples appear also in `fastapi`. [#157](https://github.com/collerek/ormar/issues/157) ## 🐛 Fixes @@ -26,7 +29,9 @@ ## 💬 Other * Add connecting to the database in QuickStart in readme [#180](https://github.com/collerek/ormar/issues/180) - +* OpenAPI schema does no longer include `ormar.Model` docstring as description, + instead just model name is provided if you do not provide your own docstring. +* Some performance improvements. # 0.10.5 diff --git a/ormar/__init__.py b/ormar/__init__.py index 1c3d104..f7228d6 100644 --- a/ormar/__init__.py +++ b/ormar/__init__.py @@ -76,7 +76,7 @@ class UndefinedType: # pragma no cover Undefined = UndefinedType() -__version__ = "0.10.5" +__version__ = "0.10.6" __all__ = [ "Integer", "BigInteger", diff --git a/ormar/fields/base.py b/ormar/fields/base.py index 271e9bf..9a217b6 100644 --- a/ormar/fields/base.py +++ b/ormar/fields/base.py @@ -31,6 +31,7 @@ class BaseField(FieldInfo): def __init__(self, **kwargs: Any) -> None: self.__type__: type = kwargs.pop("__type__", None) + self.__sample__: type = kwargs.pop("__sample__", None) self.related_name = kwargs.pop("related_name", None) self.column_type: sqlalchemy.Column = kwargs.pop("column_type", None) diff --git a/ormar/fields/foreign_key.py b/ormar/fields/foreign_key.py index feba37c..707463f 100644 --- a/ormar/fields/foreign_key.py +++ b/ormar/fields/foreign_key.py @@ -80,7 +80,7 @@ def create_dummy_model( :rtype: pydantic.BaseModel """ alias = ( - "".join(choices(string.ascii_uppercase, k=2)) + uuid.uuid4().hex[:4] + "".join(choices(string.ascii_uppercase, k=6)) # + uuid.uuid4().hex[:4] ).lower() fields = {f"{pk_field.name}": (pk_field.__type__, None)} diff --git a/ormar/fields/model_fields.py b/ormar/fields/model_fields.py index 3f4417c..4984771 100644 --- a/ormar/fields/model_fields.py +++ b/ormar/fields/model_fields.py @@ -62,6 +62,7 @@ class ModelFieldFactory: _bases: Any = (BaseField,) _type: Any = None + _sample: Any = None def __new__(cls, *args: Any, **kwargs: Any) -> BaseField: # type: ignore cls.validate(**kwargs) @@ -80,6 +81,7 @@ class ModelFieldFactory: namespace = dict( __type__=cls._type, + __sample__=cls._sample, alias=kwargs.pop("name", None), name=None, primary_key=primary_key, @@ -129,6 +131,7 @@ class String(ModelFieldFactory, str): """ _type = str + _sample = "string" def __new__( # type: ignore # noqa CFQ002 cls, @@ -185,6 +188,7 @@ class Integer(ModelFieldFactory, int): """ _type = int + _sample = 0 def __new__( # type: ignore cls, @@ -232,6 +236,7 @@ class Text(ModelFieldFactory, str): """ _type = str + _sample = "text" def __new__( # type: ignore cls, *, allow_blank: bool = True, strip_whitespace: bool = False, **kwargs: Any @@ -267,6 +272,7 @@ class Float(ModelFieldFactory, float): """ _type = float + _sample = 0.0 def __new__( # type: ignore cls, @@ -316,6 +322,7 @@ else: """ _type = bool + _sample = True @classmethod def get_column_type(cls, **kwargs: Any) -> Any: @@ -337,6 +344,7 @@ class DateTime(ModelFieldFactory, datetime.datetime): """ _type = datetime.datetime + _sample = "datetime" @classmethod def get_column_type(cls, **kwargs: Any) -> Any: @@ -358,6 +366,7 @@ class Date(ModelFieldFactory, datetime.date): """ _type = datetime.date + _sample = "date" @classmethod def get_column_type(cls, **kwargs: Any) -> Any: @@ -379,6 +388,7 @@ class Time(ModelFieldFactory, datetime.time): """ _type = datetime.time + _sample = "time" @classmethod def get_column_type(cls, **kwargs: Any) -> Any: @@ -400,6 +410,7 @@ class JSON(ModelFieldFactory, pydantic.Json): """ _type = pydantic.Json + _sample = '{"json": "json"}' @classmethod def get_column_type(cls, **kwargs: Any) -> Any: @@ -421,6 +432,7 @@ class LargeBinary(ModelFieldFactory, bytes): """ _type = bytes + _sample = "bytes" def __new__( # type: ignore # noqa CFQ002 cls, *, max_length: int = None, **kwargs: Any @@ -468,6 +480,7 @@ class BigInteger(Integer, int): """ _type = int + _sample = 0 def __new__( # type: ignore cls, @@ -515,6 +528,7 @@ class Decimal(ModelFieldFactory, decimal.Decimal): """ _type = decimal.Decimal + _sample = 0.0 def __new__( # type: ignore # noqa CFQ002 cls, @@ -587,6 +601,7 @@ class UUID(ModelFieldFactory, uuid.UUID): """ _type = uuid.UUID + _sample = "uuid" def __new__( # type: ignore # noqa CFQ002 cls, *, uuid_format: str = "hex", **kwargs: Any diff --git a/ormar/models/helpers/models.py b/ormar/models/helpers/models.py index d94f8a8..4c73712 100644 --- a/ormar/models/helpers/models.py +++ b/ormar/models/helpers/models.py @@ -3,6 +3,7 @@ import itertools import sqlite3 from typing import Any, Dict, List, TYPE_CHECKING, Tuple, Type +import pydantic from pydantic.typing import ForwardRef import ormar # noqa: I100 from ormar.models.helpers.pydantic import populate_pydantic_default_values @@ -61,6 +62,12 @@ def populate_default_options_values( else: new_model.Meta.requires_ref_update = False + new_model._json_fields = { + name + for name, field in new_model.Meta.model_fields.items() + if field.__type__ == pydantic.Json + } + class Connection(sqlite3.Connection): def __init__(self, *args: Any, **kwargs: Any) -> None: # pragma: no cover diff --git a/ormar/models/helpers/validation.py b/ormar/models/helpers/validation.py index e45f3a8..d861911 100644 --- a/ormar/models/helpers/validation.py +++ b/ormar/models/helpers/validation.py @@ -15,6 +15,7 @@ from pydantic.main import SchemaExtraCallable import ormar # noqa: I100, I202 from ormar.fields import BaseField from ormar.models.helpers.models import meta_field_not_set +from ormar.queryset.utils import translate_list_to_dict if TYPE_CHECKING: # pragma no cover from ormar import Model @@ -116,12 +117,45 @@ def choices_validator(cls: Type["Model"], values: Dict[str, Any]) -> Dict[str, A return values +def generate_model_example(model: Type["Model"], relation_map: Dict = None) -> Dict: + """ + Generates example to be included in schema in fastapi. + + :param model: ormar.Model + :type model: Type["Model"] + :param relation_map: dict with relations to follow + :type relation_map: Optional[Dict] + :return: + :rtype: Dict[str, int] + """ + example: Dict[str, Any] = dict() + relation_map = ( + relation_map + if relation_map is not None + else translate_list_to_dict(model._iterate_related_models()) + ) + for name, field in model.Meta.model_fields.items(): + if not field.is_relation: + example[name] = field.__sample__ + elif isinstance(relation_map, dict) and name in relation_map: + value = generate_model_example( + field.to, relation_map=relation_map.get(name, {}) + ) + new_value = [value] if field.is_multi or field.virtual else value + example[name] = new_value + + return example + + def construct_modify_schema_function(fields_with_choices: List) -> SchemaExtraCallable: """ Modifies the schema to include fields with choices validator. Those fields will be displayed in schema as Enum types with available choices values listed next to them. + Note that schema extra has to be a function, otherwise it's called to soon + before all the relations are expanded. + :param fields_with_choices: list of fields with choices validation :type fields_with_choices: List :return: callable that will be run by pydantic to modify the schema @@ -133,6 +167,28 @@ def construct_modify_schema_function(fields_with_choices: List) -> SchemaExtraCa if field_id in fields_with_choices: prop["enum"] = list(model.Meta.model_fields[field_id].choices) prop["description"] = prop.get("description", "") + "An enumeration." + schema["example"] = generate_model_example(model=model) + if "Main base class of ormar Model." in schema.get("description", ""): + schema["description"] = f"{model.__name__}" + + return staticmethod(schema_extra) # type: ignore + + +def construct_schema_function_without_choices() -> SchemaExtraCallable: + """ + Modifies model example and description if needed. + + Note that schema extra has to be a function, otherwise it's called to soon + before all the relations are expanded. + + :return: callable that will be run by pydantic to modify the schema + :rtype: Callable + """ + + def schema_extra(schema: Dict[str, Any], model: Type["Model"]) -> None: + schema["example"] = generate_model_example(model=model) + if "Main base class of ormar Model." in schema.get("description", ""): + schema["description"] = f"{model.__name__}" return staticmethod(schema_extra) # type: ignore @@ -162,3 +218,5 @@ def populate_choices_validators(model: Type["Model"]) -> None: # noqa CCR001 model.Config.schema_extra = construct_modify_schema_function( fields_with_choices=fields_with_choices ) + else: + model.Config.schema_extra = construct_schema_function_without_choices() diff --git a/ormar/models/metaclass.py b/ormar/models/metaclass.py index 43c8822..20d153a 100644 --- a/ormar/models/metaclass.py +++ b/ormar/models/metaclass.py @@ -94,6 +94,7 @@ def add_cached_properties(new_model: Type["Model"]) -> None: new_model._related_fields = None new_model._pydantic_fields = {name for name in new_model.__fields__} new_model._choices_fields = set() + new_model._json_fields = set() def add_property_fields(new_model: Type["Model"], attrs: Dict) -> None: # noqa: CCR001 diff --git a/ormar/models/mixins/relation_mixin.py b/ormar/models/mixins/relation_mixin.py index 6a71382..c4bd618 100644 --- a/ormar/models/mixins/relation_mixin.py +++ b/ormar/models/mixins/relation_mixin.py @@ -48,7 +48,7 @@ class RelationMixin: :return: list of related fields :rtype: List """ - if isinstance(cls._related_fields, List): + if cls._related_fields is not None: return cls._related_fields related_fields = [] @@ -66,7 +66,7 @@ class RelationMixin: :return: set of related through fields names :rtype: Set """ - if isinstance(cls._through_names, Set): + if cls._through_names is not None: return cls._through_names related_names = set() @@ -86,7 +86,7 @@ class RelationMixin: :return: set of related fields names :rtype: Set """ - if isinstance(cls._related_names, Set): + if cls._related_names is not None: return cls._related_names related_names = set() diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index cd522ba..865a48b 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -1,5 +1,4 @@ import sys -import uuid from typing import ( AbstractSet, Any, @@ -87,6 +86,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass _choices_fields: Optional[Set] _pydantic_fields: Set _quick_access_fields: Set + _json_fields: Set Meta: ModelMeta # noinspection PyMissingConstructor @@ -124,60 +124,12 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass :type kwargs: Any """ self._verify_model_can_be_initialized() - object.__setattr__(self, "_orm_id", uuid.uuid4().hex) - object.__setattr__(self, "_orm_saved", False) - object.__setattr__(self, "_pk_column", None) - object.__setattr__( - self, - "_orm", - RelationsManager( - related_fields=self.extract_related_fields(), owner=cast("Model", self), - ), - ) + self._initialize_internal_attributes() pk_only = kwargs.pop("__pk_only__", False) object.__setattr__(self, "__pk_only__", pk_only) - excluded: Set[str] = kwargs.pop("__excluded__", set()) - - if "pk" in kwargs: - kwargs[self.Meta.pkname] = kwargs.pop("pk") - - # build the models to set them and validate but don't register - # also remove property fields values from validation - try: - new_kwargs: Dict[str, Any] = { - k: self._convert_json( - k, - self.Meta.model_fields[k].expand_relationship( - v, self, to_register=False, - ) - if k in self.Meta.model_fields - else ( - v - if k in self.__fields__ - # some random key will raise KeyError - else self.__fields__["_Q*DHPQ(JAS*((JA)###*(&"] - ), - "dumps", - ) - for k, v in kwargs.items() - if k not in object.__getattribute__(self, "Meta").property_fields - } - except KeyError as e: - raise ModelError( - f"Unknown field '{e.args[0]}' for model {self.get_name(lower=False)}" - ) - - # explicitly set None to excluded fields - # as pydantic populates them with default if set - for field_to_nullify in excluded: - new_kwargs[field_to_nullify] = None - - # extract through fields - through_tmp_dict = dict() - for field_name in self.extract_through_names(): - through_tmp_dict[field_name] = new_kwargs.pop(field_name, None) + new_kwargs, through_tmp_dict = self._process_kwargs(kwargs) values, fields_set, validation_error = pydantic.validate_model( self, new_kwargs # type: ignore @@ -190,10 +142,10 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass # add back through fields new_kwargs.update(through_tmp_dict) - + model_fields = object.__getattribute__(self, "Meta").model_fields # register the columns models after initialization for related in self.extract_related_names().union(self.extract_through_names()): - self.Meta.model_fields[related].expand_relationship( + model_fields[related].expand_relationship( new_kwargs.get(related), self, to_register=True, ) @@ -314,15 +266,93 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass :return: None :rtype: None """ - if self.Meta.abstract: + if object.__getattribute__(self, "Meta").abstract: raise ModelError(f"You cannot initialize abstract model {self.get_name()}") - if self.Meta.requires_ref_update: + if object.__getattribute__(self, "Meta").requires_ref_update: raise ModelError( f"Model {self.get_name()} has not updated " f"ForwardRefs. \nBefore using the model you " f"need to call update_forward_refs()." ) + def _process_kwargs(self, kwargs: Dict) -> Tuple[Dict, Dict]: + """ + Initializes nested models. + + Removes property_fields + + Checks if field is in the model fields or pydatnic fields. + + Nullifies fields that should be excluded. + + Extracts through models from kwargs into temporary dict. + + :param kwargs: passed to init keyword arguments + :type kwargs: Dict + :return: modified kwargs + :rtype: Tuple[Dict, Dict] + """ + meta = object.__getattribute__(self, "Meta") + property_fields = meta.property_fields + model_fields = meta.model_fields + pydantic_fields = object.__getattribute__(self, "__fields__") + + # remove property fields + for prop_filed in property_fields: + kwargs.pop(prop_filed, None) + + excluded: Set[str] = kwargs.pop("__excluded__", set()) + if "pk" in kwargs: + kwargs[meta.pkname] = kwargs.pop("pk") + + # extract through fields + through_tmp_dict = dict() + for field_name in self.extract_through_names(): + through_tmp_dict[field_name] = kwargs.pop(field_name, None) + + try: + new_kwargs: Dict[str, Any] = { + k: self._convert_json( + k, + model_fields[k].expand_relationship(v, self, to_register=False,) + if k in model_fields + else ( + v + if k in pydantic_fields + else model_fields["HAP&*YA^)*GW^&QT6567q56gGG%$%"] + ), + "dumps", + ) + for k, v in kwargs.items() + } + except KeyError as e: + raise ModelError( + f"Unknown field '{e.args[0]}' for model {self.get_name(lower=False)}" + ) + + # explicitly set None to excluded fields + # as pydantic populates them with default if set + for field_to_nullify in excluded: + new_kwargs[field_to_nullify] = None + + return new_kwargs, through_tmp_dict + + def _initialize_internal_attributes(self) -> None: + """ + Initializes internal attributes during __init__() + :rtype: None + """ + # object.__setattr__(self, "_orm_id", uuid.uuid4().hex) + object.__setattr__(self, "_orm_saved", False) + object.__setattr__(self, "_pk_column", None) + object.__setattr__( + self, + "_orm", + RelationsManager( + related_fields=self.extract_related_fields(), owner=cast("Model", self), + ), + ) + def _extract_related_model_instead_of_field( self, item: str ) -> Optional[Union["Model", Sequence["Model"]]]: @@ -363,8 +393,8 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass :rtype: bool """ return ( - self._orm_id == other._orm_id - or (self.pk == other.pk and self.pk is not None) + # self._orm_id == other._orm_id + (self.pk == other.pk and self.pk is not None) or ( (self.pk is None and other.pk is None) and { @@ -748,7 +778,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass :return: converted value if needed, else original value :rtype: Any """ - if not self._is_conversion_to_json_needed(column_name): + if column_name not in object.__getattribute__(self, "_json_fields"): return value condition = ( @@ -765,20 +795,6 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass pass return value.decode("utf-8") if isinstance(value, bytes) else value - def _is_conversion_to_json_needed(self, column_name: str) -> bool: - """ - Checks if given column name is related to JSON field. - - :param column_name: name of the field - :type column_name: str - :return: result of the check - :rtype: bool - """ - return ( - column_name in self.Meta.model_fields - and self.Meta.model_fields[column_name].__type__ == pydantic.Json - ) - def _extract_own_model_fields(self) -> Dict: """ Returns a dictionary with field names and values for fields that are not diff --git a/tests/test_fastapi/test_fastapi_docs.py b/tests/test_fastapi/test_fastapi_docs.py index 03f0892..94ac9e1 100644 --- a/tests/test_fastapi/test_fastapi_docs.py +++ b/tests/test_fastapi/test_fastapi_docs.py @@ -1,6 +1,7 @@ from typing import List import databases +import pydantic import pytest import sqlalchemy from fastapi import FastAPI @@ -124,6 +125,18 @@ def test_schema_modification(): x.get("type") == "array" for x in schema["properties"]["categories"]["anyOf"] ) assert schema["properties"]["categories"]["title"] == "Categories" + assert schema["example"] == { + "id": 0, + "name": "string", + "categories": [{"id": 0, "name": "string"}], + } + + schema = Category.schema() + assert schema["example"] == { + "id": 0, + "name": "string", + "items": [{"id": 0, "name": "string"}], + } def test_schema_gen(): From 18706d884c27af3ed218085e52c1420b883e1b54 Mon Sep 17 00:00:00 2001 From: collerek Date: Fri, 30 Apr 2021 18:36:25 +0200 Subject: [PATCH 5/6] add also basic examples for native pydantic fields including models to not skip them --- ormar/models/helpers/validation.py | 75 +++++++++++++++++++++++-- ormar/models/mixins/save_mixin.py | 14 ++--- ormar/models/newbasemodel.py | 6 +- tests/test_fastapi/test_fastapi_docs.py | 31 +++++++++- 4 files changed, 106 insertions(+), 20 deletions(-) diff --git a/ormar/models/helpers/validation.py b/ormar/models/helpers/validation.py index d861911..c557de4 100644 --- a/ormar/models/helpers/validation.py +++ b/ormar/models/helpers/validation.py @@ -1,8 +1,9 @@ import datetime import decimal +import numbers import uuid from enum import Enum -from typing import Any, Dict, List, TYPE_CHECKING, Tuple, Type +from typing import Any, Dict, List, Set, TYPE_CHECKING, Tuple, Type, Union try: import orjson as json @@ -138,15 +139,79 @@ def generate_model_example(model: Type["Model"], relation_map: Dict = None) -> D if not field.is_relation: example[name] = field.__sample__ elif isinstance(relation_map, dict) and name in relation_map: - value = generate_model_example( - field.to, relation_map=relation_map.get(name, {}) + example[name] = get_nested_model_example( + name=name, field=field, relation_map=relation_map ) - new_value = [value] if field.is_multi or field.virtual else value - example[name] = new_value + to_exclude = {name for name in model.Meta.model_fields} + pydantic_repr = generate_pydantic_example(pydantic_model=model, exclude=to_exclude) + example.update(pydantic_repr) return example +def get_nested_model_example( + name: str, field: "BaseField", relation_map: Dict +) -> Union[List, Dict]: + """ + Gets representation of nested model. + + :param name: name of the field to follow + :type name: str + :param field: ormar field + :type field: BaseField + :param relation_map: dict with relation map + :type relation_map: Dict + :return: nested model or list of nested model repr + :rtype: Union[List, Dict] + """ + value = generate_model_example(field.to, relation_map=relation_map.get(name, {})) + new_value: Union[List, Dict] = [value] if field.is_multi or field.virtual else value + return new_value + + +def generate_pydantic_example( + pydantic_model: Type[pydantic.BaseModel], exclude: Set = None +) -> Dict: + """ + Generates dict with example. + + :param pydantic_model: model to parse + :type pydantic_model: Type[pydantic.BaseModel] + :param exclude: list of fields to exclude + :type exclude: Optional[Set] + :return: dict with fields and sample values + :rtype: Dict + """ + example: Dict[str, Any] = dict() + exclude = exclude or set() + for name in pydantic_model.__fields__: + if name not in exclude: + field = pydantic_model.__fields__[name] + type_ = field.type_ + if getattr(field.outer_type_, "_name", None) == "List": + example[name] = [get_pydantic_example_repr(type_)] + else: + example[name] = get_pydantic_example_repr(type_) + return example + + +def get_pydantic_example_repr(type_: Any) -> Any: + """ + Gets sample representation of pydantic field for example dict. + + :param type_: type of pydantic field + :type type_: Any + :return: representation to include in example + :rtype: Any + """ + if issubclass(type_, (numbers.Number, decimal.Decimal)): + return 0 + elif issubclass(type_, pydantic.BaseModel): + return generate_pydantic_example(pydantic_model=type_) + else: + return "string" + + def construct_modify_schema_function(fields_with_choices: List) -> SchemaExtraCallable: """ Modifies the schema to include fields with choices validator. diff --git a/ormar/models/mixins/save_mixin.py b/ormar/models/mixins/save_mixin.py index a7ac562..6a3aab0 100644 --- a/ormar/models/mixins/save_mixin.py +++ b/ormar/models/mixins/save_mixin.py @@ -275,12 +275,12 @@ class SavePrepareMixin(RelationMixin, AliasMixin): :rtype: int """ for field in fields_list: - value = getattr(self, field.name) or [] - if not isinstance(value, list): - value = [value] - for val in value: + values = getattr(self, field.name) or [] + if not isinstance(values, list): + values = [values] + for value in values: if follow: - update_count = await val.save_related( + update_count = await value.save_related( follow=follow, save_all=save_all, relation_map=self._skip_ellipsis( # type: ignore @@ -291,8 +291,8 @@ class SavePrepareMixin(RelationMixin, AliasMixin): relation_field=field, ) else: - update_count = await val._upsert_model( - instance=val, + update_count = await value._upsert_model( + instance=value, save_all=save_all, previous_model=self, relation_field=field, diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index 865a48b..c4dc612 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -316,11 +316,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass k, model_fields[k].expand_relationship(v, self, to_register=False,) if k in model_fields - else ( - v - if k in pydantic_fields - else model_fields["HAP&*YA^)*GW^&QT6567q56gGG%$%"] - ), + else (v if k in pydantic_fields else model_fields[k]), "dumps", ) for k, v in kwargs.items() diff --git a/tests/test_fastapi/test_fastapi_docs.py b/tests/test_fastapi/test_fastapi_docs.py index 94ac9e1..e19fe04 100644 --- a/tests/test_fastapi/test_fastapi_docs.py +++ b/tests/test_fastapi/test_fastapi_docs.py @@ -1,4 +1,5 @@ -from typing import List +import datetime +from typing import List, Optional import databases import pydantic @@ -35,6 +36,17 @@ class LocalMeta: database = database +class PTestA(pydantic.BaseModel): + c: str + d: bytes + e: datetime.datetime + + +class PTestP(pydantic.BaseModel): + a: int + b: Optional[PTestA] + + class Category(ormar.Model): class Meta(LocalMeta): tablename = "categories" @@ -49,6 +61,8 @@ class Item(ormar.Model): id: int = ormar.Integer(primary_key=True) name: str = ormar.String(max_length=100) + pydantic_int: Optional[int] + test_P: Optional[List[PTestP]] categories = ormar.ManyToMany(Category) @@ -126,16 +140,27 @@ def test_schema_modification(): ) assert schema["properties"]["categories"]["title"] == "Categories" assert schema["example"] == { + "categories": [{"id": 0, "name": "string"}], "id": 0, "name": "string", - "categories": [{"id": 0, "name": "string"}], + "pydantic_int": 0, + "test_P": [{"a": 0, "b": {"c": "string", "d": "string", "e": "string"}}], } schema = Category.schema() assert schema["example"] == { "id": 0, "name": "string", - "items": [{"id": 0, "name": "string"}], + "items": [ + { + "id": 0, + "name": "string", + "pydantic_int": 0, + "test_P": [ + {"a": 0, "b": {"c": "string", "d": "string", "e": "string"}} + ], + } + ], } From 1f0ceb3f48f7790c762d1cb9c5436d86613f9ae6 Mon Sep 17 00:00:00 2001 From: collerek Date: Sun, 2 May 2021 15:03:02 +0200 Subject: [PATCH 6/6] change check for pydantic list in outer type --- .github/FUNDING.yml | 1 + ormar/models/helpers/validation.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..771ea52 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: collerek \ No newline at end of file diff --git a/ormar/models/helpers/validation.py b/ormar/models/helpers/validation.py index c557de4..e9d57a9 100644 --- a/ormar/models/helpers/validation.py +++ b/ormar/models/helpers/validation.py @@ -11,6 +11,7 @@ except ImportError: # pragma: no cover import json # type: ignore import pydantic +from pydantic.fields import SHAPE_LIST from pydantic.main import SchemaExtraCallable import ormar # noqa: I100, I202 @@ -184,14 +185,14 @@ def generate_pydantic_example( """ example: Dict[str, Any] = dict() exclude = exclude or set() - for name in pydantic_model.__fields__: - if name not in exclude: - field = pydantic_model.__fields__[name] - type_ = field.type_ - if getattr(field.outer_type_, "_name", None) == "List": - example[name] = [get_pydantic_example_repr(type_)] - else: - example[name] = get_pydantic_example_repr(type_) + name_to_check = [name for name in pydantic_model.__fields__ if name not in exclude] + for name in name_to_check: + field = pydantic_model.__fields__[name] + type_ = field.type_ + if field.shape == SHAPE_LIST: + example[name] = [get_pydantic_example_repr(type_)] + else: + example[name] = get_pydantic_example_repr(type_) return example