add support for normal pydantic fields

Makefile

@@ -23,3 +23,9 @@ test:
 
 coverage:
 	pytest --cov=ormar --cov=tests --cov-fail-under=100 --cov-report=term-missing
+
+black:
+	black ormar tests
+
+mypy:
+	mypy ormar tests

docs/fields/common-parameters.md

@@ -98,7 +98,11 @@ Sets the unique constraint on a table's column.
 
 Used in sql only.
 
-## pydantic_only
+## pydantic_only (**DEPRECATED**)
+
+**This parameter is deprecated and will be removed in one of next releases!**
+
+**To check how to declare pydantic only fields that are not saved into database see [pydantic fields section](pydantic-fields.md)** 
 
 `pydantic_only`: `bool` = `False` 
 

docs/fields/pydantic-fields.md

@@ -0,0 +1,195 @@
+# Pydantic only fields
+
+Ormar allows you to declare normal `pydantic` fields in its model, so you have access to
+all basic and custom pydantic fields like `str`, `int`, `HttpUrl`, `PaymentCardNumber` etc.
+
+You can even declare fields leading to nested pydantic only Models, not only single fields.
+
+Since those fields are not stored in database (that's the whole point of those fields),
+you have to provide a meaningful value for them, either by setting a default one or 
+providing one during model initialization.
+
+If `ormar` cannot resolve the value for pydantic field it will fail during loading data from the database,
+with missing required value for declared pydantic field.
+
+Options to provide a value are described below.
+
+Of course you can combine few or all of them in one model.
+
+## Optional field
+
+If you set a field as `Optional`, it defaults to `None` if not provided and that's 
+exactly what's going to happen during loading from database.
+
+```python
+database = databases.Database(DATABASE_URL)
+metadata = sqlalchemy.MetaData()
+
+
+class BaseMeta(ormar.ModelMeta):
+    metadata = metadata
+    database = database
+    
+class ModelTest(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=200)
+    number: Optional[PaymentCardNumber]
+
+test = ModelTest(name="Test")
+assert test.name == "Test"
+assert test.number is None
+test.number = "123456789015"
+
+await test.save()
+test_check = await ModelTest.objects.get()
+
+assert test_check.name == "Test"
+# after load it's back to None
+assert test_check.number is None
+```
+
+## Field with default value
+
+By setting a default value, this value will be set on initialization and database load. 
+Note that setting a default to `None` is the same as setting the field to `Optional`.
+
+```python
+database = databases.Database(DATABASE_URL)
+metadata = sqlalchemy.MetaData()
+
+
+class BaseMeta(ormar.ModelMeta):
+    metadata = metadata
+    database = database
+    
+class ModelTest(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=200)
+    url: HttpUrl = "https://www.example.com"
+
+test = ModelTest(name="Test")
+assert test.name == "Test"
+assert test.url == "https://www.example.com"
+
+test.url = "https://www.sdta.ada.pt"
+assert test.url == "https://www.sdta.ada.pt"
+
+await test.save()
+test_check = await ModelTest.objects.get()
+
+assert test_check.name == "Test"
+# after load it's back to default
+assert test_check.url == "https://www.example.com"
+```
+
+## Default factory function
+
+By setting a `default_factory` function, this result of the function call will be set 
+on initialization and each database load.
+
+```python
+from pydantic import Field, PaymentCardNumber
+# ...
+
+database = databases.Database(DATABASE_URL)
+metadata = sqlalchemy.MetaData()
+
+
+class BaseMeta(ormar.ModelMeta):
+    metadata = metadata
+    database = database
+
+CARD_NUMBERS = [
+    "123456789007",
+    "123456789015",
+    "123456789023",
+    "123456789031",
+    "123456789049",
+]
+
+
+def get_number():
+    return random.choice(CARD_NUMBERS)
+
+
+class ModelTest2(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=200)
+    # note that you do not call the function, just pass reference
+    number: PaymentCardNumber = Field(default_factory=get_number)
+
+# note that you still CAN provide a value 
+test = ModelTest2(name="Test2", number="4000000000000002")
+assert test.name == "Test2"
+assert test.number == "4000000000000002"
+
+await test.save()
+test_check = await ModelTest2.objects.get()
+
+assert test_check.name == "Test2"
+# after load value is set to be one of the CARD_NUMBERS
+assert test_check.number in CARD_NUMBERS
+assert test_check.number != test.number
+```
+
+## Custom setup in `__init__`
+
+You can provide a value for the field in your `__init__()` method before calling a `super()` init method.
+
+```python
+from pydantic import BaseModel
+# ...
+
+database = databases.Database(DATABASE_URL)
+metadata = sqlalchemy.MetaData()
+
+
+class BaseMeta(ormar.ModelMeta):
+    metadata = metadata
+    database = database
+
+class PydanticTest(BaseModel):
+    aa: str
+    bb: int
+
+
+class ModelTest3(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    # provide your custom init function
+    def __init__(self, **kwargs):
+        # add value for required field without default value
+        kwargs["pydantic_test"] = PydanticTest(aa="random", bb=42)
+        # remember to call ormar.Model init!
+        super().__init__(**kwargs)
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=200)
+    pydantic_test: PydanticTest
+
+test = ModelTest3(name="Test3")
+assert test.name == "Test3"
+assert test.pydantic_test.bb == 42
+test.pydantic.aa = "new value"
+assert test.pydantic.aa == "new value"
+
+await test.save()
+test_check = await ModelTest3.objects.get()
+
+assert test_check.name == "Test3"
+# after load it's back to value provided in init
+assert test_check.pydantic_test.aa == "random"
+```
+
+!!!warning
+        If you do not provide a value in one of the above ways `ValidationError` will be raised on load from database.

docs/releases.md

@@ -3,10 +3,29 @@
 ## ✨ Features
 
 * Add `LargeBinary(max_length)` field type [#166](https://github.com/collerek/ormar/issues/166)
+* Add support for normal pydantic fields (including Models) instead of `pydantic_only` 
+  attribute which is now deprecated [#160](https://github.com/collerek/ormar/issues/160).
+  Pydantic fields should be declared normally as in pydantic model next to ormar fields, 
+  note that (obviously) `ormar` does not save and load the value for this field in 
+  database that mean that **ONE** of the following has to be true:
+  
+    * pydantic field declared on ormar model has to be `Optional` (defaults to None)
+    * pydantic field has to have a default value set
+    * pydantic field has `default_factory` function set
+    * ormar.Model with pydantic field has to overwrite `__init__()` and provide the value there
+    
+    If none of the above `ormar` (or rather pydantic) will fail during loading data from the database,
+    with missing required value for declared pydantic field.
+
+## 🐛 Fixes
+
+* By default `pydantic` is not validating fields during assignment, 
+  which is not a desirable setting for an ORM, now all `ormar.Models` 
+  have validation turned-on during assignment (like `model.column = 'value'`)
 
 ## 💬 Other
 
-*  Add connecting the database in quickstart in readme [#180](https://github.com/collerek/ormar/issues/180) 
+*  Add connecting to the database in QuickStart in readme [#180](https://github.com/collerek/ormar/issues/180) 
 
 
 # 0.10.5

mkdocs.yml

@@ -12,6 +12,7 @@ nav:
   - Fields:
     - Common parameters: fields/common-parameters.md
     - Fields types: fields/field-types.md
+    - Pydantic only fields: fields/pydantic-fields.md
     - Fields encryption: fields/encryption.md
   - Relations:
     - relations/index.md

ormar/fields/base.py

@@ -1,3 +1,4 @@
+import warnings
 from typing import Any, Dict, List, Optional, TYPE_CHECKING, Type, Union
 
 import sqlalchemy
@@ -43,6 +44,14 @@ class BaseField(FieldInfo):
         self.index: bool = kwargs.pop("index", False)
         self.unique: bool = kwargs.pop("unique", False)
         self.pydantic_only: bool = kwargs.pop("pydantic_only", False)
+        if self.pydantic_only:
+            warnings.warn(
+                "Parameter `pydantic_only` is deprecated and will "
+                "be removed in one of the next releases.\n You can declare "
+                "pydantic fields in a normal way. \n Check documentation: "
+                "https://collerek.github.io/ormar/fields/pydantic-fields",
+                DeprecationWarning,
+            )
         self.choices: typing.Sequence = kwargs.pop("choices", False)
 
         self.virtual: bool = kwargs.pop(

ormar/models/helpers/pydantic.py

@@ -98,6 +98,7 @@ def get_pydantic_base_orm_config() -> Type[pydantic.BaseConfig]:
 
     class Config(pydantic.BaseConfig):
         orm_mode = True
+        validate_assignment = True
 
     return Config
 

ormar/models/newbasemodel.py

@@ -12,6 +12,7 @@ from typing import (
     Sequence,
     Set,
     TYPE_CHECKING,
+    Tuple,
     Type,
     Union,
     cast,
@@ -150,6 +151,13 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
                     k,
                     self.Meta.model_fields[k].expand_relationship(
                         v, self, to_register=False,
+                    )
+                    if k in self.Meta.model_fields
+                    else (
+                        v
+                        if k in self.__fields__
+                        # some random key will raise KeyError
+                        else self.__fields__["_Q*DHPQ(JAS*((JA)###*(&"]
                     ),
                     "dumps",
                 )
@@ -243,7 +251,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
         else:
             if name in object.__getattribute__(self, "_choices_fields"):
                 validate_choices(field=self.Meta.model_fields[name], value=value)
-            super().__setattr__(name, value)
+            super().__setattr__(name, self._convert_json(name, value, op="dumps"))
             self.set_save_status(False)
 
     def __getattribute__(self, item: str) -> Any:  # noqa: CCR001

tests/test_model_definition/test_pydantic_fields.py

@@ -1,9 +1,10 @@
+import random
 from typing import Optional
 
 import databases
 import pytest
 import sqlalchemy
-from pydantic import HttpUrl
+from pydantic import BaseModel, Field, HttpUrl, PaymentCardNumber
 
 import ormar
 from tests.settings import DATABASE_URL
@@ -21,15 +22,54 @@ class ModelTest(ormar.Model):
     class Meta(BaseMeta):
         pass
 
-    def __init__(self, **kwargs):
+    id: int = ormar.Integer(primary_key=True)
-        # you need to pop non - db fields as ormar will complain that it's unknown field
+    name: str = ormar.String(max_length=200)
-        url = kwargs.pop("url", self.__fields__["url"].get_default())
+    url: HttpUrl = "https://www.example.com"
-        super().__init__(**kwargs)
+    number: Optional[PaymentCardNumber]
-        self.url = url
+
+
+CARD_NUMBERS = [
+    "123456789007",
+    "123456789015",
+    "123456789023",
+    "123456789031",
+    "123456789049",
+]
+
+
+def get_number():
+    return random.choice(CARD_NUMBERS)
+
+
+class ModelTest2(ormar.Model):
+    class Meta(BaseMeta):
+        pass
 
     id: int = ormar.Integer(primary_key=True)
     name: str = ormar.String(max_length=200)
-    url: HttpUrl = "www.example.com"  # field with default
+    url: HttpUrl = "https://www.example2.com"
+    number: PaymentCardNumber = Field(default_factory=get_number)
+
+
+class PydanticTest(BaseModel):
+    aa: str
+    bb: int
+
+
+class ModelTest3(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    def __init__(self, **kwargs):
+        kwargs["number"] = get_number()
+        kwargs["pydantic_test"] = PydanticTest(aa="random", bb=42)
+        super().__init__(**kwargs)
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=200)
+    url: HttpUrl = "https://www.example3.com"
+    number: PaymentCardNumber
+    pydantic_test: PydanticTest
 
 
 @pytest.fixture(autouse=True, scope="module")
@@ -46,16 +86,56 @@ async def test_working_with_pydantic_fields():
     async with database:
         test = ModelTest(name="Test")
         assert test.name == "Test"
-        assert test.url == "www.example.com"
+        assert test.url == "https://www.example.com"
+        assert test.number is None
+        test.number = "123456789015"
 
-        test.url = "www.sdta.ada.pt"
+        test.url = "https://www.sdta.ada.pt"
-        assert test.url == "www.sdta.ada.pt"
+        assert test.url == "https://www.sdta.ada.pt"
 
         await test.save()
         test_check = await ModelTest.objects.get()
 
         assert test_check.name == "Test"
-        assert test_check.url == "www.example.com"
+        assert test_check.url == "https://www.example.com"
+        assert test_check.number is None
 
-        # TODO add validate assignment to pydantic config
+
-        # test_check.email = 1
+@pytest.mark.asyncio
+async def test_default_factory_for_pydantic_fields():
+    async with database:
+        test = ModelTest2(name="Test2", number="4000000000000002")
+        assert test.name == "Test2"
+        assert test.url == "https://www.example2.com"
+        assert test.number == "4000000000000002"
+
+        test.url = "http://www.sdta.ada.pt"
+        assert test.url == "http://www.sdta.ada.pt"
+
+        await test.save()
+        test_check = await ModelTest2.objects.get()
+
+        assert test_check.name == "Test2"
+        assert test_check.url == "https://www.example2.com"
+        assert test_check.number in CARD_NUMBERS
+        assert test_check.number != test.number
+
+
+@pytest.mark.asyncio
+async def test_init_setting_for_pydantic_fields():
+    async with database:
+        test = ModelTest3(name="Test3")
+        assert test.name == "Test3"
+        assert test.url == "https://www.example3.com"
+        assert test.pydantic_test.bb == 42
+
+        test.url = "http://www.sdta.ada.pt"
+        assert test.url == "http://www.sdta.ada.pt"
+
+        await test.save()
+        test_check = await ModelTest3.objects.get()
+
+        assert test_check.name == "Test3"
+        assert test_check.url == "https://www.example3.com"
+        assert test_check.number in CARD_NUMBERS
+        assert test_check.pydantic_test.aa == "random"