diff --git a/docs/models/methods.md b/docs/models/methods.md index 3b81559..e9cfe72 100644 --- a/docs/models/methods.md +++ b/docs/models/methods.md @@ -198,10 +198,88 @@ or it can be a dictionary that can also contain nested items. To read more about the structure of possible values passed to `exclude` check `Queryset.fields` method documentation. !!!warning - To avoid circular updates with `follow=True` set, `save_related` keeps a set of already visited Models, + To avoid circular updates with `follow=True` set, `save_related` keeps a set of already visited Models on each branch of relation tree, and won't perform nested `save_related` on Models that were already visited. - So if you have a diamond or circular relations types you need to perform the updates in a manual way. + So if you have circular relations types you need to perform the updates in a manual way. + +Note that with `save_all=True` and `follow=True` you can use `save_related()` to save whole relation tree at once. + +Example: + +```python +class Department(ormar.Model): + class Meta: + database = database + metadata = metadata + + id: int = ormar.Integer(primary_key=True) + department_name: str = ormar.String(max_length=100) + + +class Course(ormar.Model): + class Meta: + database = database + metadata = metadata + + id: int = ormar.Integer(primary_key=True) + course_name: str = ormar.String(max_length=100) + completed: bool = ormar.Boolean() + department: Optional[Department] = ormar.ForeignKey(Department) + + +class Student(ormar.Model): + class Meta: + database = database + metadata = metadata + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + courses = ormar.ManyToMany(Course) + +to_save = { + "department_name": "Ormar", + "courses": [ + {"course_name": "basic1", + "completed": True, + "students": [ + {"name": "Jack"}, + {"name": "Abi"} + ]}, + {"course_name": "basic2", + "completed": True, + "students": [ + {"name": "Kate"}, + {"name": "Miranda"} + ] + }, + ], + } +# initializa whole tree +department = Department(**to_save) + +# save all at once (one after another) +await department.save_related(follow=True, save_all=True) + +department_check = await Department.objects.select_all(follow=True).get() + +to_exclude = { + "id": ..., + "courses": { + "id": ..., + "students": {"id", "studentcourse"} + } +} +# after excluding ids and through models you get exact same payload used to +# construct whole tree +assert department_check.dict(exclude=to_exclude) == to_save + +``` + + +!!!warning + `save_related()` iterates all relations and all models and upserts() them one by one, + so it will save all models but might not be optimal in regard of number of database queries. [fields]: ../fields.md [relations]: ../relations/index.md diff --git a/docs/relations/foreign-key.md b/docs/relations/foreign-key.md index 73977f7..1bdc4f4 100644 --- a/docs/relations/foreign-key.md +++ b/docs/relations/foreign-key.md @@ -27,6 +27,66 @@ By default it's child (source) `Model` name + s, like courses in snippet below: Reverse relation exposes API to manage related objects also from parent side. +### Skipping reverse relation + +If you are sure you don't want the reverse relation you can use `skip_reverse=True` +flag of the `ForeignKey`. + + If you set `skip_reverse` flag internally the field is still registered on the other + side of the relationship so you can: + * `filter` by related models fields from reverse model + * `order_by` by related models fields from reverse model + + But you cannot: + * access the related field from reverse model with `related_name` + * even if you `select_related` from reverse side of the model the returned models won't be populated in reversed instance (the join is not prevented so you still can `filter` and `order_by` over the relation) + * the relation won't be populated in `dict()` and `json()` + * you cannot pass the nested related objects when populating from dictionary or json (also through `fastapi`). It will be either ignored or error will be raised depending on `extra` setting in pydantic `Config`. + +Example: + +```python +class Author(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + first_name: str = ormar.String(max_length=80) + last_name: str = ormar.String(max_length=80) + + +class Post(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + title: str = ormar.String(max_length=200) + author: Optional[Author] = ormar.ForeignKey(Author, skip_reverse=True) + +# create sample data +author = Author(first_name="Test", last_name="Author") +post = Post(title="Test Post", author=author) + +assert post.author == author # ok +assert author.posts # Attribute error! + +# but still can use in order_by +authors = ( + await Author.objects.select_related("posts").order_by("posts__title").all() +) +assert authors[0].first_name == "Test" + +# note that posts are not populated for author even if explicitly +# included in select_related - note no posts in dict() +assert author.dict(exclude={"id"}) == {"first_name": "Test", "last_name": "Author"} + +# still can filter through fields of related model +authors = await Author.objects.filter(posts__title="Test Post").all() +assert authors[0].first_name == "Test" +assert len(authors) == 1 +``` + + ### add Adding child model from parent side causes adding related model to currently loaded parent relation, diff --git a/docs/relations/many-to-many.md b/docs/relations/many-to-many.md index 24be745..414f0df 100644 --- a/docs/relations/many-to-many.md +++ b/docs/relations/many-to-many.md @@ -20,6 +20,122 @@ post = await Post.objects.create(title="Hello, M2M", author=guido) news = await Category.objects.create(name="News") ``` +## Reverse relation + +`ForeignKey` fields are automatically registering reverse side of the relation. + +By default it's child (source) `Model` name + s, like courses in snippet below: + +```python +class Category(ormar.Model): + class Meta(BaseMeta): + tablename = "categories" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=40) + + +class Post(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + title: str = ormar.String(max_length=200) + categories: Optional[List[Category]] = ormar.ManyToMany(Category) + +# create some sample data +post = await Post.objects.create(title="Hello, M2M") +news = await Category.objects.create(name="News") +await post.categories.add(news) + +# now you can query and access from both sides: +post_check = Post.objects.select_related("categories").get() +assert post_check.categories[0] == news + +# query through auto registered reverse side +category_check = Category.objects.select_related("posts").get() +assert category_check.posts[0] == post +``` + +Reverse relation exposes API to manage related objects also from parent side. + +### related_name + +By default, the related_name is generated in the same way as for the `ForeignKey` relation (class.name.lower()+'s'), +but in the same way you can overwrite this name by providing `related_name` parameter like below: + +```Python +categories: Optional[Union[Category, List[Category]]] = ormar.ManyToMany( + Category, through=PostCategory, related_name="new_categories" + ) +``` + +!!!warning + When you provide multiple relations to the same model `ormar` can no longer auto generate + the `related_name` for you. Therefore, in that situation you **have to** provide `related_name` + for all but one (one can be default and generated) or all related fields. + + +### Skipping reverse relation + +If you are sure you don't want the reverse relation you can use `skip_reverse=True` +flag of the `ManyToMany`. + + If you set `skip_reverse` flag internally the field is still registered on the other + side of the relationship so you can: + * `filter` by related models fields from reverse model + * `order_by` by related models fields from reverse model + + But you cannot: + * access the related field from reverse model with `related_name` + * even if you `select_related` from reverse side of the model the returned models won't be populated in reversed instance (the join is not prevented so you still can `filter` and `order_by` over the relation) + * the relation won't be populated in `dict()` and `json()` + * you cannot pass the nested related objects when populating from dictionary or json (also through `fastapi`). It will be either ignored or error will be raised depending on `extra` setting in pydantic `Config`. + +Example: + +```python +class Category(ormar.Model): + class Meta(BaseMeta): + tablename = "categories" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=40) + + +class Post(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + title: str = ormar.String(max_length=200) + categories: Optional[List[Category]] = ormar.ManyToMany(Category, skip_reverse=True) + +# create some sample data +post = await Post.objects.create(title="Hello, M2M") +news = await Category.objects.create(name="News") +await post.categories.add(news) + +assert post.categories[0] == news # ok +assert news.posts # Attribute error! + +# but still can use in order_by +categories = ( + await Category.objects.select_related("posts").order_by("posts__title").all() +) +assert categories[0].first_name == "Test" + +# note that posts are not populated for author even if explicitly +# included in select_related - note no posts in dict() +assert news.dict(exclude={"id"}) == {"name": "News"} + +# still can filter through fields of related model +categories = await Category.objects.filter(posts__title="Hello, M2M").all() +assert categories[0].name == "News" +assert len(categories) == 1 +``` + + ## Through Model Optionally if you want to add additional fields you can explicitly create and pass @@ -220,22 +336,6 @@ Reverse relation exposes QuerysetProxy API that allows you to query related mode To read which methods of QuerySet are available read below [querysetproxy][querysetproxy] -## related_name - -By default, the related_name is generated in the same way as for the `ForeignKey` relation (class.name.lower()+'s'), -but in the same way you can overwrite this name by providing `related_name` parameter like below: - -```Python -categories: Optional[Union[Category, List[Category]]] = ormar.ManyToMany( - Category, through=PostCategory, related_name="new_categories" - ) -``` - -!!!warning - When you provide multiple relations to the same model `ormar` can no longer auto generate - the `related_name` for you. Therefore, in that situation you **have to** provide `related_name` - for all but one (one can be default and generated) or all related fields. - [queries]: ./queries.md [querysetproxy]: ./queryset-proxy.md diff --git a/docs/releases.md b/docs/releases.md index b9becc1..667f170 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -30,10 +30,12 @@ * Fix weakref `ReferenceError` error [#118](https://github.com/collerek/ormar/issues/118) * Fix error raised by Through fields when pydantic `Config.extra="forbid"` is set * Fix bug with `pydantic.PrivateAttr` not being initialized at `__init__` [#149](https://github.com/collerek/ormar/issues/149) +* Fix bug with pydantic-type `exclude` in `dict()` with `__all__` key not working ## 💬 Other * Introduce link to `sqlalchemy-to-ormar` auto-translator for models * Provide links to fastapi ecosystem libraries that support `ormar` +* Add transactions to docs (supported with `databases`) # 0.10.2 diff --git a/docs/transactions.md b/docs/transactions.md new file mode 100644 index 0000000..2b8a904 --- /dev/null +++ b/docs/transactions.md @@ -0,0 +1,88 @@ +# Transactions + +Database transactions are supported thanks to `encode/databases` which is used to issue async queries. + +## Basic usage + +To use transactions use `database.transaction` as async context manager: + +```python +async with database.transaction(): + # everyting called here will be one transaction + await Model1().save() + await Model2().save() + ... +``` + +!!!note + Note that it has to be the same `database` that the one used in Model's `Meta` class. + +To avoid passing `database` instance around in your code you can extract the instance from each `Model`. +Database provided during declaration of `ormar.Model` is available through `Meta.database` and can +be reached from both class and instance. + +```python +import databases +import sqlalchemy +import ormar + +metadata = sqlalchemy.MetaData() +database = databases.Database("sqlite:///") + +class Author(ormar.Model): + class Meta: + database=database + metadata=metadata + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=255) + +# database is accessible from class +database = Author.Meta.database + +# as well as from instance +author = Author(name="Stephen King") +database = author.Meta.database + +``` + +You can also use `.transaction()` as a function decorator on any async function: + +```python +@database.transaction() +async def create_users(request): + ... +``` + +Transaction blocks are managed as task-local state. Nested transactions +are fully supported, and are implemented using database savepoints. + +## Manual commits/ rollbacks + +For a lower-level transaction API you can trigger it manually + +```python +transaction = await database.transaction() +try: + await transaction.start() + ... +except: + await transaction.rollback() +else: + await transaction.commit() +``` + + +## Testing + +Transactions can also be useful during testing when you can apply force rollback +and you do not have to clean the data after each test. + +```python +@pytest.mark.asyncio +async def sample_test(): + async with database: + async with database.transaction(force_rollback=True): + # your test code here + ... +``` \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 5432018..65d5215 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,6 +31,7 @@ nav: - queries/pagination-and-rows-number.md - queries/aggregations.md - Signals: signals.md + - Transactions: transactions.md - Use with Fastapi: fastapi.md - Use with mypy: mypy.md - PyCharm plugin: plugin.md diff --git a/ormar/models/mixins/excludable_mixin.py b/ormar/models/mixins/excludable_mixin.py index 3a2bb04..cbe4c25 100644 --- a/ormar/models/mixins/excludable_mixin.py +++ b/ormar/models/mixins/excludable_mixin.py @@ -14,7 +14,6 @@ from typing import ( from ormar.models.excludable import ExcludableItems from ormar.models.mixins.relation_mixin import RelationMixin -from ormar.queryset.utils import translate_list_to_dict, update if TYPE_CHECKING: # pragma no cover from ormar import Model @@ -138,9 +137,7 @@ class ExcludableMixin(RelationMixin): return columns @classmethod - def _update_excluded_with_related( - cls, exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None], - ) -> Union[Set, Dict]: + def _update_excluded_with_related(cls, exclude: Union[Set, Dict, None],) -> Set: """ Used during generation of the dict(). To avoid cyclical references and max recurrence limit nested models have to @@ -151,8 +148,6 @@ class ExcludableMixin(RelationMixin): :param exclude: set/dict with fields to exclude :type exclude: Union[Set, Dict, None] - :param nested: flag setting nested models (child of previous one, not main one) - :type nested: bool :return: set or dict with excluded fields added. :rtype: Union[Set, Dict] """ @@ -160,10 +155,11 @@ class ExcludableMixin(RelationMixin): related_set = cls.extract_related_names() if isinstance(exclude, set): exclude = {s for s in exclude} - exclude.union(related_set) - else: - related_dict = translate_list_to_dict(related_set) - exclude = update(related_dict, exclude) + exclude = exclude.union(related_set) + elif isinstance(exclude, dict): + # relations are handled in ormar - take only own fields (ellipsis in dict) + exclude = {k for k, v in exclude.items() if v is Ellipsis} + exclude = exclude.union(related_set) return exclude @classmethod diff --git a/ormar/models/mixins/save_mixin.py b/ormar/models/mixins/save_mixin.py index 5ff29ad..a7ac562 100644 --- a/ormar/models/mixins/save_mixin.py +++ b/ormar/models/mixins/save_mixin.py @@ -222,9 +222,7 @@ class SavePrepareMixin(RelationMixin, AliasMixin): @staticmethod async def _upsert_through_model( - instance: "Model", - previous_model: "Model", - relation_field: "ForeignKeyField", + instance: "Model", previous_model: "Model", relation_field: "ForeignKeyField", ) -> None: """ Upsert through model for m2m relation. diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index af69e38..4e16fd0 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -514,7 +514,11 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass fields = [ field for field in fields - if field not in exclude or exclude.get(field) is not Ellipsis + if field not in exclude + or ( + exclude.get(field) is not Ellipsis + and exclude.get(field) != {"__all__"} + ) ] return fields @@ -567,6 +571,18 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass result = self.get_child(items, key) return result if result is not Ellipsis else default_return + def _convert_all(self, items: Union[Set, Dict, None]) -> Union[Set, Dict, None]: + """ + Helper to convert __all__ pydantic special index to ormar which does not + support index based exclusions. + + :param items: current include/exclude value + :type items: Union[Set, Dict, None] + """ + if isinstance(items, dict) and "__all__" in items: + return items.get("__all__") + return items + def _extract_nested_models( # noqa: CCR001 self, relation_map: Dict, @@ -603,8 +619,8 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass relation_map, field, default_return=dict() ), models=nested_model, - include=self._skip_ellipsis(include, field), - exclude=self._skip_ellipsis(exclude, field), + include=self._convert_all(self._skip_ellipsis(include, field)), + exclude=self._convert_all(self._skip_ellipsis(exclude, field)), ) elif nested_model is not None: @@ -612,8 +628,8 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass relation_map=self._skip_ellipsis( relation_map, field, default_return=dict() ), - include=self._skip_ellipsis(include, field), - exclude=self._skip_ellipsis(exclude, field), + include=self._convert_all(self._skip_ellipsis(include, field)), + exclude=self._convert_all(self._skip_ellipsis(exclude, field)), ) else: dict_instance[field] = None diff --git a/tests/test_fastapi/test_nested_saving.py b/tests/test_fastapi/test_nested_saving.py index bb388e7..c4d817b 100644 --- a/tests/test_fastapi/test_nested_saving.py +++ b/tests/test_fastapi/test_nested_saving.py @@ -1,5 +1,5 @@ import json -from typing import List, Optional +from typing import Optional import databases import pytest @@ -50,6 +50,16 @@ class Course(ormar.Model): department: Optional[Department] = ormar.ForeignKey(Department) +class Student(ormar.Model): + class Meta: + database = database + metadata = metadata + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + courses = ormar.ManyToMany(Course) + + # create db and tables @pytest.fixture(autouse=True, scope="module") def create_test_database(): @@ -59,19 +69,49 @@ def create_test_database(): metadata.drop_all(engine) -@app.post("/DepartmentWithCourses/", response_model=Department) +to_exclude = { + "id": ..., + "courses": { + "__all__": {"id": ..., "students": {"__all__": {"id", "studentcourse"}}} + }, +} + +exclude_all = {"id": ..., "courses": {"__all__"}} + +to_exclude_ormar = { + "id": ..., + "courses": {"id": ..., "students": {"id", "studentcourse"}}, +} + + +@app.post("/departments/", response_model=Department) async def create_department(department: Department): - # there is no save all - you need to split into save and save_related - await department.save() await department.save_related(follow=True, save_all=True) return department -@app.get("/DepartmentsAll/", response_model=List[Department]) -async def get_Courses(): - # if you don't provide default name it related model name + s so courses not course - departmentall = await Department.objects.select_related("courses").all() - return departmentall +@app.get("/departments/{department_name}") +async def get_department(department_name: str): + department = await Department.objects.select_all(follow=True).get( + department_name=department_name + ) + return department.dict(exclude=to_exclude) + + +@app.get("/departments/{department_name}/second") +async def get_department_exclude(department_name: str): + department = await Department.objects.select_all(follow=True).get( + department_name=department_name + ) + return department.dict(exclude=to_exclude_ormar) + + +@app.get("/departments/{department_name}/exclude") +async def get_department_exclude_all(department_name: str): + department = await Department.objects.select_all(follow=True).get( + department_name=department_name + ) + return department.dict(exclude=exclude_all) def test_saving_related_in_fastapi(): @@ -80,11 +120,19 @@ def test_saving_related_in_fastapi(): payload = { "department_name": "Ormar", "courses": [ - {"course_name": "basic1", "completed": True}, - {"course_name": "basic2", "completed": True}, + { + "course_name": "basic1", + "completed": True, + "students": [{"name": "Jack"}, {"name": "Abi"}], + }, + { + "course_name": "basic2", + "completed": True, + "students": [{"name": "Kate"}, {"name": "Miranda"}], + }, ], } - response = client.post("/DepartmentWithCourses/", data=json.dumps(payload)) + response = client.post("/departments/", data=json.dumps(payload)) department = Department(**response.json()) assert department.id is not None @@ -95,12 +143,9 @@ def test_saving_related_in_fastapi(): assert department.courses[1].course_name == "basic2" assert department.courses[1].completed - response = client.get("/DepartmentsAll/") - departments = [Department(**x) for x in response.json()] - assert departments[0].id is not None - assert len(departments[0].courses) == 2 - assert departments[0].department_name == "Ormar" - assert departments[0].courses[0].course_name == "basic1" - assert departments[0].courses[0].completed - assert departments[0].courses[1].course_name == "basic2" - assert departments[0].courses[1].completed + response = client.get("/departments/Ormar") + response2 = client.get("/departments/Ormar/second") + assert response.json() == response2.json() == payload + + response3 = client.get("/departments/Ormar/exclude") + assert response3.json() == {"department_name": "Ormar"}