fix __all__ error in exclude, update docs

docs/models/methods.md

@@ -198,10 +198,88 @@ or it can be a dictionary that can also contain nested items.
         To read more about the structure of possible values passed to `exclude` check `Queryset.fields` method documentation.
 
 !!!warning
-    To avoid circular updates with `follow=True` set, `save_related` keeps a set of already visited Models, 
+    To avoid circular updates with `follow=True` set, `save_related` keeps a set of already visited Models on each branch of relation tree, 
     and won't perform nested `save_related` on Models that were already visited.
     
-    So if you have a diamond or circular relations types you need to perform the updates in a manual way.
+    So if you have circular relations types you need to perform the updates in a manual way.
+
+Note that with `save_all=True` and `follow=True` you can use `save_related()` to save whole relation tree at once.
+
+Example:
+
+```python
+class Department(ormar.Model):
+    class Meta:
+        database = database
+        metadata = metadata
+
+    id: int = ormar.Integer(primary_key=True)
+    department_name: str = ormar.String(max_length=100)
+
+
+class Course(ormar.Model):
+    class Meta:
+        database = database
+        metadata = metadata
+
+    id: int = ormar.Integer(primary_key=True)
+    course_name: str = ormar.String(max_length=100)
+    completed: bool = ormar.Boolean()
+    department: Optional[Department] = ormar.ForeignKey(Department)
+
+
+class Student(ormar.Model):
+    class Meta:
+        database = database
+        metadata = metadata
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    courses = ormar.ManyToMany(Course)
+
+to_save = {
+            "department_name": "Ormar",
+            "courses": [
+                {"course_name": "basic1",
+                 "completed": True,
+                 "students": [
+                     {"name": "Jack"},
+                     {"name": "Abi"}
+                 ]},
+                {"course_name": "basic2",
+                 "completed": True,
+                 "students": [
+                     {"name": "Kate"},
+                     {"name": "Miranda"}
+                 ]
+                 },
+            ],
+        }
+# initializa whole tree
+department = Department(**to_save)
+
+# save all at once (one after another)
+await department.save_related(follow=True, save_all=True)
+
+department_check = await Department.objects.select_all(follow=True).get()
+
+to_exclude = {
+    "id": ...,
+    "courses": {
+        "id": ...,
+        "students": {"id", "studentcourse"}
+    }
+}
+# after excluding ids and through models you get exact same payload used to
+# construct whole tree
+assert department_check.dict(exclude=to_exclude) == to_save
+
+```
+
+
+!!!warning
+    `save_related()` iterates all relations and all models and upserts() them one by one,
+    so it will save all models but might not be optimal in regard of number of database queries.
 
 [fields]: ../fields.md
 [relations]: ../relations/index.md

docs/relations/foreign-key.md

@@ -27,6 +27,66 @@ By default it's child (source) `Model` name + s, like courses in snippet below:
 
 Reverse relation exposes API to manage related objects also from parent side.
 
+### Skipping reverse relation
+
+If you are sure you don't want the reverse relation you can use `skip_reverse=True` 
+flag of the `ForeignKey`.
+
+  If you set `skip_reverse` flag internally the field is still registered on the other 
+  side of the relationship so you can:
+  * `filter` by related models fields from reverse model
+  * `order_by` by related models fields from reverse model 
+  
+  But you cannot:
+  * access the related field from reverse model with `related_name`
+  * even if you `select_related` from reverse side of the model the returned models won't be populated in reversed instance (the join is not prevented so you still can `filter` and `order_by` over the relation)
+  * the relation won't be populated in `dict()` and `json()`
+  * you cannot pass the nested related objects when populating from dictionary or json (also through `fastapi`). It will be either ignored or error will be raised depending on `extra` setting in pydantic `Config`.
+
+Example:
+
+```python
+class Author(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    first_name: str = ormar.String(max_length=80)
+    last_name: str = ormar.String(max_length=80)
+
+
+class Post(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    title: str = ormar.String(max_length=200)
+    author: Optional[Author] = ormar.ForeignKey(Author, skip_reverse=True)
+
+# create sample data
+author = Author(first_name="Test", last_name="Author")
+post = Post(title="Test Post", author=author)
+
+assert post.author == author  # ok
+assert author.posts  # Attribute error!
+
+# but still can use in order_by
+authors = (
+    await Author.objects.select_related("posts").order_by("posts__title").all()
+)
+assert authors[0].first_name == "Test"
+
+# note that posts are not populated for author even if explicitly
+# included in select_related - note no posts in dict()
+assert author.dict(exclude={"id"}) == {"first_name": "Test", "last_name": "Author"}
+
+# still can filter through fields of related model
+authors = await Author.objects.filter(posts__title="Test Post").all()
+assert authors[0].first_name == "Test"
+assert len(authors) == 1
+```
+
+
 ### add
 
 Adding child model from parent side causes adding related model to currently loaded parent relation, 

docs/relations/many-to-many.md

@@ -20,6 +20,122 @@ post = await Post.objects.create(title="Hello, M2M", author=guido)
 news = await Category.objects.create(name="News")
 ```
 
+## Reverse relation
+
+`ForeignKey` fields are automatically registering reverse side of the relation.
+
+By default it's child (source) `Model` name + s, like courses in snippet below: 
+
+```python
+class Category(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "categories"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=40)
+
+
+class Post(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    title: str = ormar.String(max_length=200)
+    categories: Optional[List[Category]] = ormar.ManyToMany(Category)
+
+# create some sample data
+post = await Post.objects.create(title="Hello, M2M")
+news = await Category.objects.create(name="News")
+await post.categories.add(news)
+    
+# now you can query and access from both sides:
+post_check = Post.objects.select_related("categories").get()
+assert post_check.categories[0] == news
+
+# query through auto registered reverse side
+category_check = Category.objects.select_related("posts").get()
+assert category_check.posts[0] == post
+```
+
+Reverse relation exposes API to manage related objects also from parent side.
+
+### related_name
+
+By default, the related_name is generated in the same way as for the `ForeignKey` relation (class.name.lower()+'s'), 
+but in the same way you can overwrite this name by providing `related_name` parameter like below:
+
+```Python
+categories: Optional[Union[Category, List[Category]]] = ormar.ManyToMany(
+        Category, through=PostCategory, related_name="new_categories"
+    )
+```
+
+!!!warning
+    When you provide multiple relations to the same model `ormar` can no longer auto generate
+    the `related_name` for you. Therefore, in that situation you **have to** provide `related_name`
+    for all but one (one can be default and generated) or all related fields.
+
+
+### Skipping reverse relation
+
+If you are sure you don't want the reverse relation you can use `skip_reverse=True` 
+flag of the `ManyToMany`.
+
+  If you set `skip_reverse` flag internally the field is still registered on the other 
+  side of the relationship so you can:
+  * `filter` by related models fields from reverse model
+  * `order_by` by related models fields from reverse model 
+  
+  But you cannot:
+  * access the related field from reverse model with `related_name`
+  * even if you `select_related` from reverse side of the model the returned models won't be populated in reversed instance (the join is not prevented so you still can `filter` and `order_by` over the relation)
+  * the relation won't be populated in `dict()` and `json()`
+  * you cannot pass the nested related objects when populating from dictionary or json (also through `fastapi`). It will be either ignored or error will be raised depending on `extra` setting in pydantic `Config`.
+
+Example:
+
+```python
+class Category(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "categories"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=40)
+
+
+class Post(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    title: str = ormar.String(max_length=200)
+    categories: Optional[List[Category]] = ormar.ManyToMany(Category, skip_reverse=True)
+
+# create some sample data
+post = await Post.objects.create(title="Hello, M2M")
+news = await Category.objects.create(name="News")
+await post.categories.add(news)
+
+assert post.categories[0] == news  # ok
+assert news.posts  # Attribute error!
+
+# but still can use in order_by
+categories = (
+    await Category.objects.select_related("posts").order_by("posts__title").all()
+)
+assert categories[0].first_name == "Test"
+
+# note that posts are not populated for author even if explicitly
+# included in select_related - note no posts in dict()
+assert news.dict(exclude={"id"}) == {"name": "News"}
+
+# still can filter through fields of related model
+categories = await Category.objects.filter(posts__title="Hello, M2M").all()
+assert categories[0].name == "News"
+assert len(categories) == 1
+```
+
+
 ## Through Model
 
 Optionally if you want to add additional fields you can explicitly create and pass
@@ -220,22 +336,6 @@ Reverse relation exposes QuerysetProxy API that allows you to query related mode
 
 To read which methods of QuerySet are available read below [querysetproxy][querysetproxy]
 
-## related_name
-
-By default, the related_name is generated in the same way as for the `ForeignKey` relation (class.name.lower()+'s'), 
-but in the same way you can overwrite this name by providing `related_name` parameter like below:
-
-```Python
-categories: Optional[Union[Category, List[Category]]] = ormar.ManyToMany(
-        Category, through=PostCategory, related_name="new_categories"
-    )
-```
-
-!!!warning
-    When you provide multiple relations to the same model `ormar` can no longer auto generate
-    the `related_name` for you. Therefore, in that situation you **have to** provide `related_name`
-    for all but one (one can be default and generated) or all related fields.
-
 
 [queries]: ./queries.md
 [querysetproxy]: ./queryset-proxy.md

docs/releases.md

@@ -30,10 +30,12 @@
 *  Fix weakref `ReferenceError` error [#118](https://github.com/collerek/ormar/issues/118)
 *  Fix error raised by Through fields when pydantic `Config.extra="forbid"` is set
 *  Fix bug with `pydantic.PrivateAttr` not being initialized at `__init__` [#149](https://github.com/collerek/ormar/issues/149)
+*  Fix bug with pydantic-type `exclude` in `dict()` with `__all__` key not working
 
 ## 💬 Other
 *  Introduce link to `sqlalchemy-to-ormar` auto-translator for models
 *  Provide links to fastapi ecosystem libraries that support `ormar`
+*  Add transactions to docs (supported with `databases`)
 
 
 # 0.10.2

docs/transactions.md

@@ -0,0 +1,88 @@
+# Transactions
+
+Database transactions are supported thanks to `encode/databases` which is used to issue async queries.
+
+## Basic usage
+
+To use transactions use `database.transaction` as async context manager:
+
+```python
+async with database.transaction():
+    # everyting called here will be one transaction
+    await Model1().save()
+    await Model2().save()
+    ...
+```
+
+!!!note
+    Note that it has to be the same `database` that the one used in Model's `Meta` class.
+
+To avoid passing `database` instance around in your code you can extract the instance from each `Model`.
+Database provided during declaration of `ormar.Model` is available through `Meta.database` and can
+be reached from both class and instance.
+
+```python
+import databases
+import sqlalchemy
+import ormar
+
+metadata = sqlalchemy.MetaData()
+database = databases.Database("sqlite:///")
+
+class Author(ormar.Model):
+    class Meta:
+        database=database
+        metadata=metadata
+    
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=255)
+
+# database is accessible from class
+database = Author.Meta.database
+
+# as well as from instance
+author = Author(name="Stephen King")
+database = author.Meta.database
+
+```
+
+You can also use `.transaction()` as a function decorator on any async function:
+
+```python
+@database.transaction()
+async def create_users(request):
+    ...
+```
+
+Transaction blocks are managed as task-local state. Nested transactions
+are fully supported, and are implemented using database savepoints.
+
+## Manual commits/ rollbacks
+
+For a lower-level transaction API you can trigger it manually
+
+```python
+transaction = await database.transaction()
+try:
+    await transaction.start()
+    ...
+except:
+    await transaction.rollback()
+else:
+    await transaction.commit()
+```
+
+
+## Testing
+
+Transactions can also be useful during testing when you can apply force rollback 
+and you do not have to clean the data after each test.
+
+```python
+@pytest.mark.asyncio
+async def sample_test():
+    async with database:
+        async with database.transaction(force_rollback=True):
+            # your test code here
+            ...
+```

mkdocs.yml

@@ -31,6 +31,7 @@ nav:
     - queries/pagination-and-rows-number.md
     - queries/aggregations.md
   - Signals: signals.md
+  - Transactions: transactions.md
   - Use with Fastapi: fastapi.md
   - Use with mypy: mypy.md
   - PyCharm plugin: plugin.md

ormar/models/mixins/excludable_mixin.py

@@ -14,7 +14,6 @@ from typing import (
 
 from ormar.models.excludable import ExcludableItems
 from ormar.models.mixins.relation_mixin import RelationMixin
-from ormar.queryset.utils import translate_list_to_dict, update
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
@@ -138,9 +137,7 @@ class ExcludableMixin(RelationMixin):
         return columns
 
     @classmethod
-    def _update_excluded_with_related(
+    def _update_excluded_with_related(cls, exclude: Union[Set, Dict, None],) -> Set:
-        cls, exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None],
-    ) -> Union[Set, Dict]:
         """
         Used during generation of the dict().
         To avoid cyclical references and max recurrence limit nested models have to
@@ -151,8 +148,6 @@ class ExcludableMixin(RelationMixin):
 
         :param exclude: set/dict with fields to exclude
         :type exclude: Union[Set, Dict, None]
-        :param nested: flag setting nested models (child of previous one, not main one)
-        :type nested: bool
         :return: set or dict with excluded fields added.
         :rtype: Union[Set, Dict]
         """
@@ -160,10 +155,11 @@ class ExcludableMixin(RelationMixin):
         related_set = cls.extract_related_names()
         if isinstance(exclude, set):
             exclude = {s for s in exclude}
-            exclude.union(related_set)
+            exclude = exclude.union(related_set)
-        else:
+        elif isinstance(exclude, dict):
-            related_dict = translate_list_to_dict(related_set)
+            # relations are handled in ormar - take only own fields (ellipsis in dict)
-            exclude = update(related_dict, exclude)
+            exclude = {k for k, v in exclude.items() if v is Ellipsis}
+            exclude = exclude.union(related_set)
         return exclude
 
     @classmethod

ormar/models/mixins/save_mixin.py

@@ -222,9 +222,7 @@ class SavePrepareMixin(RelationMixin, AliasMixin):
 
     @staticmethod
     async def _upsert_through_model(
-        instance: "Model",
+        instance: "Model", previous_model: "Model", relation_field: "ForeignKeyField",
-        previous_model: "Model",
-        relation_field: "ForeignKeyField",
     ) -> None:
         """
         Upsert through model for m2m relation.

ormar/models/newbasemodel.py

@@ -514,7 +514,11 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
             fields = [
                 field
                 for field in fields
-                if field not in exclude or exclude.get(field) is not Ellipsis
+                if field not in exclude
+                or (
+                    exclude.get(field) is not Ellipsis
+                    and exclude.get(field) != {"__all__"}
+                )
             ]
         return fields
 
@@ -567,6 +571,18 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
         result = self.get_child(items, key)
         return result if result is not Ellipsis else default_return
 
+    def _convert_all(self, items: Union[Set, Dict, None]) -> Union[Set, Dict, None]:
+        """
+        Helper to convert __all__ pydantic special index to ormar which does not
+        support index based exclusions.
+
+        :param items: current include/exclude value
+        :type items: Union[Set, Dict, None]
+        """
+        if isinstance(items, dict) and "__all__" in items:
+            return items.get("__all__")
+        return items
+
     def _extract_nested_models(  # noqa: CCR001
         self,
         relation_map: Dict,
@@ -603,8 +619,8 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
                             relation_map, field, default_return=dict()
                         ),
                         models=nested_model,
-                        include=self._skip_ellipsis(include, field),
+                        include=self._convert_all(self._skip_ellipsis(include, field)),
-                        exclude=self._skip_ellipsis(exclude, field),
+                        exclude=self._convert_all(self._skip_ellipsis(exclude, field)),
                     )
                 elif nested_model is not None:
 
@@ -612,8 +628,8 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
                         relation_map=self._skip_ellipsis(
                             relation_map, field, default_return=dict()
                         ),
-                        include=self._skip_ellipsis(include, field),
+                        include=self._convert_all(self._skip_ellipsis(include, field)),
-                        exclude=self._skip_ellipsis(exclude, field),
+                        exclude=self._convert_all(self._skip_ellipsis(exclude, field)),
                     )
                 else:
                     dict_instance[field] = None

tests/test_fastapi/test_nested_saving.py

@@ -1,5 +1,5 @@
 import json
-from typing import List, Optional
+from typing import Optional
 
 import databases
 import pytest
@@ -50,6 +50,16 @@ class Course(ormar.Model):
     department: Optional[Department] = ormar.ForeignKey(Department)
 
 
+class Student(ormar.Model):
+    class Meta:
+        database = database
+        metadata = metadata
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    courses = ormar.ManyToMany(Course)
+
+
 # create db and tables
 @pytest.fixture(autouse=True, scope="module")
 def create_test_database():
@@ -59,19 +69,49 @@ def create_test_database():
     metadata.drop_all(engine)
 
 
-@app.post("/DepartmentWithCourses/", response_model=Department)
+to_exclude = {
+    "id": ...,
+    "courses": {
+        "__all__": {"id": ..., "students": {"__all__": {"id", "studentcourse"}}}
+    },
+}
+
+exclude_all = {"id": ..., "courses": {"__all__"}}
+
+to_exclude_ormar = {
+    "id": ...,
+    "courses": {"id": ..., "students": {"id", "studentcourse"}},
+}
+
+
+@app.post("/departments/", response_model=Department)
 async def create_department(department: Department):
-    # there is no save all - you need to split into save and save_related
-    await department.save()
     await department.save_related(follow=True, save_all=True)
     return department
 
 
-@app.get("/DepartmentsAll/", response_model=List[Department])
+@app.get("/departments/{department_name}")
-async def get_Courses():
+async def get_department(department_name: str):
-    # if you don't provide default name it related model name + s so courses not course
+    department = await Department.objects.select_all(follow=True).get(
-    departmentall = await Department.objects.select_related("courses").all()
+        department_name=department_name
-    return departmentall
+    )
+    return department.dict(exclude=to_exclude)
+
+
+@app.get("/departments/{department_name}/second")
+async def get_department_exclude(department_name: str):
+    department = await Department.objects.select_all(follow=True).get(
+        department_name=department_name
+    )
+    return department.dict(exclude=to_exclude_ormar)
+
+
+@app.get("/departments/{department_name}/exclude")
+async def get_department_exclude_all(department_name: str):
+    department = await Department.objects.select_all(follow=True).get(
+        department_name=department_name
+    )
+    return department.dict(exclude=exclude_all)
 
 
 def test_saving_related_in_fastapi():
@@ -80,11 +120,19 @@ def test_saving_related_in_fastapi():
         payload = {
             "department_name": "Ormar",
             "courses": [
-                {"course_name": "basic1", "completed": True},
+                {
-                {"course_name": "basic2", "completed": True},
+                    "course_name": "basic1",
+                    "completed": True,
+                    "students": [{"name": "Jack"}, {"name": "Abi"}],
+                },
+                {
+                    "course_name": "basic2",
+                    "completed": True,
+                    "students": [{"name": "Kate"}, {"name": "Miranda"}],
+                },
             ],
         }
-        response = client.post("/DepartmentWithCourses/", data=json.dumps(payload))
+        response = client.post("/departments/", data=json.dumps(payload))
         department = Department(**response.json())
 
         assert department.id is not None
@@ -95,12 +143,9 @@ def test_saving_related_in_fastapi():
         assert department.courses[1].course_name == "basic2"
         assert department.courses[1].completed
 
-        response = client.get("/DepartmentsAll/")
+        response = client.get("/departments/Ormar")
-        departments = [Department(**x) for x in response.json()]
+        response2 = client.get("/departments/Ormar/second")
-        assert departments[0].id is not None
+        assert response.json() == response2.json() == payload
-        assert len(departments[0].courses) == 2
+
-        assert departments[0].department_name == "Ormar"
+        response3 = client.get("/departments/Ormar/exclude")
-        assert departments[0].courses[0].course_name == "basic1"
+        assert response3.json() == {"department_name": "Ormar"}
-        assert departments[0].courses[0].completed
-        assert departments[0].courses[1].course_name == "basic2"
-        assert departments[0].courses[1].completed