update docs, fix for ellipsis for whole model, some more tests

README.md

@@ -163,8 +163,8 @@ assert len(tracks) == 1
 *  `offset(offset: int) -> QuerySet`
 *  `count() -> int`
 *  `exists() -> bool`
-*  `fields(columns: Union[List, str]) -> QuerySet`
+*  `fields(columns: Union[List, str, set, dict]) -> QuerySet`
-*  `exclude_fields(columns: Union[List, str]) -> QuerySet`
+*  `exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet`
 *  `order_by(columns:Union[List, str]) -> QuerySet`
 
 #### Relation types

docs/index.md

@@ -163,8 +163,8 @@ assert len(tracks) == 1
 *  `offset(offset: int) -> QuerySet`
 *  `count() -> int`
 *  `exists() -> bool`
-*  `fields(columns: Union[List, str]) -> QuerySet`
+*  `fields(columns: Union[List, str, set, dict]) -> QuerySet`
-*  `exclude_fields(columns: Union[List, str]) -> QuerySet`
+*  `exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet`
 *  `order_by(columns:Union[List, str]) -> QuerySet`
 
 

docs/queries.md

@@ -348,20 +348,73 @@ has_sample = await Book.objects.filter(title='Sample').exists()
 
 ### fields
 
-`fields(columns: Union[List, str]) -> QuerySet`
+`fields(columns: Union[List, str, set, dict]) -> QuerySet`
 
 With `fields()` you can select subset of model columns to limit the data load.
 
-```python hl_lines="47 59 60 66"
+Given a sample data like following:
+
+```python
 --8<-- "../docs_src/queries/docs006.py"
 ```
 
+You can select specified fields by passing a `str, List[str], Set[str] or dict` with nested definition.
+
+To include related models use notation `{related_name}__{column}[__{optional_next} etc.]`.
+
+```python hl_lines="1"
+all_cars = await Car.objects.select_related('manufacturer').fields(['id', 'name', 'manufacturer__name']).all()
+for car in all_cars:
+    # excluded columns will yield None
+    assert all(getattr(car, x) is None for x in ['year', 'gearbox_type', 'gears', 'aircon_type'])
+    # included column on related models will be available, pk column is always included
+    # even if you do not include it in fields list
+    assert car.manufacturer.name == 'Toyota'
+    # also in the nested related models - you cannot exclude pk - it's always auto added
+    assert car.manufacturer.founded is None
+```
+
+`fields()` can be called several times, building up the columns to select.
+
+If you include related models into `select_related()` call but you won't specify columns for those models in fields 
+- implies a list of all fields for those nested models.
+
+```python hl_lines="1"
+all_cars = await Car.objects.select_related('manufacturer').fields('id').fields(
+    ['name']).all()
+# all fiels from company model are selected
+assert all_cars[0].manufacturer.name == 'Toyota'
+assert all_cars[0].manufacturer.founded == 1937
+```
+
 !!!warning
     Mandatory fields cannot be excluded as it will raise `ValidationError`, to exclude a field it has to be nullable.
 
+You cannot exclude mandatory model columns - `manufacturer__name` in this example.
+
+```python
+await Car.objects.select_related('manufacturer').fields(['id', 'name', 'manufacturer__founded']).all()
+# will raise pydantic ValidationError as company.name is required
+```
+
 !!!tip
     Pk column cannot be excluded - it's always auto added even if not explicitly included.
 
+You can also pass fields to include as dictionary or set.
+
+To mark a field as included in a dictionary use it's name as key and ellipsis as value.
+
+To traverse nested models use nested dictionaries.
+
+To include fields at last level instead of nested dictionary a set can be used.
+
+To include whole nested model specify model related field name and ellipsis.
+
+Below you can see examples that are equivalent:
+
+```python
+--8<-- "../docs_src/queries/docs009.py"
+```
 
 !!!note
     All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together
@@ -372,11 +425,15 @@ With `fields()` you can select subset of model columns to limit the data load.
 
 ### exclude_fields
 
-`fields(columns: Union[List, str]) -> QuerySet`
+`exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet`
 
 With `exclude_fields()` you can select subset of model columns that will be excluded to limit the data load.
 
-It's the oposite of `fields()` method.
+It's the opposite of `fields()` method so check documentation above to see what options are available.
+
+Especially check above how you can pass also nested dictionaries and sets as a mask to exclude fields from whole hierarchy.
+
+Below you can find few simple examples:
 
 ```python hl_lines="47 48 60 61 67"
 --8<-- "../docs_src/queries/docs008.py"

docs_src/queries/docs006.py

@@ -43,25 +43,3 @@ await Car.objects.create(manufacturer=toyota, name="Yaris", year=2019, gearbox_t
 await Car.objects.create(manufacturer=toyota, name="Supreme", year=2020, gearbox_type='Auto', gears=6,
                          aircon_type='Auto')
 
-# select manufacturer but only name - to include related models use notation {model_name}__{column}
-all_cars = await Car.objects.select_related('manufacturer').fields(['id', 'name', 'company__name']).all()
-for car in all_cars:
-    # excluded columns will yield None
-    assert all(getattr(car, x) is None for x in ['year', 'gearbox_type', 'gears', 'aircon_type'])
-    # included column on related models will be available, pk column is always included
-    # even if you do not include it in fields list
-    assert car.manufacturer.name == 'Toyota'
-    # also in the nested related models - you cannot exclude pk - it's always auto added
-    assert car.manufacturer.founded is None
-
-# fields() can be called several times, building up the columns to select
-# models selected in select_related but with no columns in fields list implies all fields
-all_cars = await Car.objects.select_related('manufacturer').fields('id').fields(
-    ['name']).all()
-# all fiels from company model are selected
-assert all_cars[0].manufacturer.name == 'Toyota'
-assert all_cars[0].manufacturer.founded == 1937
-
-# cannot exclude mandatory model columns - company__name in this example
-await Car.objects.select_related('manufacturer').fields(['id', 'name', 'company__founded']).all()
-# will raise pydantic ValidationError as company.name is required

docs_src/queries/docs008.py

@@ -63,6 +63,6 @@ all_cars = await Car.objects.select_related('manufacturer').exclude_fields('year
 assert all_cars[0].manufacturer.name == 'Toyota'
 assert all_cars[0].manufacturer.founded == 1937
 
-# cannot exclude mandatory model columns - company__name in this example
+# cannot exclude mandatory model columns - company__name in this example - note usage of dict/set this time
-await Car.objects.select_related('manufacturer').exclude_fields(['company__name']).all()
+await Car.objects.select_related('manufacturer').exclude_fields([{'company': {'name'}}]).all()
 # will raise pydantic ValidationError as company.name is required

docs_src/queries/docs009.py

@@ -0,0 +1,33 @@
+# 1. like in example above
+await Car.objects.select_related('manufacturer').fields(['id', 'name', 'manufacturer__name']).all()
+
+# 2. to mark a field as required use ellipsis
+await Car.objects.select_related('manufacturer').fields({'id': ...,
+                                                         'name': ...,
+                                                         'manufacturer': {
+                                                             'name': ...}
+                                                         }).all()
+
+# 3. to include whole nested model use ellipsis
+await Car.objects.select_related('manufacturer').fields({'id': ...,
+                                                         'name': ...,
+                                                         'manufacturer': ...
+                                                         }).all()
+
+# 4. to specify fields at last nesting level you can also use set - equivalent to 2. above
+await Car.objects.select_related('manufacturer').fields({'id': ...,
+                                                         'name': ...,
+                                                         'manufacturer': {'name'}
+                                                         }).all()
+
+# 5. of course set can have multiple fields
+await Car.objects.select_related('manufacturer').fields({'id': ...,
+                                                         'name': ...,
+                                                         'manufacturer': {'name', 'founded'}
+                                                         }).all()
+
+# 6. you can include all nested fields but it will be equivalent of 3. above which is shorter
+await Car.objects.select_related('manufacturer').fields({'id': ...,
+                                                         'name': ...,
+                                                         'manufacturer': {'id', 'name', 'founded'}
+                                                         }).all()

ormar/models/excludable.py

@@ -20,6 +20,8 @@ class Excludable:
     def is_excluded(exclude: Union[Set, Dict, None], key: str = None) -> bool:
         if exclude is None:
             return False
+        if exclude is Ellipsis:  # pragma: nocover
+            return True
         to_exclude = Excludable.get_excluded(exclude=exclude, key=key)
         if isinstance(to_exclude, Set):
             return key in to_exclude
@@ -31,6 +33,8 @@ class Excludable:
     def is_included(include: Union[Set, Dict, None], key: str = None) -> bool:
         if include is None:
             return True
+        if include is Ellipsis:
+            return True
         to_include = Excludable.get_included(include=include, key=key)
         if isinstance(to_include, Set):
             return key in to_include

ormar/queryset/join.py

@@ -67,7 +67,7 @@ class SqlJoin:
         nested_name: str,
     ) -> Tuple[Optional[Union[Dict, Set]], Optional[Union[Dict, Set]]]:
         fields = model_cls.get_included(fields, nested_name)
-        exclude_fields = model_cls.get_included(exclude_fields, nested_name)
+        exclude_fields = model_cls.get_excluded(exclude_fields, nested_name)
         return fields, exclude_fields
 
     def build_join(  # noqa:  CCR001

ormar/queryset/queryset.py

@@ -196,7 +196,7 @@ class QuerySet:
         if isinstance(columns, str):
             columns = [columns]
 
-        current_included = self._exclude_columns
+        current_included = self._columns
         if not isinstance(columns, dict):
             current_included = update_dict_from_list(current_included, columns)
         else:

tests/test_selecting_subset_of_columns.py

@@ -193,7 +193,13 @@ async def test_selecting_subset():
                 assert car.manufacturer.hq.name is None
 
             all_cars_check = await Car.objects.select_related("manufacturer").all()
-            for car in all_cars_check:
+            all_cars_with_whole_nested = (
+                await Car.objects.select_related("manufacturer")
+                .fields(["id", "name", "year", "gearbox_type", "gears", "aircon_type"])
+                .fields({"manufacturer": ...})
+                .all()
+            )
+            for car in itertools.chain(all_cars_check, all_cars_with_whole_nested):
                 assert all(
                     getattr(car, x) is not None
                     for x in ["year", "gearbox_type", "gears", "aircon_type"]
@@ -201,6 +207,18 @@ async def test_selecting_subset():
                 assert car.manufacturer.name == "Toyota"
                 assert car.manufacturer.founded == 1937
 
+            all_cars_dummy = (
+                await Car.objects.select_related("manufacturer")
+                .fields(["id", "name", "year", "gearbox_type", "gears", "aircon_type"])
+                .fields({"manufacturer": ...})
+                .exclude_fields({"manufacturer": ...})
+                .fields({"manufacturer": {"name"}})
+                .exclude_fields({"manufacturer__founded"})
+                .all()
+            )
+
+            assert all_cars_dummy[0].manufacturer.founded is None
+
             with pytest.raises(pydantic.error_wrappers.ValidationError):
                 # cannot exclude mandatory model columns - company__name in this example
                 await Car.objects.select_related("manufacturer").fields(