diff --git a/README.md b/README.md index 3aeb6bb..87f3ce5 100644 --- a/README.md +++ b/README.md @@ -163,8 +163,8 @@ assert len(tracks) == 1 * `offset(offset: int) -> QuerySet` * `count() -> int` * `exists() -> bool` -* `fields(columns: Union[List, str]) -> QuerySet` -* `exclude_fields(columns: Union[List, str]) -> QuerySet` +* `fields(columns: Union[List, str, set, dict]) -> QuerySet` +* `exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet` * `order_by(columns:Union[List, str]) -> QuerySet` #### Relation types diff --git a/docs/index.md b/docs/index.md index e4d9788..a39ad7e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -163,8 +163,8 @@ assert len(tracks) == 1 * `offset(offset: int) -> QuerySet` * `count() -> int` * `exists() -> bool` -* `fields(columns: Union[List, str]) -> QuerySet` -* `exclude_fields(columns: Union[List, str]) -> QuerySet` +* `fields(columns: Union[List, str, set, dict]) -> QuerySet` +* `exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet` * `order_by(columns:Union[List, str]) -> QuerySet` diff --git a/docs/queries.md b/docs/queries.md index 957b02b..f7f3d7b 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -348,20 +348,73 @@ has_sample = await Book.objects.filter(title='Sample').exists() ### fields -`fields(columns: Union[List, str]) -> QuerySet` +`fields(columns: Union[List, str, set, dict]) -> QuerySet` With `fields()` you can select subset of model columns to limit the data load. -```python hl_lines="47 59 60 66" +Given a sample data like following: + +```python --8<-- "../docs_src/queries/docs006.py" ``` +You can select specified fields by passing a `str, List[str], Set[str] or dict` with nested definition. + +To include related models use notation `{related_name}__{column}[__{optional_next} etc.]`. + +```python hl_lines="1" +all_cars = await Car.objects.select_related('manufacturer').fields(['id', 'name', 'manufacturer__name']).all() +for car in all_cars: + # excluded columns will yield None + assert all(getattr(car, x) is None for x in ['year', 'gearbox_type', 'gears', 'aircon_type']) + # included column on related models will be available, pk column is always included + # even if you do not include it in fields list + assert car.manufacturer.name == 'Toyota' + # also in the nested related models - you cannot exclude pk - it's always auto added + assert car.manufacturer.founded is None +``` + +`fields()` can be called several times, building up the columns to select. + +If you include related models into `select_related()` call but you won't specify columns for those models in fields +- implies a list of all fields for those nested models. + +```python hl_lines="1" +all_cars = await Car.objects.select_related('manufacturer').fields('id').fields( + ['name']).all() +# all fiels from company model are selected +assert all_cars[0].manufacturer.name == 'Toyota' +assert all_cars[0].manufacturer.founded == 1937 +``` + !!!warning Mandatory fields cannot be excluded as it will raise `ValidationError`, to exclude a field it has to be nullable. +You cannot exclude mandatory model columns - `manufacturer__name` in this example. + +```python +await Car.objects.select_related('manufacturer').fields(['id', 'name', 'manufacturer__founded']).all() +# will raise pydantic ValidationError as company.name is required +``` + !!!tip Pk column cannot be excluded - it's always auto added even if not explicitly included. +You can also pass fields to include as dictionary or set. + +To mark a field as included in a dictionary use it's name as key and ellipsis as value. + +To traverse nested models use nested dictionaries. + +To include fields at last level instead of nested dictionary a set can be used. + +To include whole nested model specify model related field name and ellipsis. + +Below you can see examples that are equivalent: + +```python +--8<-- "../docs_src/queries/docs009.py" +``` !!!note All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together @@ -372,11 +425,15 @@ With `fields()` you can select subset of model columns to limit the data load. ### exclude_fields -`fields(columns: Union[List, str]) -> QuerySet` +`exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet` With `exclude_fields()` you can select subset of model columns that will be excluded to limit the data load. -It's the oposite of `fields()` method. +It's the opposite of `fields()` method so check documentation above to see what options are available. + +Especially check above how you can pass also nested dictionaries and sets as a mask to exclude fields from whole hierarchy. + +Below you can find few simple examples: ```python hl_lines="47 48 60 61 67" --8<-- "../docs_src/queries/docs008.py" diff --git a/docs_src/queries/docs006.py b/docs_src/queries/docs006.py index 936c79f..13143d2 100644 --- a/docs_src/queries/docs006.py +++ b/docs_src/queries/docs006.py @@ -43,25 +43,3 @@ await Car.objects.create(manufacturer=toyota, name="Yaris", year=2019, gearbox_t await Car.objects.create(manufacturer=toyota, name="Supreme", year=2020, gearbox_type='Auto', gears=6, aircon_type='Auto') -# select manufacturer but only name - to include related models use notation {model_name}__{column} -all_cars = await Car.objects.select_related('manufacturer').fields(['id', 'name', 'company__name']).all() -for car in all_cars: - # excluded columns will yield None - assert all(getattr(car, x) is None for x in ['year', 'gearbox_type', 'gears', 'aircon_type']) - # included column on related models will be available, pk column is always included - # even if you do not include it in fields list - assert car.manufacturer.name == 'Toyota' - # also in the nested related models - you cannot exclude pk - it's always auto added - assert car.manufacturer.founded is None - -# fields() can be called several times, building up the columns to select -# models selected in select_related but with no columns in fields list implies all fields -all_cars = await Car.objects.select_related('manufacturer').fields('id').fields( - ['name']).all() -# all fiels from company model are selected -assert all_cars[0].manufacturer.name == 'Toyota' -assert all_cars[0].manufacturer.founded == 1937 - -# cannot exclude mandatory model columns - company__name in this example -await Car.objects.select_related('manufacturer').fields(['id', 'name', 'company__founded']).all() -# will raise pydantic ValidationError as company.name is required diff --git a/docs_src/queries/docs008.py b/docs_src/queries/docs008.py index 52bbee7..2c79b61 100644 --- a/docs_src/queries/docs008.py +++ b/docs_src/queries/docs008.py @@ -63,6 +63,6 @@ all_cars = await Car.objects.select_related('manufacturer').exclude_fields('year assert all_cars[0].manufacturer.name == 'Toyota' assert all_cars[0].manufacturer.founded == 1937 -# cannot exclude mandatory model columns - company__name in this example -await Car.objects.select_related('manufacturer').exclude_fields(['company__name']).all() +# cannot exclude mandatory model columns - company__name in this example - note usage of dict/set this time +await Car.objects.select_related('manufacturer').exclude_fields([{'company': {'name'}}]).all() # will raise pydantic ValidationError as company.name is required diff --git a/docs_src/queries/docs009.py b/docs_src/queries/docs009.py new file mode 100644 index 0000000..74ecc71 --- /dev/null +++ b/docs_src/queries/docs009.py @@ -0,0 +1,33 @@ +# 1. like in example above +await Car.objects.select_related('manufacturer').fields(['id', 'name', 'manufacturer__name']).all() + +# 2. to mark a field as required use ellipsis +await Car.objects.select_related('manufacturer').fields({'id': ..., + 'name': ..., + 'manufacturer': { + 'name': ...} + }).all() + +# 3. to include whole nested model use ellipsis +await Car.objects.select_related('manufacturer').fields({'id': ..., + 'name': ..., + 'manufacturer': ... + }).all() + +# 4. to specify fields at last nesting level you can also use set - equivalent to 2. above +await Car.objects.select_related('manufacturer').fields({'id': ..., + 'name': ..., + 'manufacturer': {'name'} + }).all() + +# 5. of course set can have multiple fields +await Car.objects.select_related('manufacturer').fields({'id': ..., + 'name': ..., + 'manufacturer': {'name', 'founded'} + }).all() + +# 6. you can include all nested fields but it will be equivalent of 3. above which is shorter +await Car.objects.select_related('manufacturer').fields({'id': ..., + 'name': ..., + 'manufacturer': {'id', 'name', 'founded'} + }).all() diff --git a/ormar/fields/base.py b/ormar/fields/base.py index 66cc5aa..e902aa0 100644 --- a/ormar/fields/base.py +++ b/ormar/fields/base.py @@ -5,7 +5,7 @@ import sqlalchemy from pydantic import Field, typing from pydantic.fields import FieldInfo -import ormar # noqa I101 +import ormar # noqa I101 from ormar import ModelDefinitionError # noqa I101 if TYPE_CHECKING: # pragma no cover diff --git a/ormar/models/excludable.py b/ormar/models/excludable.py index 003f5d4..c86e0c3 100644 --- a/ormar/models/excludable.py +++ b/ormar/models/excludable.py @@ -20,6 +20,8 @@ class Excludable: def is_excluded(exclude: Union[Set, Dict, None], key: str = None) -> bool: if exclude is None: return False + if exclude is Ellipsis: # pragma: nocover + return True to_exclude = Excludable.get_excluded(exclude=exclude, key=key) if isinstance(to_exclude, Set): return key in to_exclude @@ -31,6 +33,8 @@ class Excludable: def is_included(include: Union[Set, Dict, None], key: str = None) -> bool: if include is None: return True + if include is Ellipsis: + return True to_include = Excludable.get_included(include=include, key=key) if isinstance(to_include, Set): return key in to_include diff --git a/ormar/queryset/join.py b/ormar/queryset/join.py index 101e708..1628017 100644 --- a/ormar/queryset/join.py +++ b/ormar/queryset/join.py @@ -67,7 +67,7 @@ class SqlJoin: nested_name: str, ) -> Tuple[Optional[Union[Dict, Set]], Optional[Union[Dict, Set]]]: fields = model_cls.get_included(fields, nested_name) - exclude_fields = model_cls.get_included(exclude_fields, nested_name) + exclude_fields = model_cls.get_excluded(exclude_fields, nested_name) return fields, exclude_fields def build_join( # noqa: CCR001 diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 5260cd0..448a0a9 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -196,7 +196,7 @@ class QuerySet: if isinstance(columns, str): columns = [columns] - current_included = self._exclude_columns + current_included = self._columns if not isinstance(columns, dict): current_included = update_dict_from_list(current_included, columns) else: diff --git a/tests/test_order_by.py b/tests/test_order_by.py index a05cc0f..07d5526 100644 --- a/tests/test_order_by.py +++ b/tests/test_order_by.py @@ -176,27 +176,27 @@ async def test_sort_order_on_related_model(): owner = ( await Owner.objects.select_related("toys") - .order_by("toys__name") - .filter(name="Zeus") - .get() + .order_by("toys__name") + .filter(name="Zeus") + .get() ) assert owner.toys[0].name == "Toy 1" assert owner.toys[1].name == "Toy 4" owner = ( await Owner.objects.select_related("toys") - .order_by("-toys__name") - .filter(name="Zeus") - .get() + .order_by("-toys__name") + .filter(name="Zeus") + .get() ) assert owner.toys[0].name == "Toy 4" assert owner.toys[1].name == "Toy 1" owners = ( await Owner.objects.select_related("toys") - .order_by("-toys__name") - .filter(name__in=["Zeus", "Hermes"]) - .all() + .order_by("-toys__name") + .filter(name__in=["Zeus", "Hermes"]) + .all() ) assert owners[0].toys[0].name == "Toy 6" assert owners[0].toys[1].name == "Toy 5" @@ -210,9 +210,9 @@ async def test_sort_order_on_related_model(): owners = ( await Owner.objects.select_related("toys") - .order_by("-toys__name") - .filter(name__in=["Zeus", "Hermes"]) - .all() + .order_by("-toys__name") + .filter(name__in=["Zeus", "Hermes"]) + .all() ) assert owners[0].toys[0].name == "Toy 7" assert owners[0].toys[1].name == "Toy 4" @@ -252,9 +252,9 @@ async def test_sort_order_on_many_to_many(): user = ( await User.objects.select_related("cars") - .filter(name="Mark") - .order_by("cars__name") - .get() + .filter(name="Mark") + .order_by("cars__name") + .get() ) assert user.cars[0].name == "Buggy" assert user.cars[1].name == "Ferrari" @@ -263,9 +263,9 @@ async def test_sort_order_on_many_to_many(): user = ( await User.objects.select_related("cars") - .filter(name="Mark") - .order_by("-cars__name") - .get() + .filter(name="Mark") + .order_by("-cars__name") + .get() ) assert user.cars[3].name == "Buggy" assert user.cars[2].name == "Ferrari" @@ -281,8 +281,8 @@ async def test_sort_order_on_many_to_many(): users = ( await User.objects.select_related(["cars", "cars__factory"]) - .order_by(["-cars__factory__name", "cars__name"]) - .all() + .order_by(["-cars__factory__name", "cars__name"]) + .all() ) assert users[0].name == "Julie" @@ -328,8 +328,8 @@ async def test_sort_order_with_aliases(): aliases = ( await AliasTest.objects.select_related("nested") - .order_by("-nested__name") - .all() + .order_by("-nested__name") + .all() ) assert aliases[0].nested.name == "Try4" assert aliases[1].nested.name == "Try3" diff --git a/tests/test_selecting_subset_of_columns.py b/tests/test_selecting_subset_of_columns.py index e9133b6..e3221dd 100644 --- a/tests/test_selecting_subset_of_columns.py +++ b/tests/test_selecting_subset_of_columns.py @@ -193,7 +193,13 @@ async def test_selecting_subset(): assert car.manufacturer.hq.name is None all_cars_check = await Car.objects.select_related("manufacturer").all() - for car in all_cars_check: + all_cars_with_whole_nested = ( + await Car.objects.select_related("manufacturer") + .fields(["id", "name", "year", "gearbox_type", "gears", "aircon_type"]) + .fields({"manufacturer": ...}) + .all() + ) + for car in itertools.chain(all_cars_check, all_cars_with_whole_nested): assert all( getattr(car, x) is not None for x in ["year", "gearbox_type", "gears", "aircon_type"] @@ -201,6 +207,18 @@ async def test_selecting_subset(): assert car.manufacturer.name == "Toyota" assert car.manufacturer.founded == 1937 + all_cars_dummy = ( + await Car.objects.select_related("manufacturer") + .fields(["id", "name", "year", "gearbox_type", "gears", "aircon_type"]) + .fields({"manufacturer": ...}) + .exclude_fields({"manufacturer": ...}) + .fields({"manufacturer": {"name"}}) + .exclude_fields({"manufacturer__founded"}) + .all() + ) + + assert all_cars_dummy[0].manufacturer.founded is None + with pytest.raises(pydantic.error_wrappers.ValidationError): # cannot exclude mandatory model columns - company__name in this example await Car.objects.select_related("manufacturer").fields(