From c49d21f60518ebd6723e70f51274a36268afbf65 Mon Sep 17 00:00:00 2001 From: collerek Date: Sat, 17 Apr 2021 16:24:42 +0200 Subject: [PATCH 1/9] wip adding filtering and order by with field chain access instead of strings --- docs/releases.md | 6 + ormar/__init__.py | 2 +- ormar/models/metaclass.py | 13 +- ormar/queryset/__init__.py | 2 + ormar/queryset/actions/filter_action.py | 17 ++ ormar/queryset/field_accessor.py | 102 +++++++++++ .../test_fields_access.py | 165 ++++++++++++++++++ 7 files changed, 305 insertions(+), 2 deletions(-) create mode 100644 ormar/queryset/field_accessor.py create mode 100644 tests/test_model_definition/test_fields_access.py diff --git a/docs/releases.md b/docs/releases.md index f2328cc..008c138 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,3 +1,9 @@ +# 0.10.4 + +## ✨ Features + +* Add possibility to `filter` and `order_by` with field access instead of dunder separated strings. + # 0.10.3 ## ✨ Features diff --git a/ormar/__init__.py b/ormar/__init__.py index 7436730..a57b83f 100644 --- a/ormar/__init__.py +++ b/ormar/__init__.py @@ -75,7 +75,7 @@ class UndefinedType: # pragma no cover Undefined = UndefinedType() -__version__ = "0.10.3" +__version__ = "0.10.4" __all__ = [ "Integer", "BigInteger", diff --git a/ormar/models/metaclass.py b/ormar/models/metaclass.py index bc80332..1ed1cfa 100644 --- a/ormar/models/metaclass.py +++ b/ormar/models/metaclass.py @@ -39,7 +39,7 @@ from ormar.models.helpers import ( sqlalchemy_columns_from_model_fields, ) from ormar.models.quick_access_views import quick_access_set -from ormar.queryset import QuerySet +from ormar.queryset import FieldAccessor, QuerySet from ormar.relations.alias_manager import AliasManager from ormar.signals import Signal, SignalEmitter @@ -561,3 +561,14 @@ class ModelMetaclass(pydantic.main.ModelMetaclass): f"need to call update_forward_refs()." ) return QuerySet(model_cls=cls) + + def __getattr__(self, item: str) -> Any: + if item in object.__getattribute__(self, "Meta").model_fields: + field = self.Meta.model_fields.get(item) + if field.is_relation: + return FieldAccessor( + source_model=self, model=field.to, access_chain=item + ) + else: + return FieldAccessor(source_model=self, field=field, access_chain=item) + return object.__getattribute__(self, item) diff --git a/ormar/queryset/__init__.py b/ormar/queryset/__init__.py index e75febf..909cb19 100644 --- a/ormar/queryset/__init__.py +++ b/ormar/queryset/__init__.py @@ -3,6 +3,7 @@ Contains QuerySet and different Query classes to allow for constructing of sql q """ from ormar.queryset.actions import FilterAction, OrderAction, SelectAction from ormar.queryset.clause import and_, or_ +from ormar.queryset.field_accessor import FieldAccessor from ormar.queryset.filter_query import FilterQuery from ormar.queryset.limit_query import LimitQuery from ormar.queryset.offset_query import OffsetQuery @@ -20,4 +21,5 @@ __all__ = [ "SelectAction", "and_", "or_", + "FieldAccessor", ] diff --git a/ormar/queryset/actions/filter_action.py b/ormar/queryset/actions/filter_action.py index 279e0fa..2917063 100644 --- a/ormar/queryset/actions/filter_action.py +++ b/ormar/queryset/actions/filter_action.py @@ -26,6 +26,23 @@ FILTER_OPERATORS = { "lt": "__lt__", "lte": "__le__", } +METHODS_TO_OPERATORS = { + "__eq__": "exact", + "__mod__": "contains", + "__gt__": "gt", + "__ge__": "gte", + "__lt__": "lt", + "__le__": "lte", + "iexact": "iexact", + "contains": "contains", + "icontains": "icontains", + "startswith": "startswith", + "istartswith": "istartswith", + "endswith": "endswith", + "iendswith": "iendswith", + "isnull": "isnull", + "in": "in", +} ESCAPE_CHARACTERS = ["%", "_"] diff --git a/ormar/queryset/field_accessor.py b/ormar/queryset/field_accessor.py new file mode 100644 index 0000000..02f15e6 --- /dev/null +++ b/ormar/queryset/field_accessor.py @@ -0,0 +1,102 @@ +from typing import Any + +from ormar.queryset.actions import OrderAction +from ormar.queryset.actions import FilterAction +from ormar.queryset.actions.filter_action import METHODS_TO_OPERATORS + + +class FieldAccessor: + def __init__( + self, source_model=None, field=None, model=None, access_chain: str = "" + ): + self._source_model = source_model + self._field = field + self._model = model + self._access_chain = access_chain + + def __getattr__(self, item: str) -> Any: + if self._field and item == self._field.name: + return self._field + + if item in self._model.Meta.model_fields: + field = self._model.Meta.model_fields.get(item) + if field.is_relation: + return FieldAccessor( + source_model=self._source_model, + model=field.to, + access_chain=self._access_chain + f"__{item}", + ) + else: + return FieldAccessor( + source_model=self._source_model, + field=field, + access_chain=self._access_chain + f"__{item}", + ) + return object.__getattribute__(self, item) + + def _check_field(self) -> None: + if not self._field: + raise AttributeError( + "Cannot filter by Model, you need to provide model name" + ) + + def _select_operator(self, op: str, other: Any) -> FilterAction: + self._check_field() + return FilterAction( + filter_str=self._access_chain + f"__{METHODS_TO_OPERATORS[op]}", + value=other, + model_cls=self._source_model, + ) + + def __eq__(self, other: Any) -> FilterAction: # type: ignore + return self._select_operator(op="__eq__", other=other) + + def __ge__(self, other: Any) -> FilterAction: + return self._select_operator(op="__ge__", other=other) + + def __gt__(self, other: Any) -> FilterAction: + return self._select_operator(op="__gt__", other=other) + + def __le__(self, other: Any) -> FilterAction: + return self._select_operator(op="__le__", other=other) + + def __lt__(self, other) -> FilterAction: + return self._select_operator(op="__lt__", other=other) + + def __mod__(self, other) -> FilterAction: + return self._select_operator(op="__mod__", other=other) + + def __contains__(self, item) -> FilterAction: + return self._select_operator(op="in", other=item) + + def iexact(self, other) -> FilterAction: + return self._select_operator(op="iexact", other=other) + + def contains(self, other) -> FilterAction: + return self._select_operator(op="contains", other=other) + + def icontains(self, other) -> FilterAction: + return self._select_operator(op="icontains", other=other) + + def startswith(self, other) -> FilterAction: + return self._select_operator(op="startswith", other=other) + + def istartswith(self, other) -> FilterAction: + return self._select_operator(op="istartswith", other=other) + + def endswith(self, other) -> FilterAction: + return self._select_operator(op="endswith", other=other) + + def iendswith(self, other) -> FilterAction: + return self._select_operator(op="iendswith", other=other) + + def isnull(self, other) -> FilterAction: + return self._select_operator(op="isnull", other=other) + + def asc(self) -> OrderAction: + return OrderAction(order_str=self._access_chain, model_cls=self._source_model) + + def desc(self) -> OrderAction: + return OrderAction( + order_str="-" + self._access_chain, model_cls=self._source_model + ) diff --git a/tests/test_model_definition/test_fields_access.py b/tests/test_model_definition/test_fields_access.py new file mode 100644 index 0000000..22d9941 --- /dev/null +++ b/tests/test_model_definition/test_fields_access.py @@ -0,0 +1,165 @@ +import databases +import pytest +import sqlalchemy + +import ormar +from ormar import BaseField +from tests.settings import DATABASE_URL + +database = databases.Database(DATABASE_URL, force_rollback=True) +metadata = sqlalchemy.MetaData() + + +class BaseMeta(ormar.ModelMeta): + metadata = metadata + database = database + + +class PriceList(ormar.Model): + class Meta(BaseMeta): + tablename = "price_lists" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + + +class Category(ormar.Model): + class Meta(BaseMeta): + tablename = "categories" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + price_lists = ormar.ManyToMany(PriceList, related_name="categories") + + +class Product(ormar.Model): + class Meta(BaseMeta): + tablename = "product" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + rating: float = ormar.Float(minimum=1, maximum=5) + category = ormar.ForeignKey(Category) + + +@pytest.fixture(autouse=True, scope="module") +def create_test_database(): + engine = sqlalchemy.create_engine(DATABASE_URL) + metadata.drop_all(engine) + metadata.create_all(engine) + yield + metadata.drop_all(engine) + + +def test_fields_access(): + # basic access + assert Product.id._field == Product.Meta.model_fields["id"] + assert isinstance(Product.id._field, BaseField) + assert Product.id._access_chain == "id" + assert Product.id._source_model == Product + + # nested models + curr_field = Product.category.name + assert curr_field._field == Category.Meta.model_fields["name"] + assert curr_field._access_chain == "category__name" + assert curr_field._source_model == Product + + # deeper nesting + curr_field = Product.category.price_lists.name + assert curr_field._field == PriceList.Meta.model_fields["name"] + assert curr_field._access_chain == "category__price_lists__name" + assert curr_field._source_model == Product + + # reverse nesting + curr_field = PriceList.categories.products.rating + assert curr_field._field == Product.Meta.model_fields["rating"] + assert curr_field._access_chain == "categories__products__rating" + assert curr_field._source_model == PriceList + + +@pytest.mark.parametrize( + "method, expected, expected_value", + [ + ("__eq__", "exact", "Test"), + ("__lt__", "lt", "Test"), + ("__le__", "lte", "Test"), + ("__ge__", "gte", "Test"), + ("__gt__", "gt", "Test"), + ("iexact", "iexact", "Test"), + ("contains", "contains", "%Test%"), + ("icontains", "icontains", "%Test%"), + ("startswith", "startswith", "Test%"), + ("istartswith", "istartswith", "Test%"), + ("endswith", "endswith", "%Test"), + ("iendswith", "iendswith", "%Test"), + ("isnull", "isnull", "Test"), + ("__contains__", "in", "Test"), + ("__mod__", "contains", "%Test%"), + ], +) +def test_operator_return_proper_filter_action(method, expected, expected_value): + action = getattr(Product.name, method)("Test") + assert action.source_model == Product + assert action.target_model == Product + assert action.operator == expected + assert action.filter_value == expected_value + + action = getattr(Product.category.name, method)("Test") + assert action.source_model == Product + assert action.target_model == Category + assert action.operator == expected + assert action.filter_value == expected_value + + action = getattr(PriceList.categories.products.rating, method)("Test") + assert action.source_model == PriceList + assert action.target_model == Product + assert action.operator == expected + assert action.filter_value == expected_value + + +@pytest.mark.parametrize("method, expected_direction", [("asc", ""), ("desc", "desc"),]) +def test_operator_return_proper_order_action(method, expected_direction): + action = getattr(Product.name, method)() + assert action.source_model == Product + assert action.target_model == Product + assert action.direction == expected_direction + assert action.is_source_model_order + + action = getattr(Product.category.name, method)() + assert action.source_model == Product + assert action.target_model == Category + assert action.direction == expected_direction + assert not action.is_source_model_order + + action = getattr(PriceList.categories.products.rating, method)() + assert action.source_model == PriceList + assert action.target_model == Product + assert action.direction == expected_direction + assert not action.is_source_model_order + + +# @pytest.mark.asyncio +# async def test_filtering_by_field_access(): +# async with database: +# async with database.transaction(force_rollback=True): +# category = await Category(name='Toys').save() +# product1 = await Product(name="G.I Joe", +# rating=4.7, +# category=category).save() +# product2 = await Product(name="My Little Pony", +# rating=3.8, +# category=category).save() +# +# check = Product.object.get(Product.name == "My Little Pony") +# assert check == product2 + +# TODO: Finish implementation +# * overload operators and add missing functions that return FilterAction (V) +# * return OrderAction for desc() and asc() (V) + +# * accept args in all functions that accept filters? or only filter and exclude? +# all functions: delete, first, get, get_or_none, get_or_create, all, filter, exclude +# and same from queryset, should they also accept filter groups? +# * create filter groups for & and | (and ~ - NOT?) +# * accept OrderActions in order_by +# From 7a27778b4427650476370882effe5c906fe08200 Mon Sep 17 00:00:00 2001 From: collerek Date: Mon, 19 Apr 2021 19:49:42 +0200 Subject: [PATCH 2/9] add and/or/not to filtergroups, add left and right shift to operators, add some tests, add *args to other functions that read data and use filter --- docs/releases.md | 101 +++++++++++++++++- ormar/queryset/clause.py | 40 ++++--- ormar/queryset/field_accessor.py | 55 +++++----- ormar/queryset/queryset.py | 35 +++--- ormar/relations/querysetproxy.py | 24 ++--- tests/test_fastapi/test_nested_saving.py | 21 +++- .../test_inheritance_concrete.py | 55 +++++----- .../test_fields_access.py | 88 ++++++++++----- 8 files changed, 298 insertions(+), 121 deletions(-) diff --git a/docs/releases.md b/docs/releases.md index 008c138..50f2315 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -2,7 +2,106 @@ ## ✨ Features -* Add possibility to `filter` and `order_by` with field access instead of dunder separated strings. +* Add possibility to `filter` and `order_by` with field access instead of dunder separated strings. [#51](https://github.com/collerek/ormar/issues/51) + * Accessing a field with attribute access (chain of dot notation) can be used to construct `FilterGroups` (`ormar.and_` and `ormar.or_`) + * Field access overloads set of python operators and provide a set of functions to allow same functionality as with dunder separated param names in `**kwargs`, that means that querying from sample model `Track` related to model `Album` now you have more options: + * exact - exact match to value, sql `column = ` + * OLD: `album__name__exact='Malibu'` + * NEW: can be also written as `Track.album.name == 'Malibu` + * iexact - exact match sql `column = ` (case insensitive) + * OLD: `album__name__iexact='malibu'` + * NEW: can be also written as `Track.album.name.iexact('malibu')` + * contains - sql `column LIKE '%%'` + * OLD: `album__name__contains='Mal'` + * NEW: can be also written as `Track.album.name % 'Mal')` + * NEW: can be also written as `Track.album.name.contains('Mal')` + * icontains - sql `column LIKE '%%'` (case insensitive) + * OLD: `album__name__icontains='mal'` + * NEW: can be also written as `Track.album.name.icontains('mal')` + * in - sql ` column IN (, , ...)` + * OLD: `album__name__in=['Malibu', 'Barclay']` + * NEW: can be also written as `Track.album.name << ['Malibu', 'Barclay']` + * NEW: can be also written as `Track.album.name.in_(['Malibu', 'Barclay'])` + * isnull - sql `column IS NULL` (and sql `column IS NOT NULL`) + * OLD: `album__name__isnull=True` (isnotnull `album__name__isnull=False`) + * NEW: can be also written as `Track.album.name >> None` + * NEW: can be also written as `Track.album.name.is_null(True)` + * NEW: not null can be also written as `Track.album.name.is_null(False)` + * NEW: not null can be also written as `~(Track.album.name >> None)` + * NEW: not null can be also written as `~(Track.album.name.is_null(True))` + * gt - sql `column > ` (greater than) + * OLD: `position__gt=3` + * NEW: can be also written as `Track.album.name > 3` + * gte - sql `column >= ` (greater or equal than) + * OLD: `position__gte=3` + * NEW: can be also written as `Track.album.name >= 3` + * lt - sql `column < ` (lower than) + * OLD: `position__lt=3` + * NEW: can be also written as `Track.album.name < 3` + * lte - sql `column <= ` (lower equal than) + * OLD: `position__lte=3` + * NEW: can be also written as `Track.album.name <= 3` + * startswith - sql `column LIKE '%'` (exact start match) + * OLD: `album__name__startswith='Mal'` + * NEW: can be also written as `Track.album.name.startswith('Mal')` + * istartswith - sql `column LIKE '%'` (case insensitive) + * OLD: `album__name__istartswith='mal'` + * NEW: can be also written as `Track.album.name.istartswith('mal')` + * endswith - sql `column LIKE '%'` (exact end match) + * OLD: `album__name__endswith='ibu'` + * NEW: can be also written as `Track.album.name.endswith('ibu')` + * iendswith - sql `column LIKE '%'` (case insensitive) + * OLD: `album__name__iendswith='IBU'` + * NEW: can be also written as `Track.album.name.iendswith('IBU')` +* You can provide `FilterGroups` not only in `filter()` and `exclude()` but also in: + * `get()` + * `get_or_none()` + * `get_or_create()` + * `first()` + * `all()` + * `delete()` +* With `FilterGroups` (`ormar.and_` and `ormar.or_`) you can now use: + * `&` - as `and_` instead of next level of nesting + * `|` - as `or_' instead of next level of nesting + * `~` - as negation of the filter group +* To combine groups of filters into one set of conditions use `&` (sql `AND`) and `|` (sql `OR`) + ```python + # Following queries are equivalent: + # sql: ( product.name = 'Test' AND product.rating >= 3.0 ) + + # ormar OPTION 1 - OLD one + Product.objects.filter(name='Test', rating__gte=3.0).get() + + # ormar OPTION 2 - OLD one + Product.objects.filter(ormar.and_(name='Test', rating__gte=3.0)).get() + + # ormar OPTION 3 - NEW one (field access) + Product.objects.filter((Product.name == 'Test') & (Product.rating >=3.0)).get() + ``` +* Same applies to nested complicated filters + ```python + # Following queries are equivalent: + # sql: ( product.name = 'Test' AND product.rating >= 3.0 ) + # OR (categories.name IN ('Toys', 'Books')) + + # ormar OPTION 1 - OLD one + Product.objects.filter(ormar.or_( + ormar.and_(name='Test', rating__gte=3.0), + categories__name__in=['Toys', 'Books']) + ).get() + + # ormar OPTION 2 - NEW one (instead of nested or use `|`) + Product.objects.filter( + ormar.and_(name='Test', rating__gte=3.0) | + ormar.and_(categories__name__in=['Toys', 'Books']) + ).get() + + # ormar OPTION 3 - NEW one (field access) + Product.objects.filter( + ((Product.name='Test') & (Product.rating >= 3.0)) | + (Product.categories.name << ['Toys', 'Books']) + ).get() + ``` # 0.10.3 diff --git a/ormar/queryset/clause.py b/ormar/queryset/clause.py index 504a2eb..18f2d1f 100644 --- a/ormar/queryset/clause.py +++ b/ormar/queryset/clause.py @@ -25,21 +25,34 @@ class FilterGroup: """ def __init__( - self, *args: Any, _filter_type: FilterType = FilterType.AND, **kwargs: Any, + self, *args: Any, + _filter_type: FilterType = FilterType.AND, + _exclude: bool = False, + **kwargs: Any, ) -> None: self.filter_type = _filter_type - self.exclude = False + self.exclude = _exclude self._nested_groups: List["FilterGroup"] = list(args) self._resolved = False self.is_source_model_filter = False self._kwargs_dict = kwargs self.actions: List[FilterAction] = [] + def __and__(self, other: "FilterGroup") -> "FilterGroup": + return FilterGroup(self, other) + + def __or__(self, other) -> "FilterGroup": + return FilterGroup(self, other, _filter_type=FilterType.OR) + + def __invert__(self) -> "FilterGroup": + self.exclude = not self.exclude + return self + def resolve( - self, - model_cls: Type["Model"], - select_related: List = None, - filter_clauses: List = None, + self, + model_cls: Type["Model"], + select_related: List = None, + filter_clauses: List = None, ) -> Tuple[List[FilterAction], List[str]]: """ Resolves the FilterGroups actions to use proper target model, replace @@ -107,13 +120,16 @@ class FilterGroup: :return: complied and escaped clause :rtype: sqlalchemy.sql.elements.TextClause """ + prefix = " NOT " if self.exclude else "" if self.filter_type == FilterType.AND: clause = sqlalchemy.text( - "( " + str(sqlalchemy.sql.and_(*self._get_text_clauses())) + " )" + f"{prefix}( " + str( + sqlalchemy.sql.and_(*self._get_text_clauses())) + " )" ) else: clause = sqlalchemy.text( - "( " + str(sqlalchemy.sql.or_(*self._get_text_clauses())) + " )" + f"{prefix}( " + str( + sqlalchemy.sql.or_(*self._get_text_clauses())) + " )" ) return clause @@ -166,7 +182,7 @@ class QueryClause: """ def __init__( - self, model_cls: Type["Model"], filter_clauses: List, select_related: List, + self, model_cls: Type["Model"], filter_clauses: List, select_related: List, ) -> None: self._select_related = select_related[:] @@ -176,7 +192,7 @@ class QueryClause: self.table = self.model_cls.Meta.table def prepare_filter( # noqa: A003 - self, _own_only: bool = False, **kwargs: Any + self, _own_only: bool = False, **kwargs: Any ) -> Tuple[List[FilterAction], List[str]]: """ Main external access point that processes the clauses into sqlalchemy text @@ -201,7 +217,7 @@ class QueryClause: return filter_clauses, select_related def _populate_filter_clauses( - self, _own_only: bool, **kwargs: Any + self, _own_only: bool, **kwargs: Any ) -> Tuple[List[FilterAction], List[str]]: """ Iterates all clauses and extracts used operator and field from related @@ -282,7 +298,7 @@ class QueryClause: return prefixes def _switch_filter_action_prefixes( - self, filter_clauses: List[FilterAction] + self, filter_clauses: List[FilterAction] ) -> List[FilterAction]: """ Substitutes aliases for filter action if the complex key (whole relation str) is diff --git a/ormar/queryset/field_accessor.py b/ormar/queryset/field_accessor.py index 02f15e6..f595947 100644 --- a/ormar/queryset/field_accessor.py +++ b/ormar/queryset/field_accessor.py @@ -1,8 +1,8 @@ from typing import Any from ormar.queryset.actions import OrderAction -from ormar.queryset.actions import FilterAction from ormar.queryset.actions.filter_action import METHODS_TO_OPERATORS +from ormar.queryset.clause import FilterGroup class FieldAccessor: @@ -14,6 +14,10 @@ class FieldAccessor: self._model = model self._access_chain = access_chain + def __bool__(self): + # hack to avoid pydantic name check from parent model + return False + def __getattr__(self, item: str) -> Any: if self._field and item == self._field.name: return self._field @@ -32,7 +36,7 @@ class FieldAccessor: field=field, access_chain=self._access_chain + f"__{item}", ) - return object.__getattribute__(self, item) + return object.__getattribute__(self, item) # pragma: no cover def _check_field(self) -> None: if not self._field: @@ -40,57 +44,60 @@ class FieldAccessor: "Cannot filter by Model, you need to provide model name" ) - def _select_operator(self, op: str, other: Any) -> FilterAction: + def _select_operator(self, op: str, other: Any) -> FilterGroup: self._check_field() - return FilterAction( - filter_str=self._access_chain + f"__{METHODS_TO_OPERATORS[op]}", - value=other, - model_cls=self._source_model, - ) + filter_kwg = {self._access_chain + f"__{METHODS_TO_OPERATORS[op]}": other} + return FilterGroup(**filter_kwg) - def __eq__(self, other: Any) -> FilterAction: # type: ignore + def __eq__(self, other: Any) -> FilterGroup: # type: ignore return self._select_operator(op="__eq__", other=other) - def __ge__(self, other: Any) -> FilterAction: + def __ge__(self, other: Any) -> FilterGroup: return self._select_operator(op="__ge__", other=other) - def __gt__(self, other: Any) -> FilterAction: + def __gt__(self, other: Any) -> FilterGroup: return self._select_operator(op="__gt__", other=other) - def __le__(self, other: Any) -> FilterAction: + def __le__(self, other: Any) -> FilterGroup: return self._select_operator(op="__le__", other=other) - def __lt__(self, other) -> FilterAction: + def __lt__(self, other) -> FilterGroup: return self._select_operator(op="__lt__", other=other) - def __mod__(self, other) -> FilterAction: + def __mod__(self, other) -> FilterGroup: return self._select_operator(op="__mod__", other=other) - def __contains__(self, item) -> FilterAction: - return self._select_operator(op="in", other=item) + def __lshift__(self, other) -> FilterGroup: + return self._select_operator(op="in", other=other) - def iexact(self, other) -> FilterAction: + def __rshift__(self, other) -> FilterGroup: + return self._select_operator(op="isnull", other=True) + + def in_(self, other) -> FilterGroup: + return self._select_operator(op="in", other=other) + + def iexact(self, other) -> FilterGroup: return self._select_operator(op="iexact", other=other) - def contains(self, other) -> FilterAction: + def contains(self, other) -> FilterGroup: return self._select_operator(op="contains", other=other) - def icontains(self, other) -> FilterAction: + def icontains(self, other) -> FilterGroup: return self._select_operator(op="icontains", other=other) - def startswith(self, other) -> FilterAction: + def startswith(self, other) -> FilterGroup: return self._select_operator(op="startswith", other=other) - def istartswith(self, other) -> FilterAction: + def istartswith(self, other) -> FilterGroup: return self._select_operator(op="istartswith", other=other) - def endswith(self, other) -> FilterAction: + def endswith(self, other) -> FilterGroup: return self._select_operator(op="endswith", other=other) - def iendswith(self, other) -> FilterAction: + def iendswith(self, other) -> FilterGroup: return self._select_operator(op="iendswith", other=other) - def isnull(self, other) -> FilterAction: + def isnull(self, other) -> FilterGroup: return self._select_operator(op="isnull", other=other) def asc(self) -> OrderAction: diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index c325a20..1eae1f6 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -504,7 +504,7 @@ class QuerySet(Generic[T]): """ return self.fields(columns=columns, _is_exclude=True) - def order_by(self, columns: Union[List, str]) -> "QuerySet[T]": + def order_by(self, columns: Union[List, str, OrderAction]) -> "QuerySet[T]": """ With `order_by()` you can order the results from database based on your choice of fields. @@ -541,6 +541,7 @@ class QuerySet(Generic[T]): orders_by = [ OrderAction(order_str=x, model_cls=self.model_cls) # type: ignore + if not isinstance(x, OrderAction) else x for x in columns ] @@ -671,7 +672,7 @@ class QuerySet(Generic[T]): ) return await self.database.execute(expr) - async def delete(self, each: bool = False, **kwargs: Any) -> int: + async def delete(self, *args, each: bool = False, **kwargs: Any) -> int: """ Deletes from the model table after applying the filters from kwargs. @@ -685,8 +686,8 @@ class QuerySet(Generic[T]): :return: number of deleted rows :rtype:int """ - if kwargs: - return await self.filter(**kwargs).delete() + if kwargs or args: + return await self.filter(*args, **kwargs).delete() if not each and not (self.filter_clauses or self.exclude_clauses): raise QueryDefinitionError( "You cannot delete without filtering the queryset first. " @@ -753,7 +754,7 @@ class QuerySet(Generic[T]): limit_raw_sql = self.limit_sql_raw if limit_raw_sql is None else limit_raw_sql return self.rebuild_self(offset=offset, limit_raw_sql=limit_raw_sql,) - async def first(self, **kwargs: Any) -> "T": + async def first(self, *args, **kwargs: Any) -> "T": """ Gets the first row from the db ordered by primary key column ascending. @@ -764,8 +765,8 @@ class QuerySet(Generic[T]): :return: returned model :rtype: Model """ - if kwargs: - return await self.filter(**kwargs).first() + if kwargs or args: + return await self.filter(*args, **kwargs).first() expr = self.build_select_expression( limit=1, @@ -784,7 +785,7 @@ class QuerySet(Generic[T]): self.check_single_result_rows_count(processed_rows) return processed_rows[0] # type: ignore - async def get_or_none(self, **kwargs: Any) -> Optional["T"]: + async def get_or_none(self, *args, **kwargs: Any) -> Optional["T"]: """ Get's the first row from the db meeting the criteria set by kwargs. @@ -800,11 +801,11 @@ class QuerySet(Generic[T]): :rtype: Model """ try: - return await self.get(**kwargs) + return await self.get(*args, **kwargs) except ormar.NoMatch: return None - async def get(self, **kwargs: Any) -> "T": + async def get(self, *args, **kwargs: Any) -> "T": """ Get's the first row from the db meeting the criteria set by kwargs. @@ -819,8 +820,8 @@ class QuerySet(Generic[T]): :return: returned model :rtype: Model """ - if kwargs: - return await self.filter(**kwargs).get() + if kwargs or args: + return await self.filter(*args, **kwargs).get() if not self.filter_clauses: expr = self.build_select_expression( @@ -843,7 +844,7 @@ class QuerySet(Generic[T]): self.check_single_result_rows_count(processed_rows) return processed_rows[0] # type: ignore - async def get_or_create(self, **kwargs: Any) -> "T": + async def get_or_create(self, *args, **kwargs: Any) -> "T": """ Combination of create and get methods. @@ -857,7 +858,7 @@ class QuerySet(Generic[T]): :rtype: Model """ try: - return await self.get(**kwargs) + return await self.get(*args, **kwargs) except NoMatch: return await self.create(**kwargs) @@ -878,7 +879,7 @@ class QuerySet(Generic[T]): model = await self.get(pk=kwargs[pk_name]) return await model.update(**kwargs) - async def all(self, **kwargs: Any) -> List[Optional["T"]]: # noqa: A003 + async def all(self, *args, **kwargs: Any) -> List[Optional["T"]]: # noqa: A003 """ Returns all rows from a database for given model for set filter options. @@ -891,8 +892,8 @@ class QuerySet(Generic[T]): :return: list of returned models :rtype: List[Model] """ - if kwargs: - return await self.filter(**kwargs).all() + if kwargs or args: + return await self.filter(*args, **kwargs).all() expr = self.build_select_expression() rows = await self.database.fetch_all(expr) diff --git a/ormar/relations/querysetproxy.py b/ormar/relations/querysetproxy.py index f60f263..6014e29 100644 --- a/ormar/relations/querysetproxy.py +++ b/ormar/relations/querysetproxy.py @@ -22,7 +22,7 @@ if TYPE_CHECKING: # pragma no cover from ormar.relations import Relation from ormar.models import Model, T from ormar.queryset import QuerySet - from ormar import RelationType + from ormar import OrderAction, RelationType else: T = TypeVar("T", bound="Model") @@ -276,7 +276,7 @@ class QuerysetProxy(Generic[T]): ) return await queryset.delete(**kwargs) # type: ignore - async def first(self, **kwargs: Any) -> "T": + async def first(self, *args, **kwargs: Any) -> "T": """ Gets the first row from the db ordered by primary key column ascending. @@ -289,12 +289,12 @@ class QuerysetProxy(Generic[T]): :return: :rtype: _asyncio.Future """ - first = await self.queryset.first(**kwargs) + first = await self.queryset.first(*args, **kwargs) self._clean_items_on_load() self._register_related(first) return first - async def get_or_none(self, **kwargs: Any) -> Optional["T"]: + async def get_or_none(self, *args, **kwargs: Any) -> Optional["T"]: """ Get's the first row from the db meeting the criteria set by kwargs. @@ -310,7 +310,7 @@ class QuerysetProxy(Generic[T]): :rtype: Model """ try: - get = await self.queryset.get(**kwargs) + get = await self.queryset.get(*args, **kwargs) except ormar.NoMatch: return None @@ -318,7 +318,7 @@ class QuerysetProxy(Generic[T]): self._register_related(get) return get - async def get(self, **kwargs: Any) -> "T": + async def get(self, *args, **kwargs: Any) -> "T": """ Get's the first row from the db meeting the criteria set by kwargs. @@ -337,12 +337,12 @@ class QuerysetProxy(Generic[T]): :return: returned model :rtype: Model """ - get = await self.queryset.get(**kwargs) + get = await self.queryset.get(*args, **kwargs) self._clean_items_on_load() self._register_related(get) return get - async def all(self, **kwargs: Any) -> List[Optional["T"]]: # noqa: A003 + async def all(self, *args, **kwargs: Any) -> List[Optional["T"]]: # noqa: A003 """ Returns all rows from a database for given model for set filter options. @@ -359,7 +359,7 @@ class QuerysetProxy(Generic[T]): :return: list of returned models :rtype: List[Model] """ - all_items = await self.queryset.all(**kwargs) + all_items = await self.queryset.all(*args, **kwargs) self._clean_items_on_load() self._register_related(all_items) return all_items @@ -425,7 +425,7 @@ class QuerysetProxy(Generic[T]): ) return len(children) - async def get_or_create(self, **kwargs: Any) -> "T": + async def get_or_create(self, *args, **kwargs: Any) -> "T": """ Combination of create and get methods. @@ -439,7 +439,7 @@ class QuerysetProxy(Generic[T]): :rtype: Model """ try: - return await self.get(**kwargs) + return await self.get(*args, **kwargs) except ormar.NoMatch: return await self.create(**kwargs) @@ -739,7 +739,7 @@ class QuerysetProxy(Generic[T]): relation=self.relation, type_=self.type_, to=self.to, qryset=queryset ) - def order_by(self, columns: Union[List, str]) -> "QuerysetProxy[T]": + def order_by(self, columns: Union[List, str, "OrderAction"]) -> "QuerysetProxy[T]": """ With `order_by()` you can order the results from database based on your choice of fields. diff --git a/tests/test_fastapi/test_nested_saving.py b/tests/test_fastapi/test_nested_saving.py index c4d817b..9afcc33 100644 --- a/tests/test_fastapi/test_nested_saving.py +++ b/tests/test_fastapi/test_nested_saving.py @@ -1,5 +1,5 @@ import json -from typing import Optional +from typing import Any, Dict, Optional, Set, Type, Union, cast import databases import pytest @@ -8,6 +8,7 @@ from fastapi import FastAPI from starlette.testclient import TestClient import ormar +from ormar.queryset.utils import translate_list_to_dict from tests.settings import DATABASE_URL app = FastAPI() @@ -84,6 +85,24 @@ to_exclude_ormar = { } +def auto_exclude_id_field(to_exclude: Any) -> Union[Dict, Set]: + if isinstance(to_exclude, dict): + for key in to_exclude.keys(): + to_exclude[key] = auto_exclude_id_field(to_exclude[key]) + to_exclude["id"] = Ellipsis + return to_exclude + else: + return {"id"} + + +def generate_exclude_for_ids(model: Type[ormar.Model]) -> Dict: + to_exclude_base = translate_list_to_dict(model._iterate_related_models()) + return cast(Dict, auto_exclude_id_field(to_exclude=to_exclude_base)) + + +to_exclude_auto = generate_exclude_for_ids(model=Department) + + @app.post("/departments/", response_model=Department) async def create_department(department: Department): await department.save_related(follow=True, save_all=True) diff --git a/tests/test_inheritance/test_inheritance_concrete.py b/tests/test_inheritance/test_inheritance_concrete.py index ac059e4..4bbdf5b 100644 --- a/tests/test_inheritance/test_inheritance_concrete.py +++ b/tests/test_inheritance/test_inheritance_concrete.py @@ -173,7 +173,6 @@ def test_init_of_abstract_model(): def test_duplicated_related_name_on_different_model(): with pytest.raises(ModelDefinitionError): - class Bus3(Car2): # pragma: no cover class Meta: tablename = "buses3" @@ -203,7 +202,6 @@ def test_field_redefining_in_concrete_models(): def test_model_subclassing_that_redefines_constraints_column_names(): with pytest.raises(ModelDefinitionError): - class WrongField2(DateFieldsModel): # pragma: no cover class Meta(ormar.ModelMeta): tablename = "wrongs" @@ -216,7 +214,6 @@ def test_model_subclassing_that_redefines_constraints_column_names(): def test_model_subclassing_non_abstract_raises_error(): with pytest.raises(ModelDefinitionError): - class WrongField2(DateFieldsModelNoSubclass): # pragma: no cover class Meta(ormar.ModelMeta): tablename = "wrongs" @@ -234,7 +231,7 @@ def test_params_are_inherited(): def round_date_to_seconds( - date: datetime.datetime, + date: datetime.datetime, ) -> datetime.datetime: # pragma: no cover if date.microsecond >= 500000: date = date + datetime.timedelta(seconds=1) @@ -277,9 +274,9 @@ async def test_fields_inherited_from_mixin(): sub2 = ( await Subject.objects.select_related("category") - .order_by("-created_date") - .exclude_fields("updated_date") - .get() + .order_by("-created_date") + .exclude_fields("updated_date") + .get() ) assert round_date_to_seconds(sub2.created_date) == round_date_to_seconds( sub.created_date @@ -294,9 +291,9 @@ async def test_fields_inherited_from_mixin(): sub3 = ( await Subject.objects.prefetch_related("category") - .order_by("-created_date") - .exclude_fields({"updated_date": ..., "category": {"updated_date"}}) - .get() + .order_by("-created_date") + .exclude_fields({"updated_date": ..., "category": {"updated_date"}}) + .get() ) assert round_date_to_seconds(sub3.created_date) == round_date_to_seconds( sub.created_date @@ -349,8 +346,8 @@ async def test_inheritance_with_relation(): "coowned_buses": {"created_date"}, } ) - .prefetch_related(["coowned_trucks", "coowned_buses"]) - .get(name="Joe") + .prefetch_related(["coowned_trucks", "coowned_buses"]) + .get(name="Joe") ) assert joe_check.pk == joe.pk assert joe_check.coowned_trucks[0] == shelby @@ -397,8 +394,8 @@ async def test_inheritance_with_multi_relation(): unicorn = ( await Bus2.objects.select_related(["owner", "co_owners"]) - .order_by("-co_owners__name") - .get() + .order_by("-co_owners__name") + .get() ) assert unicorn.name == "Unicorn 2" assert unicorn.owner.name == "Sam" @@ -407,8 +404,8 @@ async def test_inheritance_with_multi_relation(): unicorn = ( await Bus2.objects.select_related(["owner", "co_owners"]) - .order_by("co_owners__name") - .get() + .order_by("co_owners__name") + .get() ) assert unicorn.name == "Unicorn 2" assert unicorn.owner.name == "Sam" @@ -431,8 +428,8 @@ async def test_inheritance_with_multi_relation(): "coowned_buses2": {"created_date"}, } ) - .prefetch_related(["coowned_trucks2", "coowned_buses2"]) - .get(name="Joe") + .prefetch_related(["coowned_trucks2", "coowned_buses2"]) + .get(name="Joe") ) assert joe_check.pk == joe.pk assert joe_check.coowned_trucks2[0] == shelby @@ -446,8 +443,8 @@ async def test_inheritance_with_multi_relation(): unicorn = ( await Bus2.objects.select_related(["owner", "co_owners"]) - .filter(co_owners__name="Joe") - .get() + .filter(co_owners__name="Joe") + .get() ) assert unicorn.name == "Unicorn 2" assert unicorn.owner.name == "Sam" @@ -457,8 +454,8 @@ async def test_inheritance_with_multi_relation(): unicorn = ( await Bus2.objects.select_related(["owner", "co_owners"]) - .exclude(co_owners__name="Joe") - .get() + .exclude(co_owners__name="Joe") + .get() ) assert unicorn.name == "Unicorn 2" assert unicorn.owner.name == "Sam" @@ -480,9 +477,9 @@ async def test_inheritance_with_multi_relation(): unicorns = ( await Bus2.objects.select_related(["owner", "co_owners"]) - .filter(name__contains="Unicorn") - .order_by("-name") - .all() + .filter(name__contains="Unicorn") + .order_by("-name") + .all() ) assert unicorns[0].name == "Unicorn 3" assert unicorns[0].owner.name == "Joe" @@ -496,10 +493,10 @@ async def test_inheritance_with_multi_relation(): unicorns = ( await Bus2.objects.select_related(["owner", "co_owners"]) - .filter(name__contains="Unicorn") - .order_by("-name") - .limit(2, limit_raw_sql=True) - .all() + .filter(name__contains="Unicorn") + .order_by("-name") + .limit(2, limit_raw_sql=True) + .all() ) assert len(unicorns) == 2 assert unicorns[1].name == "Unicorn 2" diff --git a/tests/test_model_definition/test_fields_access.py b/tests/test_model_definition/test_fields_access.py index 22d9941..8298242 100644 --- a/tests/test_model_definition/test_fields_access.py +++ b/tests/test_model_definition/test_fields_access.py @@ -54,6 +54,7 @@ def create_test_database(): def test_fields_access(): # basic access assert Product.id._field == Product.Meta.model_fields["id"] + assert Product.id.id == Product.Meta.model_fields["id"] assert isinstance(Product.id._field, BaseField) assert Product.id._access_chain == "id" assert Product.id._source_model == Product @@ -76,6 +77,9 @@ def test_fields_access(): assert curr_field._access_chain == "categories__products__rating" assert curr_field._source_model == PriceList + with pytest.raises(AttributeError): + assert Product.category >= 3 + @pytest.mark.parametrize( "method, expected, expected_value", @@ -86,38 +90,33 @@ def test_fields_access(): ("__ge__", "gte", "Test"), ("__gt__", "gt", "Test"), ("iexact", "iexact", "Test"), - ("contains", "contains", "%Test%"), - ("icontains", "icontains", "%Test%"), - ("startswith", "startswith", "Test%"), - ("istartswith", "istartswith", "Test%"), - ("endswith", "endswith", "%Test"), - ("iendswith", "iendswith", "%Test"), + ("contains", "contains", "Test"), + ("icontains", "icontains", "Test"), + ("startswith", "startswith", "Test"), + ("istartswith", "istartswith", "Test"), + ("endswith", "endswith", "Test"), + ("iendswith", "iendswith", "Test"), ("isnull", "isnull", "Test"), - ("__contains__", "in", "Test"), - ("__mod__", "contains", "%Test%"), + ("in_", "in", "Test"), + ("__lshift__", "in", "Test"), + ("__rshift__", "isnull", True), + ("__mod__", "contains", "Test"), ], ) def test_operator_return_proper_filter_action(method, expected, expected_value): - action = getattr(Product.name, method)("Test") - assert action.source_model == Product - assert action.target_model == Product - assert action.operator == expected - assert action.filter_value == expected_value + group_ = getattr(Product.name, method)("Test") + assert group_._kwargs_dict == {f"name__{expected}": expected_value} - action = getattr(Product.category.name, method)("Test") - assert action.source_model == Product - assert action.target_model == Category - assert action.operator == expected - assert action.filter_value == expected_value + group_ = getattr(Product.category.name, method)("Test") + assert group_._kwargs_dict == {f"category__name__{expected}": expected_value} - action = getattr(PriceList.categories.products.rating, method)("Test") - assert action.source_model == PriceList - assert action.target_model == Product - assert action.operator == expected - assert action.filter_value == expected_value + group_ = getattr(PriceList.categories.products.rating, method)("Test") + assert group_._kwargs_dict == { + f"categories__products__rating__{expected}": expected_value} -@pytest.mark.parametrize("method, expected_direction", [("asc", ""), ("desc", "desc"),]) +@pytest.mark.parametrize("method, expected_direction", + [("asc", ""), ("desc", "desc"), ]) def test_operator_return_proper_order_action(method, expected_direction): action = getattr(Product.name, method)() assert action.source_model == Product @@ -138,6 +137,45 @@ def test_operator_return_proper_order_action(method, expected_direction): assert not action.is_source_model_order +def test_combining_groups_together(): + group = (Product.name == "Test") & (Product.rating >= 3.0) + group.resolve(model_cls=Product) + assert len(group._nested_groups) == 2 + assert str(group.get_text_clause()) == ("( ( product.name = 'Test' ) AND" + " ( product.rating >= 3.0 ) )") + + group = ~((Product.name == "Test") & (Product.rating >= 3.0)) + group.resolve(model_cls=Product) + assert len(group._nested_groups) == 2 + assert str(group.get_text_clause()) == (" NOT ( ( product.name = 'Test' ) AND" + " ( product.rating >= 3.0 ) )") + + group = ((Product.name == "Test") & (Product.rating >= 3.0)) | ( + Product.category.name << (["Toys", "Books"])) + group.resolve(model_cls=Product) + assert len(group._nested_groups) == 2 + assert len(group._nested_groups[0]._nested_groups) == 2 + group_str = str(group.get_text_clause()) + category_prefix = group._nested_groups[1].actions[0].table_prefix + assert group_str == ( + "( ( ( product.name = 'Test' ) AND ( product.rating >= 3.0 ) ) " + f"OR ( {category_prefix}_categories.name IN ('Toys', 'Books') ) )") + + group = (Product.name % "Test") | ( + (Product.category.price_lists.name.startswith("Aa")) | ( + Product.category.name << (["Toys", "Books"]))) + group.resolve(model_cls=Product) + assert len(group._nested_groups) == 2 + assert len(group._nested_groups[1]._nested_groups) == 2 + group_str = str(group.get_text_clause()) + price_list_prefix = group._nested_groups[1]._nested_groups[0].actions[ + 0].table_prefix + category_prefix = group._nested_groups[1]._nested_groups[1].actions[0].table_prefix + assert group_str == ( + f"( ( product.name LIKE '%Test%' ) " + f"OR ( ( {price_list_prefix}_price_lists.name LIKE 'Aa%' ) " + f"OR ( {category_prefix}_categories.name IN ('Toys', 'Books') ) ) )") + # @pytest.mark.asyncio # async def test_filtering_by_field_access(): # async with database: @@ -156,10 +194,10 @@ def test_operator_return_proper_order_action(method, expected_direction): # TODO: Finish implementation # * overload operators and add missing functions that return FilterAction (V) # * return OrderAction for desc() and asc() (V) +# * create filter groups for & and | (and ~ - NOT?) (V) # * accept args in all functions that accept filters? or only filter and exclude? # all functions: delete, first, get, get_or_none, get_or_create, all, filter, exclude # and same from queryset, should they also accept filter groups? -# * create filter groups for & and | (and ~ - NOT?) # * accept OrderActions in order_by # From 5e38426694faf17ec3065f6292c2b7fde99a771c Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 20 Apr 2021 11:52:41 +0200 Subject: [PATCH 3/9] finish release notes, add more test --- docs/releases.md | 9 +++ ormar/models/metaclass.py | 10 ++- ormar/queryset/actions/filter_action.py | 1 + ormar/queryset/clause.py | 37 ++++++----- ormar/queryset/field_accessor.py | 45 +++++++------ ormar/queryset/queryset.py | 15 +++-- ormar/relations/querysetproxy.py | 10 +-- .../test_inheritance_concrete.py | 55 ++++++++-------- .../test_fields_access.py | 43 ++++++------ .../test_pydantic_fields.py | 66 +++++++++++++++++++ tests/test_queries/test_or_filters.py | 28 ++++++++ tests/test_queries/test_order_by.py | 18 +++++ 12 files changed, 242 insertions(+), 95 deletions(-) create mode 100644 tests/test_model_definition/test_pydantic_fields.py diff --git a/docs/releases.md b/docs/releases.md index 50f2315..2fc282d 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -102,6 +102,15 @@ (Product.categories.name << ['Toys', 'Books']) ).get() ``` +* Now you can alos use field access to provide OrderActions to `order_by()` + * Order ascending: + * OLD: `Product.objects.order_by("name").all()` + * NEW: `Product.objects.order_by(Product.name.asc()).all()` + * Order descending: + * OLD: `Product.objects.order_by("-name").all()` + * NEW: `Product.objects.order_by(Product.name.desc()).all()` + * You can of course also combine different models and many order_bys: + `Product.objects.order_by([Product.category.name.asc(), Product.name.desc()]).all()` # 0.10.3 diff --git a/ormar/models/metaclass.py b/ormar/models/metaclass.py index 1ed1cfa..43c8822 100644 --- a/ormar/models/metaclass.py +++ b/ormar/models/metaclass.py @@ -567,8 +567,14 @@ class ModelMetaclass(pydantic.main.ModelMetaclass): field = self.Meta.model_fields.get(item) if field.is_relation: return FieldAccessor( - source_model=self, model=field.to, access_chain=item + source_model=cast(Type["Model"], self), + model=field.to, + access_chain=item, ) else: - return FieldAccessor(source_model=self, field=field, access_chain=item) + return FieldAccessor( + source_model=cast(Type["Model"], self), + field=field, + access_chain=item, + ) return object.__getattribute__(self, item) diff --git a/ormar/queryset/actions/filter_action.py b/ormar/queryset/actions/filter_action.py index 2917063..8b2c7b1 100644 --- a/ormar/queryset/actions/filter_action.py +++ b/ormar/queryset/actions/filter_action.py @@ -176,5 +176,6 @@ class FilterAction(QueryAction): clause_text = clause_text.replace( f"{self.table.name}.{self.column.name}", aliased_name ) + clause_text.replace("%%", "%") # remove doubles in some dialects clause = text(clause_text) return clause diff --git a/ormar/queryset/clause.py b/ormar/queryset/clause.py index 18f2d1f..9f5b35b 100644 --- a/ormar/queryset/clause.py +++ b/ormar/queryset/clause.py @@ -25,10 +25,11 @@ class FilterGroup: """ def __init__( - self, *args: Any, - _filter_type: FilterType = FilterType.AND, - _exclude: bool = False, - **kwargs: Any, + self, + *args: Any, + _filter_type: FilterType = FilterType.AND, + _exclude: bool = False, + **kwargs: Any, ) -> None: self.filter_type = _filter_type self.exclude = _exclude @@ -41,7 +42,7 @@ class FilterGroup: def __and__(self, other: "FilterGroup") -> "FilterGroup": return FilterGroup(self, other) - def __or__(self, other) -> "FilterGroup": + def __or__(self, other: "FilterGroup") -> "FilterGroup": return FilterGroup(self, other, _filter_type=FilterType.OR) def __invert__(self) -> "FilterGroup": @@ -49,10 +50,10 @@ class FilterGroup: return self def resolve( - self, - model_cls: Type["Model"], - select_related: List = None, - filter_clauses: List = None, + self, + model_cls: Type["Model"], + select_related: List = None, + filter_clauses: List = None, ) -> Tuple[List[FilterAction], List[str]]: """ Resolves the FilterGroups actions to use proper target model, replace @@ -123,13 +124,15 @@ class FilterGroup: prefix = " NOT " if self.exclude else "" if self.filter_type == FilterType.AND: clause = sqlalchemy.text( - f"{prefix}( " + str( - sqlalchemy.sql.and_(*self._get_text_clauses())) + " )" + f"{prefix}( " + + str(sqlalchemy.sql.and_(*self._get_text_clauses())) + + " )" ) else: clause = sqlalchemy.text( - f"{prefix}( " + str( - sqlalchemy.sql.or_(*self._get_text_clauses())) + " )" + f"{prefix}( " + + str(sqlalchemy.sql.or_(*self._get_text_clauses())) + + " )" ) return clause @@ -182,7 +185,7 @@ class QueryClause: """ def __init__( - self, model_cls: Type["Model"], filter_clauses: List, select_related: List, + self, model_cls: Type["Model"], filter_clauses: List, select_related: List, ) -> None: self._select_related = select_related[:] @@ -192,7 +195,7 @@ class QueryClause: self.table = self.model_cls.Meta.table def prepare_filter( # noqa: A003 - self, _own_only: bool = False, **kwargs: Any + self, _own_only: bool = False, **kwargs: Any ) -> Tuple[List[FilterAction], List[str]]: """ Main external access point that processes the clauses into sqlalchemy text @@ -217,7 +220,7 @@ class QueryClause: return filter_clauses, select_related def _populate_filter_clauses( - self, _own_only: bool, **kwargs: Any + self, _own_only: bool, **kwargs: Any ) -> Tuple[List[FilterAction], List[str]]: """ Iterates all clauses and extracts used operator and field from related @@ -298,7 +301,7 @@ class QueryClause: return prefixes def _switch_filter_action_prefixes( - self, filter_clauses: List[FilterAction] + self, filter_clauses: List[FilterAction] ) -> List[FilterAction]: """ Substitutes aliases for filter action if the complex key (whole relation str) is diff --git a/ormar/queryset/field_accessor.py b/ormar/queryset/field_accessor.py index f595947..52cc755 100644 --- a/ormar/queryset/field_accessor.py +++ b/ormar/queryset/field_accessor.py @@ -1,20 +1,27 @@ -from typing import Any +from typing import Any, TYPE_CHECKING, Type from ormar.queryset.actions import OrderAction from ormar.queryset.actions.filter_action import METHODS_TO_OPERATORS from ormar.queryset.clause import FilterGroup +if TYPE_CHECKING: # pragma: no cover + from ormar import BaseField, Model + class FieldAccessor: def __init__( - self, source_model=None, field=None, model=None, access_chain: str = "" - ): + self, + source_model: Type["Model"], + field: "BaseField" = None, + model: Type["Model"] = None, + access_chain: str = "", + ) -> None: self._source_model = source_model self._field = field self._model = model self._access_chain = access_chain - def __bool__(self): + def __bool__(self) -> bool: # hack to avoid pydantic name check from parent model return False @@ -22,8 +29,8 @@ class FieldAccessor: if self._field and item == self._field.name: return self._field - if item in self._model.Meta.model_fields: - field = self._model.Meta.model_fields.get(item) + if self._model and item in self._model.Meta.model_fields: + field = self._model.Meta.model_fields[item] if field.is_relation: return FieldAccessor( source_model=self._source_model, @@ -61,43 +68,43 @@ class FieldAccessor: def __le__(self, other: Any) -> FilterGroup: return self._select_operator(op="__le__", other=other) - def __lt__(self, other) -> FilterGroup: + def __lt__(self, other: Any) -> FilterGroup: return self._select_operator(op="__lt__", other=other) - def __mod__(self, other) -> FilterGroup: + def __mod__(self, other: Any) -> FilterGroup: return self._select_operator(op="__mod__", other=other) - def __lshift__(self, other) -> FilterGroup: + def __lshift__(self, other: Any) -> FilterGroup: return self._select_operator(op="in", other=other) - def __rshift__(self, other) -> FilterGroup: + def __rshift__(self, other: Any) -> FilterGroup: return self._select_operator(op="isnull", other=True) - def in_(self, other) -> FilterGroup: + def in_(self, other: Any) -> FilterGroup: return self._select_operator(op="in", other=other) - def iexact(self, other) -> FilterGroup: + def iexact(self, other: Any) -> FilterGroup: return self._select_operator(op="iexact", other=other) - def contains(self, other) -> FilterGroup: + def contains(self, other: Any) -> FilterGroup: return self._select_operator(op="contains", other=other) - def icontains(self, other) -> FilterGroup: + def icontains(self, other: Any) -> FilterGroup: return self._select_operator(op="icontains", other=other) - def startswith(self, other) -> FilterGroup: + def startswith(self, other: Any) -> FilterGroup: return self._select_operator(op="startswith", other=other) - def istartswith(self, other) -> FilterGroup: + def istartswith(self, other: Any) -> FilterGroup: return self._select_operator(op="istartswith", other=other) - def endswith(self, other) -> FilterGroup: + def endswith(self, other: Any) -> FilterGroup: return self._select_operator(op="endswith", other=other) - def iendswith(self, other) -> FilterGroup: + def iendswith(self, other: Any) -> FilterGroup: return self._select_operator(op="iendswith", other=other) - def isnull(self, other) -> FilterGroup: + def isnull(self, other: Any) -> FilterGroup: return self._select_operator(op="isnull", other=other) def asc(self) -> OrderAction: diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 1eae1f6..0024a7e 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -541,7 +541,8 @@ class QuerySet(Generic[T]): orders_by = [ OrderAction(order_str=x, model_cls=self.model_cls) # type: ignore - if not isinstance(x, OrderAction) else x + if not isinstance(x, OrderAction) + else x for x in columns ] @@ -672,7 +673,7 @@ class QuerySet(Generic[T]): ) return await self.database.execute(expr) - async def delete(self, *args, each: bool = False, **kwargs: Any) -> int: + async def delete(self, *args: Any, each: bool = False, **kwargs: Any) -> int: """ Deletes from the model table after applying the filters from kwargs. @@ -754,7 +755,7 @@ class QuerySet(Generic[T]): limit_raw_sql = self.limit_sql_raw if limit_raw_sql is None else limit_raw_sql return self.rebuild_self(offset=offset, limit_raw_sql=limit_raw_sql,) - async def first(self, *args, **kwargs: Any) -> "T": + async def first(self, *args: Any, **kwargs: Any) -> "T": """ Gets the first row from the db ordered by primary key column ascending. @@ -785,7 +786,7 @@ class QuerySet(Generic[T]): self.check_single_result_rows_count(processed_rows) return processed_rows[0] # type: ignore - async def get_or_none(self, *args, **kwargs: Any) -> Optional["T"]: + async def get_or_none(self, *args: Any, **kwargs: Any) -> Optional["T"]: """ Get's the first row from the db meeting the criteria set by kwargs. @@ -805,7 +806,7 @@ class QuerySet(Generic[T]): except ormar.NoMatch: return None - async def get(self, *args, **kwargs: Any) -> "T": + async def get(self, *args: Any, **kwargs: Any) -> "T": """ Get's the first row from the db meeting the criteria set by kwargs. @@ -844,7 +845,7 @@ class QuerySet(Generic[T]): self.check_single_result_rows_count(processed_rows) return processed_rows[0] # type: ignore - async def get_or_create(self, *args, **kwargs: Any) -> "T": + async def get_or_create(self, *args: Any, **kwargs: Any) -> "T": """ Combination of create and get methods. @@ -879,7 +880,7 @@ class QuerySet(Generic[T]): model = await self.get(pk=kwargs[pk_name]) return await model.update(**kwargs) - async def all(self, *args, **kwargs: Any) -> List[Optional["T"]]: # noqa: A003 + async def all(self, *args: Any, **kwargs: Any) -> List[Optional["T"]]: # noqa: A003 """ Returns all rows from a database for given model for set filter options. diff --git a/ormar/relations/querysetproxy.py b/ormar/relations/querysetproxy.py index 6014e29..c6ae52d 100644 --- a/ormar/relations/querysetproxy.py +++ b/ormar/relations/querysetproxy.py @@ -276,7 +276,7 @@ class QuerysetProxy(Generic[T]): ) return await queryset.delete(**kwargs) # type: ignore - async def first(self, *args, **kwargs: Any) -> "T": + async def first(self, *args: Any, **kwargs: Any) -> "T": """ Gets the first row from the db ordered by primary key column ascending. @@ -294,7 +294,7 @@ class QuerysetProxy(Generic[T]): self._register_related(first) return first - async def get_or_none(self, *args, **kwargs: Any) -> Optional["T"]: + async def get_or_none(self, *args: Any, **kwargs: Any) -> Optional["T"]: """ Get's the first row from the db meeting the criteria set by kwargs. @@ -318,7 +318,7 @@ class QuerysetProxy(Generic[T]): self._register_related(get) return get - async def get(self, *args, **kwargs: Any) -> "T": + async def get(self, *args: Any, **kwargs: Any) -> "T": """ Get's the first row from the db meeting the criteria set by kwargs. @@ -342,7 +342,7 @@ class QuerysetProxy(Generic[T]): self._register_related(get) return get - async def all(self, *args, **kwargs: Any) -> List[Optional["T"]]: # noqa: A003 + async def all(self, *args: Any, **kwargs: Any) -> List[Optional["T"]]: # noqa: A003 """ Returns all rows from a database for given model for set filter options. @@ -425,7 +425,7 @@ class QuerysetProxy(Generic[T]): ) return len(children) - async def get_or_create(self, *args, **kwargs: Any) -> "T": + async def get_or_create(self, *args: Any, **kwargs: Any) -> "T": """ Combination of create and get methods. diff --git a/tests/test_inheritance/test_inheritance_concrete.py b/tests/test_inheritance/test_inheritance_concrete.py index 4bbdf5b..ac059e4 100644 --- a/tests/test_inheritance/test_inheritance_concrete.py +++ b/tests/test_inheritance/test_inheritance_concrete.py @@ -173,6 +173,7 @@ def test_init_of_abstract_model(): def test_duplicated_related_name_on_different_model(): with pytest.raises(ModelDefinitionError): + class Bus3(Car2): # pragma: no cover class Meta: tablename = "buses3" @@ -202,6 +203,7 @@ def test_field_redefining_in_concrete_models(): def test_model_subclassing_that_redefines_constraints_column_names(): with pytest.raises(ModelDefinitionError): + class WrongField2(DateFieldsModel): # pragma: no cover class Meta(ormar.ModelMeta): tablename = "wrongs" @@ -214,6 +216,7 @@ def test_model_subclassing_that_redefines_constraints_column_names(): def test_model_subclassing_non_abstract_raises_error(): with pytest.raises(ModelDefinitionError): + class WrongField2(DateFieldsModelNoSubclass): # pragma: no cover class Meta(ormar.ModelMeta): tablename = "wrongs" @@ -231,7 +234,7 @@ def test_params_are_inherited(): def round_date_to_seconds( - date: datetime.datetime, + date: datetime.datetime, ) -> datetime.datetime: # pragma: no cover if date.microsecond >= 500000: date = date + datetime.timedelta(seconds=1) @@ -274,9 +277,9 @@ async def test_fields_inherited_from_mixin(): sub2 = ( await Subject.objects.select_related("category") - .order_by("-created_date") - .exclude_fields("updated_date") - .get() + .order_by("-created_date") + .exclude_fields("updated_date") + .get() ) assert round_date_to_seconds(sub2.created_date) == round_date_to_seconds( sub.created_date @@ -291,9 +294,9 @@ async def test_fields_inherited_from_mixin(): sub3 = ( await Subject.objects.prefetch_related("category") - .order_by("-created_date") - .exclude_fields({"updated_date": ..., "category": {"updated_date"}}) - .get() + .order_by("-created_date") + .exclude_fields({"updated_date": ..., "category": {"updated_date"}}) + .get() ) assert round_date_to_seconds(sub3.created_date) == round_date_to_seconds( sub.created_date @@ -346,8 +349,8 @@ async def test_inheritance_with_relation(): "coowned_buses": {"created_date"}, } ) - .prefetch_related(["coowned_trucks", "coowned_buses"]) - .get(name="Joe") + .prefetch_related(["coowned_trucks", "coowned_buses"]) + .get(name="Joe") ) assert joe_check.pk == joe.pk assert joe_check.coowned_trucks[0] == shelby @@ -394,8 +397,8 @@ async def test_inheritance_with_multi_relation(): unicorn = ( await Bus2.objects.select_related(["owner", "co_owners"]) - .order_by("-co_owners__name") - .get() + .order_by("-co_owners__name") + .get() ) assert unicorn.name == "Unicorn 2" assert unicorn.owner.name == "Sam" @@ -404,8 +407,8 @@ async def test_inheritance_with_multi_relation(): unicorn = ( await Bus2.objects.select_related(["owner", "co_owners"]) - .order_by("co_owners__name") - .get() + .order_by("co_owners__name") + .get() ) assert unicorn.name == "Unicorn 2" assert unicorn.owner.name == "Sam" @@ -428,8 +431,8 @@ async def test_inheritance_with_multi_relation(): "coowned_buses2": {"created_date"}, } ) - .prefetch_related(["coowned_trucks2", "coowned_buses2"]) - .get(name="Joe") + .prefetch_related(["coowned_trucks2", "coowned_buses2"]) + .get(name="Joe") ) assert joe_check.pk == joe.pk assert joe_check.coowned_trucks2[0] == shelby @@ -443,8 +446,8 @@ async def test_inheritance_with_multi_relation(): unicorn = ( await Bus2.objects.select_related(["owner", "co_owners"]) - .filter(co_owners__name="Joe") - .get() + .filter(co_owners__name="Joe") + .get() ) assert unicorn.name == "Unicorn 2" assert unicorn.owner.name == "Sam" @@ -454,8 +457,8 @@ async def test_inheritance_with_multi_relation(): unicorn = ( await Bus2.objects.select_related(["owner", "co_owners"]) - .exclude(co_owners__name="Joe") - .get() + .exclude(co_owners__name="Joe") + .get() ) assert unicorn.name == "Unicorn 2" assert unicorn.owner.name == "Sam" @@ -477,9 +480,9 @@ async def test_inheritance_with_multi_relation(): unicorns = ( await Bus2.objects.select_related(["owner", "co_owners"]) - .filter(name__contains="Unicorn") - .order_by("-name") - .all() + .filter(name__contains="Unicorn") + .order_by("-name") + .all() ) assert unicorns[0].name == "Unicorn 3" assert unicorns[0].owner.name == "Joe" @@ -493,10 +496,10 @@ async def test_inheritance_with_multi_relation(): unicorns = ( await Bus2.objects.select_related(["owner", "co_owners"]) - .filter(name__contains="Unicorn") - .order_by("-name") - .limit(2, limit_raw_sql=True) - .all() + .filter(name__contains="Unicorn") + .order_by("-name") + .limit(2, limit_raw_sql=True) + .all() ) assert len(unicorns) == 2 assert unicorns[1].name == "Unicorn 2" diff --git a/tests/test_model_definition/test_fields_access.py b/tests/test_model_definition/test_fields_access.py index 8298242..3c98cdc 100644 --- a/tests/test_model_definition/test_fields_access.py +++ b/tests/test_model_definition/test_fields_access.py @@ -112,11 +112,11 @@ def test_operator_return_proper_filter_action(method, expected, expected_value): group_ = getattr(PriceList.categories.products.rating, method)("Test") assert group_._kwargs_dict == { - f"categories__products__rating__{expected}": expected_value} + f"categories__products__rating__{expected}": expected_value + } -@pytest.mark.parametrize("method, expected_direction", - [("asc", ""), ("desc", "desc"), ]) +@pytest.mark.parametrize("method, expected_direction", [("asc", ""), ("desc", "desc"),]) def test_operator_return_proper_order_action(method, expected_direction): action = getattr(Product.name, method)() assert action.source_model == Product @@ -141,17 +141,20 @@ def test_combining_groups_together(): group = (Product.name == "Test") & (Product.rating >= 3.0) group.resolve(model_cls=Product) assert len(group._nested_groups) == 2 - assert str(group.get_text_clause()) == ("( ( product.name = 'Test' ) AND" - " ( product.rating >= 3.0 ) )") + assert str(group.get_text_clause()) == ( + "( ( product.name = 'Test' ) AND" " ( product.rating >= 3.0 ) )" + ) group = ~((Product.name == "Test") & (Product.rating >= 3.0)) group.resolve(model_cls=Product) assert len(group._nested_groups) == 2 - assert str(group.get_text_clause()) == (" NOT ( ( product.name = 'Test' ) AND" - " ( product.rating >= 3.0 ) )") + assert str(group.get_text_clause()) == ( + " NOT ( ( product.name = 'Test' ) AND" " ( product.rating >= 3.0 ) )" + ) group = ((Product.name == "Test") & (Product.rating >= 3.0)) | ( - Product.category.name << (["Toys", "Books"])) + Product.category.name << (["Toys", "Books"]) + ) group.resolve(model_cls=Product) assert len(group._nested_groups) == 2 assert len(group._nested_groups[0]._nested_groups) == 2 @@ -159,22 +162,27 @@ def test_combining_groups_together(): category_prefix = group._nested_groups[1].actions[0].table_prefix assert group_str == ( "( ( ( product.name = 'Test' ) AND ( product.rating >= 3.0 ) ) " - f"OR ( {category_prefix}_categories.name IN ('Toys', 'Books') ) )") + f"OR ( {category_prefix}_categories.name IN ('Toys', 'Books') ) )" + ) group = (Product.name % "Test") | ( - (Product.category.price_lists.name.startswith("Aa")) | ( - Product.category.name << (["Toys", "Books"]))) + (Product.category.price_lists.name.startswith("Aa")) + | (Product.category.name << (["Toys", "Books"])) + ) group.resolve(model_cls=Product) assert len(group._nested_groups) == 2 assert len(group._nested_groups[1]._nested_groups) == 2 group_str = str(group.get_text_clause()) - price_list_prefix = group._nested_groups[1]._nested_groups[0].actions[ - 0].table_prefix + price_list_prefix = ( + group._nested_groups[1]._nested_groups[0].actions[0].table_prefix + ) category_prefix = group._nested_groups[1]._nested_groups[1].actions[0].table_prefix assert group_str == ( f"( ( product.name LIKE '%Test%' ) " f"OR ( ( {price_list_prefix}_price_lists.name LIKE 'Aa%' ) " - f"OR ( {category_prefix}_categories.name IN ('Toys', 'Books') ) ) )") + f"OR ( {category_prefix}_categories.name IN ('Toys', 'Books') ) ) )" + ) + # @pytest.mark.asyncio # async def test_filtering_by_field_access(): @@ -195,9 +203,6 @@ def test_combining_groups_together(): # * overload operators and add missing functions that return FilterAction (V) # * return OrderAction for desc() and asc() (V) # * create filter groups for & and | (and ~ - NOT?) (V) - -# * accept args in all functions that accept filters? or only filter and exclude? +# * accept args in all functions that accept filters? or only filter and exclude? (V) # all functions: delete, first, get, get_or_none, get_or_create, all, filter, exclude -# and same from queryset, should they also accept filter groups? -# * accept OrderActions in order_by -# +# * accept OrderActions in order_by (V) diff --git a/tests/test_model_definition/test_pydantic_fields.py b/tests/test_model_definition/test_pydantic_fields.py new file mode 100644 index 0000000..d41669a --- /dev/null +++ b/tests/test_model_definition/test_pydantic_fields.py @@ -0,0 +1,66 @@ +from typing import Optional + +import databases +import pytest +import sqlalchemy +from pydantic import EmailStr, HttpUrl, ValidationError + +import ormar +from tests.settings import DATABASE_URL + +database = databases.Database(DATABASE_URL) +metadata = sqlalchemy.MetaData() + + +class BaseMeta(ormar.ModelMeta): + metadata = metadata + database = database + + +class Test(ormar.Model): + class Meta(BaseMeta): + pass + + def __init__(self, **kwargs): + # you need to pop non - db fields as ormar will complain that it's unknown field + email = kwargs.pop("email", self.__fields__["email"].get_default()) + url = kwargs.pop("url", self.__fields__["url"].get_default()) + super().__init__(**kwargs) + self.email = email + self.url = url + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=200) + email: Optional[EmailStr] # field optional - default to None + url: HttpUrl = "www.example.com" # field with default + + +@pytest.fixture(autouse=True, scope="module") +def create_test_database(): + engine = sqlalchemy.create_engine(DATABASE_URL) + metadata.drop_all(engine) + metadata.create_all(engine) + yield + metadata.drop_all(engine) + + +@pytest.mark.asyncio +async def test_working_with_pydantic_fields(): + async with database: + test = Test(name="Test", email="aka@go2.com") + assert test.name == "Test" + assert test.email == "aka@go2.com" + assert test.url == "www.example.com" + + test.email = "sdta@ada.pt" + assert test.email == "sdta@ada.pt" + + await test.save() + test_check = await Test.objects.get() + + assert test_check.name == "Test" + assert test_check.email is None + assert test_check.url == "www.example.com" + + # TODO add validate assignment to pydantic config + # test_check.email = 1 diff --git a/tests/test_queries/test_or_filters.py b/tests/test_queries/test_or_filters.py index 81a412c..f419d15 100644 --- a/tests/test_queries/test_or_filters.py +++ b/tests/test_queries/test_or_filters.py @@ -70,6 +70,14 @@ async def test_or_filters(): assert len(books) == 4 assert not any([x.title == "The Tower of Fools" for x in books]) + books = ( + await Book.objects.select_related("author") + .filter((Book.author.name == "J.R.R. Tolkien") | (Book.year < 1995)) + .all() + ) + assert len(books) == 4 + assert not any([x.title == "The Tower of Fools" for x in books]) + books = ( await Book.objects.select_related("author") .filter(ormar.or_(year__gt=1960, year__lt=1940)) @@ -110,6 +118,26 @@ async def test_or_filters(): assert books[0].title == "The Silmarillion" assert books[1].title == "The Witcher" + books = ( + await Book.objects.select_related("author") + .filter( + ( + ( + (Book.year > 1960) & (Book.author.name == "J.R.R. Tolkien") + | ( + (Book.year < 2000) + & (Book.author.name == "Andrzej Sapkowski") + ) + ) + & (Book.title.startswith("The")) + ), + ) + .all() + ) + assert len(books) == 2 + assert books[0].title == "The Silmarillion" + assert books[1].title == "The Witcher" + books = ( await Book.objects.select_related("author") .filter( diff --git a/tests/test_queries/test_order_by.py b/tests/test_queries/test_order_by.py index 7c2f10d..d32cf68 100644 --- a/tests/test_queries/test_order_by.py +++ b/tests/test_queries/test_order_by.py @@ -122,11 +122,21 @@ async def test_sort_order_on_main_model(): assert songs[1].name == "Song 2" assert songs[2].name == "Song 1" + songs = await Song.objects.order_by(Song.sort_order.desc()).all() + assert songs[0].name == "Song 3" + assert songs[1].name == "Song 2" + assert songs[2].name == "Song 1" + songs = await Song.objects.order_by("sort_order").all() assert songs[0].name == "Song 1" assert songs[1].name == "Song 2" assert songs[2].name == "Song 3" + songs = await Song.objects.order_by(Song.sort_order.asc()).all() + assert songs[0].name == "Song 1" + assert songs[1].name == "Song 2" + assert songs[2].name == "Song 3" + songs = await Song.objects.order_by("name").all() assert songs[0].name == "Song 1" assert songs[1].name == "Song 2" @@ -145,6 +155,14 @@ async def test_sort_order_on_main_model(): assert songs[2].name == "Song 2" assert songs[3].name == "Song 3" + songs = await Song.objects.order_by( + [Song.sort_order.asc(), Song.name.asc()] + ).all() + assert songs[0].name == "Song 1" + assert songs[1].name == "Song 4" + assert songs[2].name == "Song 2" + assert songs[3].name == "Song 3" + @pytest.mark.asyncio async def test_sort_order_on_related_model(): From 8b951a48beff648af13cb966bea12507023404a0 Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 20 Apr 2021 11:59:36 +0200 Subject: [PATCH 4/9] remove emailstr to vavoid additional deps --- tests/test_model_definition/test_pydantic_fields.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/test_model_definition/test_pydantic_fields.py b/tests/test_model_definition/test_pydantic_fields.py index d41669a..b4be0ae 100644 --- a/tests/test_model_definition/test_pydantic_fields.py +++ b/tests/test_model_definition/test_pydantic_fields.py @@ -3,7 +3,7 @@ from typing import Optional import databases import pytest import sqlalchemy -from pydantic import EmailStr, HttpUrl, ValidationError +from pydantic import HttpUrl import ormar from tests.settings import DATABASE_URL @@ -23,15 +23,12 @@ class Test(ormar.Model): def __init__(self, **kwargs): # you need to pop non - db fields as ormar will complain that it's unknown field - email = kwargs.pop("email", self.__fields__["email"].get_default()) url = kwargs.pop("url", self.__fields__["url"].get_default()) super().__init__(**kwargs) - self.email = email self.url = url id: int = ormar.Integer(primary_key=True) name: str = ormar.String(max_length=200) - email: Optional[EmailStr] # field optional - default to None url: HttpUrl = "www.example.com" # field with default @@ -47,19 +44,17 @@ def create_test_database(): @pytest.mark.asyncio async def test_working_with_pydantic_fields(): async with database: - test = Test(name="Test", email="aka@go2.com") + test = Test(name="Test") assert test.name == "Test" - assert test.email == "aka@go2.com" assert test.url == "www.example.com" - test.email = "sdta@ada.pt" - assert test.email == "sdta@ada.pt" + test.url = "www.sdta.ada.pt" + assert test.url == "www.sdta.ada.pt" await test.save() test_check = await Test.objects.get() assert test_check.name == "Test" - assert test_check.email is None assert test_check.url == "www.example.com" # TODO add validate assignment to pydantic config From d5295543d4234100916d42ea91af985f96c6b988 Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 20 Apr 2021 12:06:50 +0200 Subject: [PATCH 5/9] replace double % for some dialects --- ormar/queryset/actions/filter_action.py | 2 +- tests/test_model_definition/test_fields_access.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ormar/queryset/actions/filter_action.py b/ormar/queryset/actions/filter_action.py index 8b2c7b1..e330bed 100644 --- a/ormar/queryset/actions/filter_action.py +++ b/ormar/queryset/actions/filter_action.py @@ -176,6 +176,6 @@ class FilterAction(QueryAction): clause_text = clause_text.replace( f"{self.table.name}.{self.column.name}", aliased_name ) - clause_text.replace("%%", "%") # remove doubles in some dialects + clause_text = clause_text.replace("%%", "%") # remove doubles in some dialects clause = text(clause_text) return clause diff --git a/tests/test_model_definition/test_fields_access.py b/tests/test_model_definition/test_fields_access.py index 3c98cdc..f5f00b4 100644 --- a/tests/test_model_definition/test_fields_access.py +++ b/tests/test_model_definition/test_fields_access.py @@ -177,6 +177,7 @@ def test_combining_groups_together(): group._nested_groups[1]._nested_groups[0].actions[0].table_prefix ) category_prefix = group._nested_groups[1]._nested_groups[1].actions[0].table_prefix + group_str.replace() assert group_str == ( f"( ( product.name LIKE '%Test%' ) " f"OR ( ( {price_list_prefix}_price_lists.name LIKE 'Aa%' ) " From ac426de75705d2d9bf3f7d6a028f13289ac028de Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 20 Apr 2021 12:10:39 +0200 Subject: [PATCH 6/9] replace double % for some dialects --- tests/test_model_definition/test_fields_access.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_model_definition/test_fields_access.py b/tests/test_model_definition/test_fields_access.py index f5f00b4..3c98cdc 100644 --- a/tests/test_model_definition/test_fields_access.py +++ b/tests/test_model_definition/test_fields_access.py @@ -177,7 +177,6 @@ def test_combining_groups_together(): group._nested_groups[1]._nested_groups[0].actions[0].table_prefix ) category_prefix = group._nested_groups[1]._nested_groups[1].actions[0].table_prefix - group_str.replace() assert group_str == ( f"( ( product.name LIKE '%Test%' ) " f"OR ( ( {price_list_prefix}_price_lists.name LIKE 'Aa%' ) " From 1e68d42643272600b6bcce64ce67be31d173f3f2 Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 20 Apr 2021 12:38:11 +0200 Subject: [PATCH 7/9] exclude sqlite from replacing double % --- ormar/queryset/actions/filter_action.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ormar/queryset/actions/filter_action.py b/ormar/queryset/actions/filter_action.py index e330bed..df644a3 100644 --- a/ormar/queryset/actions/filter_action.py +++ b/ormar/queryset/actions/filter_action.py @@ -56,7 +56,7 @@ class FilterAction(QueryAction): Extracted in order to easily change table prefixes on complex relations. """ - def __init__(self, filter_str: str, value: Any, model_cls: Type["Model"],) -> None: + def __init__(self, filter_str: str, value: Any, model_cls: Type["Model"], ) -> None: super().__init__(query_str=filter_str, model_cls=model_cls) self.filter_value = value self._escape_characters_in_clause() @@ -149,7 +149,7 @@ class FilterAction(QueryAction): return clause def _compile_clause( - self, clause: sqlalchemy.sql.expression.BinaryExpression, modifiers: Dict, + self, clause: sqlalchemy.sql.expression.BinaryExpression, modifiers: Dict, ) -> sqlalchemy.sql.expression.TextClause: """ Compiles the clause to str using appropriate database dialect, replace columns @@ -176,6 +176,7 @@ class FilterAction(QueryAction): clause_text = clause_text.replace( f"{self.table.name}.{self.column.name}", aliased_name ) - clause_text = clause_text.replace("%%", "%") # remove doubles in some dialects + if self.target_model.Meta.database._backend._dialect.name != 'sqlite': + clause_text = clause_text.replace("%%", "%") # remove %% in some dialects clause = text(clause_text) return clause From ac4712f87c334eb7a1e48aba240089d2e47cb3db Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 20 Apr 2021 12:46:05 +0200 Subject: [PATCH 8/9] exclude sqlite from replacing double % --- ormar/queryset/actions/filter_action.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ormar/queryset/actions/filter_action.py b/ormar/queryset/actions/filter_action.py index df644a3..379c9e7 100644 --- a/ormar/queryset/actions/filter_action.py +++ b/ormar/queryset/actions/filter_action.py @@ -176,7 +176,8 @@ class FilterAction(QueryAction): clause_text = clause_text.replace( f"{self.table.name}.{self.column.name}", aliased_name ) - if self.target_model.Meta.database._backend._dialect.name != 'sqlite': + dialect_name = self.target_model.Meta.database._backend._dialect.name + if dialect_name != 'sqlite': # pragma: no cover clause_text = clause_text.replace("%%", "%") # remove %% in some dialects clause = text(clause_text) return clause From c11e1a870f5047cc3b013168f2e514944b050b04 Mon Sep 17 00:00:00 2001 From: collerek Date: Wed, 21 Apr 2021 11:28:49 +0200 Subject: [PATCH 9/9] update docs and readme --- README.md | 45 +++- docs/index.md | 45 +++- docs/queries/filter-and-sort.md | 307 ++++++++++++++++++++---- docs/queries/read.md | 30 +-- docs/releases.md | 10 +- examples/db.sqlite | Bin 0 -> 12288 bytes examples/script_from_readme.py | 33 ++- ormar/queryset/actions/filter_action.py | 6 +- ormar/queryset/field_accessor.py | 10 +- ormar/queryset/queryset.py | 16 +- 10 files changed, 408 insertions(+), 94 deletions(-) create mode 100644 examples/db.sqlite diff --git a/README.md b/README.md index 13a7823..582f2f5 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,10 @@ async def create(): async def read(): # Fetch an instance, without loading a foreign key relationship on it. + # Django style book = await Book.objects.get(title="The Hobbit") + # or python style + book = await Book.objects.get(Book.title == "The Hobbit") book2 = await Book.objects.first() # first() fetch the instance with lower primary key value @@ -334,20 +337,30 @@ async def filter_and_sort(): # get(), all() etc. # to use special methods or access related model fields use double # underscore like to filter by the name of the author use author__name + # Django style books = await Book.objects.all(author__name="J.R.R. Tolkien") + # python style + books = await Book.objects.all(Book.author.name == "J.R.R. Tolkien") assert len(books) == 3 # filter can accept special methods also separated with double underscore # to issue sql query ` where authors.name like "%tolkien%"` that is not # case sensitive (hence small t in Tolkien) + # Django style books = await Book.objects.filter(author__name__icontains="tolkien").all() + # python style + books = await Book.objects.filter(Book.author.name.icontains("tolkien")).all() assert len(books) == 3 # to sort use order_by() function of queryset # to sort decreasing use hyphen before the field name # same as with filter you can use double underscores to access related fields + # Django style books = await Book.objects.filter(author__name__icontains="tolkien").order_by( "-year").all() + # python style + books = await Book.objects.filter(Book.author.name.icontains("tolkien")).order_by( + Book.year.desc()).all() assert len(books) == 3 assert books[0].title == "The Silmarillion" assert books[2].title == "The Hobbit" @@ -417,12 +430,24 @@ async def pagination(): async def aggregations(): - # ormar currently supports count: + # count: assert 2 == await Author.objects.count() - # and exists + # exists: assert await Book.objects.filter(title="The Hobbit").exists() + # max: + assert 1990 == await Book.objects.max(columns=["year"]) + + # min: + assert 1937 == await Book.objects.min(columns=["year"]) + + # avg: + assert 1964.75 == await Book.objects.avg(columns=["year"]) + + # sum: + assert 7859 == await Book.objects.sum(columns=["year"]) + # to read more about aggregated functions # visit: https://collerek.github.io/ormar/queries/aggregations/ @@ -448,16 +473,16 @@ metadata.drop_all(engine) ### QuerySet methods * `create(**kwargs): -> Model` -* `get(**kwargs): -> Model` -* `get_or_none(**kwargs): -> Optional[Model]` -* `get_or_create(**kwargs) -> Model` -* `first(): -> Model` +* `get(*args, **kwargs): -> Model` +* `get_or_none(*args, **kwargs): -> Optional[Model]` +* `get_or_create(*args, **kwargs) -> Model` +* `first(*args, **kwargs): -> Model` * `update(each: bool = False, **kwargs) -> int` * `update_or_create(**kwargs) -> Model` * `bulk_create(objects: List[Model]) -> None` * `bulk_update(objects: List[Model], columns: List[str] = None) -> None` -* `delete(each: bool = False, **kwargs) -> int` -* `all(**kwargs) -> List[Optional[Model]]` +* `delete(*args, each: bool = False, **kwargs) -> int` +* `all(*args, **kwargs) -> List[Optional[Model]]` * `filter(*args, **kwargs) -> QuerySet` * `exclude(*args, **kwargs) -> QuerySet` * `select_related(related: Union[List, str]) -> QuerySet` @@ -466,6 +491,10 @@ metadata.drop_all(engine) * `offset(offset: int) -> QuerySet` * `count() -> int` * `exists() -> bool` +* `max(columns: List[str]) -> Any` +* `min(columns: List[str]) -> Any` +* `avg(columns: List[str]) -> Any` +* `sum(columns: List[str]) -> Any` * `fields(columns: Union[List, str, set, dict]) -> QuerySet` * `exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet` * `order_by(columns:Union[List, str]) -> QuerySet` diff --git a/docs/index.md b/docs/index.md index 13a7823..582f2f5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -220,7 +220,10 @@ async def create(): async def read(): # Fetch an instance, without loading a foreign key relationship on it. + # Django style book = await Book.objects.get(title="The Hobbit") + # or python style + book = await Book.objects.get(Book.title == "The Hobbit") book2 = await Book.objects.first() # first() fetch the instance with lower primary key value @@ -334,20 +337,30 @@ async def filter_and_sort(): # get(), all() etc. # to use special methods or access related model fields use double # underscore like to filter by the name of the author use author__name + # Django style books = await Book.objects.all(author__name="J.R.R. Tolkien") + # python style + books = await Book.objects.all(Book.author.name == "J.R.R. Tolkien") assert len(books) == 3 # filter can accept special methods also separated with double underscore # to issue sql query ` where authors.name like "%tolkien%"` that is not # case sensitive (hence small t in Tolkien) + # Django style books = await Book.objects.filter(author__name__icontains="tolkien").all() + # python style + books = await Book.objects.filter(Book.author.name.icontains("tolkien")).all() assert len(books) == 3 # to sort use order_by() function of queryset # to sort decreasing use hyphen before the field name # same as with filter you can use double underscores to access related fields + # Django style books = await Book.objects.filter(author__name__icontains="tolkien").order_by( "-year").all() + # python style + books = await Book.objects.filter(Book.author.name.icontains("tolkien")).order_by( + Book.year.desc()).all() assert len(books) == 3 assert books[0].title == "The Silmarillion" assert books[2].title == "The Hobbit" @@ -417,12 +430,24 @@ async def pagination(): async def aggregations(): - # ormar currently supports count: + # count: assert 2 == await Author.objects.count() - # and exists + # exists: assert await Book.objects.filter(title="The Hobbit").exists() + # max: + assert 1990 == await Book.objects.max(columns=["year"]) + + # min: + assert 1937 == await Book.objects.min(columns=["year"]) + + # avg: + assert 1964.75 == await Book.objects.avg(columns=["year"]) + + # sum: + assert 7859 == await Book.objects.sum(columns=["year"]) + # to read more about aggregated functions # visit: https://collerek.github.io/ormar/queries/aggregations/ @@ -448,16 +473,16 @@ metadata.drop_all(engine) ### QuerySet methods * `create(**kwargs): -> Model` -* `get(**kwargs): -> Model` -* `get_or_none(**kwargs): -> Optional[Model]` -* `get_or_create(**kwargs) -> Model` -* `first(): -> Model` +* `get(*args, **kwargs): -> Model` +* `get_or_none(*args, **kwargs): -> Optional[Model]` +* `get_or_create(*args, **kwargs) -> Model` +* `first(*args, **kwargs): -> Model` * `update(each: bool = False, **kwargs) -> int` * `update_or_create(**kwargs) -> Model` * `bulk_create(objects: List[Model]) -> None` * `bulk_update(objects: List[Model], columns: List[str] = None) -> None` -* `delete(each: bool = False, **kwargs) -> int` -* `all(**kwargs) -> List[Optional[Model]]` +* `delete(*args, each: bool = False, **kwargs) -> int` +* `all(*args, **kwargs) -> List[Optional[Model]]` * `filter(*args, **kwargs) -> QuerySet` * `exclude(*args, **kwargs) -> QuerySet` * `select_related(related: Union[List, str]) -> QuerySet` @@ -466,6 +491,10 @@ metadata.drop_all(engine) * `offset(offset: int) -> QuerySet` * `count() -> int` * `exists() -> bool` +* `max(columns: List[str]) -> Any` +* `min(columns: List[str]) -> Any` +* `avg(columns: List[str]) -> Any` +* `sum(columns: List[str]) -> Any` * `fields(columns: Union[List, str, set, dict]) -> QuerySet` * `exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet` * `order_by(columns:Union[List, str]) -> QuerySet` diff --git a/docs/queries/filter-and-sort.md b/docs/queries/filter-and-sort.md index ab2625b..4045f10 100644 --- a/docs/queries/filter-and-sort.md +++ b/docs/queries/filter-and-sort.md @@ -2,27 +2,27 @@ You can use following methods to filter the data (sql where clause). -* `filter(**kwargs) -> QuerySet` -* `exclude(**kwargs) -> QuerySet` -* `get(**kwargs) -> Model` -* `get_or_none(**kwargs) -> Optional[Model]` -* `get_or_create(**kwargs) -> Model` -* `all(**kwargs) -> List[Optional[Model]]` +* `filter(*args, **kwargs) -> QuerySet` +* `exclude(*args, **kwargs) -> QuerySet` +* `get(*args, **kwargs) -> Model` +* `get_or_none(*args, **kwargs) -> Optional[Model]` +* `get_or_create(*args, **kwargs) -> Model` +* `all(*args, **kwargs) -> List[Optional[Model]]` * `QuerysetProxy` - * `QuerysetProxy.filter(**kwargs)` method - * `QuerysetProxy.exclude(**kwargs)` method - * `QuerysetProxy.get(**kwargs)` method - * `QuerysetProxy.get_or_none(**kwargs)` method - * `QuerysetProxy.get_or_create(**kwargs)` method - * `QuerysetProxy.all(**kwargs)` method + * `QuerysetProxy.filter(*args, **kwargs)` method + * `QuerysetProxy.exclude(*args, **kwargs)` method + * `QuerysetProxy.get(*args, **kwargs)` method + * `QuerysetProxy.get_or_none(*args, **kwargs)` method + * `QuerysetProxy.get_or_create(*args, **kwargs)` method + * `QuerysetProxy.all(*args, **kwargs)` method And following methods to sort the data (sql order by clause). -* `order_by(columns:Union[List, str]) -> QuerySet` +* `order_by(columns:Union[List, str, OrderAction]) -> QuerySet` * `QuerysetProxy` - * `QuerysetProxy.order_by(columns:Union[List, str])` method + * `QuerysetProxy.order_by(columns:Union[List, str, OrderAction])` method ## Filtering @@ -65,24 +65,107 @@ tracks = Track.objects.filter(album__name="Fantasies").all() # will return all tracks where the columns album name = 'Fantasies' ``` +### Django style filters + You can use special filter suffix to change the filter operands: -* exact - like `album__name__exact='Malibu'` (exact match) -* iexact - like `album__name__iexact='malibu'` (exact match case insensitive) -* contains - like `album__name__contains='Mal'` (sql like) -* icontains - like `album__name__icontains='mal'` (sql like case insensitive) -* in - like `album__name__in=['Malibu', 'Barclay']` (sql in) -* isnull - like `album__name__isnull=True` (sql is null) - (isnotnull `album__name__isnull=False` (sql is not null)) -* gt - like `position__gt=3` (sql >) -* gte - like `position__gte=3` (sql >=) -* lt - like `position__lt=3` (sql <) -* lte - like `position__lte=3` (sql <=) -* startswith - like `album__name__startswith='Mal'` (exact start match) -* istartswith - like `album__name__istartswith='mal'` (exact start match case - insensitive) -* endswith - like `album__name__endswith='ibu'` (exact end match) -* iendswith - like `album__name__iendswith='IBU'` (exact end match case insensitive) +* exact - exact match to value, sql `column = ` + * can be written as`album__name__exact='Malibu'` +* iexact - exact match sql `column = ` (case insensitive) + * can be written as`album__name__iexact='malibu'` +* contains - sql `column LIKE '%%'` + * can be written as`album__name__contains='Mal'` +* icontains - sql `column LIKE '%%'` (case insensitive) + * can be written as`album__name__icontains='mal'` +* in - sql ` column IN (, , ...)` + * can be written as`album__name__in=['Malibu', 'Barclay']` +* isnull - sql `column IS NULL` (and sql `column IS NOT NULL`) + * can be written as`album__name__isnull=True` (isnotnull `album__name__isnull=False`) +* gt - sql `column > ` (greater than) + * can be written as`position__gt=3` +* gte - sql `column >= ` (greater or equal than) + * can be written as`position__gte=3` +* lt - sql `column < ` (lower than) + * can be written as`position__lt=3` +* lte - sql `column <= ` (lower equal than) + * can be written as`position__lte=3` +* startswith - sql `column LIKE '%'` (exact start match) + * can be written as`album__name__startswith='Mal'` +* istartswith - sql `column LIKE '%'` (case insensitive) + * can be written as`album__name__istartswith='mal'` +* endswith - sql `column LIKE '%'` (exact end match) + * can be written as`album__name__endswith='ibu'` +* iendswith - sql `column LIKE '%'` (case insensitive) + * can be written as`album__name__iendswith='IBU'` + +Some samples: + +```python +# sql: ( product.name = 'Test' AND product.rating >= 3.0 ) +Product.objects.filter(name='Test', rating__gte=3.0).get() + +# sql: ( product.name = 'Test' AND product.rating >= 3.0 ) +# OR (categories.name IN ('Toys', 'Books')) +Product.objects.filter( + ormar.or_( + ormar.and_(name='Test', rating__gte=3.0), + categories__name__in=['Toys', 'Books']) + ).get() +# note: to read more about and_ and or_ read complex filters section below +``` + +### Python style filters + +* exact - exact match to value, sql `column = ` + * can be written as `Track.album.name == 'Malibu` +* iexact - exact match sql `column = ` (case insensitive) + * can be written as `Track.album.name.iexact('malibu')` +* contains - sql `column LIKE '%%'` + * can be written as `Track.album.name % 'Mal')` + * can be written as `Track.album.name.contains('Mal')` +* icontains - sql `column LIKE '%%'` (case insensitive) + * can be written as `Track.album.name.icontains('mal')` +* in - sql ` column IN (, , ...)` + * can be written as `Track.album.name << ['Malibu', 'Barclay']` + * can be written as `Track.album.name.in_(['Malibu', 'Barclay'])` +* isnull - sql `column IS NULL` (and sql `column IS NOT NULL`) + * can be written as `Track.album.name >> None` + * can be written as `Track.album.name.is_null(True)` + * not null can be written as `Track.album.name.is_null(False)` + * not null can be written as `~(Track.album.name >> None)` + * not null can be written as `~(Track.album.name.is_null(True))` +* gt - sql `column > ` (greater than) + * can be written as `Track.album.name > 3` +* gte - sql `column >= ` (greater or equal than) + * can be written as `Track.album.name >= 3` +* lt - sql `column < ` (lower than) + * can be written as `Track.album.name < 3` +* lte - sql `column <= ` (lower equal than) + * can be written as `Track.album.name <= 3` +* startswith - sql `column LIKE '%'` (exact start match) + * can be written as `Track.album.name.startswith('Mal')` +* istartswith - sql `column LIKE '%'` (case insensitive) + * can be written as `Track.album.name.istartswith('mal')` +* endswith - sql `column LIKE '%'` (exact end match) + * can be written as `Track.album.name.endswith('ibu')` +* iendswith - sql `column LIKE '%'` (case insensitive) + * can be written as `Track.album.name.iendswith('IBU')` + +Some samples: + +```python +# sql: ( product.name = 'Test' AND product.rating >= 3.0 ) +Product.objects.filter( + (Product.name == 'Test') & (Product.rating >=3.0) +).get() + +# sql: ( product.name = 'Test' AND product.rating >= 3.0 ) +# OR (categories.name IN ('Toys', 'Books')) +Product.objects.filter( + ((Product.name='Test') & (Product.rating >= 3.0)) | + (Product.categories.name << ['Toys', 'Books']) + ).get() +``` !!!note All methods that do not return the rows explicitly returns a QueySet instance so @@ -155,7 +238,7 @@ In order to build `OR` and nested conditions ormar provides two functions that c `filter()` and `exclude()` in `QuerySet` and `QuerysetProxy`. !!!note - Note that you cannot provide those methods in any other method like `get()` or `all()` which accepts only keyword arguments. + Note that you can provide those methods in any other method like `get()` or `all()` that accepts `*args`. Call to `or_` and `and_` can be nested in each other, as well as combined with keyword arguments. Since it sounds more complicated than it is, let's look at some examples. @@ -208,6 +291,7 @@ Let's select books of Tolkien **OR** books written after 1970 sql: `WHERE ( authors.name = 'J.R.R. Tolkien' OR books.year > 1970 )` +### Django style ```python books = ( await Book.objects.select_related("author") @@ -217,11 +301,22 @@ books = ( assert len(books) == 5 ``` +### Python style +```python +books = ( + await Book.objects.select_related("author") + .filter((Book.author.name=="J.R.R. Tolkien") | (Book.year > 1970)) + .all() +) +assert len(books) == 5 +``` + Now let's select books written after 1960 or before 1940 which were written by Tolkien. sql: `WHERE ( books.year > 1960 OR books.year < 1940 ) AND authors.name = 'J.R.R. Tolkien'` +### Django style ```python # OPTION 1 - split and into separate call books = ( @@ -249,11 +344,38 @@ assert books[0].title == "The Hobbit" assert books[1].title == "The Silmarillion" ``` +### Python style +```python +books = ( + await Book.objects.select_related("author") + .filter((Book.year > 1960) | (Book.year < 1940)) + .filter(Book.author.name == "J.R.R. Tolkien") + .all() +) +assert len(books) == 2 + +# OPTION 2 - all in one +books = ( + await Book.objects.select_related("author") + .filter( + ( + (Book.year > 1960) | (Book.year < 1940) + ) & (Book.author.name == "J.R.R. Tolkien") + ) + .all() +) + +assert len(books) == 2 +assert books[0].title == "The Hobbit" +assert books[1].title == "The Silmarillion" +``` + Books of Sapkowski from before 2000 or books of Tolkien written after 1960 sql: `WHERE ( ( books.year > 1960 AND authors.name = 'J.R.R. Tolkien' ) OR ( books.year < 2000 AND authors.name = 'Andrzej Sapkowski' ) ) ` +### Django style ```python books = ( await Book.objects.select_related("author") @@ -268,7 +390,20 @@ books = ( assert len(books) == 2 ``` -Of course those functions can have more than 2 conditions, so if we for example want also +### Python style +```python +books = ( + await Book.objects.select_related("author") + .filter( + ((Book.year > 1960) & (Book.author.name == "J.R.R. Tolkien")) | + ((Book.year < 2000) & (Book.author.name == "Andrzej Sapkowski")) + ) + .all() +) +assert len(books) == 2 +``` + +Of course those functions can have more than 2 conditions, so if we for example want books that contains 'hobbit': sql: @@ -276,6 +411,7 @@ sql: ( books.year < 2000 AND os0cec_authors.name = 'Andrzej Sapkowski' ) OR books.title LIKE '%hobbit%' )` +### Django style ```python books = ( await Book.objects.select_related("author") @@ -290,6 +426,19 @@ books = ( ) ``` +### Python style +```python +books = ( + await Book.objects.select_related("author") + .filter( + ((Book.year > 1960) & (Book.author.name == "J.R.R. Tolkien")) | + ((Book.year < 2000) & (Book.author.name == "Andrzej Sapkowski")) | + (Book.title.icontains("hobbit")) + ) + .all() +) +``` + If you want or need to you can nest deeper conditions as deep as you want, in example to achieve a query like this: @@ -301,6 +450,28 @@ AND authors.name = 'J.R.R. Tolkien' ) OR ``` You can construct a query as follows: + +### Django style +```python +books = ( + await Book.objects.select_related("author") + .filter( + ormar.or_( + ormar.and_( + ormar.or_(year__gt=1960, year__lt=1940), + author__name="J.R.R. Tolkien", + ), + ormar.and_(year__lt=2000, author__name="Andrzej Sapkowski"), + ) + ) + .all() +) +assert len(books) == 3 +assert books[0].title == "The Hobbit" +assert books[1].title == "The Silmarillion" +assert books[2].title == "The Witcher" +``` + ```python books = ( await Book.objects.select_related("author") @@ -339,10 +510,12 @@ assert len(books) == 1 assert books[0].title == "The Witcher" ``` +Same applies to python style chaining and nesting. -!!!note - Note that you cannot provide the same keyword argument several times so queries like `filter(ormar.or_(name='Jack', name='John'))` are not allowed. If you want to check the same - column for several values simply use `in` operator: `filter(name__in=['Jack','John'])`. +### Django style + +Note that with django style you cannot provide the same keyword argument several times so queries like `filter(ormar.or_(name='Jack', name='John'))` are not allowed. If you want to check the same +column for several values simply use `in` operator: `filter(name__in=['Jack','John'])`. If you pass only one parameter to `or_` or `and_` functions it's simply wrapped in parenthesis and has no effect on actual query, so in the end all 3 queries are identical: @@ -386,13 +559,28 @@ books = ( assert len(books) == 5 ``` +### Python style + +Note that with python style you can perfectly use the same fields as many times as you want. + +```python +books = ( + await Book.objects.select_related("author") + .filter( + (Book.author.name.icontains("tolkien")) | + (Book.author.name.icontains("sapkowski")) + )) + .all() +) +``` + ## get -`get(**kwargs) -> Model` +`get(*args, **kwargs) -> Model` Get's the first row from the db meeting the criteria set by kwargs. -When any kwargs are passed it's a shortcut equivalent to calling `filter(**kwargs).get()` +When any args and/or kwargs are passed it's a shortcut equivalent to calling `filter(*args, **kwargs).get()` !!!tip To read more about `filter` go to [filter](./#filter). @@ -403,14 +591,13 @@ When any kwargs are passed it's a shortcut equivalent to calling `filter(**kwarg Exact equivalent of get described above but instead of raising the exception returns `None` if no db record matching the criteria is found. - ## get_or_create -`get_or_create(**kwargs) -> Model` +`get_or_create(*args, **kwargs) -> Model` Combination of create and get methods. -When any kwargs are passed it's a shortcut equivalent to calling `filter(**kwargs).get_or_create()` +When any args and/or kwargs are passed it's a shortcut equivalent to calling `filter(*args, **kwargs).get_or_create()` !!!tip To read more about `filter` go to [filter](./#filter). @@ -423,11 +610,11 @@ When any kwargs are passed it's a shortcut equivalent to calling `filter(**kwarg ## all -`all(**kwargs) -> List[Optional["Model"]]` +`all(*args, **kwargs) -> List[Optional["Model"]]` Returns all rows from a database for given model for set filter options. -When any kwargs are passed it's a shortcut equivalent to calling `filter(**kwargs).all()` +When any kwargs are passed it's a shortcut equivalent to calling `filter(*args, **kwargs).all()` !!!tip To read more about `filter` go to [filter](./#filter). @@ -493,7 +680,7 @@ objects from other side of the relation. ### order_by -`order_by(columns: Union[List, str]) -> QuerySet` +`order_by(columns: Union[List, str, OrderAction]) -> QuerySet` With `order_by()` you can order the results from database based on your choice of fields. @@ -534,6 +721,7 @@ Given sample Models like following: To order by main model field just provide a field name +### Django style ```python toys = await Toy.objects.select_related("owner").order_by("name").all() assert [x.name.replace("Toy ", "") for x in toys] == [ @@ -543,11 +731,23 @@ assert toys[0].owner == zeus assert toys[1].owner == aphrodite ``` +### Python style +```python +toys = await Toy.objects.select_related("owner").order_by(Toy.name.asc()).all() +assert [x.name.replace("Toy ", "") for x in toys] == [ + str(x + 1) for x in range(6) +] +assert toys[0].owner == zeus +assert toys[1].owner == aphrodite +``` + + To sort on nested models separate field names with dunder '__'. You can sort this way across all relation types -> `ForeignKey`, reverse virtual FK and `ManyToMany` fields. +### Django style ```python toys = await Toy.objects.select_related("owner").order_by("owner__name").all() assert toys[0].owner.name == toys[1].owner.name == "Aphrodite" @@ -555,8 +755,17 @@ assert toys[2].owner.name == toys[3].owner.name == "Hermes" assert toys[4].owner.name == toys[5].owner.name == "Zeus" ``` +### Python style +```python +toys = await Toy.objects.select_related("owner").order_by(Toy.owner.name.asc()).all() +assert toys[0].owner.name == toys[1].owner.name == "Aphrodite" +assert toys[2].owner.name == toys[3].owner.name == "Hermes" +assert toys[4].owner.name == toys[5].owner.name == "Zeus" +``` + To sort in descending order provide a hyphen in front of the field name +### Django style ```python owner = ( await Owner.objects.select_related("toys") @@ -568,6 +777,18 @@ assert owner.toys[0].name == "Toy 4" assert owner.toys[1].name == "Toy 1" ``` +### Python style +```python +owner = ( + await Owner.objects.select_related("toys") + .order_by(Owner.toys.name.desc()) + .filter(Owner.name == "Zeus") + .get() +) +assert owner.toys[0].name == "Toy 4" +assert owner.toys[1].name == "Toy 1" +``` + !!!note All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together diff --git a/docs/queries/read.md b/docs/queries/read.md index 40b84bb..17972b5 100644 --- a/docs/queries/read.md +++ b/docs/queries/read.md @@ -2,10 +2,10 @@ Following methods allow you to load data from the database. -* `get(**kwargs) -> Model` -* `get_or_create(**kwargs) -> Model` -* `first() -> Model` -* `all(**kwargs) -> List[Optional[Model]]` +* `get(*args, **kwargs) -> Model` +* `get_or_create(*args, **kwargs) -> Model` +* `first(*args, **kwargs) -> Model` +* `all(*args, **kwargs) -> List[Optional[Model]]` * `Model` @@ -13,20 +13,20 @@ Following methods allow you to load data from the database. * `QuerysetProxy` - * `QuerysetProxy.get(**kwargs)` method - * `QuerysetProxy.get_or_create(**kwargs)` method - * `QuerysetProxy.first()` method - * `QuerysetProxy.all(**kwargs)` method + * `QuerysetProxy.get(*args, **kwargs)` method + * `QuerysetProxy.get_or_create(*args, **kwargs)` method + * `QuerysetProxy.first(*args, **kwargs)` method + * `QuerysetProxy.all(*args, **kwargs)` method ## get -`get(**kwargs) -> Model` +`get(*args, **kwargs) -> Model` Get's the first row from the db meeting the criteria set by kwargs. If no criteria set it will return the last row in db sorted by pk column. -Passing a criteria is actually calling filter(**kwargs) method described below. +Passing a criteria is actually calling filter(*args, **kwargs) method described below. ```python class Track(ormar.Model): @@ -57,14 +57,14 @@ track == track2 ## get_or_none -`get_or_none(**kwargs) -> Model` +`get_or_none(*args, **kwargs) -> Model` Exact equivalent of get described above but instead of raising the exception returns `None` if no db record matching the criteria is found. ## get_or_create -`get_or_create(**kwargs) -> Model` +`get_or_create(*args, **kwargs) -> Model` Combination of create and get methods. @@ -102,7 +102,7 @@ assert album == album2 ## first -`first() -> Model` +`first(*args, **kwargs) -> Model` Gets the first row from the db ordered by primary key column ascending. @@ -127,11 +127,11 @@ assert album.name == 'The Cat' ## all -`all(**kwargs) -> List[Optional["Model"]]` +`all(*args, **kwargs) -> List[Optional["Model"]]` Returns all rows from a database for given model for set filter options. -Passing kwargs is a shortcut and equals to calling `filter(**kwrags).all()`. +Passing kwargs is a shortcut and equals to calling `filter(*args, **kwargs).all()`. If there are no rows meeting the criteria an empty list is returned. diff --git a/docs/releases.md b/docs/releases.md index 2fc282d..6b9a4a5 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -2,7 +2,7 @@ ## ✨ Features -* Add possibility to `filter` and `order_by` with field access instead of dunder separated strings. [#51](https://github.com/collerek/ormar/issues/51) +* Add **Python style** to `filter` and `order_by` with field access instead of dunder separated strings. [#51](https://github.com/collerek/ormar/issues/51) * Accessing a field with attribute access (chain of dot notation) can be used to construct `FilterGroups` (`ormar.and_` and `ormar.or_`) * Field access overloads set of python operators and provide a set of functions to allow same functionality as with dunder separated param names in `**kwargs`, that means that querying from sample model `Track` related to model `Album` now you have more options: * exact - exact match to value, sql `column = ` @@ -102,7 +102,7 @@ (Product.categories.name << ['Toys', 'Books']) ).get() ``` -* Now you can alos use field access to provide OrderActions to `order_by()` +* Now you can also use field access to provide OrderActions to `order_by()` * Order ascending: * OLD: `Product.objects.order_by("name").all()` * NEW: `Product.objects.order_by(Product.name.asc()).all()` @@ -112,6 +112,12 @@ * You can of course also combine different models and many order_bys: `Product.objects.order_by([Product.category.name.asc(), Product.name.desc()]).all()` +## 🐛 Fixes + +* Not really a bug but rather inconsistency. Providing a filter with nested model i.e. `album__category__name = 'AA'` + is checking if album and category models are included in `select_related()` and if not it's auto-adding them there. + The same functionality was not working for `FilterGroups` (`and_` and `or_`), now it works (also for python style filters which return `FilterGroups`). + # 0.10.3 ## ✨ Features diff --git a/examples/db.sqlite b/examples/db.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..e9f6c4221cd1530b58dbcd885cfb6d3cb358e8b0 GIT binary patch literal 12288 zcmeI%!HUyB7zgl4x}~(B*+rp;UE~ul4d}Yd3VYO8rn}v(sZOJ^r;?h~j!iR?W?2+3 zqCS8J?<>B;f-m3$c=0I~U%^S53Ki^m(SO3sgnZx3+;x~=FoKaBaslJS&n-l0ob!YX6d`^%q}&p}^02y>7hQP5nU>4wM>C^RCns-0?Vh&ixLj{1!E8RT)s*_4pna-Ie1 z=>2Fo?&putpKUbl%b>WVR#))$o}0~V%%V04-r|D0P2QtoLcVCt*01-?`nvJl?s8LKp#eNi!2#>>vJQa;)7+*C@?6wmJ$?Nb1_57D(but~xHKWArgH55&d16si z_AlSGK3iwjhl^Dvo(2LCfB*y_009U<00Izz00bcL&j?s&Y{S?#O>Y$H@A8u{AvL6y zQqi>X{mM$288>FJr!svu3X{sWdb!N1xAOd+j3#~}qe!Z_@^!=1f!#dNQAv=klj=o< WjGs>{U)BsAy`D$+)nFjg%Ew None: + def __init__(self, filter_str: str, value: Any, model_cls: Type["Model"],) -> None: super().__init__(query_str=filter_str, model_cls=model_cls) self.filter_value = value self._escape_characters_in_clause() @@ -149,7 +149,7 @@ class FilterAction(QueryAction): return clause def _compile_clause( - self, clause: sqlalchemy.sql.expression.BinaryExpression, modifiers: Dict, + self, clause: sqlalchemy.sql.expression.BinaryExpression, modifiers: Dict, ) -> sqlalchemy.sql.expression.TextClause: """ Compiles the clause to str using appropriate database dialect, replace columns @@ -177,7 +177,7 @@ class FilterAction(QueryAction): f"{self.table.name}.{self.column.name}", aliased_name ) dialect_name = self.target_model.Meta.database._backend._dialect.name - if dialect_name != 'sqlite': # pragma: no cover + if dialect_name != "sqlite": # pragma: no cover clause_text = clause_text.replace("%%", "%") # remove %% in some dialects clause = text(clause_text) return clause diff --git a/ormar/queryset/field_accessor.py b/ormar/queryset/field_accessor.py index 52cc755..2701454 100644 --- a/ormar/queryset/field_accessor.py +++ b/ormar/queryset/field_accessor.py @@ -10,11 +10,11 @@ if TYPE_CHECKING: # pragma: no cover class FieldAccessor: def __init__( - self, - source_model: Type["Model"], - field: "BaseField" = None, - model: Type["Model"] = None, - access_chain: str = "", + self, + source_model: Type["Model"], + field: "BaseField" = None, + model: Type["Model"] = None, + access_chain: str = "", ) -> None: self._source_model = source_model self._field = field diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 0024a7e..3cf4211 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -7,6 +7,7 @@ from typing import ( Sequence, Set, TYPE_CHECKING, + Tuple, Type, TypeVar, Union, @@ -180,16 +181,19 @@ class QuerySet(Generic[T]): return self.model.merge_instances_list(result_rows) # type: ignore return cast(List[Optional["T"]], result_rows) - def _resolve_filter_groups(self, groups: Any) -> List[FilterGroup]: + def _resolve_filter_groups( + self, groups: Any + ) -> Tuple[List[FilterGroup], List[str]]: """ Resolves filter groups to populate FilterAction params in group tree. :param groups: tuple of FilterGroups :type groups: Any :return: list of resolver groups - :rtype: List[FilterGroup] + :rtype: Tuple[List[FilterGroup], List[str]] """ filter_groups = [] + select_related = self._select_related if groups: for group in groups: if not isinstance(group, FilterGroup): @@ -200,13 +204,13 @@ class QuerySet(Generic[T]): "other values need to be passed by" "keyword arguments" ) - group.resolve( + _, select_related = group.resolve( model_cls=self.model, select_related=self._select_related, filter_clauses=self.filter_clauses, ) filter_groups.append(group) - return filter_groups + return filter_groups, select_related @staticmethod def check_single_result_rows_count(rows: Sequence[Optional["T"]]) -> None: @@ -304,10 +308,10 @@ class QuerySet(Generic[T]): :return: filtered QuerySet :rtype: QuerySet """ - filter_groups = self._resolve_filter_groups(groups=args) + filter_groups, select_related = self._resolve_filter_groups(groups=args) qryclause = QueryClause( model_cls=self.model, - select_related=self._select_related, + select_related=select_related, filter_clauses=self.filter_clauses, ) filter_clauses, select_related = qryclause.prepare_filter(**kwargs)