From 3fd231cf3cb2077197df1c2983ee975332edc425 Mon Sep 17 00:00:00 2001 From: collerek Date: Mon, 15 Feb 2021 17:30:14 +0100 Subject: [PATCH 01/14] wip - through models fields -> attached in queries, accesible from instances, creates in add and queryset create --- ormar/fields/__init__.py | 3 + ormar/fields/through_field.py | 73 ++++++ ormar/models/__init__.py | 3 +- ormar/models/helpers/relations.py | 36 ++- ormar/models/metaclass.py | 154 ++++++------ ormar/models/mixins/excludable_mixin.py | 3 +- ormar/models/mixins/relation_mixin.py | 29 ++- ormar/models/model.py | 252 +------------------- ormar/models/model_row.py | 303 ++++++++++++++++++++++++ ormar/models/newbasemodel.py | 6 +- ormar/models/quick_access_views.py | 2 + ormar/queryset/join.py | 20 ++ ormar/queryset/queryset.py | 2 +- ormar/relations/alias_manager.py | 5 +- ormar/relations/querysetproxy.py | 17 +- ormar/relations/relation.py | 3 +- ormar/relations/relation_manager.py | 4 +- ormar/relations/relation_proxy.py | 6 +- tests/test_m2m_through_fields.py | 130 ++++++++-- 19 files changed, 677 insertions(+), 374 deletions(-) create mode 100644 ormar/fields/through_field.py create mode 100644 ormar/models/model_row.py diff --git a/ormar/fields/__init__.py b/ormar/fields/__init__.py index 2e4a013..c5f61d4 100644 --- a/ormar/fields/__init__.py +++ b/ormar/fields/__init__.py @@ -21,6 +21,7 @@ from ormar.fields.model_fields import ( Time, UUID, ) +from ormar.fields.through_field import Through, ThroughField __all__ = [ "Decimal", @@ -41,4 +42,6 @@ __all__ = [ "BaseField", "UniqueColumns", "ForeignKeyField", + "ThroughField", + "Through", ] diff --git a/ormar/fields/through_field.py b/ormar/fields/through_field.py new file mode 100644 index 0000000..99361c7 --- /dev/null +++ b/ormar/fields/through_field.py @@ -0,0 +1,73 @@ +import sys +from typing import Any, TYPE_CHECKING, Type, Union + +from ormar.fields.base import BaseField +from ormar.fields.foreign_key import ForeignKeyField + +if TYPE_CHECKING: # pragma no cover + from ormar import Model + from pydantic.typing import ForwardRef + + if sys.version_info < (3, 7): + ToType = Type[Model] + else: + ToType = Union[Type[Model], ForwardRef] + + +def Through( # noqa CFQ002 + to: "ToType", + *, + name: str = None, + related_name: str = None, + virtual: bool = True, + **kwargs: Any, +) -> Any: + # TODO: clean docstring + """ + Despite a name it's a function that returns constructed ForeignKeyField. + This function is actually used in model declaration (as ormar.ForeignKey(ToModel)). + + Accepts number of relation setting parameters as well as all BaseField ones. + + :param to: target related ormar Model + :type to: Model class + :param name: name of the database field - later called alias + :type name: str + :param related_name: name of reversed FK relation populated for you on to model + :type related_name: str + :param virtual: marks if relation is virtual. + It is for reversed FK and auto generated FK on through model in Many2Many relations. + :type virtual: bool + :param kwargs: all other args to be populated by BaseField + :type kwargs: Any + :return: ormar ForeignKeyField with relation to selected model + :rtype: ForeignKeyField + """ + + owner = kwargs.pop("owner", None) + namespace = dict( + __type__=to, + to=to, + through=None, + alias=name, + name=kwargs.pop("real_name", None), + related_name=related_name, + virtual=virtual, + owner=owner, + nullable=False, + unique=False, + column_type=None, + primary_key=False, + index=False, + pydantic_only=False, + default=None, + server_default=None, + ) + + return type("Through", (ThroughField, BaseField), namespace) + + +class ThroughField(ForeignKeyField): + """ + Field class used to access ManyToMany model through model. + """ diff --git a/ormar/models/__init__.py b/ormar/models/__init__.py index 88a39c6..9990c4a 100644 --- a/ormar/models/__init__.py +++ b/ormar/models/__init__.py @@ -5,6 +5,7 @@ ass well as vast number of helper functions for pydantic, sqlalchemy and relatio """ from ormar.models.newbasemodel import NewBaseModel # noqa I100 +from ormar.models.model_row import ModelRow # noqa I100 from ormar.models.model import Model # noqa I100 -__all__ = ["NewBaseModel", "Model"] +__all__ = ["NewBaseModel", "Model", "ModelRow"] diff --git a/ormar/models/helpers/relations.py b/ormar/models/helpers/relations.py index 4cf19ea..af9ee61 100644 --- a/ormar/models/helpers/relations.py +++ b/ormar/models/helpers/relations.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Type import ormar from ormar import ForeignKey, ManyToMany -from ormar.fields import ManyToManyField +from ormar.fields import ManyToManyField, Through, ThroughField from ormar.fields.foreign_key import ForeignKeyField from ormar.models.helpers.sqlalchemy import adjust_through_many_to_many_model from ormar.relations import AliasManager @@ -81,7 +81,8 @@ def expand_reverse_relationships(model: Type["Model"]) -> None: :param model: model on which relation should be checked and registered :type model: Model class """ - for model_field in model.Meta.model_fields.values(): + model_fields = list(model.Meta.model_fields.values()) + for model_field in model_fields: if ( issubclass(model_field, ForeignKeyField) and not model_field.has_unresolved_forward_refs() @@ -113,6 +114,7 @@ def register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None: self_reference_primary=model_field.self_reference_primary, ) # register foreign keys on through model + register_through_shortcut_fields(model_field=model_field) adjust_through_many_to_many_model(model_field=model_field) else: model_field.to.Meta.model_fields[related_name] = ForeignKey( @@ -125,6 +127,34 @@ def register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None: ) +def register_through_shortcut_fields(model_field: Type["ManyToManyField"]) -> None: + """ + Registers m2m relation through shortcut on both ends of the relation. + + :param model_field: relation field defined in parent model + :type model_field: ManyToManyField + """ + through_model = model_field.through + through_name = through_model.get_name(lower=True) + related_name = model_field.get_related_name() + + model_field.owner.Meta.model_fields[through_name] = Through( + through_model, + real_name=through_name, + virtual=True, + related_name=model_field.name, + owner=model_field.owner, + ) + + model_field.to.Meta.model_fields[through_name] = Through( + through_model, + real_name=through_name, + virtual=True, + related_name=related_name, + owner=model_field.to, + ) + + def register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None: """ Registers the relation (and reverse relation) in alias manager. @@ -142,7 +172,7 @@ def register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None: if field.has_unresolved_forward_refs(): return register_many_to_many_relation_on_build(field=field) - elif issubclass(field, ForeignKeyField): + elif issubclass(field, ForeignKeyField) and not issubclass(field, ThroughField): if field.has_unresolved_forward_refs(): return register_relation_on_build(field=field) diff --git a/ormar/models/metaclass.py b/ormar/models/metaclass.py index 0fc6f5d..4134fd5 100644 --- a/ormar/models/metaclass.py +++ b/ormar/models/metaclass.py @@ -46,6 +46,7 @@ if TYPE_CHECKING: # pragma no cover from ormar import Model CONFIG_KEY = "Config" +PARSED_FIELDS_KEY = "__parsed_fields__" class ModelMeta: @@ -141,83 +142,6 @@ def register_signals(new_model: Type["Model"]) -> None: # noqa: CCR001 new_model.Meta.signals = signals -class ModelMetaclass(pydantic.main.ModelMetaclass): - def __new__( # type: ignore # noqa: CCR001 - mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict - ) -> "ModelMetaclass": - """ - Metaclass used by ormar Models that performs configuration - and build of ormar Models. - - - Sets pydantic configuration. - Extract model_fields and convert them to pydantic FieldInfo, - updates class namespace. - - Extracts settings and fields from parent classes. - Fetches methods decorated with @property_field decorator - to expose them later in dict(). - - Construct parent pydantic Metaclass/ Model. - - If class has Meta class declared (so actual ormar Models) it also: - - * populate sqlalchemy columns, pkname and tables from model_fields - * register reverse relationships on related models - * registers all relations in alias manager that populates table_prefixes - * exposes alias manager on each Model - * creates QuerySet for each model and exposes it on a class - - :param name: name of current class - :type name: str - :param bases: base classes - :type bases: Tuple - :param attrs: class namespace - :type attrs: Dict - """ - attrs["Config"] = get_pydantic_base_orm_config() - attrs["__name__"] = name - attrs, model_fields = extract_annotations_and_default_vals(attrs) - for base in reversed(bases): - mod = base.__module__ - if mod.startswith("ormar.models.") or mod.startswith("pydantic."): - continue - attrs, model_fields = extract_from_parents_definition( - base_class=base, curr_class=mcs, attrs=attrs, model_fields=model_fields - ) - new_model = super().__new__( # type: ignore - mcs, name, bases, attrs - ) - - add_cached_properties(new_model) - - if hasattr(new_model, "Meta"): - populate_default_options_values(new_model, model_fields) - check_required_meta_parameters(new_model) - add_property_fields(new_model, attrs) - register_signals(new_model=new_model) - populate_choices_validators(new_model) - - if not new_model.Meta.abstract: - new_model = populate_meta_tablename_columns_and_pk(name, new_model) - populate_meta_sqlalchemy_table_if_required(new_model.Meta) - expand_reverse_relationships(new_model) - for field in new_model.Meta.model_fields.values(): - register_relation_in_alias_manager(field=field) - - if new_model.Meta.pkname not in attrs["__annotations__"]: - field_name = new_model.Meta.pkname - attrs["__annotations__"][field_name] = Optional[int] # type: ignore - attrs[field_name] = None - new_model.__fields__[field_name] = get_pydantic_field( - field_name=field_name, model=new_model - ) - new_model.Meta.alias_manager = alias_manager - new_model.objects = QuerySet(new_model) - - return new_model - - def verify_constraint_names( base_class: "Model", model_fields: Dict, parent_value: List ) -> None: @@ -539,4 +463,78 @@ def update_attrs_and_fields( return updated_model_fields -PARSED_FIELDS_KEY = "__parsed_fields__" +class ModelMetaclass(pydantic.main.ModelMetaclass): + def __new__( # type: ignore # noqa: CCR001 + mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict + ) -> "ModelMetaclass": + """ + Metaclass used by ormar Models that performs configuration + and build of ormar Models. + + + Sets pydantic configuration. + Extract model_fields and convert them to pydantic FieldInfo, + updates class namespace. + + Extracts settings and fields from parent classes. + Fetches methods decorated with @property_field decorator + to expose them later in dict(). + + Construct parent pydantic Metaclass/ Model. + + If class has Meta class declared (so actual ormar Models) it also: + + * populate sqlalchemy columns, pkname and tables from model_fields + * register reverse relationships on related models + * registers all relations in alias manager that populates table_prefixes + * exposes alias manager on each Model + * creates QuerySet for each model and exposes it on a class + + :param name: name of current class + :type name: str + :param bases: base classes + :type bases: Tuple + :param attrs: class namespace + :type attrs: Dict + """ + attrs["Config"] = get_pydantic_base_orm_config() + attrs["__name__"] = name + attrs, model_fields = extract_annotations_and_default_vals(attrs) + for base in reversed(bases): + mod = base.__module__ + if mod.startswith("ormar.models.") or mod.startswith("pydantic."): + continue + attrs, model_fields = extract_from_parents_definition( + base_class=base, curr_class=mcs, attrs=attrs, model_fields=model_fields + ) + new_model = super().__new__( # type: ignore + mcs, name, bases, attrs + ) + + add_cached_properties(new_model) + + if hasattr(new_model, "Meta"): + populate_default_options_values(new_model, model_fields) + check_required_meta_parameters(new_model) + add_property_fields(new_model, attrs) + register_signals(new_model=new_model) + populate_choices_validators(new_model) + + if not new_model.Meta.abstract: + new_model = populate_meta_tablename_columns_and_pk(name, new_model) + populate_meta_sqlalchemy_table_if_required(new_model.Meta) + expand_reverse_relationships(new_model) + for field in new_model.Meta.model_fields.values(): + register_relation_in_alias_manager(field=field) + + if new_model.Meta.pkname not in attrs["__annotations__"]: + field_name = new_model.Meta.pkname + attrs["__annotations__"][field_name] = Optional[int] # type: ignore + attrs[field_name] = None + new_model.__fields__[field_name] = get_pydantic_field( + field_name=field_name, model=new_model + ) + new_model.Meta.alias_manager = alias_manager + new_model.objects = QuerySet(new_model) + + return new_model diff --git a/ormar/models/mixins/excludable_mixin.py b/ormar/models/mixins/excludable_mixin.py index 961e284..e474bd4 100644 --- a/ormar/models/mixins/excludable_mixin.py +++ b/ormar/models/mixins/excludable_mixin.py @@ -31,6 +31,7 @@ class ExcludableMixin(RelationMixin): if TYPE_CHECKING: # pragma: no cover from ormar import Model + from ormar.models import ModelRow @staticmethod def get_child( @@ -157,7 +158,7 @@ class ExcludableMixin(RelationMixin): @classmethod def own_table_columns( cls, - model: Type["Model"], + model: Union[Type["Model"], Type["ModelRow"]], fields: Optional[Union[Set, Dict]], exclude_fields: Optional[Union[Set, Dict]], use_alias: bool = False, diff --git a/ormar/models/mixins/relation_mixin.py b/ormar/models/mixins/relation_mixin.py index 48688d6..435fcc8 100644 --- a/ormar/models/mixins/relation_mixin.py +++ b/ormar/models/mixins/relation_mixin.py @@ -1,6 +1,8 @@ import inspect from typing import List, Optional, Set, TYPE_CHECKING +from ormar import ManyToManyField +from ormar.fields import ThroughField from ormar.fields.foreign_key import ForeignKeyField @@ -43,27 +45,46 @@ class RelationMixin: return cls._related_fields related_fields = [] - for name in cls.extract_related_names(): + for name in cls.extract_related_names().union(cls.extract_through_names()): related_fields.append(cls.Meta.model_fields[name]) cls._related_fields = related_fields return related_fields + @classmethod + def extract_through_names(cls) -> Set: + """ + Extracts related fields through names which are shortcuts to through models. + + :return: set of related through fields names + :rtype: Set + """ + related_fields = set() + for name in cls.extract_related_names(): + field = cls.Meta.model_fields[name] + if issubclass(field, ManyToManyField): + related_fields.add(field.through.get_name(lower=True)) + return related_fields + @classmethod def extract_related_names(cls) -> Set: """ Returns List of fields names for all relations declared on a model. List is cached in cls._related_names for quicker access. - :return: list of related fields names - :rtype: List + :return: set of related fields names + :rtype: Set """ if isinstance(cls._related_names, Set): return cls._related_names related_names = set() for name, field in cls.Meta.model_fields.items(): - if inspect.isclass(field) and issubclass(field, ForeignKeyField): + if ( + inspect.isclass(field) + and issubclass(field, ForeignKeyField) + and not issubclass(field, ThroughField) + ): related_names.add(name) cls._related_names = related_names diff --git a/ormar/models/model.py b/ormar/models/model.py index 3c66a0a..9286da9 100644 --- a/ormar/models/model.py +++ b/ormar/models/model.py @@ -1,24 +1,17 @@ from typing import ( Any, - Dict, - List, - Optional, Set, TYPE_CHECKING, Tuple, - Type, TypeVar, - Union, ) -import sqlalchemy - import ormar.queryset # noqa I100 from ormar.exceptions import ModelPersistenceError, NoMatch from ormar.fields.many_to_many import ManyToManyField from ormar.models import NewBaseModel # noqa I100 -from ormar.models.helpers.models import group_related_list from ormar.models.metaclass import ModelMeta +from ormar.models.model_row import ModelRow if TYPE_CHECKING: # pragma nocover from ormar import QuerySet @@ -26,7 +19,7 @@ if TYPE_CHECKING: # pragma nocover T = TypeVar("T", bound="Model") -class Model(NewBaseModel): +class Model(ModelRow): __abstract__ = False if TYPE_CHECKING: # pragma nocover Meta: ModelMeta @@ -36,247 +29,6 @@ class Model(NewBaseModel): _repr = {k: getattr(self, k) for k, v in self.Meta.model_fields.items()} return f"{self.__class__.__name__}({str(_repr)})" - @classmethod - def from_row( # noqa CCR001 - cls: Type[T], - row: sqlalchemy.engine.ResultProxy, - select_related: List = None, - related_models: Any = None, - previous_model: Type[T] = None, - source_model: Type[T] = None, - related_name: str = None, - fields: Optional[Union[Dict, Set]] = None, - exclude_fields: Optional[Union[Dict, Set]] = None, - current_relation_str: str = None, - ) -> Optional[T]: - """ - Model method to convert raw sql row from database into ormar.Model instance. - Traverses nested models if they were specified in select_related for query. - - Called recurrently and returns model instance if it's present in the row. - Note that it's processing one row at a time, so if there are duplicates of - parent row that needs to be joined/combined - (like parent row in sql join with 2+ child rows) - instances populated in this method are later combined in the QuerySet. - Other method working directly on raw database results is in prefetch_query, - where rows are populated in a different way as they do not have - nested models in result. - - :param current_relation_str: name of the relation field - :type current_relation_str: str - :param source_model: model on which relation was defined - :type source_model: Type[Model] - :param row: raw result row from the database - :type row: sqlalchemy.engine.result.ResultProxy - :param select_related: list of names of related models fetched from database - :type select_related: List - :param related_models: list or dict of related models - :type related_models: Union[List, Dict] - :param previous_model: internal param for nested models to specify table_prefix - :type previous_model: Model class - :param related_name: internal parameter - name of current nested model - :type related_name: str - :param fields: fields and related model fields to include - if provided only those are included - :type fields: Optional[Union[Dict, Set]] - :param exclude_fields: fields and related model fields to exclude - excludes the fields even if they are provided in fields - :type exclude_fields: Optional[Union[Dict, Set]] - :return: returns model if model is populated from database - :rtype: Optional[Model] - """ - item: Dict[str, Any] = {} - select_related = select_related or [] - related_models = related_models or [] - table_prefix = "" - - if select_related: - source_model = cls - related_models = group_related_list(select_related) - - rel_name2 = related_name - - if ( - previous_model - and related_name - and issubclass( - previous_model.Meta.model_fields[related_name], ManyToManyField - ) - ): - through_field = previous_model.Meta.model_fields[related_name] - if ( - through_field.self_reference - and related_name == through_field.self_reference_primary - ): - rel_name2 = through_field.default_source_field_name() # type: ignore - else: - rel_name2 = through_field.default_target_field_name() # type: ignore - previous_model = through_field.through # type: ignore - - if previous_model and rel_name2: - if current_relation_str and "__" in current_relation_str and source_model: - table_prefix = cls.Meta.alias_manager.resolve_relation_alias( - from_model=source_model, relation_name=current_relation_str - ) - if not table_prefix: - table_prefix = cls.Meta.alias_manager.resolve_relation_alias( - from_model=previous_model, relation_name=rel_name2 - ) - - item = cls.populate_nested_models_from_row( - item=item, - row=row, - related_models=related_models, - fields=fields, - exclude_fields=exclude_fields, - current_relation_str=current_relation_str, - source_model=source_model, - ) - item = cls.extract_prefixed_table_columns( - item=item, - row=row, - table_prefix=table_prefix, - fields=fields, - exclude_fields=exclude_fields, - ) - - instance: Optional[T] = None - if item.get(cls.Meta.pkname, None) is not None: - item["__excluded__"] = cls.get_names_to_exclude( - fields=fields, exclude_fields=exclude_fields - ) - instance = cls(**item) - instance.set_save_status(True) - return instance - - @classmethod - def populate_nested_models_from_row( # noqa: CFQ002 - cls, - item: dict, - row: sqlalchemy.engine.ResultProxy, - related_models: Any, - fields: Optional[Union[Dict, Set]] = None, - exclude_fields: Optional[Union[Dict, Set]] = None, - current_relation_str: str = None, - source_model: Type[T] = None, - ) -> dict: - """ - Traverses structure of related models and populates the nested models - from the database row. - Related models can be a list if only directly related models are to be - populated, converted to dict if related models also have their own related - models to be populated. - - Recurrently calls from_row method on nested instances and create nested - instances. In the end those instances are added to the final model dictionary. - - :param source_model: source model from which relation started - :type source_model: Type[Model] - :param current_relation_str: joined related parts into one string - :type current_relation_str: str - :param item: dictionary of already populated nested models, otherwise empty dict - :type item: Dict - :param row: raw result row from the database - :type row: sqlalchemy.engine.result.ResultProxy - :param related_models: list or dict of related models - :type related_models: Union[Dict, List] - :param fields: fields and related model fields to include - - if provided only those are included - :type fields: Optional[Union[Dict, Set]] - :param exclude_fields: fields and related model fields to exclude - excludes the fields even if they are provided in fields - :type exclude_fields: Optional[Union[Dict, Set]] - :return: dictionary with keys corresponding to model fields names - and values are database values - :rtype: Dict - """ - - for related in related_models: - relation_str = ( - "__".join([current_relation_str, related]) - if current_relation_str - else related - ) - fields = cls.get_included(fields, related) - exclude_fields = cls.get_excluded(exclude_fields, related) - model_cls = cls.Meta.model_fields[related].to - - remainder = None - if isinstance(related_models, dict) and related_models[related]: - remainder = related_models[related] - child = model_cls.from_row( - row, - related_models=remainder, - previous_model=cls, - related_name=related, - fields=fields, - exclude_fields=exclude_fields, - current_relation_str=relation_str, - source_model=source_model, - ) - item[model_cls.get_column_name_from_alias(related)] = child - - return item - - @classmethod - def extract_prefixed_table_columns( # noqa CCR001 - cls, - item: dict, - row: sqlalchemy.engine.result.ResultProxy, - table_prefix: str, - fields: Optional[Union[Dict, Set]] = None, - exclude_fields: Optional[Union[Dict, Set]] = None, - ) -> dict: - """ - Extracts own fields from raw sql result, using a given prefix. - Prefix changes depending on the table's position in a join. - - If the table is a main table, there is no prefix. - All joined tables have prefixes to allow duplicate column names, - as well as duplicated joins to the same table from multiple different tables. - - Extracted fields populates the related dict later used to construct a Model. - - Used in Model.from_row and PrefetchQuery._populate_rows methods. - - :param item: dictionary of already populated nested models, otherwise empty dict - :type item: Dict - :param row: raw result row from the database - :type row: sqlalchemy.engine.result.ResultProxy - :param table_prefix: prefix of the table from AliasManager - each pair of tables have own prefix (two of them depending on direction) - - used in joins to allow multiple joins to the same table. - :type table_prefix: str - :param fields: fields and related model fields to include - - if provided only those are included - :type fields: Optional[Union[Dict, Set]] - :param exclude_fields: fields and related model fields to exclude - excludes the fields even if they are provided in fields - :type exclude_fields: Optional[Union[Dict, Set]] - :return: dictionary with keys corresponding to model fields names - and values are database values - :rtype: Dict - """ - # databases does not keep aliases in Record for postgres, change to raw row - source = row._row if cls.db_backend_name() == "postgresql" else row - - selected_columns = cls.own_table_columns( - model=cls, - fields=fields or {}, - exclude_fields=exclude_fields or {}, - use_alias=False, - ) - - for column in cls.Meta.table.columns: - alias = cls.get_column_name_from_alias(column.name) - if alias not in item and alias in selected_columns: - prefixed_name = ( - f'{table_prefix + "_" if table_prefix else ""}{column.name}' - ) - item[alias] = source[prefixed_name] - - return item - async def upsert(self: T, **kwargs: Any) -> T: """ Performs either a save or an update depending on the presence of the pk. diff --git a/ormar/models/model_row.py b/ormar/models/model_row.py new file mode 100644 index 0000000..9d33a64 --- /dev/null +++ b/ormar/models/model_row.py @@ -0,0 +1,303 @@ +from typing import ( + Any, + Dict, + List, + Optional, + Set, + Type, + TypeVar, + Union, +) + +import sqlalchemy + +from ormar import ManyToManyField # noqa: I202 +from ormar.models import NewBaseModel +from ormar.models.helpers.models import group_related_list + +T = TypeVar("T", bound="ModelRow") + + +class ModelRow(NewBaseModel): + @classmethod + def from_row( # noqa CCR001 + cls: Type[T], + row: sqlalchemy.engine.ResultProxy, + select_related: List = None, + related_models: Any = None, + previous_model: Type[T] = None, + source_model: Type[T] = None, + related_name: str = None, + fields: Optional[Union[Dict, Set]] = None, + exclude_fields: Optional[Union[Dict, Set]] = None, + current_relation_str: str = None, + ) -> Optional[T]: + """ + Model method to convert raw sql row from database into ormar.Model instance. + Traverses nested models if they were specified in select_related for query. + + Called recurrently and returns model instance if it's present in the row. + Note that it's processing one row at a time, so if there are duplicates of + parent row that needs to be joined/combined + (like parent row in sql join with 2+ child rows) + instances populated in this method are later combined in the QuerySet. + Other method working directly on raw database results is in prefetch_query, + where rows are populated in a different way as they do not have + nested models in result. + + :param current_relation_str: name of the relation field + :type current_relation_str: str + :param source_model: model on which relation was defined + :type source_model: Type[Model] + :param row: raw result row from the database + :type row: sqlalchemy.engine.result.ResultProxy + :param select_related: list of names of related models fetched from database + :type select_related: List + :param related_models: list or dict of related models + :type related_models: Union[List, Dict] + :param previous_model: internal param for nested models to specify table_prefix + :type previous_model: Model class + :param related_name: internal parameter - name of current nested model + :type related_name: str + :param fields: fields and related model fields to include + if provided only those are included + :type fields: Optional[Union[Dict, Set]] + :param exclude_fields: fields and related model fields to exclude + excludes the fields even if they are provided in fields + :type exclude_fields: Optional[Union[Dict, Set]] + :return: returns model if model is populated from database + :rtype: Optional[Model] + """ + item: Dict[str, Any] = {} + select_related = select_related or [] + related_models = related_models or [] + table_prefix = "" + + if select_related: + source_model = cls + related_models = group_related_list(select_related) + + rel_name2 = related_name + + # TODO: refactor this into field classes? + if ( + previous_model + and related_name + and issubclass( + previous_model.Meta.model_fields[related_name], ManyToManyField + ) + ): + through_field = previous_model.Meta.model_fields[related_name] + if ( + through_field.self_reference + and related_name == through_field.self_reference_primary + ): + rel_name2 = through_field.default_source_field_name() # type: ignore + else: + rel_name2 = through_field.default_target_field_name() # type: ignore + previous_model = through_field.through # type: ignore + + if previous_model and rel_name2: + if current_relation_str and "__" in current_relation_str and source_model: + table_prefix = cls.Meta.alias_manager.resolve_relation_alias( + from_model=source_model, relation_name=current_relation_str + ) + if not table_prefix: + table_prefix = cls.Meta.alias_manager.resolve_relation_alias( + from_model=previous_model, relation_name=rel_name2 + ) + + item = cls.populate_nested_models_from_row( + item=item, + row=row, + related_models=related_models, + fields=fields, + exclude_fields=exclude_fields, + current_relation_str=current_relation_str, + source_model=source_model, + ) + item = cls.extract_prefixed_table_columns( + item=item, + row=row, + table_prefix=table_prefix, + fields=fields, + exclude_fields=exclude_fields, + ) + + instance: Optional[T] = None + if item.get(cls.Meta.pkname, None) is not None: + item["__excluded__"] = cls.get_names_to_exclude( + fields=fields, exclude_fields=exclude_fields + ) + instance = cls(**item) + instance.set_save_status(True) + return instance + + @classmethod + def populate_nested_models_from_row( # noqa: CFQ002 + cls, + item: dict, + row: sqlalchemy.engine.ResultProxy, + related_models: Any, + fields: Optional[Union[Dict, Set]] = None, + exclude_fields: Optional[Union[Dict, Set]] = None, + current_relation_str: str = None, + source_model: Type[T] = None, + ) -> dict: + """ + Traverses structure of related models and populates the nested models + from the database row. + Related models can be a list if only directly related models are to be + populated, converted to dict if related models also have their own related + models to be populated. + + Recurrently calls from_row method on nested instances and create nested + instances. In the end those instances are added to the final model dictionary. + + :param source_model: source model from which relation started + :type source_model: Type[Model] + :param current_relation_str: joined related parts into one string + :type current_relation_str: str + :param item: dictionary of already populated nested models, otherwise empty dict + :type item: Dict + :param row: raw result row from the database + :type row: sqlalchemy.engine.result.ResultProxy + :param related_models: list or dict of related models + :type related_models: Union[Dict, List] + :param fields: fields and related model fields to include - + if provided only those are included + :type fields: Optional[Union[Dict, Set]] + :param exclude_fields: fields and related model fields to exclude + excludes the fields even if they are provided in fields + :type exclude_fields: Optional[Union[Dict, Set]] + :return: dictionary with keys corresponding to model fields names + and values are database values + :rtype: Dict + """ + + for related in related_models: + relation_str = ( + "__".join([current_relation_str, related]) + if current_relation_str + else related + ) + field = cls.Meta.model_fields[related] + fields = cls.get_included(fields, related) + exclude_fields = cls.get_excluded(exclude_fields, related) + model_cls = field.to + + remainder = None + if isinstance(related_models, dict) and related_models[related]: + remainder = related_models[related] + child = model_cls.from_row( + row, + related_models=remainder, + previous_model=cls, + related_name=related, + fields=fields, + exclude_fields=exclude_fields, + current_relation_str=relation_str, + source_model=source_model, + ) + item[model_cls.get_column_name_from_alias(related)] = child + if issubclass(field, ManyToManyField) and child: + # TODO: way to figure out which side should be populated? + through_name = cls.Meta.model_fields[related].through.get_name() + # for now it's nested dict, should be instance? + through_child = cls.populate_through_instance( + row=row, + related=related, + through_name=through_name, + fields=fields, + exclude_fields=exclude_fields, + ) + item[through_name] = through_child + setattr(child, through_name, through_child) + child.set_save_status(True) + + return item + + @classmethod + def populate_through_instance( + cls, + row: sqlalchemy.engine.ResultProxy, + through_name: str, + related: str, + fields: Optional[Union[Dict, Set]] = None, + exclude_fields: Optional[Union[Dict, Set]] = None, + ) -> Dict: + # TODO: fix excludes and includes + fields = cls.get_included(fields, through_name) + # exclude_fields = cls.get_excluded(exclude_fields, through_name) + model_cls = cls.Meta.model_fields[through_name].to + exclude_fields = model_cls.extract_related_names() + table_prefix = cls.Meta.alias_manager.resolve_relation_alias( + from_model=cls, relation_name=related + ) + child = model_cls.extract_prefixed_table_columns( + item={}, + row=row, + table_prefix=table_prefix, + fields=fields, + exclude_fields=exclude_fields, + ) + return child + + @classmethod + def extract_prefixed_table_columns( # noqa CCR001 + cls, + item: dict, + row: sqlalchemy.engine.result.ResultProxy, + table_prefix: str, + fields: Optional[Union[Dict, Set]] = None, + exclude_fields: Optional[Union[Dict, Set]] = None, + ) -> dict: + """ + Extracts own fields from raw sql result, using a given prefix. + Prefix changes depending on the table's position in a join. + + If the table is a main table, there is no prefix. + All joined tables have prefixes to allow duplicate column names, + as well as duplicated joins to the same table from multiple different tables. + + Extracted fields populates the related dict later used to construct a Model. + + Used in Model.from_row and PrefetchQuery._populate_rows methods. + + :param item: dictionary of already populated nested models, otherwise empty dict + :type item: Dict + :param row: raw result row from the database + :type row: sqlalchemy.engine.result.ResultProxy + :param table_prefix: prefix of the table from AliasManager + each pair of tables have own prefix (two of them depending on direction) - + used in joins to allow multiple joins to the same table. + :type table_prefix: str + :param fields: fields and related model fields to include - + if provided only those are included + :type fields: Optional[Union[Dict, Set]] + :param exclude_fields: fields and related model fields to exclude + excludes the fields even if they are provided in fields + :type exclude_fields: Optional[Union[Dict, Set]] + :return: dictionary with keys corresponding to model fields names + and values are database values + :rtype: Dict + """ + # databases does not keep aliases in Record for postgres, change to raw row + source = row._row if cls.db_backend_name() == "postgresql" else row + + selected_columns = cls.own_table_columns( + model=cls, + fields=fields or {}, + exclude_fields=exclude_fields or {}, + use_alias=False, + ) + + for column in cls.Meta.table.columns: + alias = cls.get_column_name_from_alias(column.name) + if alias not in item and alias in selected_columns: + prefixed_name = ( + f'{table_prefix + "_" if table_prefix else ""}{column.name}' + ) + item[alias] = source[prefixed_name] + + return item diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index 3aa359e..6975753 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -172,7 +172,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass object.__setattr__(self, "__fields_set__", fields_set) # register the columns models after initialization - for related in self.extract_related_names(): + for related in self.extract_related_names().union(self.extract_through_names()): self.Meta.model_fields[related].expand_relationship( new_kwargs.get(related), self, to_register=True, ) @@ -267,6 +267,10 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass return object.__getattribute__( self, "_extract_related_model_instead_of_field" )(item) + if item in object.__getattribute__(self, "extract_through_names")(): + return object.__getattribute__( + self, "_extract_related_model_instead_of_field" + )(item) if item in object.__getattribute__(self, "Meta").property_fields: value = object.__getattribute__(self, item) return value() if callable(value) else value diff --git a/ormar/models/quick_access_views.py b/ormar/models/quick_access_views.py index c09b672..c960898 100644 --- a/ormar/models/quick_access_views.py +++ b/ormar/models/quick_access_views.py @@ -34,10 +34,12 @@ quick_access_set = { "_skip_ellipsis", "_update_and_follow", "_update_excluded_with_related_not_required", + "_verify_model_can_be_initialized", "copy", "delete", "dict", "extract_related_names", + "extract_through_names", "update_from_dict", "get_column_alias", "get_column_name_from_alias", diff --git a/ormar/queryset/join.py b/ormar/queryset/join.py index 0b44078..9ce306d 100644 --- a/ormar/queryset/join.py +++ b/ormar/queryset/join.py @@ -291,6 +291,8 @@ class SqlJoin: self.get_order_bys( to_table=to_table, pkname_alias=pkname_alias, ) + else: + self.select_through_model_fields() self_related_fields = self.next_model.own_table_columns( model=self.next_model, @@ -305,6 +307,24 @@ class SqlJoin: ) self.used_aliases.append(self.next_alias) + def select_through_model_fields(self) -> None: + # TODO: add docstring + next_alias = self.alias_manager.resolve_relation_alias( + from_model=self.target_field.owner, relation_name=self.relation_name + ) + # TODO: fix fields and exclusions + self_related_fields = self.target_field.through.own_table_columns( + model=self.target_field.through, + fields=None, + exclude_fields=self.target_field.through.extract_related_names(), + use_alias=True, + ) + self.columns.extend( + self.alias_manager.prefixed_columns( + next_alias, self.target_field.through.Meta.table, self_related_fields + ) + ) + def _replace_many_to_many_order_by_columns(self, part: str, new_part: str) -> None: """ Substitutes the name of the relation with actual model name in m2m order bys. diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 6adec49..77ec421 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -197,7 +197,7 @@ class QuerySet: limit_raw_sql=self.limit_sql_raw, ) exp = qry.build_select_expression() - # print("\n", exp.compile(compile_kwargs={"literal_binds": True})) + print("\n", exp.compile(compile_kwargs={"literal_binds": True})) return exp def filter(self, _exclude: bool = False, **kwargs: Any) -> "QuerySet": # noqa: A003 diff --git a/ormar/relations/alias_manager.py b/ormar/relations/alias_manager.py index cd3dc8b..c99dfdb 100644 --- a/ormar/relations/alias_manager.py +++ b/ormar/relations/alias_manager.py @@ -1,13 +1,14 @@ import string import uuid from random import choices -from typing import Any, Dict, List, TYPE_CHECKING, Type +from typing import Any, Dict, List, TYPE_CHECKING, Type, Union import sqlalchemy from sqlalchemy import text if TYPE_CHECKING: # pragma: no cover from ormar import Model + from ormar.models import ModelRow def get_table_alias() -> str: @@ -133,7 +134,7 @@ class AliasManager: return alias def resolve_relation_alias( - self, from_model: Type["Model"], relation_name: str + self, from_model: Union[Type["Model"], Type["ModelRow"]], relation_name: str ) -> str: """ Given model and relation name returns the alias for this relation. diff --git a/ormar/relations/querysetproxy.py b/ormar/relations/querysetproxy.py index 360e863..c23dcb7 100644 --- a/ormar/relations/querysetproxy.py +++ b/ormar/relations/querysetproxy.py @@ -44,6 +44,11 @@ class QuerysetProxy(ormar.QuerySetProtocol): ].get_related_name() self.related_field = self.relation.to.Meta.model_fields[self.related_field_name] self.owner_pk_value = self._owner.pk + self.through_model_name = ( + self.related_field.through.get_name() + if self.type_ == ormar.RelationType.MULTIPLE + else None + ) @property def queryset(self) -> "QuerySet": @@ -99,17 +104,20 @@ class QuerysetProxy(ormar.QuerySetProtocol): for item in self.relation.related_models[:]: self.relation.remove(item) - async def create_through_instance(self, child: "T") -> None: + async def create_through_instance(self, child: "T", **kwargs: Any) -> None: """ Crete a through model instance in the database for m2m relations. + :param kwargs: dict of additional keyword arguments for through instance + :type kwargs: Any :param child: child model instance :type child: Model """ model_cls = self.relation.through owner_column = self.related_field.default_target_field_name() # type: ignore child_column = self.related_field.default_source_field_name() # type: ignore - kwargs = {owner_column: self._owner.pk, child_column: child.pk} + rel_kwargs = {owner_column: self._owner.pk, child_column: child.pk} + final_kwargs = {**rel_kwargs, **kwargs} if child.pk is None: raise ModelPersistenceError( f"You cannot save {child.get_name()} " @@ -117,7 +125,7 @@ class QuerysetProxy(ormar.QuerySetProtocol): f"Save the child model first." ) expr = model_cls.Meta.table.insert() - expr = expr.values(**kwargs) + expr = expr.values(**final_kwargs) # print("\n", expr.compile(compile_kwargs={"literal_binds": True})) await model_cls.Meta.database.execute(expr) @@ -270,12 +278,13 @@ class QuerysetProxy(ormar.QuerySetProtocol): :return: created model :rtype: Model """ + through_kwargs = kwargs.pop(self.through_model_name, {}) if self.type_ == ormar.RelationType.REVERSE: kwargs[self.related_field.name] = self._owner created = await self.queryset.create(**kwargs) self._register_related(created) if self.type_ == ormar.RelationType.MULTIPLE: - await self.create_through_instance(created) + await self.create_through_instance(created, **through_kwargs) return created async def get_or_create(self, **kwargs: Any) -> "Model": diff --git a/ormar/relations/relation.py b/ormar/relations/relation.py index cf191e0..0ae2f59 100644 --- a/ormar/relations/relation.py +++ b/ormar/relations/relation.py @@ -26,6 +26,7 @@ class RelationType(Enum): PRIMARY = 1 REVERSE = 2 MULTIPLE = 3 + THROUGH = 4 class Relation: @@ -128,7 +129,7 @@ class Relation: :type child: Model """ relation_name = self.field_name - if self._type == RelationType.PRIMARY: + if self._type in (RelationType.PRIMARY, RelationType.THROUGH): self.related_models = child self._owner.__dict__[relation_name] = child else: diff --git a/ormar/relations/relation_manager.py b/ormar/relations/relation_manager.py index 511dd7b..addfcf1 100644 --- a/ormar/relations/relation_manager.py +++ b/ormar/relations/relation_manager.py @@ -1,7 +1,7 @@ from typing import Dict, List, Optional, Sequence, TYPE_CHECKING, Type, TypeVar, Union from weakref import proxy -from ormar.fields import BaseField +from ormar.fields import BaseField, ThroughField from ormar.fields.foreign_key import ForeignKeyField from ormar.fields.many_to_many import ManyToManyField from ormar.relations.relation import Relation, RelationType @@ -42,6 +42,8 @@ class RelationsManager: """ if issubclass(field, ManyToManyField): return RelationType.MULTIPLE + if issubclass(field, ThroughField): + return RelationType.THROUGH return RelationType.PRIMARY if not field.virtual else RelationType.REVERSE def _add_relation(self, field: Type[BaseField]) -> None: diff --git a/ormar/relations/relation_proxy.py b/ormar/relations/relation_proxy.py index 518fc71..87bb05f 100644 --- a/ormar/relations/relation_proxy.py +++ b/ormar/relations/relation_proxy.py @@ -163,19 +163,21 @@ class RelationProxy(list): else: await item.delete() - async def add(self, item: "Model") -> None: + async def add(self, item: "Model", **kwargs: Any) -> None: """ Adds child model to relation. For ManyToMany relations through instance is automatically created. + :param kwargs: dict of additional keyword arguments for through instance + :type kwargs: Any :param item: child to add to relation :type item: Model """ relation_name = self.related_field_name self._check_if_model_saved() if self.type_ == ormar.RelationType.MULTIPLE: - await self.queryset_proxy.create_through_instance(item) + await self.queryset_proxy.create_through_instance(item, **kwargs) setattr(item, relation_name, self._owner) else: setattr(item, relation_name, self._owner) diff --git a/tests/test_m2m_through_fields.py b/tests/test_m2m_through_fields.py index 96c7f89..229eac4 100644 --- a/tests/test_m2m_through_fields.py +++ b/tests/test_m2m_through_fields.py @@ -1,6 +1,7 @@ import databases import pytest import sqlalchemy +from pydantic.typing import ForwardRef import ormar from tests.settings import DATABASE_URL @@ -39,28 +40,107 @@ class Post(ormar.Model): categories = ormar.ManyToMany(Category, through=PostCategory) -# -# @pytest.fixture(autouse=True, scope="module") -# async def create_test_database(): -# engine = sqlalchemy.create_engine(DATABASE_URL) -# metadata.create_all(engine) -# yield -# metadata.drop_all(engine) -# -# -# @pytest.mark.asyncio -# async def test_setting_fields_on_through_model(): -# async with database: -# # TODO: check/ modify following -# # loading the data into model instance of though model? -# # <- attach to other side? both sides? access by through, or add to fields? -# # creating while adding to relation (kwargs in add?) -# # creating in query (dividing kwargs between final and through) -# # updating in query -# # sorting in filter (special __through__ notation?) -# # ordering by in order_by -# # accessing from instance (both sides?) -# # modifying from instance (both sides?) -# # including/excluding in fields? -# # allowing to change fk fields names in through model? -# pass +@pytest.fixture(autouse=True, scope="module") +def create_test_database(): + engine = sqlalchemy.create_engine(DATABASE_URL) + metadata.drop_all(engine) + metadata.create_all(engine) + yield + metadata.drop_all(engine) + + +class PostCategory2(ormar.Model): + class Meta(BaseMeta): + tablename = "posts_x_categories2" + + id: int = ormar.Integer(primary_key=True) + sort_order: int = ormar.Integer(nullable=True) + + +@pytest.mark.asyncio +async def test_forward_ref_is_updated(): + async with database: + + class Post2(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + title: str = ormar.String(max_length=200) + categories = ormar.ManyToMany(Category, through=ForwardRef("PostCategory2")) + + assert Post2.Meta.requires_ref_update + Post2.update_forward_refs() + + assert Post2.Meta.model_fields["postcategory2"].to == PostCategory2 + + +@pytest.mark.asyncio +async def test_setting_fields_on_through_model(): + async with database: + post = await Post(title="Test post").save() + category = await Category(name="Test category").save() + await post.categories.add(category) + + assert hasattr(post.categories[0], "postcategory") + assert post.categories[0].postcategory is None + + +@pytest.mark.asyncio +async def test_setting_additional_fields_on_through_model_in_add(): + async with database: + post = await Post(title="Test post").save() + category = await Category(name="Test category").save() + await post.categories.add(category, sort_order=1) + postcat = await PostCategory.objects.get() + assert postcat.sort_order == 1 + + +@pytest.mark.asyncio +async def test_setting_additional_fields_on_through_model_in_create(): + async with database: + post = await Post(title="Test post").save() + await post.categories.create( + name="Test category2", postcategory={"sort_order": 2} + ) + postcat = await PostCategory.objects.get() + assert postcat.sort_order == 2 + + +@pytest.mark.asyncio +async def test_getting_additional_fields_from_queryset(): + async with database: + post = await Post(title="Test post").save() + await post.categories.create( + name="Test category1", postcategory={"sort_order": 1} + ) + await post.categories.create( + name="Test category2", postcategory={"sort_order": 2} + ) + + await post.categories.all() + assert post.categories[0].postcategory.sort_order == 1 + assert post.categories[1].postcategory.sort_order == 2 + + post = await Post.objects.select_related("categories").get( + categories__name="Test category2" + ) + assert post.categories[0].postcategory.sort_order == 2 + + +# TODO: check/ modify following + +# add to fields with class lower name (V) +# forward refs update (V) +# creating while adding to relation (kwargs in add) (V) +# creating in queryset proxy (dict with through name and kwargs) (V) +# loading the data into model instance of though model (V) <- fix fields ane exclude +# accessing from instance (V) <- no both sides only nested one is relevant, fix one side + +# updating in query +# sorting in filter (special __through__ notation?) +# ordering by in order_by +# modifying from instance (both sides?) +# including/excluding in fields? +# allowing to change fk fields names in through model? +# make through optional? auto-generated for cases other fields are missing? From 5d40fb6bff098670795dc271fd40263b311d924e Mon Sep 17 00:00:00 2001 From: collerek Date: Wed, 17 Feb 2021 13:51:38 +0100 Subject: [PATCH 02/14] WIP - further work and refactoring --- ormar/fields/foreign_key.py | 21 +++++++++ ormar/fields/many_to_many.py | 23 ++++++++++ ormar/models/mixins/excludable_mixin.py | 4 +- ormar/models/model_row.py | 57 ++++++++----------------- ormar/queryset/queryset.py | 1 + ormar/relations/alias_manager.py | 33 ++++++++++++++ tests/test_m2m_through_fields.py | 18 ++++---- 7 files changed, 107 insertions(+), 50 deletions(-) diff --git a/ormar/fields/foreign_key.py b/ormar/fields/foreign_key.py index 30602ae..16bb23d 100644 --- a/ormar/fields/foreign_key.py +++ b/ormar/fields/foreign_key.py @@ -450,3 +450,24 @@ class ForeignKeyField(BaseField): value.__class__.__name__, cls._construct_model_from_pk )(value, child, to_register) return model + + @classmethod + def get_relation_name(cls) -> str: + """ + Returns name of the relation, which can be a own name or through model + names for m2m models + + :return: result of the check + :rtype: bool + """ + return cls.name + + @classmethod + def get_source_model(cls) -> Type["Model"]: + """ + Returns model from which the relation comes -> either owner or through model + + :return: source model + :rtype: Type["Model"] + """ + return cls.owner diff --git a/ormar/fields/many_to_many.py b/ormar/fields/many_to_many.py index 2228121..ad7e6d9 100644 --- a/ormar/fields/many_to_many.py +++ b/ormar/fields/many_to_many.py @@ -187,3 +187,26 @@ class ManyToManyField(ForeignKeyField, ormar.QuerySetProtocol, ormar.RelationPro globalns, localns or None, ) + + @classmethod + def get_relation_name(cls) -> str: + """ + Returns name of the relation, which can be a own name or through model + names for m2m models + + :return: result of the check + :rtype: bool + """ + if cls.self_reference and cls.name == cls.self_reference_primary: + return cls.default_source_field_name() + return cls.default_target_field_name() + + @classmethod + def get_source_model(cls) -> Type["Model"]: + """ + Returns model from which the relation comes -> either owner or through model + + :return: source model + :rtype: Type["Model"] + """ + return cls.through diff --git a/ormar/models/mixins/excludable_mixin.py b/ormar/models/mixins/excludable_mixin.py index e474bd4..4b096d9 100644 --- a/ormar/models/mixins/excludable_mixin.py +++ b/ormar/models/mixins/excludable_mixin.py @@ -131,7 +131,9 @@ class ExcludableMixin(RelationMixin): @staticmethod def _populate_pk_column( - model: Type["Model"], columns: List[str], use_alias: bool = False, + model: Union[Type["Model"], Type["ModelRow"]], + columns: List[str], + use_alias: bool = False, ) -> List[str]: """ Adds primary key column/alias (depends on use_alias flag) to list of diff --git a/ormar/models/model_row.py b/ormar/models/model_row.py index 9d33a64..f184bb8 100644 --- a/ormar/models/model_row.py +++ b/ormar/models/model_row.py @@ -4,6 +4,7 @@ from typing import ( List, Optional, Set, + TYPE_CHECKING, Type, TypeVar, Union, @@ -17,20 +18,22 @@ from ormar.models.helpers.models import group_related_list T = TypeVar("T", bound="ModelRow") +if TYPE_CHECKING: + from ormar.fields import ForeignKeyField + class ModelRow(NewBaseModel): @classmethod - def from_row( # noqa CCR001 + def from_row( cls: Type[T], row: sqlalchemy.engine.ResultProxy, + source_model: Type[T], select_related: List = None, related_models: Any = None, - previous_model: Type[T] = None, - source_model: Type[T] = None, - related_name: str = None, + related_field: Type["ForeignKeyField"] = None, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None, - current_relation_str: str = None, + current_relation_str: str = "", ) -> Optional[T]: """ Model method to convert raw sql row from database into ormar.Model instance. @@ -55,10 +58,8 @@ class ModelRow(NewBaseModel): :type select_related: List :param related_models: list or dict of related models :type related_models: Union[List, Dict] - :param previous_model: internal param for nested models to specify table_prefix - :type previous_model: Model class - :param related_name: internal parameter - name of current nested model - :type related_name: str + :param related_field: field with relation declaration + :type related_field: Type[ForeignKeyField] :param fields: fields and related model fields to include if provided only those are included :type fields: Optional[Union[Dict, Set]] @@ -77,35 +78,12 @@ class ModelRow(NewBaseModel): source_model = cls related_models = group_related_list(select_related) - rel_name2 = related_name - - # TODO: refactor this into field classes? - if ( - previous_model - and related_name - and issubclass( - previous_model.Meta.model_fields[related_name], ManyToManyField + if related_field: + table_prefix = cls.Meta.alias_manager.resolve_relation_alias_after_complex( + source_model=source_model, + relation_str=current_relation_str, + relation_field=related_field, ) - ): - through_field = previous_model.Meta.model_fields[related_name] - if ( - through_field.self_reference - and related_name == through_field.self_reference_primary - ): - rel_name2 = through_field.default_source_field_name() # type: ignore - else: - rel_name2 = through_field.default_target_field_name() # type: ignore - previous_model = through_field.through # type: ignore - - if previous_model and rel_name2: - if current_relation_str and "__" in current_relation_str and source_model: - table_prefix = cls.Meta.alias_manager.resolve_relation_alias( - from_model=source_model, relation_name=current_relation_str - ) - if not table_prefix: - table_prefix = cls.Meta.alias_manager.resolve_relation_alias( - from_model=previous_model, relation_name=rel_name2 - ) item = cls.populate_nested_models_from_row( item=item, @@ -138,11 +116,11 @@ class ModelRow(NewBaseModel): cls, item: dict, row: sqlalchemy.engine.ResultProxy, + source_model: Type[T], related_models: Any, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None, current_relation_str: str = None, - source_model: Type[T] = None, ) -> dict: """ Traverses structure of related models and populates the nested models @@ -192,8 +170,7 @@ class ModelRow(NewBaseModel): child = model_cls.from_row( row, related_models=remainder, - previous_model=cls, - related_name=related, + related_field=field, fields=fields, exclude_fields=exclude_fields, current_relation_str=relation_str, diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 77ec421..051e695 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -128,6 +128,7 @@ class QuerySet: select_related=self._select_related, fields=self._columns, exclude_fields=self._exclude_columns, + source_model=self.model, ) for row in rows ] diff --git a/ormar/relations/alias_manager.py b/ormar/relations/alias_manager.py index c99dfdb..815a4dc 100644 --- a/ormar/relations/alias_manager.py +++ b/ormar/relations/alias_manager.py @@ -9,6 +9,7 @@ from sqlalchemy import text if TYPE_CHECKING: # pragma: no cover from ormar import Model from ormar.models import ModelRow + from ormar.fields import ForeignKeyField def get_table_alias() -> str: @@ -148,3 +149,35 @@ class AliasManager: """ alias = self._aliases_new.get(f"{from_model.get_name()}_{relation_name}", "") return alias + + def resolve_relation_alias_after_complex( + self, + source_model: Union[Type["Model"], Type["ModelRow"]], + relation_str: str, + relation_field: Type["ForeignKeyField"], + ) -> str: + """ + Given source model and relation string returns the alias for this complex + relation if it exists, otherwise fallback to normal relation from a relation + field definition. + + :param relation_field: field with direct relation definition + :type relation_field: Type["ForeignKeyField"] + :param source_model: model with query starts + :type source_model: source Model + :param relation_str: string with relation joins defined + :type relation_str: str + :return: alias of the relation + :rtype: str + """ + alias = "" + if relation_str and "__" in relation_str: + alias = self.resolve_relation_alias( + from_model=source_model, relation_name=relation_str + ) + if not alias: + alias = self.resolve_relation_alias( + from_model=relation_field.get_source_model(), + relation_name=relation_field.get_relation_name(), + ) + return alias diff --git a/tests/test_m2m_through_fields.py b/tests/test_m2m_through_fields.py index 229eac4..cb123fe 100644 --- a/tests/test_m2m_through_fields.py +++ b/tests/test_m2m_through_fields.py @@ -57,18 +57,18 @@ class PostCategory2(ormar.Model): sort_order: int = ormar.Integer(nullable=True) +class Post2(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + title: str = ormar.String(max_length=200) + categories = ormar.ManyToMany(Category, through=ForwardRef("PostCategory2")) + + @pytest.mark.asyncio async def test_forward_ref_is_updated(): async with database: - - class Post2(ormar.Model): - class Meta(BaseMeta): - pass - - id: int = ormar.Integer(primary_key=True) - title: str = ormar.String(max_length=200) - categories = ormar.ManyToMany(Category, through=ForwardRef("PostCategory2")) - assert Post2.Meta.requires_ref_update Post2.update_forward_refs() From e697235172b95ff8fa3f7426dcfb94b32bbb13cb Mon Sep 17 00:00:00 2001 From: collerek Date: Sun, 21 Feb 2021 17:46:06 +0100 Subject: [PATCH 03/14] intorduce relation flags on basefield and simplify imports --- ormar/fields/base.py | 8 ++- ormar/fields/foreign_key.py | 3 +- ormar/fields/many_to_many.py | 2 + ormar/fields/through_field.py | 11 ++-- ormar/models/__init__.py | 4 +- ormar/models/helpers/models.py | 2 +- ormar/models/helpers/pydantic.py | 7 ++- .../helpers/related_names_validation.py | 2 +- ormar/models/helpers/relations.py | 24 ++++---- ormar/models/helpers/sqlalchemy.py | 6 +- ormar/models/metaclass.py | 7 ++- ormar/models/mixins/prefetch_mixin.py | 12 ++-- ormar/models/mixins/relation_mixin.py | 12 +--- ormar/models/model.py | 6 +- ormar/models/model_row.py | 19 +++--- ormar/models/newbasemodel.py | 18 +++--- ormar/queryset/__init__.py | 4 +- ormar/queryset/join.py | 7 +-- ormar/queryset/prefetch_query.py | 11 ++-- ormar/queryset/queryset.py | 55 +++++++++++------- ormar/queryset/utils.py | 3 +- ormar/relations/querysetproxy.py | 25 ++++---- ormar/relations/relation.py | 18 +++--- ormar/relations/relation_manager.py | 25 ++++---- ormar/relations/relation_proxy.py | 4 +- test.db-journal | Bin 0 -> 4616 bytes tests/test_m2m_through_fields.py | 17 ++++-- 27 files changed, 163 insertions(+), 149 deletions(-) create mode 100644 test.db-journal diff --git a/ormar/fields/base.py b/ormar/fields/base.py index 08308d8..95645c0 100644 --- a/ormar/fields/base.py +++ b/ormar/fields/base.py @@ -36,9 +36,13 @@ class BaseField(FieldInfo): index: bool unique: bool pydantic_only: bool - virtual: bool = False choices: typing.Sequence + virtual: bool = False # ManyToManyFields and reverse ForeignKeyFields + is_multi: bool = False # ManyToManyField + is_relation: bool = False # ForeignKeyField + subclasses + is_through: bool = False # ThroughFields + owner: Type["Model"] to: Type["Model"] through: Type["Model"] @@ -62,7 +66,7 @@ class BaseField(FieldInfo): :return: result of the check :rtype: bool """ - return not issubclass(cls, ormar.fields.ManyToManyField) and not cls.virtual + return not cls.is_multi and not cls.virtual @classmethod def get_alias(cls) -> str: diff --git a/ormar/fields/foreign_key.py b/ormar/fields/foreign_key.py index f6bccfa..8dda14f 100644 --- a/ormar/fields/foreign_key.py +++ b/ormar/fields/foreign_key.py @@ -48,7 +48,7 @@ def create_dummy_instance(fk: Type["Model"], pk: Any = None) -> "Model": **{ k: create_dummy_instance(v.to) for k, v in fk.Meta.model_fields.items() - if isinstance(v, ForeignKeyField) and not v.nullable and not v.virtual + if v.is_relation and not v.nullable and not v.virtual }, } return fk(**init_dict) @@ -217,6 +217,7 @@ def ForeignKey( # noqa CFQ002 ondelete=ondelete, owner=owner, self_reference=self_reference, + is_relation=True, ) return type("ForeignKey", (ForeignKeyField, BaseField), namespace) diff --git a/ormar/fields/many_to_many.py b/ormar/fields/many_to_many.py index ad7e6d9..2b2b300 100644 --- a/ormar/fields/many_to_many.py +++ b/ormar/fields/many_to_many.py @@ -103,6 +103,8 @@ def ManyToMany( server_default=None, owner=owner, self_reference=self_reference, + is_relation=True, + is_multi=True, ) return type("ManyToMany", (ManyToManyField, BaseField), namespace) diff --git a/ormar/fields/through_field.py b/ormar/fields/through_field.py index 99361c7..e5e4a24 100644 --- a/ormar/fields/through_field.py +++ b/ormar/fields/through_field.py @@ -15,12 +15,7 @@ if TYPE_CHECKING: # pragma no cover def Through( # noqa CFQ002 - to: "ToType", - *, - name: str = None, - related_name: str = None, - virtual: bool = True, - **kwargs: Any, + to: "ToType", *, name: str = None, related_name: str = None, **kwargs: Any, ) -> Any: # TODO: clean docstring """ @@ -52,7 +47,7 @@ def Through( # noqa CFQ002 alias=name, name=kwargs.pop("real_name", None), related_name=related_name, - virtual=virtual, + virtual=True, owner=owner, nullable=False, unique=False, @@ -62,6 +57,8 @@ def Through( # noqa CFQ002 pydantic_only=False, default=None, server_default=None, + is_relation=True, + is_through=True, ) return type("Through", (ThroughField, BaseField), namespace) diff --git a/ormar/models/__init__.py b/ormar/models/__init__.py index 9990c4a..58372b7 100644 --- a/ormar/models/__init__.py +++ b/ormar/models/__init__.py @@ -6,6 +6,6 @@ ass well as vast number of helper functions for pydantic, sqlalchemy and relatio from ormar.models.newbasemodel import NewBaseModel # noqa I100 from ormar.models.model_row import ModelRow # noqa I100 -from ormar.models.model import Model # noqa I100 +from ormar.models.model import Model, T # noqa I100 -__all__ = ["NewBaseModel", "Model", "ModelRow"] +__all__ = ["T", "NewBaseModel", "Model", "ModelRow"] diff --git a/ormar/models/helpers/models.py b/ormar/models/helpers/models.py index 449a920..6d67e91 100644 --- a/ormar/models/helpers/models.py +++ b/ormar/models/helpers/models.py @@ -21,7 +21,7 @@ def is_field_an_forward_ref(field: Type["BaseField"]) -> bool: :return: result of the check :rtype: bool """ - return issubclass(field, ormar.ForeignKeyField) and ( + return field.is_relation and ( field.to.__class__ == ForwardRef or field.through.__class__ == ForwardRef ) diff --git a/ormar/models/helpers/pydantic.py b/ormar/models/helpers/pydantic.py index 6a8f8a5..5797598 100644 --- a/ormar/models/helpers/pydantic.py +++ b/ormar/models/helpers/pydantic.py @@ -6,14 +6,15 @@ from pydantic.fields import ModelField from pydantic.utils import lenient_issubclass import ormar # noqa: I100, I202 -from ormar.fields import BaseField, ManyToManyField +from ormar.fields import BaseField if TYPE_CHECKING: # pragma no cover from ormar import Model + from ormar.fields import ManyToManyField def create_pydantic_field( - field_name: str, model: Type["Model"], model_field: Type[ManyToManyField] + field_name: str, model: Type["Model"], model_field: Type["ManyToManyField"] ) -> None: """ Registers pydantic field on through model that leads to passed model @@ -59,7 +60,7 @@ def get_pydantic_field(field_name: str, model: Type["Model"]) -> "ModelField": def populate_default_pydantic_field_value( - ormar_field: Type[BaseField], field_name: str, attrs: dict + ormar_field: Type["BaseField"], field_name: str, attrs: dict ) -> dict: """ Grabs current value of the ormar Field in class namespace diff --git a/ormar/models/helpers/related_names_validation.py b/ormar/models/helpers/related_names_validation.py index 8bc32c1..56497b2 100644 --- a/ormar/models/helpers/related_names_validation.py +++ b/ormar/models/helpers/related_names_validation.py @@ -25,7 +25,7 @@ def validate_related_names_in_relations( # noqa CCR001 """ already_registered: Dict[str, List[Optional[str]]] = dict() for field in model_fields.values(): - if issubclass(field, ormar.ForeignKeyField): + if field.is_relation: to_name = ( field.to.get_name() if not field.to.__class__ == ForwardRef diff --git a/ormar/models/helpers/relations.py b/ormar/models/helpers/relations.py index af9ee61..48b35be 100644 --- a/ormar/models/helpers/relations.py +++ b/ormar/models/helpers/relations.py @@ -1,14 +1,14 @@ -from typing import TYPE_CHECKING, Type +from typing import TYPE_CHECKING, Type, cast import ormar from ormar import ForeignKey, ManyToMany -from ormar.fields import ManyToManyField, Through, ThroughField -from ormar.fields.foreign_key import ForeignKeyField +from ormar.fields import Through from ormar.models.helpers.sqlalchemy import adjust_through_many_to_many_model from ormar.relations import AliasManager if TYPE_CHECKING: # pragma no cover from ormar import Model + from ormar.fields import ManyToManyField, ForeignKeyField alias_manager = AliasManager() @@ -32,7 +32,7 @@ def register_relation_on_build(field: Type["ForeignKeyField"]) -> None: ) -def register_many_to_many_relation_on_build(field: Type[ManyToManyField]) -> None: +def register_many_to_many_relation_on_build(field: Type["ManyToManyField"]) -> None: """ Registers connection between through model and both sides of the m2m relation. Registration include also reverse relation side to be able to join both sides. @@ -83,10 +83,8 @@ def expand_reverse_relationships(model: Type["Model"]) -> None: """ model_fields = list(model.Meta.model_fields.values()) for model_field in model_fields: - if ( - issubclass(model_field, ForeignKeyField) - and not model_field.has_unresolved_forward_refs() - ): + if model_field.is_relation and not model_field.has_unresolved_forward_refs(): + model_field = cast(Type["ForeignKeyField"], model_field) expand_reverse_relationship(model_field=model_field) @@ -102,7 +100,7 @@ def register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None: :type model_field: relation Field """ related_name = model_field.get_related_name() - if issubclass(model_field, ManyToManyField): + if model_field.is_multi: model_field.to.Meta.model_fields[related_name] = ManyToMany( model_field.owner, through=model_field.through, @@ -114,6 +112,7 @@ def register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None: self_reference_primary=model_field.self_reference_primary, ) # register foreign keys on through model + model_field = cast(Type["ManyToManyField"], model_field) register_through_shortcut_fields(model_field=model_field) adjust_through_many_to_many_model(model_field=model_field) else: @@ -155,7 +154,7 @@ def register_through_shortcut_fields(model_field: Type["ManyToManyField"]) -> No ) -def register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None: +def register_relation_in_alias_manager(field: Type["ForeignKeyField"]) -> None: """ Registers the relation (and reverse relation) in alias manager. The m2m relations require registration of through model between @@ -168,11 +167,12 @@ def register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None: :param field: relation field :type field: ForeignKey or ManyToManyField class """ - if issubclass(field, ManyToManyField): + if field.is_multi: if field.has_unresolved_forward_refs(): return + field = cast(Type["ManyToManyField"], field) register_many_to_many_relation_on_build(field=field) - elif issubclass(field, ForeignKeyField) and not issubclass(field, ThroughField): + elif field.is_relation and not field.is_through: if field.has_unresolved_forward_refs(): return register_relation_on_build(field=field) diff --git a/ormar/models/helpers/sqlalchemy.py b/ormar/models/helpers/sqlalchemy.py index a17e786..b641969 100644 --- a/ormar/models/helpers/sqlalchemy.py +++ b/ormar/models/helpers/sqlalchemy.py @@ -156,11 +156,7 @@ def sqlalchemy_columns_from_model_fields( field.owner = new_model if field.primary_key: pkname = check_pk_column_validity(field_name, field, pkname) - if ( - not field.pydantic_only - and not field.virtual - and not issubclass(field, ormar.ManyToManyField) - ): + if not field.pydantic_only and not field.virtual and not field.is_multi: columns.append(field.get_column(field.get_alias())) return pkname, columns diff --git a/ormar/models/metaclass.py b/ormar/models/metaclass.py index 4134fd5..304fd19 100644 --- a/ormar/models/metaclass.py +++ b/ormar/models/metaclass.py @@ -262,7 +262,7 @@ def copy_and_replace_m2m_through_model( new_meta.model_fields = { name: field for name, field in new_meta.model_fields.items() - if not issubclass(field, ForeignKeyField) + if not field.is_relation } _, columns = sqlalchemy_columns_from_model_fields( new_meta.model_fields, copy_through @@ -329,7 +329,8 @@ def copy_data_from_parent_model( # noqa: CCR001 else attrs.get("__name__", "").lower() + "s" ) for field_name, field in base_class.Meta.model_fields.items(): - if issubclass(field, ManyToManyField): + if field.is_multi: + field = cast(Type["ManyToManyField"], field) copy_and_replace_m2m_through_model( field=field, field_name=field_name, @@ -339,7 +340,7 @@ def copy_data_from_parent_model( # noqa: CCR001 meta=meta, ) - elif issubclass(field, ForeignKeyField) and field.related_name: + elif field.is_relation and field.related_name: copy_field = type( # type: ignore field.__name__, (ForeignKeyField, BaseField), dict(field.__dict__) ) diff --git a/ormar/models/mixins/prefetch_mixin.py b/ormar/models/mixins/prefetch_mixin.py index 273dd01..d8ee350 100644 --- a/ormar/models/mixins/prefetch_mixin.py +++ b/ormar/models/mixins/prefetch_mixin.py @@ -1,9 +1,10 @@ -from typing import Callable, Dict, List, TYPE_CHECKING, Tuple, Type +from typing import Callable, Dict, List, TYPE_CHECKING, Tuple, Type, cast -import ormar -from ormar.fields.foreign_key import ForeignKeyField from ormar.models.mixins.relation_mixin import RelationMixin +if TYPE_CHECKING: + from ormar.fields import ForeignKeyField, ManyToManyField + class PrefetchQueryMixin(RelationMixin): """ @@ -39,7 +40,8 @@ class PrefetchQueryMixin(RelationMixin): if reverse: field_name = parent_model.Meta.model_fields[related].get_related_name() field = target_model.Meta.model_fields[field_name] - if issubclass(field, ormar.fields.ManyToManyField): + if field.is_multi: + field = cast(Type["ManyToManyField"], field) field_name = field.default_target_field_name() sub_field = field.through.Meta.model_fields[field_name] return field.through, sub_field.get_alias() @@ -87,7 +89,7 @@ class PrefetchQueryMixin(RelationMixin): :return: name of the field :rtype: str """ - if issubclass(target_field, ormar.fields.ManyToManyField): + if target_field.is_multi: return cls.get_name() if target_field.virtual: return target_field.get_related_name() diff --git a/ormar/models/mixins/relation_mixin.py b/ormar/models/mixins/relation_mixin.py index 435fcc8..aebaa20 100644 --- a/ormar/models/mixins/relation_mixin.py +++ b/ormar/models/mixins/relation_mixin.py @@ -1,10 +1,6 @@ import inspect from typing import List, Optional, Set, TYPE_CHECKING -from ormar import ManyToManyField -from ormar.fields import ThroughField -from ormar.fields.foreign_key import ForeignKeyField - class RelationMixin: """ @@ -62,7 +58,7 @@ class RelationMixin: related_fields = set() for name in cls.extract_related_names(): field = cls.Meta.model_fields[name] - if issubclass(field, ManyToManyField): + if field.is_multi: related_fields.add(field.through.get_name(lower=True)) return related_fields @@ -80,11 +76,7 @@ class RelationMixin: related_names = set() for name, field in cls.Meta.model_fields.items(): - if ( - inspect.isclass(field) - and issubclass(field, ForeignKeyField) - and not issubclass(field, ThroughField) - ): + if inspect.isclass(field) and field.is_relation and not field.is_through: related_names.add(name) cls._related_names = related_names diff --git a/ormar/models/model.py b/ormar/models/model.py index 9286da9..c20368d 100644 --- a/ormar/models/model.py +++ b/ormar/models/model.py @@ -8,7 +8,6 @@ from typing import ( import ormar.queryset # noqa I100 from ormar.exceptions import ModelPersistenceError, NoMatch -from ormar.fields.many_to_many import ManyToManyField from ormar.models import NewBaseModel # noqa I100 from ormar.models.metaclass import ModelMeta from ormar.models.model_row import ModelRow @@ -139,8 +138,9 @@ class Model(ModelRow): visited.add(self.__class__) for related in self.extract_related_names(): - if self.Meta.model_fields[related].virtual or issubclass( - self.Meta.model_fields[related], ManyToManyField + if ( + self.Meta.model_fields[related].virtual + or self.Meta.model_fields[related].is_multi ): for rel in getattr(self, related): update_count, visited = await self._update_and_follow( diff --git a/ormar/models/model_row.py b/ormar/models/model_row.py index f184bb8..f0a4a31 100644 --- a/ormar/models/model_row.py +++ b/ormar/models/model_row.py @@ -8,24 +8,26 @@ from typing import ( Type, TypeVar, Union, + cast, ) import sqlalchemy -from ormar import ManyToManyField # noqa: I202 -from ormar.models import NewBaseModel +from ormar.models import NewBaseModel # noqa: I202 from ormar.models.helpers.models import group_related_list -T = TypeVar("T", bound="ModelRow") if TYPE_CHECKING: from ormar.fields import ForeignKeyField + from ormar.models import T +else: + T = TypeVar("T", bound="ModelRow") class ModelRow(NewBaseModel): @classmethod - def from_row( - cls: Type[T], + def from_row( # noqa: CFQ002 + cls: Type["ModelRow"], row: sqlalchemy.engine.ResultProxy, source_model: Type[T], select_related: List = None, @@ -75,7 +77,7 @@ class ModelRow(NewBaseModel): table_prefix = "" if select_related: - source_model = cls + source_model = cast(Type[T], cls) related_models = group_related_list(select_related) if related_field: @@ -107,7 +109,7 @@ class ModelRow(NewBaseModel): item["__excluded__"] = cls.get_names_to_exclude( fields=fields, exclude_fields=exclude_fields ) - instance = cls(**item) + instance = cast(T, cls(**item)) instance.set_save_status(True) return instance @@ -160,6 +162,7 @@ class ModelRow(NewBaseModel): else related ) field = cls.Meta.model_fields[related] + field = cast(Type["ForeignKeyField"], field) fields = cls.get_included(fields, related) exclude_fields = cls.get_excluded(exclude_fields, related) model_cls = field.to @@ -177,7 +180,7 @@ class ModelRow(NewBaseModel): source_model=source_model, ) item[model_cls.get_column_name_from_alias(related)] = child - if issubclass(field, ManyToManyField) and child: + if field.is_multi and child: # TODO: way to figure out which side should be populated? through_name = cls.Meta.model_fields[related].through.get_name() # for now it's nested dict, should be instance? diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index 6975753..cc20807 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -46,15 +46,15 @@ from ormar.relations.alias_manager import AliasManager from ormar.relations.relation_manager import RelationsManager if TYPE_CHECKING: # pragma no cover - from ormar import Model + from ormar.models import Model, T from ormar.signals import SignalEmitter - T = TypeVar("T", bound=Model) - IntStr = Union[int, str] DictStrAny = Dict[str, Any] AbstractSetIntStr = AbstractSet[IntStr] MappingIntStrAny = Mapping[IntStr, Any] +else: + T = TypeVar("T") class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass): @@ -89,7 +89,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass Meta: ModelMeta # noinspection PyMissingConstructor - def __init__(self, *args: Any, **kwargs: Any) -> None: # type: ignore + def __init__(self: T, *args: Any, **kwargs: Any) -> None: # type: ignore """ Initializer that creates a new ormar Model that is also pydantic Model at the same time. @@ -129,7 +129,9 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass object.__setattr__( self, "_orm", - RelationsManager(related_fields=self.extract_related_fields(), owner=self,), + RelationsManager( + related_fields=self.extract_related_fields(), owner=cast(T, self), + ), ) pk_only = kwargs.pop("__pk_only__", False) @@ -298,7 +300,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass def _extract_related_model_instead_of_field( self, item: str - ) -> Optional[Union["T", Sequence["T"]]]: + ) -> Optional[Union["Model", Sequence["Model"]]]: """ Retrieves the related model/models from RelationshipManager. @@ -755,9 +757,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass :return: value of pk if set :rtype: Optional[int] """ - if target_field.virtual or issubclass( - target_field, ormar.fields.ManyToManyField - ): + if target_field.virtual or target_field.is_multi: return self.pk related_name = target_field.name related_model = getattr(self, related_name) diff --git a/ormar/queryset/__init__.py b/ormar/queryset/__init__.py index 8528b05..ebfab7b 100644 --- a/ormar/queryset/__init__.py +++ b/ormar/queryset/__init__.py @@ -5,6 +5,6 @@ from ormar.queryset.filter_query import FilterQuery from ormar.queryset.limit_query import LimitQuery from ormar.queryset.offset_query import OffsetQuery from ormar.queryset.order_query import OrderQuery -from ormar.queryset.queryset import QuerySet +from ormar.queryset.queryset import QuerySet, T -__all__ = ["QuerySet", "FilterQuery", "LimitQuery", "OffsetQuery", "OrderQuery"] +__all__ = ["T", "QuerySet", "FilterQuery", "LimitQuery", "OffsetQuery", "OrderQuery"] diff --git a/ormar/queryset/join.py b/ormar/queryset/join.py index 9ce306d..e90e49d 100644 --- a/ormar/queryset/join.py +++ b/ormar/queryset/join.py @@ -15,7 +15,6 @@ import sqlalchemy from sqlalchemy import text from ormar.exceptions import RelationshipInstanceError # noqa I100 -from ormar.fields import BaseField, ManyToManyField # noqa I100 from ormar.relations import AliasManager if TYPE_CHECKING: # pragma no cover @@ -118,7 +117,7 @@ class SqlJoin: :return: list of used aliases, select from, list of aliased columns, sort orders :rtype: Tuple[List[str], Join, List[TextClause], collections.OrderedDict] """ - if issubclass(self.target_field, ManyToManyField): + if self.target_field.is_multi: self.process_m2m_through_table() self.next_model = self.target_field.to @@ -287,7 +286,7 @@ class SqlJoin: ) pkname_alias = self.next_model.get_column_alias(self.next_model.Meta.pkname) - if not issubclass(self.target_field, ManyToManyField): + if not self.target_field.is_multi: self.get_order_bys( to_table=to_table, pkname_alias=pkname_alias, ) @@ -415,7 +414,7 @@ class SqlJoin: :return: to key and from key :rtype: Tuple[str, str] """ - if issubclass(self.target_field, ManyToManyField): + if self.target_field.is_multi: to_key = self.process_m2m_related_name_change(reverse=True) from_key = self.main_model.get_column_alias(self.main_model.Meta.pkname) diff --git a/ormar/queryset/prefetch_query.py b/ormar/queryset/prefetch_query.py index 4c8c6d7..7abf4c6 100644 --- a/ormar/queryset/prefetch_query.py +++ b/ormar/queryset/prefetch_query.py @@ -13,14 +13,13 @@ from typing import ( ) import ormar -from ormar.fields import BaseField, ManyToManyField -from ormar.fields.foreign_key import ForeignKeyField from ormar.queryset.clause import QueryClause from ormar.queryset.query import Query from ormar.queryset.utils import extract_models_to_dict_of_lists, translate_list_to_dict if TYPE_CHECKING: # pragma: no cover from ormar import Model + from ormar.fields import ForeignKeyField, BaseField def add_relation_field_to_fields( @@ -316,7 +315,7 @@ class PrefetchQuery: for related in related_to_extract: target_field = model.Meta.model_fields[related] - target_field = cast(Type[ForeignKeyField], target_field) + target_field = cast(Type["ForeignKeyField"], target_field) target_model = target_field.to.get_name() model_id = model.get_relation_model_id(target_field=target_field) @@ -424,9 +423,9 @@ class PrefetchQuery: fields = target_model.get_included(fields, related) exclude_fields = target_model.get_excluded(exclude_fields, related) target_field = target_model.Meta.model_fields[related] - target_field = cast(Type[ForeignKeyField], target_field) + target_field = cast(Type["ForeignKeyField"], target_field) reverse = False - if target_field.virtual or issubclass(target_field, ManyToManyField): + if target_field.virtual or target_field.is_multi: reverse = True parent_model = target_model @@ -522,7 +521,7 @@ class PrefetchQuery: select_related = [] query_target = target_model table_prefix = "" - if issubclass(target_field, ManyToManyField): + if target_field.is_multi: query_target = target_field.through select_related = [target_name] table_prefix = target_field.to.Meta.alias_manager.resolve_relation_alias( diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 051e695..ba55586 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -1,4 +1,17 @@ -from typing import Any, Dict, List, Optional, Sequence, Set, TYPE_CHECKING, Type, Union +from typing import ( + Any, + Dict, + Generic, + List, + Optional, + Sequence, + Set, + TYPE_CHECKING, + Type, + TypeVar, + Union, + cast, +) import databases import sqlalchemy @@ -14,19 +27,21 @@ from ormar.queryset.query import Query from ormar.queryset.utils import update, update_dict_from_list if TYPE_CHECKING: # pragma no cover - from ormar import Model + from ormar.models import T from ormar.models.metaclass import ModelMeta from ormar.relations.querysetproxy import QuerysetProxy +else: + T = TypeVar("T") -class QuerySet: +class QuerySet(Generic[T]): """ Main class to perform database queries, exposed on each model as objects attribute. """ def __init__( # noqa CFQ002 self, - model_cls: Type["Model"] = None, + model_cls: Optional[Type[T]] = None, filter_clauses: List = None, exclude_clauses: List = None, select_related: List = None, @@ -53,7 +68,7 @@ class QuerySet: def __get__( self, instance: Optional[Union["QuerySet", "QuerysetProxy"]], - owner: Union[Type["Model"], Type["QuerysetProxy"]], + owner: Union[Type[T], Type["QuerysetProxy"]], ) -> "QuerySet": if issubclass(owner, ormar.Model): if owner.Meta.requires_ref_update: @@ -62,7 +77,7 @@ class QuerySet: f"ForwardRefs. \nBefore using the model you " f"need to call update_forward_refs()." ) - if issubclass(owner, ormar.Model): + owner = cast(Type[T], owner) return self.__class__(model_cls=owner) return self.__class__() # pragma: no cover @@ -79,7 +94,7 @@ class QuerySet: return self.model_cls.Meta @property - def model(self) -> Type["Model"]: + def model(self) -> Type[T]: """ Shortcut to model class set on QuerySet. @@ -91,8 +106,8 @@ class QuerySet: return self.model_cls async def _prefetch_related_models( - self, models: Sequence[Optional["Model"]], rows: List - ) -> Sequence[Optional["Model"]]: + self, models: Sequence[Optional["T"]], rows: List + ) -> Sequence[Optional["T"]]: """ Performs prefetch query for selected models names. @@ -113,7 +128,7 @@ class QuerySet: ) return await query.prefetch_related(models=models, rows=rows) # type: ignore - def _process_query_result_rows(self, rows: List) -> Sequence[Optional["Model"]]: + def _process_query_result_rows(self, rows: List) -> Sequence[Optional[T]]: """ Process database rows and initialize ormar Model from each of the rows. @@ -137,7 +152,7 @@ class QuerySet: return result_rows @staticmethod - def check_single_result_rows_count(rows: Sequence[Optional["Model"]]) -> None: + def check_single_result_rows_count(rows: Sequence[Optional[T]]) -> None: """ Verifies if the result has one and only one row. @@ -198,7 +213,7 @@ class QuerySet: limit_raw_sql=self.limit_sql_raw, ) exp = qry.build_select_expression() - print("\n", exp.compile(compile_kwargs={"literal_binds": True})) + # print("\n", exp.compile(compile_kwargs={"literal_binds": True})) return exp def filter(self, _exclude: bool = False, **kwargs: Any) -> "QuerySet": # noqa: A003 @@ -683,7 +698,7 @@ class QuerySet: limit_raw_sql=limit_raw_sql, ) - async def first(self, **kwargs: Any) -> "Model": + async def first(self, **kwargs: Any) -> T: """ Gets the first row from the db ordered by primary key column ascending. @@ -707,7 +722,7 @@ class QuerySet: self.check_single_result_rows_count(processed_rows) return processed_rows[0] # type: ignore - async def get(self, **kwargs: Any) -> "Model": + async def get(self, **kwargs: Any) -> T: """ Get's the first row from the db meeting the criteria set by kwargs. @@ -739,7 +754,7 @@ class QuerySet: self.check_single_result_rows_count(processed_rows) return processed_rows[0] # type: ignore - async def get_or_create(self, **kwargs: Any) -> "Model": + async def get_or_create(self, **kwargs: Any) -> T: """ Combination of create and get methods. @@ -757,7 +772,7 @@ class QuerySet: except NoMatch: return await self.create(**kwargs) - async def update_or_create(self, **kwargs: Any) -> "Model": + async def update_or_create(self, **kwargs: Any) -> T: """ Updates the model, or in case there is no match in database creates a new one. @@ -774,7 +789,7 @@ class QuerySet: model = await self.get(pk=kwargs[pk_name]) return await model.update(**kwargs) - async def all(self, **kwargs: Any) -> Sequence[Optional["Model"]]: # noqa: A003 + async def all(self, **kwargs: Any) -> Sequence[Optional[T]]: # noqa: A003 """ Returns all rows from a database for given model for set filter options. @@ -798,7 +813,7 @@ class QuerySet: return result_rows - async def create(self, **kwargs: Any) -> "Model": + async def create(self, **kwargs: Any) -> T: """ Creates the model instance, saves it in a database and returns the updates model (with pk populated if not passed and autoincrement is set). @@ -841,7 +856,7 @@ class QuerySet: ) return instance - async def bulk_create(self, objects: List["Model"]) -> None: + async def bulk_create(self, objects: List[T]) -> None: """ Performs a bulk update in one database session to speed up the process. @@ -867,7 +882,7 @@ class QuerySet: objt.set_save_status(True) async def bulk_update( # noqa: CCR001 - self, objects: List["Model"], columns: List[str] = None + self, objects: List[T], columns: List[str] = None ) -> None: """ Performs bulk update in one database session to speed up the process. diff --git a/ormar/queryset/utils.py b/ormar/queryset/utils.py index e2cf33a..f1cbf43 100644 --- a/ormar/queryset/utils.py +++ b/ormar/queryset/utils.py @@ -12,7 +12,6 @@ from typing import ( Union, ) -from ormar.fields import ManyToManyField if TYPE_CHECKING: # pragma no cover from ormar import Model @@ -236,7 +235,7 @@ def get_relationship_alias_model_and_str( manager = model_cls.Meta.alias_manager for relation in related_parts: related_field = model_cls.Meta.model_fields[relation] - if issubclass(related_field, ManyToManyField): + if related_field.is_multi: previous_model = related_field.through relation = related_field.default_target_field_name() # type: ignore table_prefix = manager.resolve_relation_alias( diff --git a/ormar/relations/querysetproxy.py b/ormar/relations/querysetproxy.py index c23dcb7..031684b 100644 --- a/ormar/relations/querysetproxy.py +++ b/ormar/relations/querysetproxy.py @@ -1,6 +1,7 @@ from typing import ( Any, Dict, + Generic, List, MutableSequence, Optional, @@ -9,6 +10,7 @@ from typing import ( TYPE_CHECKING, TypeVar, Union, + cast, ) import ormar @@ -16,14 +18,14 @@ from ormar.exceptions import ModelPersistenceError if TYPE_CHECKING: # pragma no cover from ormar.relations import Relation - from ormar.models import Model + from ormar.models import Model, T from ormar.queryset import QuerySet from ormar import RelationType - - T = TypeVar("T", bound=Model) +else: + T = TypeVar("T") -class QuerysetProxy(ormar.QuerySetProtocol): +class QuerysetProxy(Generic[T]): """ Exposes QuerySet methods on relations, but also handles creating and removing of through Models for m2m relations. @@ -47,7 +49,7 @@ class QuerysetProxy(ormar.QuerySetProtocol): self.through_model_name = ( self.related_field.through.get_name() if self.type_ == ormar.RelationType.MULTIPLE - else None + else "" ) @property @@ -94,6 +96,7 @@ class QuerysetProxy(ormar.QuerySetProtocol): self._assign_child_to_parent(subchild) else: assert isinstance(child, ormar.Model) + child = cast(T, child) self._assign_child_to_parent(child) def _clean_items_on_load(self) -> None: @@ -198,7 +201,7 @@ class QuerysetProxy(ormar.QuerySetProtocol): ) return await queryset.delete(**kwargs) # type: ignore - async def first(self, **kwargs: Any) -> "Model": + async def first(self, **kwargs: Any) -> T: """ Gets the first row from the db ordered by primary key column ascending. @@ -216,7 +219,7 @@ class QuerysetProxy(ormar.QuerySetProtocol): self._register_related(first) return first - async def get(self, **kwargs: Any) -> "Model": + async def get(self, **kwargs: Any) -> "T": """ Get's the first row from the db meeting the criteria set by kwargs. @@ -240,7 +243,7 @@ class QuerysetProxy(ormar.QuerySetProtocol): self._register_related(get) return get - async def all(self, **kwargs: Any) -> Sequence[Optional["Model"]]: # noqa: A003 + async def all(self, **kwargs: Any) -> Sequence[Optional["T"]]: # noqa: A003 """ Returns all rows from a database for given model for set filter options. @@ -262,7 +265,7 @@ class QuerysetProxy(ormar.QuerySetProtocol): self._register_related(all_items) return all_items - async def create(self, **kwargs: Any) -> "Model": + async def create(self, **kwargs: Any) -> "T": """ Creates the model instance, saves it in a database and returns the updates model (with pk populated if not passed and autoincrement is set). @@ -287,7 +290,7 @@ class QuerysetProxy(ormar.QuerySetProtocol): await self.create_through_instance(created, **through_kwargs) return created - async def get_or_create(self, **kwargs: Any) -> "Model": + async def get_or_create(self, **kwargs: Any) -> "T": """ Combination of create and get methods. @@ -305,7 +308,7 @@ class QuerysetProxy(ormar.QuerySetProtocol): except ormar.NoMatch: return await self.create(**kwargs) - async def update_or_create(self, **kwargs: Any) -> "Model": + async def update_or_create(self, **kwargs: Any) -> "T": """ Updates the model, or in case there is no match in database creates a new one. diff --git a/ormar/relations/relation.py b/ormar/relations/relation.py index 0ae2f59..6d4da36 100644 --- a/ormar/relations/relation.py +++ b/ormar/relations/relation.py @@ -1,17 +1,13 @@ from enum import Enum -from typing import List, Optional, Set, TYPE_CHECKING, Type, TypeVar, Union +from typing import List, Optional, Set, TYPE_CHECKING, Type, Union import ormar # noqa I100 from ormar.exceptions import RelationshipInstanceError # noqa I100 -from ormar.fields.foreign_key import ForeignKeyField # noqa I100 from ormar.relations.relation_proxy import RelationProxy if TYPE_CHECKING: # pragma no cover - from ormar import Model from ormar.relations import RelationsManager - from ormar.models import NewBaseModel - - T = TypeVar("T", bound=Model) + from ormar.models import Model, NewBaseModel, T class RelationType(Enum): @@ -39,7 +35,7 @@ class Relation: manager: "RelationsManager", type_: RelationType, field_name: str, - to: Type["T"], + to: Type["Model"], through: Type["T"] = None, ) -> None: """ @@ -63,10 +59,10 @@ class Relation: self._owner: "Model" = manager.owner self._type: RelationType = type_ self._to_remove: Set = set() - self.to: Type["T"] = to - self._through: Optional[Type["T"]] = through + self.to: Type["Model"] = to + self._through = through self.field_name: str = field_name - self.related_models: Optional[Union[RelationProxy, "T"]] = ( + self.related_models: Optional[Union[RelationProxy, "Model"]] = ( RelationProxy(relation=self, type_=type_, field_name=field_name) if type_ in (RelationType.REVERSE, RelationType.MULTIPLE) else None @@ -161,7 +157,7 @@ class Relation: self.related_models.pop(position) # type: ignore del self._owner.__dict__[relation_name][position] - def get(self) -> Optional[Union[List["T"], "T"]]: + def get(self) -> Optional[Union[List["Model"], "Model"]]: """ Return the related model or models from RelationProxy. diff --git a/ormar/relations/relation_manager.py b/ormar/relations/relation_manager.py index addfcf1..a718b09 100644 --- a/ormar/relations/relation_manager.py +++ b/ormar/relations/relation_manager.py @@ -1,17 +1,12 @@ -from typing import Dict, List, Optional, Sequence, TYPE_CHECKING, Type, TypeVar, Union +from typing import Dict, List, Optional, Sequence, TYPE_CHECKING, Type, Union from weakref import proxy -from ormar.fields import BaseField, ThroughField -from ormar.fields.foreign_key import ForeignKeyField -from ormar.fields.many_to_many import ManyToManyField from ormar.relations.relation import Relation, RelationType from ormar.relations.utils import get_relations_sides_and_names if TYPE_CHECKING: # pragma no cover - from ormar import Model - from ormar.models import NewBaseModel - - T = TypeVar("T", bound=Model) + from ormar.models import NewBaseModel, T, Model + from ormar.fields import ForeignKeyField, BaseField class RelationsManager: @@ -21,8 +16,8 @@ class RelationsManager: def __init__( self, - related_fields: List[Type[ForeignKeyField]] = None, - owner: "NewBaseModel" = None, + related_fields: List[Type["ForeignKeyField"]] = None, + owner: Optional["T"] = None, ) -> None: self.owner = proxy(owner) self._related_fields = related_fields or [] @@ -31,7 +26,7 @@ class RelationsManager: for field in self._related_fields: self._add_relation(field) - def _get_relation_type(self, field: Type[BaseField]) -> RelationType: + def _get_relation_type(self, field: Type["BaseField"]) -> RelationType: """ Returns type of the relation declared on a field. @@ -40,13 +35,13 @@ class RelationsManager: :return: type of the relation defined on field :rtype: RelationType """ - if issubclass(field, ManyToManyField): + if field.is_multi: return RelationType.MULTIPLE - if issubclass(field, ThroughField): + if field.is_through: return RelationType.THROUGH return RelationType.PRIMARY if not field.virtual else RelationType.REVERSE - def _add_relation(self, field: Type[BaseField]) -> None: + def _add_relation(self, field: Type["BaseField"]) -> None: """ Registers relation in the manager. Adds Relation instance under field.name. @@ -73,7 +68,7 @@ class RelationsManager: """ return item in self._related_names - def get(self, name: str) -> Optional[Union["T", Sequence["T"]]]: + def get(self, name: str) -> Optional[Union["Model", Sequence["Model"]]]: """ Returns the related model/models if relation is set. Actual call is delegated to Relation instance registered under relation name. diff --git a/ormar/relations/relation_proxy.py b/ormar/relations/relation_proxy.py index 87bb05f..58d6e9e 100644 --- a/ormar/relations/relation_proxy.py +++ b/ormar/relations/relation_proxy.py @@ -27,7 +27,9 @@ class RelationProxy(list): self.type_: "RelationType" = type_ self.field_name = field_name self._owner: "Model" = self.relation.manager.owner - self.queryset_proxy = QuerysetProxy(relation=self.relation, type_=type_) + self.queryset_proxy: QuerysetProxy = QuerysetProxy( + relation=self.relation, type_=type_ + ) self._related_field_name: Optional[str] = None @property diff --git a/test.db-journal b/test.db-journal new file mode 100644 index 0000000000000000000000000000000000000000..f5538646e9b3593b93b04ca5c2a9d379bdd826c2 GIT binary patch literal 4616 zcmZQzK!AcPQ+$9tRt5$pASHm17-bFT5Mbs7`EE42hjSc{`fW4>MnhmU1V%$(Gz3ON aU^E0qLtr!nMnhmU1V%$(Gz1tx<30d0eguF3 literal 0 HcmV?d00001 diff --git a/tests/test_m2m_through_fields.py b/tests/test_m2m_through_fields.py index cb123fe..279d1a8 100644 --- a/tests/test_m2m_through_fields.py +++ b/tests/test_m2m_through_fields.py @@ -1,3 +1,5 @@ +from typing import Any + import databases import pytest import sqlalchemy @@ -19,8 +21,8 @@ class Category(ormar.Model): class Meta(BaseMeta): tablename = "categories" - id: int = ormar.Integer(primary_key=True) - name: str = ormar.String(max_length=40) + id = ormar.Integer(primary_key=True) + name = ormar.String(max_length=40) class PostCategory(ormar.Model): @@ -107,8 +109,12 @@ async def test_setting_additional_fields_on_through_model_in_create(): assert postcat.sort_order == 2 +def process_post(post: Post): + pass + + @pytest.mark.asyncio -async def test_getting_additional_fields_from_queryset(): +async def test_getting_additional_fields_from_queryset() -> Any: async with database: post = await Post(title="Test post").save() await post.categories.create( @@ -122,10 +128,11 @@ async def test_getting_additional_fields_from_queryset(): assert post.categories[0].postcategory.sort_order == 1 assert post.categories[1].postcategory.sort_order == 2 - post = await Post.objects.select_related("categories").get( + post2 = await Post.objects.select_related("categories").get( categories__name="Test category2" ) - assert post.categories[0].postcategory.sort_order == 2 + assert post2.categories[0].postcategory.sort_order == 2 + process_post(post2) # TODO: check/ modify following From c139ca4f6179ef4dbad743242a6574b057a7a7fd Mon Sep 17 00:00:00 2001 From: collerek Date: Wed, 24 Feb 2021 18:14:25 +0100 Subject: [PATCH 04/14] add possibility to filter on through models fields --- .gitignore | 1 + ormar/models/model.py | 2 +- ormar/models/newbasemodel.py | 2 +- ormar/queryset/clause.py | 1 + ormar/queryset/filter_action.py | 2 ++ ormar/queryset/join.py | 21 +----------------- ormar/queryset/utils.py | 15 +++++++++---- ormar/relations/querysetproxy.py | 6 ++--- test.db-journal | Bin 4616 -> 0 bytes tests/test_m2m_through_fields.py | 37 +++++++++++++++++++++++++------ 10 files changed, 51 insertions(+), 36 deletions(-) delete mode 100644 test.db-journal diff --git a/.gitignore b/.gitignore index fc07f13..6c5114b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ dist site profile.py *.db +*.db-journal diff --git a/ormar/models/model.py b/ormar/models/model.py index c20368d..1535113 100644 --- a/ormar/models/model.py +++ b/ormar/models/model.py @@ -22,7 +22,7 @@ class Model(ModelRow): __abstract__ = False if TYPE_CHECKING: # pragma nocover Meta: ModelMeta - objects: "QuerySet" + objects: "QuerySet[Model]" def __repr__(self) -> str: # pragma nocover _repr = {k: getattr(self, k) for k, v in self.Meta.model_fields.items()} diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index cc20807..d400c0f 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -310,7 +310,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass :rtype: Optional[Union[Model, List[Model]]] """ if item in self._orm: - return self._orm.get(item) + return self._orm.get(item) # type: ignore return None # pragma no cover def __eq__(self, other: object) -> bool: diff --git a/ormar/queryset/clause.py b/ormar/queryset/clause.py index e52ae4a..b5a3c5b 100644 --- a/ormar/queryset/clause.py +++ b/ormar/queryset/clause.py @@ -16,6 +16,7 @@ class Prefix: table_prefix: str model_cls: Type["Model"] relation_str: str + is_through: bool @property def alias_key(self) -> str: diff --git a/ormar/queryset/filter_action.py b/ormar/queryset/filter_action.py index 4f26864..d2d8e45 100644 --- a/ormar/queryset/filter_action.py +++ b/ormar/queryset/filter_action.py @@ -53,6 +53,7 @@ class FilterAction: self.table_prefix = "" self.source_model = model_cls self.target_model = model_cls + self.is_through = False self._determine_filter_target_table() self._escape_characters_in_clause() @@ -100,6 +101,7 @@ class FilterAction: self.table_prefix, self.target_model, self.related_str, + self.is_through, ) = get_relationship_alias_model_and_str(self.source_model, self.related_parts) def _escape_characters_in_clause(self) -> None: diff --git a/ormar/queryset/join.py b/ormar/queryset/join.py index e90e49d..4626fbd 100644 --- a/ormar/queryset/join.py +++ b/ormar/queryset/join.py @@ -290,9 +290,8 @@ class SqlJoin: self.get_order_bys( to_table=to_table, pkname_alias=pkname_alias, ) - else: - self.select_through_model_fields() + # TODO: fix fields and exclusions for through model? self_related_fields = self.next_model.own_table_columns( model=self.next_model, fields=self.fields, @@ -306,24 +305,6 @@ class SqlJoin: ) self.used_aliases.append(self.next_alias) - def select_through_model_fields(self) -> None: - # TODO: add docstring - next_alias = self.alias_manager.resolve_relation_alias( - from_model=self.target_field.owner, relation_name=self.relation_name - ) - # TODO: fix fields and exclusions - self_related_fields = self.target_field.through.own_table_columns( - model=self.target_field.through, - fields=None, - exclude_fields=self.target_field.through.extract_related_names(), - use_alias=True, - ) - self.columns.extend( - self.alias_manager.prefixed_columns( - next_alias, self.target_field.through.Meta.table, self_related_fields - ) - ) - def _replace_many_to_many_order_by_columns(self, part: str, new_part: str) -> None: """ Substitutes the name of the relation with actual model name in m2m order bys. diff --git a/ormar/queryset/utils.py b/ormar/queryset/utils.py index f1cbf43..6b98028 100644 --- a/ormar/queryset/utils.py +++ b/ormar/queryset/utils.py @@ -12,7 +12,6 @@ from typing import ( Union, ) - if TYPE_CHECKING: # pragma no cover from ormar import Model @@ -218,7 +217,7 @@ def extract_models_to_dict_of_lists( def get_relationship_alias_model_and_str( source_model: Type["Model"], related_parts: List -) -> Tuple[str, Type["Model"], str]: +) -> Tuple[str, Type["Model"], str, bool]: """ Walks the relation to retrieve the actual model on which the clause should be constructed, extracts alias based on last relation leading to target model. @@ -230,11 +229,19 @@ def get_relationship_alias_model_and_str( :rtype: Tuple[str, Type["Model"], str] """ table_prefix = "" + is_through = False model_cls = source_model previous_model = model_cls manager = model_cls.Meta.alias_manager - for relation in related_parts: + for relation in related_parts[:]: related_field = model_cls.Meta.model_fields[relation] + if related_field.is_through: + is_through = True + related_parts = [ + x.replace(relation, related_field.related_name) if x == relation else x + for x in related_parts + ] + relation = related_field.related_name if related_field.is_multi: previous_model = related_field.through relation = related_field.default_target_field_name() # type: ignore @@ -245,4 +252,4 @@ def get_relationship_alias_model_and_str( previous_model = model_cls relation_str = "__".join(related_parts) - return table_prefix, model_cls, relation_str + return table_prefix, model_cls, relation_str, is_through diff --git a/ormar/relations/querysetproxy.py b/ormar/relations/querysetproxy.py index 031684b..85b7832 100644 --- a/ormar/relations/querysetproxy.py +++ b/ormar/relations/querysetproxy.py @@ -139,7 +139,7 @@ class QuerysetProxy(Generic[T]): :param child: child model instance :type child: Model """ - queryset = ormar.QuerySet(model_cls=self.relation.through) + queryset = ormar.QuerySet(model_cls=self.relation.through) # type: ignore owner_column = self.related_field.default_target_field_name() # type: ignore child_column = self.related_field.default_source_field_name() # type: ignore kwargs = {owner_column: self._owner, child_column: child} @@ -187,10 +187,10 @@ class QuerysetProxy(Generic[T]): :rtype: int """ if self.type_ == ormar.RelationType.MULTIPLE: - queryset = ormar.QuerySet(model_cls=self.relation.through) + queryset = ormar.QuerySet(model_cls=self.relation.through) # type: ignore owner_column = self._owner.get_name() else: - queryset = ormar.QuerySet(model_cls=self.relation.to) + queryset = ormar.QuerySet(model_cls=self.relation.to) # type: ignore owner_column = self.related_field.name kwargs = {owner_column: self._owner} self._clean_items_on_load() diff --git a/test.db-journal b/test.db-journal deleted file mode 100644 index f5538646e9b3593b93b04ca5c2a9d379bdd826c2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4616 zcmZQzK!AcPQ+$9tRt5$pASHm17-bFT5Mbs7`EE42hjSc{`fW4>MnhmU1V%$(Gz3ON aU^E0qLtr!nMnhmU1V%$(Gz1tx<30d0eguF3 diff --git a/tests/test_m2m_through_fields.py b/tests/test_m2m_through_fields.py index 279d1a8..9d7f38a 100644 --- a/tests/test_m2m_through_fields.py +++ b/tests/test_m2m_through_fields.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, TYPE_CHECKING import databases import pytest @@ -31,6 +31,7 @@ class PostCategory(ormar.Model): id: int = ormar.Integer(primary_key=True) sort_order: int = ormar.Integer(nullable=True) + param_name: str = ormar.String(default="Name", max_length=200) class Post(ormar.Model): @@ -109,10 +110,6 @@ async def test_setting_additional_fields_on_through_model_in_create(): assert postcat.sort_order == 2 -def process_post(post: Post): - pass - - @pytest.mark.asyncio async def test_getting_additional_fields_from_queryset() -> Any: async with database: @@ -132,9 +129,35 @@ async def test_getting_additional_fields_from_queryset() -> Any: categories__name="Test category2" ) assert post2.categories[0].postcategory.sort_order == 2 - process_post(post2) + # if TYPE_CHECKING: + # reveal_type(post2) +@pytest.mark.asyncio +async def test_filtering_by_through_model() -> Any: + async with database: + post = await Post(title="Test post").save() + await post.categories.create( + name="Test category1", + postcategory={"sort_order": 1, "param_name": "volume"}, + ) + await post.categories.create( + name="Test category2", postcategory={"sort_order": 2, "param_name": "area"} + ) + + post2 = ( + await Post.objects.filter(postcategory__sort_order__gt=1) + .select_related("categories") + .get() + ) + assert len(post2.categories) == 1 + assert post2.categories[0].postcategory.sort_order == 2 + + post3 = await Post.objects.filter( + categories__postcategory__param_name="volume").get() + assert len(post3.categories) == 1 + assert post3.categories[0].postcategory.param_name == "volume" + # TODO: check/ modify following # add to fields with class lower name (V) @@ -143,9 +166,9 @@ async def test_getting_additional_fields_from_queryset() -> Any: # creating in queryset proxy (dict with through name and kwargs) (V) # loading the data into model instance of though model (V) <- fix fields ane exclude # accessing from instance (V) <- no both sides only nested one is relevant, fix one side +# filtering in filter (through name normally) (V) < - table prefix from normal relation, check if is_through needed # updating in query -# sorting in filter (special __through__ notation?) # ordering by in order_by # modifying from instance (both sides?) # including/excluding in fields? From 503f589fa74bb40862f30aefb31b0c720bc49857 Mon Sep 17 00:00:00 2001 From: collerek Date: Thu, 25 Feb 2021 17:28:05 +0100 Subject: [PATCH 05/14] refactor order bys into OrderQuery, add ordering to through models too --- ormar/__init__.py | 3 +- ormar/models/mixins/prefetch_mixin.py | 12 +- ormar/models/model_row.py | 2 +- ormar/queryset/__init__.py | 12 +- ormar/queryset/actions/__init__.py | 4 + ormar/queryset/{ => actions}/filter_action.py | 76 +++------- ormar/queryset/actions/order_action.py | 68 +++++++++ ormar/queryset/actions/query_action.py | 93 ++++++++++++ ormar/queryset/clause.py | 2 +- ormar/queryset/filter_query.py | 2 +- ormar/queryset/join.py | 134 ++++++------------ ormar/queryset/prefetch_query.py | 7 +- ormar/queryset/query.py | 48 ++----- ormar/queryset/queryset.py | 26 +++- ormar/queryset/utils.py | 18 ++- tests/test_m2m_through_fields.py | 91 +++++++++++- 16 files changed, 388 insertions(+), 210 deletions(-) create mode 100644 ormar/queryset/actions/__init__.py rename ormar/queryset/{ => actions}/filter_action.py (71%) create mode 100644 ormar/queryset/actions/order_action.py create mode 100644 ormar/queryset/actions/query_action.py diff --git a/ormar/__init__.py b/ormar/__init__.py index b2c7020..328e894 100644 --- a/ormar/__init__.py +++ b/ormar/__init__.py @@ -56,7 +56,7 @@ from ormar.fields import ( ) # noqa: I100 from ormar.models import Model from ormar.models.metaclass import ModelMeta -from ormar.queryset import QuerySet +from ormar.queryset import OrderAction, QuerySet from ormar.relations import RelationType from ormar.signals import Signal @@ -106,4 +106,5 @@ __all__ = [ "BaseField", "ManyToManyField", "ForeignKeyField", + "OrderAction", ] diff --git a/ormar/models/mixins/prefetch_mixin.py b/ormar/models/mixins/prefetch_mixin.py index d8ee350..440052a 100644 --- a/ormar/models/mixins/prefetch_mixin.py +++ b/ormar/models/mixins/prefetch_mixin.py @@ -2,7 +2,7 @@ from typing import Callable, Dict, List, TYPE_CHECKING, Tuple, Type, cast from ormar.models.mixins.relation_mixin import RelationMixin -if TYPE_CHECKING: +if TYPE_CHECKING: # pragma: no cover from ormar.fields import ForeignKeyField, ManyToManyField @@ -18,10 +18,10 @@ class PrefetchQueryMixin(RelationMixin): @staticmethod def get_clause_target_and_filter_column_name( - parent_model: Type["Model"], - target_model: Type["Model"], - reverse: bool, - related: str, + parent_model: Type["Model"], + target_model: Type["Model"], + reverse: bool, + related: str, ) -> Tuple[Type["Model"], str]: """ Returns Model on which query clause should be performed and name of the column. @@ -51,7 +51,7 @@ class PrefetchQueryMixin(RelationMixin): @staticmethod def get_column_name_for_id_extraction( - parent_model: Type["Model"], reverse: bool, related: str, use_raw: bool, + parent_model: Type["Model"], reverse: bool, related: str, use_raw: bool, ) -> str: """ Returns name of the column that should be used to extract ids from model. diff --git a/ormar/models/model_row.py b/ormar/models/model_row.py index f0a4a31..476e274 100644 --- a/ormar/models/model_row.py +++ b/ormar/models/model_row.py @@ -17,7 +17,7 @@ from ormar.models import NewBaseModel # noqa: I202 from ormar.models.helpers.models import group_related_list -if TYPE_CHECKING: +if TYPE_CHECKING: # pragma: no cover from ormar.fields import ForeignKeyField from ormar.models import T else: diff --git a/ormar/queryset/__init__.py b/ormar/queryset/__init__.py index ebfab7b..11b072e 100644 --- a/ormar/queryset/__init__.py +++ b/ormar/queryset/__init__.py @@ -1,10 +1,20 @@ """ Contains QuerySet and different Query classes to allow for constructing of sql queries. """ +from ormar.queryset.actions import FilterAction, OrderAction from ormar.queryset.filter_query import FilterQuery from ormar.queryset.limit_query import LimitQuery from ormar.queryset.offset_query import OffsetQuery from ormar.queryset.order_query import OrderQuery from ormar.queryset.queryset import QuerySet, T -__all__ = ["T", "QuerySet", "FilterQuery", "LimitQuery", "OffsetQuery", "OrderQuery"] +__all__ = [ + "T", + "QuerySet", + "FilterQuery", + "LimitQuery", + "OffsetQuery", + "OrderQuery", + "FilterAction", + "OrderAction", +] diff --git a/ormar/queryset/actions/__init__.py b/ormar/queryset/actions/__init__.py new file mode 100644 index 0000000..088d68a --- /dev/null +++ b/ormar/queryset/actions/__init__.py @@ -0,0 +1,4 @@ +from ormar.queryset.actions.filter_action import FilterAction +from ormar.queryset.actions.order_action import OrderAction + +__all__ = ["FilterAction", "OrderAction"] diff --git a/ormar/queryset/filter_action.py b/ormar/queryset/actions/filter_action.py similarity index 71% rename from ormar/queryset/filter_action.py rename to ormar/queryset/actions/filter_action.py index d2d8e45..43c71df 100644 --- a/ormar/queryset/filter_action.py +++ b/ormar/queryset/actions/filter_action.py @@ -1,11 +1,11 @@ -from typing import Any, Dict, List, TYPE_CHECKING, Type +from typing import Any, Dict, TYPE_CHECKING, Type import sqlalchemy from sqlalchemy import text import ormar # noqa: I100, I202 from ormar.exceptions import QueryDefinitionError -from ormar.queryset.utils import get_relationship_alias_model_and_str +from ormar.queryset.actions.query_action import QueryAction if TYPE_CHECKING: # pragma: nocover from ormar import Model @@ -28,7 +28,7 @@ FILTER_OPERATORS = { ESCAPE_CHARACTERS = ["%", "_"] -class FilterAction: +class FilterAction(QueryAction): """ Filter Actions is populated by queryset when filter() is called. @@ -39,7 +39,18 @@ class FilterAction: """ def __init__(self, filter_str: str, value: Any, model_cls: Type["Model"]) -> None: - parts = filter_str.split("__") + super().__init__(query_str=filter_str, model_cls=model_cls) + self.filter_value = value + self._escape_characters_in_clause() + + def has_escaped_characters(self) -> bool: + """Check if value is a string that contains characters to escape""" + return isinstance(self.filter_value, str) and any( + c for c in ESCAPE_CHARACTERS if c in self.filter_value + ) + + def _split_value_into_parts(self, query_str: str) -> None: + parts = query_str.split("__") if parts[-1] in FILTER_OPERATORS: self.operator = parts[-1] self.field_name = parts[-2] @@ -49,61 +60,6 @@ class FilterAction: self.field_name = parts[-1] self.related_parts = parts[:-1] - self.filter_value = value - self.table_prefix = "" - self.source_model = model_cls - self.target_model = model_cls - self.is_through = False - self._determine_filter_target_table() - self._escape_characters_in_clause() - - @property - def table(self) -> sqlalchemy.Table: - """Shortcut to sqlalchemy Table of filtered target model""" - return self.target_model.Meta.table - - @property - def column(self) -> sqlalchemy.Column: - """Shortcut to sqlalchemy column of filtered target model""" - aliased_name = self.target_model.get_column_alias(self.field_name) - return self.target_model.Meta.table.columns[aliased_name] - - def has_escaped_characters(self) -> bool: - """Check if value is a string that contains characters to escape""" - return isinstance(self.filter_value, str) and any( - c for c in ESCAPE_CHARACTERS if c in self.filter_value - ) - - def update_select_related(self, select_related: List[str]) -> List[str]: - """ - Updates list of select related with related part included in the filter key. - That way If you want to just filter by relation you do not have to provide - select_related separately. - - :param select_related: list of relation join strings - :type select_related: List[str] - :return: list of relation joins with implied joins from filter added - :rtype: List[str] - """ - select_related = select_related[:] - if self.related_str and not any( - rel.startswith(self.related_str) for rel in select_related - ): - select_related.append(self.related_str) - return select_related - - def _determine_filter_target_table(self) -> None: - """ - Walks the relation to retrieve the actual model on which the clause should be - constructed, extracts alias based on last relation leading to target model. - """ - ( - self.table_prefix, - self.target_model, - self.related_str, - self.is_through, - ) = get_relationship_alias_model_and_str(self.source_model, self.related_parts) - def _escape_characters_in_clause(self) -> None: """ Escapes the special characters ["%", "_"] if needed. @@ -151,7 +107,7 @@ class FilterAction: sufix = "%" if "end" not in self.operator else "" self.filter_value = f"{prefix}{self.filter_value}{sufix}" - def get_text_clause(self,) -> sqlalchemy.sql.expression.TextClause: + def get_text_clause(self) -> sqlalchemy.sql.expression.TextClause: """ Escapes characters if it's required. Substitutes values of the models if value is a ormar Model with its pk value. diff --git a/ormar/queryset/actions/order_action.py b/ormar/queryset/actions/order_action.py new file mode 100644 index 0000000..2173e24 --- /dev/null +++ b/ormar/queryset/actions/order_action.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING, Type + +import sqlalchemy +from sqlalchemy import text + +from ormar.queryset.actions.query_action import QueryAction # noqa: I100, I202 + +if TYPE_CHECKING: # pragma: nocover + from ormar import Model + + +class OrderAction(QueryAction): + """ + Order Actions is populated by queryset when order_by() is called. + + All required params are extracted but kept raw until actual filter clause value + is required -> then the action is converted into text() clause. + + Extracted in order to easily change table prefixes on complex relations. + """ + + def __init__( + self, order_str: str, model_cls: Type["Model"], alias: str = None + ) -> None: + self.direction: str = "" + super().__init__(query_str=order_str, model_cls=model_cls) + self.is_source_model_order = False + if alias: + self.table_prefix = alias + if self.source_model == self.target_model and "__" not in self.related_str: + self.is_source_model_order = True + + @property + def field_alias(self) -> str: + return self.target_model.get_column_alias(self.field_name) + + def get_text_clause(self) -> sqlalchemy.sql.expression.TextClause: + """ + Escapes characters if it's required. + Substitutes values of the models if value is a ormar Model with its pk value. + Compiles the clause. + + :return: complied and escaped clause + :rtype: sqlalchemy.sql.elements.TextClause + """ + prefix = f"{self.table_prefix}_" if self.table_prefix else "" + return text(f"{prefix}{self.table}" f".{self.field_alias} {self.direction}") + + def _split_value_into_parts(self, order_str: str) -> None: + if order_str.startswith("-"): + self.direction = "desc" + order_str = order_str[1:] + parts = order_str.split("__") + self.field_name = parts[-1] + self.related_parts = parts[:-1] + + def check_if_filter_apply(self, target_model: Type["Model"], alias: str) -> bool: + """ + Checks filter conditions to find if they apply to current join. + + :param target_model: model which is now processed + :type target_model: Type["Model"] + :param alias: prefix of the relation + :type alias: str + :return: result of the check + :rtype: bool + """ + return target_model == self.target_model and alias == self.table_prefix diff --git a/ormar/queryset/actions/query_action.py b/ormar/queryset/actions/query_action.py new file mode 100644 index 0000000..2c6ee84 --- /dev/null +++ b/ormar/queryset/actions/query_action.py @@ -0,0 +1,93 @@ +import abc +from typing import Any, List, TYPE_CHECKING, Type + +import sqlalchemy + +from ormar.queryset.utils import get_relationship_alias_model_and_str # noqa: I202 + +if TYPE_CHECKING: # pragma: nocover + from ormar import Model + + +class QueryAction(abc.ABC): + """ + Base QueryAction class with common params for Filter and Order actions. + """ + + def __init__(self, query_str: str, model_cls: Type["Model"]) -> None: + self.query_str = query_str + self.field_name: str = "" + self.related_parts: List[str] = [] + self.related_str: str = "" + + self.table_prefix = "" + self.source_model = model_cls + self.target_model = model_cls + self.is_through = False + + self._split_value_into_parts(query_str) + self._determine_filter_target_table() + + def __eq__(self, other: object) -> bool: # pragma: no cover + if not isinstance(other, QueryAction): + return False + return self.query_str == other.query_str + + def __hash__(self) -> Any: + return hash((self.table_prefix, self.query_str)) + + @abc.abstractmethod + def _split_value_into_parts(self, query_str: str) -> None: # pragma: no cover + """ + Splits string into related parts and field_name + :param query_str: query action string to split (i..e filter or order by) + :type query_str: str + """ + pass + + @abc.abstractmethod + def get_text_clause( + self, + ) -> sqlalchemy.sql.expression.TextClause: # pragma: no cover + pass + + @property + def table(self) -> sqlalchemy.Table: + """Shortcut to sqlalchemy Table of filtered target model""" + return self.target_model.Meta.table + + @property + def column(self) -> sqlalchemy.Column: + """Shortcut to sqlalchemy column of filtered target model""" + aliased_name = self.target_model.get_column_alias(self.field_name) + return self.target_model.Meta.table.columns[aliased_name] + + def update_select_related(self, select_related: List[str]) -> List[str]: + """ + Updates list of select related with related part included in the filter key. + That way If you want to just filter by relation you do not have to provide + select_related separately. + + :param select_related: list of relation join strings + :type select_related: List[str] + :return: list of relation joins with implied joins from filter added + :rtype: List[str] + """ + select_related = select_related[:] + if self.related_str and not any( + rel.startswith(self.related_str) for rel in select_related + ): + select_related.append(self.related_str) + return select_related + + def _determine_filter_target_table(self) -> None: + """ + Walks the relation to retrieve the actual model on which the clause should be + constructed, extracts alias based on last relation leading to target model. + """ + ( + self.table_prefix, + self.target_model, + self.related_str, + self.is_through, + ) = get_relationship_alias_model_and_str(self.source_model, self.related_parts) diff --git a/ormar/queryset/clause.py b/ormar/queryset/clause.py index b5a3c5b..b98616d 100644 --- a/ormar/queryset/clause.py +++ b/ormar/queryset/clause.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from typing import Any, List, TYPE_CHECKING, Tuple, Type import ormar # noqa I100 -from ormar.queryset.filter_action import FilterAction +from ormar.queryset.actions.filter_action import FilterAction from ormar.queryset.utils import get_relationship_alias_model_and_str if TYPE_CHECKING: # pragma no cover diff --git a/ormar/queryset/filter_query.py b/ormar/queryset/filter_query.py index 4100f16..cb9b880 100644 --- a/ormar/queryset/filter_query.py +++ b/ormar/queryset/filter_query.py @@ -1,7 +1,7 @@ from typing import List import sqlalchemy -from ormar.queryset.filter_action import FilterAction +from ormar.queryset.actions.filter_action import FilterAction class FilterQuery: diff --git a/ormar/queryset/join.py b/ormar/queryset/join.py index 4626fbd..f18c81d 100644 --- a/ormar/queryset/join.py +++ b/ormar/queryset/join.py @@ -14,11 +14,13 @@ from typing import ( import sqlalchemy from sqlalchemy import text -from ormar.exceptions import RelationshipInstanceError # noqa I100 +import ormar # noqa I100 +from ormar.exceptions import RelationshipInstanceError from ormar.relations import AliasManager if TYPE_CHECKING: # pragma no cover from ormar import Model + from ormar.queryset import OrderAction class SqlJoin: @@ -29,7 +31,7 @@ class SqlJoin: columns: List[sqlalchemy.Column], fields: Optional[Union[Set, Dict]], exclude_fields: Optional[Union[Set, Dict]], - order_columns: Optional[List], + order_columns: Optional[List["OrderAction"]], sorted_orders: OrderedDict, main_model: Type["Model"], relation_name: str, @@ -89,7 +91,18 @@ class SqlJoin: """ return self.main_model.Meta.alias_manager - def on_clause(self, previous_alias: str, from_clause: str, to_clause: str,) -> text: + @property + def to_table(self) -> str: + """ + Shortcut to table name of the next model + :return: name of the target table + :rtype: str + """ + return self.next_model.Meta.table.name + + def _on_clause( + self, previous_alias: str, from_clause: str, to_clause: str, + ) -> text: """ Receives aliases and names of both ends of the join and combines them into one text clause used in joins. @@ -118,7 +131,7 @@ class SqlJoin: :rtype: Tuple[List[str], Join, List[TextClause], collections.OrderedDict] """ if self.target_field.is_multi: - self.process_m2m_through_table() + self._process_m2m_through_table() self.next_model = self.target_field.to self._forward_join() @@ -207,7 +220,7 @@ class SqlJoin: self.sorted_orders, ) = sql_join.build_join() - def process_m2m_through_table(self) -> None: + def _process_m2m_through_table(self) -> None: """ Process Through table of the ManyToMany relation so that source table is linked to the through table (one additional join) @@ -222,8 +235,7 @@ class SqlJoin: To point to through model """ - new_part = self.process_m2m_related_name_change() - self._replace_many_to_many_order_by_columns(self.relation_name, new_part) + new_part = self._process_m2m_related_name_change() self.next_model = self.target_field.through self._forward_join() @@ -232,7 +244,7 @@ class SqlJoin: self.own_alias = self.next_alias self.target_field = self.next_model.Meta.model_fields[self.relation_name] - def process_m2m_related_name_change(self, reverse: bool = False) -> str: + def _process_m2m_related_name_change(self, reverse: bool = False) -> str: """ Extracts relation name to link join through the Through model declared on relation field. @@ -272,24 +284,21 @@ class SqlJoin: Process order_by causes for non m2m relations. """ - to_table = self.next_model.Meta.table.name - to_key, from_key = self.get_to_and_from_keys() + to_key, from_key = self._get_to_and_from_keys() - on_clause = self.on_clause( + on_clause = self._on_clause( previous_alias=self.own_alias, from_clause=f"{self.target_field.owner.Meta.tablename}.{from_key}", - to_clause=f"{to_table}.{to_key}", + to_clause=f"{self.to_table}.{to_key}", + ) + target_table = self.alias_manager.prefixed_table_name( + self.next_alias, self.to_table ) - target_table = self.alias_manager.prefixed_table_name(self.next_alias, to_table) self.select_from = sqlalchemy.sql.outerjoin( self.select_from, target_table, on_clause ) - pkname_alias = self.next_model.get_column_alias(self.next_model.Meta.pkname) - if not self.target_field.is_multi: - self.get_order_bys( - to_table=to_table, pkname_alias=pkname_alias, - ) + self._get_order_bys() # TODO: fix fields and exclusions for through model? self_related_fields = self.next_model.own_table_columns( @@ -305,88 +314,35 @@ class SqlJoin: ) self.used_aliases.append(self.next_alias) - def _replace_many_to_many_order_by_columns(self, part: str, new_part: str) -> None: - """ - Substitutes the name of the relation with actual model name in m2m order bys. - - :param part: name of the field with relation - :type part: str - :param new_part: name of the target model - :type new_part: str - """ - if self.order_columns: - split_order_columns = [ - x.split("__") for x in self.order_columns if "__" in x - ] - for condition in split_order_columns: - if self._check_if_condition_apply(condition, part): - condition[-2] = condition[-2].replace(part, new_part) - self.order_columns = [x for x in self.order_columns if "__" not in x] + [ - "__".join(x) for x in split_order_columns - ] - - @staticmethod - def _check_if_condition_apply(condition: List, part: str) -> bool: - """ - Checks filter conditions to find if they apply to current join. - - :param condition: list of parts of condition split by '__' - :type condition: List[str] - :param part: name of the current relation join. - :type part: str - :return: result of the check - :rtype: bool - """ - return len(condition) >= 2 and ( - condition[-2] == part or condition[-2][1:] == part + def _set_default_primary_key_order_by(self) -> None: + clause = ormar.OrderAction( + order_str=self.next_model.Meta.pkname, + model_cls=self.next_model, + alias=self.next_alias, ) + self.sorted_orders[clause] = clause.get_text_clause() - def set_aliased_order_by(self, condition: List[str], to_table: str,) -> None: - """ - Substitute hyphens ('-') with descending order. - Construct actual sqlalchemy text clause using aliased table and column name. - - :param condition: list of parts of a current condition split by '__' - :type condition: List[str] - :param to_table: target table - :type to_table: sqlalchemy.sql.elements.quoted_name - """ - direction = f"{'desc' if condition[0][0] == '-' else ''}" - column_alias = self.next_model.get_column_alias(condition[-1]) - order = text(f"{self.next_alias}_{to_table}.{column_alias} {direction}") - self.sorted_orders["__".join(condition)] = order - - def get_order_bys(self, to_table: str, pkname_alias: str,) -> None: # noqa: CCR001 + def _get_order_bys(self) -> None: # noqa: CCR001 """ Triggers construction of order bys if they are given. Otherwise by default each table is sorted by a primary key column asc. - - :param to_table: target table - :type to_table: sqlalchemy.sql.elements.quoted_name - :param pkname_alias: alias of the primary key column - :type pkname_alias: str """ alias = self.next_alias if self.order_columns: current_table_sorted = False - split_order_columns = [ - x.split("__") for x in self.order_columns if "__" in x - ] - for condition in split_order_columns: - if self._check_if_condition_apply(condition, self.relation_name): + for condition in self.order_columns: + if condition.check_if_filter_apply( + target_model=self.next_model, alias=alias + ): current_table_sorted = True - self.set_aliased_order_by( - condition=condition, to_table=to_table, - ) - if not current_table_sorted: - order = text(f"{alias}_{to_table}.{pkname_alias}") - self.sorted_orders[f"{alias}.{pkname_alias}"] = order + self.sorted_orders[condition] = condition.get_text_clause() + if not current_table_sorted and not self.target_field.is_multi: + self._set_default_primary_key_order_by() - else: - order = text(f"{alias}_{to_table}.{pkname_alias}") - self.sorted_orders[f"{alias}.{pkname_alias}"] = order + elif not self.target_field.is_multi: + self._set_default_primary_key_order_by() - def get_to_and_from_keys(self) -> Tuple[str, str]: + def _get_to_and_from_keys(self) -> Tuple[str, str]: """ Based on the relation type, name of the relation and previous models and parts stored in JoinParameters it resolves the current to and from keys, which are @@ -396,7 +352,7 @@ class SqlJoin: :rtype: Tuple[str, str] """ if self.target_field.is_multi: - to_key = self.process_m2m_related_name_change(reverse=True) + to_key = self._process_m2m_related_name_change(reverse=True) from_key = self.main_model.get_column_alias(self.main_model.Meta.pkname) elif self.target_field.virtual: diff --git a/ormar/queryset/prefetch_query.py b/ormar/queryset/prefetch_query.py index 7abf4c6..08d2675 100644 --- a/ormar/queryset/prefetch_query.py +++ b/ormar/queryset/prefetch_query.py @@ -20,6 +20,7 @@ from ormar.queryset.utils import extract_models_to_dict_of_lists, translate_list if TYPE_CHECKING: # pragma: no cover from ormar import Model from ormar.fields import ForeignKeyField, BaseField + from ormar.queryset import OrderAction def add_relation_field_to_fields( @@ -128,7 +129,7 @@ class PrefetchQuery: exclude_fields: Optional[Union[Dict, Set]], prefetch_related: List, select_related: List, - orders_by: List, + orders_by: List["OrderAction"], ) -> None: self.model = model_cls @@ -141,7 +142,9 @@ class PrefetchQuery: self.models: Dict = {} self.select_dict = translate_list_to_dict(self._select_related) self.orders_by = orders_by or [] - self.order_dict = translate_list_to_dict(self.orders_by, is_order=True) + self.order_dict = translate_list_to_dict( + [x.query_str for x in self.orders_by], is_order=True + ) async def prefetch_related( self, models: Sequence["Model"], rows: List diff --git a/ormar/queryset/query.py b/ormar/queryset/query.py index edb28c1..7c5a211 100644 --- a/ormar/queryset/query.py +++ b/ormar/queryset/query.py @@ -8,11 +8,12 @@ from sqlalchemy import text import ormar # noqa I100 from ormar.models.helpers.models import group_related_list from ormar.queryset import FilterQuery, LimitQuery, OffsetQuery, OrderQuery -from ormar.queryset.filter_action import FilterAction +from ormar.queryset.actions.filter_action import FilterAction from ormar.queryset.join import SqlJoin if TYPE_CHECKING: # pragma no cover from ormar import Model + from ormar.queryset import OrderAction class Query: @@ -26,7 +27,7 @@ class Query: offset: Optional[int], fields: Optional[Union[Dict, Set]], exclude_fields: Optional[Union[Dict, Set]], - order_bys: Optional[List], + order_bys: Optional[List["OrderAction"]], limit_raw_sql: bool, ) -> None: self.query_offset = offset @@ -45,7 +46,7 @@ class Query: self.select_from: List[str] = [] self.columns = [sqlalchemy.Column] self.order_columns = order_bys - self.sorted_orders: OrderedDict = OrderedDict() + self.sorted_orders: OrderedDict[OrderAction, text] = OrderedDict() self._init_sorted_orders() self.limit_raw_sql = limit_raw_sql @@ -58,28 +59,6 @@ class Query: for clause in self.order_columns: self.sorted_orders[clause] = None - @property - def prefixed_pk_name(self) -> str: - """ - Shortcut for extracting prefixed with alias primary key column name from main - model - :return: alias of pk column prefix with table name. - :rtype: str - """ - pkname_alias = self.model_cls.get_column_alias(self.model_cls.Meta.pkname) - return f"{self.table.name}.{pkname_alias}" - - def alias(self, name: str) -> str: - """ - Shortcut to extracting column alias from given master model. - - :param name: name of column - :type name: str - :return: alias of given column name - :rtype: str - """ - return self.model_cls.get_column_alias(name) - def apply_order_bys_for_primary_model(self) -> None: # noqa: CCR001 """ Applies order_by queries on main model when it's used as a subquery. @@ -88,16 +67,13 @@ class Query: """ if self.order_columns: for clause in self.order_columns: - if "__" not in clause: - text_clause = ( - text(f"{self.table.name}.{self.alias(clause[1:])} desc") - if clause.startswith("-") - else text(f"{self.table.name}.{self.alias(clause)}") - ) - self.sorted_orders[clause] = text_clause + if clause.is_source_model_order: + self.sorted_orders[clause] = clause.get_text_clause() else: - order = text(self.prefixed_pk_name) - self.sorted_orders[self.prefixed_pk_name] = order + clause = ormar.OrderAction( + order_str=self.model_cls.Meta.pkname, model_cls=self.model_cls + ) + self.sorted_orders[clause] = clause.get_text_clause() def _pagination_query_required(self) -> bool: """ @@ -208,7 +184,9 @@ class Query: for filter_clause in self.exclude_clauses if filter_clause.table_prefix == "" ] - sorts_to_use = {k: v for k, v in self.sorted_orders.items() if "__" not in k} + sorts_to_use = { + k: v for k, v in self.sorted_orders.items() if k.is_source_model_order + } expr = FilterQuery(filter_clauses=filters_to_use).apply(expr) expr = FilterQuery(filter_clauses=excludes_to_use, exclude=True).apply(expr) expr = OrderQuery(sorted_orders=sorts_to_use).apply(expr) diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index ba55586..2202abd 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -21,6 +21,7 @@ import ormar # noqa I100 from ormar import MultipleMatches, NoMatch from ormar.exceptions import ModelError, ModelPersistenceError, QueryDefinitionError from ormar.queryset import FilterQuery +from ormar.queryset.actions.order_action import OrderAction from ormar.queryset.clause import QueryClause from ormar.queryset.prefetch_query import PrefetchQuery from ormar.queryset.query import Query @@ -514,7 +515,12 @@ class QuerySet(Generic[T]): if not isinstance(columns, list): columns = [columns] - order_bys = self.order_bys + [x for x in columns if x not in self.order_bys] + orders_by = [ + OrderAction(order_str=x, model_cls=self.model_cls) # type: ignore + for x in columns + ] + + order_bys = self.order_bys + [x for x in orders_by if x not in self.order_bys] return self.__class__( model_cls=self.model, filter_clauses=self.filter_clauses, @@ -713,7 +719,14 @@ class QuerySet(Generic[T]): return await self.filter(**kwargs).first() expr = self.build_select_expression( - limit=1, order_bys=[f"{self.model.Meta.pkname}"] + self.order_bys + limit=1, + order_bys=[ + OrderAction( + order_str=f"{self.model.Meta.pkname}", + model_cls=self.model_cls, # type: ignore + ) + ] + + self.order_bys, ) rows = await self.database.fetch_all(expr) processed_rows = self._process_query_result_rows(rows) @@ -742,7 +755,14 @@ class QuerySet(Generic[T]): if not self.filter_clauses: expr = self.build_select_expression( - limit=1, order_bys=[f"-{self.model.Meta.pkname}"] + self.order_bys + limit=1, + order_bys=[ + OrderAction( + order_str=f"-{self.model.Meta.pkname}", + model_cls=self.model_cls, # type: ignore + ) + ] + + self.order_bys, ) else: expr = self.build_select_expression() diff --git a/ormar/queryset/utils.py b/ormar/queryset/utils.py index 6b98028..9445fe0 100644 --- a/ormar/queryset/utils.py +++ b/ormar/queryset/utils.py @@ -232,16 +232,24 @@ def get_relationship_alias_model_and_str( is_through = False model_cls = source_model previous_model = model_cls + previous_models = [model_cls] manager = model_cls.Meta.alias_manager for relation in related_parts[:]: related_field = model_cls.Meta.model_fields[relation] + if related_field.is_through: + # through is always last - cannot go further is_through = True - related_parts = [ - x.replace(relation, related_field.related_name) if x == relation else x - for x in related_parts + related_parts.remove(relation) + through_field = related_field.owner.Meta.model_fields[ + related_field.related_name or "" ] - relation = related_field.related_name + if len(previous_models) > 1 and previous_models[-2] == through_field.to: + previous_model = through_field.to + relation = through_field.related_name + else: + relation = related_field.related_name + if related_field.is_multi: previous_model = related_field.through relation = related_field.default_target_field_name() # type: ignore @@ -250,6 +258,8 @@ def get_relationship_alias_model_and_str( ) model_cls = related_field.to previous_model = model_cls + if not is_through: + previous_models.append(previous_model) relation_str = "__".join(related_parts) return table_prefix, model_cls, relation_str, is_through diff --git a/tests/test_m2m_through_fields.py b/tests/test_m2m_through_fields.py index 9d7f38a..5c1b44f 100644 --- a/tests/test_m2m_through_fields.py +++ b/tests/test_m2m_through_fields.py @@ -34,6 +34,14 @@ class PostCategory(ormar.Model): param_name: str = ormar.String(default="Name", max_length=200) +class Blog(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + title: str = ormar.String(max_length=200) + + class Post(ormar.Model): class Meta(BaseMeta): pass @@ -41,6 +49,7 @@ class Post(ormar.Model): id: int = ormar.Integer(primary_key=True) title: str = ormar.String(max_length=200) categories = ormar.ManyToMany(Category, through=PostCategory) + blog = ormar.ForeignKey(Blog) @pytest.fixture(autouse=True, scope="module") @@ -146,18 +155,86 @@ async def test_filtering_by_through_model() -> Any: ) post2 = ( - await Post.objects.filter(postcategory__sort_order__gt=1) - .select_related("categories") - .get() + await Post.objects.select_related("categories") + .filter(postcategory__sort_order__gt=1) + .get() ) assert len(post2.categories) == 1 assert post2.categories[0].postcategory.sort_order == 2 post3 = await Post.objects.filter( - categories__postcategory__param_name="volume").get() + categories__postcategory__param_name="volume" + ).get() assert len(post3.categories) == 1 assert post3.categories[0].postcategory.param_name == "volume" + +@pytest.mark.asyncio +async def test_deep_filtering_by_through_model() -> Any: + async with database: + blog = await Blog(title="My Blog").save() + post = await Post(title="Test post", blog=blog).save() + + await post.categories.create( + name="Test category1", + postcategory={"sort_order": 1, "param_name": "volume"}, + ) + await post.categories.create( + name="Test category2", postcategory={"sort_order": 2, "param_name": "area"} + ) + + blog2 = ( + await Blog.objects.select_related("posts__categories") + .filter(posts__postcategory__sort_order__gt=1) + .get() + ) + assert len(blog2.posts) == 1 + assert len(blog2.posts[0].categories) == 1 + assert blog2.posts[0].categories[0].postcategory.sort_order == 2 + + blog3 = await Blog.objects.filter( + posts__categories__postcategory__param_name="volume" + ).get() + assert len(blog3.posts) == 1 + assert len(blog3.posts[0].categories) == 1 + assert blog3.posts[0].categories[0].postcategory.param_name == "volume" + + +@pytest.mark.asyncio +async def test_ordering_by_through_model() -> Any: + async with database: + post = await Post(title="Test post").save() + await post.categories.create( + name="Test category1", + postcategory={"sort_order": 2, "param_name": "volume"}, + ) + await post.categories.create( + name="Test category2", postcategory={"sort_order": 1, "param_name": "area"} + ) + await post.categories.create( + name="Test category3", + postcategory={"sort_order": 3, "param_name": "velocity"}, + ) + + post2 = ( + await Post.objects.select_related("categories") + .order_by("-postcategory__sort_order") + .get() + ) + assert len(post2.categories) == 3 + assert post2.categories[0].name == "Test category3" + assert post2.categories[2].name == "Test category2" + + post3 = ( + await Post.objects.select_related("categories") + .order_by("categories__postcategory__param_name") + .get() + ) + assert len(post3.categories) == 3 + assert post3.categories[0].postcategory.param_name == "area" + assert post3.categories[2].postcategory.param_name == "volume" + + # TODO: check/ modify following # add to fields with class lower name (V) @@ -166,10 +243,12 @@ async def test_filtering_by_through_model() -> Any: # creating in queryset proxy (dict with through name and kwargs) (V) # loading the data into model instance of though model (V) <- fix fields ane exclude # accessing from instance (V) <- no both sides only nested one is relevant, fix one side -# filtering in filter (through name normally) (V) < - table prefix from normal relation, check if is_through needed +# filtering in filter (through name normally) (V) < - table prefix from normal relation, +# check if is_through needed, resolved side of relation +# ordering by in order_by + # updating in query -# ordering by in order_by # modifying from instance (both sides?) # including/excluding in fields? # allowing to change fk fields names in through model? From 7bf781098fc1134b688d9867ff7606d62c3ef886 Mon Sep 17 00:00:00 2001 From: collerek Date: Fri, 26 Feb 2021 11:28:44 +0100 Subject: [PATCH 06/14] add update to queryset, add update_through_instance, start to update docs --- docs/releases.md | 24 ++++ ormar/models/mixins/prefetch_mixin.py | 10 +- ormar/protocols/queryset_protocol.py | 3 + ormar/queryset/prefetch_query.py | 1 + ormar/queryset/queryset.py | 12 +- ormar/relations/querysetproxy.py | 51 ++++++++- tests/test_m2m_through_fields.py | 65 ++++++++++- tests/test_queryproxy_on_m2m_models.py | 40 +++++++ tests/test_wekref_exclusion.py | 147 +++++++++++++++++++++++++ 9 files changed, 339 insertions(+), 14 deletions(-) create mode 100644 tests/test_wekref_exclusion.py diff --git a/docs/releases.md b/docs/releases.md index 30e6fcb..9d98f9f 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,3 +1,27 @@ +# 0.9.5 + +## Features +* Add `update` method to `QuerysetProxy` so now it's possible to update related models directly from parent model + in `ManyToMany` relations and in reverse `ForeignKey` relations. Note that update like in `QuerySet` `update` returns number of + updated models and **does not update related models in place** on praent model. To get the refreshed data on parent model you need to refresh + the related models (i.e. `await model_instance.related.all()`) +* Added possibility to add more fields on `Through` model for `ManyToMany` relationships: + * name of the through model field is the lowercase name of the Through class + * you can pass additional fields when calling `add(child, **kwargs)` on relation (on `QuerysetProxy`) + * you can pass additional fields when calling `create(**kwargs)` on relation (on `QuerysetProxy`) + when one of the keyword arguments should be the through model name with a dict of values + * you can order by on through model fields + * you can filter on through model fields + * you can include and exclude fields on through models + * through models are attached only to related models (i.e. if you query from A to B -> only on B) + * check the updated docs for more information + +# Other +* Updated docs and api docs +* Refactors and optimisations mainly related to filters and order bys + + + # 0.9.4 ## Fixes diff --git a/ormar/models/mixins/prefetch_mixin.py b/ormar/models/mixins/prefetch_mixin.py index 440052a..85faec2 100644 --- a/ormar/models/mixins/prefetch_mixin.py +++ b/ormar/models/mixins/prefetch_mixin.py @@ -18,10 +18,10 @@ class PrefetchQueryMixin(RelationMixin): @staticmethod def get_clause_target_and_filter_column_name( - parent_model: Type["Model"], - target_model: Type["Model"], - reverse: bool, - related: str, + parent_model: Type["Model"], + target_model: Type["Model"], + reverse: bool, + related: str, ) -> Tuple[Type["Model"], str]: """ Returns Model on which query clause should be performed and name of the column. @@ -51,7 +51,7 @@ class PrefetchQueryMixin(RelationMixin): @staticmethod def get_column_name_for_id_extraction( - parent_model: Type["Model"], reverse: bool, related: str, use_raw: bool, + parent_model: Type["Model"], reverse: bool, related: str, use_raw: bool, ) -> str: """ Returns name of the column that should be used to extract ids from model. diff --git a/ormar/protocols/queryset_protocol.py b/ormar/protocols/queryset_protocol.py index 7eb7092..397f58b 100644 --- a/ormar/protocols/queryset_protocol.py +++ b/ormar/protocols/queryset_protocol.py @@ -52,6 +52,9 @@ class QuerySetProtocol(Protocol): # pragma: nocover async def create(self, **kwargs: Any) -> "Model": ... + async def update(self, each: bool = False, **kwargs: Any) -> int: + ... + async def get_or_create(self, **kwargs: Any) -> "Model": ... diff --git a/ormar/queryset/prefetch_query.py b/ormar/queryset/prefetch_query.py index 08d2675..533f92c 100644 --- a/ormar/queryset/prefetch_query.py +++ b/ormar/queryset/prefetch_query.py @@ -142,6 +142,7 @@ class PrefetchQuery: self.models: Dict = {} self.select_dict = translate_list_to_dict(self._select_related) self.orders_by = orders_by or [] + # TODO: refactor OrderActions to use it instead of strings from it self.order_dict = translate_list_to_dict( [x.query_str for x in self.orders_by], is_order=True ) diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 2202abd..46d679a 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -573,17 +573,19 @@ class QuerySet(Generic[T]): :return: number of updated rows :rtype: int """ + if not each and not self.filter_clauses: + raise QueryDefinitionError( + "You cannot update without filtering the queryset first. " + "If you want to update all rows use update(each=True, **kwargs)" + ) + self_fields = self.model.extract_db_own_fields().union( self.model.extract_related_names() ) updates = {k: v for k, v in kwargs.items() if k in self_fields} updates = self.model.validate_choices(updates) updates = self.model.translate_columns_to_aliases(updates) - if not each and not self.filter_clauses: - raise QueryDefinitionError( - "You cannot update without filtering the queryset first. " - "If you want to update all rows use update(each=True, **kwargs)" - ) + expr = FilterQuery(filter_clauses=self.filter_clauses).apply( self.table.update().values(**updates) ) diff --git a/ormar/relations/querysetproxy.py b/ormar/relations/querysetproxy.py index 85b7832..157e72c 100644 --- a/ormar/relations/querysetproxy.py +++ b/ormar/relations/querysetproxy.py @@ -14,7 +14,7 @@ from typing import ( ) import ormar -from ormar.exceptions import ModelPersistenceError +from ormar.exceptions import ModelPersistenceError, QueryDefinitionError if TYPE_CHECKING: # pragma no cover from ormar.relations import Relation @@ -132,6 +132,22 @@ class QuerysetProxy(Generic[T]): # print("\n", expr.compile(compile_kwargs={"literal_binds": True})) await model_cls.Meta.database.execute(expr) + async def update_through_instance(self, child: "T", **kwargs: Any) -> None: + """ + Updates a through model instance in the database for m2m relations. + + :param kwargs: dict of additional keyword arguments for through instance + :type kwargs: Any + :param child: child model instance + :type child: Model + """ + model_cls = self.relation.through + owner_column = self.related_field.default_target_field_name() # type: ignore + child_column = self.related_field.default_source_field_name() # type: ignore + rel_kwargs = {owner_column: self._owner.pk, child_column: child.pk} + through_model = await model_cls.objects.get(**rel_kwargs) + await through_model.update(**kwargs) + async def delete_through_instance(self, child: "T") -> None: """ Removes through model instance from the database for m2m relations. @@ -290,6 +306,39 @@ class QuerysetProxy(Generic[T]): await self.create_through_instance(created, **through_kwargs) return created + async def update(self, each: bool = False, **kwargs: Any) -> int: + """ + Updates the model table after applying the filters from kwargs. + + You have to either pass a filter to narrow down a query or explicitly pass + each=True flag to affect whole table. + + :param each: flag if whole table should be affected if no filter is passed + :type each: bool + :param kwargs: fields names and proper value types + :type kwargs: Any + :return: number of updated rows + :rtype: int + """ + # queryset proxy always have one filter for pk of parent model + if not each and len(self.queryset.filter_clauses) == 1: + raise QueryDefinitionError( + "You cannot update without filtering the queryset first. " + "If you want to update all rows use update(each=True, **kwargs)" + ) + + through_kwargs = kwargs.pop(self.through_model_name, {}) + children = await self.queryset.all() + for child in children: + if child: + await child.update(**kwargs) + if self.type_ == ormar.RelationType.MULTIPLE and through_kwargs: + await self.update_through_instance( + child=child, # type: ignore + **through_kwargs, + ) + return len(children) + async def get_or_create(self, **kwargs: Any) -> "T": """ Combination of create and get methods. diff --git a/tests/test_m2m_through_fields.py b/tests/test_m2m_through_fields.py index 5c1b44f..898f103 100644 --- a/tests/test_m2m_through_fields.py +++ b/tests/test_m2m_through_fields.py @@ -235,6 +235,64 @@ async def test_ordering_by_through_model() -> Any: assert post3.categories[2].postcategory.param_name == "volume" +@pytest.mark.asyncio +async def test_update_through_models_from_queryset_on_through() -> Any: + async with database: + post = await Post(title="Test post").save() + await post.categories.create( + name="Test category1", + postcategory={"sort_order": 2, "param_name": "volume"}, + ) + await post.categories.create( + name="Test category2", postcategory={"sort_order": 1, "param_name": "area"} + ) + await post.categories.create( + name="Test category3", + postcategory={"sort_order": 3, "param_name": "velocity"}, + ) + + await PostCategory.objects.filter(param_name="volume", post=post.id).update( + sort_order=4 + ) + post2 = ( + await Post.objects.select_related("categories") + .order_by("-postcategory__sort_order") + .get() + ) + assert len(post2.categories) == 3 + assert post2.categories[0].postcategory.param_name == "volume" + assert post2.categories[2].postcategory.param_name == "area" + + +@pytest.mark.asyncio +async def test_update_through_from_related() -> Any: + async with database: + post = await Post(title="Test post").save() + await post.categories.create( + name="Test category1", + postcategory={"sort_order": 2, "param_name": "volume"}, + ) + await post.categories.create( + name="Test category2", postcategory={"sort_order": 1, "param_name": "area"} + ) + await post.categories.create( + name="Test category3", + postcategory={"sort_order": 3, "param_name": "velocity"}, + ) + + await post.categories.filter(name="Test category3").update( + postcategory={"sort_order": 4} + ) + + post2 = ( + await Post.objects.select_related("categories") + .order_by("postcategory__sort_order") + .get() + ) + assert len(post2.categories) == 3 + assert post2.categories[2].postcategory.sort_order == 4 + + # TODO: check/ modify following # add to fields with class lower name (V) @@ -245,11 +303,12 @@ async def test_ordering_by_through_model() -> Any: # accessing from instance (V) <- no both sides only nested one is relevant, fix one side # filtering in filter (through name normally) (V) < - table prefix from normal relation, # check if is_through needed, resolved side of relation -# ordering by in order_by +# ordering by in order_by (V) +# updating in query (V) +# updating from querysetproxy (V) +# modifying from instance (both sides?) (X) <= no, the loaded one doesn't have relations -# updating in query -# modifying from instance (both sides?) # including/excluding in fields? # allowing to change fk fields names in through model? # make through optional? auto-generated for cases other fields are missing? diff --git a/tests/test_queryproxy_on_m2m_models.py b/tests/test_queryproxy_on_m2m_models.py index d33aa5d..a91c4f8 100644 --- a/tests/test_queryproxy_on_m2m_models.py +++ b/tests/test_queryproxy_on_m2m_models.py @@ -6,6 +6,7 @@ import pytest import sqlalchemy import ormar +from ormar.exceptions import QueryDefinitionError from tests.settings import DATABASE_URL database = databases.Database(DATABASE_URL, force_rollback=True) @@ -180,3 +181,42 @@ async def test_queryset_methods(): assert len(categories) == 3 == len(post.categories) for cat in post.categories: assert cat.subject.name is not None + + +@pytest.mark.asyncio +async def test_queryset_update(): + async with database: + async with database.transaction(force_rollback=True): + guido = await Author.objects.create( + first_name="Guido", last_name="Van Rossum" + ) + subject = await Subject(name="Random").save() + post = await Post.objects.create(title="Hello, M2M", author=guido) + await post.categories.create(name="News", sort_order=1, subject=subject) + await post.categories.create(name="Breaking", sort_order=3, subject=subject) + + await post.categories.order_by("sort_order").all() + assert len(post.categories) == 2 + assert post.categories[0].sort_order == 1 + assert post.categories[0].name == "News" + assert post.categories[1].sort_order == 3 + assert post.categories[1].name == "Breaking" + + updated = await post.categories.update(each=True, name="Test") + assert updated == 2 + + await post.categories.order_by("sort_order").all() + assert len(post.categories) == 2 + assert post.categories[0].name == "Test" + assert post.categories[1].name == "Test" + + updated = await post.categories.filter(sort_order=3).update(name="Test 2") + assert updated == 1 + + await post.categories.order_by("sort_order").all() + assert len(post.categories) == 2 + assert post.categories[0].name == "Test" + assert post.categories[1].name == "Test 2" + + with pytest.raises(QueryDefinitionError): + await post.categories.update(name="Test WRONG") diff --git a/tests/test_wekref_exclusion.py b/tests/test_wekref_exclusion.py new file mode 100644 index 0000000..a1140f7 --- /dev/null +++ b/tests/test_wekref_exclusion.py @@ -0,0 +1,147 @@ +from typing import List, Optional +from uuid import UUID, uuid4 + +import databases +import pydantic +import pytest +import sqlalchemy +from fastapi import FastAPI +from starlette.testclient import TestClient + +import ormar +from tests.settings import DATABASE_URL + +app = FastAPI() + +database = databases.Database(DATABASE_URL, force_rollback=True) +metadata = sqlalchemy.MetaData() + +app.state.database = database + + +@app.on_event("startup") +async def startup() -> None: + database_ = app.state.database + if not database_.is_connected: + await database_.connect() + + +@app.on_event("shutdown") +async def shutdown() -> None: + database_ = app.state.database + if database_.is_connected: + await database_.disconnect() + + +@pytest.fixture(autouse=True, scope="module") +def create_test_database(): + engine = sqlalchemy.create_engine(DATABASE_URL) + metadata.create_all(engine) + yield + metadata.drop_all(engine) + + +class BaseMeta(ormar.ModelMeta): + database = database + metadata = metadata + + +class OtherThing(ormar.Model): + class Meta(BaseMeta): + tablename = "other_things" + + id: UUID = ormar.UUID(primary_key=True, default=uuid4) + name: str = ormar.Text(default="") + ot_contents: str = ormar.Text(default="") + + +class Thing(ormar.Model): + class Meta(BaseMeta): + tablename = "things" + + id: UUID = ormar.UUID(primary_key=True, default=uuid4) + name: str = ormar.Text(default="") + js: pydantic.Json = ormar.JSON(nullable=True) + other_thing: Optional[OtherThing] = ormar.ForeignKey(OtherThing, nullable=True) + + +@app.post("/test/1") +async def post_test_1(): + # don't split initialization and attribute assignment + ot = await OtherThing(ot_contents="otc").save() + await Thing(other_thing=ot, name="t1").save() + await Thing(other_thing=ot, name="t2").save() + await Thing(other_thing=ot, name="t3").save() + + # if you do not care about returned object you can even go with bulk_create + # all of them are created in one transaction + # things = [Thing(other_thing=ot, name='t1'), + # Thing(other_thing=ot, name="t2"), + # Thing(other_thing=ot, name="t3")] + # await Thing.objects.bulk_create(things) + + +@app.get("/test/2", response_model=List[Thing]) +async def get_test_2(): + # if you only query for one use get or first + ot = await OtherThing.objects.get() + ts = await ot.things.all() + # specifically null out the relation on things before return + for t in ts: + t.remove(ot, name="other_thing") + return ts + + +@app.get("/test/3", response_model=List[Thing]) +async def get_test_3(): + ot = await OtherThing.objects.select_related("things").get() + # exclude unwanted field while ot is still in scope + # in order not to pass it to fastapi + return [t.dict(exclude={"other_thing"}) for t in ot.things] + + +@app.get("/test/4", response_model=List[Thing], response_model_exclude={"other_thing"}) +async def get_test_4(): + ot = await OtherThing.objects.get() + # query from the active side + return await Thing.objects.all(other_thing=ot) + + +@app.get("/get_ot/", response_model=OtherThing) +async def get_ot(): + return await OtherThing.objects.get() + + +# more real life (usually) is not getting some random OT and get it's Things +# but query for a specific one by some kind of id +@app.get( + "/test/5/{thing_id}", + response_model=List[Thing], + response_model_exclude={"other_thing"}, +) +async def get_test_5(thing_id: UUID): + return await Thing.objects.all(other_thing__id=thing_id) + + +def test_endpoints(): + client = TestClient(app) + with client: + resp = client.post("/test/1") + assert resp.status_code == 200 + + resp2 = client.get("/test/2") + assert resp2.status_code == 200 + assert len(resp2.json()) == 3 + + resp3 = client.get("/test/3") + assert resp3.status_code == 200 + assert len(resp3.json()) == 3 + + resp4 = client.get("/test/4") + assert resp4.status_code == 200 + assert len(resp4.json()) == 3 + + ot = OtherThing(**client.get("/get_ot/").json()) + resp5 = client.get(f"/test/5/{ot.id}") + assert resp5.status_code == 200 + assert len(resp5.json()) == 3 From ad9d065c6d40113987fa4e04daf7a8d07990cda0 Mon Sep 17 00:00:00 2001 From: collerek Date: Fri, 26 Feb 2021 17:47:52 +0100 Subject: [PATCH 07/14] start to refactor fields and eclude_fields into ExcludableItems to simplify access --- ormar/models/excludable.py | 162 ++++++++++++++++ ormar/queryset/actions/filter_action.py | 3 + ormar/queryset/query.py | 4 +- ormar/queryset/queryset.py | 1 + ormar/queryset/utils.py | 16 +- ormar/relations/querysetproxy.py | 13 +- tests/test_excludable_items.py | 218 ++++++++++++++++++++++ tests/test_m2m_through_fields.py | 33 +++- tests/test_selecting_subset_of_columns.py | 4 +- 9 files changed, 434 insertions(+), 20 deletions(-) create mode 100644 ormar/models/excludable.py create mode 100644 tests/test_excludable_items.py diff --git a/ormar/models/excludable.py b/ormar/models/excludable.py new file mode 100644 index 0000000..b754eb4 --- /dev/null +++ b/ormar/models/excludable.py @@ -0,0 +1,162 @@ +from dataclasses import dataclass, field +from typing import Dict, List, Set, TYPE_CHECKING, Tuple, Type, Union + +from ormar.queryset.utils import get_relationship_alias_model_and_str + +if TYPE_CHECKING: # pragma: no cover + from ormar import Model + + +@dataclass +class Excludable: + include: Set = field(default_factory=set) + exclude: Set = field(default_factory=set) + + def set_values(self, value: Set, is_exclude: bool) -> None: + prop = "exclude" if is_exclude else "include" + if ... in getattr(self, prop) or ... in value: + setattr(self, prop, {...}) + else: + current_value = getattr(self, prop) + current_value.update(value) + setattr(self, prop, current_value) + + def is_included(self, key: str) -> bool: + return (... in self.include or key in self.include) if self.include else True + + def is_excluded(self, key: str) -> bool: + return (... in self.exclude or key in self.exclude) if self.exclude else False + + +class ExcludableItems: + """ + Keeps a dictionary of Excludables by alias + model_name keys + to allow quick lookup by nested models without need to travers + deeply nested dictionaries and passing include/exclude around + """ + + def __init__(self) -> None: + self.items: Dict[str, Excludable] = dict() + + def get(self, model_cls: Type["Model"], alias: str = "") -> Excludable: + key = f"{alias + '_' if alias else ''}{model_cls.get_name(lower=True)}" + return self.items.get(key, Excludable()) + + def build( + self, + items: Union[List[str], str, Tuple[str], Set[str], Dict], + model_cls: Type["Model"], + is_exclude: bool = False, + ) -> None: + + if isinstance(items, str): + items = {items} + + if isinstance(items, Dict): + self._traverse_dict( + values=items, + source_model=model_cls, + model_cls=model_cls, + is_exclude=is_exclude, + ) + + else: + items = set(items) + nested_items = set(x for x in items if "__" in x) + items.difference_update(nested_items) + self._set_excludes( + items=items, + model_name=model_cls.get_name(lower=True), + is_exclude=is_exclude, + ) + if nested_items: + self._traverse_list( + values=nested_items, model_cls=model_cls, is_exclude=is_exclude + ) + + def _set_excludes( + self, items: Set, model_name: str, is_exclude: bool, alias: str = "" + ) -> None: + + key = f"{alias + '_' if alias else ''}{model_name}" + excludable = self.items.get(key) + if not excludable: + excludable = Excludable() + excludable.set_values(value=items, is_exclude=is_exclude) + self.items[key] = excludable + + def _traverse_dict( # noqa: CFQ002 + self, + values: Dict, + source_model: Type["Model"], + model_cls: Type["Model"], + is_exclude: bool, + related_items: List = None, + alias: str = "", + ) -> None: + + self_fields = set() + related_items = related_items[:] if related_items else [] + for key, value in values.items(): + if value is ...: + self_fields.add(key) + elif isinstance(value, set): + related_items.append(key) + ( + table_prefix, + target_model, + _, + _, + ) = get_relationship_alias_model_and_str( + source_model=source_model, related_parts=related_items + ) + self._set_excludes( + items=value, + model_name=target_model.get_name(), + is_exclude=is_exclude, + alias=table_prefix, + ) + else: + # dict + related_items.append(key) + ( + table_prefix, + target_model, + _, + _, + ) = get_relationship_alias_model_and_str( + source_model=source_model, related_parts=related_items + ) + self._traverse_dict( + values=value, + source_model=source_model, + model_cls=target_model, + is_exclude=is_exclude, + related_items=related_items, + alias=table_prefix, + ) + if self_fields: + self._set_excludes( + items=self_fields, + model_name=model_cls.get_name(), + is_exclude=is_exclude, + alias=alias, + ) + + def _traverse_list( + self, values: Set[str], model_cls: Type["Model"], is_exclude: bool + ) -> None: + + # here we have only nested related keys + for key in values: + key_split = key.split("__") + related_items, field_name = key_split[:-1], key_split[-1] + (table_prefix, target_model, _, _) = get_relationship_alias_model_and_str( + source_model=model_cls, related_parts=related_items + ) + self._set_excludes( + items={field_name}, + model_name=target_model.get_name(), + is_exclude=is_exclude, + alias=table_prefix, + ) diff --git a/ormar/queryset/actions/filter_action.py b/ormar/queryset/actions/filter_action.py index 43c71df..ed6277d 100644 --- a/ormar/queryset/actions/filter_action.py +++ b/ormar/queryset/actions/filter_action.py @@ -42,6 +42,9 @@ class FilterAction(QueryAction): super().__init__(query_str=filter_str, model_cls=model_cls) self.filter_value = value self._escape_characters_in_clause() + self.is_source_model_filter = False + if self.source_model == self.target_model and "__" not in self.related_str: + self.is_source_model_filter = True def has_escaped_characters(self) -> bool: """Check if value is a string that contains characters to escape""" diff --git a/ormar/queryset/query.py b/ormar/queryset/query.py index 7c5a211..d6b10d2 100644 --- a/ormar/queryset/query.py +++ b/ormar/queryset/query.py @@ -177,12 +177,12 @@ class Query: filters_to_use = [ filter_clause for filter_clause in self.filter_clauses - if filter_clause.table_prefix == "" + if filter_clause.is_source_model_filter ] excludes_to_use = [ filter_clause for filter_clause in self.exclude_clauses - if filter_clause.table_prefix == "" + if filter_clause.is_source_model_filter ] sorts_to_use = { k: v for k, v in self.sorted_orders.items() if k.is_source_model_order diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 46d679a..7c664ac 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -410,6 +410,7 @@ class QuerySet(Generic[T]): if isinstance(columns, str): columns = [columns] + # TODO: Flatten all excludes into one dict-like structure with alias + model key current_included = self._columns if not isinstance(columns, dict): current_included = update_dict_from_list(current_included, columns) diff --git a/ormar/queryset/utils.py b/ormar/queryset/utils.py index 9445fe0..ca3358d 100644 --- a/ormar/queryset/utils.py +++ b/ormar/queryset/utils.py @@ -230,12 +230,12 @@ def get_relationship_alias_model_and_str( """ table_prefix = "" is_through = False - model_cls = source_model - previous_model = model_cls - previous_models = [model_cls] - manager = model_cls.Meta.alias_manager + target_model = source_model + previous_model = target_model + previous_models = [target_model] + manager = target_model.Meta.alias_manager for relation in related_parts[:]: - related_field = model_cls.Meta.model_fields[relation] + related_field = target_model.Meta.model_fields[relation] if related_field.is_through: # through is always last - cannot go further @@ -256,10 +256,10 @@ def get_relationship_alias_model_and_str( table_prefix = manager.resolve_relation_alias( from_model=previous_model, relation_name=relation ) - model_cls = related_field.to - previous_model = model_cls + target_model = related_field.to + previous_model = target_model if not is_through: previous_models.append(previous_model) relation_str = "__".join(related_parts) - return table_prefix, model_cls, relation_str, is_through + return table_prefix, target_model, relation_str, is_through diff --git a/ormar/relations/querysetproxy.py b/ormar/relations/querysetproxy.py index 157e72c..952a6c7 100644 --- a/ormar/relations/querysetproxy.py +++ b/ormar/relations/querysetproxy.py @@ -330,13 +330,12 @@ class QuerysetProxy(Generic[T]): through_kwargs = kwargs.pop(self.through_model_name, {}) children = await self.queryset.all() for child in children: - if child: - await child.update(**kwargs) - if self.type_ == ormar.RelationType.MULTIPLE and through_kwargs: - await self.update_through_instance( - child=child, # type: ignore - **through_kwargs, - ) + await child.update(**kwargs) # type: ignore + if self.type_ == ormar.RelationType.MULTIPLE and through_kwargs: + await self.update_through_instance( + child=child, # type: ignore + **through_kwargs, + ) return len(children) async def get_or_create(self, **kwargs: Any) -> "T": diff --git a/tests/test_excludable_items.py b/tests/test_excludable_items.py new file mode 100644 index 0000000..95d1319 --- /dev/null +++ b/tests/test_excludable_items.py @@ -0,0 +1,218 @@ +from typing import List, Optional + +import databases +import sqlalchemy + +import ormar +from ormar.models.excludable import ExcludableItems +from tests.settings import DATABASE_URL + +database = databases.Database(DATABASE_URL, force_rollback=True) +metadata = sqlalchemy.MetaData() + + +class BaseMeta(ormar.ModelMeta): + database = database + metadata = metadata + + +class NickNames(ormar.Model): + class Meta(BaseMeta): + tablename = "nicks" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, nullable=False, name="hq_name") + is_lame: bool = ormar.Boolean(nullable=True) + + +class NicksHq(ormar.Model): + class Meta(BaseMeta): + tablename = "nicks_x_hq" + + +class HQ(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, nullable=False, name="hq_name") + nicks: List[NickNames] = ormar.ManyToMany(NickNames, through=NicksHq) + + +class Company(ormar.Model): + class Meta(BaseMeta): + tablename = "companies" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, nullable=False, name="company_name") + founded: int = ormar.Integer(nullable=True) + hq: HQ = ormar.ForeignKey(HQ) + + +class Car(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + manufacturer: Optional[Company] = ormar.ForeignKey(Company) + name: str = ormar.String(max_length=100) + year: int = ormar.Integer(nullable=True) + gearbox_type: str = ormar.String(max_length=20, nullable=True) + gears: int = ormar.Integer(nullable=True) + aircon_type: str = ormar.String(max_length=20, nullable=True) + + +def compare_results(excludable): + car_excludable = excludable.get(Car) + assert car_excludable.exclude == {"year", "gearbox_type", "gears", "aircon_type"} + assert car_excludable.include == set() + + assert car_excludable.is_excluded("year") + + alias = Company.Meta.alias_manager.resolve_relation_alias(Car, "manufacturer") + manu_excludable = excludable.get(Company, alias=alias) + assert manu_excludable.exclude == {"founded"} + assert manu_excludable.include == set() + + assert manu_excludable.is_excluded("founded") + + +def compare_results_include(excludable): + manager = Company.Meta.alias_manager + car_excludable = excludable.get(Car) + assert car_excludable.include == {"id", "name"} + assert car_excludable.exclude == set() + + assert car_excludable.is_included("name") + assert not car_excludable.is_included("gears") + + alias = manager.resolve_relation_alias(Car, "manufacturer") + manu_excludable = excludable.get(Company, alias=alias) + assert manu_excludable.include == {"name"} + assert manu_excludable.exclude == set() + + assert manu_excludable.is_included("name") + assert not manu_excludable.is_included("founded") + + alias = manager.resolve_relation_alias(Company, "hq") + hq_excludable = excludable.get(HQ, alias=alias) + assert hq_excludable.include == {"name"} + assert hq_excludable.exclude == set() + + alias = manager.resolve_relation_alias(NicksHq, "nicknames") + nick_excludable = excludable.get(NickNames, alias=alias) + assert nick_excludable.include == {"name"} + assert nick_excludable.exclude == set() + + +def test_excluding_fields_from_list(): + fields = [ + "gearbox_type", + "gears", + "aircon_type", + "year", + "manufacturer__founded", + ] + excludable = ExcludableItems() + excludable.build(items=fields, model_cls=Car, is_exclude=True) + compare_results(excludable) + + +def test_excluding_fields_from_dict(): + fields = { + "gearbox_type": ..., + "gears": ..., + "aircon_type": ..., + "year": ..., + "manufacturer": {"founded": ...}, + } + excludable = ExcludableItems() + excludable.build(items=fields, model_cls=Car, is_exclude=True) + compare_results(excludable) + + +def test_excluding_fields_from_dict_with_set(): + fields = { + "gearbox_type": ..., + "gears": ..., + "aircon_type": ..., + "year": ..., + "manufacturer": {"founded"}, + } + excludable = ExcludableItems() + excludable.build(items=fields, model_cls=Car, is_exclude=True) + compare_results(excludable) + + +def test_gradual_build_from_lists(): + fields_col = [ + "year", + ["gearbox_type", "gears"], + "aircon_type", + ["manufacturer__founded"], + ] + excludable = ExcludableItems() + for fields in fields_col: + excludable.build(items=fields, model_cls=Car, is_exclude=True) + compare_results(excludable) + + +def test_nested_includes(): + fields = [ + "id", + "name", + "manufacturer__name", + "manufacturer__hq__name", + "manufacturer__hq__nicks__name", + ] + excludable = ExcludableItems() + excludable.build(items=fields, model_cls=Car, is_exclude=False) + compare_results_include(excludable) + + +def test_nested_includes_from_dict(): + fields = { + "id": ..., + "name": ..., + "manufacturer": {"name": ..., "hq": {"name": ..., "nicks": {"name": ...}},}, + } + excludable = ExcludableItems() + excludable.build(items=fields, model_cls=Car, is_exclude=False) + compare_results_include(excludable) + + +def test_nested_includes_from_dict_with_set(): + fields = { + "id": ..., + "name": ..., + "manufacturer": {"name": ..., "hq": {"name": ..., "nicks": {"name"}},}, + } + excludable = ExcludableItems() + excludable.build(items=fields, model_cls=Car, is_exclude=False) + compare_results_include(excludable) + + +def test_includes_and_excludes_combo(): + fields_inc1 = ["id", "name", "year", "gearbox_type", "gears"] + fields_inc2 = {"manufacturer": {"name"}} + fields_exc1 = {"manufacturer__founded"} + fields_exc2 = "aircon_type" + excludable = ExcludableItems() + excludable.build(items=fields_inc1, model_cls=Car, is_exclude=False) + excludable.build(items=fields_inc2, model_cls=Car, is_exclude=False) + excludable.build(items=fields_exc1, model_cls=Car, is_exclude=True) + excludable.build(items=fields_exc2, model_cls=Car, is_exclude=True) + + car_excludable = excludable.get(Car) + assert car_excludable.include == {"id", "name", "year", "gearbox_type", "gears"} + assert car_excludable.exclude == {"aircon_type"} + + assert car_excludable.is_excluded("aircon_type") + assert car_excludable.is_included("name") + + alias = Company.Meta.alias_manager.resolve_relation_alias(Car, "manufacturer") + manu_excludable = excludable.get(Company, alias=alias) + assert manu_excludable.include == {"name"} + assert manu_excludable.exclude == {"founded"} + + assert manu_excludable.is_excluded("founded") diff --git a/tests/test_m2m_through_fields.py b/tests/test_m2m_through_fields.py index 898f103..8dd8bba 100644 --- a/tests/test_m2m_through_fields.py +++ b/tests/test_m2m_through_fields.py @@ -1,4 +1,4 @@ -from typing import Any, TYPE_CHECKING +from typing import Any import databases import pytest @@ -293,6 +293,37 @@ async def test_update_through_from_related() -> Any: assert post2.categories[2].postcategory.sort_order == 4 +@pytest.mark.asyncio +@pytest.mark.skip # TODO: Restore after finished exclude refactor +async def test_excluding_fields_on_through_model() -> Any: + async with database: + post = await Post(title="Test post").save() + await post.categories.create( + name="Test category1", + postcategory={"sort_order": 2, "param_name": "volume"}, + ) + await post.categories.create( + name="Test category2", postcategory={"sort_order": 1, "param_name": "area"} + ) + await post.categories.create( + name="Test category3", + postcategory={"sort_order": 3, "param_name": "velocity"}, + ) + + post2 = ( + await Post.objects.select_related("categories") + .exclude_fields("postcategory__param_name") + .order_by("postcategory__sort_order") + .get() + ) + assert len(post2.categories) == 3 + assert post2.categories[0].postcategory.param_name is None + assert post2.categories[0].postcategory.sort_order == 1 + + assert post2.categories[2].postcategory.param_name is None + assert post2.categories[2].postcategory.sort_order == 3 + + # TODO: check/ modify following # add to fields with class lower name (V) diff --git a/tests/test_selecting_subset_of_columns.py b/tests/test_selecting_subset_of_columns.py index a2d57db..809b508 100644 --- a/tests/test_selecting_subset_of_columns.py +++ b/tests/test_selecting_subset_of_columns.py @@ -204,8 +204,8 @@ async def test_selecting_subset(): all_cars_dummy = ( await Car.objects.select_related("manufacturer") .fields(["id", "name", "year", "gearbox_type", "gears", "aircon_type"]) - .fields({"manufacturer": ...}) - .exclude_fields({"manufacturer": ...}) + # .fields({"manufacturer": ...}) + # .exclude_fields({"manufacturer": ...}) .fields({"manufacturer": {"name"}}) .exclude_fields({"manufacturer__founded"}) .all() From fd38ae2a40b85cc25d2b04db697224288a7b520c Mon Sep 17 00:00:00 2001 From: collerek Date: Sun, 28 Feb 2021 08:19:02 +0100 Subject: [PATCH 08/14] wip with m2m fields --- ormar/models/excludable.py | 47 +++++--- ormar/models/mixins/excludable_mixin.py | 77 ++++++------- ormar/models/model_row.py | 76 +++++-------- ormar/queryset/join.py | 16 +-- ormar/queryset/prefetch_query.py | 141 +++++++++++------------- ormar/queryset/query.py | 37 +++---- ormar/queryset/queryset.py | 141 +++++++++--------------- 7 files changed, 231 insertions(+), 304 deletions(-) diff --git a/ormar/models/excludable.py b/ormar/models/excludable.py index b754eb4..b8832d2 100644 --- a/ormar/models/excludable.py +++ b/ormar/models/excludable.py @@ -12,6 +12,20 @@ class Excludable: include: Set = field(default_factory=set) exclude: Set = field(default_factory=set) + @property + def include_all(self): + return ... in self.include + + @property + def exclude_all(self): + return ... in self.exclude + + def get_copy(self) -> "Excludable": + _copy = self.__class__() + _copy.include = {x for x in self.include} + _copy.exclude = {x for x in self.exclude} + return _copy + def set_values(self, value: Set, is_exclude: bool) -> None: prop = "exclude" if is_exclude else "include" if ... in getattr(self, prop) or ... in value: @@ -38,15 +52,22 @@ class ExcludableItems: def __init__(self) -> None: self.items: Dict[str, Excludable] = dict() + @classmethod + def from_excludable(cls, other: "ExcludableItems") -> "ExcludableItems": + new_excludable = cls() + for key, value in other.items.items(): + new_excludable.items[key] = value.get_copy() + return new_excludable + def get(self, model_cls: Type["Model"], alias: str = "") -> Excludable: key = f"{alias + '_' if alias else ''}{model_cls.get_name(lower=True)}" return self.items.get(key, Excludable()) def build( - self, - items: Union[List[str], str, Tuple[str], Set[str], Dict], - model_cls: Type["Model"], - is_exclude: bool = False, + self, + items: Union[List[str], str, Tuple[str], Set[str], Dict], + model_cls: Type["Model"], + is_exclude: bool = False, ) -> None: if isinstance(items, str): @@ -75,7 +96,7 @@ class ExcludableItems: ) def _set_excludes( - self, items: Set, model_name: str, is_exclude: bool, alias: str = "" + self, items: Set, model_name: str, is_exclude: bool, alias: str = "" ) -> None: key = f"{alias + '_' if alias else ''}{model_name}" @@ -86,13 +107,13 @@ class ExcludableItems: self.items[key] = excludable def _traverse_dict( # noqa: CFQ002 - self, - values: Dict, - source_model: Type["Model"], - model_cls: Type["Model"], - is_exclude: bool, - related_items: List = None, - alias: str = "", + self, + values: Dict, + source_model: Type["Model"], + model_cls: Type["Model"], + is_exclude: bool, + related_items: List = None, + alias: str = "", ) -> None: self_fields = set() @@ -144,7 +165,7 @@ class ExcludableItems: ) def _traverse_list( - self, values: Set[str], model_cls: Type["Model"], is_exclude: bool + self, values: Set[str], model_cls: Type["Model"], is_exclude: bool ) -> None: # here we have only nested related keys diff --git a/ormar/models/mixins/excludable_mixin.py b/ormar/models/mixins/excludable_mixin.py index 4b096d9..4b25035 100644 --- a/ormar/models/mixins/excludable_mixin.py +++ b/ormar/models/mixins/excludable_mixin.py @@ -9,9 +9,10 @@ from typing import ( TYPE_CHECKING, Type, TypeVar, - Union, + Union, cast, ) +from ormar.models.excludable import ExcludableItems from ormar.models.mixins.relation_mixin import RelationMixin from ormar.queryset.utils import translate_list_to_dict, update @@ -35,7 +36,7 @@ class ExcludableMixin(RelationMixin): @staticmethod def get_child( - items: Union[Set, Dict, None], key: str = None + items: Union[Set, Dict, None], key: str = None ) -> Union[Set, Dict, None]: """ Used to get nested dictionaries keys if they exists otherwise returns @@ -53,7 +54,7 @@ class ExcludableMixin(RelationMixin): @staticmethod def get_excluded( - exclude: Union[Set, Dict, None], key: str = None + exclude: Union[Set, Dict, None], key: str = None ) -> Union[Set, Dict, None]: """ Proxy to ExcludableMixin.get_child for exclusions. @@ -69,7 +70,7 @@ class ExcludableMixin(RelationMixin): @staticmethod def get_included( - include: Union[Set, Dict, None], key: str = None + include: Union[Set, Dict, None], key: str = None ) -> Union[Set, Dict, None]: """ Proxy to ExcludableMixin.get_child for inclusions. @@ -131,9 +132,9 @@ class ExcludableMixin(RelationMixin): @staticmethod def _populate_pk_column( - model: Union[Type["Model"], Type["ModelRow"]], - columns: List[str], - use_alias: bool = False, + model: Union[Type["Model"], Type["ModelRow"]], + columns: List[str], + use_alias: bool = False, ) -> List[str]: """ Adds primary key column/alias (depends on use_alias flag) to list of @@ -159,12 +160,13 @@ class ExcludableMixin(RelationMixin): @classmethod def own_table_columns( - cls, - model: Union[Type["Model"], Type["ModelRow"]], - fields: Optional[Union[Set, Dict]], - exclude_fields: Optional[Union[Set, Dict]], - use_alias: bool = False, + cls, + model: Union[Type["Model"], Type["ModelRow"]], + excludable: ExcludableItems, + alias: str = '', + use_alias: bool = False, ) -> List[str]: + # TODO update docstring """ Returns list of aliases or field names for given model. Aliases/names switch is use_alias flag. @@ -176,15 +178,12 @@ class ExcludableMixin(RelationMixin): :param model: model on columns are selected :type model: Type["Model"] - :param fields: set/dict of fields to include - :type fields: Optional[Union[Set, Dict]] - :param exclude_fields: set/dict of fields to exclude - :type exclude_fields: Optional[Union[Set, Dict]] :param use_alias: flag if aliases or field names should be used :type use_alias: bool :return: list of column field names or aliases :rtype: List[str] """ + model_excludable = excludable.get(model_cls=model, alias=alias) columns = [ model.get_column_name_from_alias(col.name) if not use_alias else col.name for col in model.Meta.table.columns @@ -193,17 +192,17 @@ class ExcludableMixin(RelationMixin): model.get_column_name_from_alias(col.name) for col in model.Meta.table.columns ] - if fields: + if model_excludable.include: columns = [ col for col, name in zip(columns, field_names) - if model.is_included(fields, name) + if model_excludable.is_included(name) ] - if exclude_fields: + if model_excludable.exclude: columns = [ col for col, name in zip(columns, field_names) - if not model.is_excluded(exclude_fields, name) + if not model_excludable.is_excluded(name) ] # always has to return pk column for ormar to work @@ -215,9 +214,9 @@ class ExcludableMixin(RelationMixin): @classmethod def _update_excluded_with_related_not_required( - cls, - exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None], - nested: bool = False, + cls, + exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None], + nested: bool = False, ) -> Union[Set, Dict]: """ Used during generation of the dict(). @@ -245,9 +244,9 @@ class ExcludableMixin(RelationMixin): @classmethod def get_names_to_exclude( - cls, - fields: Optional[Union[Dict, Set]] = None, - exclude_fields: Optional[Union[Dict, Set]] = None, + cls, + excludable: ExcludableItems, + alias: str ) -> Set: """ Returns a set of models field names that should be explicitly excluded @@ -259,33 +258,27 @@ class ExcludableMixin(RelationMixin): Used in parsing data from database rows that construct Models by initializing them with dicts constructed from those db rows. - :param fields: set/dict of fields to include - :type fields: Optional[Union[Set, Dict]] - :param exclude_fields: set/dict of fields to exclude - :type exclude_fields: Optional[Union[Set, Dict]] + :param alias: alias of current relation + :type alias: str + :param excludable: structure of fields to include and exclude + :type excludable: ExcludableItems :return: set of field names that should be excluded :rtype: Set """ + model = cast(Type["Model"], cls) + model_excludable = excludable.get(model_cls=model, alias=alias) fields_names = cls.extract_db_own_fields() - if fields and fields is not Ellipsis: - fields_to_keep = {name for name in fields if name in fields_names} + if model_excludable.include and model_excludable.include_all: + fields_to_keep = model_excludable.include.intersection(fields_names) else: fields_to_keep = fields_names fields_to_exclude = fields_names - fields_to_keep - if isinstance(exclude_fields, Set): + if model_excludable.exclude: fields_to_exclude = fields_to_exclude.union( - {name for name in exclude_fields if name in fields_names} + model_excludable.exclude.intersection(fields_names) ) - elif isinstance(exclude_fields, Dict): - new_to_exclude = { - name - for name in exclude_fields - if name in fields_names and exclude_fields[name] is Ellipsis - } - fields_to_exclude = fields_to_exclude.union(new_to_exclude) - fields_to_exclude = fields_to_exclude - {cls.Meta.pkname} return fields_to_exclude diff --git a/ormar/models/model_row.py b/ormar/models/model_row.py index 476e274..6a6cb0e 100644 --- a/ormar/models/model_row.py +++ b/ormar/models/model_row.py @@ -14,6 +14,7 @@ from typing import ( import sqlalchemy from ormar.models import NewBaseModel # noqa: I202 +from ormar.models.excludable import ExcludableItems from ormar.models.helpers.models import group_related_list @@ -33,8 +34,7 @@ class ModelRow(NewBaseModel): select_related: List = None, related_models: Any = None, related_field: Type["ForeignKeyField"] = None, - fields: Optional[Union[Dict, Set]] = None, - exclude_fields: Optional[Union[Dict, Set]] = None, + excludable: ExcludableItems = None, current_relation_str: str = "", ) -> Optional[T]: """ @@ -50,6 +50,8 @@ class ModelRow(NewBaseModel): where rows are populated in a different way as they do not have nested models in result. + :param excludable: structure of fields to include and exclude + :type excludable: ExcludableItems :param current_relation_str: name of the relation field :type current_relation_str: str :param source_model: model on which relation was defined @@ -62,12 +64,6 @@ class ModelRow(NewBaseModel): :type related_models: Union[List, Dict] :param related_field: field with relation declaration :type related_field: Type[ForeignKeyField] - :param fields: fields and related model fields to include - if provided only those are included - :type fields: Optional[Union[Dict, Set]] - :param exclude_fields: fields and related model fields to exclude - excludes the fields even if they are provided in fields - :type exclude_fields: Optional[Union[Dict, Set]] :return: returns model if model is populated from database :rtype: Optional[Model] """ @@ -75,6 +71,7 @@ class ModelRow(NewBaseModel): select_related = select_related or [] related_models = related_models or [] table_prefix = "" + excludable = excludable or ExcludableItems() if select_related: source_model = cast(Type[T], cls) @@ -87,12 +84,11 @@ class ModelRow(NewBaseModel): relation_field=related_field, ) - item = cls.populate_nested_models_from_row( + item = cls._populate_nested_models_from_row( item=item, row=row, related_models=related_models, - fields=fields, - exclude_fields=exclude_fields, + excludable=excludable, current_relation_str=current_relation_str, source_model=source_model, ) @@ -100,28 +96,26 @@ class ModelRow(NewBaseModel): item=item, row=row, table_prefix=table_prefix, - fields=fields, - exclude_fields=exclude_fields, + excludable=excludable ) instance: Optional[T] = None if item.get(cls.Meta.pkname, None) is not None: item["__excluded__"] = cls.get_names_to_exclude( - fields=fields, exclude_fields=exclude_fields + excludable=excludable, alias=table_prefix ) instance = cast(T, cls(**item)) instance.set_save_status(True) return instance @classmethod - def populate_nested_models_from_row( # noqa: CFQ002 + def _populate_nested_models_from_row( # noqa: CFQ002 cls, item: dict, row: sqlalchemy.engine.ResultProxy, source_model: Type[T], related_models: Any, - fields: Optional[Union[Dict, Set]] = None, - exclude_fields: Optional[Union[Dict, Set]] = None, + excludable: ExcludableItems, current_relation_str: str = None, ) -> dict: """ @@ -134,6 +128,8 @@ class ModelRow(NewBaseModel): Recurrently calls from_row method on nested instances and create nested instances. In the end those instances are added to the final model dictionary. + :param excludable: structure of fields to include and exclude + :type excludable: ExcludableItems :param source_model: source model from which relation started :type source_model: Type[Model] :param current_relation_str: joined related parts into one string @@ -144,12 +140,6 @@ class ModelRow(NewBaseModel): :type row: sqlalchemy.engine.result.ResultProxy :param related_models: list or dict of related models :type related_models: Union[Dict, List] - :param fields: fields and related model fields to include - - if provided only those are included - :type fields: Optional[Union[Dict, Set]] - :param exclude_fields: fields and related model fields to exclude - excludes the fields even if they are provided in fields - :type exclude_fields: Optional[Union[Dict, Set]] :return: dictionary with keys corresponding to model fields names and values are database values :rtype: Dict @@ -163,8 +153,6 @@ class ModelRow(NewBaseModel): ) field = cls.Meta.model_fields[related] field = cast(Type["ForeignKeyField"], field) - fields = cls.get_included(fields, related) - exclude_fields = cls.get_excluded(exclude_fields, related) model_cls = field.to remainder = None @@ -174,8 +162,7 @@ class ModelRow(NewBaseModel): row, related_models=remainder, related_field=field, - fields=fields, - exclude_fields=exclude_fields, + excludable=excludable, current_relation_str=relation_str, source_model=source_model, ) @@ -188,8 +175,7 @@ class ModelRow(NewBaseModel): row=row, related=related, through_name=through_name, - fields=fields, - exclude_fields=exclude_fields, + excludable=excludable ) item[through_name] = through_child setattr(child, through_name, through_child) @@ -203,35 +189,29 @@ class ModelRow(NewBaseModel): row: sqlalchemy.engine.ResultProxy, through_name: str, related: str, - fields: Optional[Union[Dict, Set]] = None, - exclude_fields: Optional[Union[Dict, Set]] = None, + excludable: ExcludableItems ) -> Dict: - # TODO: fix excludes and includes - fields = cls.get_included(fields, through_name) - # exclude_fields = cls.get_excluded(exclude_fields, through_name) + # TODO: fix excludes and includes and docstring model_cls = cls.Meta.model_fields[through_name].to - exclude_fields = model_cls.extract_related_names() table_prefix = cls.Meta.alias_manager.resolve_relation_alias( from_model=cls, relation_name=related ) child = model_cls.extract_prefixed_table_columns( item={}, row=row, - table_prefix=table_prefix, - fields=fields, - exclude_fields=exclude_fields, + excludable=excludable, + table_prefix=table_prefix ) return child @classmethod - def extract_prefixed_table_columns( # noqa CCR001 + def extract_prefixed_table_columns( cls, item: dict, row: sqlalchemy.engine.result.ResultProxy, table_prefix: str, - fields: Optional[Union[Dict, Set]] = None, - exclude_fields: Optional[Union[Dict, Set]] = None, - ) -> dict: + excludable: ExcludableItems + ) -> Dict: """ Extracts own fields from raw sql result, using a given prefix. Prefix changes depending on the table's position in a join. @@ -244,6 +224,8 @@ class ModelRow(NewBaseModel): Used in Model.from_row and PrefetchQuery._populate_rows methods. + :param excludable: structure of fields to include and exclude + :type excludable: ExcludableItems :param item: dictionary of already populated nested models, otherwise empty dict :type item: Dict :param row: raw result row from the database @@ -252,12 +234,6 @@ class ModelRow(NewBaseModel): each pair of tables have own prefix (two of them depending on direction) - used in joins to allow multiple joins to the same table. :type table_prefix: str - :param fields: fields and related model fields to include - - if provided only those are included - :type fields: Optional[Union[Dict, Set]] - :param exclude_fields: fields and related model fields to exclude - excludes the fields even if they are provided in fields - :type exclude_fields: Optional[Union[Dict, Set]] :return: dictionary with keys corresponding to model fields names and values are database values :rtype: Dict @@ -267,8 +243,8 @@ class ModelRow(NewBaseModel): selected_columns = cls.own_table_columns( model=cls, - fields=fields or {}, - exclude_fields=exclude_fields or {}, + excludable=excludable, + alias=table_prefix, use_alias=False, ) diff --git a/ormar/queryset/join.py b/ormar/queryset/join.py index f18c81d..a6f1e93 100644 --- a/ormar/queryset/join.py +++ b/ormar/queryset/join.py @@ -16,6 +16,7 @@ from sqlalchemy import text import ormar # noqa I100 from ormar.exceptions import RelationshipInstanceError +from ormar.models.excludable import ExcludableItems from ormar.relations import AliasManager if TYPE_CHECKING: # pragma no cover @@ -29,8 +30,7 @@ class SqlJoin: used_aliases: List, select_from: sqlalchemy.sql.select, columns: List[sqlalchemy.Column], - fields: Optional[Union[Set, Dict]], - exclude_fields: Optional[Union[Set, Dict]], + excludable: ExcludableItems, order_columns: Optional[List["OrderAction"]], sorted_orders: OrderedDict, main_model: Type["Model"], @@ -44,8 +44,7 @@ class SqlJoin: self.related_models = related_models or [] self.select_from = select_from self.columns = columns - self.fields = fields - self.exclude_fields = exclude_fields + self.excludable=excludable self.order_columns = order_columns self.sorted_orders = sorted_orders self.main_model = main_model @@ -200,10 +199,7 @@ class SqlJoin: used_aliases=self.used_aliases, select_from=self.select_from, columns=self.columns, - fields=self.main_model.get_excluded(self.fields, related_name), - exclude_fields=self.main_model.get_excluded( - self.exclude_fields, related_name - ), + excludable=self.excludable, order_columns=self.order_columns, sorted_orders=self.sorted_orders, main_model=self.next_model, @@ -303,8 +299,8 @@ class SqlJoin: # TODO: fix fields and exclusions for through model? self_related_fields = self.next_model.own_table_columns( model=self.next_model, - fields=self.fields, - exclude_fields=self.exclude_fields, + excludable=self.excludable, + alias=self.next_alias, use_alias=True, ) self.columns.extend( diff --git a/ormar/queryset/prefetch_query.py b/ormar/queryset/prefetch_query.py index 533f92c..88fc8e3 100644 --- a/ormar/queryset/prefetch_query.py +++ b/ormar/queryset/prefetch_query.py @@ -13,6 +13,7 @@ from typing import ( ) import ormar +from ormar.models.excludable import ExcludableItems from ormar.queryset.clause import QueryClause from ormar.queryset.query import Query from ormar.queryset.utils import extract_models_to_dict_of_lists, translate_list_to_dict @@ -24,7 +25,7 @@ if TYPE_CHECKING: # pragma: no cover def add_relation_field_to_fields( - fields: Union[Set[Any], Dict[Any, Any], None], related_field_name: str + fields: Union[Set[Any], Dict[Any, Any], None], related_field_name: str ) -> Union[Set[Any], Dict[Any, Any], None]: """ Adds related field into fields to include as otherwise it would be skipped. @@ -73,12 +74,12 @@ def sort_models(models: List["Model"], orders_by: Dict) -> List["Model"]: def set_children_on_model( # noqa: CCR001 - model: "Model", - related: str, - children: Dict, - model_id: int, - models: Dict, - orders_by: Dict, + model: "Model", + related: str, + children: Dict, + model_id: int, + models: Dict, + orders_by: Dict, ) -> None: """ Extract ids of child models by given relation id key value. @@ -123,21 +124,19 @@ class PrefetchQuery: """ def __init__( # noqa: CFQ002 - self, - model_cls: Type["Model"], - fields: Optional[Union[Dict, Set]], - exclude_fields: Optional[Union[Dict, Set]], - prefetch_related: List, - select_related: List, - orders_by: List["OrderAction"], + self, + model_cls: Type["Model"], + excludable: ExcludableItems, + prefetch_related: List, + select_related: List, + orders_by: List["OrderAction"], ) -> None: self.model = model_cls self.database = self.model.Meta.database self._prefetch_related = prefetch_related self._select_related = select_related - self._exclude_columns = exclude_fields - self._columns = fields + self.excludable = excludable self.already_extracted: Dict = dict() self.models: Dict = {} self.select_dict = translate_list_to_dict(self._select_related) @@ -148,7 +147,7 @@ class PrefetchQuery: ) async def prefetch_related( - self, models: Sequence["Model"], rows: List + self, models: Sequence["Model"], rows: List ) -> Sequence["Model"]: """ Main entry point for prefetch_query. @@ -173,7 +172,7 @@ class PrefetchQuery: return await self._prefetch_related_models(models=models, rows=rows) def _extract_ids_from_raw_data( - self, parent_model: Type["Model"], column_name: str + self, parent_model: Type["Model"], column_name: str ) -> Set: """ Iterates over raw rows and extract id values of relation columns by using @@ -196,7 +195,7 @@ class PrefetchQuery: return list_of_ids def _extract_ids_from_preloaded_models( - self, parent_model: Type["Model"], column_name: str + self, parent_model: Type["Model"], column_name: str ) -> Set: """ Extracts relation ids from already populated models if they were included @@ -219,7 +218,7 @@ class PrefetchQuery: return list_of_ids def _extract_required_ids( - self, parent_model: Type["Model"], reverse: bool, related: str, + self, parent_model: Type["Model"], reverse: bool, related: str, ) -> Set: """ Delegates extraction of the fields to either get ids from raw sql response @@ -253,11 +252,11 @@ class PrefetchQuery: ) def _get_filter_for_prefetch( - self, - parent_model: Type["Model"], - target_model: Type["Model"], - reverse: bool, - related: str, + self, + parent_model: Type["Model"], + target_model: Type["Model"], + reverse: bool, + related: str, ) -> List: """ Populates where clause with condition to return only models within the @@ -298,7 +297,7 @@ class PrefetchQuery: return [] def _populate_nested_related( - self, model: "Model", prefetch_dict: Dict, orders_by: Dict, + self, model: "Model", prefetch_dict: Dict, orders_by: Dict, ) -> "Model": """ Populates all related models children of parent model that are @@ -342,7 +341,7 @@ class PrefetchQuery: return model async def _prefetch_related_models( - self, models: Sequence["Model"], rows: List + self, models: Sequence["Model"], rows: List ) -> Sequence["Model"]: """ Main method of the query. @@ -366,8 +365,6 @@ class PrefetchQuery: select_dict = translate_list_to_dict(self._select_related) prefetch_dict = translate_list_to_dict(self._prefetch_related) target_model = self.model - fields = self._columns - exclude_fields = self._exclude_columns orders_by = self.order_dict for related in prefetch_dict.keys(): await self._extract_related_models( @@ -375,8 +372,7 @@ class PrefetchQuery: target_model=target_model, prefetch_dict=prefetch_dict.get(related, {}), select_dict=select_dict.get(related, {}), - fields=fields, - exclude_fields=exclude_fields, + excludable=self.excludable, orders_by=orders_by.get(related, {}), ) final_models = [] @@ -389,14 +385,13 @@ class PrefetchQuery: return models async def _extract_related_models( # noqa: CFQ002, CCR001 - self, - related: str, - target_model: Type["Model"], - prefetch_dict: Dict, - select_dict: Dict, - fields: Union[Set[Any], Dict[Any, Any], None], - exclude_fields: Union[Set[Any], Dict[Any, Any], None], - orders_by: Dict, + self, + related: str, + target_model: Type["Model"], + prefetch_dict: Dict, + select_dict: Dict, + excludable: ExcludableItems, + orders_by: Dict, ) -> None: """ Constructs queries with required ids and extracts data with fields that should @@ -424,8 +419,6 @@ class PrefetchQuery: :return: None :rtype: None """ - fields = target_model.get_included(fields, related) - exclude_fields = target_model.get_excluded(exclude_fields, related) target_field = target_model.Meta.model_fields[related] target_field = cast(Type["ForeignKeyField"], target_field) reverse = False @@ -450,14 +443,11 @@ class PrefetchQuery: related_field_name = parent_model.get_related_field_name( target_field=target_field ) - fields = add_relation_field_to_fields( - fields=fields, related_field_name=related_field_name - ) table_prefix, rows = await self._run_prefetch_query( target_field=target_field, - fields=fields, - exclude_fields=exclude_fields, + excludable=excludable, filter_clauses=filter_clauses, + related_field_name=related_field_name ) else: rows = [] @@ -472,8 +462,7 @@ class PrefetchQuery: select_dict=self._get_select_related_if_apply( subrelated, select_dict ), - fields=fields, - exclude_fields=exclude_fields, + excludable=excludable, orders_by=self._get_select_related_if_apply(subrelated, orders_by), ) @@ -483,8 +472,7 @@ class PrefetchQuery: parent_model=parent_model, target_field=target_field, table_prefix=table_prefix, - fields=fields, - exclude_fields=exclude_fields, + excludable=excludable, prefetch_dict=prefetch_dict, orders_by=orders_by, ) @@ -496,11 +484,11 @@ class PrefetchQuery: ) async def _run_prefetch_query( - self, - target_field: Type["BaseField"], - fields: Union[Set[Any], Dict[Any, Any], None], - exclude_fields: Union[Set[Any], Dict[Any, Any], None], - filter_clauses: List, + self, + target_field: Type["BaseField"], + excludable: ExcludableItems, + filter_clauses: List, + related_field_name: str ) -> Tuple[str, List]: """ Actually runs the queries against the database and populates the raw response @@ -511,10 +499,6 @@ class PrefetchQuery: :param target_field: ormar field with relation definition :type target_field: Type["BaseField"] - :param fields: fields to include - :type fields: Union[Set[Any], Dict[Any, Any], None] - :param exclude_fields: fields to exclude - :type exclude_fields: Union[Set[Any], Dict[Any, Any], None] :param filter_clauses: list of clauses, actually one clause with ids of relation :type filter_clauses: List[sqlalchemy.sql.elements.TextClause] :return: table prefix and raw rows from sql response @@ -533,6 +517,11 @@ class PrefetchQuery: ) self.already_extracted.setdefault(target_name, {})["prefix"] = table_prefix + model_excludable = excludable.get(model_cls=target_model, alias=table_prefix) + if model_excludable.include and not model_excludable.is_included( + related_field_name): + model_excludable.set_values({related_field_name}, is_exclude=False) + qry = Query( model_cls=query_target, select_related=select_related, @@ -540,8 +529,7 @@ class PrefetchQuery: exclude_clauses=[], offset=None, limit_count=None, - fields=fields, - exclude_fields=exclude_fields, + excludable=excludable, order_bys=None, limit_raw_sql=False, ) @@ -571,7 +559,7 @@ class PrefetchQuery: ) def _update_already_loaded_rows( # noqa: CFQ002 - self, target_field: Type["BaseField"], prefetch_dict: Dict, orders_by: Dict, + self, target_field: Type["BaseField"], prefetch_dict: Dict, orders_by: Dict, ) -> None: """ Updates models that are already loaded, usually children of children. @@ -590,15 +578,14 @@ class PrefetchQuery: ) def _populate_rows( # noqa: CFQ002 - self, - rows: List, - target_field: Type["ForeignKeyField"], - parent_model: Type["Model"], - table_prefix: str, - fields: Union[Set[Any], Dict[Any, Any], None], - exclude_fields: Union[Set[Any], Dict[Any, Any], None], - prefetch_dict: Dict, - orders_by: Dict, + self, + rows: List, + target_field: Type["ForeignKeyField"], + parent_model: Type["Model"], + table_prefix: str, + excludable: ExcludableItems, + prefetch_dict: Dict, + orders_by: Dict, ) -> None: """ Instantiates children models extracted from given relation. @@ -610,6 +597,8 @@ class PrefetchQuery: already_extracted dictionary. Later those instances will be fetched by ids and set on the parent model after sorting if needed. + :param excludable: structure of fields to include and exclude + :type excludable: ExcludableItems :param rows: raw sql response from the prefetch query :type rows: List[sqlalchemy.engine.result.RowProxy] :param target_field: field with relation definition from parent model @@ -618,10 +607,6 @@ class PrefetchQuery: :type parent_model: Type[Model] :param table_prefix: prefix of the target table from current relation :type table_prefix: str - :param fields: fields to include - :type fields: Union[Set[Any], Dict[Any, Any], None] - :param exclude_fields: fields to exclude - :type exclude_fields: Union[Set[Any], Dict[Any, Any], None] :param prefetch_dict: dictionaries of related models to prefetch :type prefetch_dict: Dict :param orders_by: dictionary of order by clauses by model @@ -629,16 +614,16 @@ class PrefetchQuery: """ target_model = target_field.to for row in rows: + # TODO Fix fields field_name = parent_model.get_related_field_name(target_field=target_field) item = target_model.extract_prefixed_table_columns( item={}, row=row, table_prefix=table_prefix, - fields=fields, - exclude_fields=exclude_fields, + excludable=excludable, ) item["__excluded__"] = target_model.get_names_to_exclude( - fields=fields, exclude_fields=exclude_fields + excludable=excludable, alias=table_prefix ) instance = target_model(**item) instance = self._populate_nested_related( diff --git a/ormar/queryset/query.py b/ormar/queryset/query.py index d6b10d2..2e88212 100644 --- a/ormar/queryset/query.py +++ b/ormar/queryset/query.py @@ -6,6 +6,7 @@ import sqlalchemy from sqlalchemy import text import ormar # noqa I100 +from ormar.models.excludable import ExcludableItems from ormar.models.helpers.models import group_related_list from ormar.queryset import FilterQuery, LimitQuery, OffsetQuery, OrderQuery from ormar.queryset.actions.filter_action import FilterAction @@ -18,25 +19,23 @@ if TYPE_CHECKING: # pragma no cover class Query: def __init__( # noqa CFQ002 - self, - model_cls: Type["Model"], - filter_clauses: List[FilterAction], - exclude_clauses: List[FilterAction], - select_related: List, - limit_count: Optional[int], - offset: Optional[int], - fields: Optional[Union[Dict, Set]], - exclude_fields: Optional[Union[Dict, Set]], - order_bys: Optional[List["OrderAction"]], - limit_raw_sql: bool, + self, + model_cls: Type["Model"], + filter_clauses: List[FilterAction], + exclude_clauses: List[FilterAction], + select_related: List, + limit_count: Optional[int], + offset: Optional[int], + excludable: ExcludableItems, + order_bys: Optional[List["OrderAction"]], + limit_raw_sql: bool, ) -> None: self.query_offset = offset self.limit_count = limit_count self._select_related = select_related[:] self.filter_clauses = filter_clauses[:] self.exclude_clauses = exclude_clauses[:] - self.fields = copy.deepcopy(fields) if fields else {} - self.exclude_fields = copy.deepcopy(exclude_fields) if exclude_fields else {} + self.excludable = excludable self.model_cls = model_cls self.table = self.model_cls.Meta.table @@ -105,8 +104,7 @@ class Query: """ self_related_fields = self.model_cls.own_table_columns( model=self.model_cls, - fields=self.fields, - exclude_fields=self.exclude_fields, + excludable=self.excludable, use_alias=True, ) self.columns = self.model_cls.Meta.alias_manager.prefixed_columns( @@ -121,8 +119,6 @@ class Query: related_models = group_related_list(self._select_related) for related in related_models: - fields = self.model_cls.get_included(self.fields, related) - exclude_fields = self.model_cls.get_excluded(self.exclude_fields, related) remainder = None if isinstance(related_models, dict) and related_models[related]: remainder = related_models[related] @@ -130,8 +126,7 @@ class Query: used_aliases=self.used_aliases, select_from=self.select_from, columns=self.columns, - fields=fields, - exclude_fields=exclude_fields, + excludable=self.excludable, order_columns=self.order_columns, sorted_orders=self.sorted_orders, main_model=self.model_cls, @@ -196,7 +191,7 @@ class Query: return expr def _apply_expression_modifiers( - self, expr: sqlalchemy.sql.select + self, expr: sqlalchemy.sql.select ) -> sqlalchemy.sql.select: """ Receives the select query (might be join) and applies: @@ -231,5 +226,3 @@ class Query: self.select_from = [] self.columns = [] self.used_aliases = [] - self.fields = {} - self.exclude_fields = {} diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 7c664ac..1c93590 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -20,6 +20,7 @@ from sqlalchemy import bindparam import ormar # noqa I100 from ormar import MultipleMatches, NoMatch from ormar.exceptions import ModelError, ModelPersistenceError, QueryDefinitionError +from ormar.models.excludable import ExcludableItems from ormar.queryset import FilterQuery from ormar.queryset.actions.order_action import OrderAction from ormar.queryset.clause import QueryClause @@ -41,18 +42,17 @@ class QuerySet(Generic[T]): """ def __init__( # noqa CFQ002 - self, - model_cls: Optional[Type[T]] = None, - filter_clauses: List = None, - exclude_clauses: List = None, - select_related: List = None, - limit_count: int = None, - offset: int = None, - columns: Dict = None, - exclude_columns: Dict = None, - order_bys: List = None, - prefetch_related: List = None, - limit_raw_sql: bool = False, + self, + model_cls: Optional[Type[T]] = None, + filter_clauses: List = None, + exclude_clauses: List = None, + select_related: List = None, + limit_count: int = None, + offset: int = None, + excludable: ExcludableItems = None, + order_bys: List = None, + prefetch_related: List = None, + limit_raw_sql: bool = False, ) -> None: self.model_cls = model_cls self.filter_clauses = [] if filter_clauses is None else filter_clauses @@ -61,15 +61,14 @@ class QuerySet(Generic[T]): self._prefetch_related = [] if prefetch_related is None else prefetch_related self.limit_count = limit_count self.query_offset = offset - self._columns = columns or {} - self._exclude_columns = exclude_columns or {} + self._excludable = excludable or ExcludableItems() self.order_bys = order_bys or [] self.limit_sql_raw = limit_raw_sql def __get__( - self, - instance: Optional[Union["QuerySet", "QuerysetProxy"]], - owner: Union[Type[T], Type["QuerysetProxy"]], + self, + instance: Optional[Union["QuerySet", "QuerysetProxy"]], + owner: Union[Type[T], Type["QuerysetProxy"]], ) -> "QuerySet": if issubclass(owner, ormar.Model): if owner.Meta.requires_ref_update: @@ -107,7 +106,7 @@ class QuerySet(Generic[T]): return self.model_cls async def _prefetch_related_models( - self, models: Sequence[Optional["T"]], rows: List + self, models: Sequence[Optional["T"]], rows: List ) -> Sequence[Optional["T"]]: """ Performs prefetch query for selected models names. @@ -121,8 +120,7 @@ class QuerySet(Generic[T]): """ query = PrefetchQuery( model_cls=self.model, - fields=self._columns, - exclude_fields=self._exclude_columns, + excludable=self._excludable, prefetch_related=self._prefetch_related, select_related=self._select_related, orders_by=self.order_bys, @@ -142,8 +140,7 @@ class QuerySet(Generic[T]): self.model.from_row( row=row, select_related=self._select_related, - fields=self._columns, - exclude_fields=self._exclude_columns, + excludable=self._excludable, source_model=self.model, ) for row in rows @@ -186,7 +183,7 @@ class QuerySet(Generic[T]): return self.model_meta.table def build_select_expression( - self, limit: int = None, offset: int = None, order_bys: List = None, + self, limit: int = None, offset: int = None, order_bys: List = None, ) -> sqlalchemy.sql.select: """ Constructs the actual database query used in the QuerySet. @@ -208,8 +205,7 @@ class QuerySet(Generic[T]): exclude_clauses=self.exclude_clauses, offset=offset or self.query_offset, limit_count=limit or self.limit_count, - fields=self._columns, - exclude_fields=self._exclude_columns, + excludable=self._excludable, order_bys=order_bys or self.order_bys, limit_raw_sql=self.limit_sql_raw, ) @@ -265,8 +261,7 @@ class QuerySet(Generic[T]): select_related=select_related, limit_count=self.limit_count, offset=self.query_offset, - columns=self._columns, - exclude_columns=self._exclude_columns, + excludable=self._excludable, order_bys=self.order_bys, prefetch_related=self._prefetch_related, limit_raw_sql=self.limit_sql_raw, @@ -321,8 +316,7 @@ class QuerySet(Generic[T]): select_related=related, limit_count=self.limit_count, offset=self.query_offset, - columns=self._columns, - exclude_columns=self._exclude_columns, + excludable=self._excludable, order_bys=self.order_bys, prefetch_related=self._prefetch_related, limit_raw_sql=self.limit_sql_raw, @@ -357,14 +351,14 @@ class QuerySet(Generic[T]): select_related=self._select_related, limit_count=self.limit_count, offset=self.query_offset, - columns=self._columns, - exclude_columns=self._exclude_columns, + excludable=self._excludable, order_bys=self.order_bys, prefetch_related=related, limit_raw_sql=self.limit_sql_raw, ) - def fields(self, columns: Union[List, str, Set, Dict]) -> "QuerySet": + def fields(self, columns: Union[List, str, Set, Dict], + _is_exclude: bool = False) -> "QuerySet": """ With `fields()` you can select subset of model columns to limit the data load. @@ -407,15 +401,10 @@ class QuerySet(Generic[T]): :return: QuerySet :rtype: QuerySet """ - if isinstance(columns, str): - columns = [columns] - - # TODO: Flatten all excludes into one dict-like structure with alias + model key - current_included = self._columns - if not isinstance(columns, dict): - current_included = update_dict_from_list(current_included, columns) - else: - current_included = update(current_included, columns) + excludable = ExcludableItems.from_excludable(self._excludable) + excludable.build(items=columns, + model_cls=self.model_cls, + is_exclude=_is_exclude) return self.__class__( model_cls=self.model, @@ -424,8 +413,7 @@ class QuerySet(Generic[T]): select_related=self._select_related, limit_count=self.limit_count, offset=self.query_offset, - columns=current_included, - exclude_columns=self._exclude_columns, + excludable=excludable, order_bys=self.order_bys, prefetch_related=self._prefetch_related, limit_raw_sql=self.limit_sql_raw, @@ -458,28 +446,7 @@ class QuerySet(Generic[T]): :return: QuerySet :rtype: QuerySet """ - if isinstance(columns, str): - columns = [columns] - - current_excluded = self._exclude_columns - if not isinstance(columns, dict): - current_excluded = update_dict_from_list(current_excluded, columns) - else: - current_excluded = update(current_excluded, columns) - - return self.__class__( - model_cls=self.model, - filter_clauses=self.filter_clauses, - exclude_clauses=self.exclude_clauses, - select_related=self._select_related, - limit_count=self.limit_count, - offset=self.query_offset, - columns=self._columns, - exclude_columns=current_excluded, - order_bys=self.order_bys, - prefetch_related=self._prefetch_related, - limit_raw_sql=self.limit_sql_raw, - ) + return self.fields(columns=columns, _is_exclude=True) def order_by(self, columns: Union[List, str]) -> "QuerySet": """ @@ -529,8 +496,7 @@ class QuerySet(Generic[T]): select_related=self._select_related, limit_count=self.limit_count, offset=self.query_offset, - columns=self._columns, - exclude_columns=self._exclude_columns, + excludable=self._excludable, order_bys=order_bys, prefetch_related=self._prefetch_related, limit_raw_sql=self.limit_sql_raw, @@ -642,8 +608,7 @@ class QuerySet(Generic[T]): select_related=self._select_related, limit_count=limit_count, offset=query_offset, - columns=self._columns, - exclude_columns=self._exclude_columns, + excludable=self._excludable, order_bys=self.order_bys, prefetch_related=self._prefetch_related, limit_raw_sql=self.limit_sql_raw, @@ -671,8 +636,7 @@ class QuerySet(Generic[T]): select_related=self._select_related, limit_count=limit_count, offset=self.query_offset, - columns=self._columns, - exclude_columns=self._exclude_columns, + excludable=self._excludable, order_bys=self.order_bys, prefetch_related=self._prefetch_related, limit_raw_sql=limit_raw_sql, @@ -700,8 +664,7 @@ class QuerySet(Generic[T]): select_related=self._select_related, limit_count=self.limit_count, offset=offset, - columns=self._columns, - exclude_columns=self._exclude_columns, + excludable=self._excludable, order_bys=self.order_bys, prefetch_related=self._prefetch_related, limit_raw_sql=limit_raw_sql, @@ -724,12 +687,12 @@ class QuerySet(Generic[T]): expr = self.build_select_expression( limit=1, order_bys=[ - OrderAction( - order_str=f"{self.model.Meta.pkname}", - model_cls=self.model_cls, # type: ignore - ) - ] - + self.order_bys, + OrderAction( + order_str=f"{self.model.Meta.pkname}", + model_cls=self.model_cls, # type: ignore + ) + ] + + self.order_bys, ) rows = await self.database.fetch_all(expr) processed_rows = self._process_query_result_rows(rows) @@ -760,12 +723,12 @@ class QuerySet(Generic[T]): expr = self.build_select_expression( limit=1, order_bys=[ - OrderAction( - order_str=f"-{self.model.Meta.pkname}", - model_cls=self.model_cls, # type: ignore - ) - ] - + self.order_bys, + OrderAction( + order_str=f"-{self.model.Meta.pkname}", + model_cls=self.model_cls, # type: ignore + ) + ] + + self.order_bys, ) else: expr = self.build_select_expression() @@ -868,9 +831,9 @@ class QuerySet(Generic[T]): # refresh server side defaults if any( - field.server_default is not None - for name, field in self.model.Meta.model_fields.items() - if name not in kwargs + field.server_default is not None + for name, field in self.model.Meta.model_fields.items() + if name not in kwargs ): instance = await instance.load() instance.set_save_status(True) @@ -905,7 +868,7 @@ class QuerySet(Generic[T]): objt.set_save_status(True) async def bulk_update( # noqa: CCR001 - self, objects: List[T], columns: List[str] = None + self, objects: List[T], columns: List[str] = None ) -> None: """ Performs bulk update in one database session to speed up the process. From 0c781c4d521a1c32ac27d9aa587de16fe2c841b3 Mon Sep 17 00:00:00 2001 From: collerek Date: Sun, 28 Feb 2021 09:20:42 +0100 Subject: [PATCH 09/14] apply fix for new pydantic --- ormar/fields/base.py | 22 ++++++++++++---------- setup.py | 5 ++++- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/ormar/fields/base.py b/ormar/fields/base.py index 95645c0..155b1c9 100644 --- a/ormar/fields/base.py +++ b/ormar/fields/base.py @@ -93,9 +93,11 @@ class BaseField(FieldInfo): :rtype: bool """ return ( - field_name not in ["default", "default_factory", "alias"] - and not field_name.startswith("__") - and hasattr(cls, field_name) + field_name not in ["default", "default_factory", "alias", + "allow_mutation"] + and not field_name.startswith("__") + and hasattr(cls, field_name) + and not callable(getattr(cls, field_name)) ) @classmethod @@ -204,7 +206,7 @@ class BaseField(FieldInfo): :rtype: bool """ return cls.default is not None or ( - cls.server_default is not None and use_server + cls.server_default is not None and use_server ) @classmethod @@ -237,7 +239,7 @@ class BaseField(FieldInfo): ondelete=con.ondelete, onupdate=con.onupdate, name=f"fk_{cls.owner.Meta.tablename}_{cls.to.Meta.tablename}" - f"_{cls.to.get_column_alias(cls.to.Meta.pkname)}_{cls.name}", + f"_{cls.to.get_column_alias(cls.to.Meta.pkname)}_{cls.name}", ) for con in cls.constraints ] @@ -270,10 +272,10 @@ class BaseField(FieldInfo): @classmethod def expand_relationship( - cls, - value: Any, - child: Union["Model", "NewBaseModel"], - to_register: bool = True, + cls, + value: Any, + child: Union["Model", "NewBaseModel"], + to_register: bool = True, ) -> Any: """ Function overwritten for relations, in basic field the value is returned as is. @@ -301,7 +303,7 @@ class BaseField(FieldInfo): :rtype: None """ if cls.owner is not None and ( - cls.owner == cls.to or cls.owner.Meta == cls.to.Meta + cls.owner == cls.to or cls.owner.Meta == cls.to.Meta ): cls.self_reference = True cls.self_reference_primary = cls.name diff --git a/setup.py b/setup.py index 909a738..2c5c25b 100644 --- a/setup.py +++ b/setup.py @@ -52,8 +52,11 @@ setup( package_data={PACKAGE: ["py.typed"]}, include_package_data=True, zip_safe=False, + python_requires=">=3.6", data_files=[("", ["LICENSE.md"])], - install_requires=["databases", "pydantic>=1.5", "sqlalchemy", "typing_extensions"], + install_requires=["databases>=0.3.2,<=0.4.1", "pydantic>=1.6.1,<=1.8", + "sqlalchemy>=1.3.18,<=1.3.23", + "typing_extensions>=3.7,<=3.7.4.3"], extras_require={ "postgresql": ["asyncpg", "psycopg2"], "mysql": ["aiomysql", "pymysql"], From a99000d2c0a52cccdbc7ae145d16eaa4364a73c0 Mon Sep 17 00:00:00 2001 From: collerek Date: Mon, 1 Mar 2021 19:26:33 +0100 Subject: [PATCH 10/14] add possibility to exclude/include fields (refactor to excludableitems), fix for through model only on related side of the relation, fix for exclude of through model related models --- ormar/__init__.py | 3 +- ormar/fields/base.py | 23 ++- ormar/fields/through_field.py | 12 +- ormar/models/__init__.py | 3 +- ormar/models/excludable.py | 53 +++--- ormar/models/mixins/excludable_mixin.py | 121 +++--------- ormar/models/model_row.py | 51 +++--- ormar/queryset/join.py | 10 +- ormar/queryset/prefetch_query.py | 143 ++++++--------- ormar/queryset/query.py | 31 ++-- ormar/queryset/queryset.py | 234 ++++++++++-------------- ormar/relations/relation_proxy.py | 4 +- tests/test_choices_schema.py | 6 +- tests/test_m2m_through_fields.py | 44 ++++- tests/test_queryset_utils.py | 5 - 15 files changed, 313 insertions(+), 430 deletions(-) diff --git a/ormar/__init__.py b/ormar/__init__.py index 328e894..4868e8f 100644 --- a/ormar/__init__.py +++ b/ormar/__init__.py @@ -54,7 +54,7 @@ from ormar.fields import ( UUID, UniqueColumns, ) # noqa: I100 -from ormar.models import Model +from ormar.models import ExcludableItems, Model from ormar.models.metaclass import ModelMeta from ormar.queryset import OrderAction, QuerySet from ormar.relations import RelationType @@ -107,4 +107,5 @@ __all__ = [ "ManyToManyField", "ForeignKeyField", "OrderAction", + "ExcludableItems", ] diff --git a/ormar/fields/base.py b/ormar/fields/base.py index 155b1c9..1fada90 100644 --- a/ormar/fields/base.py +++ b/ormar/fields/base.py @@ -93,11 +93,10 @@ class BaseField(FieldInfo): :rtype: bool """ return ( - field_name not in ["default", "default_factory", "alias", - "allow_mutation"] - and not field_name.startswith("__") - and hasattr(cls, field_name) - and not callable(getattr(cls, field_name)) + field_name not in ["default", "default_factory", "alias", "allow_mutation"] + and not field_name.startswith("__") + and hasattr(cls, field_name) + and not callable(getattr(cls, field_name)) ) @classmethod @@ -206,7 +205,7 @@ class BaseField(FieldInfo): :rtype: bool """ return cls.default is not None or ( - cls.server_default is not None and use_server + cls.server_default is not None and use_server ) @classmethod @@ -239,7 +238,7 @@ class BaseField(FieldInfo): ondelete=con.ondelete, onupdate=con.onupdate, name=f"fk_{cls.owner.Meta.tablename}_{cls.to.Meta.tablename}" - f"_{cls.to.get_column_alias(cls.to.Meta.pkname)}_{cls.name}", + f"_{cls.to.get_column_alias(cls.to.Meta.pkname)}_{cls.name}", ) for con in cls.constraints ] @@ -272,10 +271,10 @@ class BaseField(FieldInfo): @classmethod def expand_relationship( - cls, - value: Any, - child: Union["Model", "NewBaseModel"], - to_register: bool = True, + cls, + value: Any, + child: Union["Model", "NewBaseModel"], + to_register: bool = True, ) -> Any: """ Function overwritten for relations, in basic field the value is returned as is. @@ -303,7 +302,7 @@ class BaseField(FieldInfo): :rtype: None """ if cls.owner is not None and ( - cls.owner == cls.to or cls.owner.Meta == cls.to.Meta + cls.owner == cls.to or cls.owner.Meta == cls.to.Meta ): cls.self_reference = True cls.self_reference_primary = cls.name diff --git a/ormar/fields/through_field.py b/ormar/fields/through_field.py index e5e4a24..b25e94b 100644 --- a/ormar/fields/through_field.py +++ b/ormar/fields/through_field.py @@ -17,11 +17,9 @@ if TYPE_CHECKING: # pragma no cover def Through( # noqa CFQ002 to: "ToType", *, name: str = None, related_name: str = None, **kwargs: Any, ) -> Any: - # TODO: clean docstring """ - Despite a name it's a function that returns constructed ForeignKeyField. - This function is actually used in model declaration (as ormar.ForeignKey(ToModel)). - + Despite a name it's a function that returns constructed ThroughField. + It's a special field populated only for m2m relations. Accepts number of relation setting parameters as well as all BaseField ones. :param to: target related ormar Model @@ -30,15 +28,13 @@ def Through( # noqa CFQ002 :type name: str :param related_name: name of reversed FK relation populated for you on to model :type related_name: str - :param virtual: marks if relation is virtual. It is for reversed FK and auto generated FK on through model in Many2Many relations. - :type virtual: bool :param kwargs: all other args to be populated by BaseField :type kwargs: Any :return: ormar ForeignKeyField with relation to selected model :rtype: ForeignKeyField """ - + nullable = kwargs.pop("nullable", False) owner = kwargs.pop("owner", None) namespace = dict( __type__=to, @@ -49,7 +45,7 @@ def Through( # noqa CFQ002 related_name=related_name, virtual=True, owner=owner, - nullable=False, + nullable=nullable, unique=False, column_type=None, primary_key=False, diff --git a/ormar/models/__init__.py b/ormar/models/__init__.py index 58372b7..0ecd9cc 100644 --- a/ormar/models/__init__.py +++ b/ormar/models/__init__.py @@ -7,5 +7,6 @@ ass well as vast number of helper functions for pydantic, sqlalchemy and relatio from ormar.models.newbasemodel import NewBaseModel # noqa I100 from ormar.models.model_row import ModelRow # noqa I100 from ormar.models.model import Model, T # noqa I100 +from ormar.models.excludable import ExcludableItems # noqa I100 -__all__ = ["T", "NewBaseModel", "Model", "ModelRow"] +__all__ = ["T", "NewBaseModel", "Model", "ModelRow", "ExcludableItems"] diff --git a/ormar/models/excludable.py b/ormar/models/excludable.py index b8832d2..203d8a0 100644 --- a/ormar/models/excludable.py +++ b/ormar/models/excludable.py @@ -7,19 +7,12 @@ if TYPE_CHECKING: # pragma: no cover from ormar import Model +# TODO: Add docstrings @dataclass class Excludable: include: Set = field(default_factory=set) exclude: Set = field(default_factory=set) - @property - def include_all(self): - return ... in self.include - - @property - def exclude_all(self): - return ... in self.exclude - def get_copy(self) -> "Excludable": _copy = self.__class__() _copy.include = {x for x in self.include} @@ -28,12 +21,9 @@ class Excludable: def set_values(self, value: Set, is_exclude: bool) -> None: prop = "exclude" if is_exclude else "include" - if ... in getattr(self, prop) or ... in value: - setattr(self, prop, {...}) - else: - current_value = getattr(self, prop) - current_value.update(value) - setattr(self, prop, current_value) + current_value = getattr(self, prop) + current_value.update(value) + setattr(self, prop, current_value) def is_included(self, key: str) -> bool: return (... in self.include or key in self.include) if self.include else True @@ -61,13 +51,17 @@ class ExcludableItems: def get(self, model_cls: Type["Model"], alias: str = "") -> Excludable: key = f"{alias + '_' if alias else ''}{model_cls.get_name(lower=True)}" - return self.items.get(key, Excludable()) + excludable = self.items.get(key) + if not excludable: + excludable = Excludable() + self.items[key] = excludable + return excludable def build( - self, - items: Union[List[str], str, Tuple[str], Set[str], Dict], - model_cls: Type["Model"], - is_exclude: bool = False, + self, + items: Union[List[str], str, Tuple[str], Set[str], Dict], + model_cls: Type["Model"], + is_exclude: bool = False, ) -> None: if isinstance(items, str): @@ -96,7 +90,7 @@ class ExcludableItems: ) def _set_excludes( - self, items: Set, model_name: str, is_exclude: bool, alias: str = "" + self, items: Set, model_name: str, is_exclude: bool, alias: str = "" ) -> None: key = f"{alias + '_' if alias else ''}{model_name}" @@ -107,13 +101,13 @@ class ExcludableItems: self.items[key] = excludable def _traverse_dict( # noqa: CFQ002 - self, - values: Dict, - source_model: Type["Model"], - model_cls: Type["Model"], - is_exclude: bool, - related_items: List = None, - alias: str = "", + self, + values: Dict, + source_model: Type["Model"], + model_cls: Type["Model"], + is_exclude: bool, + related_items: List = None, + alias: str = "", ) -> None: self_fields = set() @@ -122,14 +116,13 @@ class ExcludableItems: if value is ...: self_fields.add(key) elif isinstance(value, set): - related_items.append(key) ( table_prefix, target_model, _, _, ) = get_relationship_alias_model_and_str( - source_model=source_model, related_parts=related_items + source_model=source_model, related_parts=related_items + [key] ) self._set_excludes( items=value, @@ -165,7 +158,7 @@ class ExcludableItems: ) def _traverse_list( - self, values: Set[str], model_cls: Type["Model"], is_exclude: bool + self, values: Set[str], model_cls: Type["Model"], is_exclude: bool ) -> None: # here we have only nested related keys diff --git a/ormar/models/mixins/excludable_mixin.py b/ormar/models/mixins/excludable_mixin.py index 4b25035..a7850d5 100644 --- a/ormar/models/mixins/excludable_mixin.py +++ b/ormar/models/mixins/excludable_mixin.py @@ -4,12 +4,12 @@ from typing import ( Dict, List, Mapping, - Optional, Set, TYPE_CHECKING, Type, TypeVar, - Union, cast, + Union, + cast, ) from ormar.models.excludable import ExcludableItems @@ -36,7 +36,7 @@ class ExcludableMixin(RelationMixin): @staticmethod def get_child( - items: Union[Set, Dict, None], key: str = None + items: Union[Set, Dict, None], key: str = None ) -> Union[Set, Dict, None]: """ Used to get nested dictionaries keys if they exists otherwise returns @@ -52,89 +52,11 @@ class ExcludableMixin(RelationMixin): return items.get(key, {}) return items - @staticmethod - def get_excluded( - exclude: Union[Set, Dict, None], key: str = None - ) -> Union[Set, Dict, None]: - """ - Proxy to ExcludableMixin.get_child for exclusions. - - :param exclude: bag of items to exclude - :type exclude: Union[Set, Dict, None] - :param key: name of the child to extract - :type key: str - :return: child extracted from items if exists - :rtype: Union[Set, Dict, None] - """ - return ExcludableMixin.get_child(items=exclude, key=key) - - @staticmethod - def get_included( - include: Union[Set, Dict, None], key: str = None - ) -> Union[Set, Dict, None]: - """ - Proxy to ExcludableMixin.get_child for inclusions. - - :param include: bag of items to include - :type include: Union[Set, Dict, None] - :param key: name of the child to extract - :type key: str - :return: child extracted from items if exists - :rtype: Union[Set, Dict, None] - """ - return ExcludableMixin.get_child(items=include, key=key) - - @staticmethod - def is_excluded(exclude: Union[Set, Dict, None], key: str = None) -> bool: - """ - Checks if given key should be excluded on model/ dict. - - :param exclude: bag of items to exclude - :type exclude: Union[Set, Dict, None] - :param key: name of the child to extract - :type key: str - :return: child extracted from items if exists - :rtype: Union[Set, Dict, None] - """ - if exclude is None: - return False - if exclude is Ellipsis: # pragma: nocover - return True - to_exclude = ExcludableMixin.get_excluded(exclude=exclude, key=key) - if isinstance(to_exclude, Set): - return key in to_exclude - if to_exclude is ...: - return True - return False - - @staticmethod - def is_included(include: Union[Set, Dict, None], key: str = None) -> bool: - """ - Checks if given key should be included on model/ dict. - - :param include: bag of items to include - :type include: Union[Set, Dict, None] - :param key: name of the child to extract - :type key: str - :return: child extracted from items if exists - :rtype: Union[Set, Dict, None] - """ - if include is None: - return True - if include is Ellipsis: - return True - to_include = ExcludableMixin.get_included(include=include, key=key) - if isinstance(to_include, Set): - return key in to_include - if to_include is ...: - return True - return False - @staticmethod def _populate_pk_column( - model: Union[Type["Model"], Type["ModelRow"]], - columns: List[str], - use_alias: bool = False, + model: Union[Type["Model"], Type["ModelRow"]], + columns: List[str], + use_alias: bool = False, ) -> List[str]: """ Adds primary key column/alias (depends on use_alias flag) to list of @@ -160,13 +82,12 @@ class ExcludableMixin(RelationMixin): @classmethod def own_table_columns( - cls, - model: Union[Type["Model"], Type["ModelRow"]], - excludable: ExcludableItems, - alias: str = '', - use_alias: bool = False, + cls, + model: Union[Type["Model"], Type["ModelRow"]], + excludable: ExcludableItems, + alias: str = "", + use_alias: bool = False, ) -> List[str]: - # TODO update docstring """ Returns list of aliases or field names for given model. Aliases/names switch is use_alias flag. @@ -176,6 +97,10 @@ class ExcludableMixin(RelationMixin): Primary key field is always added and cannot be excluded (will be added anyway). + :param alias: relation prefix + :type alias: str + :param excludable: structure of fields to include and exclude + :type excludable: ExcludableItems :param model: model on columns are selected :type model: Type["Model"] :param use_alias: flag if aliases or field names should be used @@ -183,7 +108,7 @@ class ExcludableMixin(RelationMixin): :return: list of column field names or aliases :rtype: List[str] """ - model_excludable = excludable.get(model_cls=model, alias=alias) + model_excludable = excludable.get(model_cls=model, alias=alias) # type: ignore columns = [ model.get_column_name_from_alias(col.name) if not use_alias else col.name for col in model.Meta.table.columns @@ -214,9 +139,9 @@ class ExcludableMixin(RelationMixin): @classmethod def _update_excluded_with_related_not_required( - cls, - exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None], - nested: bool = False, + cls, + exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None], + nested: bool = False, ) -> Union[Set, Dict]: """ Used during generation of the dict(). @@ -243,11 +168,7 @@ class ExcludableMixin(RelationMixin): return exclude @classmethod - def get_names_to_exclude( - cls, - excludable: ExcludableItems, - alias: str - ) -> Set: + def get_names_to_exclude(cls, excludable: ExcludableItems, alias: str) -> Set: """ Returns a set of models field names that should be explicitly excluded during model initialization. @@ -268,7 +189,7 @@ class ExcludableMixin(RelationMixin): model = cast(Type["Model"], cls) model_excludable = excludable.get(model_cls=model, alias=alias) fields_names = cls.extract_db_own_fields() - if model_excludable.include and model_excludable.include_all: + if model_excludable.include: fields_to_keep = model_excludable.include.intersection(fields_names) else: fields_to_keep = fields_names diff --git a/ormar/models/model_row.py b/ormar/models/model_row.py index 6a6cb0e..1b15e6d 100644 --- a/ormar/models/model_row.py +++ b/ormar/models/model_row.py @@ -3,11 +3,9 @@ from typing import ( Dict, List, Optional, - Set, TYPE_CHECKING, Type, TypeVar, - Union, cast, ) @@ -17,7 +15,6 @@ from ormar.models import NewBaseModel # noqa: I202 from ormar.models.excludable import ExcludableItems from ormar.models.helpers.models import group_related_list - if TYPE_CHECKING: # pragma: no cover from ormar.fields import ForeignKeyField from ormar.models import T @@ -36,6 +33,7 @@ class ModelRow(NewBaseModel): related_field: Type["ForeignKeyField"] = None, excludable: ExcludableItems = None, current_relation_str: str = "", + proxy_source_model: Optional[Type["ModelRow"]] = None, ) -> Optional[T]: """ Model method to convert raw sql row from database into ormar.Model instance. @@ -91,12 +89,10 @@ class ModelRow(NewBaseModel): excludable=excludable, current_relation_str=current_relation_str, source_model=source_model, + proxy_source_model=proxy_source_model, # type: ignore ) item = cls.extract_prefixed_table_columns( - item=item, - row=row, - table_prefix=table_prefix, - excludable=excludable + item=item, row=row, table_prefix=table_prefix, excludable=excludable ) instance: Optional[T] = None @@ -117,6 +113,7 @@ class ModelRow(NewBaseModel): related_models: Any, excludable: ExcludableItems, current_relation_str: str = None, + proxy_source_model: Type[T] = None, ) -> dict: """ Traverses structure of related models and populates the nested models @@ -165,20 +162,22 @@ class ModelRow(NewBaseModel): excludable=excludable, current_relation_str=relation_str, source_model=source_model, + proxy_source_model=proxy_source_model, ) item[model_cls.get_column_name_from_alias(related)] = child if field.is_multi and child: - # TODO: way to figure out which side should be populated? through_name = cls.Meta.model_fields[related].through.get_name() - # for now it's nested dict, should be instance? through_child = cls.populate_through_instance( row=row, related=related, through_name=through_name, - excludable=excludable + excludable=excludable, ) - item[through_name] = through_child - setattr(child, through_name, through_child) + + if child.__class__ != proxy_source_model: + setattr(child, through_name, through_child) + else: + item[through_name] = through_child child.set_save_status(True) return item @@ -189,19 +188,24 @@ class ModelRow(NewBaseModel): row: sqlalchemy.engine.ResultProxy, through_name: str, related: str, - excludable: ExcludableItems - ) -> Dict: - # TODO: fix excludes and includes and docstring + excludable: ExcludableItems, + ) -> "ModelRow": model_cls = cls.Meta.model_fields[through_name].to table_prefix = cls.Meta.alias_manager.resolve_relation_alias( from_model=cls, relation_name=related ) - child = model_cls.extract_prefixed_table_columns( - item={}, - row=row, - excludable=excludable, - table_prefix=table_prefix + # remove relations on through field + model_excludable = excludable.get(model_cls=model_cls, alias=table_prefix) + model_excludable.set_values( + value=model_cls.extract_related_names(), is_exclude=True ) + child_dict = model_cls.extract_prefixed_table_columns( + item={}, row=row, excludable=excludable, table_prefix=table_prefix + ) + child_dict["__excluded__"] = model_cls.get_names_to_exclude( + excludable=excludable, alias=table_prefix + ) + child = model_cls(**child_dict) # type: ignore return child @classmethod @@ -210,7 +214,7 @@ class ModelRow(NewBaseModel): item: dict, row: sqlalchemy.engine.result.ResultProxy, table_prefix: str, - excludable: ExcludableItems + excludable: ExcludableItems, ) -> Dict: """ Extracts own fields from raw sql result, using a given prefix. @@ -242,10 +246,7 @@ class ModelRow(NewBaseModel): source = row._row if cls.db_backend_name() == "postgresql" else row selected_columns = cls.own_table_columns( - model=cls, - excludable=excludable, - alias=table_prefix, - use_alias=False, + model=cls, excludable=excludable, alias=table_prefix, use_alias=False, ) for column in cls.Meta.table.columns: diff --git a/ormar/queryset/join.py b/ormar/queryset/join.py index a6f1e93..b9e71df 100644 --- a/ormar/queryset/join.py +++ b/ormar/queryset/join.py @@ -1,14 +1,11 @@ from collections import OrderedDict from typing import ( Any, - Dict, List, Optional, - Set, TYPE_CHECKING, Tuple, Type, - Union, ) import sqlalchemy @@ -16,12 +13,12 @@ from sqlalchemy import text import ormar # noqa I100 from ormar.exceptions import RelationshipInstanceError -from ormar.models.excludable import ExcludableItems from ormar.relations import AliasManager if TYPE_CHECKING: # pragma no cover from ormar import Model from ormar.queryset import OrderAction + from ormar.models.excludable import ExcludableItems class SqlJoin: @@ -30,7 +27,7 @@ class SqlJoin: used_aliases: List, select_from: sqlalchemy.sql.select, columns: List[sqlalchemy.Column], - excludable: ExcludableItems, + excludable: "ExcludableItems", order_columns: Optional[List["OrderAction"]], sorted_orders: OrderedDict, main_model: Type["Model"], @@ -44,7 +41,7 @@ class SqlJoin: self.related_models = related_models or [] self.select_from = select_from self.columns = columns - self.excludable=excludable + self.excludable = excludable self.order_columns = order_columns self.sorted_orders = sorted_orders self.main_model = main_model @@ -296,7 +293,6 @@ class SqlJoin: self._get_order_bys() - # TODO: fix fields and exclusions for through model? self_related_fields = self.next_model.own_table_columns( model=self.next_model, excludable=self.excludable, diff --git a/ormar/queryset/prefetch_query.py b/ormar/queryset/prefetch_query.py index 88fc8e3..a661b73 100644 --- a/ormar/queryset/prefetch_query.py +++ b/ormar/queryset/prefetch_query.py @@ -1,19 +1,15 @@ from typing import ( - Any, Dict, List, - Optional, Sequence, Set, TYPE_CHECKING, Tuple, Type, - Union, cast, ) import ormar -from ormar.models.excludable import ExcludableItems from ormar.queryset.clause import QueryClause from ormar.queryset.query import Query from ormar.queryset.utils import extract_models_to_dict_of_lists, translate_list_to_dict @@ -22,29 +18,7 @@ if TYPE_CHECKING: # pragma: no cover from ormar import Model from ormar.fields import ForeignKeyField, BaseField from ormar.queryset import OrderAction - - -def add_relation_field_to_fields( - fields: Union[Set[Any], Dict[Any, Any], None], related_field_name: str -) -> Union[Set[Any], Dict[Any, Any], None]: - """ - Adds related field into fields to include as otherwise it would be skipped. - Related field is added only if fields are already populated. - Empty fields implies all fields. - - :param fields: Union[Set[Any], Dict[Any, Any], None] - :type fields: Dict - :param related_field_name: name of the field with relation - :type related_field_name: str - :return: updated fields dict - :rtype: Union[Set[Any], Dict[Any, Any], None] - """ - if fields and related_field_name not in fields: - if isinstance(fields, dict): - fields[related_field_name] = ... - elif isinstance(fields, set): - fields.add(related_field_name) - return fields + from ormar.models.excludable import ExcludableItems def sort_models(models: List["Model"], orders_by: Dict) -> List["Model"]: @@ -74,12 +48,12 @@ def sort_models(models: List["Model"], orders_by: Dict) -> List["Model"]: def set_children_on_model( # noqa: CCR001 - model: "Model", - related: str, - children: Dict, - model_id: int, - models: Dict, - orders_by: Dict, + model: "Model", + related: str, + children: Dict, + model_id: int, + models: Dict, + orders_by: Dict, ) -> None: """ Extract ids of child models by given relation id key value. @@ -124,12 +98,12 @@ class PrefetchQuery: """ def __init__( # noqa: CFQ002 - self, - model_cls: Type["Model"], - excludable: ExcludableItems, - prefetch_related: List, - select_related: List, - orders_by: List["OrderAction"], + self, + model_cls: Type["Model"], + excludable: "ExcludableItems", + prefetch_related: List, + select_related: List, + orders_by: List["OrderAction"], ) -> None: self.model = model_cls @@ -147,7 +121,7 @@ class PrefetchQuery: ) async def prefetch_related( - self, models: Sequence["Model"], rows: List + self, models: Sequence["Model"], rows: List ) -> Sequence["Model"]: """ Main entry point for prefetch_query. @@ -172,7 +146,7 @@ class PrefetchQuery: return await self._prefetch_related_models(models=models, rows=rows) def _extract_ids_from_raw_data( - self, parent_model: Type["Model"], column_name: str + self, parent_model: Type["Model"], column_name: str ) -> Set: """ Iterates over raw rows and extract id values of relation columns by using @@ -195,7 +169,7 @@ class PrefetchQuery: return list_of_ids def _extract_ids_from_preloaded_models( - self, parent_model: Type["Model"], column_name: str + self, parent_model: Type["Model"], column_name: str ) -> Set: """ Extracts relation ids from already populated models if they were included @@ -218,7 +192,7 @@ class PrefetchQuery: return list_of_ids def _extract_required_ids( - self, parent_model: Type["Model"], reverse: bool, related: str, + self, parent_model: Type["Model"], reverse: bool, related: str, ) -> Set: """ Delegates extraction of the fields to either get ids from raw sql response @@ -252,11 +226,11 @@ class PrefetchQuery: ) def _get_filter_for_prefetch( - self, - parent_model: Type["Model"], - target_model: Type["Model"], - reverse: bool, - related: str, + self, + parent_model: Type["Model"], + target_model: Type["Model"], + reverse: bool, + related: str, ) -> List: """ Populates where clause with condition to return only models within the @@ -297,7 +271,7 @@ class PrefetchQuery: return [] def _populate_nested_related( - self, model: "Model", prefetch_dict: Dict, orders_by: Dict, + self, model: "Model", prefetch_dict: Dict, orders_by: Dict, ) -> "Model": """ Populates all related models children of parent model that are @@ -341,7 +315,7 @@ class PrefetchQuery: return model async def _prefetch_related_models( - self, models: Sequence["Model"], rows: List + self, models: Sequence["Model"], rows: List ) -> Sequence["Model"]: """ Main method of the query. @@ -385,13 +359,13 @@ class PrefetchQuery: return models async def _extract_related_models( # noqa: CFQ002, CCR001 - self, - related: str, - target_model: Type["Model"], - prefetch_dict: Dict, - select_dict: Dict, - excludable: ExcludableItems, - orders_by: Dict, + self, + related: str, + target_model: Type["Model"], + prefetch_dict: Dict, + select_dict: Dict, + excludable: "ExcludableItems", + orders_by: Dict, ) -> None: """ Constructs queries with required ids and extracts data with fields that should @@ -443,15 +417,16 @@ class PrefetchQuery: related_field_name = parent_model.get_related_field_name( target_field=target_field ) - table_prefix, rows = await self._run_prefetch_query( + table_prefix, exclude_prefix, rows = await self._run_prefetch_query( target_field=target_field, excludable=excludable, filter_clauses=filter_clauses, - related_field_name=related_field_name + related_field_name=related_field_name, ) else: rows = [] table_prefix = "" + exclude_prefix = "" if prefetch_dict and prefetch_dict is not Ellipsis: for subrelated in prefetch_dict.keys(): @@ -472,6 +447,7 @@ class PrefetchQuery: parent_model=parent_model, target_field=target_field, table_prefix=table_prefix, + exclude_prefix=exclude_prefix, excludable=excludable, prefetch_dict=prefetch_dict, orders_by=orders_by, @@ -484,12 +460,12 @@ class PrefetchQuery: ) async def _run_prefetch_query( - self, - target_field: Type["BaseField"], - excludable: ExcludableItems, - filter_clauses: List, - related_field_name: str - ) -> Tuple[str, List]: + self, + target_field: Type["BaseField"], + excludable: "ExcludableItems", + filter_clauses: List, + related_field_name: str, + ) -> Tuple[str, str, List]: """ Actually runs the queries against the database and populates the raw response for given related model. @@ -509,17 +485,22 @@ class PrefetchQuery: select_related = [] query_target = target_model table_prefix = "" + exclude_prefix = target_field.to.Meta.alias_manager.resolve_relation_alias( + from_model=target_field.owner, relation_name=target_field.name + ) if target_field.is_multi: query_target = target_field.through select_related = [target_name] table_prefix = target_field.to.Meta.alias_manager.resolve_relation_alias( from_model=query_target, relation_name=target_name ) + exclude_prefix = table_prefix self.already_extracted.setdefault(target_name, {})["prefix"] = table_prefix - model_excludable = excludable.get(model_cls=target_model, alias=table_prefix) + model_excludable = excludable.get(model_cls=target_model, alias=exclude_prefix) if model_excludable.include and not model_excludable.is_included( - related_field_name): + related_field_name + ): model_excludable.set_values({related_field_name}, is_exclude=False) qry = Query( @@ -537,7 +518,7 @@ class PrefetchQuery: # print(expr.compile(compile_kwargs={"literal_binds": True})) rows = await self.database.fetch_all(expr) self.already_extracted.setdefault(target_name, {}).update({"raw": rows}) - return table_prefix, rows + return table_prefix, exclude_prefix, rows @staticmethod def _get_select_related_if_apply(related: str, select_dict: Dict) -> Dict: @@ -559,7 +540,7 @@ class PrefetchQuery: ) def _update_already_loaded_rows( # noqa: CFQ002 - self, target_field: Type["BaseField"], prefetch_dict: Dict, orders_by: Dict, + self, target_field: Type["BaseField"], prefetch_dict: Dict, orders_by: Dict, ) -> None: """ Updates models that are already loaded, usually children of children. @@ -578,14 +559,15 @@ class PrefetchQuery: ) def _populate_rows( # noqa: CFQ002 - self, - rows: List, - target_field: Type["ForeignKeyField"], - parent_model: Type["Model"], - table_prefix: str, - excludable: ExcludableItems, - prefetch_dict: Dict, - orders_by: Dict, + self, + rows: List, + target_field: Type["ForeignKeyField"], + parent_model: Type["Model"], + table_prefix: str, + exclude_prefix: str, + excludable: "ExcludableItems", + prefetch_dict: Dict, + orders_by: Dict, ) -> None: """ Instantiates children models extracted from given relation. @@ -617,13 +599,10 @@ class PrefetchQuery: # TODO Fix fields field_name = parent_model.get_related_field_name(target_field=target_field) item = target_model.extract_prefixed_table_columns( - item={}, - row=row, - table_prefix=table_prefix, - excludable=excludable, + item={}, row=row, table_prefix=table_prefix, excludable=excludable, ) item["__excluded__"] = target_model.get_names_to_exclude( - excludable=excludable, alias=table_prefix + excludable=excludable, alias=exclude_prefix ) instance = target_model(**item) instance = self._populate_nested_related( diff --git a/ormar/queryset/query.py b/ormar/queryset/query.py index 2e88212..0987bac 100644 --- a/ormar/queryset/query.py +++ b/ormar/queryset/query.py @@ -1,12 +1,10 @@ -import copy from collections import OrderedDict -from typing import Dict, List, Optional, Set, TYPE_CHECKING, Tuple, Type, Union +from typing import List, Optional, TYPE_CHECKING, Tuple, Type import sqlalchemy from sqlalchemy import text import ormar # noqa I100 -from ormar.models.excludable import ExcludableItems from ormar.models.helpers.models import group_related_list from ormar.queryset import FilterQuery, LimitQuery, OffsetQuery, OrderQuery from ormar.queryset.actions.filter_action import FilterAction @@ -15,20 +13,21 @@ from ormar.queryset.join import SqlJoin if TYPE_CHECKING: # pragma no cover from ormar import Model from ormar.queryset import OrderAction + from ormar.models.excludable import ExcludableItems class Query: def __init__( # noqa CFQ002 - self, - model_cls: Type["Model"], - filter_clauses: List[FilterAction], - exclude_clauses: List[FilterAction], - select_related: List, - limit_count: Optional[int], - offset: Optional[int], - excludable: ExcludableItems, - order_bys: Optional[List["OrderAction"]], - limit_raw_sql: bool, + self, + model_cls: Type["Model"], + filter_clauses: List[FilterAction], + exclude_clauses: List[FilterAction], + select_related: List, + limit_count: Optional[int], + offset: Optional[int], + excludable: "ExcludableItems", + order_bys: Optional[List["OrderAction"]], + limit_raw_sql: bool, ) -> None: self.query_offset = offset self.limit_count = limit_count @@ -103,9 +102,7 @@ class Query: :rtype: sqlalchemy.sql.selectable.Select """ self_related_fields = self.model_cls.own_table_columns( - model=self.model_cls, - excludable=self.excludable, - use_alias=True, + model=self.model_cls, excludable=self.excludable, use_alias=True, ) self.columns = self.model_cls.Meta.alias_manager.prefixed_columns( "", self.table, self_related_fields @@ -191,7 +188,7 @@ class Query: return expr def _apply_expression_modifiers( - self, expr: sqlalchemy.sql.select + self, expr: sqlalchemy.sql.select ) -> sqlalchemy.sql.select: """ Receives the select query (might be join) and applies: diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 1c93590..f4f9fd0 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -20,18 +20,17 @@ from sqlalchemy import bindparam import ormar # noqa I100 from ormar import MultipleMatches, NoMatch from ormar.exceptions import ModelError, ModelPersistenceError, QueryDefinitionError -from ormar.models.excludable import ExcludableItems from ormar.queryset import FilterQuery from ormar.queryset.actions.order_action import OrderAction from ormar.queryset.clause import QueryClause from ormar.queryset.prefetch_query import PrefetchQuery from ormar.queryset.query import Query -from ormar.queryset.utils import update, update_dict_from_list if TYPE_CHECKING: # pragma no cover from ormar.models import T from ormar.models.metaclass import ModelMeta from ormar.relations.querysetproxy import QuerysetProxy + from ormar.models.excludable import ExcludableItems else: T = TypeVar("T") @@ -42,18 +41,20 @@ class QuerySet(Generic[T]): """ def __init__( # noqa CFQ002 - self, - model_cls: Optional[Type[T]] = None, - filter_clauses: List = None, - exclude_clauses: List = None, - select_related: List = None, - limit_count: int = None, - offset: int = None, - excludable: ExcludableItems = None, - order_bys: List = None, - prefetch_related: List = None, - limit_raw_sql: bool = False, + self, + model_cls: Optional[Type[T]] = None, + filter_clauses: List = None, + exclude_clauses: List = None, + select_related: List = None, + limit_count: int = None, + offset: int = None, + excludable: "ExcludableItems" = None, + order_bys: List = None, + prefetch_related: List = None, + limit_raw_sql: bool = False, + proxy_source_model: Optional[Type[T]] = None, ) -> None: + self.proxy_source_model = proxy_source_model self.model_cls = model_cls self.filter_clauses = [] if filter_clauses is None else filter_clauses self.exclude_clauses = [] if exclude_clauses is None else exclude_clauses @@ -61,14 +62,14 @@ class QuerySet(Generic[T]): self._prefetch_related = [] if prefetch_related is None else prefetch_related self.limit_count = limit_count self.query_offset = offset - self._excludable = excludable or ExcludableItems() + self._excludable = excludable or ormar.ExcludableItems() self.order_bys = order_bys or [] self.limit_sql_raw = limit_raw_sql def __get__( - self, - instance: Optional[Union["QuerySet", "QuerysetProxy"]], - owner: Union[Type[T], Type["QuerysetProxy"]], + self, + instance: Optional[Union["QuerySet", "QuerysetProxy"]], + owner: Union[Type[T], Type["QuerysetProxy"]], ) -> "QuerySet": if issubclass(owner, ormar.Model): if owner.Meta.requires_ref_update: @@ -105,8 +106,53 @@ class QuerySet(Generic[T]): raise ValueError("Model class of QuerySet is not initialized") return self.model_cls + def rebuild_self( # noqa: CFQ002 + self, + filter_clauses: List = None, + exclude_clauses: List = None, + select_related: List = None, + limit_count: int = None, + offset: int = None, + excludable: "ExcludableItems" = None, + order_bys: List = None, + prefetch_related: List = None, + limit_raw_sql: bool = None, + proxy_source_model: Optional[Type[T]] = None, + ) -> "QuerySet": + """ + Method that returns new instance of queryset based on passed params, + all not passed params are taken from current values. + """ + overwrites = { + "select_related": "_select_related", + "offset": "query_offset", + "excludable": "_excludable", + "prefetch_related": "_prefetch_related", + "limit_raw_sql": "limit_sql_raw", + } + passed_args = locals() + + def replace_if_none(arg_name: str) -> Any: + if passed_args.get(arg_name) is None: + return getattr(self, overwrites.get(arg_name, arg_name)) + return passed_args.get(arg_name) + + return self.__class__( + model_cls=self.model_cls, + filter_clauses=replace_if_none("filter_clauses"), + exclude_clauses=replace_if_none("exclude_clauses"), + select_related=replace_if_none("select_related"), + limit_count=replace_if_none("limit_count"), + offset=replace_if_none("offset"), + excludable=replace_if_none("excludable"), + order_bys=replace_if_none("order_bys"), + prefetch_related=replace_if_none("prefetch_related"), + limit_raw_sql=replace_if_none("limit_raw_sql"), + proxy_source_model=replace_if_none("proxy_source_model"), + ) + async def _prefetch_related_models( - self, models: Sequence[Optional["T"]], rows: List + self, models: Sequence[Optional["T"]], rows: List ) -> Sequence[Optional["T"]]: """ Performs prefetch query for selected models names. @@ -142,6 +188,7 @@ class QuerySet(Generic[T]): select_related=self._select_related, excludable=self._excludable, source_model=self.model, + proxy_source_model=self.proxy_source_model, ) for row in rows ] @@ -183,7 +230,7 @@ class QuerySet(Generic[T]): return self.model_meta.table def build_select_expression( - self, limit: int = None, offset: int = None, order_bys: List = None, + self, limit: int = None, offset: int = None, order_bys: List = None, ) -> sqlalchemy.sql.select: """ Constructs the actual database query used in the QuerySet. @@ -254,17 +301,10 @@ class QuerySet(Generic[T]): exclude_clauses = self.exclude_clauses filter_clauses = filter_clauses - return self.__class__( - model_cls=self.model, + return self.rebuild_self( filter_clauses=filter_clauses, exclude_clauses=exclude_clauses, select_related=select_related, - limit_count=self.limit_count, - offset=self.query_offset, - excludable=self._excludable, - order_bys=self.order_bys, - prefetch_related=self._prefetch_related, - limit_raw_sql=self.limit_sql_raw, ) def exclude(self, **kwargs: Any) -> "QuerySet": # noqa: A003 @@ -309,18 +349,7 @@ class QuerySet(Generic[T]): related = [related] related = list(set(list(self._select_related) + related)) - return self.__class__( - model_cls=self.model, - filter_clauses=self.filter_clauses, - exclude_clauses=self.exclude_clauses, - select_related=related, - limit_count=self.limit_count, - offset=self.query_offset, - excludable=self._excludable, - order_bys=self.order_bys, - prefetch_related=self._prefetch_related, - limit_raw_sql=self.limit_sql_raw, - ) + return self.rebuild_self(select_related=related,) def prefetch_related(self, related: Union[List, str]) -> "QuerySet": """ @@ -344,21 +373,11 @@ class QuerySet(Generic[T]): related = [related] related = list(set(list(self._prefetch_related) + related)) - return self.__class__( - model_cls=self.model, - filter_clauses=self.filter_clauses, - exclude_clauses=self.exclude_clauses, - select_related=self._select_related, - limit_count=self.limit_count, - offset=self.query_offset, - excludable=self._excludable, - order_bys=self.order_bys, - prefetch_related=related, - limit_raw_sql=self.limit_sql_raw, - ) + return self.rebuild_self(prefetch_related=related,) - def fields(self, columns: Union[List, str, Set, Dict], - _is_exclude: bool = False) -> "QuerySet": + def fields( + self, columns: Union[List, str, Set, Dict], _is_exclude: bool = False + ) -> "QuerySet": """ With `fields()` you can select subset of model columns to limit the data load. @@ -396,29 +415,22 @@ class QuerySet(Generic[T]): To include whole nested model specify model related field name and ellipsis. + :param _is_exclude: flag if it's exclude or include operation + :type _is_exclude: bool :param columns: columns to include :type columns: Union[List, str, Set, Dict] :return: QuerySet :rtype: QuerySet """ - excludable = ExcludableItems.from_excludable(self._excludable) - excludable.build(items=columns, - model_cls=self.model_cls, - is_exclude=_is_exclude) - - return self.__class__( - model_cls=self.model, - filter_clauses=self.filter_clauses, - exclude_clauses=self.exclude_clauses, - select_related=self._select_related, - limit_count=self.limit_count, - offset=self.query_offset, - excludable=excludable, - order_bys=self.order_bys, - prefetch_related=self._prefetch_related, - limit_raw_sql=self.limit_sql_raw, + excludable = ormar.ExcludableItems.from_excludable(self._excludable) + excludable.build( + items=columns, + model_cls=self.model_cls, # type: ignore + is_exclude=_is_exclude, ) + return self.rebuild_self(excludable=excludable,) + def exclude_fields(self, columns: Union[List, str, Set, Dict]) -> "QuerySet": """ With `exclude_fields()` you can select subset of model columns that will @@ -489,18 +501,7 @@ class QuerySet(Generic[T]): ] order_bys = self.order_bys + [x for x in orders_by if x not in self.order_bys] - return self.__class__( - model_cls=self.model, - filter_clauses=self.filter_clauses, - exclude_clauses=self.exclude_clauses, - select_related=self._select_related, - limit_count=self.limit_count, - offset=self.query_offset, - excludable=self._excludable, - order_bys=order_bys, - prefetch_related=self._prefetch_related, - limit_raw_sql=self.limit_sql_raw, - ) + return self.rebuild_self(order_bys=order_bys,) async def exists(self) -> bool: """ @@ -601,18 +602,7 @@ class QuerySet(Generic[T]): limit_count = page_size query_offset = (page - 1) * page_size - return self.__class__( - model_cls=self.model, - filter_clauses=self.filter_clauses, - exclude_clauses=self.exclude_clauses, - select_related=self._select_related, - limit_count=limit_count, - offset=query_offset, - excludable=self._excludable, - order_bys=self.order_bys, - prefetch_related=self._prefetch_related, - limit_raw_sql=self.limit_sql_raw, - ) + return self.rebuild_self(limit_count=limit_count, offset=query_offset,) def limit(self, limit_count: int, limit_raw_sql: bool = None) -> "QuerySet": """ @@ -629,18 +619,7 @@ class QuerySet(Generic[T]): :rtype: QuerySet """ limit_raw_sql = self.limit_sql_raw if limit_raw_sql is None else limit_raw_sql - return self.__class__( - model_cls=self.model, - filter_clauses=self.filter_clauses, - exclude_clauses=self.exclude_clauses, - select_related=self._select_related, - limit_count=limit_count, - offset=self.query_offset, - excludable=self._excludable, - order_bys=self.order_bys, - prefetch_related=self._prefetch_related, - limit_raw_sql=limit_raw_sql, - ) + return self.rebuild_self(limit_count=limit_count, limit_raw_sql=limit_raw_sql,) def offset(self, offset: int, limit_raw_sql: bool = None) -> "QuerySet": """ @@ -657,18 +636,7 @@ class QuerySet(Generic[T]): :rtype: QuerySet """ limit_raw_sql = self.limit_sql_raw if limit_raw_sql is None else limit_raw_sql - return self.__class__( - model_cls=self.model, - filter_clauses=self.filter_clauses, - exclude_clauses=self.exclude_clauses, - select_related=self._select_related, - limit_count=self.limit_count, - offset=offset, - excludable=self._excludable, - order_bys=self.order_bys, - prefetch_related=self._prefetch_related, - limit_raw_sql=limit_raw_sql, - ) + return self.rebuild_self(offset=offset, limit_raw_sql=limit_raw_sql,) async def first(self, **kwargs: Any) -> T: """ @@ -687,12 +655,12 @@ class QuerySet(Generic[T]): expr = self.build_select_expression( limit=1, order_bys=[ - OrderAction( - order_str=f"{self.model.Meta.pkname}", - model_cls=self.model_cls, # type: ignore - ) - ] - + self.order_bys, + OrderAction( + order_str=f"{self.model.Meta.pkname}", + model_cls=self.model_cls, # type: ignore + ) + ] + + self.order_bys, ) rows = await self.database.fetch_all(expr) processed_rows = self._process_query_result_rows(rows) @@ -723,12 +691,12 @@ class QuerySet(Generic[T]): expr = self.build_select_expression( limit=1, order_bys=[ - OrderAction( - order_str=f"-{self.model.Meta.pkname}", - model_cls=self.model_cls, # type: ignore - ) - ] - + self.order_bys, + OrderAction( + order_str=f"-{self.model.Meta.pkname}", + model_cls=self.model_cls, # type: ignore + ) + ] + + self.order_bys, ) else: expr = self.build_select_expression() @@ -831,9 +799,9 @@ class QuerySet(Generic[T]): # refresh server side defaults if any( - field.server_default is not None - for name, field in self.model.Meta.model_fields.items() - if name not in kwargs + field.server_default is not None + for name, field in self.model.Meta.model_fields.items() + if name not in kwargs ): instance = await instance.load() instance.set_save_status(True) @@ -868,7 +836,7 @@ class QuerySet(Generic[T]): objt.set_save_status(True) async def bulk_update( # noqa: CCR001 - self, objects: List[T], columns: List[str] = None + self, objects: List[T], columns: List[str] = None ) -> None: """ Performs bulk update in one database session to speed up the process. diff --git a/ormar/relations/relation_proxy.py b/ormar/relations/relation_proxy.py index 58d6e9e..596f594 100644 --- a/ormar/relations/relation_proxy.py +++ b/ormar/relations/relation_proxy.py @@ -119,7 +119,9 @@ class RelationProxy(list): self._check_if_model_saved() kwargs = {f"{related_field.get_alias()}__{pkname}": self._owner.pk} queryset = ( - ormar.QuerySet(model_cls=self.relation.to) + ormar.QuerySet( + model_cls=self.relation.to, proxy_source_model=self._owner.__class__ + ) .select_related(related_field.name) .filter(**kwargs) ) diff --git a/tests/test_choices_schema.py b/tests/test_choices_schema.py index 0cf1852..978c3f6 100644 --- a/tests/test_choices_schema.py +++ b/tests/test_choices_schema.py @@ -22,7 +22,7 @@ uuid1 = uuid.uuid4() uuid2 = uuid.uuid4() -class TestEnum(Enum): +class EnumTest(Enum): val1 = "Val1" val2 = "Val2" @@ -56,7 +56,7 @@ class Organisation(ormar.Model): ) random_json: pydantic.Json = ormar.JSON(choices=["aa", '{"aa":"bb"}']) random_uuid: uuid.UUID = ormar.UUID(choices=[uuid1, uuid2]) - enum_string: str = ormar.String(max_length=100, choices=list(TestEnum)) + enum_string: str = ormar.String(max_length=100, choices=list(EnumTest)) @app.on_event("startup") @@ -110,7 +110,7 @@ def test_all_endpoints(): "random_decimal": 12.4, "random_json": '{"aa":"bb"}', "random_uuid": str(uuid1), - "enum_string": TestEnum.val1.value, + "enum_string": EnumTest.val1.value, }, ) diff --git a/tests/test_m2m_through_fields.py b/tests/test_m2m_through_fields.py index 8dd8bba..6c4211a 100644 --- a/tests/test_m2m_through_fields.py +++ b/tests/test_m2m_through_fields.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, List, Sequence, cast import databases import pytest @@ -131,6 +131,7 @@ async def test_getting_additional_fields_from_queryset() -> Any: ) await post.categories.all() + assert post.postcategory is None assert post.categories[0].postcategory.sort_order == 1 assert post.categories[1].postcategory.sort_order == 2 @@ -138,8 +139,31 @@ async def test_getting_additional_fields_from_queryset() -> Any: categories__name="Test category2" ) assert post2.categories[0].postcategory.sort_order == 2 - # if TYPE_CHECKING: - # reveal_type(post2) + + +@pytest.mark.asyncio +async def test_only_one_side_has_through() -> Any: + async with database: + post = await Post(title="Test post").save() + await post.categories.create( + name="Test category1", postcategory={"sort_order": 1} + ) + await post.categories.create( + name="Test category2", postcategory={"sort_order": 2} + ) + + post2 = await Post.objects.select_related("categories").get() + assert post2.postcategory is None + assert post2.categories[0].postcategory is not None + + await post2.categories.all() + assert post2.postcategory is None + assert post2.categories[0].postcategory is not None + + categories = await Category.objects.select_related("posts").all() + categories = cast(Sequence[Category], categories) + assert categories[0].postcategory is None + assert categories[0].posts[0].postcategory is not None @pytest.mark.asyncio @@ -294,7 +318,6 @@ async def test_update_through_from_related() -> Any: @pytest.mark.asyncio -@pytest.mark.skip # TODO: Restore after finished exclude refactor async def test_excluding_fields_on_through_model() -> Any: async with database: post = await Post(title="Test post").save() @@ -323,6 +346,17 @@ async def test_excluding_fields_on_through_model() -> Any: assert post2.categories[2].postcategory.param_name is None assert post2.categories[2].postcategory.sort_order == 3 + post3 = ( + await Post.objects.select_related("categories") + .fields({"postcategory": ..., "title": ...}) + .exclude_fields({"postcategory": {"param_name", "sort_order"}}) + .get() + ) + assert len(post3.categories) == 3 + for category in post3.categories: + assert category.postcategory.param_name is None + assert category.postcategory.sort_order is None + # TODO: check/ modify following @@ -337,9 +371,9 @@ async def test_excluding_fields_on_through_model() -> Any: # ordering by in order_by (V) # updating in query (V) # updating from querysetproxy (V) +# including/excluding in fields? # modifying from instance (both sides?) (X) <= no, the loaded one doesn't have relations -# including/excluding in fields? # allowing to change fk fields names in through model? # make through optional? auto-generated for cases other fields are missing? diff --git a/tests/test_queryset_utils.py b/tests/test_queryset_utils.py index daae2b4..cd96dc8 100644 --- a/tests/test_queryset_utils.py +++ b/tests/test_queryset_utils.py @@ -8,11 +8,6 @@ from ormar.queryset.utils import translate_list_to_dict, update_dict_from_list, from tests.settings import DATABASE_URL -def test_empty_excludable(): - assert ExcludableMixin.is_included(None, "key") # all fields included if empty - assert not ExcludableMixin.is_excluded(None, "key") # none field excluded if empty - - def test_list_to_dict_translation(): tet_list = ["aa", "bb", "cc__aa", "cc__bb", "cc__aa__xx", "cc__aa__yy"] test = translate_list_to_dict(tet_list) From 9ad1528cc0ce233e70607f349731a80a4507eb8e Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 2 Mar 2021 19:10:59 +0100 Subject: [PATCH 11/14] add docstrings, clean types test --- ormar/fields/foreign_key.py | 2 +- ormar/models/__init__.py | 4 +- ormar/models/excludable.py | 97 ++++++++++++++++++++++- ormar/models/model.py | 17 ++-- ormar/models/model_row.py | 43 ++++++---- ormar/models/newbasemodel.py | 11 +-- ormar/queryset/__init__.py | 3 +- ormar/queryset/prefetch_query.py | 1 - ormar/queryset/queryset.py | 44 +++++----- ormar/relations/querysetproxy.py | 39 +++++---- ormar/relations/relation.py | 8 +- ormar/relations/relation_manager.py | 4 +- tests/test_excluding_fields_in_fastapi.py | 15 ++-- 13 files changed, 190 insertions(+), 98 deletions(-) diff --git a/ormar/fields/foreign_key.py b/ormar/fields/foreign_key.py index 8dda14f..643b88d 100644 --- a/ormar/fields/foreign_key.py +++ b/ormar/fields/foreign_key.py @@ -73,7 +73,7 @@ def create_dummy_model( "".join(choices(string.ascii_uppercase, k=2)) + uuid.uuid4().hex[:4] ).lower() fields = {f"{pk_field.name}": (pk_field.__type__, None)} - dummy_model = create_model( + dummy_model = create_model( # type: ignore f"PkOnly{base_model.get_name(lower=False)}{alias}", __module__=base_model.__module__, **fields, # type: ignore diff --git a/ormar/models/__init__.py b/ormar/models/__init__.py index 0ecd9cc..eb6bdd7 100644 --- a/ormar/models/__init__.py +++ b/ormar/models/__init__.py @@ -6,7 +6,7 @@ ass well as vast number of helper functions for pydantic, sqlalchemy and relatio from ormar.models.newbasemodel import NewBaseModel # noqa I100 from ormar.models.model_row import ModelRow # noqa I100 -from ormar.models.model import Model, T # noqa I100 +from ormar.models.model import Model # noqa I100 from ormar.models.excludable import ExcludableItems # noqa I100 -__all__ = ["T", "NewBaseModel", "Model", "ModelRow", "ExcludableItems"] +__all__ = ["NewBaseModel", "Model", "ModelRow", "ExcludableItems"] diff --git a/ormar/models/excludable.py b/ormar/models/excludable.py index 203d8a0..9b888b0 100644 --- a/ormar/models/excludable.py +++ b/ormar/models/excludable.py @@ -7,28 +7,58 @@ if TYPE_CHECKING: # pragma: no cover from ormar import Model -# TODO: Add docstrings @dataclass class Excludable: + """ + Class that keeps sets of fields to exclude and include + """ + include: Set = field(default_factory=set) exclude: Set = field(default_factory=set) def get_copy(self) -> "Excludable": + """ + Return copy of self to avoid in place modifications + :return: copy of self with copied sets + :rtype: ormar.models.excludable.Excludable + """ _copy = self.__class__() _copy.include = {x for x in self.include} _copy.exclude = {x for x in self.exclude} return _copy def set_values(self, value: Set, is_exclude: bool) -> None: + """ + Appends the data to include/exclude sets. + + :param value: set of values to add + :type value: set + :param is_exclude: flag if values are to be excluded or included + :type is_exclude: bool + """ prop = "exclude" if is_exclude else "include" current_value = getattr(self, prop) current_value.update(value) setattr(self, prop, current_value) def is_included(self, key: str) -> bool: + """ + Check if field in included (in set or set is {...}) + :param key: key to check + :type key: str + :return: result of the check + :rtype: bool + """ return (... in self.include or key in self.include) if self.include else True def is_excluded(self, key: str) -> bool: + """ + Check if field in excluded (in set or set is {...}) + :param key: key to check + :type key: str + :return: result of the check + :rtype: bool + """ return (... in self.exclude or key in self.exclude) if self.exclude else False @@ -44,12 +74,30 @@ class ExcludableItems: @classmethod def from_excludable(cls, other: "ExcludableItems") -> "ExcludableItems": + """ + Copy passed ExcludableItems to avoid inplace modifications. + + :param other: other excludable items to be copied + :type other: ormar.models.excludable.ExcludableItems + :return: copy of other + :rtype: ormar.models.excludable.ExcludableItems + """ new_excludable = cls() for key, value in other.items.items(): new_excludable.items[key] = value.get_copy() return new_excludable def get(self, model_cls: Type["Model"], alias: str = "") -> Excludable: + """ + Return Excludable for given model and alias. + + :param model_cls: target model to check + :type model_cls: ormar.models.metaclass.ModelMetaclass + :param alias: table alias from relation manager + :type alias: str + :return: Excludable for given model and alias + :rtype: ormar.models.excludable.Excludable + """ key = f"{alias + '_' if alias else ''}{model_cls.get_name(lower=True)}" excludable = self.items.get(key) if not excludable: @@ -63,7 +111,19 @@ class ExcludableItems: model_cls: Type["Model"], is_exclude: bool = False, ) -> None: + """ + Receives the one of the types of items and parses them as to achieve + a end situation with one excludable per alias/model in relation. + Each excludable has two sets of values - one to include, one to exclude. + + :param items: values to be included or excluded + :type items: Union[List[str], str, Tuple[str], Set[str], Dict] + :param model_cls: source model from which relations are constructed + :type model_cls: ormar.models.metaclass.ModelMetaclass + :param is_exclude: flag if items should be included or excluded + :type is_exclude: bool + """ if isinstance(items, str): items = {items} @@ -92,7 +152,18 @@ class ExcludableItems: def _set_excludes( self, items: Set, model_name: str, is_exclude: bool, alias: str = "" ) -> None: + """ + Sets set of values to be included or excluded for given key and model. + :param items: items to include/exclude + :type items: set + :param model_name: name of model to construct key + :type model_name: str + :param is_exclude: flag if values should be included or excluded + :type is_exclude: bool + :param alias: + :type alias: str + """ key = f"{alias + '_' if alias else ''}{model_name}" excludable = self.items.get(key) if not excludable: @@ -109,7 +180,22 @@ class ExcludableItems: related_items: List = None, alias: str = "", ) -> None: + """ + Goes through dict of nested values and construct/update Excludables. + :param values: items to include/exclude + :type values: Dict + :param source_model: source model from which relations are constructed + :type source_model: ormar.models.metaclass.ModelMetaclass + :param model_cls: model from which current relation is constructed + :type model_cls: ormar.models.metaclass.ModelMetaclass + :param is_exclude: flag if values should be included or excluded + :type is_exclude: bool + :param related_items: list of names of related fields chain + :type related_items: List + :param alias: alias of relation + :type alias: str + """ self_fields = set() related_items = related_items[:] if related_items else [] for key, value in values.items(): @@ -160,7 +246,16 @@ class ExcludableItems: def _traverse_list( self, values: Set[str], model_cls: Type["Model"], is_exclude: bool ) -> None: + """ + Goes through list of values and construct/update Excludables. + :param values: items to include/exclude + :type values: set + :param model_cls: model from which current relation is constructed + :type model_cls: ormar.models.metaclass.ModelMetaclass + :param is_exclude: flag if values should be included or excluded + :type is_exclude: bool + """ # here we have only nested related keys for key in values: key_split = key.split("__") diff --git a/ormar/models/model.py b/ormar/models/model.py index 1535113..2c14888 100644 --- a/ormar/models/model.py +++ b/ormar/models/model.py @@ -3,7 +3,6 @@ from typing import ( Set, TYPE_CHECKING, Tuple, - TypeVar, ) import ormar.queryset # noqa I100 @@ -15,20 +14,18 @@ from ormar.models.model_row import ModelRow if TYPE_CHECKING: # pragma nocover from ormar import QuerySet -T = TypeVar("T", bound="Model") - class Model(ModelRow): __abstract__ = False if TYPE_CHECKING: # pragma nocover Meta: ModelMeta - objects: "QuerySet[Model]" + objects: "QuerySet" def __repr__(self) -> str: # pragma nocover _repr = {k: getattr(self, k) for k, v in self.Meta.model_fields.items()} return f"{self.__class__.__name__}({str(_repr)})" - async def upsert(self: T, **kwargs: Any) -> T: + async def upsert(self, **kwargs: Any) -> "Model": """ Performs either a save or an update depending on the presence of the pk. If the pk field is filled it's an update, otherwise the save is performed. @@ -43,7 +40,7 @@ class Model(ModelRow): return await self.save() return await self.update(**kwargs) - async def save(self: T) -> T: + async def save(self) -> "Model": """ Performs a save of given Model instance. If primary key is already saved, db backend will throw integrity error. @@ -160,7 +157,7 @@ class Model(ModelRow): @staticmethod async def _update_and_follow( - rel: T, follow: bool, visited: Set, update_count: int + rel: "Model", follow: bool, visited: Set, update_count: int ) -> Tuple[int, Set]: """ Internal method used in save_related to follow related models and update numbers @@ -189,7 +186,7 @@ class Model(ModelRow): update_count += 1 return update_count, visited - async def update(self: T, **kwargs: Any) -> T: + async def update(self, **kwargs: Any) -> "Model": """ Performs update of Model instance in the database. Fields can be updated before or you can pass them as kwargs. @@ -225,7 +222,7 @@ class Model(ModelRow): await self.signals.post_update.send(sender=self.__class__, instance=self) return self - async def delete(self: T) -> int: + async def delete(self) -> int: """ Removes the Model instance from the database. @@ -248,7 +245,7 @@ class Model(ModelRow): await self.signals.post_delete.send(sender=self.__class__, instance=self) return result - async def load(self: T) -> T: + async def load(self) -> "Model": """ Allow to refresh existing Models fields from database. Be careful as the related models can be overwritten by pk_only models in load. diff --git a/ormar/models/model_row.py b/ormar/models/model_row.py index 1b15e6d..c4ea9d0 100644 --- a/ormar/models/model_row.py +++ b/ormar/models/model_row.py @@ -5,7 +5,6 @@ from typing import ( Optional, TYPE_CHECKING, Type, - TypeVar, cast, ) @@ -17,24 +16,22 @@ from ormar.models.helpers.models import group_related_list if TYPE_CHECKING: # pragma: no cover from ormar.fields import ForeignKeyField - from ormar.models import T -else: - T = TypeVar("T", bound="ModelRow") + from ormar.models import Model class ModelRow(NewBaseModel): @classmethod def from_row( # noqa: CFQ002 - cls: Type["ModelRow"], + cls, row: sqlalchemy.engine.ResultProxy, - source_model: Type[T], + source_model: Type["Model"], select_related: List = None, related_models: Any = None, related_field: Type["ForeignKeyField"] = None, excludable: ExcludableItems = None, current_relation_str: str = "", - proxy_source_model: Optional[Type["ModelRow"]] = None, - ) -> Optional[T]: + proxy_source_model: Optional[Type["Model"]] = None, + ) -> Optional["Model"]: """ Model method to convert raw sql row from database into ormar.Model instance. Traverses nested models if they were specified in select_related for query. @@ -48,6 +45,8 @@ class ModelRow(NewBaseModel): where rows are populated in a different way as they do not have nested models in result. + :param proxy_source_model: source model from which querysetproxy is constructed + :type proxy_source_model: Optional[Type["ModelRow"]] :param excludable: structure of fields to include and exclude :type excludable: ExcludableItems :param current_relation_str: name of the relation field @@ -72,7 +71,6 @@ class ModelRow(NewBaseModel): excludable = excludable or ExcludableItems() if select_related: - source_model = cast(Type[T], cls) related_models = group_related_list(select_related) if related_field: @@ -88,19 +86,19 @@ class ModelRow(NewBaseModel): related_models=related_models, excludable=excludable, current_relation_str=current_relation_str, - source_model=source_model, + source_model=source_model, # type: ignore proxy_source_model=proxy_source_model, # type: ignore ) item = cls.extract_prefixed_table_columns( item=item, row=row, table_prefix=table_prefix, excludable=excludable ) - instance: Optional[T] = None + instance: Optional["Model"] = None if item.get(cls.Meta.pkname, None) is not None: item["__excluded__"] = cls.get_names_to_exclude( excludable=excludable, alias=table_prefix ) - instance = cast(T, cls(**item)) + instance = cast("Model", cls(**item)) instance.set_save_status(True) return instance @@ -109,11 +107,11 @@ class ModelRow(NewBaseModel): cls, item: dict, row: sqlalchemy.engine.ResultProxy, - source_model: Type[T], + source_model: Type["Model"], related_models: Any, excludable: ExcludableItems, current_relation_str: str = None, - proxy_source_model: Type[T] = None, + proxy_source_model: Type["Model"] = None, ) -> dict: """ Traverses structure of related models and populates the nested models @@ -125,6 +123,8 @@ class ModelRow(NewBaseModel): Recurrently calls from_row method on nested instances and create nested instances. In the end those instances are added to the final model dictionary. + :param proxy_source_model: source model from which querysetproxy is constructed + :type proxy_source_model: Optional[Type["ModelRow"]] :param excludable: structure of fields to include and exclude :type excludable: ExcludableItems :param source_model: source model from which relation started @@ -190,6 +190,21 @@ class ModelRow(NewBaseModel): related: str, excludable: ExcludableItems, ) -> "ModelRow": + """ + Initialize the through model from db row. + Excluded all relation fields and other exclude/include set in excludable. + + :param row: loaded row from database + :type row: sqlalchemy.engine.ResultProxy + :param through_name: name of the through field + :type through_name: str + :param related: name of the relation + :type related: str + :param excludable: structure of fields to include and exclude + :type excludable: ExcludableItems + :return: initialized through model without relation + :rtype: "ModelRow" + """ model_cls = cls.Meta.model_fields[through_name].to table_prefix = cls.Meta.alias_manager.resolve_relation_alias( from_model=cls, relation_name=related diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index d400c0f..9308d83 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -13,7 +13,6 @@ from typing import ( Set, TYPE_CHECKING, Type, - TypeVar, Union, cast, ) @@ -46,15 +45,13 @@ from ormar.relations.alias_manager import AliasManager from ormar.relations.relation_manager import RelationsManager if TYPE_CHECKING: # pragma no cover - from ormar.models import Model, T + from ormar.models import Model from ormar.signals import SignalEmitter IntStr = Union[int, str] DictStrAny = Dict[str, Any] AbstractSetIntStr = AbstractSet[IntStr] MappingIntStrAny = Mapping[IntStr, Any] -else: - T = TypeVar("T") class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass): @@ -89,7 +86,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass Meta: ModelMeta # noinspection PyMissingConstructor - def __init__(self: T, *args: Any, **kwargs: Any) -> None: # type: ignore + def __init__(self, *args: Any, **kwargs: Any) -> None: # type: ignore """ Initializer that creates a new ormar Model that is also pydantic Model at the same time. @@ -130,7 +127,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass self, "_orm", RelationsManager( - related_fields=self.extract_related_fields(), owner=cast(T, self), + related_fields=self.extract_related_fields(), owner=cast("Model", self), ), ) @@ -397,7 +394,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass cause some dialect require different treatment""" return cls.Meta.database._backend._dialect.name - def remove(self, parent: "T", name: str) -> None: + def remove(self, parent: "Model", name: str) -> None: """Removes child from relation with given name in RelationshipManager""" self._orm.remove_parent(self, parent, name) diff --git a/ormar/queryset/__init__.py b/ormar/queryset/__init__.py index 11b072e..161d9bb 100644 --- a/ormar/queryset/__init__.py +++ b/ormar/queryset/__init__.py @@ -6,10 +6,9 @@ from ormar.queryset.filter_query import FilterQuery from ormar.queryset.limit_query import LimitQuery from ormar.queryset.offset_query import OffsetQuery from ormar.queryset.order_query import OrderQuery -from ormar.queryset.queryset import QuerySet, T +from ormar.queryset.queryset import QuerySet __all__ = [ - "T", "QuerySet", "FilterQuery", "LimitQuery", diff --git a/ormar/queryset/prefetch_query.py b/ormar/queryset/prefetch_query.py index a661b73..d224c22 100644 --- a/ormar/queryset/prefetch_query.py +++ b/ormar/queryset/prefetch_query.py @@ -596,7 +596,6 @@ class PrefetchQuery: """ target_model = target_field.to for row in rows: - # TODO Fix fields field_name = parent_model.get_related_field_name(target_field=target_field) item = target_model.extract_prefixed_table_columns( item={}, row=row, table_prefix=table_prefix, excludable=excludable, diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index f4f9fd0..7fadcb8 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -1,14 +1,12 @@ from typing import ( Any, Dict, - Generic, List, Optional, Sequence, Set, TYPE_CHECKING, Type, - TypeVar, Union, cast, ) @@ -27,22 +25,20 @@ from ormar.queryset.prefetch_query import PrefetchQuery from ormar.queryset.query import Query if TYPE_CHECKING: # pragma no cover - from ormar.models import T + from ormar import Model from ormar.models.metaclass import ModelMeta from ormar.relations.querysetproxy import QuerysetProxy from ormar.models.excludable import ExcludableItems -else: - T = TypeVar("T") -class QuerySet(Generic[T]): +class QuerySet: """ Main class to perform database queries, exposed on each model as objects attribute. """ def __init__( # noqa CFQ002 self, - model_cls: Optional[Type[T]] = None, + model_cls: Optional[Type["Model"]] = None, filter_clauses: List = None, exclude_clauses: List = None, select_related: List = None, @@ -52,7 +48,7 @@ class QuerySet(Generic[T]): order_bys: List = None, prefetch_related: List = None, limit_raw_sql: bool = False, - proxy_source_model: Optional[Type[T]] = None, + proxy_source_model: Optional[Type["Model"]] = None, ) -> None: self.proxy_source_model = proxy_source_model self.model_cls = model_cls @@ -69,7 +65,7 @@ class QuerySet(Generic[T]): def __get__( self, instance: Optional[Union["QuerySet", "QuerysetProxy"]], - owner: Union[Type[T], Type["QuerysetProxy"]], + owner: Union[Type["Model"], Type["QuerysetProxy"]], ) -> "QuerySet": if issubclass(owner, ormar.Model): if owner.Meta.requires_ref_update: @@ -78,7 +74,7 @@ class QuerySet(Generic[T]): f"ForwardRefs. \nBefore using the model you " f"need to call update_forward_refs()." ) - owner = cast(Type[T], owner) + owner = cast(Type["Model"], owner) return self.__class__(model_cls=owner) return self.__class__() # pragma: no cover @@ -95,7 +91,7 @@ class QuerySet(Generic[T]): return self.model_cls.Meta @property - def model(self) -> Type[T]: + def model(self) -> Type["Model"]: """ Shortcut to model class set on QuerySet. @@ -117,7 +113,7 @@ class QuerySet(Generic[T]): order_bys: List = None, prefetch_related: List = None, limit_raw_sql: bool = None, - proxy_source_model: Optional[Type[T]] = None, + proxy_source_model: Optional[Type["Model"]] = None, ) -> "QuerySet": """ Method that returns new instance of queryset based on passed params, @@ -152,8 +148,8 @@ class QuerySet(Generic[T]): ) async def _prefetch_related_models( - self, models: Sequence[Optional["T"]], rows: List - ) -> Sequence[Optional["T"]]: + self, models: Sequence[Optional["Model"]], rows: List + ) -> Sequence[Optional["Model"]]: """ Performs prefetch query for selected models names. @@ -173,7 +169,7 @@ class QuerySet(Generic[T]): ) return await query.prefetch_related(models=models, rows=rows) # type: ignore - def _process_query_result_rows(self, rows: List) -> Sequence[Optional[T]]: + def _process_query_result_rows(self, rows: List) -> Sequence[Optional["Model"]]: """ Process database rows and initialize ormar Model from each of the rows. @@ -197,7 +193,7 @@ class QuerySet(Generic[T]): return result_rows @staticmethod - def check_single_result_rows_count(rows: Sequence[Optional[T]]) -> None: + def check_single_result_rows_count(rows: Sequence[Optional["Model"]]) -> None: """ Verifies if the result has one and only one row. @@ -638,7 +634,7 @@ class QuerySet(Generic[T]): limit_raw_sql = self.limit_sql_raw if limit_raw_sql is None else limit_raw_sql return self.rebuild_self(offset=offset, limit_raw_sql=limit_raw_sql,) - async def first(self, **kwargs: Any) -> T: + async def first(self, **kwargs: Any) -> "Model": """ Gets the first row from the db ordered by primary key column ascending. @@ -669,7 +665,7 @@ class QuerySet(Generic[T]): self.check_single_result_rows_count(processed_rows) return processed_rows[0] # type: ignore - async def get(self, **kwargs: Any) -> T: + async def get(self, **kwargs: Any) -> "Model": """ Get's the first row from the db meeting the criteria set by kwargs. @@ -708,7 +704,7 @@ class QuerySet(Generic[T]): self.check_single_result_rows_count(processed_rows) return processed_rows[0] # type: ignore - async def get_or_create(self, **kwargs: Any) -> T: + async def get_or_create(self, **kwargs: Any) -> "Model": """ Combination of create and get methods. @@ -726,7 +722,7 @@ class QuerySet(Generic[T]): except NoMatch: return await self.create(**kwargs) - async def update_or_create(self, **kwargs: Any) -> T: + async def update_or_create(self, **kwargs: Any) -> "Model": """ Updates the model, or in case there is no match in database creates a new one. @@ -743,7 +739,7 @@ class QuerySet(Generic[T]): model = await self.get(pk=kwargs[pk_name]) return await model.update(**kwargs) - async def all(self, **kwargs: Any) -> Sequence[Optional[T]]: # noqa: A003 + async def all(self, **kwargs: Any) -> Sequence[Optional["Model"]]: # noqa: A003 """ Returns all rows from a database for given model for set filter options. @@ -767,7 +763,7 @@ class QuerySet(Generic[T]): return result_rows - async def create(self, **kwargs: Any) -> T: + async def create(self, **kwargs: Any) -> "Model": """ Creates the model instance, saves it in a database and returns the updates model (with pk populated if not passed and autoincrement is set). @@ -810,7 +806,7 @@ class QuerySet(Generic[T]): ) return instance - async def bulk_create(self, objects: List[T]) -> None: + async def bulk_create(self, objects: List["Model"]) -> None: """ Performs a bulk update in one database session to speed up the process. @@ -836,7 +832,7 @@ class QuerySet(Generic[T]): objt.set_save_status(True) async def bulk_update( # noqa: CCR001 - self, objects: List[T], columns: List[str] = None + self, objects: List["Model"], columns: List[str] = None ) -> None: """ Performs bulk update in one database session to speed up the process. diff --git a/ormar/relations/querysetproxy.py b/ormar/relations/querysetproxy.py index 952a6c7..190dd7b 100644 --- a/ormar/relations/querysetproxy.py +++ b/ormar/relations/querysetproxy.py @@ -1,14 +1,13 @@ -from typing import ( +from _weakref import CallableProxyType +from typing import ( # noqa: I100, I201 Any, Dict, - Generic, List, MutableSequence, Optional, Sequence, Set, TYPE_CHECKING, - TypeVar, Union, cast, ) @@ -18,14 +17,12 @@ from ormar.exceptions import ModelPersistenceError, QueryDefinitionError if TYPE_CHECKING: # pragma no cover from ormar.relations import Relation - from ormar.models import Model, T + from ormar.models import Model from ormar.queryset import QuerySet from ormar import RelationType -else: - T = TypeVar("T") -class QuerysetProxy(Generic[T]): +class QuerysetProxy: """ Exposes QuerySet methods on relations, but also handles creating and removing of through Models for m2m relations. @@ -40,7 +37,7 @@ class QuerysetProxy(Generic[T]): self.relation: Relation = relation self._queryset: Optional["QuerySet"] = qryset self.type_: "RelationType" = type_ - self._owner: "Model" = self.relation.manager.owner + self._owner: Union[CallableProxyType, "Model"] = self.relation.manager.owner self.related_field_name = self._owner.Meta.model_fields[ self.relation.field_name ].get_related_name() @@ -72,7 +69,7 @@ class QuerysetProxy(Generic[T]): """ self._queryset = value - def _assign_child_to_parent(self, child: Optional["T"]) -> None: + def _assign_child_to_parent(self, child: Optional["Model"]) -> None: """ Registers child in parents RelationManager. @@ -84,7 +81,9 @@ class QuerysetProxy(Generic[T]): rel_name = self.relation.field_name setattr(owner, rel_name, child) - def _register_related(self, child: Union["T", Sequence[Optional["T"]]]) -> None: + def _register_related( + self, child: Union["Model", Sequence[Optional["Model"]]] + ) -> None: """ Registers child/ children in parents RelationManager. @@ -96,7 +95,7 @@ class QuerysetProxy(Generic[T]): self._assign_child_to_parent(subchild) else: assert isinstance(child, ormar.Model) - child = cast(T, child) + child = cast("Model", child) self._assign_child_to_parent(child) def _clean_items_on_load(self) -> None: @@ -107,7 +106,7 @@ class QuerysetProxy(Generic[T]): for item in self.relation.related_models[:]: self.relation.remove(item) - async def create_through_instance(self, child: "T", **kwargs: Any) -> None: + async def create_through_instance(self, child: "Model", **kwargs: Any) -> None: """ Crete a through model instance in the database for m2m relations. @@ -132,7 +131,7 @@ class QuerysetProxy(Generic[T]): # print("\n", expr.compile(compile_kwargs={"literal_binds": True})) await model_cls.Meta.database.execute(expr) - async def update_through_instance(self, child: "T", **kwargs: Any) -> None: + async def update_through_instance(self, child: "Model", **kwargs: Any) -> None: """ Updates a through model instance in the database for m2m relations. @@ -148,7 +147,7 @@ class QuerysetProxy(Generic[T]): through_model = await model_cls.objects.get(**rel_kwargs) await through_model.update(**kwargs) - async def delete_through_instance(self, child: "T") -> None: + async def delete_through_instance(self, child: "Model") -> None: """ Removes through model instance from the database for m2m relations. @@ -217,7 +216,7 @@ class QuerysetProxy(Generic[T]): ) return await queryset.delete(**kwargs) # type: ignore - async def first(self, **kwargs: Any) -> T: + async def first(self, **kwargs: Any) -> "Model": """ Gets the first row from the db ordered by primary key column ascending. @@ -235,7 +234,7 @@ class QuerysetProxy(Generic[T]): self._register_related(first) return first - async def get(self, **kwargs: Any) -> "T": + async def get(self, **kwargs: Any) -> "Model": """ Get's the first row from the db meeting the criteria set by kwargs. @@ -259,7 +258,7 @@ class QuerysetProxy(Generic[T]): self._register_related(get) return get - async def all(self, **kwargs: Any) -> Sequence[Optional["T"]]: # noqa: A003 + async def all(self, **kwargs: Any) -> Sequence[Optional["Model"]]: # noqa: A003 """ Returns all rows from a database for given model for set filter options. @@ -281,7 +280,7 @@ class QuerysetProxy(Generic[T]): self._register_related(all_items) return all_items - async def create(self, **kwargs: Any) -> "T": + async def create(self, **kwargs: Any) -> "Model": """ Creates the model instance, saves it in a database and returns the updates model (with pk populated if not passed and autoincrement is set). @@ -338,7 +337,7 @@ class QuerysetProxy(Generic[T]): ) return len(children) - async def get_or_create(self, **kwargs: Any) -> "T": + async def get_or_create(self, **kwargs: Any) -> "Model": """ Combination of create and get methods. @@ -356,7 +355,7 @@ class QuerysetProxy(Generic[T]): except ormar.NoMatch: return await self.create(**kwargs) - async def update_or_create(self, **kwargs: Any) -> "T": + async def update_or_create(self, **kwargs: Any) -> "Model": """ Updates the model, or in case there is no match in database creates a new one. diff --git a/ormar/relations/relation.py b/ormar/relations/relation.py index 6d4da36..0bdc671 100644 --- a/ormar/relations/relation.py +++ b/ormar/relations/relation.py @@ -7,7 +7,7 @@ from ormar.relations.relation_proxy import RelationProxy if TYPE_CHECKING: # pragma no cover from ormar.relations import RelationsManager - from ormar.models import Model, NewBaseModel, T + from ormar.models import Model, NewBaseModel class RelationType(Enum): @@ -36,7 +36,7 @@ class Relation: type_: RelationType, field_name: str, to: Type["Model"], - through: Type["T"] = None, + through: Type["Model"] = None, ) -> None: """ Initialize the Relation and keep the related models either as instances of @@ -69,7 +69,7 @@ class Relation: ) @property - def through(self) -> Type["T"]: + def through(self) -> Type["Model"]: if not self._through: # pragma: no cover raise RelationshipInstanceError("Relation does not have through model!") return self._through @@ -116,7 +116,7 @@ class Relation: self._to_remove.add(ind) return None - def add(self, child: "T") -> None: + def add(self, child: "Model") -> None: """ Adds child Model to relation, either sets child as related model or adds it to the list in RelationProxy depending on relation type. diff --git a/ormar/relations/relation_manager.py b/ormar/relations/relation_manager.py index a718b09..0719550 100644 --- a/ormar/relations/relation_manager.py +++ b/ormar/relations/relation_manager.py @@ -5,7 +5,7 @@ from ormar.relations.relation import Relation, RelationType from ormar.relations.utils import get_relations_sides_and_names if TYPE_CHECKING: # pragma no cover - from ormar.models import NewBaseModel, T, Model + from ormar.models import NewBaseModel, Model from ormar.fields import ForeignKeyField, BaseField @@ -17,7 +17,7 @@ class RelationsManager: def __init__( self, related_fields: List[Type["ForeignKeyField"]] = None, - owner: Optional["T"] = None, + owner: Optional["Model"] = None, ) -> None: self.owner = proxy(owner) self._related_fields = related_fields or [] diff --git a/tests/test_excluding_fields_in_fastapi.py b/tests/test_excluding_fields_in_fastapi.py index 6568d9b..dd4dd84 100644 --- a/tests/test_excluding_fields_in_fastapi.py +++ b/tests/test_excluding_fields_in_fastapi.py @@ -124,8 +124,7 @@ async def create_user(user: User): @app.post("/users2/", response_model=User) async def create_user2(user: User): - user = await user.save() - return user.dict(exclude={"password"}) + return (await user.save()).dict(exclude={"password"}) @app.post("/users3/", response_model=UserBase) @@ -135,26 +134,22 @@ async def create_user3(user: User2): @app.post("/users4/") async def create_user4(user: User2): - user = await user.save() - return user.dict(exclude={"password"}) + return (await user.save()).dict(exclude={"password"}) @app.post("/random/", response_model=RandomModel) async def create_user5(user: RandomModel): - user = await user.save() - return user + return await user.save() @app.post("/random2/", response_model=RandomModel) async def create_user6(user: RandomModel): - user = await user.save() - return user.dict() + return await user.save() @app.post("/random3/", response_model=RandomModel, response_model_exclude={"full_name"}) async def create_user7(user: RandomModel): - user = await user.save() - return user.dict() + return await user.save() def test_excluding_fields_in_endpoints(): From a8ae50276e9104758af8e1472c79a68a809736e1 Mon Sep 17 00:00:00 2001 From: collerek Date: Wed, 3 Mar 2021 19:48:40 +0100 Subject: [PATCH 12/14] update docs, add load_all(), tests for load_all, make through field optional --- README.md | 2 +- docs/api/fields/base-field.md | 21 ++ docs/api/fields/foreign-key.md | 29 +++ docs/api/fields/many-to-many.md | 41 +++- docs/api/models/excludable-items.md | 188 ++++++++++++++++++ docs/api/models/helpers/models.md | 42 ++-- docs/api/models/helpers/pydantic.md | 6 +- .../helpers/related-names-validation.md | 25 +++ docs/api/models/helpers/relations.md | 17 +- docs/api/models/helpers/sqlalchemy.md | 8 +- docs/api/models/helpers/validation.md | 120 +++++++++++ docs/api/models/mixins/excludable-mixin.md | 90 +-------- docs/api/models/mixins/relation-mixin.md | 39 +++- docs/api/models/mixins/save-prepare-mixin.md | 19 ++ docs/api/models/model-metaclass.md | 114 ++--------- docs/api/models/model-row.md | 132 ++++++++++++ docs/api/models/model.md | 157 ++++----------- docs/api/models/new-basemodel.md | 4 +- docs/api/query-set/join.md | 97 +++------ docs/api/query-set/prefetch-query.md | 31 +-- docs/api/query-set/query-set.md | 13 +- docs/api/query-set/query.md | 32 --- docs/api/query-set/utils.md | 2 +- docs/api/relations/alias-manager.md | 23 ++- docs/api/relations/queryset-proxy.md | 46 ++++- docs/api/relations/relation-manager.md | 98 ++++----- docs/api/relations/relation-proxy.md | 3 +- docs/api/relations/relation.md | 6 +- docs/index.md | 2 +- docs/models/methods.md | 34 ++++ docs/relations/index.md | 60 ++++-- docs/relations/many-to-many.md | 166 +++++++++++++++- docs/relations/queryset-proxy.md | 47 +++++ docs/releases.md | 26 ++- docs_src/relations/docs002.py | 13 +- docs_src/relations/docs004.py | 29 +++ mkdocs.yml | 2 + ormar/fields/many_to_many.py | 22 +- ormar/models/helpers/sqlalchemy.py | 2 + ormar/models/metaclass.py | 10 +- ormar/models/mixins/merge_mixin.py | 4 +- ormar/models/mixins/relation_mixin.py | 53 ++++- ormar/models/model.py | 45 +++++ ormar/models/model_row.py | 15 +- ormar/queryset/queryset.py | 6 +- ormar/relations/relation.py | 8 + ormar/relations/relation_manager.py | 94 ++++----- ormar/relations/relation_proxy.py | 3 + pydoc-markdown.yml | 6 + tests/test_aliases.py | 13 +- tests/test_fastapi_docs.py | 7 +- tests/test_inheritance_concrete.py | 14 +- tests/test_load_all.py | 171 ++++++++++++++++ tests/test_m2m_through_fields.py | 26 ++- tests/test_many_to_many.py | 14 +- tests/test_order_by.py | 9 +- 56 files changed, 1653 insertions(+), 653 deletions(-) create mode 100644 docs/api/models/excludable-items.md create mode 100644 docs/api/models/helpers/related-names-validation.md create mode 100644 docs/api/models/helpers/validation.md create mode 100644 docs/api/models/model-row.md create mode 100644 docs_src/relations/docs004.py create mode 100644 tests/test_load_all.py diff --git a/README.md b/README.md index c6c5f41..d4655e5 100644 --- a/README.md +++ b/README.md @@ -306,7 +306,7 @@ async def joins(): # visit: https://collerek.github.io/ormar/relations/ # to read more about joins and subqueries - # visit: https://collerek.github.io/ormar/queries/delete/ + # visit: https://collerek.github.io/ormar/queries/joins-and-subqueries/ async def filter_and_sort(): diff --git a/docs/api/fields/base-field.md b/docs/api/fields/base-field.md index ac7ea01..82b99d3 100644 --- a/docs/api/fields/base-field.md +++ b/docs/api/fields/base-field.md @@ -72,6 +72,27 @@ Excludes defaults and alias as they are populated separately `(bool)`: True if field is present on pydantic.FieldInfo + +#### get\_base\_pydantic\_field\_info + +```python + | @classmethod + | get_base_pydantic_field_info(cls, allow_null: bool) -> FieldInfo +``` + +Generates base pydantic.FieldInfo with only default and optionally +required to fix pydantic Json field being set to required=False. +Used in an ormar Model Metaclass. + +**Arguments**: + +- `allow_null (bool)`: flag if the default value can be None +or if it should be populated by pydantic Undefined + +**Returns**: + +`(pydantic.FieldInfo)`: instance of base pydantic.FieldInfo + #### convert\_to\_pydantic\_field\_info diff --git a/docs/api/fields/foreign-key.md b/docs/api/fields/foreign-key.md index 019e2c9..055b661 100644 --- a/docs/api/fields/foreign-key.md +++ b/docs/api/fields/foreign-key.md @@ -332,3 +332,32 @@ Selects the appropriate constructor based on a passed value. `(Optional[Union["Model", List["Model"]]])`: returns a Model or a list of Models + +#### get\_relation\_name + +```python + | @classmethod + | get_relation_name(cls) -> str +``` + +Returns name of the relation, which can be a own name or through model +names for m2m models + +**Returns**: + +`(bool)`: result of the check + + +#### get\_source\_model + +```python + | @classmethod + | get_source_model(cls) -> Type["Model"] +``` + +Returns model from which the relation comes -> either owner or through model + +**Returns**: + +`(Type["Model"])`: source model + diff --git a/docs/api/fields/many-to-many.md b/docs/api/fields/many-to-many.md index 72c95e1..89570aa 100644 --- a/docs/api/fields/many-to-many.md +++ b/docs/api/fields/many-to-many.md @@ -24,7 +24,7 @@ pydantic field to use and type of the target column field. #### ManyToMany ```python -ManyToMany(to: "ToType", through: "ToType", *, name: str = None, unique: bool = False, virtual: bool = False, **kwargs: Any, ,) -> Any +ManyToMany(to: "ToType", through: Optional["ToType"] = None, *, name: str = None, unique: bool = False, virtual: bool = False, **kwargs: Any, ,) -> Any ``` Despite a name it's a function that returns constructed ManyToManyField. @@ -134,3 +134,42 @@ Evaluates the ForwardRef to actual Field based on global and local namespaces `(None)`: None + +#### get\_relation\_name + +```python + | @classmethod + | get_relation_name(cls) -> str +``` + +Returns name of the relation, which can be a own name or through model +names for m2m models + +**Returns**: + +`(bool)`: result of the check + + +#### get\_source\_model + +```python + | @classmethod + | get_source_model(cls) -> Type["Model"] +``` + +Returns model from which the relation comes -> either owner or through model + +**Returns**: + +`(Type["Model"])`: source model + + +#### create\_default\_through\_model + +```python + | @classmethod + | create_default_through_model(cls) -> None +``` + +Creates default empty through model if no additional fields are required. + diff --git a/docs/api/models/excludable-items.md b/docs/api/models/excludable-items.md new file mode 100644 index 0000000..ee12586 --- /dev/null +++ b/docs/api/models/excludable-items.md @@ -0,0 +1,188 @@ + +# models.excludable + + +## Excludable Objects + +```python +@dataclass +class Excludable() +``` + +Class that keeps sets of fields to exclude and include + + +#### get\_copy + +```python + | get_copy() -> "Excludable" +``` + +Return copy of self to avoid in place modifications + +**Returns**: + +`(ormar.models.excludable.Excludable)`: copy of self with copied sets + + +#### set\_values + +```python + | set_values(value: Set, is_exclude: bool) -> None +``` + +Appends the data to include/exclude sets. + +**Arguments**: + +- `value (set)`: set of values to add +- `is_exclude (bool)`: flag if values are to be excluded or included + + +#### is\_included + +```python + | is_included(key: str) -> bool +``` + +Check if field in included (in set or set is {...}) + +**Arguments**: + +- `key (str)`: key to check + +**Returns**: + +`(bool)`: result of the check + + +#### is\_excluded + +```python + | is_excluded(key: str) -> bool +``` + +Check if field in excluded (in set or set is {...}) + +**Arguments**: + +- `key (str)`: key to check + +**Returns**: + +`(bool)`: result of the check + + +## ExcludableItems Objects + +```python +class ExcludableItems() +``` + +Keeps a dictionary of Excludables by alias + model_name keys +to allow quick lookup by nested models without need to travers +deeply nested dictionaries and passing include/exclude around + + +#### from\_excludable + +```python + | @classmethod + | from_excludable(cls, other: "ExcludableItems") -> "ExcludableItems" +``` + +Copy passed ExcludableItems to avoid inplace modifications. + +**Arguments**: + +- `other (ormar.models.excludable.ExcludableItems)`: other excludable items to be copied + +**Returns**: + +`(ormar.models.excludable.ExcludableItems)`: copy of other + + +#### get + +```python + | get(model_cls: Type["Model"], alias: str = "") -> Excludable +``` + +Return Excludable for given model and alias. + +**Arguments**: + +- `model_cls (ormar.models.metaclass.ModelMetaclass)`: target model to check +- `alias (str)`: table alias from relation manager + +**Returns**: + +`(ormar.models.excludable.Excludable)`: Excludable for given model and alias + + +#### build + +```python + | build(items: Union[List[str], str, Tuple[str], Set[str], Dict], model_cls: Type["Model"], is_exclude: bool = False) -> None +``` + +Receives the one of the types of items and parses them as to achieve +a end situation with one excludable per alias/model in relation. + +Each excludable has two sets of values - one to include, one to exclude. + +**Arguments**: + +- `items (Union[List[str], str, Tuple[str], Set[str], Dict])`: values to be included or excluded +- `model_cls (ormar.models.metaclass.ModelMetaclass)`: source model from which relations are constructed +- `is_exclude (bool)`: flag if items should be included or excluded + + +#### \_set\_excludes + +```python + | _set_excludes(items: Set, model_name: str, is_exclude: bool, alias: str = "") -> None +``` + +Sets set of values to be included or excluded for given key and model. + +**Arguments**: + +- `items (set)`: items to include/exclude +- `model_name (str)`: name of model to construct key +- `is_exclude (bool)`: flag if values should be included or excluded +- `alias (str)`: + + +#### \_traverse\_dict + +```python + | _traverse_dict(values: Dict, source_model: Type["Model"], model_cls: Type["Model"], is_exclude: bool, related_items: List = None, alias: str = "") -> None +``` + +Goes through dict of nested values and construct/update Excludables. + +**Arguments**: + +- `values (Dict)`: items to include/exclude +- `source_model (ormar.models.metaclass.ModelMetaclass)`: source model from which relations are constructed +- `model_cls (ormar.models.metaclass.ModelMetaclass)`: model from which current relation is constructed +- `is_exclude (bool)`: flag if values should be included or excluded +- `related_items (List)`: list of names of related fields chain +- `alias (str)`: alias of relation + + +#### \_traverse\_list + +```python + | _traverse_list(values: Set[str], model_cls: Type["Model"], is_exclude: bool) -> None +``` + +Goes through list of values and construct/update Excludables. + +**Arguments**: + +- `values (set)`: items to include/exclude +- `model_cls (ormar.models.metaclass.ModelMetaclass)`: model from which current relation is constructed +- `is_exclude (bool)`: flag if values should be included or excluded + diff --git a/docs/api/models/helpers/models.md b/docs/api/models/helpers/models.md index b3100c6..2537a70 100644 --- a/docs/api/models/helpers/models.md +++ b/docs/api/models/helpers/models.md @@ -87,28 +87,6 @@ extraction of ormar model_fields. `(Tuple[Dict, Dict])`: namespace of the class updated, dict of extracted model_fields - -#### validate\_related\_names\_in\_relations - -```python -validate_related_names_in_relations(model_fields: Dict, new_model: Type["Model"]) -> None -``` - -Performs a validation of relation_names in relation fields. -If multiple fields are leading to the same related model -only one can have empty related_name param -(populated by default as model.name.lower()+'s'). -Also related_names have to be unique for given related model. - -**Raises**: - -- `ModelDefinitionError`: if validation of related_names fail - -**Arguments**: - -- `model_fields (Dict[str, ormar.Field])`: dictionary of declared ormar model fields -- `new_model (Model class)`: - #### group\_related\_list @@ -134,3 +112,23 @@ Result dictionary is sorted by length of the values and by key `(Dict[str, List])`: list converted to dictionary to avoid repetition and group nested models + +#### meta\_field\_not\_set + +```python +meta_field_not_set(model: Type["Model"], field_name: str) -> bool +``` + +Checks if field with given name is already present in model.Meta. +Then check if it's set to something truthful +(in practice meaning not None, as it's non or ormar Field only). + +**Arguments**: + +- `model (Model class)`: newly constructed model +- `field_name (str)`: name of the ormar field + +**Returns**: + +`(bool)`: result of the check + diff --git a/docs/api/models/helpers/pydantic.md b/docs/api/models/helpers/pydantic.md index 8b38eaf..49e44a5 100644 --- a/docs/api/models/helpers/pydantic.md +++ b/docs/api/models/helpers/pydantic.md @@ -5,7 +5,7 @@ #### create\_pydantic\_field ```python -create_pydantic_field(field_name: str, model: Type["Model"], model_field: Type[ManyToManyField]) -> None +create_pydantic_field(field_name: str, model: Type["Model"], model_field: Type["ManyToManyField"]) -> None ``` Registers pydantic field on through model that leads to passed model @@ -42,7 +42,7 @@ field_name. Returns a pydantic field with type of field_name field type. #### populate\_default\_pydantic\_field\_value ```python -populate_default_pydantic_field_value(ormar_field: Type[BaseField], field_name: str, attrs: dict) -> dict +populate_default_pydantic_field_value(ormar_field: Type["BaseField"], field_name: str, attrs: dict) -> dict ``` Grabs current value of the ormar Field in class namespace @@ -94,7 +94,7 @@ Those annotations are later used by pydantic to construct it's own fields. #### get\_pydantic\_base\_orm\_config ```python -get_pydantic_base_orm_config() -> Type[BaseConfig] +get_pydantic_base_orm_config() -> Type[pydantic.BaseConfig] ``` Returns empty pydantic Config with orm_mode set to True. diff --git a/docs/api/models/helpers/related-names-validation.md b/docs/api/models/helpers/related-names-validation.md new file mode 100644 index 0000000..9fa93cc --- /dev/null +++ b/docs/api/models/helpers/related-names-validation.md @@ -0,0 +1,25 @@ + +# models.helpers.related\_names\_validation + + +#### validate\_related\_names\_in\_relations + +```python +validate_related_names_in_relations(model_fields: Dict, new_model: Type["Model"]) -> None +``` + +Performs a validation of relation_names in relation fields. +If multiple fields are leading to the same related model +only one can have empty related_name param +(populated by default as model.name.lower()+'s'). +Also related_names have to be unique for given related model. + +**Raises**: + +- `ModelDefinitionError`: if validation of related_names fail + +**Arguments**: + +- `model_fields (Dict[str, ormar.Field])`: dictionary of declared ormar model fields +- `new_model (Model class)`: + diff --git a/docs/api/models/helpers/relations.md b/docs/api/models/helpers/relations.md index d470756..8da7561 100644 --- a/docs/api/models/helpers/relations.md +++ b/docs/api/models/helpers/relations.md @@ -23,7 +23,7 @@ aliases for proper sql joins. #### register\_many\_to\_many\_relation\_on\_build ```python -register_many_to_many_relation_on_build(field: Type[ManyToManyField]) -> None +register_many_to_many_relation_on_build(field: Type["ManyToManyField"]) -> None ``` Registers connection between through model and both sides of the m2m relation. @@ -89,11 +89,24 @@ Autogenerated reverse fields also set related_name to the original field name. - `model_field (relation Field)`: original relation ForeignKey field + +#### register\_through\_shortcut\_fields + +```python +register_through_shortcut_fields(model_field: Type["ManyToManyField"]) -> None +``` + +Registers m2m relation through shortcut on both ends of the relation. + +**Arguments**: + +- `model_field (ManyToManyField)`: relation field defined in parent model + #### register\_relation\_in\_alias\_manager ```python -register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None +register_relation_in_alias_manager(field: Type["ForeignKeyField"]) -> None ``` Registers the relation (and reverse relation) in alias manager. diff --git a/docs/api/models/helpers/sqlalchemy.md b/docs/api/models/helpers/sqlalchemy.md index 02c43c1..473b599 100644 --- a/docs/api/models/helpers/sqlalchemy.md +++ b/docs/api/models/helpers/sqlalchemy.md @@ -5,7 +5,7 @@ #### adjust\_through\_many\_to\_many\_model ```python -adjust_through_many_to_many_model(model_field: Type[ManyToManyField]) -> None +adjust_through_many_to_many_model(model_field: Type["ManyToManyField"]) -> None ``` Registers m2m relation on through model. @@ -21,7 +21,7 @@ Sets pydantic fields with child and parent model types. #### create\_and\_append\_m2m\_fk ```python -create_and_append_m2m_fk(model: Type["Model"], model_field: Type[ManyToManyField], field_name: str) -> None +create_and_append_m2m_fk(model: Type["Model"], model_field: Type["ManyToManyField"], field_name: str) -> None ``` Registers sqlalchemy Column with sqlalchemy.ForeignKey leading to the model. @@ -38,7 +38,7 @@ Newly created field is added to m2m relation through model Meta columns and tabl #### check\_pk\_column\_validity ```python -check_pk_column_validity(field_name: str, field: BaseField, pkname: Optional[str]) -> Optional[str] +check_pk_column_validity(field_name: str, field: "BaseField", pkname: Optional[str]) -> Optional[str] ``` Receives the field marked as primary key and verifies if the pkname @@ -165,7 +165,7 @@ It populates name, metadata, columns and constraints. #### update\_column\_definition ```python -update_column_definition(model: Union[Type["Model"], Type["NewBaseModel"]], field: Type[ForeignKeyField]) -> None +update_column_definition(model: Union[Type["Model"], Type["NewBaseModel"]], field: Type["ForeignKeyField"]) -> None ``` Updates a column with a new type column based on updated parameters in FK fields. diff --git a/docs/api/models/helpers/validation.md b/docs/api/models/helpers/validation.md new file mode 100644 index 0000000..9c2717b --- /dev/null +++ b/docs/api/models/helpers/validation.md @@ -0,0 +1,120 @@ + +# models.helpers.validation + + +#### check\_if\_field\_has\_choices + +```python +check_if_field_has_choices(field: Type[BaseField]) -> bool +``` + +Checks if given field has choices populated. +A if it has one, a validator for this field needs to be attached. + +**Arguments**: + +- `field (BaseField)`: ormar field to check + +**Returns**: + +`(bool)`: result of the check + + +#### convert\_choices\_if\_needed + +```python +convert_choices_if_needed(field: Type["BaseField"], value: Any) -> Tuple[Any, List] +``` + +Converts dates to isoformat as fastapi can check this condition in routes +and the fields are not yet parsed. + +Converts enums to list of it's values. + +Converts uuids to strings. + +Converts decimal to float with given scale. + +**Arguments**: + +- `field (Type[BaseField])`: ormar field to check with choices +- `values (Dict)`: current values of the model to verify + +**Returns**: + +`(Tuple[Any, List])`: value, choices list + + +#### validate\_choices + +```python +validate_choices(field: Type["BaseField"], value: Any) -> None +``` + +Validates if given value is in provided choices. + +**Raises**: + +- `ValueError`: If value is not in choices. + +**Arguments**: + +- `field (Type[BaseField])`: field to validate +- `value (Any)`: value of the field + + +#### choices\_validator + +```python +choices_validator(cls: Type["Model"], values: Dict[str, Any]) -> Dict[str, Any] +``` + +Validator that is attached to pydantic model pre root validators. +Validator checks if field value is in field.choices list. + +**Raises**: + +- `ValueError`: if field value is outside of allowed choices. + +**Arguments**: + +- `cls (Model class)`: constructed class +- `values (Dict[str, Any])`: dictionary of field values (pydantic side) + +**Returns**: + +`(Dict[str, Any])`: values if pass validation, otherwise exception is raised + + +#### construct\_modify\_schema\_function + +```python +construct_modify_schema_function(fields_with_choices: List) -> SchemaExtraCallable +``` + +Modifies the schema to include fields with choices validator. +Those fields will be displayed in schema as Enum types with available choices +values listed next to them. + +**Arguments**: + +- `fields_with_choices (List)`: list of fields with choices validation + +**Returns**: + +`(Callable)`: callable that will be run by pydantic to modify the schema + + +#### populate\_choices\_validators + +```python +populate_choices_validators(model: Type["Model"]) -> None +``` + +Checks if Model has any fields with choices set. +If yes it adds choices validation into pre root validators. + +**Arguments**: + +- `model (Model class)`: newly constructed Model + diff --git a/docs/api/models/mixins/excludable-mixin.md b/docs/api/models/mixins/excludable-mixin.md index a4d9c79..b2ad2f6 100644 --- a/docs/api/models/mixins/excludable-mixin.md +++ b/docs/api/models/mixins/excludable-mixin.md @@ -30,88 +30,12 @@ passed items. `(Union[Set, Dict, None])`: child extracted from items if exists - -#### get\_excluded - -```python - | @staticmethod - | get_excluded(exclude: Union[Set, Dict, None], key: str = None) -> Union[Set, Dict, None] -``` - -Proxy to ExcludableMixin.get_child for exclusions. - -**Arguments**: - -- `exclude (Union[Set, Dict, None])`: bag of items to exclude -- `key (str)`: name of the child to extract - -**Returns**: - -`(Union[Set, Dict, None])`: child extracted from items if exists - - -#### get\_included - -```python - | @staticmethod - | get_included(include: Union[Set, Dict, None], key: str = None) -> Union[Set, Dict, None] -``` - -Proxy to ExcludableMixin.get_child for inclusions. - -**Arguments**: - -- `include (Union[Set, Dict, None])`: bag of items to include -- `key (str)`: name of the child to extract - -**Returns**: - -`(Union[Set, Dict, None])`: child extracted from items if exists - - -#### is\_excluded - -```python - | @staticmethod - | is_excluded(exclude: Union[Set, Dict, None], key: str = None) -> bool -``` - -Checks if given key should be excluded on model/ dict. - -**Arguments**: - -- `exclude (Union[Set, Dict, None])`: bag of items to exclude -- `key (str)`: name of the child to extract - -**Returns**: - -`(Union[Set, Dict, None])`: child extracted from items if exists - - -#### is\_included - -```python - | @staticmethod - | is_included(include: Union[Set, Dict, None], key: str = None) -> bool -``` - -Checks if given key should be included on model/ dict. - -**Arguments**: - -- `include (Union[Set, Dict, None])`: bag of items to include -- `key (str)`: name of the child to extract - -**Returns**: - -`(Union[Set, Dict, None])`: child extracted from items if exists - #### \_populate\_pk\_column ```python | @staticmethod - | _populate_pk_column(model: Type["Model"], columns: List[str], use_alias: bool = False) -> List[str] + | _populate_pk_column(model: Union[Type["Model"], Type["ModelRow"]], columns: List[str], use_alias: bool = False) -> List[str] ``` Adds primary key column/alias (depends on use_alias flag) to list of @@ -132,7 +56,7 @@ column names that are selected. ```python | @classmethod - | own_table_columns(cls, model: Type["Model"], fields: Optional[Union[Set, Dict]], exclude_fields: Optional[Union[Set, Dict]], use_alias: bool = False) -> List[str] + | own_table_columns(cls, model: Union[Type["Model"], Type["ModelRow"]], excludable: ExcludableItems, alias: str = "", use_alias: bool = False) -> List[str] ``` Returns list of aliases or field names for given model. @@ -145,9 +69,9 @@ Primary key field is always added and cannot be excluded (will be added anyway). **Arguments**: +- `alias (str)`: relation prefix +- `excludable (ExcludableItems)`: structure of fields to include and exclude - `model (Type["Model"])`: model on columns are selected -- `fields (Optional[Union[Set, Dict]])`: set/dict of fields to include -- `exclude_fields (Optional[Union[Set, Dict]])`: set/dict of fields to exclude - `use_alias (bool)`: flag if aliases or field names should be used **Returns**: @@ -183,7 +107,7 @@ exclusion, for nested models all related models are excluded. ```python | @classmethod - | get_names_to_exclude(cls, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None) -> Set + | get_names_to_exclude(cls, excludable: ExcludableItems, alias: str) -> Set ``` Returns a set of models field names that should be explicitly excluded @@ -197,8 +121,8 @@ them with dicts constructed from those db rows. **Arguments**: -- `fields (Optional[Union[Set, Dict]])`: set/dict of fields to include -- `exclude_fields (Optional[Union[Set, Dict]])`: set/dict of fields to exclude +- `alias (str)`: alias of current relation +- `excludable (ExcludableItems)`: structure of fields to include and exclude **Returns**: diff --git a/docs/api/models/mixins/relation-mixin.md b/docs/api/models/mixins/relation-mixin.md index 5e94eb9..50ccb79 100644 --- a/docs/api/models/mixins/relation-mixin.md +++ b/docs/api/models/mixins/relation-mixin.md @@ -40,12 +40,26 @@ List is cached in cls._related_fields for quicker access. `(List)`: list of related fields + +#### extract\_through\_names + +```python + | @classmethod + | extract_through_names(cls) -> Set +``` + +Extracts related fields through names which are shortcuts to through models. + +**Returns**: + +`(Set)`: set of related through fields names + #### extract\_related\_names ```python | @classmethod - | extract_related_names(cls) -> Set + | extract_related_names(cls) -> Set[str] ``` Returns List of fields names for all relations declared on a model. @@ -53,7 +67,7 @@ List is cached in cls._related_names for quicker access. **Returns**: -`(List)`: list of related fields names +`(Set)`: set of related fields names #### \_extract\_db\_related\_names @@ -91,3 +105,24 @@ for nested models all related models are returned. `(Set)`: set of non mandatory related fields + +#### \_iterate\_related\_models + +```python + | @classmethod + | _iterate_related_models(cls, visited: Set[Union[Type["Model"], Type["RelationMixin"]]] = None, source_relation: str = None, source_model: Union[Type["Model"], Type["RelationMixin"]] = None) -> List[str] +``` + +Iterates related models recursively to extract relation strings of +nested not visited models. + +**Arguments**: + +- `visited (Set[str])`: set of already visited models +- `source_relation (str)`: name of the current relation +- `source_model (Type["Model"])`: model from which relation comes in nested relations + +**Returns**: + +`(List[str])`: list of relation strings to be passed to select_related + diff --git a/docs/api/models/mixins/save-prepare-mixin.md b/docs/api/models/mixins/save-prepare-mixin.md index d8c7e57..a3f14cb 100644 --- a/docs/api/models/mixins/save-prepare-mixin.md +++ b/docs/api/models/mixins/save-prepare-mixin.md @@ -91,3 +91,22 @@ passed by the user. `(Dict)`: dictionary of model that is about to be saved + +#### validate\_choices + +```python + | @classmethod + | validate_choices(cls, new_kwargs: Dict) -> Dict +``` + +Receives dictionary of model that is about to be saved and validates the +fields with choices set to see if the value is allowed. + +**Arguments**: + +- `new_kwargs (Dict)`: dictionary of model that is about to be saved + +**Returns**: + +`(Dict)`: dictionary of model that is about to be saved + diff --git a/docs/api/models/model-metaclass.md b/docs/api/models/model-metaclass.md index c31abe9..957a9f7 100644 --- a/docs/api/models/model-metaclass.md +++ b/docs/api/models/model-metaclass.md @@ -12,61 +12,6 @@ Class used for type hinting. Users can subclass this one for convenience but it's not required. The only requirement is that ormar.Model has to have inner class with name Meta. - -#### check\_if\_field\_has\_choices - -```python -check_if_field_has_choices(field: Type[BaseField]) -> bool -``` - -Checks if given field has choices populated. -A if it has one, a validator for this field needs to be attached. - -**Arguments**: - -- `field (BaseField)`: ormar field to check - -**Returns**: - -`(bool)`: result of the check - - -#### choices\_validator - -```python -choices_validator(cls: Type["Model"], values: Dict[str, Any]) -> Dict[str, Any] -``` - -Validator that is attached to pydantic model pre root validators. -Validator checks if field value is in field.choices list. - -**Raises**: - -- `ValueError`: if field value is outside of allowed choices. - -**Arguments**: - -- `cls (Model class)`: constructed class -- `values (Dict[str, Any])`: dictionary of field values (pydantic side) - -**Returns**: - -`(Dict[str, Any])`: values if pass validation, otherwise exception is raised - - -#### populate\_choices\_validators - -```python -populate_choices_validators(model: Type["Model"]) -> None -``` - -Checks if Model has any fields with choices set. -If yes it adds choices validation into pre root validators. - -**Arguments**: - -- `model (Model class)`: newly constructed Model - #### add\_cached\_properties @@ -87,26 +32,6 @@ All properties here are used as "cache" to not recalculate them constantly. - `new_model (Model class)`: newly constructed Model - -#### meta\_field\_not\_set - -```python -meta_field_not_set(model: Type["Model"], field_name: str) -> bool -``` - -Checks if field with given name is already present in model.Meta. -Then check if it's set to something truthful -(in practice meaning not None, as it's non or ormar Field only). - -**Arguments**: - -- `model (Model class)`: newly constructed model -- `field_name (str)`: name of the ormar field - -**Returns**: - -`(bool)`: result of the check - #### add\_property\_fields @@ -141,24 +66,6 @@ Signals are emitted in both model own methods and in selected queryset ones. - `new_model (Model class)`: newly constructed model - -#### update\_attrs\_and\_fields - -```python -update_attrs_and_fields(attrs: Dict, new_attrs: Dict, model_fields: Dict, new_model_fields: Dict, new_fields: Set) -> Dict -``` - -Updates __annotations__, values of model fields (so pydantic FieldInfos) -as well as model.Meta.model_fields definitions from parents. - -**Arguments**: - -- `attrs (Dict)`: new namespace for class being constructed -- `new_attrs (Dict)`: related of the namespace extracted from parent class -- `model_fields (Dict[str, BaseField])`: ormar fields in defined in current class -- `new_model_fields (Dict[str, BaseField])`: ormar fields defined in parent classes -- `new_fields (Set[str])`: set of new fields names - #### verify\_constraint\_names @@ -195,7 +102,7 @@ Updates Meta parameters in child from parent if needed. #### copy\_and\_replace\_m2m\_through\_model ```python -copy_and_replace_m2m_through_model(field: Type[ManyToManyField], field_name: str, table_name: str, parent_fields: Dict, attrs: Dict, meta: ModelMeta) -> None +copy_and_replace_m2m_through_model(field: Type[ManyToManyField], field_name: str, table_name: str, parent_fields: Dict, attrs: Dict, meta: ModelMeta, base_class: Type["Model"]) -> None ``` Clones class with Through model for m2m relations, appends child name to the name @@ -211,6 +118,7 @@ Removes the original sqlalchemy table from metadata if it was not removed. **Arguments**: +- `base_class (Type["Model"])`: base class model - `field (Type[ManyToManyField])`: field with relations definition - `field_name (str)`: name of the relation field - `table_name (str)`: name of the table @@ -281,6 +189,24 @@ If the class is a ormar.Model it is skipped. `(Tuple[Dict, Dict])`: updated attrs and model_fields + +#### update\_attrs\_and\_fields + +```python +update_attrs_and_fields(attrs: Dict, new_attrs: Dict, model_fields: Dict, new_model_fields: Dict, new_fields: Set) -> Dict +``` + +Updates __annotations__, values of model fields (so pydantic FieldInfos) +as well as model.Meta.model_fields definitions from parents. + +**Arguments**: + +- `attrs (Dict)`: new namespace for class being constructed +- `new_attrs (Dict)`: related of the namespace extracted from parent class +- `model_fields (Dict[str, BaseField])`: ormar fields in defined in current class +- `new_model_fields (Dict[str, BaseField])`: ormar fields defined in parent classes +- `new_fields (Set[str])`: set of new fields names + ## ModelMetaclass Objects diff --git a/docs/api/models/model-row.md b/docs/api/models/model-row.md new file mode 100644 index 0000000..60f0b3a --- /dev/null +++ b/docs/api/models/model-row.md @@ -0,0 +1,132 @@ + +# models.model\_row + + +## ModelRow Objects + +```python +class ModelRow(NewBaseModel) +``` + + +#### from\_row + +```python + | @classmethod + | from_row(cls, row: sqlalchemy.engine.ResultProxy, source_model: Type["Model"], select_related: List = None, related_models: Any = None, related_field: Type["ForeignKeyField"] = None, excludable: ExcludableItems = None, current_relation_str: str = "", proxy_source_model: Optional[Type["Model"]] = None) -> Optional["Model"] +``` + +Model method to convert raw sql row from database into ormar.Model instance. +Traverses nested models if they were specified in select_related for query. + +Called recurrently and returns model instance if it's present in the row. +Note that it's processing one row at a time, so if there are duplicates of +parent row that needs to be joined/combined +(like parent row in sql join with 2+ child rows) +instances populated in this method are later combined in the QuerySet. +Other method working directly on raw database results is in prefetch_query, +where rows are populated in a different way as they do not have +nested models in result. + +**Arguments**: + +- `proxy_source_model (Optional[Type["ModelRow"]])`: source model from which querysetproxy is constructed +- `excludable (ExcludableItems)`: structure of fields to include and exclude +- `current_relation_str (str)`: name of the relation field +- `source_model (Type[Model])`: model on which relation was defined +- `row (sqlalchemy.engine.result.ResultProxy)`: raw result row from the database +- `select_related (List)`: list of names of related models fetched from database +- `related_models (Union[List, Dict])`: list or dict of related models +- `related_field (Type[ForeignKeyField])`: field with relation declaration + +**Returns**: + +`(Optional[Model])`: returns model if model is populated from database + + +#### \_populate\_nested\_models\_from\_row + +```python + | @classmethod + | _populate_nested_models_from_row(cls, item: dict, row: sqlalchemy.engine.ResultProxy, source_model: Type["Model"], related_models: Any, excludable: ExcludableItems, table_prefix: str, current_relation_str: str = None, proxy_source_model: Type["Model"] = None) -> dict +``` + +Traverses structure of related models and populates the nested models +from the database row. +Related models can be a list if only directly related models are to be +populated, converted to dict if related models also have their own related +models to be populated. + +Recurrently calls from_row method on nested instances and create nested +instances. In the end those instances are added to the final model dictionary. + +**Arguments**: + +- `proxy_source_model (Optional[Type["ModelRow"]])`: source model from which querysetproxy is constructed +- `excludable (ExcludableItems)`: structure of fields to include and exclude +- `source_model (Type[Model])`: source model from which relation started +- `current_relation_str (str)`: joined related parts into one string +- `item (Dict)`: dictionary of already populated nested models, otherwise empty dict +- `row (sqlalchemy.engine.result.ResultProxy)`: raw result row from the database +- `related_models (Union[Dict, List])`: list or dict of related models + +**Returns**: + +`(Dict)`: dictionary with keys corresponding to model fields names +and values are database values + + +#### populate\_through\_instance + +```python + | @classmethod + | populate_through_instance(cls, row: sqlalchemy.engine.ResultProxy, through_name: str, related: str, excludable: ExcludableItems) -> "ModelRow" +``` + +Initialize the through model from db row. +Excluded all relation fields and other exclude/include set in excludable. + +**Arguments**: + +- `row (sqlalchemy.engine.ResultProxy)`: loaded row from database +- `through_name (str)`: name of the through field +- `related (str)`: name of the relation +- `excludable (ExcludableItems)`: structure of fields to include and exclude + +**Returns**: + +`("ModelRow")`: initialized through model without relation + + +#### extract\_prefixed\_table\_columns + +```python + | @classmethod + | extract_prefixed_table_columns(cls, item: dict, row: sqlalchemy.engine.result.ResultProxy, table_prefix: str, excludable: ExcludableItems) -> Dict +``` + +Extracts own fields from raw sql result, using a given prefix. +Prefix changes depending on the table's position in a join. + +If the table is a main table, there is no prefix. +All joined tables have prefixes to allow duplicate column names, +as well as duplicated joins to the same table from multiple different tables. + +Extracted fields populates the related dict later used to construct a Model. + +Used in Model.from_row and PrefetchQuery._populate_rows methods. + +**Arguments**: + +- `excludable (ExcludableItems)`: structure of fields to include and exclude +- `item (Dict)`: dictionary of already populated nested models, otherwise empty dict +- `row (sqlalchemy.engine.result.ResultProxy)`: raw result row from the database +- `table_prefix (str)`: prefix of the table from AliasManager +each pair of tables have own prefix (two of them depending on direction) - +used in joins to allow multiple joins to the same table. + +**Returns**: + +`(Dict)`: dictionary with keys corresponding to model fields names +and values are database values + diff --git a/docs/api/models/model.md b/docs/api/models/model.md index e78825f..facb8f4 100644 --- a/docs/api/models/model.md +++ b/docs/api/models/model.md @@ -5,122 +5,14 @@ ## Model Objects ```python -class Model(NewBaseModel) +class Model(ModelRow) ``` - -#### from\_row - -```python - | @classmethod - | from_row(cls: Type[T], row: sqlalchemy.engine.ResultProxy, select_related: List = None, related_models: Any = None, previous_model: Type[T] = None, source_model: Type[T] = None, related_name: str = None, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None, current_relation_str: str = None) -> Optional[T] -``` - -Model method to convert raw sql row from database into ormar.Model instance. -Traverses nested models if they were specified in select_related for query. - -Called recurrently and returns model instance if it's present in the row. -Note that it's processing one row at a time, so if there are duplicates of -parent row that needs to be joined/combined -(like parent row in sql join with 2+ child rows) -instances populated in this method are later combined in the QuerySet. -Other method working directly on raw database results is in prefetch_query, -where rows are populated in a different way as they do not have -nested models in result. - -**Arguments**: - -- `current_relation_str (str)`: name of the relation field -- `source_model (Type[Model])`: model on which relation was defined -- `row (sqlalchemy.engine.result.ResultProxy)`: raw result row from the database -- `select_related (List)`: list of names of related models fetched from database -- `related_models (Union[List, Dict])`: list or dict of related models -- `previous_model (Model class)`: internal param for nested models to specify table_prefix -- `related_name (str)`: internal parameter - name of current nested model -- `fields (Optional[Union[Dict, Set]])`: fields and related model fields to include -if provided only those are included -- `exclude_fields (Optional[Union[Dict, Set]])`: fields and related model fields to exclude -excludes the fields even if they are provided in fields - -**Returns**: - -`(Optional[Model])`: returns model if model is populated from database - - -#### populate\_nested\_models\_from\_row - -```python - | @classmethod - | populate_nested_models_from_row(cls, item: dict, row: sqlalchemy.engine.ResultProxy, related_models: Any, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None, current_relation_str: str = None, source_model: Type[T] = None) -> dict -``` - -Traverses structure of related models and populates the nested models -from the database row. -Related models can be a list if only directly related models are to be -populated, converted to dict if related models also have their own related -models to be populated. - -Recurrently calls from_row method on nested instances and create nested -instances. In the end those instances are added to the final model dictionary. - -**Arguments**: - -- `source_model (Type[Model])`: source model from which relation started -- `current_relation_str (str)`: joined related parts into one string -- `item (Dict)`: dictionary of already populated nested models, otherwise empty dict -- `row (sqlalchemy.engine.result.ResultProxy)`: raw result row from the database -- `related_models (Union[Dict, List])`: list or dict of related models -- `fields (Optional[Union[Dict, Set]])`: fields and related model fields to include - -if provided only those are included -- `exclude_fields (Optional[Union[Dict, Set]])`: fields and related model fields to exclude -excludes the fields even if they are provided in fields - -**Returns**: - -`(Dict)`: dictionary with keys corresponding to model fields names -and values are database values - - -#### extract\_prefixed\_table\_columns - -```python - | @classmethod - | extract_prefixed_table_columns(cls, item: dict, row: sqlalchemy.engine.result.ResultProxy, table_prefix: str, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None) -> dict -``` - -Extracts own fields from raw sql result, using a given prefix. -Prefix changes depending on the table's position in a join. - -If the table is a main table, there is no prefix. -All joined tables have prefixes to allow duplicate column names, -as well as duplicated joins to the same table from multiple different tables. - -Extracted fields populates the related dict later used to construct a Model. - -Used in Model.from_row and PrefetchQuery._populate_rows methods. - -**Arguments**: - -- `item (Dict)`: dictionary of already populated nested models, otherwise empty dict -- `row (sqlalchemy.engine.result.ResultProxy)`: raw result row from the database -- `table_prefix (str)`: prefix of the table from AliasManager -each pair of tables have own prefix (two of them depending on direction) - -used in joins to allow multiple joins to the same table. -- `fields (Optional[Union[Dict, Set]])`: fields and related model fields to include - -if provided only those are included -- `exclude_fields (Optional[Union[Dict, Set]])`: fields and related model fields to exclude -excludes the fields even if they are provided in fields - -**Returns**: - -`(Dict)`: dictionary with keys corresponding to model fields names -and values are database values - #### upsert ```python - | async upsert(**kwargs: Any) -> T + | async upsert(**kwargs: Any) -> "Model" ``` Performs either a save or an update depending on the presence of the pk. @@ -139,7 +31,7 @@ For save kwargs are ignored, used only in update if provided. #### save ```python - | async save() -> T + | async save() -> "Model" ``` Performs a save of given Model instance. @@ -203,7 +95,7 @@ number of updated instances ```python | @staticmethod - | async _update_and_follow(rel: T, follow: bool, visited: Set, update_count: int) -> Tuple[int, Set] + | async _update_and_follow(rel: "Model", follow: bool, visited: Set, update_count: int) -> Tuple[int, Set] ``` Internal method used in save_related to follow related models and update numbers @@ -227,7 +119,7 @@ number of updated instances #### update ```python - | async update(**kwargs: Any) -> T + | async update(**kwargs: Any) -> "Model" ``` Performs update of Model instance in the database. @@ -274,7 +166,7 @@ or update and the Model will be saved in database again. #### load ```python - | async load() -> T + | async load() -> "Model" ``` Allow to refresh existing Models fields from database. @@ -289,3 +181,40 @@ Does NOT refresh the related models fields if they were loaded before. `(Model)`: reloaded Model + +#### load\_all + +```python + | async load_all(follow: bool = False, exclude: Union[List, str, Set, Dict] = None) -> "Model" +``` + +Allow to refresh existing Models fields from database. +Performs refresh of the related models fields. + +By default loads only self and the directly related ones. + +If follow=True is set it loads also related models of related models. + +To not get stuck in an infinite loop as related models also keep a relation +to parent model visited models set is kept. + +That way already visited models that are nested are loaded, but the load do not +follow them inside. So Model A -> Model B -> Model C -> Model A -> Model X +will load second Model A but will never follow into Model X. +Nested relations of those kind need to be loaded manually. + +**Raises**: + +- `NoMatch`: If given pk is not found in database. + +**Arguments**: + +- `exclude ()`: +- `follow (bool)`: flag to trigger deep save - +by default only directly related models are saved +with follow=True also related models of related models are saved + +**Returns**: + +`(Model)`: reloaded Model + diff --git a/docs/api/models/new-basemodel.md b/docs/api/models/new-basemodel.md index 6499cf0..88b4ed7 100644 --- a/docs/api/models/new-basemodel.md +++ b/docs/api/models/new-basemodel.md @@ -146,7 +146,7 @@ Raises exception if model is abstract or has ForwardRefs in relation fields. #### \_extract\_related\_model\_instead\_of\_field ```python - | _extract_related_model_instead_of_field(item: str) -> Optional[Union["T", Sequence["T"]]] + | _extract_related_model_instead_of_field(item: str) -> Optional[Union["Model", Sequence["Model"]]] ``` Retrieves the related model/models from RelationshipManager. @@ -276,7 +276,7 @@ cause some dialect require different treatment #### remove ```python - | remove(parent: "T", name: str) -> None + | remove(parent: "Model", name: str) -> None ``` Removes child from relation with given name in RelationshipManager diff --git a/docs/api/query-set/join.md b/docs/api/query-set/join.md index 213d4c0..9a8f898 100644 --- a/docs/api/query-set/join.md +++ b/docs/api/query-set/join.md @@ -22,11 +22,25 @@ Shortcut for ormar's model AliasManager stored on Meta. `(AliasManager)`: alias manager from model's Meta - -#### on\_clause + +#### to\_table ```python - | on_clause(previous_alias: str, from_clause: str, to_clause: str) -> text + | @property + | to_table() -> str +``` + +Shortcut to table name of the next model + +**Returns**: + +`(str)`: name of the target table + + +#### \_on\_clause + +```python + | _on_clause(previous_alias: str, from_clause: str, to_clause: str) -> text ``` Receives aliases and names of both ends of the join and combines them @@ -99,11 +113,11 @@ Updated are: - `related_name (str)`: name of the relation to follow - `remainder (Any)`: deeper tables if there are more nested joins - -#### process\_m2m\_through\_table + +#### \_process\_m2m\_through\_table ```python - | process_m2m_through_table() -> None + | _process_m2m_through_table() -> None ``` Process Through table of the ManyToMany relation so that source table is @@ -119,11 +133,11 @@ Replaces needed parameters like: To point to through model - -#### process\_m2m\_related\_name\_change + +#### \_process\_m2m\_related\_name\_change ```python - | process_m2m_related_name_change(reverse: bool = False) -> str + | _process_m2m_related_name_change(reverse: bool = False) -> str ``` Extracts relation name to link join through the Through model declared on @@ -158,74 +172,21 @@ Updates the used aliases list directly. Process order_by causes for non m2m relations. - -#### \_replace\_many\_to\_many\_order\_by\_columns + +#### \_get\_order\_bys ```python - | _replace_many_to_many_order_by_columns(part: str, new_part: str) -> None -``` - -Substitutes the name of the relation with actual model name in m2m order bys. - -**Arguments**: - -- `part (str)`: name of the field with relation -- `new_part (str)`: name of the target model - - -#### \_check\_if\_condition\_apply - -```python - | @staticmethod - | _check_if_condition_apply(condition: List, part: str) -> bool -``` - -Checks filter conditions to find if they apply to current join. - -**Arguments**: - -- `condition (List[str])`: list of parts of condition split by '__' -- `part (str)`: name of the current relation join. - -**Returns**: - -`(bool)`: result of the check - - -#### set\_aliased\_order\_by - -```python - | set_aliased_order_by(condition: List[str], to_table: str) -> None -``` - -Substitute hyphens ('-') with descending order. -Construct actual sqlalchemy text clause using aliased table and column name. - -**Arguments**: - -- `condition (List[str])`: list of parts of a current condition split by '__' -- `to_table (sqlalchemy.sql.elements.quoted_name)`: target table - - -#### get\_order\_bys - -```python - | get_order_bys(to_table: str, pkname_alias: str) -> None + | _get_order_bys() -> None ``` Triggers construction of order bys if they are given. Otherwise by default each table is sorted by a primary key column asc. -**Arguments**: - -- `to_table (sqlalchemy.sql.elements.quoted_name)`: target table -- `pkname_alias (str)`: alias of the primary key column - - -#### get\_to\_and\_from\_keys + +#### \_get\_to\_and\_from\_keys ```python - | get_to_and_from_keys() -> Tuple[str, str] + | _get_to_and_from_keys() -> Tuple[str, str] ``` Based on the relation type, name of the relation and previous models and parts diff --git a/docs/api/query-set/prefetch-query.md b/docs/api/query-set/prefetch-query.md index cc848f0..ff0a64c 100644 --- a/docs/api/query-set/prefetch-query.md +++ b/docs/api/query-set/prefetch-query.md @@ -1,26 +1,6 @@ # queryset.prefetch\_query - -#### add\_relation\_field\_to\_fields - -```python -add_relation_field_to_fields(fields: Union[Set[Any], Dict[Any, Any], None], related_field_name: str) -> Union[Set[Any], Dict[Any, Any], None] -``` - -Adds related field into fields to include as otherwise it would be skipped. -Related field is added only if fields are already populated. -Empty fields implies all fields. - -**Arguments**: - -- `fields (Dict)`: Union[Set[Any], Dict[Any, Any], None] -- `related_field_name (str)`: name of the field with relation - -**Returns**: - -`(Union[Set[Any], Dict[Any, Any], None])`: updated fields dict - #### sort\_models @@ -232,7 +212,7 @@ on each of the parent models from list. #### \_extract\_related\_models ```python - | async _extract_related_models(related: str, target_model: Type["Model"], prefetch_dict: Dict, select_dict: Dict, fields: Union[Set[Any], Dict[Any, Any], None], exclude_fields: Union[Set[Any], Dict[Any, Any], None], orders_by: Dict) -> None + | async _extract_related_models(related: str, target_model: Type["Model"], prefetch_dict: Dict, select_dict: Dict, excludable: "ExcludableItems", orders_by: Dict) -> None ``` Constructs queries with required ids and extracts data with fields that should @@ -261,7 +241,7 @@ Calls itself recurrently to extract deeper nested relations of related model. #### \_run\_prefetch\_query ```python - | async _run_prefetch_query(target_field: Type["BaseField"], fields: Union[Set[Any], Dict[Any, Any], None], exclude_fields: Union[Set[Any], Dict[Any, Any], None], filter_clauses: List) -> Tuple[str, List] + | async _run_prefetch_query(target_field: Type["BaseField"], excludable: "ExcludableItems", filter_clauses: List, related_field_name: str) -> Tuple[str, str, List] ``` Actually runs the queries against the database and populates the raw response @@ -273,8 +253,6 @@ models. **Arguments**: - `target_field (Type["BaseField"])`: ormar field with relation definition -- `fields (Union[Set[Any], Dict[Any, Any], None])`: fields to include -- `exclude_fields (Union[Set[Any], Dict[Any, Any], None])`: fields to exclude - `filter_clauses (List[sqlalchemy.sql.elements.TextClause])`: list of clauses, actually one clause with ids of relation **Returns**: @@ -320,7 +298,7 @@ Updates models that are already loaded, usually children of children. #### \_populate\_rows ```python - | _populate_rows(rows: List, target_field: Type["ForeignKeyField"], parent_model: Type["Model"], table_prefix: str, fields: Union[Set[Any], Dict[Any, Any], None], exclude_fields: Union[Set[Any], Dict[Any, Any], None], prefetch_dict: Dict, orders_by: Dict) -> None + | _populate_rows(rows: List, target_field: Type["ForeignKeyField"], parent_model: Type["Model"], table_prefix: str, exclude_prefix: str, excludable: "ExcludableItems", prefetch_dict: Dict, orders_by: Dict) -> None ``` Instantiates children models extracted from given relation. @@ -334,12 +312,11 @@ and set on the parent model after sorting if needed. **Arguments**: +- `excludable (ExcludableItems)`: structure of fields to include and exclude - `rows (List[sqlalchemy.engine.result.RowProxy])`: raw sql response from the prefetch query - `target_field (Type["BaseField"])`: field with relation definition from parent model - `parent_model (Type[Model])`: model with relation definition - `table_prefix (str)`: prefix of the target table from current relation -- `fields (Union[Set[Any], Dict[Any, Any], None])`: fields to include -- `exclude_fields (Union[Set[Any], Dict[Any, Any], None])`: fields to exclude - `prefetch_dict (Dict)`: dictionaries of related models to prefetch - `orders_by (Dict)`: dictionary of order by clauses by model diff --git a/docs/api/query-set/query-set.md b/docs/api/query-set/query-set.md index 4c2ad6a..34f0945 100644 --- a/docs/api/query-set/query-set.md +++ b/docs/api/query-set/query-set.md @@ -38,6 +38,16 @@ Shortcut to model class set on QuerySet. `(Type[Model])`: model class + +#### rebuild\_self + +```python + | rebuild_self(filter_clauses: List = None, exclude_clauses: List = None, select_related: List = None, limit_count: int = None, offset: int = None, excludable: "ExcludableItems" = None, order_bys: List = None, prefetch_related: List = None, limit_raw_sql: bool = None, proxy_source_model: Optional[Type["Model"]] = None) -> "QuerySet" +``` + +Method that returns new instance of queryset based on passed params, +all not passed params are taken from current values. + #### \_prefetch\_related\_models @@ -252,7 +262,7 @@ To chain related `Models` relation use double underscores between names. #### fields ```python - | fields(columns: Union[List, str, Set, Dict]) -> "QuerySet" + | fields(columns: Union[List, str, Set, Dict], _is_exclude: bool = False) -> "QuerySet" ``` With `fields()` you can select subset of model columns to limit the data load. @@ -293,6 +303,7 @@ To include whole nested model specify model related field name and ellipsis. **Arguments**: +- `_is_exclude (bool)`: flag if it's exclude or include operation - `columns (Union[List, str, Set, Dict])`: columns to include **Returns**: diff --git a/docs/api/query-set/query.md b/docs/api/query-set/query.md index 4715c36..dc3314d 100644 --- a/docs/api/query-set/query.md +++ b/docs/api/query-set/query.md @@ -17,38 +17,6 @@ class Query() Initialize empty order_by dict to be populated later during the query call - -#### prefixed\_pk\_name - -```python - | @property - | prefixed_pk_name() -> str -``` - -Shortcut for extracting prefixed with alias primary key column name from main -model - -**Returns**: - -`(str)`: alias of pk column prefix with table name. - - -#### alias - -```python - | alias(name: str) -> str -``` - -Shortcut to extracting column alias from given master model. - -**Arguments**: - -- `name (str)`: name of column - -**Returns**: - -`(str)`: alias of given column name - #### apply\_order\_bys\_for\_primary\_model diff --git a/docs/api/query-set/utils.md b/docs/api/query-set/utils.md index 0ba9471..f42e340 100644 --- a/docs/api/query-set/utils.md +++ b/docs/api/query-set/utils.md @@ -154,7 +154,7 @@ with all children models under their relation keys. #### get\_relationship\_alias\_model\_and\_str ```python -get_relationship_alias_model_and_str(source_model: Type["Model"], related_parts: List) -> Tuple[str, Type["Model"], str] +get_relationship_alias_model_and_str(source_model: Type["Model"], related_parts: List) -> Tuple[str, Type["Model"], str, bool] ``` Walks the relation to retrieve the actual model on which the clause should be diff --git a/docs/api/relations/alias-manager.md b/docs/api/relations/alias-manager.md index 24016de..d563cf8 100644 --- a/docs/api/relations/alias-manager.md +++ b/docs/api/relations/alias-manager.md @@ -120,7 +120,7 @@ Adds alias to the dictionary of aliases under given key. #### resolve\_relation\_alias ```python - | resolve_relation_alias(from_model: Type["Model"], relation_name: str) -> str + | resolve_relation_alias(from_model: Union[Type["Model"], Type["ModelRow"]], relation_name: str) -> str ``` Given model and relation name returns the alias for this relation. @@ -134,3 +134,24 @@ Given model and relation name returns the alias for this relation. `(str)`: alias of the relation + +#### resolve\_relation\_alias\_after\_complex + +```python + | resolve_relation_alias_after_complex(source_model: Union[Type["Model"], Type["ModelRow"]], relation_str: str, relation_field: Type["ForeignKeyField"]) -> str +``` + +Given source model and relation string returns the alias for this complex +relation if it exists, otherwise fallback to normal relation from a relation +field definition. + +**Arguments**: + +- `relation_field (Type["ForeignKeyField"])`: field with direct relation definition +- `source_model (source Model)`: model with query starts +- `relation_str (str)`: string with relation joins defined + +**Returns**: + +`(str)`: alias of the relation + diff --git a/docs/api/relations/queryset-proxy.md b/docs/api/relations/queryset-proxy.md index 627e995..1eb9637 100644 --- a/docs/api/relations/queryset-proxy.md +++ b/docs/api/relations/queryset-proxy.md @@ -5,7 +5,7 @@ ## QuerysetProxy Objects ```python -class QuerysetProxy(ormar.QuerySetProtocol) +class QuerysetProxy() ``` Exposes QuerySet methods on relations, but also handles creating and removing @@ -43,7 +43,7 @@ Set's the queryset. Initialized in RelationProxy. #### \_assign\_child\_to\_parent ```python - | _assign_child_to_parent(child: Optional["T"]) -> None + | _assign_child_to_parent(child: Optional["Model"]) -> None ``` Registers child in parents RelationManager. @@ -56,7 +56,7 @@ Registers child in parents RelationManager. #### \_register\_related ```python - | _register_related(child: Union["T", Sequence[Optional["T"]]]) -> None + | _register_related(child: Union["Model", Sequence[Optional["Model"]]]) -> None ``` Registers child/ children in parents RelationManager. @@ -78,20 +78,35 @@ Cleans the current list of the related models. #### create\_through\_instance ```python - | async create_through_instance(child: "T") -> None + | async create_through_instance(child: "Model", **kwargs: Any) -> None ``` Crete a through model instance in the database for m2m relations. **Arguments**: +- `kwargs (Any)`: dict of additional keyword arguments for through instance +- `child (Model)`: child model instance + + +#### update\_through\_instance + +```python + | async update_through_instance(child: "Model", **kwargs: Any) -> None +``` + +Updates a through model instance in the database for m2m relations. + +**Arguments**: + +- `kwargs (Any)`: dict of additional keyword arguments for through instance - `child (Model)`: child model instance #### delete\_through\_instance ```python - | async delete_through_instance(child: "T") -> None + | async delete_through_instance(child: "Model") -> None ``` Removes through model instance from the database for m2m relations. @@ -256,6 +271,27 @@ Actual call delegated to QuerySet. `(Model)`: created model + +#### update + +```python + | async update(each: bool = False, **kwargs: Any) -> int +``` + +Updates the model table after applying the filters from kwargs. + +You have to either pass a filter to narrow down a query or explicitly pass +each=True flag to affect whole table. + +**Arguments**: + +- `each (bool)`: flag if whole table should be affected if no filter is passed +- `kwargs (Any)`: fields names and proper value types + +**Returns**: + +`(int)`: number of updated rows + #### get\_or\_create diff --git a/docs/api/relations/relation-manager.md b/docs/api/relations/relation-manager.md index 57ad512..d83febe 100644 --- a/docs/api/relations/relation-manager.md +++ b/docs/api/relations/relation-manager.md @@ -10,37 +10,6 @@ class RelationsManager() Manages relations on a Model, each Model has it's own instance. - -#### \_get\_relation\_type - -```python - | _get_relation_type(field: Type[BaseField]) -> RelationType -``` - -Returns type of the relation declared on a field. - -**Arguments**: - -- `field (Type[BaseField])`: field with relation declaration - -**Returns**: - -`(RelationType)`: type of the relation defined on field - - -#### \_add\_relation - -```python - | _add_relation(field: Type[BaseField]) -> None -``` - -Registers relation in the manager. -Adds Relation instance under field.name. - -**Arguments**: - -- `field (Type[BaseField])`: field with relation declaration - #### \_\_contains\_\_ @@ -62,7 +31,7 @@ Checks if relation with given name is already registered. #### get ```python - | get(name: str) -> Optional[Union["T", Sequence["T"]]] + | get(name: str) -> Optional[Union["Model", Sequence["Model"]]] ``` Returns the related model/models if relation is set. @@ -76,23 +45,6 @@ Actual call is delegated to Relation instance registered under relation name. `(Optional[Union[Model, List[Model]])`: related model or list of related models if set - -#### \_get - -```python - | _get(name: str) -> Optional[Relation] -``` - -Returns the actual relation and not the related model(s). - -**Arguments**: - -- `name (str)`: name of the relation - -**Returns**: - -`(ormar.relations.relation.Relation)`: Relation instance - #### add @@ -148,3 +100,51 @@ of relation from which you want to remove the parent. - `parent (Model)`: parent Model - `name (str)`: name of the relation + +#### \_get + +```python + | _get(name: str) -> Optional[Relation] +``` + +Returns the actual relation and not the related model(s). + +**Arguments**: + +- `name (str)`: name of the relation + +**Returns**: + +`(ormar.relations.relation.Relation)`: Relation instance + + +#### \_get\_relation\_type + +```python + | _get_relation_type(field: Type["BaseField"]) -> RelationType +``` + +Returns type of the relation declared on a field. + +**Arguments**: + +- `field (Type[BaseField])`: field with relation declaration + +**Returns**: + +`(RelationType)`: type of the relation defined on field + + +#### \_add\_relation + +```python + | _add_relation(field: Type["BaseField"]) -> None +``` + +Registers relation in the manager. +Adds Relation instance under field.name. + +**Arguments**: + +- `field (Type[BaseField])`: field with relation declaration + diff --git a/docs/api/relations/relation-proxy.md b/docs/api/relations/relation-proxy.md index 645bb2a..1d122a7 100644 --- a/docs/api/relations/relation-proxy.md +++ b/docs/api/relations/relation-proxy.md @@ -131,7 +131,7 @@ will be deleted, and not only removed from relation). #### add ```python - | async add(item: "Model") -> None + | async add(item: "Model", **kwargs: Any) -> None ``` Adds child model to relation. @@ -140,5 +140,6 @@ For ManyToMany relations through instance is automatically created. **Arguments**: +- `kwargs (Any)`: dict of additional keyword arguments for through instance - `item (Model)`: child to add to relation diff --git a/docs/api/relations/relation.md b/docs/api/relations/relation.md index 1c50b36..29e8ab7 100644 --- a/docs/api/relations/relation.md +++ b/docs/api/relations/relation.md @@ -27,7 +27,7 @@ Keeps related Models and handles adding/removing of the children. #### \_\_init\_\_ ```python - | __init__(manager: "RelationsManager", type_: RelationType, field_name: str, to: Type["T"], through: Type["T"] = None) -> None + | __init__(manager: "RelationsManager", type_: RelationType, field_name: str, to: Type["Model"], through: Type["Model"] = None) -> None ``` Initialize the Relation and keep the related models either as instances of @@ -73,7 +73,7 @@ Find child model in RelationProxy if exists. #### add ```python - | add(child: "T") -> None + | add(child: "Model") -> None ``` Adds child Model to relation, either sets child as related model or adds @@ -101,7 +101,7 @@ it from the list in RelationProxy depending on relation type. #### get ```python - | get() -> Optional[Union[List["T"], "T"]] + | get() -> Optional[Union[List["Model"], "Model"]] ``` Return the related model or models from RelationProxy. diff --git a/docs/index.md b/docs/index.md index c6c5f41..d4655e5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -306,7 +306,7 @@ async def joins(): # visit: https://collerek.github.io/ormar/relations/ # to read more about joins and subqueries - # visit: https://collerek.github.io/ormar/queries/delete/ + # visit: https://collerek.github.io/ormar/queries/joins-and-subqueries/ async def filter_and_sort(): diff --git a/docs/models/methods.md b/docs/models/methods.md index 084ba25..d8d25b1 100644 --- a/docs/models/methods.md +++ b/docs/models/methods.md @@ -27,6 +27,39 @@ await track.album.load() track.album.name # will return 'Malibu' ``` +## load_all + +`load_all(follow: bool = False, exclude: Union[List, str, Set, Dict] = None) -> Model` + +Method works like `load()` but also goes through all relations of the `Model` on which the method is called, +and reloads them from database. + +By default the `load_all` method loads only models that are directly related (one step away) to the model on which the method is called. + +But you can specify the `follow=True` parameter to traverse through nested models and load all of them in the relation tree. + +!!!warning + To avoid circular updates with `follow=True` set, `load_all` keeps a set of already visited Models, + and won't perform nested `loads` on Models that were already visited. + + So if you have a diamond or circular relations types you need to perform the loads in a manual way. + + ```python + # in example like this the second Street (coming from City) won't be load_all, so ZipCode won't be reloaded + Street -> District -> City -> Street -> ZipCode + ``` + +Method accepts also optional exclude parameter that works exactly the same as exclude_fields method in `QuerySet`. +That way you can remove fields from related models being refreshed or skip whole related models. + +Method performs one database query so it's more efficient than nested calls to `load()` and `all()` on related models. + +!!!tip + To read more about `exclude` read [exclude_fields][exclude_fields] + +!!!warning + All relations are cleared on `load_all()`, so if you exclude some nested models they will be empty after call. + ## save `save() -> self` @@ -128,3 +161,4 @@ But you can specify the `follow=True` parameter to traverse through nested model [alembic]: https://alembic.sqlalchemy.org/en/latest/tutorial.html [save status]: ../models/index/#model-save-status [Internals]: #internals +[exclude_fields]: ../queries/select-columns.md#exclude_fields diff --git a/docs/relations/index.md b/docs/relations/index.md index 0896c13..76ab3ba 100644 --- a/docs/relations/index.md +++ b/docs/relations/index.md @@ -52,7 +52,7 @@ class Department(ormar.Model): To define many-to-many relation use `ManyToMany` field. -```python hl_lines="25-26" +```python hl_lines="18" class Category(ormar.Model): class Meta: tablename = "categories" @@ -62,13 +62,6 @@ class Category(ormar.Model): id: int = ormar.Integer(primary_key=True) name: str = ormar.String(max_length=40) -# note: you need to specify through model -class PostCategory(ormar.Model): - class Meta: - tablename = "posts_categories" - database = database - metadata = metadata - class Post(ormar.Model): class Meta: tablename = "posts" @@ -77,9 +70,7 @@ class Post(ormar.Model): id: int = ormar.Integer(primary_key=True) title: str = ormar.String(max_length=200) - categories: Optional[Union[Category, List[Category]]] = ormar.ManyToMany( - Category, through=PostCategory - ) + categories: Optional[List[Category]] = ormar.ManyToMany(Category) ``` @@ -92,7 +83,52 @@ class Post(ormar.Model): It allows you to use `await post.categories.all()` but also `await category.posts.all()` to fetch data related only to specific post, category etc. -##Self-reference and postponed references +## Through fields + +As part of the `ManyToMany` relation you can define a through model, that can contain additional +fields that you can use to filter, order etc. Fields defined like this are exposed on the reverse +side of the current query for m2m models. + +So if you query from model `A` to model `B`, only model `B` has through field exposed. +Which kind of make sense, since it's a one through model/field for each of related models. + +```python hl_lines="10-15" +class Category(ormar.Model): + class Meta(BaseMeta): + tablename = "categories" + + id = ormar.Integer(primary_key=True) + name = ormar.String(max_length=40) + +# you can specify additional fields on through model +class PostCategory(ormar.Model): + class Meta(BaseMeta): + tablename = "posts_x_categories" + + id: int = ormar.Integer(primary_key=True) + sort_order: int = ormar.Integer(nullable=True) + param_name: str = ormar.String(default="Name", max_length=200) + + +class Post(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + title: str = ormar.String(max_length=200) + categories = ormar.ManyToMany(Category, through=PostCategory) +``` + +!!!tip + To read more about many-to-many relations and through fields visit [many-to-many][many-to-many] section + + +!!!tip + ManyToMany allows you to query the related models with [queryset-proxy][queryset-proxy]. + + It allows you to use `await post.categories.all()` but also `await category.posts.all()` to fetch data related only to specific post, category etc. + +## Self-reference and postponed references In order to create auto-relation or create two models that reference each other in at least two different relations (remember the reverse side is auto-registered for you), you need to use diff --git a/docs/relations/many-to-many.md b/docs/relations/many-to-many.md index 8038a6f..24be745 100644 --- a/docs/relations/many-to-many.md +++ b/docs/relations/many-to-many.md @@ -1,6 +1,6 @@ # ManyToMany -`ManyToMany(to, through)` has required parameters `to` and `through` that takes target and relation `Model` classes. +`ManyToMany(to, through)` has required parameters `to` and optional `through` that takes target and relation `Model` classes. Sqlalchemy column and Type are automatically taken from target `Model`. @@ -9,7 +9,7 @@ Sqlalchemy column and Type are automatically taken from target `Model`. ## Defining Models -```Python hl_lines="32 49-50" +```Python hl_lines="40" --8<-- "../docs_src/relations/docs002.py" ``` @@ -20,8 +20,154 @@ post = await Post.objects.create(title="Hello, M2M", author=guido) news = await Category.objects.create(name="News") ``` +## Through Model + +Optionally if you want to add additional fields you can explicitly create and pass +the through model class. + +```Python hl_lines="14-20 29" +--8<-- "../docs_src/relations/docs004.py" +``` + +!!!warning + Note that even of you do not provide through model it's going to be created for you automatically and + still has to be included in example in `alembic` migrations. + +!!!tip + Note that you need to provide `through` model if you want to + customize the `Through` model name or the database table name of this model. + +If you do not provide the Through field it will be generated for you. + +The default naming convention is: + +* for class name it's union of both classes name (parent+other) so in example above + it would be `PostCategory` +* for table name it similar but with underscore in between and s in the end of class + lowercase name, in example above would be `posts_categorys` + +## Through Fields + +The through field is auto added to the reverse side of the relation. + +The exposed field is named as lowercase `Through` class name. + +The exposed field **explicitly has no relations loaded** as the relation is already populated in `ManyToMany` field, +so it's useful only when additional fields are provided on `Through` model. + +In a sample model setup as following: + +```Python hl_lines="14-20 29" +--8<-- "../docs_src/relations/docs004.py" +``` + +the through field can be used as a normal model field in most of the QuerySet operations. + +Note that through field is attached only to related side of the query so: + +```python +post = await Post.objects.select_related("categories").get() +# source model has no through field +assert post.postcategory is None +# related models have through field +assert post.categories[0].postcategory is not None + +# same is applicable for reversed query +category = await Category.objects.select_related("posts").get() +assert category.postcategory is None +assert category.posts[0].postcategory is not None +``` + +Through field can be used for filtering the data. +```python +post = ( + await Post.objects.select_related("categories") + .filter(postcategory__sort_order__gt=1) + .get() + ) +``` + +!!!tip + Note that despite that the actual instance is not populated on source model, + in queries, order by statements etc you can access through model from both sides. + So below query has exactly the same effect (note access through `categories`) + + ```python + post = ( + await Post.objects.select_related("categories") + .filter(categories__postcategory__sort_order__gt=1) + .get() + ) + ``` + +Through model can be used in order by queries. +```python +post = ( + await Post.objects.select_related("categories") + .order_by("-postcategory__sort_order") + .get() + ) +``` + +You can also select subset of the columns in a normal `QuerySet` way with `fields` +and `exclude_fields`. + +```python +post2 = ( + await Post.objects.select_related("categories") + .exclude_fields("postcategory__param_name") + .get() + ) +``` + +!!!warning + Note that because through fields explicitly nullifies all relation fields, as relation + is populated in ManyToMany field, you should not use the standard model methods like + `save()` and `update()` before re-loading the field from database. + +If you want to modify the through field in place remember to reload it from database. +Otherwise you will set relations to None so effectively make the field useless! + +```python +# always reload the field before modification +await post2.categories[0].postcategory.load() +# only then update the field +await post2.categories[0].postcategory.update(sort_order=3) +``` +Note that reloading the model effectively reloads the relations as `pk_only` models +(only primary key is set) so they are not fully populated, but it's enough to preserve +the relation on update. + +!!!warning + If you use i.e. `fastapi` the partially loaded related models on through field might cause + `pydantic` validation errors (that's the primary reason why they are not populated by default). + So either you need to exclude the related fields in your response, or fully load the related + models. In example above it would mean: + ```python + await post2.categories[0].postcategory.post.load() + await post2.categories[0].postcategory.category.load() + ``` + Alternatively you can use `load_all()`: + ```python + await post2.categories[0].postcategory.load_all() + ``` + +**Preferred way of update is through queryset proxy `update()` method** + +```python +# filter the desired related model with through field and update only through field params +await post2.categories.filter(name='Test category').update(postcategory={"sort_order": 3}) +``` + + +## Relation methods + ### add +`add(item: Model, **kwargs)` + +Allows you to add model to ManyToMany relation. + ```python # Add a category to a post. await post.categories.add(news) @@ -30,10 +176,24 @@ await news.posts.add(post) ``` !!!warning - In all not None cases the primary key value for related model **has to exist in database**. + In all not `None` cases the primary key value for related model **has to exist in database**. Otherwise an IntegrityError will be raised by your database driver library. +If you declare your models with a Through model with additional fields, you can populate them +during adding child model to relation. + +In order to do so, pass keyword arguments with field names and values to `add()` call. + +Note that this works only for `ManyToMany` relations. + +```python +post = await Post(title="Test post").save() +category = await Category(name="Test category").save() +# apart from model pass arguments referencing through model fields +await post.categories.add(category, sort_order=1, param_name='test') +``` + ### remove Removal of the related model one by one. diff --git a/docs/relations/queryset-proxy.md b/docs/relations/queryset-proxy.md index 9e033b6..db1343d 100644 --- a/docs/relations/queryset-proxy.md +++ b/docs/relations/queryset-proxy.md @@ -104,6 +104,29 @@ assert len(await post.categories.all()) == 2 !!!tip Read more in queries documentation [create][create] +For `ManyToMany` relations there is an additional functionality of passing parameters +that will be used to create a through model if you declared additional fields on explicitly +provided Through model. + +Given sample like this: + +```Python hl_lines="14-20, 29" +--8<-- "../docs_src/relations/docs004.py" +``` + +You can populate fields on through model in the `create()` call in a following way: + +```python + +post = await Post(title="Test post").save() +await post.categories.create( + name="Test category1", + # in arguments pass a dictionary with name of the through field and keys + # corresponding to through model fields + postcategory={"sort_order": 1, "param_name": "volume"}, +) +``` + ### get_or_create `get_or_create(**kwargs) -> Model` @@ -122,6 +145,29 @@ Updates the model, or in case there is no match in database creates a new one. !!!tip Read more in queries documentation [update_or_create][update_or_create] +### update + +`update(**kwargs, each:bool = False) -> int` + +Updates the related model with provided keyword arguments, return number of updated rows. + +!!!tip + Read more in queries documentation [update][update] + +Note that for `ManyToMany` relations update can also accept an argument with through field +name and a dictionary of fields. + +```Python hl_lines="14-20 29" +--8<-- "../docs_src/relations/docs004.py" +``` + +In example above you can update attributes of `postcategory` in a following call: +```python +await post.categories.filter(name="Test category3").update( + postcategory={"sort_order": 4} + ) +``` + ## Filtering and sorting ### filter @@ -251,6 +297,7 @@ Returns a bool value to confirm if there are rows matching the given criteria (a [create]: ../queries/create.md#create [get_or_create]: ../queries/read.md#get_or_create [update_or_create]: ../queries/update.md#update_or_create +[update]: ../queries/update.md#update [filter]: ../queries/filter-and-sort.md#filter [exclude]: ../queries/filter-and-sort.md#exclude [select_related]: ../queries/joins-and-subqueries.md#select_related diff --git a/docs/releases.md b/docs/releases.md index 9d98f9f..ef6342a 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,9 +1,19 @@ -# 0.9.5 +# 0.9.6 + +##Important +* `Through` model for `ManyToMany` relations now **becomes optional**. It's not a breaking change + since if you provide it everything works just fine as it used to. So if you don't want or need any additional + fields on `Through` model you can skip it. Note that it's going to be created for you automatically and + still has to be included in example in `alembic` migrations. + If you want to delete existing one check the default naming convention to adjust your existing database structure. + + Note that you still need to provide it if you want to + customize the `Through` model name or the database table name. ## Features * Add `update` method to `QuerysetProxy` so now it's possible to update related models directly from parent model in `ManyToMany` relations and in reverse `ForeignKey` relations. Note that update like in `QuerySet` `update` returns number of - updated models and **does not update related models in place** on praent model. To get the refreshed data on parent model you need to refresh + updated models and **does not update related models in place** on parent model. To get the refreshed data on parent model you need to refresh the related models (i.e. `await model_instance.related.all()`) * Added possibility to add more fields on `Through` model for `ManyToMany` relationships: * name of the through model field is the lowercase name of the Through class @@ -14,13 +24,21 @@ * you can filter on through model fields * you can include and exclude fields on through models * through models are attached only to related models (i.e. if you query from A to B -> only on B) - * check the updated docs for more information + * note that through models are explicitly loaded without relations -> relation is already populated in ManyToMany field. + * note that just like before you cannot declare the relation fields on through model, they will be populated for you by `ormar` + * check the updated ManyToMany relation docs for more information # Other * Updated docs and api docs -* Refactors and optimisations mainly related to filters and order bys +* Refactors and optimisations mainly related to filters, exclusions and order bys +# 0.9.5 + +## Fixes +* Fix creation of `pydantic` FieldInfo after update of `pydantic` to version >=1.8 +* Pin required dependency versions to avoid such situations in the future + # 0.9.4 diff --git a/docs_src/relations/docs002.py b/docs_src/relations/docs002.py index 8dd0566..9831ccb 100644 --- a/docs_src/relations/docs002.py +++ b/docs_src/relations/docs002.py @@ -29,15 +29,6 @@ class Category(ormar.Model): name: str = ormar.String(max_length=40) -class PostCategory(ormar.Model): - class Meta: - tablename = "posts_categories" - database = database - metadata = metadata - - # If there are no additional columns id will be created automatically as Integer - - class Post(ormar.Model): class Meta: tablename = "posts" @@ -46,7 +37,5 @@ class Post(ormar.Model): id: int = ormar.Integer(primary_key=True) title: str = ormar.String(max_length=200) - categories: Optional[Union[Category, List[Category]]] = ormar.ManyToMany( - Category, through=PostCategory - ) + categories: Optional[List[Category]] = ormar.ManyToMany(Category) author: Optional[Author] = ormar.ForeignKey(Author) diff --git a/docs_src/relations/docs004.py b/docs_src/relations/docs004.py new file mode 100644 index 0000000..9a3b0c0 --- /dev/null +++ b/docs_src/relations/docs004.py @@ -0,0 +1,29 @@ +class BaseMeta(ormar.ModelMeta): + database = database + metadata = metadata + + +class Category(ormar.Model): + class Meta(BaseMeta): + tablename = "categories" + + id = ormar.Integer(primary_key=True) + name = ormar.String(max_length=40) + + +class PostCategory(ormar.Model): + class Meta(BaseMeta): + tablename = "posts_x_categories" + + id: int = ormar.Integer(primary_key=True) + sort_order: int = ormar.Integer(nullable=True) + param_name: str = ormar.String(default="Name", max_length=200) + + +class Post(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + title: str = ormar.String(max_length=200) + categories = ormar.ManyToMany(Category, through=PostCategory) diff --git a/mkdocs.yml b/mkdocs.yml index 2bdafce..8aba336 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -53,9 +53,11 @@ nav: - Relation Mixin: api/models/mixins/relation-mixin.md - Save Prepare Mixin: api/models/mixins/save-prepare-mixin.md - api/models/model.md + - Model Row: api/models/model-row.md - New BaseModel: api/models/new-basemodel.md - Model Table Proxy: api/models/model-table-proxy.md - Model Metaclass: api/models/model-metaclass.md + - Excludable Items: api/models/excludable-items.md - Fields: - Base Field: api/fields/base-field.md - Model Fields: api/fields/model-fields.md diff --git a/ormar/fields/many_to_many.py b/ormar/fields/many_to_many.py index 2b2b300..0e7602e 100644 --- a/ormar/fields/many_to_many.py +++ b/ormar/fields/many_to_many.py @@ -1,5 +1,5 @@ import sys -from typing import Any, List, Optional, TYPE_CHECKING, Tuple, Type, Union +from typing import Any, List, Optional, TYPE_CHECKING, Tuple, Type, Union, cast from pydantic.typing import ForwardRef, evaluate_forwardref import ormar # noqa: I100 @@ -43,7 +43,7 @@ def populate_m2m_params_based_on_to_model( def ManyToMany( to: "ToType", - through: "ToType", + through: Optional["ToType"] = None, *, name: str = None, unique: bool = False, @@ -212,3 +212,21 @@ class ManyToManyField(ForeignKeyField, ormar.QuerySetProtocol, ormar.RelationPro :rtype: Type["Model"] """ return cls.through + + @classmethod + def create_default_through_model(cls) -> None: + """ + Creates default empty through model if no additional fields are required. + """ + owner_name = cls.owner.get_name(lower=False) + to_name = cls.to.get_name(lower=False) + class_name = f"{owner_name}{to_name}" + table_name = f"{owner_name.lower()}s_{to_name.lower()}s" + new_meta_namespace = { + "tablename": table_name, + "database": cls.owner.Meta.database, + "metadata": cls.owner.Meta.metadata, + } + new_meta = type("Meta", (), new_meta_namespace) + through_model = type(class_name, (ormar.Model,), {"Meta": new_meta}) + cls.through = cast(Type["Model"], through_model) diff --git a/ormar/models/helpers/sqlalchemy.py b/ormar/models/helpers/sqlalchemy.py index b641969..3475c66 100644 --- a/ormar/models/helpers/sqlalchemy.py +++ b/ormar/models/helpers/sqlalchemy.py @@ -154,6 +154,8 @@ def sqlalchemy_columns_from_model_fields( pkname = None for field_name, field in model_fields.items(): field.owner = new_model + if field.is_multi and not field.through: + field.create_default_through_model() if field.primary_key: pkname = check_pk_column_validity(field_name, field, pkname) if not field.pydantic_only and not field.virtual and not field.is_multi: diff --git a/ormar/models/metaclass.py b/ormar/models/metaclass.py index 304fd19..cbef18d 100644 --- a/ormar/models/metaclass.py +++ b/ormar/models/metaclass.py @@ -209,13 +209,14 @@ def update_attrs_from_base_meta( # noqa: CCR001 setattr(attrs["Meta"], param, parent_value) -def copy_and_replace_m2m_through_model( +def copy_and_replace_m2m_through_model( # noqa: CFQ002 field: Type[ManyToManyField], field_name: str, table_name: str, parent_fields: Dict, attrs: Dict, meta: ModelMeta, + base_class: Type["Model"], ) -> None: """ Clones class with Through model for m2m relations, appends child name to the name @@ -229,6 +230,8 @@ def copy_and_replace_m2m_through_model( Removes the original sqlalchemy table from metadata if it was not removed. + :param base_class: base class model + :type base_class: Type["Model"] :param field: field with relations definition :type field: Type[ManyToManyField] :param field_name: name of the relation field @@ -249,6 +252,10 @@ def copy_and_replace_m2m_through_model( copy_field.related_name = related_name # type: ignore through_class = field.through + if not through_class: + field.owner = base_class + field.create_default_through_model() + through_class = field.through new_meta: ormar.ModelMeta = type( # type: ignore "Meta", (), dict(through_class.Meta.__dict__), ) @@ -338,6 +345,7 @@ def copy_data_from_parent_model( # noqa: CCR001 parent_fields=parent_fields, attrs=attrs, meta=meta, + base_class=base_class, # type: ignore ) elif field.is_relation and field.related_name: diff --git a/ormar/models/mixins/merge_mixin.py b/ormar/models/mixins/merge_mixin.py index 0ad471d..32d7288 100644 --- a/ormar/models/mixins/merge_mixin.py +++ b/ormar/models/mixins/merge_mixin.py @@ -1,5 +1,5 @@ from collections import OrderedDict -from typing import List, Sequence, TYPE_CHECKING +from typing import List, TYPE_CHECKING import ormar @@ -17,7 +17,7 @@ class MergeModelMixin: """ @classmethod - def merge_instances_list(cls, result_rows: Sequence["Model"]) -> Sequence["Model"]: + def merge_instances_list(cls, result_rows: List["Model"]) -> List["Model"]: """ Merges a list of models into list of unique models. diff --git a/ormar/models/mixins/relation_mixin.py b/ormar/models/mixins/relation_mixin.py index aebaa20..4fdd9e7 100644 --- a/ormar/models/mixins/relation_mixin.py +++ b/ormar/models/mixins/relation_mixin.py @@ -1,5 +1,5 @@ import inspect -from typing import List, Optional, Set, TYPE_CHECKING +from typing import List, Optional, Set, TYPE_CHECKING, Type, Union class RelationMixin: @@ -8,7 +8,7 @@ class RelationMixin: """ if TYPE_CHECKING: # pragma no cover - from ormar import ModelMeta + from ormar import ModelMeta, Model Meta: ModelMeta _related_names: Optional[Set] @@ -63,7 +63,7 @@ class RelationMixin: return related_fields @classmethod - def extract_related_names(cls) -> Set: + def extract_related_names(cls) -> Set[str]: """ Returns List of fields names for all relations declared on a model. List is cached in cls._related_names for quicker access. @@ -118,3 +118,50 @@ class RelationMixin: name for name in related_names if cls.Meta.model_fields[name].nullable } return related_names + + @classmethod + def _iterate_related_models( + cls, + visited: Set[Union[Type["Model"], Type["RelationMixin"]]] = None, + source_relation: str = None, + source_model: Union[Type["Model"], Type["RelationMixin"]] = None, + ) -> List[str]: + """ + Iterates related models recursively to extract relation strings of + nested not visited models. + + :param visited: set of already visited models + :type visited: Set[str] + :param source_relation: name of the current relation + :type source_relation: str + :param source_model: model from which relation comes in nested relations + :type source_model: Type["Model"] + :return: list of relation strings to be passed to select_related + :rtype: List[str] + """ + visited = visited or set() + visited.add(cls) + relations = cls.extract_related_names() + processed_relations = [] + for relation in relations: + target_model = cls.Meta.model_fields[relation].to + if source_model and target_model == source_model: + continue + if target_model not in visited: + visited.add(target_model) + deep_relations = target_model._iterate_related_models( + visited=visited, source_relation=relation, source_model=cls + ) + processed_relations.extend(deep_relations) + # TODO add test for circular deps + else: # pragma: no cover + processed_relations.append(relation) + if processed_relations: + final_relations = [ + f"{source_relation + '__' if source_relation else ''}{relation}" + for relation in processed_relations + ] + else: + final_relations = [source_relation] if source_relation else [] + + return final_relations diff --git a/ormar/models/model.py b/ormar/models/model.py index 2c14888..26ef420 100644 --- a/ormar/models/model.py +++ b/ormar/models/model.py @@ -1,8 +1,11 @@ from typing import ( Any, + Dict, + List, Set, TYPE_CHECKING, Tuple, + Union, ) import ormar.queryset # noqa I100 @@ -265,3 +268,45 @@ class Model(ModelRow): self.update_from_dict(kwargs) self.set_save_status(True) return self + + async def load_all( + self, follow: bool = False, exclude: Union[List, str, Set, Dict] = None + ) -> "Model": + """ + Allow to refresh existing Models fields from database. + Performs refresh of the related models fields. + + By default loads only self and the directly related ones. + + If follow=True is set it loads also related models of related models. + + To not get stuck in an infinite loop as related models also keep a relation + to parent model visited models set is kept. + + That way already visited models that are nested are loaded, but the load do not + follow them inside. So Model A -> Model B -> Model C -> Model A -> Model X + will load second Model A but will never follow into Model X. + Nested relations of those kind need to be loaded manually. + + :raises NoMatch: If given pk is not found in database. + + :param exclude: related models to exclude + :type exclude: Union[List, str, Set, Dict] + :param follow: flag to trigger deep save - + by default only directly related models are saved + with follow=True also related models of related models are saved + :type follow: bool + :return: reloaded Model + :rtype: Model + """ + relations = list(self.extract_related_names()) + if follow: + relations = self._iterate_related_models() + queryset = self.__class__.objects + print(relations) + if exclude: + queryset = queryset.exclude_fields(exclude) + instance = await queryset.select_related(relations).get(pk=self.pk) + self._orm.clear() + self.update_from_dict(instance.dict()) + return self diff --git a/ormar/models/model_row.py b/ormar/models/model_row.py index c4ea9d0..a14d418 100644 --- a/ormar/models/model_row.py +++ b/ormar/models/model_row.py @@ -88,6 +88,7 @@ class ModelRow(NewBaseModel): current_relation_str=current_relation_str, source_model=source_model, # type: ignore proxy_source_model=proxy_source_model, # type: ignore + table_prefix=table_prefix, ) item = cls.extract_prefixed_table_columns( item=item, row=row, table_prefix=table_prefix, excludable=excludable @@ -110,6 +111,7 @@ class ModelRow(NewBaseModel): source_model: Type["Model"], related_models: Any, excludable: ExcludableItems, + table_prefix: str, current_relation_str: str = None, proxy_source_model: Type["Model"] = None, ) -> dict: @@ -143,15 +145,20 @@ class ModelRow(NewBaseModel): """ for related in related_models: + field = cls.Meta.model_fields[related] + field = cast(Type["ForeignKeyField"], field) + model_cls = field.to + model_excludable = excludable.get( + model_cls=cast(Type["Model"], cls), alias=table_prefix + ) + if model_excludable.is_excluded(related): + return item + relation_str = ( "__".join([current_relation_str, related]) if current_relation_str else related ) - field = cls.Meta.model_fields[related] - field = cast(Type["ForeignKeyField"], field) - model_cls = field.to - remainder = None if isinstance(related_models, dict) and related_models[related]: remainder = related_models[related] diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 7fadcb8..8adc4f3 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -148,8 +148,8 @@ class QuerySet: ) async def _prefetch_related_models( - self, models: Sequence[Optional["Model"]], rows: List - ) -> Sequence[Optional["Model"]]: + self, models: List[Optional["Model"]], rows: List + ) -> List[Optional["Model"]]: """ Performs prefetch query for selected models names. @@ -169,7 +169,7 @@ class QuerySet: ) return await query.prefetch_related(models=models, rows=rows) # type: ignore - def _process_query_result_rows(self, rows: List) -> Sequence[Optional["Model"]]: + def _process_query_result_rows(self, rows: List) -> List[Optional["Model"]]: """ Process database rows and initialize ormar Model from each of the rows. diff --git a/ormar/relations/relation.py b/ormar/relations/relation.py index 0bdc671..0f9be3d 100644 --- a/ormar/relations/relation.py +++ b/ormar/relations/relation.py @@ -68,6 +68,14 @@ class Relation: else None ) + def clear(self) -> None: + if self._type in (RelationType.PRIMARY, RelationType.THROUGH): + self.related_models = None + self._owner.__dict__[self.field_name] = None + elif self.related_models is not None: + self.related_models._clear() + self._owner.__dict__[self.field_name] = [] + @property def through(self) -> Type["Model"]: if not self._through: # pragma: no cover diff --git a/ormar/relations/relation_manager.py b/ormar/relations/relation_manager.py index 0719550..19c0dc5 100644 --- a/ormar/relations/relation_manager.py +++ b/ormar/relations/relation_manager.py @@ -26,37 +26,6 @@ class RelationsManager: for field in self._related_fields: self._add_relation(field) - def _get_relation_type(self, field: Type["BaseField"]) -> RelationType: - """ - Returns type of the relation declared on a field. - - :param field: field with relation declaration - :type field: Type[BaseField] - :return: type of the relation defined on field - :rtype: RelationType - """ - if field.is_multi: - return RelationType.MULTIPLE - if field.is_through: - return RelationType.THROUGH - return RelationType.PRIMARY if not field.virtual else RelationType.REVERSE - - def _add_relation(self, field: Type["BaseField"]) -> None: - """ - Registers relation in the manager. - Adds Relation instance under field.name. - - :param field: field with relation declaration - :type field: Type[BaseField] - """ - self._relations[field.name] = Relation( - manager=self, - type_=self._get_relation_type(field), - field_name=field.name, - to=field.to, - through=getattr(field, "through", None), - ) - def __contains__(self, item: str) -> bool: """ Checks if relation with given name is already registered. @@ -68,6 +37,10 @@ class RelationsManager: """ return item in self._related_names + def clear(self) -> None: + for relation in self._relations.values(): + relation.clear() + def get(self, name: str) -> Optional[Union["Model", Sequence["Model"]]]: """ Returns the related model/models if relation is set. @@ -83,20 +56,6 @@ class RelationsManager: return relation.get() return None # pragma nocover - def _get(self, name: str) -> Optional[Relation]: - """ - Returns the actual relation and not the related model(s). - - :param name: name of the relation - :type name: str - :return: Relation instance - :rtype: ormar.relations.relation.Relation - """ - relation = self._relations.get(name, None) - if relation is not None: - return relation - return None - @staticmethod def add(parent: "Model", child: "Model", field: Type["ForeignKeyField"],) -> None: """ @@ -164,3 +123,48 @@ class RelationsManager: relation_name = item.Meta.model_fields[name].get_related_name() item._orm.remove(name, parent) parent._orm.remove(relation_name, item) + + def _get(self, name: str) -> Optional[Relation]: + """ + Returns the actual relation and not the related model(s). + + :param name: name of the relation + :type name: str + :return: Relation instance + :rtype: ormar.relations.relation.Relation + """ + relation = self._relations.get(name, None) + if relation is not None: + return relation + return None + + def _get_relation_type(self, field: Type["BaseField"]) -> RelationType: + """ + Returns type of the relation declared on a field. + + :param field: field with relation declaration + :type field: Type[BaseField] + :return: type of the relation defined on field + :rtype: RelationType + """ + if field.is_multi: + return RelationType.MULTIPLE + if field.is_through: + return RelationType.THROUGH + return RelationType.PRIMARY if not field.virtual else RelationType.REVERSE + + def _add_relation(self, field: Type["BaseField"]) -> None: + """ + Registers relation in the manager. + Adds Relation instance under field.name. + + :param field: field with relation declaration + :type field: Type[BaseField] + """ + self._relations[field.name] = Relation( + manager=self, + type_=self._get_relation_type(field), + field_name=field.name, + to=field.to, + through=getattr(field, "through", None), + ) diff --git a/ormar/relations/relation_proxy.py b/ormar/relations/relation_proxy.py index 596f594..ce4b86f 100644 --- a/ormar/relations/relation_proxy.py +++ b/ormar/relations/relation_proxy.py @@ -75,6 +75,9 @@ class RelationProxy(list): self._initialize_queryset() return getattr(self.queryset_proxy, item) + def _clear(self) -> None: + super().clear() + def _initialize_queryset(self) -> None: """ Initializes the QuerySetProxy if not yet initialized. diff --git a/pydoc-markdown.yml b/pydoc-markdown.yml index a7ba311..6f0188f 100644 --- a/pydoc-markdown.yml +++ b/pydoc-markdown.yml @@ -21,9 +21,15 @@ renderer: - title: Model contents: - models.model.* + - title: Model Row + contents: + - models.model_row.* - title: New BaseModel contents: - models.newbasemodel.* + - title: Excludable Items + contents: + - models.excludable.* - title: Model Table Proxy contents: - models.modelproxy.* diff --git a/tests/test_aliases.py b/tests/test_aliases.py index 239c182..bb6b40f 100644 --- a/tests/test_aliases.py +++ b/tests/test_aliases.py @@ -1,4 +1,4 @@ -from typing import Optional, Union, List +from typing import List, Optional import databases import pytest @@ -23,13 +23,6 @@ class Child(ormar.Model): born_year: int = ormar.Integer(name="year_born", nullable=True) -class ArtistChildren(ormar.Model): - class Meta: - tablename = "children_x_artists" - metadata = metadata - database = database - - class Artist(ormar.Model): class Meta: tablename = "artists" @@ -40,9 +33,7 @@ class Artist(ormar.Model): first_name: str = ormar.String(name="fname", max_length=100) last_name: str = ormar.String(name="lname", max_length=100) born_year: int = ormar.Integer(name="year") - children: Optional[Union[Child, List[Child]]] = ormar.ManyToMany( - Child, through=ArtistChildren - ) + children: Optional[List[Child]] = ormar.ManyToMany(Child) class Album(ormar.Model): diff --git a/tests/test_fastapi_docs.py b/tests/test_fastapi_docs.py index 08118ea..03f0892 100644 --- a/tests/test_fastapi_docs.py +++ b/tests/test_fastapi_docs.py @@ -42,18 +42,13 @@ class Category(ormar.Model): name: str = ormar.String(max_length=100) -class ItemsXCategories(ormar.Model): - class Meta(LocalMeta): - tablename = "items_x_categories" - - class Item(ormar.Model): class Meta(LocalMeta): pass id: int = ormar.Integer(primary_key=True) name: str = ormar.String(max_length=100) - categories = ormar.ManyToMany(Category, through=ItemsXCategories) + categories = ormar.ManyToMany(Category) @pytest.fixture(autouse=True, scope="module") diff --git a/tests/test_inheritance_concrete.py b/tests/test_inheritance_concrete.py index 2ac22ff..6bc3859 100644 --- a/tests/test_inheritance_concrete.py +++ b/tests/test_inheritance_concrete.py @@ -121,11 +121,11 @@ class Bus(Car): max_persons: int = ormar.Integer() -class PersonsCar(ormar.Model): - class Meta: - tablename = "cars_x_persons" - metadata = metadata - database = db +# class PersonsCar(ormar.Model): +# class Meta: +# tablename = "cars_x_persons" +# metadata = metadata +# database = db class Car2(ormar.Model): @@ -138,7 +138,9 @@ class Car2(ormar.Model): name: str = ormar.String(max_length=50) owner: Person = ormar.ForeignKey(Person, related_name="owned") co_owners: List[Person] = ormar.ManyToMany( - Person, through=PersonsCar, related_name="coowned" + Person, + # through=PersonsCar, + related_name="coowned", ) created_date: datetime.datetime = ormar.DateTime(default=datetime.datetime.now) diff --git a/tests/test_load_all.py b/tests/test_load_all.py new file mode 100644 index 0000000..3b4bde5 --- /dev/null +++ b/tests/test_load_all.py @@ -0,0 +1,171 @@ +from typing import List + +import databases +import pytest +import sqlalchemy + +import ormar +from tests.settings import DATABASE_URL + +database = databases.Database(DATABASE_URL, force_rollback=True) +metadata = sqlalchemy.MetaData() + + +class BaseMeta(ormar.ModelMeta): + database = database + metadata = metadata + + +class Language(ormar.Model): + class Meta(BaseMeta): + tablename = "languages" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + level: str = ormar.String(max_length=150, default="Beginner") + + +class CringeLevel(ormar.Model): + class Meta(BaseMeta): + tablename = "levels" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + language = ormar.ForeignKey(Language) + + +class NickName(ormar.Model): + class Meta(BaseMeta): + tablename = "nicks" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, nullable=False, name="hq_name") + is_lame: bool = ormar.Boolean(nullable=True) + level: CringeLevel = ormar.ForeignKey(CringeLevel) + + +class HQ(ormar.Model): + class Meta(BaseMeta): + tablename = "hqs" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, nullable=False, name="hq_name") + nicks: List[NickName] = ormar.ManyToMany(NickName) + + +class Company(ormar.Model): + class Meta(BaseMeta): + tablename = "companies" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, nullable=False, name="company_name") + founded: int = ormar.Integer(nullable=True) + hq: HQ = ormar.ForeignKey(HQ, related_name="companies") + + +@pytest.fixture(autouse=True, scope="module") +def create_test_database(): + engine = sqlalchemy.create_engine(DATABASE_URL) + metadata.drop_all(engine) + metadata.create_all(engine) + yield + metadata.drop_all(engine) + + +@pytest.mark.asyncio +async def test_load_all_fk_rel(): + async with database: + async with database.transaction(force_rollback=True): + hq = await HQ.objects.create(name="Main") + company = await Company.objects.create(name="Banzai", founded=1988, hq=hq) + + hq = await HQ.objects.get(name="Main") + await hq.load_all() + + assert hq.companies[0] == company + assert hq.companies[0].name == "Banzai" + assert hq.companies[0].founded == 1988 + + +@pytest.mark.asyncio +async def test_load_all_many_to_many(): + async with database: + async with database.transaction(force_rollback=True): + nick1 = await NickName.objects.create(name="BazingaO", is_lame=False) + nick2 = await NickName.objects.create(name="Bazinga20", is_lame=True) + hq = await HQ.objects.create(name="Main") + await hq.nicks.add(nick1) + await hq.nicks.add(nick2) + + hq = await HQ.objects.get(name="Main") + await hq.load_all() + + assert hq.nicks[0] == nick1 + assert hq.nicks[0].name == "BazingaO" + + assert hq.nicks[1] == nick2 + assert hq.nicks[1].name == "Bazinga20" + + +@pytest.mark.asyncio +async def test_loading_reversed_relation(): + async with database: + async with database.transaction(force_rollback=True): + hq = await HQ.objects.create(name="Main") + await Company.objects.create(name="Banzai", founded=1988, hq=hq) + + company = await Company.objects.get(name="Banzai") + await company.load_all() + + assert company.hq == hq + + +@pytest.mark.asyncio +async def test_loading_nested(): + async with database: + async with database.transaction(force_rollback=True): + language = await Language.objects.create(name="English") + level = await CringeLevel.objects.create(name="High", language=language) + level2 = await CringeLevel.objects.create(name="Low", language=language) + nick1 = await NickName.objects.create( + name="BazingaO", is_lame=False, level=level + ) + nick2 = await NickName.objects.create( + name="Bazinga20", is_lame=True, level=level2 + ) + hq = await HQ.objects.create(name="Main") + await hq.nicks.add(nick1) + await hq.nicks.add(nick2) + + hq = await HQ.objects.get(name="Main") + await hq.load_all(follow=True) + + assert hq.nicks[0] == nick1 + assert hq.nicks[0].name == "BazingaO" + assert hq.nicks[0].level.name == "High" + assert hq.nicks[0].level.language.name == "English" + + assert hq.nicks[1] == nick2 + assert hq.nicks[1].name == "Bazinga20" + assert hq.nicks[1].level.name == "Low" + assert hq.nicks[1].level.language.name == "English" + + await hq.load_all(follow=True, exclude="nicks__level__language") + assert len(hq.nicks) == 2 + assert hq.nicks[0].level.language is None + assert hq.nicks[1].level.language is None + + await hq.load_all(follow=True, exclude="nicks__level__language__level") + assert len(hq.nicks) == 2 + assert hq.nicks[0].level.language is not None + assert hq.nicks[0].level.language.level is None + assert hq.nicks[1].level.language is not None + assert hq.nicks[1].level.language.level is None + + await hq.load_all(follow=True, exclude="nicks__level") + assert len(hq.nicks) == 2 + assert hq.nicks[0].level is None + assert hq.nicks[1].level is None + + await hq.load_all(follow=True, exclude="nicks") + assert len(hq.nicks) == 0 diff --git a/tests/test_m2m_through_fields.py b/tests/test_m2m_through_fields.py index 6c4211a..3f600e0 100644 --- a/tests/test_m2m_through_fields.py +++ b/tests/test_m2m_through_fields.py @@ -288,6 +288,25 @@ async def test_update_through_models_from_queryset_on_through() -> Any: assert post2.categories[2].postcategory.param_name == "area" +@pytest.mark.asyncio +async def test_update_through_model_after_load() -> Any: + async with database: + post = await Post(title="Test post").save() + await post.categories.create( + name="Test category1", + postcategory={"sort_order": 2, "param_name": "volume"}, + ) + post2 = await Post.objects.select_related("categories").get() + assert len(post2.categories) == 1 + + await post2.categories[0].postcategory.load() + await post2.categories[0].postcategory.update(sort_order=3) + + post3 = await Post.objects.select_related("categories").get() + assert len(post3.categories) == 1 + assert post3.categories[0].postcategory.sort_order == 3 + + @pytest.mark.asyncio async def test_update_through_from_related() -> Any: async with database: @@ -371,9 +390,10 @@ async def test_excluding_fields_on_through_model() -> Any: # ordering by in order_by (V) # updating in query (V) # updating from querysetproxy (V) -# including/excluding in fields? +# including/excluding in fields? (V) +# make through optional? auto-generated for cases other fields are missing? (V) # modifying from instance (both sides?) (X) <= no, the loaded one doesn't have relations +# allowing to change fk fields names in through model? (X) <= separate issue -# allowing to change fk fields names in through model? -# make through optional? auto-generated for cases other fields are missing? +# prevent adding relation on through field definition diff --git a/tests/test_many_to_many.py b/tests/test_many_to_many.py index 8b10eae..3989df8 100644 --- a/tests/test_many_to_many.py +++ b/tests/test_many_to_many.py @@ -1,5 +1,5 @@ import asyncio -from typing import List, Union, Optional +from typing import List, Optional import databases import pytest @@ -34,13 +34,6 @@ class Category(ormar.Model): name: str = ormar.String(max_length=40) -class PostCategory(ormar.Model): - class Meta: - tablename = "posts_categories" - database = database - metadata = metadata - - class Post(ormar.Model): class Meta: tablename = "posts" @@ -49,9 +42,7 @@ class Post(ormar.Model): id: int = ormar.Integer(primary_key=True) title: str = ormar.String(max_length=200) - categories: Optional[Union[Category, List[Category]]] = ormar.ManyToMany( - Category, through=PostCategory - ) + categories: Optional[List[Category]] = ormar.ManyToMany(Category) author: Optional[Author] = ormar.ForeignKey(Author) @@ -74,6 +65,7 @@ async def create_test_database(): async def cleanup(): yield async with database: + PostCategory = Post.Meta.model_fields["categories"].through await PostCategory.objects.delete(each=True) await Post.objects.delete(each=True) await Category.objects.delete(each=True) diff --git a/tests/test_order_by.py b/tests/test_order_by.py index 02639ca..bbb6385 100644 --- a/tests/test_order_by.py +++ b/tests/test_order_by.py @@ -85,13 +85,6 @@ class Car(ormar.Model): factory: Optional[Factory] = ormar.ForeignKey(Factory) -class UsersCar(ormar.Model): - class Meta: - tablename = "cars_x_users" - metadata = metadata - database = database - - class User(ormar.Model): class Meta: tablename = "users" @@ -100,7 +93,7 @@ class User(ormar.Model): id: int = ormar.Integer(primary_key=True) name: str = ormar.String(max_length=100) - cars: List[Car] = ormar.ManyToMany(Car, through=UsersCar) + cars: List[Car] = ormar.ManyToMany(Car) @pytest.fixture(autouse=True, scope="module") From 4e27f07a7e06ca838adddeb01fa9ce501a1dd96e Mon Sep 17 00:00:00 2001 From: collerek Date: Thu, 4 Mar 2021 13:12:07 +0100 Subject: [PATCH 13/14] som types fixes, fix for wrong prefixes in model_row for complex relations, test load_all with repeating tables, add docs --- docs/releases.md | 4 +++ ormar/fields/foreign_key.py | 4 +-- ormar/models/helpers/models.py | 7 ++-- ormar/models/mixins/relation_mixin.py | 40 +++++++++++++++++------ ormar/models/model.py | 16 +++++---- ormar/models/model_row.py | 25 +++++++++++--- ormar/queryset/queryset.py | 2 +- ormar/relations/relation.py | 2 +- tests/test_excluding_fields_in_fastapi.py | 3 +- tests/test_m2m_through_fields.py | 2 +- tests/test_more_same_table_joins.py | 14 ++++++++ 11 files changed, 90 insertions(+), 29 deletions(-) diff --git a/docs/releases.md b/docs/releases.md index ef6342a..2671b3d 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -15,6 +15,10 @@ in `ManyToMany` relations and in reverse `ForeignKey` relations. Note that update like in `QuerySet` `update` returns number of updated models and **does not update related models in place** on parent model. To get the refreshed data on parent model you need to refresh the related models (i.e. `await model_instance.related.all()`) +* Add `load_all(follow=False, exclude=None)` model method that allows to load current instance of the model + with all related models in one call. By default it loads only directly related models but setting + `follow=True` causes traversing the tree (avoiding loops). You can also pass `exclude` parameter + that works the same as `QuerySet.exclude_fields()` method. * Added possibility to add more fields on `Through` model for `ManyToMany` relationships: * name of the through model field is the lowercase name of the Through class * you can pass additional fields when calling `add(child, **kwargs)` on relation (on `QuerysetProxy`) diff --git a/ormar/fields/foreign_key.py b/ormar/fields/foreign_key.py index 643b88d..65a9df0 100644 --- a/ormar/fields/foreign_key.py +++ b/ormar/fields/foreign_key.py @@ -460,7 +460,7 @@ class ForeignKeyField(BaseField): return model @classmethod - def get_relation_name(cls) -> str: + def get_relation_name(cls) -> str: # pragma: no cover """ Returns name of the relation, which can be a own name or through model names for m2m models @@ -471,7 +471,7 @@ class ForeignKeyField(BaseField): return cls.name @classmethod - def get_source_model(cls) -> Type["Model"]: + def get_source_model(cls) -> Type["Model"]: # pragma: no cover """ Returns model from which the relation comes -> either owner or through model diff --git a/ormar/models/helpers/models.py b/ormar/models/helpers/models.py index 6d67e91..e0b5d3c 100644 --- a/ormar/models/helpers/models.py +++ b/ormar/models/helpers/models.py @@ -1,3 +1,4 @@ +import collections import itertools import sqlite3 from typing import Any, Dict, List, TYPE_CHECKING, Tuple, Type @@ -123,7 +124,7 @@ def extract_annotations_and_default_vals(attrs: Dict) -> Tuple[Dict, Dict]: return attrs, model_fields -def group_related_list(list_: List) -> Dict: +def group_related_list(list_: List) -> collections.OrderedDict: """ Translates the list of related strings into a dictionary. That way nested models are grouped to traverse them in a right order @@ -152,7 +153,9 @@ def group_related_list(list_: List) -> Dict: result_dict[key] = group_related_list(new) else: result_dict.setdefault(key, []).extend(new) - return {k: v for k, v in sorted(result_dict.items(), key=lambda item: len(item[1]))} + return collections.OrderedDict( + sorted(result_dict.items(), key=lambda item: len(item[1])) + ) def meta_field_not_set(model: Type["Model"], field_name: str) -> bool: diff --git a/ormar/models/mixins/relation_mixin.py b/ormar/models/mixins/relation_mixin.py index 4fdd9e7..53fab3c 100644 --- a/ormar/models/mixins/relation_mixin.py +++ b/ormar/models/mixins/relation_mixin.py @@ -1,5 +1,13 @@ import inspect -from typing import List, Optional, Set, TYPE_CHECKING, Type, Union +from typing import ( + Callable, + List, + Optional, + Set, + TYPE_CHECKING, + Type, + Union, +) class RelationMixin: @@ -13,6 +21,7 @@ class RelationMixin: Meta: ModelMeta _related_names: Optional[Set] _related_fields: Optional[List] + get_name: Callable @classmethod def extract_db_own_fields(cls) -> Set: @@ -122,7 +131,8 @@ class RelationMixin: @classmethod def _iterate_related_models( cls, - visited: Set[Union[Type["Model"], Type["RelationMixin"]]] = None, + visited: Set[str] = None, + source_visited: Set[str] = None, source_relation: str = None, source_model: Union[Type["Model"], Type["RelationMixin"]] = None, ) -> List[str]: @@ -139,22 +149,24 @@ class RelationMixin: :return: list of relation strings to be passed to select_related :rtype: List[str] """ - visited = visited or set() - visited.add(cls) + source_visited = source_visited or set() + if not source_model: + source_visited = cls._populate_source_model_prefixes() relations = cls.extract_related_names() processed_relations = [] for relation in relations: target_model = cls.Meta.model_fields[relation].to if source_model and target_model == source_model: continue - if target_model not in visited: - visited.add(target_model) + if target_model not in source_visited or not source_model: deep_relations = target_model._iterate_related_models( - visited=visited, source_relation=relation, source_model=cls + visited=visited, + source_visited=source_visited, + source_relation=relation, + source_model=cls, ) processed_relations.extend(deep_relations) - # TODO add test for circular deps - else: # pragma: no cover + else: processed_relations.append(relation) if processed_relations: final_relations = [ @@ -163,5 +175,13 @@ class RelationMixin: ] else: final_relations = [source_relation] if source_relation else [] - return final_relations + + @classmethod + def _populate_source_model_prefixes(cls) -> Set: + relations = cls.extract_related_names() + visited = {cls} + for relation in relations: + target_model = cls.Meta.model_fields[relation].to + visited.add(target_model) + return visited diff --git a/ormar/models/model.py b/ormar/models/model.py index 26ef420..48b9f58 100644 --- a/ormar/models/model.py +++ b/ormar/models/model.py @@ -5,6 +5,7 @@ from typing import ( Set, TYPE_CHECKING, Tuple, + TypeVar, Union, ) @@ -17,6 +18,8 @@ from ormar.models.model_row import ModelRow if TYPE_CHECKING: # pragma nocover from ormar import QuerySet +T = TypeVar("T", bound="Model") + class Model(ModelRow): __abstract__ = False @@ -28,7 +31,7 @@ class Model(ModelRow): _repr = {k: getattr(self, k) for k, v in self.Meta.model_fields.items()} return f"{self.__class__.__name__}({str(_repr)})" - async def upsert(self, **kwargs: Any) -> "Model": + async def upsert(self: T, **kwargs: Any) -> T: """ Performs either a save or an update depending on the presence of the pk. If the pk field is filled it's an update, otherwise the save is performed. @@ -43,7 +46,7 @@ class Model(ModelRow): return await self.save() return await self.update(**kwargs) - async def save(self) -> "Model": + async def save(self: T) -> T: """ Performs a save of given Model instance. If primary key is already saved, db backend will throw integrity error. @@ -189,7 +192,7 @@ class Model(ModelRow): update_count += 1 return update_count, visited - async def update(self, **kwargs: Any) -> "Model": + async def update(self: T, **kwargs: Any) -> T: """ Performs update of Model instance in the database. Fields can be updated before or you can pass them as kwargs. @@ -248,7 +251,7 @@ class Model(ModelRow): await self.signals.post_delete.send(sender=self.__class__, instance=self) return result - async def load(self) -> "Model": + async def load(self: T) -> T: """ Allow to refresh existing Models fields from database. Be careful as the related models can be overwritten by pk_only models in load. @@ -270,8 +273,8 @@ class Model(ModelRow): return self async def load_all( - self, follow: bool = False, exclude: Union[List, str, Set, Dict] = None - ) -> "Model": + self: T, follow: bool = False, exclude: Union[List, str, Set, Dict] = None + ) -> T: """ Allow to refresh existing Models fields from database. Performs refresh of the related models fields. @@ -303,7 +306,6 @@ class Model(ModelRow): if follow: relations = self._iterate_related_models() queryset = self.__class__.objects - print(relations) if exclude: queryset = queryset.exclude_fields(exclude) instance = await queryset.select_related(relations).get(pk=self.pk) diff --git a/ormar/models/model_row.py b/ormar/models/model_row.py index a14d418..d9c674d 100644 --- a/ormar/models/model_row.py +++ b/ormar/models/model_row.py @@ -31,6 +31,7 @@ class ModelRow(NewBaseModel): excludable: ExcludableItems = None, current_relation_str: str = "", proxy_source_model: Optional[Type["Model"]] = None, + used_prefixes: List[str] = None, ) -> Optional["Model"]: """ Model method to convert raw sql row from database into ormar.Model instance. @@ -45,6 +46,8 @@ class ModelRow(NewBaseModel): where rows are populated in a different way as they do not have nested models in result. + :param used_prefixes: list of already extracted prefixes + :type used_prefixes: List[str] :param proxy_source_model: source model from which querysetproxy is constructed :type proxy_source_model: Optional[Type["ModelRow"]] :param excludable: structure of fields to include and exclude @@ -68,17 +71,28 @@ class ModelRow(NewBaseModel): select_related = select_related or [] related_models = related_models or [] table_prefix = "" + used_prefixes = used_prefixes if used_prefixes is not None else [] excludable = excludable or ExcludableItems() if select_related: related_models = group_related_list(select_related) if related_field: - table_prefix = cls.Meta.alias_manager.resolve_relation_alias_after_complex( - source_model=source_model, - relation_str=current_relation_str, - relation_field=related_field, + if related_field.is_multi: + previous_model = related_field.through + else: + previous_model = related_field.owner + table_prefix = cls.Meta.alias_manager.resolve_relation_alias( + from_model=previous_model, relation_name=related_field.name ) + if not table_prefix or table_prefix in used_prefixes: + manager = cls.Meta.alias_manager + table_prefix = manager.resolve_relation_alias_after_complex( + source_model=source_model, + relation_str=current_relation_str, + relation_field=related_field, + ) + used_prefixes.append(table_prefix) item = cls._populate_nested_models_from_row( item=item, @@ -89,6 +103,7 @@ class ModelRow(NewBaseModel): source_model=source_model, # type: ignore proxy_source_model=proxy_source_model, # type: ignore table_prefix=table_prefix, + used_prefixes=used_prefixes, ) item = cls.extract_prefixed_table_columns( item=item, row=row, table_prefix=table_prefix, excludable=excludable @@ -112,6 +127,7 @@ class ModelRow(NewBaseModel): related_models: Any, excludable: ExcludableItems, table_prefix: str, + used_prefixes: List[str], current_relation_str: str = None, proxy_source_model: Type["Model"] = None, ) -> dict: @@ -170,6 +186,7 @@ class ModelRow(NewBaseModel): current_relation_str=relation_str, source_model=source_model, proxy_source_model=proxy_source_model, + used_prefixes=used_prefixes, ) item[model_cls.get_column_name_from_alias(related)] = child if field.is_multi and child: diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 8adc4f3..d0679f5 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -344,7 +344,7 @@ class QuerySet: if not isinstance(related, list): related = [related] - related = list(set(list(self._select_related) + related)) + related = sorted(list(set(list(self._select_related) + related))) return self.rebuild_self(select_related=related,) def prefetch_related(self, related: Union[List, str]) -> "QuerySet": diff --git a/ormar/relations/relation.py b/ormar/relations/relation.py index 0f9be3d..bb7abd1 100644 --- a/ormar/relations/relation.py +++ b/ormar/relations/relation.py @@ -74,7 +74,7 @@ class Relation: self._owner.__dict__[self.field_name] = None elif self.related_models is not None: self.related_models._clear() - self._owner.__dict__[self.field_name] = [] + self._owner.__dict__[self.field_name] = None @property def through(self) -> Type["Model"]: diff --git a/tests/test_excluding_fields_in_fastapi.py b/tests/test_excluding_fields_in_fastapi.py index dd4dd84..1f0950f 100644 --- a/tests/test_excluding_fields_in_fastapi.py +++ b/tests/test_excluding_fields_in_fastapi.py @@ -124,7 +124,8 @@ async def create_user(user: User): @app.post("/users2/", response_model=User) async def create_user2(user: User): - return (await user.save()).dict(exclude={"password"}) + user = await user.save() + return user.dict(exclude={"password"}) @app.post("/users3/", response_model=UserBase) diff --git a/tests/test_m2m_through_fields.py b/tests/test_m2m_through_fields.py index 3f600e0..4ed7023 100644 --- a/tests/test_m2m_through_fields.py +++ b/tests/test_m2m_through_fields.py @@ -1,4 +1,4 @@ -from typing import Any, List, Sequence, cast +from typing import Any, Sequence, cast import databases import pytest diff --git a/tests/test_more_same_table_joins.py b/tests/test_more_same_table_joins.py index 9dc086e..b991d13 100644 --- a/tests/test_more_same_table_joins.py +++ b/tests/test_more_same_table_joins.py @@ -108,3 +108,17 @@ async def test_model_multiple_instances_of_same_table_in_schema(): assert len(classes[0].dict().get("students")) == 2 assert classes[0].teachers[0].category.department.name == "Law Department" assert classes[0].students[0].category.department.name == "Math Department" + + +@pytest.mark.asyncio +async def test_load_all_multiple_instances_of_same_table_in_schema(): + async with database: + await create_data() + math_class = await SchoolClass.objects.get(name="Math") + assert math_class.name == "Math" + + await math_class.load_all(follow=True) + assert math_class.students[0].name == "Jane" + assert len(math_class.dict().get("students")) == 2 + assert math_class.teachers[0].category.department.name == "Law Department" + assert math_class.students[0].category.department.name == "Math Department" From 8682427910492db44170c9a2299f06b8eb2ace2f Mon Sep 17 00:00:00 2001 From: collerek Date: Fri, 5 Mar 2021 12:02:41 +0100 Subject: [PATCH 14/14] for now revert type changes to avoid mypy errors, add validation for through models, clean docs etc --- docs/releases.md | 5 +-- ormar/fields/many_to_many.py | 19 +++++++++++ tests/test_m2m_through_fields.py | 22 ------------ tests/test_through_relations_fail.py | 51 ++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 24 deletions(-) create mode 100644 tests/test_through_relations_fail.py diff --git a/docs/releases.md b/docs/releases.md index 2671b3d..5e61041 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -29,9 +29,10 @@ * you can include and exclude fields on through models * through models are attached only to related models (i.e. if you query from A to B -> only on B) * note that through models are explicitly loaded without relations -> relation is already populated in ManyToMany field. - * note that just like before you cannot declare the relation fields on through model, they will be populated for you by `ormar` + * note that just like before you cannot declare the relation fields on through model, they will be populated for you by `ormar`, + but now if you try to do so `ModelDefinitionError` will be thrown * check the updated ManyToMany relation docs for more information - + # Other * Updated docs and api docs * Refactors and optimisations mainly related to filters, exclusions and order bys diff --git a/ormar/fields/many_to_many.py b/ormar/fields/many_to_many.py index 0e7602e..ec8a6f1 100644 --- a/ormar/fields/many_to_many.py +++ b/ormar/fields/many_to_many.py @@ -3,6 +3,7 @@ from typing import Any, List, Optional, TYPE_CHECKING, Tuple, Type, Union, cast from pydantic.typing import ForwardRef, evaluate_forwardref import ormar # noqa: I100 +from ormar import ModelDefinitionError from ormar.fields import BaseField from ormar.fields.foreign_key import ForeignKeyField @@ -17,6 +18,21 @@ if TYPE_CHECKING: # pragma no cover REF_PREFIX = "#/components/schemas/" +def forbid_through_relations(through: Type["Model"]) -> None: + """ + Verifies if the through model does not have relations. + + :param through: through Model to be checked + :type through: Type['Model] + """ + if any(field.is_relation for field in through.Meta.model_fields.values()): + raise ModelDefinitionError( + f"Through Models cannot have explicit relations " + f"defined. Remove the relations from Model " + f"{through.get_name(lower=False)}" + ) + + def populate_m2m_params_based_on_to_model( to: Type["Model"], nullable: bool ) -> Tuple[Any, Any]: @@ -77,6 +93,8 @@ def ManyToMany( nullable = kwargs.pop("nullable", True) owner = kwargs.pop("owner", None) self_reference = kwargs.pop("self_reference", False) + if through is not None and through.__class__ != ForwardRef: + forbid_through_relations(cast(Type["Model"], through)) if to.__class__ == ForwardRef: __type__ = to if not nullable else Optional[to] @@ -189,6 +207,7 @@ class ManyToManyField(ForeignKeyField, ormar.QuerySetProtocol, ormar.RelationPro globalns, localns or None, ) + forbid_through_relations(cls.through) @classmethod def get_relation_name(cls) -> str: diff --git a/tests/test_m2m_through_fields.py b/tests/test_m2m_through_fields.py index 4ed7023..ef9847c 100644 --- a/tests/test_m2m_through_fields.py +++ b/tests/test_m2m_through_fields.py @@ -375,25 +375,3 @@ async def test_excluding_fields_on_through_model() -> Any: for category in post3.categories: assert category.postcategory.param_name is None assert category.postcategory.sort_order is None - - -# TODO: check/ modify following - -# add to fields with class lower name (V) -# forward refs update (V) -# creating while adding to relation (kwargs in add) (V) -# creating in queryset proxy (dict with through name and kwargs) (V) -# loading the data into model instance of though model (V) <- fix fields ane exclude -# accessing from instance (V) <- no both sides only nested one is relevant, fix one side -# filtering in filter (through name normally) (V) < - table prefix from normal relation, -# check if is_through needed, resolved side of relation -# ordering by in order_by (V) -# updating in query (V) -# updating from querysetproxy (V) -# including/excluding in fields? (V) -# make through optional? auto-generated for cases other fields are missing? (V) - -# modifying from instance (both sides?) (X) <= no, the loaded one doesn't have relations -# allowing to change fk fields names in through model? (X) <= separate issue - -# prevent adding relation on through field definition diff --git a/tests/test_through_relations_fail.py b/tests/test_through_relations_fail.py new file mode 100644 index 0000000..472a8a1 --- /dev/null +++ b/tests/test_through_relations_fail.py @@ -0,0 +1,51 @@ +# type: ignore + +import databases +import pytest +import sqlalchemy + +import ormar +from ormar import ModelDefinitionError +from tests.settings import DATABASE_URL + +database = databases.Database(DATABASE_URL, force_rollback=True) +metadata = sqlalchemy.MetaData() + + +def test_through_with_relation_fails(): + class BaseMeta(ormar.ModelMeta): + database = database + metadata = metadata + + class Category(ormar.Model): + class Meta(BaseMeta): + tablename = "categories" + + id = ormar.Integer(primary_key=True) + name = ormar.String(max_length=40) + + class Blog(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + title: str = ormar.String(max_length=200) + + class PostCategory(ormar.Model): + class Meta(BaseMeta): + tablename = "posts_x_categories" + + id: int = ormar.Integer(primary_key=True) + sort_order: int = ormar.Integer(nullable=True) + param_name: str = ormar.String(default="Name", max_length=200) + blog = ormar.ForeignKey(Blog) + + with pytest.raises(ModelDefinitionError): + + class Post(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + title: str = ormar.String(max_length=200) + categories = ormar.ManyToMany(Category, through=PostCategory)