diff --git a/ormar/fields/__init__.py b/ormar/fields/__init__.py index 2e4a013..c5f61d4 100644 --- a/ormar/fields/__init__.py +++ b/ormar/fields/__init__.py @@ -21,6 +21,7 @@ from ormar.fields.model_fields import ( Time, UUID, ) +from ormar.fields.through_field import Through, ThroughField __all__ = [ "Decimal", @@ -41,4 +42,6 @@ __all__ = [ "BaseField", "UniqueColumns", "ForeignKeyField", + "ThroughField", + "Through", ] diff --git a/ormar/fields/through_field.py b/ormar/fields/through_field.py new file mode 100644 index 0000000..99361c7 --- /dev/null +++ b/ormar/fields/through_field.py @@ -0,0 +1,73 @@ +import sys +from typing import Any, TYPE_CHECKING, Type, Union + +from ormar.fields.base import BaseField +from ormar.fields.foreign_key import ForeignKeyField + +if TYPE_CHECKING: # pragma no cover + from ormar import Model + from pydantic.typing import ForwardRef + + if sys.version_info < (3, 7): + ToType = Type[Model] + else: + ToType = Union[Type[Model], ForwardRef] + + +def Through( # noqa CFQ002 + to: "ToType", + *, + name: str = None, + related_name: str = None, + virtual: bool = True, + **kwargs: Any, +) -> Any: + # TODO: clean docstring + """ + Despite a name it's a function that returns constructed ForeignKeyField. + This function is actually used in model declaration (as ormar.ForeignKey(ToModel)). + + Accepts number of relation setting parameters as well as all BaseField ones. + + :param to: target related ormar Model + :type to: Model class + :param name: name of the database field - later called alias + :type name: str + :param related_name: name of reversed FK relation populated for you on to model + :type related_name: str + :param virtual: marks if relation is virtual. + It is for reversed FK and auto generated FK on through model in Many2Many relations. + :type virtual: bool + :param kwargs: all other args to be populated by BaseField + :type kwargs: Any + :return: ormar ForeignKeyField with relation to selected model + :rtype: ForeignKeyField + """ + + owner = kwargs.pop("owner", None) + namespace = dict( + __type__=to, + to=to, + through=None, + alias=name, + name=kwargs.pop("real_name", None), + related_name=related_name, + virtual=virtual, + owner=owner, + nullable=False, + unique=False, + column_type=None, + primary_key=False, + index=False, + pydantic_only=False, + default=None, + server_default=None, + ) + + return type("Through", (ThroughField, BaseField), namespace) + + +class ThroughField(ForeignKeyField): + """ + Field class used to access ManyToMany model through model. + """ diff --git a/ormar/models/__init__.py b/ormar/models/__init__.py index 88a39c6..9990c4a 100644 --- a/ormar/models/__init__.py +++ b/ormar/models/__init__.py @@ -5,6 +5,7 @@ ass well as vast number of helper functions for pydantic, sqlalchemy and relatio """ from ormar.models.newbasemodel import NewBaseModel # noqa I100 +from ormar.models.model_row import ModelRow # noqa I100 from ormar.models.model import Model # noqa I100 -__all__ = ["NewBaseModel", "Model"] +__all__ = ["NewBaseModel", "Model", "ModelRow"] diff --git a/ormar/models/helpers/relations.py b/ormar/models/helpers/relations.py index 4cf19ea..af9ee61 100644 --- a/ormar/models/helpers/relations.py +++ b/ormar/models/helpers/relations.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Type import ormar from ormar import ForeignKey, ManyToMany -from ormar.fields import ManyToManyField +from ormar.fields import ManyToManyField, Through, ThroughField from ormar.fields.foreign_key import ForeignKeyField from ormar.models.helpers.sqlalchemy import adjust_through_many_to_many_model from ormar.relations import AliasManager @@ -81,7 +81,8 @@ def expand_reverse_relationships(model: Type["Model"]) -> None: :param model: model on which relation should be checked and registered :type model: Model class """ - for model_field in model.Meta.model_fields.values(): + model_fields = list(model.Meta.model_fields.values()) + for model_field in model_fields: if ( issubclass(model_field, ForeignKeyField) and not model_field.has_unresolved_forward_refs() @@ -113,6 +114,7 @@ def register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None: self_reference_primary=model_field.self_reference_primary, ) # register foreign keys on through model + register_through_shortcut_fields(model_field=model_field) adjust_through_many_to_many_model(model_field=model_field) else: model_field.to.Meta.model_fields[related_name] = ForeignKey( @@ -125,6 +127,34 @@ def register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None: ) +def register_through_shortcut_fields(model_field: Type["ManyToManyField"]) -> None: + """ + Registers m2m relation through shortcut on both ends of the relation. + + :param model_field: relation field defined in parent model + :type model_field: ManyToManyField + """ + through_model = model_field.through + through_name = through_model.get_name(lower=True) + related_name = model_field.get_related_name() + + model_field.owner.Meta.model_fields[through_name] = Through( + through_model, + real_name=through_name, + virtual=True, + related_name=model_field.name, + owner=model_field.owner, + ) + + model_field.to.Meta.model_fields[through_name] = Through( + through_model, + real_name=through_name, + virtual=True, + related_name=related_name, + owner=model_field.to, + ) + + def register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None: """ Registers the relation (and reverse relation) in alias manager. @@ -142,7 +172,7 @@ def register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None: if field.has_unresolved_forward_refs(): return register_many_to_many_relation_on_build(field=field) - elif issubclass(field, ForeignKeyField): + elif issubclass(field, ForeignKeyField) and not issubclass(field, ThroughField): if field.has_unresolved_forward_refs(): return register_relation_on_build(field=field) diff --git a/ormar/models/metaclass.py b/ormar/models/metaclass.py index 0fc6f5d..4134fd5 100644 --- a/ormar/models/metaclass.py +++ b/ormar/models/metaclass.py @@ -46,6 +46,7 @@ if TYPE_CHECKING: # pragma no cover from ormar import Model CONFIG_KEY = "Config" +PARSED_FIELDS_KEY = "__parsed_fields__" class ModelMeta: @@ -141,83 +142,6 @@ def register_signals(new_model: Type["Model"]) -> None: # noqa: CCR001 new_model.Meta.signals = signals -class ModelMetaclass(pydantic.main.ModelMetaclass): - def __new__( # type: ignore # noqa: CCR001 - mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict - ) -> "ModelMetaclass": - """ - Metaclass used by ormar Models that performs configuration - and build of ormar Models. - - - Sets pydantic configuration. - Extract model_fields and convert them to pydantic FieldInfo, - updates class namespace. - - Extracts settings and fields from parent classes. - Fetches methods decorated with @property_field decorator - to expose them later in dict(). - - Construct parent pydantic Metaclass/ Model. - - If class has Meta class declared (so actual ormar Models) it also: - - * populate sqlalchemy columns, pkname and tables from model_fields - * register reverse relationships on related models - * registers all relations in alias manager that populates table_prefixes - * exposes alias manager on each Model - * creates QuerySet for each model and exposes it on a class - - :param name: name of current class - :type name: str - :param bases: base classes - :type bases: Tuple - :param attrs: class namespace - :type attrs: Dict - """ - attrs["Config"] = get_pydantic_base_orm_config() - attrs["__name__"] = name - attrs, model_fields = extract_annotations_and_default_vals(attrs) - for base in reversed(bases): - mod = base.__module__ - if mod.startswith("ormar.models.") or mod.startswith("pydantic."): - continue - attrs, model_fields = extract_from_parents_definition( - base_class=base, curr_class=mcs, attrs=attrs, model_fields=model_fields - ) - new_model = super().__new__( # type: ignore - mcs, name, bases, attrs - ) - - add_cached_properties(new_model) - - if hasattr(new_model, "Meta"): - populate_default_options_values(new_model, model_fields) - check_required_meta_parameters(new_model) - add_property_fields(new_model, attrs) - register_signals(new_model=new_model) - populate_choices_validators(new_model) - - if not new_model.Meta.abstract: - new_model = populate_meta_tablename_columns_and_pk(name, new_model) - populate_meta_sqlalchemy_table_if_required(new_model.Meta) - expand_reverse_relationships(new_model) - for field in new_model.Meta.model_fields.values(): - register_relation_in_alias_manager(field=field) - - if new_model.Meta.pkname not in attrs["__annotations__"]: - field_name = new_model.Meta.pkname - attrs["__annotations__"][field_name] = Optional[int] # type: ignore - attrs[field_name] = None - new_model.__fields__[field_name] = get_pydantic_field( - field_name=field_name, model=new_model - ) - new_model.Meta.alias_manager = alias_manager - new_model.objects = QuerySet(new_model) - - return new_model - - def verify_constraint_names( base_class: "Model", model_fields: Dict, parent_value: List ) -> None: @@ -539,4 +463,78 @@ def update_attrs_and_fields( return updated_model_fields -PARSED_FIELDS_KEY = "__parsed_fields__" +class ModelMetaclass(pydantic.main.ModelMetaclass): + def __new__( # type: ignore # noqa: CCR001 + mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict + ) -> "ModelMetaclass": + """ + Metaclass used by ormar Models that performs configuration + and build of ormar Models. + + + Sets pydantic configuration. + Extract model_fields and convert them to pydantic FieldInfo, + updates class namespace. + + Extracts settings and fields from parent classes. + Fetches methods decorated with @property_field decorator + to expose them later in dict(). + + Construct parent pydantic Metaclass/ Model. + + If class has Meta class declared (so actual ormar Models) it also: + + * populate sqlalchemy columns, pkname and tables from model_fields + * register reverse relationships on related models + * registers all relations in alias manager that populates table_prefixes + * exposes alias manager on each Model + * creates QuerySet for each model and exposes it on a class + + :param name: name of current class + :type name: str + :param bases: base classes + :type bases: Tuple + :param attrs: class namespace + :type attrs: Dict + """ + attrs["Config"] = get_pydantic_base_orm_config() + attrs["__name__"] = name + attrs, model_fields = extract_annotations_and_default_vals(attrs) + for base in reversed(bases): + mod = base.__module__ + if mod.startswith("ormar.models.") or mod.startswith("pydantic."): + continue + attrs, model_fields = extract_from_parents_definition( + base_class=base, curr_class=mcs, attrs=attrs, model_fields=model_fields + ) + new_model = super().__new__( # type: ignore + mcs, name, bases, attrs + ) + + add_cached_properties(new_model) + + if hasattr(new_model, "Meta"): + populate_default_options_values(new_model, model_fields) + check_required_meta_parameters(new_model) + add_property_fields(new_model, attrs) + register_signals(new_model=new_model) + populate_choices_validators(new_model) + + if not new_model.Meta.abstract: + new_model = populate_meta_tablename_columns_and_pk(name, new_model) + populate_meta_sqlalchemy_table_if_required(new_model.Meta) + expand_reverse_relationships(new_model) + for field in new_model.Meta.model_fields.values(): + register_relation_in_alias_manager(field=field) + + if new_model.Meta.pkname not in attrs["__annotations__"]: + field_name = new_model.Meta.pkname + attrs["__annotations__"][field_name] = Optional[int] # type: ignore + attrs[field_name] = None + new_model.__fields__[field_name] = get_pydantic_field( + field_name=field_name, model=new_model + ) + new_model.Meta.alias_manager = alias_manager + new_model.objects = QuerySet(new_model) + + return new_model diff --git a/ormar/models/mixins/excludable_mixin.py b/ormar/models/mixins/excludable_mixin.py index 961e284..e474bd4 100644 --- a/ormar/models/mixins/excludable_mixin.py +++ b/ormar/models/mixins/excludable_mixin.py @@ -31,6 +31,7 @@ class ExcludableMixin(RelationMixin): if TYPE_CHECKING: # pragma: no cover from ormar import Model + from ormar.models import ModelRow @staticmethod def get_child( @@ -157,7 +158,7 @@ class ExcludableMixin(RelationMixin): @classmethod def own_table_columns( cls, - model: Type["Model"], + model: Union[Type["Model"], Type["ModelRow"]], fields: Optional[Union[Set, Dict]], exclude_fields: Optional[Union[Set, Dict]], use_alias: bool = False, diff --git a/ormar/models/mixins/relation_mixin.py b/ormar/models/mixins/relation_mixin.py index 48688d6..435fcc8 100644 --- a/ormar/models/mixins/relation_mixin.py +++ b/ormar/models/mixins/relation_mixin.py @@ -1,6 +1,8 @@ import inspect from typing import List, Optional, Set, TYPE_CHECKING +from ormar import ManyToManyField +from ormar.fields import ThroughField from ormar.fields.foreign_key import ForeignKeyField @@ -43,27 +45,46 @@ class RelationMixin: return cls._related_fields related_fields = [] - for name in cls.extract_related_names(): + for name in cls.extract_related_names().union(cls.extract_through_names()): related_fields.append(cls.Meta.model_fields[name]) cls._related_fields = related_fields return related_fields + @classmethod + def extract_through_names(cls) -> Set: + """ + Extracts related fields through names which are shortcuts to through models. + + :return: set of related through fields names + :rtype: Set + """ + related_fields = set() + for name in cls.extract_related_names(): + field = cls.Meta.model_fields[name] + if issubclass(field, ManyToManyField): + related_fields.add(field.through.get_name(lower=True)) + return related_fields + @classmethod def extract_related_names(cls) -> Set: """ Returns List of fields names for all relations declared on a model. List is cached in cls._related_names for quicker access. - :return: list of related fields names - :rtype: List + :return: set of related fields names + :rtype: Set """ if isinstance(cls._related_names, Set): return cls._related_names related_names = set() for name, field in cls.Meta.model_fields.items(): - if inspect.isclass(field) and issubclass(field, ForeignKeyField): + if ( + inspect.isclass(field) + and issubclass(field, ForeignKeyField) + and not issubclass(field, ThroughField) + ): related_names.add(name) cls._related_names = related_names diff --git a/ormar/models/model.py b/ormar/models/model.py index 3c66a0a..9286da9 100644 --- a/ormar/models/model.py +++ b/ormar/models/model.py @@ -1,24 +1,17 @@ from typing import ( Any, - Dict, - List, - Optional, Set, TYPE_CHECKING, Tuple, - Type, TypeVar, - Union, ) -import sqlalchemy - import ormar.queryset # noqa I100 from ormar.exceptions import ModelPersistenceError, NoMatch from ormar.fields.many_to_many import ManyToManyField from ormar.models import NewBaseModel # noqa I100 -from ormar.models.helpers.models import group_related_list from ormar.models.metaclass import ModelMeta +from ormar.models.model_row import ModelRow if TYPE_CHECKING: # pragma nocover from ormar import QuerySet @@ -26,7 +19,7 @@ if TYPE_CHECKING: # pragma nocover T = TypeVar("T", bound="Model") -class Model(NewBaseModel): +class Model(ModelRow): __abstract__ = False if TYPE_CHECKING: # pragma nocover Meta: ModelMeta @@ -36,247 +29,6 @@ class Model(NewBaseModel): _repr = {k: getattr(self, k) for k, v in self.Meta.model_fields.items()} return f"{self.__class__.__name__}({str(_repr)})" - @classmethod - def from_row( # noqa CCR001 - cls: Type[T], - row: sqlalchemy.engine.ResultProxy, - select_related: List = None, - related_models: Any = None, - previous_model: Type[T] = None, - source_model: Type[T] = None, - related_name: str = None, - fields: Optional[Union[Dict, Set]] = None, - exclude_fields: Optional[Union[Dict, Set]] = None, - current_relation_str: str = None, - ) -> Optional[T]: - """ - Model method to convert raw sql row from database into ormar.Model instance. - Traverses nested models if they were specified in select_related for query. - - Called recurrently and returns model instance if it's present in the row. - Note that it's processing one row at a time, so if there are duplicates of - parent row that needs to be joined/combined - (like parent row in sql join with 2+ child rows) - instances populated in this method are later combined in the QuerySet. - Other method working directly on raw database results is in prefetch_query, - where rows are populated in a different way as they do not have - nested models in result. - - :param current_relation_str: name of the relation field - :type current_relation_str: str - :param source_model: model on which relation was defined - :type source_model: Type[Model] - :param row: raw result row from the database - :type row: sqlalchemy.engine.result.ResultProxy - :param select_related: list of names of related models fetched from database - :type select_related: List - :param related_models: list or dict of related models - :type related_models: Union[List, Dict] - :param previous_model: internal param for nested models to specify table_prefix - :type previous_model: Model class - :param related_name: internal parameter - name of current nested model - :type related_name: str - :param fields: fields and related model fields to include - if provided only those are included - :type fields: Optional[Union[Dict, Set]] - :param exclude_fields: fields and related model fields to exclude - excludes the fields even if they are provided in fields - :type exclude_fields: Optional[Union[Dict, Set]] - :return: returns model if model is populated from database - :rtype: Optional[Model] - """ - item: Dict[str, Any] = {} - select_related = select_related or [] - related_models = related_models or [] - table_prefix = "" - - if select_related: - source_model = cls - related_models = group_related_list(select_related) - - rel_name2 = related_name - - if ( - previous_model - and related_name - and issubclass( - previous_model.Meta.model_fields[related_name], ManyToManyField - ) - ): - through_field = previous_model.Meta.model_fields[related_name] - if ( - through_field.self_reference - and related_name == through_field.self_reference_primary - ): - rel_name2 = through_field.default_source_field_name() # type: ignore - else: - rel_name2 = through_field.default_target_field_name() # type: ignore - previous_model = through_field.through # type: ignore - - if previous_model and rel_name2: - if current_relation_str and "__" in current_relation_str and source_model: - table_prefix = cls.Meta.alias_manager.resolve_relation_alias( - from_model=source_model, relation_name=current_relation_str - ) - if not table_prefix: - table_prefix = cls.Meta.alias_manager.resolve_relation_alias( - from_model=previous_model, relation_name=rel_name2 - ) - - item = cls.populate_nested_models_from_row( - item=item, - row=row, - related_models=related_models, - fields=fields, - exclude_fields=exclude_fields, - current_relation_str=current_relation_str, - source_model=source_model, - ) - item = cls.extract_prefixed_table_columns( - item=item, - row=row, - table_prefix=table_prefix, - fields=fields, - exclude_fields=exclude_fields, - ) - - instance: Optional[T] = None - if item.get(cls.Meta.pkname, None) is not None: - item["__excluded__"] = cls.get_names_to_exclude( - fields=fields, exclude_fields=exclude_fields - ) - instance = cls(**item) - instance.set_save_status(True) - return instance - - @classmethod - def populate_nested_models_from_row( # noqa: CFQ002 - cls, - item: dict, - row: sqlalchemy.engine.ResultProxy, - related_models: Any, - fields: Optional[Union[Dict, Set]] = None, - exclude_fields: Optional[Union[Dict, Set]] = None, - current_relation_str: str = None, - source_model: Type[T] = None, - ) -> dict: - """ - Traverses structure of related models and populates the nested models - from the database row. - Related models can be a list if only directly related models are to be - populated, converted to dict if related models also have their own related - models to be populated. - - Recurrently calls from_row method on nested instances and create nested - instances. In the end those instances are added to the final model dictionary. - - :param source_model: source model from which relation started - :type source_model: Type[Model] - :param current_relation_str: joined related parts into one string - :type current_relation_str: str - :param item: dictionary of already populated nested models, otherwise empty dict - :type item: Dict - :param row: raw result row from the database - :type row: sqlalchemy.engine.result.ResultProxy - :param related_models: list or dict of related models - :type related_models: Union[Dict, List] - :param fields: fields and related model fields to include - - if provided only those are included - :type fields: Optional[Union[Dict, Set]] - :param exclude_fields: fields and related model fields to exclude - excludes the fields even if they are provided in fields - :type exclude_fields: Optional[Union[Dict, Set]] - :return: dictionary with keys corresponding to model fields names - and values are database values - :rtype: Dict - """ - - for related in related_models: - relation_str = ( - "__".join([current_relation_str, related]) - if current_relation_str - else related - ) - fields = cls.get_included(fields, related) - exclude_fields = cls.get_excluded(exclude_fields, related) - model_cls = cls.Meta.model_fields[related].to - - remainder = None - if isinstance(related_models, dict) and related_models[related]: - remainder = related_models[related] - child = model_cls.from_row( - row, - related_models=remainder, - previous_model=cls, - related_name=related, - fields=fields, - exclude_fields=exclude_fields, - current_relation_str=relation_str, - source_model=source_model, - ) - item[model_cls.get_column_name_from_alias(related)] = child - - return item - - @classmethod - def extract_prefixed_table_columns( # noqa CCR001 - cls, - item: dict, - row: sqlalchemy.engine.result.ResultProxy, - table_prefix: str, - fields: Optional[Union[Dict, Set]] = None, - exclude_fields: Optional[Union[Dict, Set]] = None, - ) -> dict: - """ - Extracts own fields from raw sql result, using a given prefix. - Prefix changes depending on the table's position in a join. - - If the table is a main table, there is no prefix. - All joined tables have prefixes to allow duplicate column names, - as well as duplicated joins to the same table from multiple different tables. - - Extracted fields populates the related dict later used to construct a Model. - - Used in Model.from_row and PrefetchQuery._populate_rows methods. - - :param item: dictionary of already populated nested models, otherwise empty dict - :type item: Dict - :param row: raw result row from the database - :type row: sqlalchemy.engine.result.ResultProxy - :param table_prefix: prefix of the table from AliasManager - each pair of tables have own prefix (two of them depending on direction) - - used in joins to allow multiple joins to the same table. - :type table_prefix: str - :param fields: fields and related model fields to include - - if provided only those are included - :type fields: Optional[Union[Dict, Set]] - :param exclude_fields: fields and related model fields to exclude - excludes the fields even if they are provided in fields - :type exclude_fields: Optional[Union[Dict, Set]] - :return: dictionary with keys corresponding to model fields names - and values are database values - :rtype: Dict - """ - # databases does not keep aliases in Record for postgres, change to raw row - source = row._row if cls.db_backend_name() == "postgresql" else row - - selected_columns = cls.own_table_columns( - model=cls, - fields=fields or {}, - exclude_fields=exclude_fields or {}, - use_alias=False, - ) - - for column in cls.Meta.table.columns: - alias = cls.get_column_name_from_alias(column.name) - if alias not in item and alias in selected_columns: - prefixed_name = ( - f'{table_prefix + "_" if table_prefix else ""}{column.name}' - ) - item[alias] = source[prefixed_name] - - return item - async def upsert(self: T, **kwargs: Any) -> T: """ Performs either a save or an update depending on the presence of the pk. diff --git a/ormar/models/model_row.py b/ormar/models/model_row.py new file mode 100644 index 0000000..9d33a64 --- /dev/null +++ b/ormar/models/model_row.py @@ -0,0 +1,303 @@ +from typing import ( + Any, + Dict, + List, + Optional, + Set, + Type, + TypeVar, + Union, +) + +import sqlalchemy + +from ormar import ManyToManyField # noqa: I202 +from ormar.models import NewBaseModel +from ormar.models.helpers.models import group_related_list + +T = TypeVar("T", bound="ModelRow") + + +class ModelRow(NewBaseModel): + @classmethod + def from_row( # noqa CCR001 + cls: Type[T], + row: sqlalchemy.engine.ResultProxy, + select_related: List = None, + related_models: Any = None, + previous_model: Type[T] = None, + source_model: Type[T] = None, + related_name: str = None, + fields: Optional[Union[Dict, Set]] = None, + exclude_fields: Optional[Union[Dict, Set]] = None, + current_relation_str: str = None, + ) -> Optional[T]: + """ + Model method to convert raw sql row from database into ormar.Model instance. + Traverses nested models if they were specified in select_related for query. + + Called recurrently and returns model instance if it's present in the row. + Note that it's processing one row at a time, so if there are duplicates of + parent row that needs to be joined/combined + (like parent row in sql join with 2+ child rows) + instances populated in this method are later combined in the QuerySet. + Other method working directly on raw database results is in prefetch_query, + where rows are populated in a different way as they do not have + nested models in result. + + :param current_relation_str: name of the relation field + :type current_relation_str: str + :param source_model: model on which relation was defined + :type source_model: Type[Model] + :param row: raw result row from the database + :type row: sqlalchemy.engine.result.ResultProxy + :param select_related: list of names of related models fetched from database + :type select_related: List + :param related_models: list or dict of related models + :type related_models: Union[List, Dict] + :param previous_model: internal param for nested models to specify table_prefix + :type previous_model: Model class + :param related_name: internal parameter - name of current nested model + :type related_name: str + :param fields: fields and related model fields to include + if provided only those are included + :type fields: Optional[Union[Dict, Set]] + :param exclude_fields: fields and related model fields to exclude + excludes the fields even if they are provided in fields + :type exclude_fields: Optional[Union[Dict, Set]] + :return: returns model if model is populated from database + :rtype: Optional[Model] + """ + item: Dict[str, Any] = {} + select_related = select_related or [] + related_models = related_models or [] + table_prefix = "" + + if select_related: + source_model = cls + related_models = group_related_list(select_related) + + rel_name2 = related_name + + # TODO: refactor this into field classes? + if ( + previous_model + and related_name + and issubclass( + previous_model.Meta.model_fields[related_name], ManyToManyField + ) + ): + through_field = previous_model.Meta.model_fields[related_name] + if ( + through_field.self_reference + and related_name == through_field.self_reference_primary + ): + rel_name2 = through_field.default_source_field_name() # type: ignore + else: + rel_name2 = through_field.default_target_field_name() # type: ignore + previous_model = through_field.through # type: ignore + + if previous_model and rel_name2: + if current_relation_str and "__" in current_relation_str and source_model: + table_prefix = cls.Meta.alias_manager.resolve_relation_alias( + from_model=source_model, relation_name=current_relation_str + ) + if not table_prefix: + table_prefix = cls.Meta.alias_manager.resolve_relation_alias( + from_model=previous_model, relation_name=rel_name2 + ) + + item = cls.populate_nested_models_from_row( + item=item, + row=row, + related_models=related_models, + fields=fields, + exclude_fields=exclude_fields, + current_relation_str=current_relation_str, + source_model=source_model, + ) + item = cls.extract_prefixed_table_columns( + item=item, + row=row, + table_prefix=table_prefix, + fields=fields, + exclude_fields=exclude_fields, + ) + + instance: Optional[T] = None + if item.get(cls.Meta.pkname, None) is not None: + item["__excluded__"] = cls.get_names_to_exclude( + fields=fields, exclude_fields=exclude_fields + ) + instance = cls(**item) + instance.set_save_status(True) + return instance + + @classmethod + def populate_nested_models_from_row( # noqa: CFQ002 + cls, + item: dict, + row: sqlalchemy.engine.ResultProxy, + related_models: Any, + fields: Optional[Union[Dict, Set]] = None, + exclude_fields: Optional[Union[Dict, Set]] = None, + current_relation_str: str = None, + source_model: Type[T] = None, + ) -> dict: + """ + Traverses structure of related models and populates the nested models + from the database row. + Related models can be a list if only directly related models are to be + populated, converted to dict if related models also have their own related + models to be populated. + + Recurrently calls from_row method on nested instances and create nested + instances. In the end those instances are added to the final model dictionary. + + :param source_model: source model from which relation started + :type source_model: Type[Model] + :param current_relation_str: joined related parts into one string + :type current_relation_str: str + :param item: dictionary of already populated nested models, otherwise empty dict + :type item: Dict + :param row: raw result row from the database + :type row: sqlalchemy.engine.result.ResultProxy + :param related_models: list or dict of related models + :type related_models: Union[Dict, List] + :param fields: fields and related model fields to include - + if provided only those are included + :type fields: Optional[Union[Dict, Set]] + :param exclude_fields: fields and related model fields to exclude + excludes the fields even if they are provided in fields + :type exclude_fields: Optional[Union[Dict, Set]] + :return: dictionary with keys corresponding to model fields names + and values are database values + :rtype: Dict + """ + + for related in related_models: + relation_str = ( + "__".join([current_relation_str, related]) + if current_relation_str + else related + ) + field = cls.Meta.model_fields[related] + fields = cls.get_included(fields, related) + exclude_fields = cls.get_excluded(exclude_fields, related) + model_cls = field.to + + remainder = None + if isinstance(related_models, dict) and related_models[related]: + remainder = related_models[related] + child = model_cls.from_row( + row, + related_models=remainder, + previous_model=cls, + related_name=related, + fields=fields, + exclude_fields=exclude_fields, + current_relation_str=relation_str, + source_model=source_model, + ) + item[model_cls.get_column_name_from_alias(related)] = child + if issubclass(field, ManyToManyField) and child: + # TODO: way to figure out which side should be populated? + through_name = cls.Meta.model_fields[related].through.get_name() + # for now it's nested dict, should be instance? + through_child = cls.populate_through_instance( + row=row, + related=related, + through_name=through_name, + fields=fields, + exclude_fields=exclude_fields, + ) + item[through_name] = through_child + setattr(child, through_name, through_child) + child.set_save_status(True) + + return item + + @classmethod + def populate_through_instance( + cls, + row: sqlalchemy.engine.ResultProxy, + through_name: str, + related: str, + fields: Optional[Union[Dict, Set]] = None, + exclude_fields: Optional[Union[Dict, Set]] = None, + ) -> Dict: + # TODO: fix excludes and includes + fields = cls.get_included(fields, through_name) + # exclude_fields = cls.get_excluded(exclude_fields, through_name) + model_cls = cls.Meta.model_fields[through_name].to + exclude_fields = model_cls.extract_related_names() + table_prefix = cls.Meta.alias_manager.resolve_relation_alias( + from_model=cls, relation_name=related + ) + child = model_cls.extract_prefixed_table_columns( + item={}, + row=row, + table_prefix=table_prefix, + fields=fields, + exclude_fields=exclude_fields, + ) + return child + + @classmethod + def extract_prefixed_table_columns( # noqa CCR001 + cls, + item: dict, + row: sqlalchemy.engine.result.ResultProxy, + table_prefix: str, + fields: Optional[Union[Dict, Set]] = None, + exclude_fields: Optional[Union[Dict, Set]] = None, + ) -> dict: + """ + Extracts own fields from raw sql result, using a given prefix. + Prefix changes depending on the table's position in a join. + + If the table is a main table, there is no prefix. + All joined tables have prefixes to allow duplicate column names, + as well as duplicated joins to the same table from multiple different tables. + + Extracted fields populates the related dict later used to construct a Model. + + Used in Model.from_row and PrefetchQuery._populate_rows methods. + + :param item: dictionary of already populated nested models, otherwise empty dict + :type item: Dict + :param row: raw result row from the database + :type row: sqlalchemy.engine.result.ResultProxy + :param table_prefix: prefix of the table from AliasManager + each pair of tables have own prefix (two of them depending on direction) - + used in joins to allow multiple joins to the same table. + :type table_prefix: str + :param fields: fields and related model fields to include - + if provided only those are included + :type fields: Optional[Union[Dict, Set]] + :param exclude_fields: fields and related model fields to exclude + excludes the fields even if they are provided in fields + :type exclude_fields: Optional[Union[Dict, Set]] + :return: dictionary with keys corresponding to model fields names + and values are database values + :rtype: Dict + """ + # databases does not keep aliases in Record for postgres, change to raw row + source = row._row if cls.db_backend_name() == "postgresql" else row + + selected_columns = cls.own_table_columns( + model=cls, + fields=fields or {}, + exclude_fields=exclude_fields or {}, + use_alias=False, + ) + + for column in cls.Meta.table.columns: + alias = cls.get_column_name_from_alias(column.name) + if alias not in item and alias in selected_columns: + prefixed_name = ( + f'{table_prefix + "_" if table_prefix else ""}{column.name}' + ) + item[alias] = source[prefixed_name] + + return item diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index 3aa359e..6975753 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -172,7 +172,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass object.__setattr__(self, "__fields_set__", fields_set) # register the columns models after initialization - for related in self.extract_related_names(): + for related in self.extract_related_names().union(self.extract_through_names()): self.Meta.model_fields[related].expand_relationship( new_kwargs.get(related), self, to_register=True, ) @@ -267,6 +267,10 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass return object.__getattribute__( self, "_extract_related_model_instead_of_field" )(item) + if item in object.__getattribute__(self, "extract_through_names")(): + return object.__getattribute__( + self, "_extract_related_model_instead_of_field" + )(item) if item in object.__getattribute__(self, "Meta").property_fields: value = object.__getattribute__(self, item) return value() if callable(value) else value diff --git a/ormar/models/quick_access_views.py b/ormar/models/quick_access_views.py index c09b672..c960898 100644 --- a/ormar/models/quick_access_views.py +++ b/ormar/models/quick_access_views.py @@ -34,10 +34,12 @@ quick_access_set = { "_skip_ellipsis", "_update_and_follow", "_update_excluded_with_related_not_required", + "_verify_model_can_be_initialized", "copy", "delete", "dict", "extract_related_names", + "extract_through_names", "update_from_dict", "get_column_alias", "get_column_name_from_alias", diff --git a/ormar/queryset/join.py b/ormar/queryset/join.py index 0b44078..9ce306d 100644 --- a/ormar/queryset/join.py +++ b/ormar/queryset/join.py @@ -291,6 +291,8 @@ class SqlJoin: self.get_order_bys( to_table=to_table, pkname_alias=pkname_alias, ) + else: + self.select_through_model_fields() self_related_fields = self.next_model.own_table_columns( model=self.next_model, @@ -305,6 +307,24 @@ class SqlJoin: ) self.used_aliases.append(self.next_alias) + def select_through_model_fields(self) -> None: + # TODO: add docstring + next_alias = self.alias_manager.resolve_relation_alias( + from_model=self.target_field.owner, relation_name=self.relation_name + ) + # TODO: fix fields and exclusions + self_related_fields = self.target_field.through.own_table_columns( + model=self.target_field.through, + fields=None, + exclude_fields=self.target_field.through.extract_related_names(), + use_alias=True, + ) + self.columns.extend( + self.alias_manager.prefixed_columns( + next_alias, self.target_field.through.Meta.table, self_related_fields + ) + ) + def _replace_many_to_many_order_by_columns(self, part: str, new_part: str) -> None: """ Substitutes the name of the relation with actual model name in m2m order bys. diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 6adec49..77ec421 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -197,7 +197,7 @@ class QuerySet: limit_raw_sql=self.limit_sql_raw, ) exp = qry.build_select_expression() - # print("\n", exp.compile(compile_kwargs={"literal_binds": True})) + print("\n", exp.compile(compile_kwargs={"literal_binds": True})) return exp def filter(self, _exclude: bool = False, **kwargs: Any) -> "QuerySet": # noqa: A003 diff --git a/ormar/relations/alias_manager.py b/ormar/relations/alias_manager.py index cd3dc8b..c99dfdb 100644 --- a/ormar/relations/alias_manager.py +++ b/ormar/relations/alias_manager.py @@ -1,13 +1,14 @@ import string import uuid from random import choices -from typing import Any, Dict, List, TYPE_CHECKING, Type +from typing import Any, Dict, List, TYPE_CHECKING, Type, Union import sqlalchemy from sqlalchemy import text if TYPE_CHECKING: # pragma: no cover from ormar import Model + from ormar.models import ModelRow def get_table_alias() -> str: @@ -133,7 +134,7 @@ class AliasManager: return alias def resolve_relation_alias( - self, from_model: Type["Model"], relation_name: str + self, from_model: Union[Type["Model"], Type["ModelRow"]], relation_name: str ) -> str: """ Given model and relation name returns the alias for this relation. diff --git a/ormar/relations/querysetproxy.py b/ormar/relations/querysetproxy.py index 360e863..c23dcb7 100644 --- a/ormar/relations/querysetproxy.py +++ b/ormar/relations/querysetproxy.py @@ -44,6 +44,11 @@ class QuerysetProxy(ormar.QuerySetProtocol): ].get_related_name() self.related_field = self.relation.to.Meta.model_fields[self.related_field_name] self.owner_pk_value = self._owner.pk + self.through_model_name = ( + self.related_field.through.get_name() + if self.type_ == ormar.RelationType.MULTIPLE + else None + ) @property def queryset(self) -> "QuerySet": @@ -99,17 +104,20 @@ class QuerysetProxy(ormar.QuerySetProtocol): for item in self.relation.related_models[:]: self.relation.remove(item) - async def create_through_instance(self, child: "T") -> None: + async def create_through_instance(self, child: "T", **kwargs: Any) -> None: """ Crete a through model instance in the database for m2m relations. + :param kwargs: dict of additional keyword arguments for through instance + :type kwargs: Any :param child: child model instance :type child: Model """ model_cls = self.relation.through owner_column = self.related_field.default_target_field_name() # type: ignore child_column = self.related_field.default_source_field_name() # type: ignore - kwargs = {owner_column: self._owner.pk, child_column: child.pk} + rel_kwargs = {owner_column: self._owner.pk, child_column: child.pk} + final_kwargs = {**rel_kwargs, **kwargs} if child.pk is None: raise ModelPersistenceError( f"You cannot save {child.get_name()} " @@ -117,7 +125,7 @@ class QuerysetProxy(ormar.QuerySetProtocol): f"Save the child model first." ) expr = model_cls.Meta.table.insert() - expr = expr.values(**kwargs) + expr = expr.values(**final_kwargs) # print("\n", expr.compile(compile_kwargs={"literal_binds": True})) await model_cls.Meta.database.execute(expr) @@ -270,12 +278,13 @@ class QuerysetProxy(ormar.QuerySetProtocol): :return: created model :rtype: Model """ + through_kwargs = kwargs.pop(self.through_model_name, {}) if self.type_ == ormar.RelationType.REVERSE: kwargs[self.related_field.name] = self._owner created = await self.queryset.create(**kwargs) self._register_related(created) if self.type_ == ormar.RelationType.MULTIPLE: - await self.create_through_instance(created) + await self.create_through_instance(created, **through_kwargs) return created async def get_or_create(self, **kwargs: Any) -> "Model": diff --git a/ormar/relations/relation.py b/ormar/relations/relation.py index cf191e0..0ae2f59 100644 --- a/ormar/relations/relation.py +++ b/ormar/relations/relation.py @@ -26,6 +26,7 @@ class RelationType(Enum): PRIMARY = 1 REVERSE = 2 MULTIPLE = 3 + THROUGH = 4 class Relation: @@ -128,7 +129,7 @@ class Relation: :type child: Model """ relation_name = self.field_name - if self._type == RelationType.PRIMARY: + if self._type in (RelationType.PRIMARY, RelationType.THROUGH): self.related_models = child self._owner.__dict__[relation_name] = child else: diff --git a/ormar/relations/relation_manager.py b/ormar/relations/relation_manager.py index 511dd7b..addfcf1 100644 --- a/ormar/relations/relation_manager.py +++ b/ormar/relations/relation_manager.py @@ -1,7 +1,7 @@ from typing import Dict, List, Optional, Sequence, TYPE_CHECKING, Type, TypeVar, Union from weakref import proxy -from ormar.fields import BaseField +from ormar.fields import BaseField, ThroughField from ormar.fields.foreign_key import ForeignKeyField from ormar.fields.many_to_many import ManyToManyField from ormar.relations.relation import Relation, RelationType @@ -42,6 +42,8 @@ class RelationsManager: """ if issubclass(field, ManyToManyField): return RelationType.MULTIPLE + if issubclass(field, ThroughField): + return RelationType.THROUGH return RelationType.PRIMARY if not field.virtual else RelationType.REVERSE def _add_relation(self, field: Type[BaseField]) -> None: diff --git a/ormar/relations/relation_proxy.py b/ormar/relations/relation_proxy.py index 518fc71..87bb05f 100644 --- a/ormar/relations/relation_proxy.py +++ b/ormar/relations/relation_proxy.py @@ -163,19 +163,21 @@ class RelationProxy(list): else: await item.delete() - async def add(self, item: "Model") -> None: + async def add(self, item: "Model", **kwargs: Any) -> None: """ Adds child model to relation. For ManyToMany relations through instance is automatically created. + :param kwargs: dict of additional keyword arguments for through instance + :type kwargs: Any :param item: child to add to relation :type item: Model """ relation_name = self.related_field_name self._check_if_model_saved() if self.type_ == ormar.RelationType.MULTIPLE: - await self.queryset_proxy.create_through_instance(item) + await self.queryset_proxy.create_through_instance(item, **kwargs) setattr(item, relation_name, self._owner) else: setattr(item, relation_name, self._owner) diff --git a/tests/test_m2m_through_fields.py b/tests/test_m2m_through_fields.py index 96c7f89..229eac4 100644 --- a/tests/test_m2m_through_fields.py +++ b/tests/test_m2m_through_fields.py @@ -1,6 +1,7 @@ import databases import pytest import sqlalchemy +from pydantic.typing import ForwardRef import ormar from tests.settings import DATABASE_URL @@ -39,28 +40,107 @@ class Post(ormar.Model): categories = ormar.ManyToMany(Category, through=PostCategory) -# -# @pytest.fixture(autouse=True, scope="module") -# async def create_test_database(): -# engine = sqlalchemy.create_engine(DATABASE_URL) -# metadata.create_all(engine) -# yield -# metadata.drop_all(engine) -# -# -# @pytest.mark.asyncio -# async def test_setting_fields_on_through_model(): -# async with database: -# # TODO: check/ modify following -# # loading the data into model instance of though model? -# # <- attach to other side? both sides? access by through, or add to fields? -# # creating while adding to relation (kwargs in add?) -# # creating in query (dividing kwargs between final and through) -# # updating in query -# # sorting in filter (special __through__ notation?) -# # ordering by in order_by -# # accessing from instance (both sides?) -# # modifying from instance (both sides?) -# # including/excluding in fields? -# # allowing to change fk fields names in through model? -# pass +@pytest.fixture(autouse=True, scope="module") +def create_test_database(): + engine = sqlalchemy.create_engine(DATABASE_URL) + metadata.drop_all(engine) + metadata.create_all(engine) + yield + metadata.drop_all(engine) + + +class PostCategory2(ormar.Model): + class Meta(BaseMeta): + tablename = "posts_x_categories2" + + id: int = ormar.Integer(primary_key=True) + sort_order: int = ormar.Integer(nullable=True) + + +@pytest.mark.asyncio +async def test_forward_ref_is_updated(): + async with database: + + class Post2(ormar.Model): + class Meta(BaseMeta): + pass + + id: int = ormar.Integer(primary_key=True) + title: str = ormar.String(max_length=200) + categories = ormar.ManyToMany(Category, through=ForwardRef("PostCategory2")) + + assert Post2.Meta.requires_ref_update + Post2.update_forward_refs() + + assert Post2.Meta.model_fields["postcategory2"].to == PostCategory2 + + +@pytest.mark.asyncio +async def test_setting_fields_on_through_model(): + async with database: + post = await Post(title="Test post").save() + category = await Category(name="Test category").save() + await post.categories.add(category) + + assert hasattr(post.categories[0], "postcategory") + assert post.categories[0].postcategory is None + + +@pytest.mark.asyncio +async def test_setting_additional_fields_on_through_model_in_add(): + async with database: + post = await Post(title="Test post").save() + category = await Category(name="Test category").save() + await post.categories.add(category, sort_order=1) + postcat = await PostCategory.objects.get() + assert postcat.sort_order == 1 + + +@pytest.mark.asyncio +async def test_setting_additional_fields_on_through_model_in_create(): + async with database: + post = await Post(title="Test post").save() + await post.categories.create( + name="Test category2", postcategory={"sort_order": 2} + ) + postcat = await PostCategory.objects.get() + assert postcat.sort_order == 2 + + +@pytest.mark.asyncio +async def test_getting_additional_fields_from_queryset(): + async with database: + post = await Post(title="Test post").save() + await post.categories.create( + name="Test category1", postcategory={"sort_order": 1} + ) + await post.categories.create( + name="Test category2", postcategory={"sort_order": 2} + ) + + await post.categories.all() + assert post.categories[0].postcategory.sort_order == 1 + assert post.categories[1].postcategory.sort_order == 2 + + post = await Post.objects.select_related("categories").get( + categories__name="Test category2" + ) + assert post.categories[0].postcategory.sort_order == 2 + + +# TODO: check/ modify following + +# add to fields with class lower name (V) +# forward refs update (V) +# creating while adding to relation (kwargs in add) (V) +# creating in queryset proxy (dict with through name and kwargs) (V) +# loading the data into model instance of though model (V) <- fix fields ane exclude +# accessing from instance (V) <- no both sides only nested one is relevant, fix one side + +# updating in query +# sorting in filter (special __through__ notation?) +# ordering by in order_by +# modifying from instance (both sides?) +# including/excluding in fields? +# allowing to change fk fields names in through model? +# make through optional? auto-generated for cases other fields are missing?