wip - through models fields -> attached in queries, accesible from instances, creates in add and queryset create

ormar/fields/__init__.py

@@ -21,6 +21,7 @@ from ormar.fields.model_fields import (
     Time,
     UUID,
 )
+from ormar.fields.through_field import Through, ThroughField
 
 __all__ = [
     "Decimal",
@@ -41,4 +42,6 @@ __all__ = [
     "BaseField",
     "UniqueColumns",
     "ForeignKeyField",
+    "ThroughField",
+    "Through",
 ]

ormar/fields/through_field.py

@@ -0,0 +1,73 @@
+import sys
+from typing import Any, TYPE_CHECKING, Type, Union
+
+from ormar.fields.base import BaseField
+from ormar.fields.foreign_key import ForeignKeyField
+
+if TYPE_CHECKING:  # pragma no cover
+    from ormar import Model
+    from pydantic.typing import ForwardRef
+
+    if sys.version_info < (3, 7):
+        ToType = Type[Model]
+    else:
+        ToType = Union[Type[Model], ForwardRef]
+
+
+def Through(  # noqa CFQ002
+    to: "ToType",
+    *,
+    name: str = None,
+    related_name: str = None,
+    virtual: bool = True,
+    **kwargs: Any,
+) -> Any:
+    # TODO: clean docstring
+    """
+    Despite a name it's a function that returns constructed ForeignKeyField.
+    This function is actually used in model declaration (as ormar.ForeignKey(ToModel)).
+
+    Accepts number of relation setting parameters as well as all BaseField ones.
+
+    :param to: target related ormar Model
+    :type to: Model class
+    :param name: name of the database field - later called alias
+    :type name: str
+    :param related_name: name of reversed FK relation populated for you on to model
+    :type related_name: str
+    :param virtual: marks if relation is virtual.
+    It is for reversed FK and auto generated FK on through model in Many2Many relations.
+    :type virtual: bool
+    :param kwargs: all other args to be populated by BaseField
+    :type kwargs: Any
+    :return: ormar ForeignKeyField with relation to selected model
+    :rtype: ForeignKeyField
+    """
+
+    owner = kwargs.pop("owner", None)
+    namespace = dict(
+        __type__=to,
+        to=to,
+        through=None,
+        alias=name,
+        name=kwargs.pop("real_name", None),
+        related_name=related_name,
+        virtual=virtual,
+        owner=owner,
+        nullable=False,
+        unique=False,
+        column_type=None,
+        primary_key=False,
+        index=False,
+        pydantic_only=False,
+        default=None,
+        server_default=None,
+    )
+
+    return type("Through", (ThroughField, BaseField), namespace)
+
+
+class ThroughField(ForeignKeyField):
+    """
+    Field class used to access ManyToMany model through model.
+    """

ormar/models/__init__.py

@@ -5,6 +5,7 @@ ass well as vast number of helper functions for pydantic, sqlalchemy and relatio
 """
 
 from ormar.models.newbasemodel import NewBaseModel  # noqa I100
+from ormar.models.model_row import ModelRow  # noqa I100
 from ormar.models.model import Model  # noqa I100
 
-__all__ = ["NewBaseModel", "Model"]
+__all__ = ["NewBaseModel", "Model", "ModelRow"]

ormar/models/helpers/relations.py

@@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Type
 
 import ormar
 from ormar import ForeignKey, ManyToMany
-from ormar.fields import ManyToManyField
+from ormar.fields import ManyToManyField, Through, ThroughField
 from ormar.fields.foreign_key import ForeignKeyField
 from ormar.models.helpers.sqlalchemy import adjust_through_many_to_many_model
 from ormar.relations import AliasManager
@@ -81,7 +81,8 @@ def expand_reverse_relationships(model: Type["Model"]) -> None:
     :param model: model on which relation should be checked and registered
     :type model: Model class
     """
-    for model_field in model.Meta.model_fields.values():
+    model_fields = list(model.Meta.model_fields.values())
+    for model_field in model_fields:
         if (
             issubclass(model_field, ForeignKeyField)
             and not model_field.has_unresolved_forward_refs()
@@ -113,6 +114,7 @@ def register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None:
             self_reference_primary=model_field.self_reference_primary,
         )
         # register foreign keys on through model
+        register_through_shortcut_fields(model_field=model_field)
         adjust_through_many_to_many_model(model_field=model_field)
     else:
         model_field.to.Meta.model_fields[related_name] = ForeignKey(
@@ -125,6 +127,34 @@ def register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None:
         )
 
 
+def register_through_shortcut_fields(model_field: Type["ManyToManyField"]) -> None:
+    """
+    Registers m2m relation through shortcut on both ends of the relation.
+
+    :param model_field: relation field defined in parent model
+    :type model_field: ManyToManyField
+    """
+    through_model = model_field.through
+    through_name = through_model.get_name(lower=True)
+    related_name = model_field.get_related_name()
+
+    model_field.owner.Meta.model_fields[through_name] = Through(
+        through_model,
+        real_name=through_name,
+        virtual=True,
+        related_name=model_field.name,
+        owner=model_field.owner,
+    )
+
+    model_field.to.Meta.model_fields[through_name] = Through(
+        through_model,
+        real_name=through_name,
+        virtual=True,
+        related_name=related_name,
+        owner=model_field.to,
+    )
+
+
 def register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None:
     """
     Registers the relation (and reverse relation) in alias manager.
@@ -142,7 +172,7 @@ def register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None:
         if field.has_unresolved_forward_refs():
             return
         register_many_to_many_relation_on_build(field=field)
-    elif issubclass(field, ForeignKeyField):
+    elif issubclass(field, ForeignKeyField) and not issubclass(field, ThroughField):
         if field.has_unresolved_forward_refs():
             return
         register_relation_on_build(field=field)

ormar/models/metaclass.py

@@ -46,6 +46,7 @@ if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
 
 CONFIG_KEY = "Config"
+PARSED_FIELDS_KEY = "__parsed_fields__"
 
 
 class ModelMeta:
@@ -141,83 +142,6 @@ def register_signals(new_model: Type["Model"]) -> None:  # noqa: CCR001
         new_model.Meta.signals = signals
 
 
-class ModelMetaclass(pydantic.main.ModelMetaclass):
-    def __new__(  # type: ignore # noqa: CCR001
-        mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict
-    ) -> "ModelMetaclass":
-        """
-        Metaclass used by ormar Models that performs configuration
-        and build of ormar Models.
-
-
-        Sets pydantic configuration.
-        Extract model_fields and convert them to pydantic FieldInfo,
-        updates class namespace.
-
-        Extracts settings and fields from parent classes.
-        Fetches methods decorated with @property_field decorator
-        to expose them later in dict().
-
-        Construct parent pydantic Metaclass/ Model.
-
-        If class has Meta class declared (so actual ormar Models) it also:
-
-        * populate sqlalchemy columns, pkname and tables from model_fields
-        * register reverse relationships on related models
-        * registers all relations in alias manager that populates table_prefixes
-        * exposes alias manager on each Model
-        * creates QuerySet for each model and exposes it on a class
-
-        :param name: name of current class
-        :type name: str
-        :param bases: base classes
-        :type bases: Tuple
-        :param attrs: class namespace
-        :type attrs: Dict
-        """
-        attrs["Config"] = get_pydantic_base_orm_config()
-        attrs["__name__"] = name
-        attrs, model_fields = extract_annotations_and_default_vals(attrs)
-        for base in reversed(bases):
-            mod = base.__module__
-            if mod.startswith("ormar.models.") or mod.startswith("pydantic."):
-                continue
-            attrs, model_fields = extract_from_parents_definition(
-                base_class=base, curr_class=mcs, attrs=attrs, model_fields=model_fields
-            )
-        new_model = super().__new__(  # type: ignore
-            mcs, name, bases, attrs
-        )
-
-        add_cached_properties(new_model)
-
-        if hasattr(new_model, "Meta"):
-            populate_default_options_values(new_model, model_fields)
-            check_required_meta_parameters(new_model)
-            add_property_fields(new_model, attrs)
-            register_signals(new_model=new_model)
-            populate_choices_validators(new_model)
-
-            if not new_model.Meta.abstract:
-                new_model = populate_meta_tablename_columns_and_pk(name, new_model)
-                populate_meta_sqlalchemy_table_if_required(new_model.Meta)
-                expand_reverse_relationships(new_model)
-                for field in new_model.Meta.model_fields.values():
-                    register_relation_in_alias_manager(field=field)
-
-                if new_model.Meta.pkname not in attrs["__annotations__"]:
-                    field_name = new_model.Meta.pkname
-                    attrs["__annotations__"][field_name] = Optional[int]  # type: ignore
-                    attrs[field_name] = None
-                    new_model.__fields__[field_name] = get_pydantic_field(
-                        field_name=field_name, model=new_model
-                    )
-                new_model.Meta.alias_manager = alias_manager
-                new_model.objects = QuerySet(new_model)
-
-        return new_model
-
-
 def verify_constraint_names(
     base_class: "Model", model_fields: Dict, parent_value: List
 ) -> None:
@@ -539,4 +463,78 @@ def update_attrs_and_fields(
     return updated_model_fields
 
 
-PARSED_FIELDS_KEY = "__parsed_fields__"
+class ModelMetaclass(pydantic.main.ModelMetaclass):
+    def __new__(  # type: ignore # noqa: CCR001
+        mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict
+    ) -> "ModelMetaclass":
+        """
+        Metaclass used by ormar Models that performs configuration
+        and build of ormar Models.
+
+
+        Sets pydantic configuration.
+        Extract model_fields and convert them to pydantic FieldInfo,
+        updates class namespace.
+
+        Extracts settings and fields from parent classes.
+        Fetches methods decorated with @property_field decorator
+        to expose them later in dict().
+
+        Construct parent pydantic Metaclass/ Model.
+
+        If class has Meta class declared (so actual ormar Models) it also:
+
+        * populate sqlalchemy columns, pkname and tables from model_fields
+        * register reverse relationships on related models
+        * registers all relations in alias manager that populates table_prefixes
+        * exposes alias manager on each Model
+        * creates QuerySet for each model and exposes it on a class
+
+        :param name: name of current class
+        :type name: str
+        :param bases: base classes
+        :type bases: Tuple
+        :param attrs: class namespace
+        :type attrs: Dict
+        """
+        attrs["Config"] = get_pydantic_base_orm_config()
+        attrs["__name__"] = name
+        attrs, model_fields = extract_annotations_and_default_vals(attrs)
+        for base in reversed(bases):
+            mod = base.__module__
+            if mod.startswith("ormar.models.") or mod.startswith("pydantic."):
+                continue
+            attrs, model_fields = extract_from_parents_definition(
+                base_class=base, curr_class=mcs, attrs=attrs, model_fields=model_fields
+            )
+        new_model = super().__new__(  # type: ignore
+            mcs, name, bases, attrs
+        )
+
+        add_cached_properties(new_model)
+
+        if hasattr(new_model, "Meta"):
+            populate_default_options_values(new_model, model_fields)
+            check_required_meta_parameters(new_model)
+            add_property_fields(new_model, attrs)
+            register_signals(new_model=new_model)
+            populate_choices_validators(new_model)
+
+            if not new_model.Meta.abstract:
+                new_model = populate_meta_tablename_columns_and_pk(name, new_model)
+                populate_meta_sqlalchemy_table_if_required(new_model.Meta)
+                expand_reverse_relationships(new_model)
+                for field in new_model.Meta.model_fields.values():
+                    register_relation_in_alias_manager(field=field)
+
+                if new_model.Meta.pkname not in attrs["__annotations__"]:
+                    field_name = new_model.Meta.pkname
+                    attrs["__annotations__"][field_name] = Optional[int]  # type: ignore
+                    attrs[field_name] = None
+                    new_model.__fields__[field_name] = get_pydantic_field(
+                        field_name=field_name, model=new_model
+                    )
+                new_model.Meta.alias_manager = alias_manager
+                new_model.objects = QuerySet(new_model)
+
+        return new_model

ormar/models/mixins/excludable_mixin.py

@@ -31,6 +31,7 @@ class ExcludableMixin(RelationMixin):
 
     if TYPE_CHECKING:  # pragma: no cover
         from ormar import Model
+        from ormar.models import ModelRow
 
     @staticmethod
     def get_child(
@@ -157,7 +158,7 @@ class ExcludableMixin(RelationMixin):
     @classmethod
     def own_table_columns(
         cls,
-        model: Type["Model"],
+        model: Union[Type["Model"], Type["ModelRow"]],
         fields: Optional[Union[Set, Dict]],
         exclude_fields: Optional[Union[Set, Dict]],
         use_alias: bool = False,

ormar/models/mixins/relation_mixin.py

@@ -1,6 +1,8 @@
 import inspect
 from typing import List, Optional, Set, TYPE_CHECKING
 
+from ormar import ManyToManyField
+from ormar.fields import ThroughField
 from ormar.fields.foreign_key import ForeignKeyField
 
 
@@ -43,27 +45,46 @@ class RelationMixin:
             return cls._related_fields
 
         related_fields = []
-        for name in cls.extract_related_names():
+        for name in cls.extract_related_names().union(cls.extract_through_names()):
             related_fields.append(cls.Meta.model_fields[name])
         cls._related_fields = related_fields
 
         return related_fields
 
+    @classmethod
+    def extract_through_names(cls) -> Set:
+        """
+        Extracts related fields through names which are shortcuts to through models.
+
+        :return: set of related through fields names
+        :rtype: Set
+        """
+        related_fields = set()
+        for name in cls.extract_related_names():
+            field = cls.Meta.model_fields[name]
+            if issubclass(field, ManyToManyField):
+                related_fields.add(field.through.get_name(lower=True))
+        return related_fields
+
     @classmethod
     def extract_related_names(cls) -> Set:
         """
         Returns List of fields names for all relations declared on a model.
         List is cached in cls._related_names for quicker access.
 
-        :return: list of related fields names
+        :return: set of related fields names
-        :rtype: List
+        :rtype: Set
         """
         if isinstance(cls._related_names, Set):
             return cls._related_names
 
         related_names = set()
         for name, field in cls.Meta.model_fields.items():
-            if inspect.isclass(field) and issubclass(field, ForeignKeyField):
+            if (
+                inspect.isclass(field)
+                and issubclass(field, ForeignKeyField)
+                and not issubclass(field, ThroughField)
+            ):
                 related_names.add(name)
         cls._related_names = related_names
 

ormar/models/model.py

@@ -1,24 +1,17 @@
 from typing import (
     Any,
-    Dict,
-    List,
-    Optional,
     Set,
     TYPE_CHECKING,
     Tuple,
-    Type,
     TypeVar,
-    Union,
 )
 
-import sqlalchemy
-
 import ormar.queryset  # noqa I100
 from ormar.exceptions import ModelPersistenceError, NoMatch
 from ormar.fields.many_to_many import ManyToManyField
 from ormar.models import NewBaseModel  # noqa I100
-from ormar.models.helpers.models import group_related_list
 from ormar.models.metaclass import ModelMeta
+from ormar.models.model_row import ModelRow
 
 if TYPE_CHECKING:  # pragma nocover
     from ormar import QuerySet
@@ -26,7 +19,7 @@ if TYPE_CHECKING:  # pragma nocover
 T = TypeVar("T", bound="Model")
 
 
-class Model(NewBaseModel):
+class Model(ModelRow):
     __abstract__ = False
     if TYPE_CHECKING:  # pragma nocover
         Meta: ModelMeta
@@ -36,247 +29,6 @@ class Model(NewBaseModel):
         _repr = {k: getattr(self, k) for k, v in self.Meta.model_fields.items()}
         return f"{self.__class__.__name__}({str(_repr)})"
 
-    @classmethod
-    def from_row(  # noqa CCR001
-        cls: Type[T],
-        row: sqlalchemy.engine.ResultProxy,
-        select_related: List = None,
-        related_models: Any = None,
-        previous_model: Type[T] = None,
-        source_model: Type[T] = None,
-        related_name: str = None,
-        fields: Optional[Union[Dict, Set]] = None,
-        exclude_fields: Optional[Union[Dict, Set]] = None,
-        current_relation_str: str = None,
-    ) -> Optional[T]:
-        """
-        Model method to convert raw sql row from database into ormar.Model instance.
-        Traverses nested models if they were specified in select_related for query.
-
-        Called recurrently and returns model instance if it's present in the row.
-        Note that it's processing one row at a time, so if there are duplicates of
-        parent row that needs to be joined/combined
-        (like parent row in sql join with 2+ child rows)
-        instances populated in this method are later combined in the QuerySet.
-        Other method working directly on raw database results is in prefetch_query,
-        where rows are populated in a different way as they do not have
-        nested models in result.
-
-        :param current_relation_str: name of the relation field
-        :type current_relation_str: str
-        :param source_model: model on which relation was defined
-        :type source_model: Type[Model]
-        :param row: raw result row from the database
-        :type row: sqlalchemy.engine.result.ResultProxy
-        :param select_related: list of names of related models fetched from database
-        :type select_related: List
-        :param related_models: list or dict of related models
-        :type related_models: Union[List, Dict]
-        :param previous_model: internal param for nested models to specify table_prefix
-        :type previous_model: Model class
-        :param related_name: internal parameter - name of current nested model
-        :type related_name: str
-        :param fields: fields and related model fields to include
-        if provided only those are included
-        :type fields: Optional[Union[Dict, Set]]
-        :param exclude_fields: fields and related model fields to exclude
-        excludes the fields even if they are provided in fields
-        :type exclude_fields: Optional[Union[Dict, Set]]
-        :return: returns model if model is populated from database
-        :rtype: Optional[Model]
-        """
-        item: Dict[str, Any] = {}
-        select_related = select_related or []
-        related_models = related_models or []
-        table_prefix = ""
-
-        if select_related:
-            source_model = cls
-            related_models = group_related_list(select_related)
-
-        rel_name2 = related_name
-
-        if (
-            previous_model
-            and related_name
-            and issubclass(
-                previous_model.Meta.model_fields[related_name], ManyToManyField
-            )
-        ):
-            through_field = previous_model.Meta.model_fields[related_name]
-            if (
-                through_field.self_reference
-                and related_name == through_field.self_reference_primary
-            ):
-                rel_name2 = through_field.default_source_field_name()  # type: ignore
-            else:
-                rel_name2 = through_field.default_target_field_name()  # type: ignore
-            previous_model = through_field.through  # type: ignore
-
-        if previous_model and rel_name2:
-            if current_relation_str and "__" in current_relation_str and source_model:
-                table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
-                    from_model=source_model, relation_name=current_relation_str
-                )
-            if not table_prefix:
-                table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
-                    from_model=previous_model, relation_name=rel_name2
-                )
-
-        item = cls.populate_nested_models_from_row(
-            item=item,
-            row=row,
-            related_models=related_models,
-            fields=fields,
-            exclude_fields=exclude_fields,
-            current_relation_str=current_relation_str,
-            source_model=source_model,
-        )
-        item = cls.extract_prefixed_table_columns(
-            item=item,
-            row=row,
-            table_prefix=table_prefix,
-            fields=fields,
-            exclude_fields=exclude_fields,
-        )
-
-        instance: Optional[T] = None
-        if item.get(cls.Meta.pkname, None) is not None:
-            item["__excluded__"] = cls.get_names_to_exclude(
-                fields=fields, exclude_fields=exclude_fields
-            )
-            instance = cls(**item)
-            instance.set_save_status(True)
-        return instance
-
-    @classmethod
-    def populate_nested_models_from_row(  # noqa: CFQ002
-        cls,
-        item: dict,
-        row: sqlalchemy.engine.ResultProxy,
-        related_models: Any,
-        fields: Optional[Union[Dict, Set]] = None,
-        exclude_fields: Optional[Union[Dict, Set]] = None,
-        current_relation_str: str = None,
-        source_model: Type[T] = None,
-    ) -> dict:
-        """
-        Traverses structure of related models and populates the nested models
-        from the database row.
-        Related models can be a list if only directly related models are to be
-        populated, converted to dict if related models also have their own related
-        models to be populated.
-
-        Recurrently calls from_row method on nested instances and create nested
-        instances. In the end those instances are added to the final model dictionary.
-
-        :param source_model: source model from which relation started
-        :type source_model: Type[Model]
-        :param current_relation_str: joined related parts into one string
-        :type current_relation_str: str
-        :param item: dictionary of already populated nested models, otherwise empty dict
-        :type item: Dict
-        :param row: raw result row from the database
-        :type row: sqlalchemy.engine.result.ResultProxy
-        :param related_models: list or dict of related models
-        :type related_models: Union[Dict, List]
-        :param fields: fields and related model fields to include -
-        if provided only those are included
-        :type fields: Optional[Union[Dict, Set]]
-        :param exclude_fields: fields and related model fields to exclude
-        excludes the fields even if they are provided in fields
-        :type exclude_fields: Optional[Union[Dict, Set]]
-        :return: dictionary with keys corresponding to model fields names
-        and values are database values
-        :rtype: Dict
-        """
-
-        for related in related_models:
-            relation_str = (
-                "__".join([current_relation_str, related])
-                if current_relation_str
-                else related
-            )
-            fields = cls.get_included(fields, related)
-            exclude_fields = cls.get_excluded(exclude_fields, related)
-            model_cls = cls.Meta.model_fields[related].to
-
-            remainder = None
-            if isinstance(related_models, dict) and related_models[related]:
-                remainder = related_models[related]
-            child = model_cls.from_row(
-                row,
-                related_models=remainder,
-                previous_model=cls,
-                related_name=related,
-                fields=fields,
-                exclude_fields=exclude_fields,
-                current_relation_str=relation_str,
-                source_model=source_model,
-            )
-            item[model_cls.get_column_name_from_alias(related)] = child
-
-        return item
-
-    @classmethod
-    def extract_prefixed_table_columns(  # noqa CCR001
-        cls,
-        item: dict,
-        row: sqlalchemy.engine.result.ResultProxy,
-        table_prefix: str,
-        fields: Optional[Union[Dict, Set]] = None,
-        exclude_fields: Optional[Union[Dict, Set]] = None,
-    ) -> dict:
-        """
-        Extracts own fields from raw sql result, using a given prefix.
-        Prefix changes depending on the table's position in a join.
-
-        If the table is a main table, there is no prefix.
-        All joined tables have prefixes to allow duplicate column names,
-        as well as duplicated joins to the same table from multiple different tables.
-
-        Extracted fields populates the related dict later used to construct a Model.
-
-        Used in Model.from_row and PrefetchQuery._populate_rows methods.
-
-        :param item: dictionary of already populated nested models, otherwise empty dict
-        :type item: Dict
-        :param row: raw result row from the database
-        :type row: sqlalchemy.engine.result.ResultProxy
-        :param table_prefix: prefix of the table from AliasManager
-        each pair of tables have own prefix (two of them depending on direction) -
-        used in joins to allow multiple joins to the same table.
-        :type table_prefix: str
-        :param fields: fields and related model fields to include -
-        if provided only those are included
-        :type fields: Optional[Union[Dict, Set]]
-        :param exclude_fields: fields and related model fields to exclude
-        excludes the fields even if they are provided in fields
-        :type exclude_fields: Optional[Union[Dict, Set]]
-        :return: dictionary with keys corresponding to model fields names
-        and values are database values
-        :rtype: Dict
-        """
-        # databases does not keep aliases in Record for postgres, change to raw row
-        source = row._row if cls.db_backend_name() == "postgresql" else row
-
-        selected_columns = cls.own_table_columns(
-            model=cls,
-            fields=fields or {},
-            exclude_fields=exclude_fields or {},
-            use_alias=False,
-        )
-
-        for column in cls.Meta.table.columns:
-            alias = cls.get_column_name_from_alias(column.name)
-            if alias not in item and alias in selected_columns:
-                prefixed_name = (
-                    f'{table_prefix + "_" if table_prefix else ""}{column.name}'
-                )
-                item[alias] = source[prefixed_name]
-
-        return item
-
     async def upsert(self: T, **kwargs: Any) -> T:
         """
         Performs either a save or an update depending on the presence of the pk.

ormar/models/model_row.py

@@ -0,0 +1,303 @@
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    Set,
+    Type,
+    TypeVar,
+    Union,
+)
+
+import sqlalchemy
+
+from ormar import ManyToManyField  # noqa: I202
+from ormar.models import NewBaseModel
+from ormar.models.helpers.models import group_related_list
+
+T = TypeVar("T", bound="ModelRow")
+
+
+class ModelRow(NewBaseModel):
+    @classmethod
+    def from_row(  # noqa CCR001
+        cls: Type[T],
+        row: sqlalchemy.engine.ResultProxy,
+        select_related: List = None,
+        related_models: Any = None,
+        previous_model: Type[T] = None,
+        source_model: Type[T] = None,
+        related_name: str = None,
+        fields: Optional[Union[Dict, Set]] = None,
+        exclude_fields: Optional[Union[Dict, Set]] = None,
+        current_relation_str: str = None,
+    ) -> Optional[T]:
+        """
+        Model method to convert raw sql row from database into ormar.Model instance.
+        Traverses nested models if they were specified in select_related for query.
+
+        Called recurrently and returns model instance if it's present in the row.
+        Note that it's processing one row at a time, so if there are duplicates of
+        parent row that needs to be joined/combined
+        (like parent row in sql join with 2+ child rows)
+        instances populated in this method are later combined in the QuerySet.
+        Other method working directly on raw database results is in prefetch_query,
+        where rows are populated in a different way as they do not have
+        nested models in result.
+
+        :param current_relation_str: name of the relation field
+        :type current_relation_str: str
+        :param source_model: model on which relation was defined
+        :type source_model: Type[Model]
+        :param row: raw result row from the database
+        :type row: sqlalchemy.engine.result.ResultProxy
+        :param select_related: list of names of related models fetched from database
+        :type select_related: List
+        :param related_models: list or dict of related models
+        :type related_models: Union[List, Dict]
+        :param previous_model: internal param for nested models to specify table_prefix
+        :type previous_model: Model class
+        :param related_name: internal parameter - name of current nested model
+        :type related_name: str
+        :param fields: fields and related model fields to include
+        if provided only those are included
+        :type fields: Optional[Union[Dict, Set]]
+        :param exclude_fields: fields and related model fields to exclude
+        excludes the fields even if they are provided in fields
+        :type exclude_fields: Optional[Union[Dict, Set]]
+        :return: returns model if model is populated from database
+        :rtype: Optional[Model]
+        """
+        item: Dict[str, Any] = {}
+        select_related = select_related or []
+        related_models = related_models or []
+        table_prefix = ""
+
+        if select_related:
+            source_model = cls
+            related_models = group_related_list(select_related)
+
+        rel_name2 = related_name
+
+        # TODO: refactor this into field classes?
+        if (
+            previous_model
+            and related_name
+            and issubclass(
+                previous_model.Meta.model_fields[related_name], ManyToManyField
+            )
+        ):
+            through_field = previous_model.Meta.model_fields[related_name]
+            if (
+                through_field.self_reference
+                and related_name == through_field.self_reference_primary
+            ):
+                rel_name2 = through_field.default_source_field_name()  # type: ignore
+            else:
+                rel_name2 = through_field.default_target_field_name()  # type: ignore
+            previous_model = through_field.through  # type: ignore
+
+        if previous_model and rel_name2:
+            if current_relation_str and "__" in current_relation_str and source_model:
+                table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
+                    from_model=source_model, relation_name=current_relation_str
+                )
+            if not table_prefix:
+                table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
+                    from_model=previous_model, relation_name=rel_name2
+                )
+
+        item = cls.populate_nested_models_from_row(
+            item=item,
+            row=row,
+            related_models=related_models,
+            fields=fields,
+            exclude_fields=exclude_fields,
+            current_relation_str=current_relation_str,
+            source_model=source_model,
+        )
+        item = cls.extract_prefixed_table_columns(
+            item=item,
+            row=row,
+            table_prefix=table_prefix,
+            fields=fields,
+            exclude_fields=exclude_fields,
+        )
+
+        instance: Optional[T] = None
+        if item.get(cls.Meta.pkname, None) is not None:
+            item["__excluded__"] = cls.get_names_to_exclude(
+                fields=fields, exclude_fields=exclude_fields
+            )
+            instance = cls(**item)
+            instance.set_save_status(True)
+        return instance
+
+    @classmethod
+    def populate_nested_models_from_row(  # noqa: CFQ002
+        cls,
+        item: dict,
+        row: sqlalchemy.engine.ResultProxy,
+        related_models: Any,
+        fields: Optional[Union[Dict, Set]] = None,
+        exclude_fields: Optional[Union[Dict, Set]] = None,
+        current_relation_str: str = None,
+        source_model: Type[T] = None,
+    ) -> dict:
+        """
+        Traverses structure of related models and populates the nested models
+        from the database row.
+        Related models can be a list if only directly related models are to be
+        populated, converted to dict if related models also have their own related
+        models to be populated.
+
+        Recurrently calls from_row method on nested instances and create nested
+        instances. In the end those instances are added to the final model dictionary.
+
+        :param source_model: source model from which relation started
+        :type source_model: Type[Model]
+        :param current_relation_str: joined related parts into one string
+        :type current_relation_str: str
+        :param item: dictionary of already populated nested models, otherwise empty dict
+        :type item: Dict
+        :param row: raw result row from the database
+        :type row: sqlalchemy.engine.result.ResultProxy
+        :param related_models: list or dict of related models
+        :type related_models: Union[Dict, List]
+        :param fields: fields and related model fields to include -
+        if provided only those are included
+        :type fields: Optional[Union[Dict, Set]]
+        :param exclude_fields: fields and related model fields to exclude
+        excludes the fields even if they are provided in fields
+        :type exclude_fields: Optional[Union[Dict, Set]]
+        :return: dictionary with keys corresponding to model fields names
+        and values are database values
+        :rtype: Dict
+        """
+
+        for related in related_models:
+            relation_str = (
+                "__".join([current_relation_str, related])
+                if current_relation_str
+                else related
+            )
+            field = cls.Meta.model_fields[related]
+            fields = cls.get_included(fields, related)
+            exclude_fields = cls.get_excluded(exclude_fields, related)
+            model_cls = field.to
+
+            remainder = None
+            if isinstance(related_models, dict) and related_models[related]:
+                remainder = related_models[related]
+            child = model_cls.from_row(
+                row,
+                related_models=remainder,
+                previous_model=cls,
+                related_name=related,
+                fields=fields,
+                exclude_fields=exclude_fields,
+                current_relation_str=relation_str,
+                source_model=source_model,
+            )
+            item[model_cls.get_column_name_from_alias(related)] = child
+            if issubclass(field, ManyToManyField) and child:
+                # TODO: way to figure out which side should be populated?
+                through_name = cls.Meta.model_fields[related].through.get_name()
+                # for now it's nested dict, should be instance?
+                through_child = cls.populate_through_instance(
+                    row=row,
+                    related=related,
+                    through_name=through_name,
+                    fields=fields,
+                    exclude_fields=exclude_fields,
+                )
+                item[through_name] = through_child
+                setattr(child, through_name, through_child)
+                child.set_save_status(True)
+
+        return item
+
+    @classmethod
+    def populate_through_instance(
+        cls,
+        row: sqlalchemy.engine.ResultProxy,
+        through_name: str,
+        related: str,
+        fields: Optional[Union[Dict, Set]] = None,
+        exclude_fields: Optional[Union[Dict, Set]] = None,
+    ) -> Dict:
+        # TODO: fix excludes and includes
+        fields = cls.get_included(fields, through_name)
+        # exclude_fields = cls.get_excluded(exclude_fields, through_name)
+        model_cls = cls.Meta.model_fields[through_name].to
+        exclude_fields = model_cls.extract_related_names()
+        table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
+            from_model=cls, relation_name=related
+        )
+        child = model_cls.extract_prefixed_table_columns(
+            item={},
+            row=row,
+            table_prefix=table_prefix,
+            fields=fields,
+            exclude_fields=exclude_fields,
+        )
+        return child
+
+    @classmethod
+    def extract_prefixed_table_columns(  # noqa CCR001
+        cls,
+        item: dict,
+        row: sqlalchemy.engine.result.ResultProxy,
+        table_prefix: str,
+        fields: Optional[Union[Dict, Set]] = None,
+        exclude_fields: Optional[Union[Dict, Set]] = None,
+    ) -> dict:
+        """
+        Extracts own fields from raw sql result, using a given prefix.
+        Prefix changes depending on the table's position in a join.
+
+        If the table is a main table, there is no prefix.
+        All joined tables have prefixes to allow duplicate column names,
+        as well as duplicated joins to the same table from multiple different tables.
+
+        Extracted fields populates the related dict later used to construct a Model.
+
+        Used in Model.from_row and PrefetchQuery._populate_rows methods.
+
+        :param item: dictionary of already populated nested models, otherwise empty dict
+        :type item: Dict
+        :param row: raw result row from the database
+        :type row: sqlalchemy.engine.result.ResultProxy
+        :param table_prefix: prefix of the table from AliasManager
+        each pair of tables have own prefix (two of them depending on direction) -
+        used in joins to allow multiple joins to the same table.
+        :type table_prefix: str
+        :param fields: fields and related model fields to include -
+        if provided only those are included
+        :type fields: Optional[Union[Dict, Set]]
+        :param exclude_fields: fields and related model fields to exclude
+        excludes the fields even if they are provided in fields
+        :type exclude_fields: Optional[Union[Dict, Set]]
+        :return: dictionary with keys corresponding to model fields names
+        and values are database values
+        :rtype: Dict
+        """
+        # databases does not keep aliases in Record for postgres, change to raw row
+        source = row._row if cls.db_backend_name() == "postgresql" else row
+
+        selected_columns = cls.own_table_columns(
+            model=cls,
+            fields=fields or {},
+            exclude_fields=exclude_fields or {},
+            use_alias=False,
+        )
+
+        for column in cls.Meta.table.columns:
+            alias = cls.get_column_name_from_alias(column.name)
+            if alias not in item and alias in selected_columns:
+                prefixed_name = (
+                    f'{table_prefix + "_" if table_prefix else ""}{column.name}'
+                )
+                item[alias] = source[prefixed_name]
+
+        return item

ormar/models/newbasemodel.py

@@ -172,7 +172,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
         object.__setattr__(self, "__fields_set__", fields_set)
 
         # register the columns models after initialization
-        for related in self.extract_related_names():
+        for related in self.extract_related_names().union(self.extract_through_names()):
             self.Meta.model_fields[related].expand_relationship(
                 new_kwargs.get(related), self, to_register=True,
             )
@@ -267,6 +267,10 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
             return object.__getattribute__(
                 self, "_extract_related_model_instead_of_field"
             )(item)
+        if item in object.__getattribute__(self, "extract_through_names")():
+            return object.__getattribute__(
+                self, "_extract_related_model_instead_of_field"
+            )(item)
         if item in object.__getattribute__(self, "Meta").property_fields:
             value = object.__getattribute__(self, item)
             return value() if callable(value) else value

ormar/models/quick_access_views.py

@@ -34,10 +34,12 @@ quick_access_set = {
     "_skip_ellipsis",
     "_update_and_follow",
     "_update_excluded_with_related_not_required",
+    "_verify_model_can_be_initialized",
     "copy",
     "delete",
     "dict",
     "extract_related_names",
+    "extract_through_names",
     "update_from_dict",
     "get_column_alias",
     "get_column_name_from_alias",

ormar/queryset/join.py

@@ -291,6 +291,8 @@ class SqlJoin:
             self.get_order_bys(
                 to_table=to_table, pkname_alias=pkname_alias,
             )
+        else:
+            self.select_through_model_fields()
 
         self_related_fields = self.next_model.own_table_columns(
             model=self.next_model,
@@ -305,6 +307,24 @@ class SqlJoin:
         )
         self.used_aliases.append(self.next_alias)
 
+    def select_through_model_fields(self) -> None:
+        # TODO: add docstring
+        next_alias = self.alias_manager.resolve_relation_alias(
+            from_model=self.target_field.owner, relation_name=self.relation_name
+        )
+        # TODO: fix fields and exclusions
+        self_related_fields = self.target_field.through.own_table_columns(
+            model=self.target_field.through,
+            fields=None,
+            exclude_fields=self.target_field.through.extract_related_names(),
+            use_alias=True,
+        )
+        self.columns.extend(
+            self.alias_manager.prefixed_columns(
+                next_alias, self.target_field.through.Meta.table, self_related_fields
+            )
+        )
+
     def _replace_many_to_many_order_by_columns(self, part: str, new_part: str) -> None:
         """
         Substitutes the name of the relation with actual model name in m2m order bys.

ormar/queryset/queryset.py

@@ -197,7 +197,7 @@ class QuerySet:
             limit_raw_sql=self.limit_sql_raw,
         )
         exp = qry.build_select_expression()
-        # print("\n", exp.compile(compile_kwargs={"literal_binds": True}))
+        print("\n", exp.compile(compile_kwargs={"literal_binds": True}))
         return exp
 
     def filter(self, _exclude: bool = False, **kwargs: Any) -> "QuerySet":  # noqa: A003

ormar/relations/alias_manager.py

@@ -1,13 +1,14 @@
 import string
 import uuid
 from random import choices
-from typing import Any, Dict, List, TYPE_CHECKING, Type
+from typing import Any, Dict, List, TYPE_CHECKING, Type, Union
 
 import sqlalchemy
 from sqlalchemy import text
 
 if TYPE_CHECKING:  # pragma: no cover
     from ormar import Model
+    from ormar.models import ModelRow
 
 
 def get_table_alias() -> str:
@@ -133,7 +134,7 @@ class AliasManager:
         return alias
 
     def resolve_relation_alias(
-        self, from_model: Type["Model"], relation_name: str
+        self, from_model: Union[Type["Model"], Type["ModelRow"]], relation_name: str
     ) -> str:
         """
         Given model and relation name returns the alias for this relation.

ormar/relations/querysetproxy.py

@@ -44,6 +44,11 @@ class QuerysetProxy(ormar.QuerySetProtocol):
         ].get_related_name()
         self.related_field = self.relation.to.Meta.model_fields[self.related_field_name]
         self.owner_pk_value = self._owner.pk
+        self.through_model_name = (
+            self.related_field.through.get_name()
+            if self.type_ == ormar.RelationType.MULTIPLE
+            else None
+        )
 
     @property
     def queryset(self) -> "QuerySet":
@@ -99,17 +104,20 @@ class QuerysetProxy(ormar.QuerySetProtocol):
             for item in self.relation.related_models[:]:
                 self.relation.remove(item)
 
-    async def create_through_instance(self, child: "T") -> None:
+    async def create_through_instance(self, child: "T", **kwargs: Any) -> None:
         """
         Crete a through model instance in the database for m2m relations.
 
+        :param kwargs: dict of additional keyword arguments for through instance
+        :type kwargs: Any
         :param child: child model instance
         :type child: Model
         """
         model_cls = self.relation.through
         owner_column = self.related_field.default_target_field_name()  # type: ignore
         child_column = self.related_field.default_source_field_name()  # type: ignore
-        kwargs = {owner_column: self._owner.pk, child_column: child.pk}
+        rel_kwargs = {owner_column: self._owner.pk, child_column: child.pk}
+        final_kwargs = {**rel_kwargs, **kwargs}
         if child.pk is None:
             raise ModelPersistenceError(
                 f"You cannot save {child.get_name()} "
@@ -117,7 +125,7 @@ class QuerysetProxy(ormar.QuerySetProtocol):
                 f"Save the child model first."
             )
         expr = model_cls.Meta.table.insert()
-        expr = expr.values(**kwargs)
+        expr = expr.values(**final_kwargs)
         # print("\n", expr.compile(compile_kwargs={"literal_binds": True}))
         await model_cls.Meta.database.execute(expr)
 
@@ -270,12 +278,13 @@ class QuerysetProxy(ormar.QuerySetProtocol):
         :return: created model
         :rtype: Model
         """
+        through_kwargs = kwargs.pop(self.through_model_name, {})
         if self.type_ == ormar.RelationType.REVERSE:
             kwargs[self.related_field.name] = self._owner
         created = await self.queryset.create(**kwargs)
         self._register_related(created)
         if self.type_ == ormar.RelationType.MULTIPLE:
-            await self.create_through_instance(created)
+            await self.create_through_instance(created, **through_kwargs)
         return created
 
     async def get_or_create(self, **kwargs: Any) -> "Model":

ormar/relations/relation.py

@@ -26,6 +26,7 @@ class RelationType(Enum):
     PRIMARY = 1
     REVERSE = 2
     MULTIPLE = 3
+    THROUGH = 4
 
 
 class Relation:
@@ -128,7 +129,7 @@ class Relation:
         :type child: Model
         """
         relation_name = self.field_name
-        if self._type == RelationType.PRIMARY:
+        if self._type in (RelationType.PRIMARY, RelationType.THROUGH):
             self.related_models = child
             self._owner.__dict__[relation_name] = child
         else:

ormar/relations/relation_manager.py

@@ -1,7 +1,7 @@
 from typing import Dict, List, Optional, Sequence, TYPE_CHECKING, Type, TypeVar, Union
 from weakref import proxy
 
-from ormar.fields import BaseField
+from ormar.fields import BaseField, ThroughField
 from ormar.fields.foreign_key import ForeignKeyField
 from ormar.fields.many_to_many import ManyToManyField
 from ormar.relations.relation import Relation, RelationType
@@ -42,6 +42,8 @@ class RelationsManager:
         """
         if issubclass(field, ManyToManyField):
             return RelationType.MULTIPLE
+        if issubclass(field, ThroughField):
+            return RelationType.THROUGH
         return RelationType.PRIMARY if not field.virtual else RelationType.REVERSE
 
     def _add_relation(self, field: Type[BaseField]) -> None:

ormar/relations/relation_proxy.py

@@ -163,19 +163,21 @@ class RelationProxy(list):
             else:
                 await item.delete()
 
-    async def add(self, item: "Model") -> None:
+    async def add(self, item: "Model", **kwargs: Any) -> None:
         """
         Adds child model to relation.
 
         For ManyToMany relations through instance is automatically created.
 
+        :param kwargs: dict of additional keyword arguments for through instance
+        :type kwargs: Any
         :param item: child to add to relation
         :type item: Model
         """
         relation_name = self.related_field_name
         self._check_if_model_saved()
         if self.type_ == ormar.RelationType.MULTIPLE:
-            await self.queryset_proxy.create_through_instance(item)
+            await self.queryset_proxy.create_through_instance(item, **kwargs)
             setattr(item, relation_name, self._owner)
         else:
             setattr(item, relation_name, self._owner)

tests/test_m2m_through_fields.py

@@ -1,6 +1,7 @@
 import databases
 import pytest
 import sqlalchemy
+from pydantic.typing import ForwardRef
 
 import ormar
 from tests.settings import DATABASE_URL
@@ -39,28 +40,107 @@ class Post(ormar.Model):
     categories = ormar.ManyToMany(Category, through=PostCategory)
 
 
-#
+@pytest.fixture(autouse=True, scope="module")
-# @pytest.fixture(autouse=True, scope="module")
+def create_test_database():
-# async def create_test_database():
+    engine = sqlalchemy.create_engine(DATABASE_URL)
-#     engine = sqlalchemy.create_engine(DATABASE_URL)
+    metadata.drop_all(engine)
-#     metadata.create_all(engine)
+    metadata.create_all(engine)
-#     yield
+    yield
-#     metadata.drop_all(engine)
+    metadata.drop_all(engine)
-#
+
-#
+
-# @pytest.mark.asyncio
+class PostCategory2(ormar.Model):
-# async def test_setting_fields_on_through_model():
+    class Meta(BaseMeta):
-#     async with database:
+        tablename = "posts_x_categories2"
-#         # TODO: check/ modify following
+
-#         # loading the data into model instance of though model?
+    id: int = ormar.Integer(primary_key=True)
-#         # <- attach to other side? both sides? access by through, or add to fields?
+    sort_order: int = ormar.Integer(nullable=True)
-#         # creating while adding to relation (kwargs in add?)
+
-#         # creating in query (dividing kwargs between final and through)
+
-#         # updating in query
+@pytest.mark.asyncio
-#         # sorting in filter (special __through__<field_name> notation?)
+async def test_forward_ref_is_updated():
-#         # ordering by in order_by
+    async with database:
-#         # accessing from instance (both sides?)
+
-#         # modifying from instance (both sides?)
+        class Post2(ormar.Model):
-#         # including/excluding in fields?
+            class Meta(BaseMeta):
-#         # allowing to change fk fields names in through model?
+                pass
-#         pass
+
+            id: int = ormar.Integer(primary_key=True)
+            title: str = ormar.String(max_length=200)
+            categories = ormar.ManyToMany(Category, through=ForwardRef("PostCategory2"))
+
+        assert Post2.Meta.requires_ref_update
+        Post2.update_forward_refs()
+
+        assert Post2.Meta.model_fields["postcategory2"].to == PostCategory2
+
+
+@pytest.mark.asyncio
+async def test_setting_fields_on_through_model():
+    async with database:
+        post = await Post(title="Test post").save()
+        category = await Category(name="Test category").save()
+        await post.categories.add(category)
+
+        assert hasattr(post.categories[0], "postcategory")
+        assert post.categories[0].postcategory is None
+
+
+@pytest.mark.asyncio
+async def test_setting_additional_fields_on_through_model_in_add():
+    async with database:
+        post = await Post(title="Test post").save()
+        category = await Category(name="Test category").save()
+        await post.categories.add(category, sort_order=1)
+        postcat = await PostCategory.objects.get()
+        assert postcat.sort_order == 1
+
+
+@pytest.mark.asyncio
+async def test_setting_additional_fields_on_through_model_in_create():
+    async with database:
+        post = await Post(title="Test post").save()
+        await post.categories.create(
+            name="Test category2", postcategory={"sort_order": 2}
+        )
+        postcat = await PostCategory.objects.get()
+        assert postcat.sort_order == 2
+
+
+@pytest.mark.asyncio
+async def test_getting_additional_fields_from_queryset():
+    async with database:
+        post = await Post(title="Test post").save()
+        await post.categories.create(
+            name="Test category1", postcategory={"sort_order": 1}
+        )
+        await post.categories.create(
+            name="Test category2", postcategory={"sort_order": 2}
+        )
+
+        await post.categories.all()
+        assert post.categories[0].postcategory.sort_order == 1
+        assert post.categories[1].postcategory.sort_order == 2
+
+        post = await Post.objects.select_related("categories").get(
+            categories__name="Test category2"
+        )
+        assert post.categories[0].postcategory.sort_order == 2
+
+
+# TODO: check/ modify following
+
+# add to fields with class lower name (V)
+# forward refs update (V)
+# creating while adding to relation (kwargs in add) (V)
+# creating in queryset proxy (dict with through name and kwargs) (V)
+# loading the data into model instance of though model (V) <- fix fields ane exclude
+# accessing from instance (V) <- no both sides only nested one is relevant, fix one side
+
+# updating in query
+# sorting in filter (special __through__<field_name> notation?)
+# ordering by in order_by
+# modifying from instance (both sides?)
+# including/excluding in fields?
+# allowing to change fk fields names in through model?
+# make through optional? auto-generated for cases other fields are missing?