Prodream/ormar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip - through models fields -> attached in queries, accesible from instances, creates in add and queryset create

Browse Source
This commit is contained in:
collerek
2021-02-15 17:30:14 +01:00
parent 868243686d
commit 3fd231cf3c
19 changed files with 677 additions and 374 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
ormar/fields/__init__.py
View File

@ -21,6 +21,7 @@ from ormar.fields.model_fields import (
Time,
UUID,
)
from ormar.fields.through_field import Through, ThroughField
__all__ = [
"Decimal",
@ -41,4 +42,6 @@ __all__ = [
"BaseField",
"UniqueColumns",
"ForeignKeyField",
"ThroughField",
"Through",
]

73
ormar/fields/through_field.py Normal file
View File

@ -0,0 +1,73 @@
import sys
from typing import Any, TYPE_CHECKING, Type, Union
from ormar.fields.base import BaseField
from ormar.fields.foreign_key import ForeignKeyField
if TYPE_CHECKING: # pragma no cover
from ormar import Model
from pydantic.typing import ForwardRef
if sys.version_info < (3, 7):
ToType = Type[Model]
else:
ToType = Union[Type[Model], ForwardRef]
def Through( # noqa CFQ002
to: "ToType",
*,
name: str = None,
related_name: str = None,
virtual: bool = True,
**kwargs: Any,
) -> Any:
# TODO: clean docstring
"""
Despite a name it's a function that returns constructed ForeignKeyField.
This function is actually used in model declaration (as ormar.ForeignKey(ToModel)).
Accepts number of relation setting parameters as well as all BaseField ones.
:param to: target related ormar Model
:type to: Model class
:param name: name of the database field - later called alias
:type name: str
:param related_name: name of reversed FK relation populated for you on to model
:type related_name: str
:param virtual: marks if relation is virtual.
It is for reversed FK and auto generated FK on through model in Many2Many relations.
:type virtual: bool
:param kwargs: all other args to be populated by BaseField
:type kwargs: Any
:return: ormar ForeignKeyField with relation to selected model
:rtype: ForeignKeyField
"""
owner = kwargs.pop("owner", None)
namespace = dict(
__type__=to,
to=to,
through=None,
alias=name,
name=kwargs.pop("real_name", None),
related_name=related_name,
virtual=virtual,
owner=owner,
nullable=False,
unique=False,
column_type=None,
primary_key=False,
index=False,
pydantic_only=False,
default=None,
server_default=None,
)
return type("Through", (ThroughField, BaseField), namespace)
class ThroughField(ForeignKeyField):
"""
Field class used to access ManyToMany model through model.
"""

3
ormar/models/__init__.py
View File

@ -5,6 +5,7 @@ ass well as vast number of helper functions for pydantic, sqlalchemy and relatio
"""
from ormar.models.newbasemodel import NewBaseModel # noqa I100
from ormar.models.model_row import ModelRow # noqa I100
from ormar.models.model import Model # noqa I100
__all__ = ["NewBaseModel", "Model"]
__all__ = ["NewBaseModel", "Model", "ModelRow"]

36
ormar/models/helpers/relations.py
View File

@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Type
import ormar
from ormar import ForeignKey, ManyToMany
from ormar.fields import ManyToManyField
from ormar.fields import ManyToManyField, Through, ThroughField
from ormar.fields.foreign_key import ForeignKeyField
from ormar.models.helpers.sqlalchemy import adjust_through_many_to_many_model
from ormar.relations import AliasManager
@ -81,7 +81,8 @@ def expand_reverse_relationships(model: Type["Model"]) -> None:
:param model: model on which relation should be checked and registered
:type model: Model class
"""
for model_field in model.Meta.model_fields.values():
model_fields = list(model.Meta.model_fields.values())
for model_field in model_fields:
if (
issubclass(model_field, ForeignKeyField)
and not model_field.has_unresolved_forward_refs()
@ -113,6 +114,7 @@ def register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None:
self_reference_primary=model_field.self_reference_primary,
)
# register foreign keys on through model
register_through_shortcut_fields(model_field=model_field)
adjust_through_many_to_many_model(model_field=model_field)
else:
model_field.to.Meta.model_fields[related_name] = ForeignKey(
@ -125,6 +127,34 @@ def register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None:
)
def register_through_shortcut_fields(model_field: Type["ManyToManyField"]) -> None:
"""
Registers m2m relation through shortcut on both ends of the relation.
:param model_field: relation field defined in parent model
:type model_field: ManyToManyField
"""
through_model = model_field.through
through_name = through_model.get_name(lower=True)
related_name = model_field.get_related_name()
model_field.owner.Meta.model_fields[through_name] = Through(
through_model,
real_name=through_name,
virtual=True,
related_name=model_field.name,
owner=model_field.owner,
)
model_field.to.Meta.model_fields[through_name] = Through(
through_model,
real_name=through_name,
virtual=True,
related_name=related_name,
owner=model_field.to,
)
def register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None:
"""
Registers the relation (and reverse relation) in alias manager.
@ -142,7 +172,7 @@ def register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None:
if field.has_unresolved_forward_refs():
return
register_many_to_many_relation_on_build(field=field)
elif issubclass(field, ForeignKeyField):
elif issubclass(field, ForeignKeyField) and not issubclass(field, ThroughField):
if field.has_unresolved_forward_refs():
return
register_relation_on_build(field=field)

154
ormar/models/metaclass.py
View File

@ -46,6 +46,7 @@ if TYPE_CHECKING: # pragma no cover
from ormar import Model
CONFIG_KEY = "Config"
PARSED_FIELDS_KEY = "__parsed_fields__"
class ModelMeta:
@ -141,83 +142,6 @@ def register_signals(new_model: Type["Model"]) -> None: # noqa: CCR001
new_model.Meta.signals = signals
class ModelMetaclass(pydantic.main.ModelMetaclass):
def __new__( # type: ignore # noqa: CCR001
mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict
) -> "ModelMetaclass":
"""
Metaclass used by ormar Models that performs configuration
and build of ormar Models.
Sets pydantic configuration.
Extract model_fields and convert them to pydantic FieldInfo,
updates class namespace.
Extracts settings and fields from parent classes.
Fetches methods decorated with @property_field decorator
to expose them later in dict().
Construct parent pydantic Metaclass/ Model.
If class has Meta class declared (so actual ormar Models) it also:
* populate sqlalchemy columns, pkname and tables from model_fields
* register reverse relationships on related models
* registers all relations in alias manager that populates table_prefixes
* exposes alias manager on each Model
* creates QuerySet for each model and exposes it on a class
:param name: name of current class
:type name: str
:param bases: base classes
:type bases: Tuple
:param attrs: class namespace
:type attrs: Dict
"""
attrs["Config"] = get_pydantic_base_orm_config()
attrs["__name__"] = name
attrs, model_fields = extract_annotations_and_default_vals(attrs)
for base in reversed(bases):
mod = base.__module__
if mod.startswith("ormar.models.") or mod.startswith("pydantic."):
continue
attrs, model_fields = extract_from_parents_definition(
base_class=base, curr_class=mcs, attrs=attrs, model_fields=model_fields
)
new_model = super().__new__( # type: ignore
mcs, name, bases, attrs
)
add_cached_properties(new_model)
if hasattr(new_model, "Meta"):
populate_default_options_values(new_model, model_fields)
check_required_meta_parameters(new_model)
add_property_fields(new_model, attrs)
register_signals(new_model=new_model)
populate_choices_validators(new_model)
if not new_model.Meta.abstract:
new_model = populate_meta_tablename_columns_and_pk(name, new_model)
populate_meta_sqlalchemy_table_if_required(new_model.Meta)
expand_reverse_relationships(new_model)
for field in new_model.Meta.model_fields.values():
register_relation_in_alias_manager(field=field)
if new_model.Meta.pkname not in attrs["__annotations__"]:
field_name = new_model.Meta.pkname
attrs["__annotations__"][field_name] = Optional[int] # type: ignore
attrs[field_name] = None
new_model.__fields__[field_name] = get_pydantic_field(
field_name=field_name, model=new_model
)
new_model.Meta.alias_manager = alias_manager
new_model.objects = QuerySet(new_model)
return new_model
def verify_constraint_names(
base_class: "Model", model_fields: Dict, parent_value: List
) -> None:
@ -539,4 +463,78 @@ def update_attrs_and_fields(
return updated_model_fields
PARSED_FIELDS_KEY = "__parsed_fields__"
class ModelMetaclass(pydantic.main.ModelMetaclass):
def __new__( # type: ignore # noqa: CCR001
mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict
) -> "ModelMetaclass":
"""
Metaclass used by ormar Models that performs configuration
and build of ormar Models.
Sets pydantic configuration.
Extract model_fields and convert them to pydantic FieldInfo,
updates class namespace.
Extracts settings and fields from parent classes.
Fetches methods decorated with @property_field decorator
to expose them later in dict().
Construct parent pydantic Metaclass/ Model.
If class has Meta class declared (so actual ormar Models) it also:
* populate sqlalchemy columns, pkname and tables from model_fields
* register reverse relationships on related models
* registers all relations in alias manager that populates table_prefixes
* exposes alias manager on each Model
* creates QuerySet for each model and exposes it on a class
:param name: name of current class
:type name: str
:param bases: base classes
:type bases: Tuple
:param attrs: class namespace
:type attrs: Dict
"""
attrs["Config"] = get_pydantic_base_orm_config()
attrs["__name__"] = name
attrs, model_fields = extract_annotations_and_default_vals(attrs)
for base in reversed(bases):
mod = base.__module__
if mod.startswith("ormar.models.") or mod.startswith("pydantic."):
continue
attrs, model_fields = extract_from_parents_definition(
base_class=base, curr_class=mcs, attrs=attrs, model_fields=model_fields
)
new_model = super().__new__( # type: ignore
mcs, name, bases, attrs
)
add_cached_properties(new_model)
if hasattr(new_model, "Meta"):
populate_default_options_values(new_model, model_fields)
check_required_meta_parameters(new_model)
add_property_fields(new_model, attrs)
register_signals(new_model=new_model)
populate_choices_validators(new_model)
if not new_model.Meta.abstract:
new_model = populate_meta_tablename_columns_and_pk(name, new_model)
populate_meta_sqlalchemy_table_if_required(new_model.Meta)
expand_reverse_relationships(new_model)
for field in new_model.Meta.model_fields.values():
register_relation_in_alias_manager(field=field)
if new_model.Meta.pkname not in attrs["__annotations__"]:
field_name = new_model.Meta.pkname
attrs["__annotations__"][field_name] = Optional[int] # type: ignore
attrs[field_name] = None
new_model.__fields__[field_name] = get_pydantic_field(
field_name=field_name, model=new_model
)
new_model.Meta.alias_manager = alias_manager
new_model.objects = QuerySet(new_model)
return new_model

3
ormar/models/mixins/excludable_mixin.py
View File

@ -31,6 +31,7 @@ class ExcludableMixin(RelationMixin):
if TYPE_CHECKING: # pragma: no cover
from ormar import Model
from ormar.models import ModelRow
@staticmethod
def get_child(
@ -157,7 +158,7 @@ class ExcludableMixin(RelationMixin):
@classmethod
def own_table_columns(
cls,
model: Type["Model"],
model: Union[Type["Model"], Type["ModelRow"]],
fields: Optional[Union[Set, Dict]],
exclude_fields: Optional[Union[Set, Dict]],
use_alias: bool = False,

29
ormar/models/mixins/relation_mixin.py
View File

@ -1,6 +1,8 @@
import inspect
from typing import List, Optional, Set, TYPE_CHECKING
from ormar import ManyToManyField
from ormar.fields import ThroughField
from ormar.fields.foreign_key import ForeignKeyField
@ -43,27 +45,46 @@ class RelationMixin:
return cls._related_fields
related_fields = []
for name in cls.extract_related_names():
for name in cls.extract_related_names().union(cls.extract_through_names()):
related_fields.append(cls.Meta.model_fields[name])
cls._related_fields = related_fields
return related_fields
@classmethod
def extract_through_names(cls) -> Set:
"""
Extracts related fields through names which are shortcuts to through models.
:return: set of related through fields names
:rtype: Set
"""
related_fields = set()
for name in cls.extract_related_names():
field = cls.Meta.model_fields[name]
if issubclass(field, ManyToManyField):
related_fields.add(field.through.get_name(lower=True))
return related_fields
@classmethod
def extract_related_names(cls) -> Set:
"""
Returns List of fields names for all relations declared on a model.
List is cached in cls._related_names for quicker access.
:return: list of related fields names
:rtype: List
:return: set of related fields names
:rtype: Set
"""
if isinstance(cls._related_names, Set):
return cls._related_names
related_names = set()
for name, field in cls.Meta.model_fields.items():
if inspect.isclass(field) and issubclass(field, ForeignKeyField):
if (
inspect.isclass(field)
and issubclass(field, ForeignKeyField)
and not issubclass(field, ThroughField)
):
related_names.add(name)
cls._related_names = related_names

252
ormar/models/model.py
View File

@ -1,24 +1,17 @@
from typing import (
Any,
Dict,
List,
Optional,
Set,
TYPE_CHECKING,
Tuple,
Type,
TypeVar,
Union,
)
import sqlalchemy
import ormar.queryset # noqa I100
from ormar.exceptions import ModelPersistenceError, NoMatch
from ormar.fields.many_to_many import ManyToManyField
from ormar.models import NewBaseModel # noqa I100
from ormar.models.helpers.models import group_related_list
from ormar.models.metaclass import ModelMeta
from ormar.models.model_row import ModelRow
if TYPE_CHECKING: # pragma nocover
from ormar import QuerySet
@ -26,7 +19,7 @@ if TYPE_CHECKING: # pragma nocover
T = TypeVar("T", bound="Model")
class Model(NewBaseModel):
class Model(ModelRow):
__abstract__ = False
if TYPE_CHECKING: # pragma nocover
Meta: ModelMeta
@ -36,247 +29,6 @@ class Model(NewBaseModel):
_repr = {k: getattr(self, k) for k, v in self.Meta.model_fields.items()}
return f"{self.__class__.__name__}({str(_repr)})"
@classmethod
def from_row( # noqa CCR001
cls: Type[T],
row: sqlalchemy.engine.ResultProxy,
select_related: List = None,
related_models: Any = None,
previous_model: Type[T] = None,
source_model: Type[T] = None,
related_name: str = None,
fields: Optional[Union[Dict, Set]] = None,
exclude_fields: Optional[Union[Dict, Set]] = None,
current_relation_str: str = None,
) -> Optional[T]:
"""
Model method to convert raw sql row from database into ormar.Model instance.
Traverses nested models if they were specified in select_related for query.
Called recurrently and returns model instance if it's present in the row.
Note that it's processing one row at a time, so if there are duplicates of
parent row that needs to be joined/combined
(like parent row in sql join with 2+ child rows)
instances populated in this method are later combined in the QuerySet.
Other method working directly on raw database results is in prefetch_query,
where rows are populated in a different way as they do not have
nested models in result.
:param current_relation_str: name of the relation field
:type current_relation_str: str
:param source_model: model on which relation was defined
:type source_model: Type[Model]
:param row: raw result row from the database
:type row: sqlalchemy.engine.result.ResultProxy
:param select_related: list of names of related models fetched from database
:type select_related: List
:param related_models: list or dict of related models
:type related_models: Union[List, Dict]
:param previous_model: internal param for nested models to specify table_prefix
:type previous_model: Model class
:param related_name: internal parameter - name of current nested model
:type related_name: str
:param fields: fields and related model fields to include
if provided only those are included
:type fields: Optional[Union[Dict, Set]]
:param exclude_fields: fields and related model fields to exclude
excludes the fields even if they are provided in fields
:type exclude_fields: Optional[Union[Dict, Set]]
:return: returns model if model is populated from database
:rtype: Optional[Model]
"""
item: Dict[str, Any] = {}
select_related = select_related or []
related_models = related_models or []
table_prefix = ""
if select_related:
source_model = cls
related_models = group_related_list(select_related)
rel_name2 = related_name
if (
previous_model
and related_name
and issubclass(
previous_model.Meta.model_fields[related_name], ManyToManyField
)
):
through_field = previous_model.Meta.model_fields[related_name]
if (
through_field.self_reference
and related_name == through_field.self_reference_primary
):
rel_name2 = through_field.default_source_field_name() # type: ignore
else:
rel_name2 = through_field.default_target_field_name() # type: ignore
previous_model = through_field.through # type: ignore
if previous_model and rel_name2:
if current_relation_str and "__" in current_relation_str and source_model:
table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
from_model=source_model, relation_name=current_relation_str
)
if not table_prefix:
table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
from_model=previous_model, relation_name=rel_name2
)
item = cls.populate_nested_models_from_row(
item=item,
row=row,
related_models=related_models,
fields=fields,
exclude_fields=exclude_fields,
current_relation_str=current_relation_str,
source_model=source_model,
)
item = cls.extract_prefixed_table_columns(
item=item,
row=row,
table_prefix=table_prefix,
fields=fields,
exclude_fields=exclude_fields,
)
instance: Optional[T] = None
if item.get(cls.Meta.pkname, None) is not None:
item["__excluded__"] = cls.get_names_to_exclude(
fields=fields, exclude_fields=exclude_fields
)
instance = cls(**item)
instance.set_save_status(True)
return instance
@classmethod
def populate_nested_models_from_row( # noqa: CFQ002
cls,
item: dict,
row: sqlalchemy.engine.ResultProxy,
related_models: Any,
fields: Optional[Union[Dict, Set]] = None,
exclude_fields: Optional[Union[Dict, Set]] = None,
current_relation_str: str = None,
source_model: Type[T] = None,
) -> dict:
"""
Traverses structure of related models and populates the nested models
from the database row.
Related models can be a list if only directly related models are to be
populated, converted to dict if related models also have their own related
models to be populated.
Recurrently calls from_row method on nested instances and create nested
instances. In the end those instances are added to the final model dictionary.
:param source_model: source model from which relation started
:type source_model: Type[Model]
:param current_relation_str: joined related parts into one string
:type current_relation_str: str
:param item: dictionary of already populated nested models, otherwise empty dict
:type item: Dict
:param row: raw result row from the database
:type row: sqlalchemy.engine.result.ResultProxy
:param related_models: list or dict of related models
:type related_models: Union[Dict, List]
:param fields: fields and related model fields to include -
if provided only those are included
:type fields: Optional[Union[Dict, Set]]
:param exclude_fields: fields and related model fields to exclude
excludes the fields even if they are provided in fields
:type exclude_fields: Optional[Union[Dict, Set]]
:return: dictionary with keys corresponding to model fields names
and values are database values
:rtype: Dict
"""
for related in related_models:
relation_str = (
"__".join([current_relation_str, related])
if current_relation_str
else related
)
fields = cls.get_included(fields, related)
exclude_fields = cls.get_excluded(exclude_fields, related)
model_cls = cls.Meta.model_fields[related].to
remainder = None
if isinstance(related_models, dict) and related_models[related]:
remainder = related_models[related]
child = model_cls.from_row(
row,
related_models=remainder,
previous_model=cls,
related_name=related,
fields=fields,
exclude_fields=exclude_fields,
current_relation_str=relation_str,
source_model=source_model,
)
item[model_cls.get_column_name_from_alias(related)] = child
return item
@classmethod
def extract_prefixed_table_columns( # noqa CCR001
cls,
item: dict,
row: sqlalchemy.engine.result.ResultProxy,
table_prefix: str,
fields: Optional[Union[Dict, Set]] = None,
exclude_fields: Optional[Union[Dict, Set]] = None,
) -> dict:
"""
Extracts own fields from raw sql result, using a given prefix.
Prefix changes depending on the table's position in a join.
If the table is a main table, there is no prefix.
All joined tables have prefixes to allow duplicate column names,
as well as duplicated joins to the same table from multiple different tables.
Extracted fields populates the related dict later used to construct a Model.
Used in Model.from_row and PrefetchQuery._populate_rows methods.
:param item: dictionary of already populated nested models, otherwise empty dict
:type item: Dict
:param row: raw result row from the database
:type row: sqlalchemy.engine.result.ResultProxy
:param table_prefix: prefix of the table from AliasManager
each pair of tables have own prefix (two of them depending on direction) -
used in joins to allow multiple joins to the same table.
:type table_prefix: str
:param fields: fields and related model fields to include -
if provided only those are included
:type fields: Optional[Union[Dict, Set]]
:param exclude_fields: fields and related model fields to exclude
excludes the fields even if they are provided in fields
:type exclude_fields: Optional[Union[Dict, Set]]
:return: dictionary with keys corresponding to model fields names
and values are database values
:rtype: Dict
"""
# databases does not keep aliases in Record for postgres, change to raw row
source = row._row if cls.db_backend_name() == "postgresql" else row
selected_columns = cls.own_table_columns(
model=cls,
fields=fields or {},
exclude_fields=exclude_fields or {},
use_alias=False,
)
for column in cls.Meta.table.columns:
alias = cls.get_column_name_from_alias(column.name)
if alias not in item and alias in selected_columns:
prefixed_name = (
f'{table_prefix + "_" if table_prefix else ""}{column.name}'
)
item[alias] = source[prefixed_name]
return item
async def upsert(self: T, **kwargs: Any) -> T:
"""
Performs either a save or an update depending on the presence of the pk.

303
ormar/models/model_row.py Normal file
View File

@ -0,0 +1,303 @@
from typing import (
Any,
Dict,
List,
Optional,
Set,
Type,
TypeVar,
Union,
)
import sqlalchemy
from ormar import ManyToManyField # noqa: I202
from ormar.models import NewBaseModel
from ormar.models.helpers.models import group_related_list
T = TypeVar("T", bound="ModelRow")
class ModelRow(NewBaseModel):
@classmethod
def from_row( # noqa CCR001
cls: Type[T],
row: sqlalchemy.engine.ResultProxy,
select_related: List = None,
related_models: Any = None,
previous_model: Type[T] = None,
source_model: Type[T] = None,
related_name: str = None,
fields: Optional[Union[Dict, Set]] = None,
exclude_fields: Optional[Union[Dict, Set]] = None,
current_relation_str: str = None,
) -> Optional[T]:
"""
Model method to convert raw sql row from database into ormar.Model instance.
Traverses nested models if they were specified in select_related for query.
Called recurrently and returns model instance if it's present in the row.
Note that it's processing one row at a time, so if there are duplicates of
parent row that needs to be joined/combined
(like parent row in sql join with 2+ child rows)
instances populated in this method are later combined in the QuerySet.
Other method working directly on raw database results is in prefetch_query,
where rows are populated in a different way as they do not have
nested models in result.
:param current_relation_str: name of the relation field
:type current_relation_str: str
:param source_model: model on which relation was defined
:type source_model: Type[Model]
:param row: raw result row from the database
:type row: sqlalchemy.engine.result.ResultProxy
:param select_related: list of names of related models fetched from database
:type select_related: List
:param related_models: list or dict of related models
:type related_models: Union[List, Dict]
:param previous_model: internal param for nested models to specify table_prefix
:type previous_model: Model class
:param related_name: internal parameter - name of current nested model
:type related_name: str
:param fields: fields and related model fields to include
if provided only those are included
:type fields: Optional[Union[Dict, Set]]
:param exclude_fields: fields and related model fields to exclude
excludes the fields even if they are provided in fields
:type exclude_fields: Optional[Union[Dict, Set]]
:return: returns model if model is populated from database
:rtype: Optional[Model]
"""
item: Dict[str, Any] = {}
select_related = select_related or []
related_models = related_models or []
table_prefix = ""
if select_related:
source_model = cls
related_models = group_related_list(select_related)
rel_name2 = related_name
# TODO: refactor this into field classes?
if (
previous_model
and related_name
and issubclass(
previous_model.Meta.model_fields[related_name], ManyToManyField
)
):
through_field = previous_model.Meta.model_fields[related_name]
if (
through_field.self_reference
and related_name == through_field.self_reference_primary
):
rel_name2 = through_field.default_source_field_name() # type: ignore
else:
rel_name2 = through_field.default_target_field_name() # type: ignore
previous_model = through_field.through # type: ignore
if previous_model and rel_name2:
if current_relation_str and "__" in current_relation_str and source_model:
table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
from_model=source_model, relation_name=current_relation_str
)
if not table_prefix:
table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
from_model=previous_model, relation_name=rel_name2
)
item = cls.populate_nested_models_from_row(
item=item,
row=row,
related_models=related_models,
fields=fields,
exclude_fields=exclude_fields,
current_relation_str=current_relation_str,
source_model=source_model,
)
item = cls.extract_prefixed_table_columns(
item=item,
row=row,
table_prefix=table_prefix,
fields=fields,
exclude_fields=exclude_fields,
)
instance: Optional[T] = None
if item.get(cls.Meta.pkname, None) is not None:
item["__excluded__"] = cls.get_names_to_exclude(
fields=fields, exclude_fields=exclude_fields
)
instance = cls(**item)
instance.set_save_status(True)
return instance
@classmethod
def populate_nested_models_from_row( # noqa: CFQ002
cls,
item: dict,
row: sqlalchemy.engine.ResultProxy,
related_models: Any,
fields: Optional[Union[Dict, Set]] = None,
exclude_fields: Optional[Union[Dict, Set]] = None,
current_relation_str: str = None,
source_model: Type[T] = None,
) -> dict:
"""
Traverses structure of related models and populates the nested models
from the database row.
Related models can be a list if only directly related models are to be
populated, converted to dict if related models also have their own related
models to be populated.
Recurrently calls from_row method on nested instances and create nested
instances. In the end those instances are added to the final model dictionary.
:param source_model: source model from which relation started
:type source_model: Type[Model]
:param current_relation_str: joined related parts into one string
:type current_relation_str: str
:param item: dictionary of already populated nested models, otherwise empty dict
:type item: Dict
:param row: raw result row from the database
:type row: sqlalchemy.engine.result.ResultProxy
:param related_models: list or dict of related models
:type related_models: Union[Dict, List]
:param fields: fields and related model fields to include -
if provided only those are included
:type fields: Optional[Union[Dict, Set]]
:param exclude_fields: fields and related model fields to exclude
excludes the fields even if they are provided in fields
:type exclude_fields: Optional[Union[Dict, Set]]
:return: dictionary with keys corresponding to model fields names
and values are database values
:rtype: Dict
"""
for related in related_models:
relation_str = (
"__".join([current_relation_str, related])
if current_relation_str
else related
)
field = cls.Meta.model_fields[related]
fields = cls.get_included(fields, related)
exclude_fields = cls.get_excluded(exclude_fields, related)
model_cls = field.to
remainder = None
if isinstance(related_models, dict) and related_models[related]:
remainder = related_models[related]
child = model_cls.from_row(
row,
related_models=remainder,
previous_model=cls,
related_name=related,
fields=fields,
exclude_fields=exclude_fields,
current_relation_str=relation_str,
source_model=source_model,
)
item[model_cls.get_column_name_from_alias(related)] = child
if issubclass(field, ManyToManyField) and child:
# TODO: way to figure out which side should be populated?
through_name = cls.Meta.model_fields[related].through.get_name()
# for now it's nested dict, should be instance?
through_child = cls.populate_through_instance(
row=row,
related=related,
through_name=through_name,
fields=fields,
exclude_fields=exclude_fields,
)
item[through_name] = through_child
setattr(child, through_name, through_child)
child.set_save_status(True)
return item
@classmethod
def populate_through_instance(
cls,
row: sqlalchemy.engine.ResultProxy,
through_name: str,
related: str,
fields: Optional[Union[Dict, Set]] = None,
exclude_fields: Optional[Union[Dict, Set]] = None,
) -> Dict:
# TODO: fix excludes and includes
fields = cls.get_included(fields, through_name)
# exclude_fields = cls.get_excluded(exclude_fields, through_name)
model_cls = cls.Meta.model_fields[through_name].to
exclude_fields = model_cls.extract_related_names()
table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
from_model=cls, relation_name=related
)
child = model_cls.extract_prefixed_table_columns(
item={},
row=row,
table_prefix=table_prefix,
fields=fields,
exclude_fields=exclude_fields,
)
return child
@classmethod
def extract_prefixed_table_columns( # noqa CCR001
cls,
item: dict,
row: sqlalchemy.engine.result.ResultProxy,
table_prefix: str,
fields: Optional[Union[Dict, Set]] = None,
exclude_fields: Optional[Union[Dict, Set]] = None,
) -> dict:
"""
Extracts own fields from raw sql result, using a given prefix.
Prefix changes depending on the table's position in a join.
If the table is a main table, there is no prefix.
All joined tables have prefixes to allow duplicate column names,
as well as duplicated joins to the same table from multiple different tables.
Extracted fields populates the related dict later used to construct a Model.
Used in Model.from_row and PrefetchQuery._populate_rows methods.
:param item: dictionary of already populated nested models, otherwise empty dict
:type item: Dict
:param row: raw result row from the database
:type row: sqlalchemy.engine.result.ResultProxy
:param table_prefix: prefix of the table from AliasManager
each pair of tables have own prefix (two of them depending on direction) -
used in joins to allow multiple joins to the same table.
:type table_prefix: str
:param fields: fields and related model fields to include -
if provided only those are included
:type fields: Optional[Union[Dict, Set]]
:param exclude_fields: fields and related model fields to exclude
excludes the fields even if they are provided in fields
:type exclude_fields: Optional[Union[Dict, Set]]
:return: dictionary with keys corresponding to model fields names
and values are database values
:rtype: Dict
"""
# databases does not keep aliases in Record for postgres, change to raw row
source = row._row if cls.db_backend_name() == "postgresql" else row
selected_columns = cls.own_table_columns(
model=cls,
fields=fields or {},
exclude_fields=exclude_fields or {},
use_alias=False,
)
for column in cls.Meta.table.columns:
alias = cls.get_column_name_from_alias(column.name)
if alias not in item and alias in selected_columns:
prefixed_name = (
f'{table_prefix + "_" if table_prefix else ""}{column.name}'
)
item[alias] = source[prefixed_name]
return item

6
ormar/models/newbasemodel.py
View File

@ -172,7 +172,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
object.__setattr__(self, "__fields_set__", fields_set)
# register the columns models after initialization
for related in self.extract_related_names():
for related in self.extract_related_names().union(self.extract_through_names()):
self.Meta.model_fields[related].expand_relationship(
new_kwargs.get(related), self, to_register=True,
)
@ -267,6 +267,10 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
return object.__getattribute__(
self, "_extract_related_model_instead_of_field"
)(item)
if item in object.__getattribute__(self, "extract_through_names")():
return object.__getattribute__(
self, "_extract_related_model_instead_of_field"
)(item)
if item in object.__getattribute__(self, "Meta").property_fields:
value = object.__getattribute__(self, item)
return value() if callable(value) else value

2
ormar/models/quick_access_views.py
View File

@ -34,10 +34,12 @@ quick_access_set = {
"_skip_ellipsis",
"_update_and_follow",
"_update_excluded_with_related_not_required",
"_verify_model_can_be_initialized",
"copy",
"delete",
"dict",
"extract_related_names",
"extract_through_names",
"update_from_dict",
"get_column_alias",
"get_column_name_from_alias",

20
ormar/queryset/join.py
View File

@ -291,6 +291,8 @@ class SqlJoin:
self.get_order_bys(
to_table=to_table, pkname_alias=pkname_alias,
)
else:
self.select_through_model_fields()
self_related_fields = self.next_model.own_table_columns(
model=self.next_model,
@ -305,6 +307,24 @@ class SqlJoin:
)
self.used_aliases.append(self.next_alias)
def select_through_model_fields(self) -> None:
# TODO: add docstring
next_alias = self.alias_manager.resolve_relation_alias(
from_model=self.target_field.owner, relation_name=self.relation_name
)
# TODO: fix fields and exclusions
self_related_fields = self.target_field.through.own_table_columns(
model=self.target_field.through,
fields=None,
exclude_fields=self.target_field.through.extract_related_names(),
use_alias=True,
)
self.columns.extend(
self.alias_manager.prefixed_columns(
next_alias, self.target_field.through.Meta.table, self_related_fields
)
)
def _replace_many_to_many_order_by_columns(self, part: str, new_part: str) -> None:
"""
Substitutes the name of the relation with actual model name in m2m order bys.

2
ormar/queryset/queryset.py
View File

@ -197,7 +197,7 @@ class QuerySet:
limit_raw_sql=self.limit_sql_raw,
)
exp = qry.build_select_expression()
# print("\n", exp.compile(compile_kwargs={"literal_binds": True}))
print("\n", exp.compile(compile_kwargs={"literal_binds": True}))
return exp
def filter(self, _exclude: bool = False, **kwargs: Any) -> "QuerySet": # noqa: A003

5
ormar/relations/alias_manager.py
View File

@ -1,13 +1,14 @@
import string
import uuid
from random import choices
from typing import Any, Dict, List, TYPE_CHECKING, Type
from typing import Any, Dict, List, TYPE_CHECKING, Type, Union
import sqlalchemy
from sqlalchemy import text
if TYPE_CHECKING: # pragma: no cover
from ormar import Model
from ormar.models import ModelRow
def get_table_alias() -> str:
@ -133,7 +134,7 @@ class AliasManager:
return alias
def resolve_relation_alias(
self, from_model: Type["Model"], relation_name: str
self, from_model: Union[Type["Model"], Type["ModelRow"]], relation_name: str
) -> str:
"""
Given model and relation name returns the alias for this relation.

17
ormar/relations/querysetproxy.py
View File

@ -44,6 +44,11 @@ class QuerysetProxy(ormar.QuerySetProtocol):
].get_related_name()
self.related_field = self.relation.to.Meta.model_fields[self.related_field_name]
self.owner_pk_value = self._owner.pk
self.through_model_name = (
self.related_field.through.get_name()
if self.type_ == ormar.RelationType.MULTIPLE
else None
)
@property
def queryset(self) -> "QuerySet":
@ -99,17 +104,20 @@ class QuerysetProxy(ormar.QuerySetProtocol):
for item in self.relation.related_models[:]:
self.relation.remove(item)
async def create_through_instance(self, child: "T") -> None:
async def create_through_instance(self, child: "T", **kwargs: Any) -> None:
"""
Crete a through model instance in the database for m2m relations.
:param kwargs: dict of additional keyword arguments for through instance
:type kwargs: Any
:param child: child model instance
:type child: Model
"""
model_cls = self.relation.through
owner_column = self.related_field.default_target_field_name() # type: ignore
child_column = self.related_field.default_source_field_name() # type: ignore
kwargs = {owner_column: self._owner.pk, child_column: child.pk}
rel_kwargs = {owner_column: self._owner.pk, child_column: child.pk}
final_kwargs = {**rel_kwargs, **kwargs}
if child.pk is None:
raise ModelPersistenceError(
f"You cannot save {child.get_name()} "
@ -117,7 +125,7 @@ class QuerysetProxy(ormar.QuerySetProtocol):
f"Save the child model first."
)
expr = model_cls.Meta.table.insert()
expr = expr.values(**kwargs)
expr = expr.values(**final_kwargs)
# print("\n", expr.compile(compile_kwargs={"literal_binds": True}))
await model_cls.Meta.database.execute(expr)
@ -270,12 +278,13 @@ class QuerysetProxy(ormar.QuerySetProtocol):
:return: created model
:rtype: Model
"""
through_kwargs = kwargs.pop(self.through_model_name, {})
if self.type_ == ormar.RelationType.REVERSE:
kwargs[self.related_field.name] = self._owner
created = await self.queryset.create(**kwargs)
self._register_related(created)
if self.type_ == ormar.RelationType.MULTIPLE:
await self.create_through_instance(created)
await self.create_through_instance(created, **through_kwargs)
return created
async def get_or_create(self, **kwargs: Any) -> "Model":

3
ormar/relations/relation.py
View File

@ -26,6 +26,7 @@ class RelationType(Enum):
PRIMARY = 1
REVERSE = 2
MULTIPLE = 3
THROUGH = 4
class Relation:
@ -128,7 +129,7 @@ class Relation:
:type child: Model
"""
relation_name = self.field_name
if self._type == RelationType.PRIMARY:
if self._type in (RelationType.PRIMARY, RelationType.THROUGH):
self.related_models = child
self._owner.__dict__[relation_name] = child
else:

4
ormar/relations/relation_manager.py
View File

@ -1,7 +1,7 @@
from typing import Dict, List, Optional, Sequence, TYPE_CHECKING, Type, TypeVar, Union
from weakref import proxy
from ormar.fields import BaseField
from ormar.fields import BaseField, ThroughField
from ormar.fields.foreign_key import ForeignKeyField
from ormar.fields.many_to_many import ManyToManyField
from ormar.relations.relation import Relation, RelationType
@ -42,6 +42,8 @@ class RelationsManager:
"""
if issubclass(field, ManyToManyField):
return RelationType.MULTIPLE
if issubclass(field, ThroughField):
return RelationType.THROUGH
return RelationType.PRIMARY if not field.virtual else RelationType.REVERSE
def _add_relation(self, field: Type[BaseField]) -> None:

6
ormar/relations/relation_proxy.py
View File

@ -163,19 +163,21 @@ class RelationProxy(list):
else:
await item.delete()
async def add(self, item: "Model") -> None:
async def add(self, item: "Model", **kwargs: Any) -> None:
"""
Adds child model to relation.
For ManyToMany relations through instance is automatically created.
:param kwargs: dict of additional keyword arguments for through instance
:type kwargs: Any
:param item: child to add to relation
:type item: Model
"""
relation_name = self.related_field_name
self._check_if_model_saved()
if self.type_ == ormar.RelationType.MULTIPLE:
await self.queryset_proxy.create_through_instance(item)
await self.queryset_proxy.create_through_instance(item, **kwargs)
setattr(item, relation_name, self._owner)
else:
setattr(item, relation_name, self._owner)

130
tests/test_m2m_through_fields.py
View File

@ -1,6 +1,7 @@
import databases
import pytest
import sqlalchemy
from pydantic.typing import ForwardRef
import ormar
from tests.settings import DATABASE_URL
@ -39,28 +40,107 @@ class Post(ormar.Model):
categories = ormar.ManyToMany(Category, through=PostCategory)
#
# @pytest.fixture(autouse=True, scope="module")
# async def create_test_database():
# engine = sqlalchemy.create_engine(DATABASE_URL)
# metadata.create_all(engine)
# yield
# metadata.drop_all(engine)
#
#
# @pytest.mark.asyncio
# async def test_setting_fields_on_through_model():
# async with database:
# # TODO: check/ modify following
# # loading the data into model instance of though model?
# # <- attach to other side? both sides? access by through, or add to fields?
# # creating while adding to relation (kwargs in add?)
# # creating in query (dividing kwargs between final and through)
# # updating in query
# # sorting in filter (special __through__<field_name> notation?)
# # ordering by in order_by
# # accessing from instance (both sides?)
# # modifying from instance (both sides?)
# # including/excluding in fields?
# # allowing to change fk fields names in through model?
# pass
@pytest.fixture(autouse=True, scope="module")
def create_test_database():
engine = sqlalchemy.create_engine(DATABASE_URL)
metadata.drop_all(engine)
metadata.create_all(engine)
yield
metadata.drop_all(engine)
class PostCategory2(ormar.Model):
class Meta(BaseMeta):
tablename = "posts_x_categories2"
id: int = ormar.Integer(primary_key=True)
sort_order: int = ormar.Integer(nullable=True)
@pytest.mark.asyncio
async def test_forward_ref_is_updated():
async with database:
class Post2(ormar.Model):
class Meta(BaseMeta):
pass
id: int = ormar.Integer(primary_key=True)
title: str = ormar.String(max_length=200)
categories = ormar.ManyToMany(Category, through=ForwardRef("PostCategory2"))
assert Post2.Meta.requires_ref_update
Post2.update_forward_refs()
assert Post2.Meta.model_fields["postcategory2"].to == PostCategory2
@pytest.mark.asyncio
async def test_setting_fields_on_through_model():
async with database:
post = await Post(title="Test post").save()
category = await Category(name="Test category").save()
await post.categories.add(category)
assert hasattr(post.categories[0], "postcategory")
assert post.categories[0].postcategory is None
@pytest.mark.asyncio
async def test_setting_additional_fields_on_through_model_in_add():
async with database:
post = await Post(title="Test post").save()
category = await Category(name="Test category").save()
await post.categories.add(category, sort_order=1)
postcat = await PostCategory.objects.get()
assert postcat.sort_order == 1
@pytest.mark.asyncio
async def test_setting_additional_fields_on_through_model_in_create():
async with database:
post = await Post(title="Test post").save()
await post.categories.create(
name="Test category2", postcategory={"sort_order": 2}
)
postcat = await PostCategory.objects.get()
assert postcat.sort_order == 2
@pytest.mark.asyncio
async def test_getting_additional_fields_from_queryset():
async with database:
post = await Post(title="Test post").save()
await post.categories.create(
name="Test category1", postcategory={"sort_order": 1}
)
await post.categories.create(
name="Test category2", postcategory={"sort_order": 2}
)
await post.categories.all()
assert post.categories[0].postcategory.sort_order == 1
assert post.categories[1].postcategory.sort_order == 2
post = await Post.objects.select_related("categories").get(
categories__name="Test category2"
)
assert post.categories[0].postcategory.sort_order == 2
# TODO: check/ modify following
# add to fields with class lower name (V)
# forward refs update (V)
# creating while adding to relation (kwargs in add) (V)
# creating in queryset proxy (dict with through name and kwargs) (V)
# loading the data into model instance of though model (V) <- fix fields ane exclude
# accessing from instance (V) <- no both sides only nested one is relevant, fix one side
# updating in query
# sorting in filter (special __through__<field_name> notation?)
# ordering by in order_by
# modifying from instance (both sides?)
# including/excluding in fields?
# allowing to change fk fields names in through model?
# make through optional? auto-generated for cases other fields are missing?