Merge pull request #121 from collerek/m2m_fields

Add through fields, load_all() method and make through models optional

.gitignore

@@ -13,3 +13,4 @@ dist
 site
 profile.py
 *.db
+*.db-journal

README.md

@@ -306,7 +306,7 @@ async def joins():
     # visit: https://collerek.github.io/ormar/relations/
 
     # to read more about joins and subqueries
-    # visit: https://collerek.github.io/ormar/queries/delete/
+    # visit: https://collerek.github.io/ormar/queries/joins-and-subqueries/
 
 
 async def filter_and_sort():

docs/api/fields/base-field.md

@@ -72,6 +72,27 @@ Excludes defaults and alias as they are populated separately
 
 `(bool)`: True if field is present on pydantic.FieldInfo
 
+<a name="fields.base.BaseField.get_base_pydantic_field_info"></a>
+#### get\_base\_pydantic\_field\_info
+
+```python
+ | @classmethod
+ | get_base_pydantic_field_info(cls, allow_null: bool) -> FieldInfo
+```
+
+Generates base pydantic.FieldInfo with only default and optionally
+required to fix pydantic Json field being set to required=False.
+Used in an ormar Model Metaclass.
+
+**Arguments**:
+
+- `allow_null (bool)`: flag if the default value can be None
+or if it should be populated by pydantic Undefined
+
+**Returns**:
+
+`(pydantic.FieldInfo)`: instance of base pydantic.FieldInfo
+
 <a name="fields.base.BaseField.convert_to_pydantic_field_info"></a>
 #### convert\_to\_pydantic\_field\_info
 

docs/api/fields/foreign-key.md

@@ -332,3 +332,32 @@ Selects the appropriate constructor based on a passed value.
 
 `(Optional[Union["Model", List["Model"]]])`: returns a Model or a list of Models
 
+<a name="fields.foreign_key.ForeignKeyField.get_relation_name"></a>
+#### get\_relation\_name
+
+```python
+ | @classmethod
+ | get_relation_name(cls) -> str
+```
+
+Returns name of the relation, which can be a own name or through model
+names for m2m models
+
+**Returns**:
+
+`(bool)`: result of the check
+
+<a name="fields.foreign_key.ForeignKeyField.get_source_model"></a>
+#### get\_source\_model
+
+```python
+ | @classmethod
+ | get_source_model(cls) -> Type["Model"]
+```
+
+Returns model from which the relation comes -> either owner or through model
+
+**Returns**:
+
+`(Type["Model"])`: source model
+

docs/api/fields/many-to-many.md

@@ -24,7 +24,7 @@ pydantic field to use and type of the target column field.
 #### ManyToMany
 
 ```python
-ManyToMany(to: "ToType", through: "ToType", *, name: str = None, unique: bool = False, virtual: bool = False, **kwargs: Any, ,) -> Any
+ManyToMany(to: "ToType", through: Optional["ToType"] = None, *, name: str = None, unique: bool = False, virtual: bool = False, **kwargs: Any, ,) -> Any
 ```
 
 Despite a name it's a function that returns constructed ManyToManyField.
@@ -134,3 +134,42 @@ Evaluates the ForwardRef to actual Field based on global and local namespaces
 
 `(None)`: None
 
+<a name="fields.many_to_many.ManyToManyField.get_relation_name"></a>
+#### get\_relation\_name
+
+```python
+ | @classmethod
+ | get_relation_name(cls) -> str
+```
+
+Returns name of the relation, which can be a own name or through model
+names for m2m models
+
+**Returns**:
+
+`(bool)`: result of the check
+
+<a name="fields.many_to_many.ManyToManyField.get_source_model"></a>
+#### get\_source\_model
+
+```python
+ | @classmethod
+ | get_source_model(cls) -> Type["Model"]
+```
+
+Returns model from which the relation comes -> either owner or through model
+
+**Returns**:
+
+`(Type["Model"])`: source model
+
+<a name="fields.many_to_many.ManyToManyField.create_default_through_model"></a>
+#### create\_default\_through\_model
+
+```python
+ | @classmethod
+ | create_default_through_model(cls) -> None
+```
+
+Creates default empty through model if no additional fields are required.
+

docs/api/models/excludable-items.md

@@ -0,0 +1,188 @@
+<a name="models.excludable"></a>
+# models.excludable
+
+<a name="models.excludable.Excludable"></a>
+## Excludable Objects
+
+```python
+@dataclass
+class Excludable()
+```
+
+Class that keeps sets of fields to exclude and include
+
+<a name="models.excludable.Excludable.get_copy"></a>
+#### get\_copy
+
+```python
+ | get_copy() -> "Excludable"
+```
+
+Return copy of self to avoid in place modifications
+
+**Returns**:
+
+`(ormar.models.excludable.Excludable)`: copy of self with copied sets
+
+<a name="models.excludable.Excludable.set_values"></a>
+#### set\_values
+
+```python
+ | set_values(value: Set, is_exclude: bool) -> None
+```
+
+Appends the data to include/exclude sets.
+
+**Arguments**:
+
+- `value (set)`: set of values to add
+- `is_exclude (bool)`: flag if values are to be excluded or included
+
+<a name="models.excludable.Excludable.is_included"></a>
+#### is\_included
+
+```python
+ | is_included(key: str) -> bool
+```
+
+Check if field in included (in set or set is {...})
+
+**Arguments**:
+
+- `key (str)`: key to check
+
+**Returns**:
+
+`(bool)`: result of the check
+
+<a name="models.excludable.Excludable.is_excluded"></a>
+#### is\_excluded
+
+```python
+ | is_excluded(key: str) -> bool
+```
+
+Check if field in excluded (in set or set is {...})
+
+**Arguments**:
+
+- `key (str)`: key to check
+
+**Returns**:
+
+`(bool)`: result of the check
+
+<a name="models.excludable.ExcludableItems"></a>
+## ExcludableItems Objects
+
+```python
+class ExcludableItems()
+```
+
+Keeps a dictionary of Excludables by alias + model_name keys
+to allow quick lookup by nested models without need to travers
+deeply nested dictionaries and passing include/exclude around
+
+<a name="models.excludable.ExcludableItems.from_excludable"></a>
+#### from\_excludable
+
+```python
+ | @classmethod
+ | from_excludable(cls, other: "ExcludableItems") -> "ExcludableItems"
+```
+
+Copy passed ExcludableItems to avoid inplace modifications.
+
+**Arguments**:
+
+- `other (ormar.models.excludable.ExcludableItems)`: other excludable items to be copied
+
+**Returns**:
+
+`(ormar.models.excludable.ExcludableItems)`: copy of other
+
+<a name="models.excludable.ExcludableItems.get"></a>
+#### get
+
+```python
+ | get(model_cls: Type["Model"], alias: str = "") -> Excludable
+```
+
+Return Excludable for given model and alias.
+
+**Arguments**:
+
+- `model_cls (ormar.models.metaclass.ModelMetaclass)`: target model to check
+- `alias (str)`: table alias from relation manager
+
+**Returns**:
+
+`(ormar.models.excludable.Excludable)`: Excludable for given model and alias
+
+<a name="models.excludable.ExcludableItems.build"></a>
+#### build
+
+```python
+ | build(items: Union[List[str], str, Tuple[str], Set[str], Dict], model_cls: Type["Model"], is_exclude: bool = False) -> None
+```
+
+Receives the one of the types of items and parses them as to achieve
+a end situation with one excludable per alias/model in relation.
+
+Each excludable has two sets of values - one to include, one to exclude.
+
+**Arguments**:
+
+- `items (Union[List[str], str, Tuple[str], Set[str], Dict])`: values to be included or excluded
+- `model_cls (ormar.models.metaclass.ModelMetaclass)`: source model from which relations are constructed
+- `is_exclude (bool)`: flag if items should be included or excluded
+
+<a name="models.excludable.ExcludableItems._set_excludes"></a>
+#### \_set\_excludes
+
+```python
+ | _set_excludes(items: Set, model_name: str, is_exclude: bool, alias: str = "") -> None
+```
+
+Sets set of values to be included or excluded for given key and model.
+
+**Arguments**:
+
+- `items (set)`: items to include/exclude
+- `model_name (str)`: name of model to construct key
+- `is_exclude (bool)`: flag if values should be included or excluded
+- `alias (str)`: 
+
+<a name="models.excludable.ExcludableItems._traverse_dict"></a>
+#### \_traverse\_dict
+
+```python
+ | _traverse_dict(values: Dict, source_model: Type["Model"], model_cls: Type["Model"], is_exclude: bool, related_items: List = None, alias: str = "") -> None
+```
+
+Goes through dict of nested values and construct/update Excludables.
+
+**Arguments**:
+
+- `values (Dict)`: items to include/exclude
+- `source_model (ormar.models.metaclass.ModelMetaclass)`: source model from which relations are constructed
+- `model_cls (ormar.models.metaclass.ModelMetaclass)`: model from which current relation is constructed
+- `is_exclude (bool)`: flag if values should be included or excluded
+- `related_items (List)`: list of names of related fields chain
+- `alias (str)`: alias of relation
+
+<a name="models.excludable.ExcludableItems._traverse_list"></a>
+#### \_traverse\_list
+
+```python
+ | _traverse_list(values: Set[str], model_cls: Type["Model"], is_exclude: bool) -> None
+```
+
+Goes through list of values and construct/update Excludables.
+
+**Arguments**:
+
+- `values (set)`: items to include/exclude
+- `model_cls (ormar.models.metaclass.ModelMetaclass)`: model from which current relation is constructed
+- `is_exclude (bool)`: flag if values should be included or excluded
+

docs/api/models/helpers/models.md

@@ -87,28 +87,6 @@ extraction of ormar model_fields.
 
 `(Tuple[Dict, Dict])`: namespace of the class updated, dict of extracted model_fields
 
-<a name="models.helpers.models.validate_related_names_in_relations"></a>
-#### validate\_related\_names\_in\_relations
-
-```python
-validate_related_names_in_relations(model_fields: Dict, new_model: Type["Model"]) -> None
-```
-
-Performs a validation of relation_names in relation fields.
-If multiple fields are leading to the same related model
-only one can have empty related_name param
-(populated by default as model.name.lower()+'s').
-Also related_names have to be unique for given related model.
-
-**Raises**:
-
-- `ModelDefinitionError`: if validation of related_names fail
-
-**Arguments**:
-
-- `model_fields (Dict[str, ormar.Field])`: dictionary of declared ormar model fields
-- `new_model (Model class)`: 
-
 <a name="models.helpers.models.group_related_list"></a>
 #### group\_related\_list
 
@@ -134,3 +112,23 @@ Result dictionary is sorted by length of the values and by key
 
 `(Dict[str, List])`: list converted to dictionary to avoid repetition and group nested models
 
+<a name="models.helpers.models.meta_field_not_set"></a>
+#### meta\_field\_not\_set
+
+```python
+meta_field_not_set(model: Type["Model"], field_name: str) -> bool
+```
+
+Checks if field with given name is already present in model.Meta.
+Then check if it's set to something truthful
+(in practice meaning not None, as it's non or ormar Field only).
+
+**Arguments**:
+
+- `model (Model class)`: newly constructed model
+- `field_name (str)`: name of the ormar field
+
+**Returns**:
+
+`(bool)`: result of the check
+

docs/api/models/helpers/pydantic.md

@@ -5,7 +5,7 @@
 #### create\_pydantic\_field
 
 ```python
-create_pydantic_field(field_name: str, model: Type["Model"], model_field: Type[ManyToManyField]) -> None
+create_pydantic_field(field_name: str, model: Type["Model"], model_field: Type["ManyToManyField"]) -> None
 ```
 
 Registers pydantic field on through model that leads to passed model
@@ -42,7 +42,7 @@ field_name. Returns a pydantic field with type of field_name field type.
 #### populate\_default\_pydantic\_field\_value
 
 ```python
-populate_default_pydantic_field_value(ormar_field: Type[BaseField], field_name: str, attrs: dict) -> dict
+populate_default_pydantic_field_value(ormar_field: Type["BaseField"], field_name: str, attrs: dict) -> dict
 ```
 
 Grabs current value of the ormar Field in class namespace
@@ -94,7 +94,7 @@ Those annotations are later used by pydantic to construct it's own fields.
 #### get\_pydantic\_base\_orm\_config
 
 ```python
-get_pydantic_base_orm_config() -> Type[BaseConfig]
+get_pydantic_base_orm_config() -> Type[pydantic.BaseConfig]
 ```
 
 Returns empty pydantic Config with orm_mode set to True.

docs/api/models/helpers/related-names-validation.md

@@ -0,0 +1,25 @@
+<a name="models.helpers.related_names_validation"></a>
+# models.helpers.related\_names\_validation
+
+<a name="models.helpers.related_names_validation.validate_related_names_in_relations"></a>
+#### validate\_related\_names\_in\_relations
+
+```python
+validate_related_names_in_relations(model_fields: Dict, new_model: Type["Model"]) -> None
+```
+
+Performs a validation of relation_names in relation fields.
+If multiple fields are leading to the same related model
+only one can have empty related_name param
+(populated by default as model.name.lower()+'s').
+Also related_names have to be unique for given related model.
+
+**Raises**:
+
+- `ModelDefinitionError`: if validation of related_names fail
+
+**Arguments**:
+
+- `model_fields (Dict[str, ormar.Field])`: dictionary of declared ormar model fields
+- `new_model (Model class)`: 
+

docs/api/models/helpers/relations.md

@@ -23,7 +23,7 @@ aliases for proper sql joins.
 #### register\_many\_to\_many\_relation\_on\_build
 
 ```python
-register_many_to_many_relation_on_build(field: Type[ManyToManyField]) -> None
+register_many_to_many_relation_on_build(field: Type["ManyToManyField"]) -> None
 ```
 
 Registers connection between through model and both sides of the m2m relation.
@@ -89,11 +89,24 @@ Autogenerated reverse fields also set related_name to the original field name.
 
 - `model_field (relation Field)`: original relation ForeignKey field
 
+<a name="models.helpers.relations.register_through_shortcut_fields"></a>
+#### register\_through\_shortcut\_fields
+
+```python
+register_through_shortcut_fields(model_field: Type["ManyToManyField"]) -> None
+```
+
+Registers m2m relation through shortcut on both ends of the relation.
+
+**Arguments**:
+
+- `model_field (ManyToManyField)`: relation field defined in parent model
+
 <a name="models.helpers.relations.register_relation_in_alias_manager"></a>
 #### register\_relation\_in\_alias\_manager
 
 ```python
-register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None
+register_relation_in_alias_manager(field: Type["ForeignKeyField"]) -> None
 ```
 
 Registers the relation (and reverse relation) in alias manager.

docs/api/models/helpers/sqlalchemy.md

@@ -5,7 +5,7 @@
 #### adjust\_through\_many\_to\_many\_model
 
 ```python
-adjust_through_many_to_many_model(model_field: Type[ManyToManyField]) -> None
+adjust_through_many_to_many_model(model_field: Type["ManyToManyField"]) -> None
 ```
 
 Registers m2m relation on through model.
@@ -21,7 +21,7 @@ Sets pydantic fields with child and parent model types.
 #### create\_and\_append\_m2m\_fk
 
 ```python
-create_and_append_m2m_fk(model: Type["Model"], model_field: Type[ManyToManyField], field_name: str) -> None
+create_and_append_m2m_fk(model: Type["Model"], model_field: Type["ManyToManyField"], field_name: str) -> None
 ```
 
 Registers sqlalchemy Column with sqlalchemy.ForeignKey leading to the model.
@@ -38,7 +38,7 @@ Newly created field is added to m2m relation through model Meta columns and tabl
 #### check\_pk\_column\_validity
 
 ```python
-check_pk_column_validity(field_name: str, field: BaseField, pkname: Optional[str]) -> Optional[str]
+check_pk_column_validity(field_name: str, field: "BaseField", pkname: Optional[str]) -> Optional[str]
 ```
 
 Receives the field marked as primary key and verifies if the pkname
@@ -165,7 +165,7 @@ It populates name, metadata, columns and constraints.
 #### update\_column\_definition
 
 ```python
-update_column_definition(model: Union[Type["Model"], Type["NewBaseModel"]], field: Type[ForeignKeyField]) -> None
+update_column_definition(model: Union[Type["Model"], Type["NewBaseModel"]], field: Type["ForeignKeyField"]) -> None
 ```
 
 Updates a column with a new type column based on updated parameters in FK fields.

docs/api/models/helpers/validation.md

@@ -0,0 +1,120 @@
+<a name="models.helpers.validation"></a>
+# models.helpers.validation
+
+<a name="models.helpers.validation.check_if_field_has_choices"></a>
+#### check\_if\_field\_has\_choices
+
+```python
+check_if_field_has_choices(field: Type[BaseField]) -> bool
+```
+
+Checks if given field has choices populated.
+A if it has one, a validator for this field needs to be attached.
+
+**Arguments**:
+
+- `field (BaseField)`: ormar field to check
+
+**Returns**:
+
+`(bool)`: result of the check
+
+<a name="models.helpers.validation.convert_choices_if_needed"></a>
+#### convert\_choices\_if\_needed
+
+```python
+convert_choices_if_needed(field: Type["BaseField"], value: Any) -> Tuple[Any, List]
+```
+
+Converts dates to isoformat as fastapi can check this condition in routes
+and the fields are not yet parsed.
+
+Converts enums to list of it's values.
+
+Converts uuids to strings.
+
+Converts decimal to float with given scale.
+
+**Arguments**:
+
+- `field (Type[BaseField])`: ormar field to check with choices
+- `values (Dict)`: current values of the model to verify
+
+**Returns**:
+
+`(Tuple[Any, List])`: value, choices list
+
+<a name="models.helpers.validation.validate_choices"></a>
+#### validate\_choices
+
+```python
+validate_choices(field: Type["BaseField"], value: Any) -> None
+```
+
+Validates if given value is in provided choices.
+
+**Raises**:
+
+- `ValueError`: If value is not in choices.
+
+**Arguments**:
+
+- `field (Type[BaseField])`: field to validate
+- `value (Any)`: value of the field
+
+<a name="models.helpers.validation.choices_validator"></a>
+#### choices\_validator
+
+```python
+choices_validator(cls: Type["Model"], values: Dict[str, Any]) -> Dict[str, Any]
+```
+
+Validator that is attached to pydantic model pre root validators.
+Validator checks if field value is in field.choices list.
+
+**Raises**:
+
+- `ValueError`: if field value is outside of allowed choices.
+
+**Arguments**:
+
+- `cls (Model class)`: constructed class
+- `values (Dict[str, Any])`: dictionary of field values (pydantic side)
+
+**Returns**:
+
+`(Dict[str, Any])`: values if pass validation, otherwise exception is raised
+
+<a name="models.helpers.validation.construct_modify_schema_function"></a>
+#### construct\_modify\_schema\_function
+
+```python
+construct_modify_schema_function(fields_with_choices: List) -> SchemaExtraCallable
+```
+
+Modifies the schema to include fields with choices validator.
+Those fields will be displayed in schema as Enum types with available choices
+values listed next to them.
+
+**Arguments**:
+
+- `fields_with_choices (List)`: list of fields with choices validation
+
+**Returns**:
+
+`(Callable)`: callable that will be run by pydantic to modify the schema
+
+<a name="models.helpers.validation.populate_choices_validators"></a>
+#### populate\_choices\_validators
+
+```python
+populate_choices_validators(model: Type["Model"]) -> None
+```
+
+Checks if Model has any fields with choices set.
+If yes it adds choices validation into pre root validators.
+
+**Arguments**:
+
+- `model (Model class)`: newly constructed Model
+

docs/api/models/mixins/excludable-mixin.md

@@ -30,88 +30,12 @@ passed items.
 
 `(Union[Set, Dict, None])`: child extracted from items if exists
 
-<a name="models.mixins.excludable_mixin.ExcludableMixin.get_excluded"></a>
-#### get\_excluded
-
-```python
- | @staticmethod
- | get_excluded(exclude: Union[Set, Dict, None], key: str = None) -> Union[Set, Dict, None]
-```
-
-Proxy to ExcludableMixin.get_child for exclusions.
-
-**Arguments**:
-
-- `exclude (Union[Set, Dict, None])`: bag of items to exclude
-- `key (str)`: name of the child to extract
-
-**Returns**:
-
-`(Union[Set, Dict, None])`: child extracted from items if exists
-
-<a name="models.mixins.excludable_mixin.ExcludableMixin.get_included"></a>
-#### get\_included
-
-```python
- | @staticmethod
- | get_included(include: Union[Set, Dict, None], key: str = None) -> Union[Set, Dict, None]
-```
-
-Proxy to ExcludableMixin.get_child for inclusions.
-
-**Arguments**:
-
-- `include (Union[Set, Dict, None])`: bag of items to include
-- `key (str)`: name of the child to extract
-
-**Returns**:
-
-`(Union[Set, Dict, None])`: child extracted from items if exists
-
-<a name="models.mixins.excludable_mixin.ExcludableMixin.is_excluded"></a>
-#### is\_excluded
-
-```python
- | @staticmethod
- | is_excluded(exclude: Union[Set, Dict, None], key: str = None) -> bool
-```
-
-Checks if given key should be excluded on model/ dict.
-
-**Arguments**:
-
-- `exclude (Union[Set, Dict, None])`: bag of items to exclude
-- `key (str)`: name of the child to extract
-
-**Returns**:
-
-`(Union[Set, Dict, None])`: child extracted from items if exists
-
-<a name="models.mixins.excludable_mixin.ExcludableMixin.is_included"></a>
-#### is\_included
-
-```python
- | @staticmethod
- | is_included(include: Union[Set, Dict, None], key: str = None) -> bool
-```
-
-Checks if given key should be included on model/ dict.
-
-**Arguments**:
-
-- `include (Union[Set, Dict, None])`: bag of items to include
-- `key (str)`: name of the child to extract
-
-**Returns**:
-
-`(Union[Set, Dict, None])`: child extracted from items if exists
-
 <a name="models.mixins.excludable_mixin.ExcludableMixin._populate_pk_column"></a>
 #### \_populate\_pk\_column
 
 ```python
  | @staticmethod
- | _populate_pk_column(model: Type["Model"], columns: List[str], use_alias: bool = False) -> List[str]
+ | _populate_pk_column(model: Union[Type["Model"], Type["ModelRow"]], columns: List[str], use_alias: bool = False) -> List[str]
 ```
 
 Adds primary key column/alias (depends on use_alias flag) to list of
@@ -132,7 +56,7 @@ column names that are selected.
 
 ```python
  | @classmethod
- | own_table_columns(cls, model: Type["Model"], fields: Optional[Union[Set, Dict]], exclude_fields: Optional[Union[Set, Dict]], use_alias: bool = False) -> List[str]
+ | own_table_columns(cls, model: Union[Type["Model"], Type["ModelRow"]], excludable: ExcludableItems, alias: str = "", use_alias: bool = False) -> List[str]
 ```
 
 Returns list of aliases or field names for given model.
@@ -145,9 +69,9 @@ Primary key field is always added and cannot be excluded (will be added anyway).
 
 **Arguments**:
 
+- `alias (str)`: relation prefix
+- `excludable (ExcludableItems)`: structure of fields to include and exclude
 - `model (Type["Model"])`: model on columns are selected
-- `fields (Optional[Union[Set, Dict]])`: set/dict of fields to include
-- `exclude_fields (Optional[Union[Set, Dict]])`: set/dict of fields to exclude
 - `use_alias (bool)`: flag if aliases or field names should be used
 
 **Returns**:
@@ -183,7 +107,7 @@ exclusion, for nested models all related models are excluded.
 
 ```python
  | @classmethod
- | get_names_to_exclude(cls, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None) -> Set
+ | get_names_to_exclude(cls, excludable: ExcludableItems, alias: str) -> Set
 ```
 
 Returns a set of models field names that should be explicitly excluded
@@ -197,8 +121,8 @@ them with dicts constructed from those db rows.
 
 **Arguments**:
 
-- `fields (Optional[Union[Set, Dict]])`: set/dict of fields to include
-- `exclude_fields (Optional[Union[Set, Dict]])`: set/dict of fields to exclude
+- `alias (str)`: alias of current relation
+- `excludable (ExcludableItems)`: structure of fields to include and exclude
 
 **Returns**:
 

docs/api/models/mixins/relation-mixin.md

@@ -40,12 +40,26 @@ List is cached in cls._related_fields for quicker access.
 
 `(List)`: list of related fields
 
+<a name="models.mixins.relation_mixin.RelationMixin.extract_through_names"></a>
+#### extract\_through\_names
+
+```python
+ | @classmethod
+ | extract_through_names(cls) -> Set
+```
+
+Extracts related fields through names which are shortcuts to through models.
+
+**Returns**:
+
+`(Set)`: set of related through fields names
+
 <a name="models.mixins.relation_mixin.RelationMixin.extract_related_names"></a>
 #### extract\_related\_names
 
 ```python
  | @classmethod
- | extract_related_names(cls) -> Set
+ | extract_related_names(cls) -> Set[str]
 ```
 
 Returns List of fields names for all relations declared on a model.
@@ -53,7 +67,7 @@ List is cached in cls._related_names for quicker access.
 
 **Returns**:
 
-`(List)`: list of related fields names
+`(Set)`: set of related fields names
 
 <a name="models.mixins.relation_mixin.RelationMixin._extract_db_related_names"></a>
 #### \_extract\_db\_related\_names
@@ -91,3 +105,24 @@ for nested models all related models are returned.
 
 `(Set)`: set of non mandatory related fields
 
+<a name="models.mixins.relation_mixin.RelationMixin._iterate_related_models"></a>
+#### \_iterate\_related\_models
+
+```python
+ | @classmethod
+ | _iterate_related_models(cls, visited: Set[Union[Type["Model"], Type["RelationMixin"]]] = None, source_relation: str = None, source_model: Union[Type["Model"], Type["RelationMixin"]] = None) -> List[str]
+```
+
+Iterates related models recursively to extract relation strings of
+nested not visited models.
+
+**Arguments**:
+
+- `visited (Set[str])`: set of already visited models
+- `source_relation (str)`: name of the current relation
+- `source_model (Type["Model"])`: model from which relation comes in nested relations
+
+**Returns**:
+
+`(List[str])`: list of relation strings to be passed to select_related
+

docs/api/models/mixins/save-prepare-mixin.md

@@ -91,3 +91,22 @@ passed by the user.
 
 `(Dict)`: dictionary of model that is about to be saved
 
+<a name="models.mixins.save_mixin.SavePrepareMixin.validate_choices"></a>
+#### validate\_choices
+
+```python
+ | @classmethod
+ | validate_choices(cls, new_kwargs: Dict) -> Dict
+```
+
+Receives dictionary of model that is about to be saved and validates the
+fields with choices set to see if the value is allowed.
+
+**Arguments**:
+
+- `new_kwargs (Dict)`: dictionary of model that is about to be saved
+
+**Returns**:
+
+`(Dict)`: dictionary of model that is about to be saved
+

docs/api/models/model-metaclass.md

@@ -12,61 +12,6 @@ Class used for type hinting.
 Users can subclass this one for convenience but it's not required.
 The only requirement is that ormar.Model has to have inner class with name Meta.
 
-<a name="models.metaclass.check_if_field_has_choices"></a>
-#### check\_if\_field\_has\_choices
-
-```python
-check_if_field_has_choices(field: Type[BaseField]) -> bool
-```
-
-Checks if given field has choices populated.
-A if it has one, a validator for this field needs to be attached.
-
-**Arguments**:
-
-- `field (BaseField)`: ormar field to check
-
-**Returns**:
-
-`(bool)`: result of the check
-
-<a name="models.metaclass.choices_validator"></a>
-#### choices\_validator
-
-```python
-choices_validator(cls: Type["Model"], values: Dict[str, Any]) -> Dict[str, Any]
-```
-
-Validator that is attached to pydantic model pre root validators.
-Validator checks if field value is in field.choices list.
-
-**Raises**:
-
-- `ValueError`: if field value is outside of allowed choices.
-
-**Arguments**:
-
-- `cls (Model class)`: constructed class
-- `values (Dict[str, Any])`: dictionary of field values (pydantic side)
-
-**Returns**:
-
-`(Dict[str, Any])`: values if pass validation, otherwise exception is raised
-
-<a name="models.metaclass.populate_choices_validators"></a>
-#### populate\_choices\_validators
-
-```python
-populate_choices_validators(model: Type["Model"]) -> None
-```
-
-Checks if Model has any fields with choices set.
-If yes it adds choices validation into pre root validators.
-
-**Arguments**:
-
-- `model (Model class)`: newly constructed Model
-
 <a name="models.metaclass.add_cached_properties"></a>
 #### add\_cached\_properties
 
@@ -87,26 +32,6 @@ All properties here are used as "cache" to not recalculate them constantly.
 
 - `new_model (Model class)`: newly constructed Model
 
-<a name="models.metaclass.meta_field_not_set"></a>
-#### meta\_field\_not\_set
-
-```python
-meta_field_not_set(model: Type["Model"], field_name: str) -> bool
-```
-
-Checks if field with given name is already present in model.Meta.
-Then check if it's set to something truthful
-(in practice meaning not None, as it's non or ormar Field only).
-
-**Arguments**:
-
-- `model (Model class)`: newly constructed model
-- `field_name (str)`: name of the ormar field
-
-**Returns**:
-
-`(bool)`: result of the check
-
 <a name="models.metaclass.add_property_fields"></a>
 #### add\_property\_fields
 
@@ -141,24 +66,6 @@ Signals are emitted in both model own methods and in selected queryset ones.
 
 - `new_model (Model class)`: newly constructed model
 
-<a name="models.metaclass.update_attrs_and_fields"></a>
-#### update\_attrs\_and\_fields
-
-```python
-update_attrs_and_fields(attrs: Dict, new_attrs: Dict, model_fields: Dict, new_model_fields: Dict, new_fields: Set) -> Dict
-```
-
-Updates __annotations__, values of model fields (so pydantic FieldInfos)
-as well as model.Meta.model_fields definitions from parents.
-
-**Arguments**:
-
-- `attrs (Dict)`: new namespace for class being constructed
-- `new_attrs (Dict)`: related of the namespace extracted from parent class
-- `model_fields (Dict[str, BaseField])`: ormar fields in defined in current class
-- `new_model_fields (Dict[str, BaseField])`: ormar fields defined in parent classes
-- `new_fields (Set[str])`: set of new fields names
-
 <a name="models.metaclass.verify_constraint_names"></a>
 #### verify\_constraint\_names
 
@@ -195,7 +102,7 @@ Updates Meta parameters in child from parent if needed.
 #### copy\_and\_replace\_m2m\_through\_model
 
 ```python
-copy_and_replace_m2m_through_model(field: Type[ManyToManyField], field_name: str, table_name: str, parent_fields: Dict, attrs: Dict, meta: ModelMeta) -> None
+copy_and_replace_m2m_through_model(field: Type[ManyToManyField], field_name: str, table_name: str, parent_fields: Dict, attrs: Dict, meta: ModelMeta, base_class: Type["Model"]) -> None
 ```
 
 Clones class with Through model for m2m relations, appends child name to the name
@@ -211,6 +118,7 @@ Removes the original sqlalchemy table from metadata if it was not removed.
 
 **Arguments**:
 
+- `base_class (Type["Model"])`: base class model
 - `field (Type[ManyToManyField])`: field with relations definition
 - `field_name (str)`: name of the relation field
 - `table_name (str)`: name of the table
@@ -281,6 +189,24 @@ If the class is a ormar.Model it is skipped.
 
 `(Tuple[Dict, Dict])`: updated attrs and model_fields
 
+<a name="models.metaclass.update_attrs_and_fields"></a>
+#### update\_attrs\_and\_fields
+
+```python
+update_attrs_and_fields(attrs: Dict, new_attrs: Dict, model_fields: Dict, new_model_fields: Dict, new_fields: Set) -> Dict
+```
+
+Updates __annotations__, values of model fields (so pydantic FieldInfos)
+as well as model.Meta.model_fields definitions from parents.
+
+**Arguments**:
+
+- `attrs (Dict)`: new namespace for class being constructed
+- `new_attrs (Dict)`: related of the namespace extracted from parent class
+- `model_fields (Dict[str, BaseField])`: ormar fields in defined in current class
+- `new_model_fields (Dict[str, BaseField])`: ormar fields defined in parent classes
+- `new_fields (Set[str])`: set of new fields names
+
 <a name="models.metaclass.ModelMetaclass"></a>
 ## ModelMetaclass Objects
 

docs/api/models/model-row.md

@@ -0,0 +1,132 @@
+<a name="models.model_row"></a>
+# models.model\_row
+
+<a name="models.model_row.ModelRow"></a>
+## ModelRow Objects
+
+```python
+class ModelRow(NewBaseModel)
+```
+
+<a name="models.model_row.ModelRow.from_row"></a>
+#### from\_row
+
+```python
+ | @classmethod
+ | from_row(cls, row: sqlalchemy.engine.ResultProxy, source_model: Type["Model"], select_related: List = None, related_models: Any = None, related_field: Type["ForeignKeyField"] = None, excludable: ExcludableItems = None, current_relation_str: str = "", proxy_source_model: Optional[Type["Model"]] = None) -> Optional["Model"]
+```
+
+Model method to convert raw sql row from database into ormar.Model instance.
+Traverses nested models if they were specified in select_related for query.
+
+Called recurrently and returns model instance if it's present in the row.
+Note that it's processing one row at a time, so if there are duplicates of
+parent row that needs to be joined/combined
+(like parent row in sql join with 2+ child rows)
+instances populated in this method are later combined in the QuerySet.
+Other method working directly on raw database results is in prefetch_query,
+where rows are populated in a different way as they do not have
+nested models in result.
+
+**Arguments**:
+
+- `proxy_source_model (Optional[Type["ModelRow"]])`: source model from which querysetproxy is constructed
+- `excludable (ExcludableItems)`: structure of fields to include and exclude
+- `current_relation_str (str)`: name of the relation field
+- `source_model (Type[Model])`: model on which relation was defined
+- `row (sqlalchemy.engine.result.ResultProxy)`: raw result row from the database
+- `select_related (List)`: list of names of related models fetched from database
+- `related_models (Union[List, Dict])`: list or dict of related models
+- `related_field (Type[ForeignKeyField])`: field with relation declaration
+
+**Returns**:
+
+`(Optional[Model])`: returns model if model is populated from database
+
+<a name="models.model_row.ModelRow._populate_nested_models_from_row"></a>
+#### \_populate\_nested\_models\_from\_row
+
+```python
+ | @classmethod
+ | _populate_nested_models_from_row(cls, item: dict, row: sqlalchemy.engine.ResultProxy, source_model: Type["Model"], related_models: Any, excludable: ExcludableItems, table_prefix: str, current_relation_str: str = None, proxy_source_model: Type["Model"] = None) -> dict
+```
+
+Traverses structure of related models and populates the nested models
+from the database row.
+Related models can be a list if only directly related models are to be
+populated, converted to dict if related models also have their own related
+models to be populated.
+
+Recurrently calls from_row method on nested instances and create nested
+instances. In the end those instances are added to the final model dictionary.
+
+**Arguments**:
+
+- `proxy_source_model (Optional[Type["ModelRow"]])`: source model from which querysetproxy is constructed
+- `excludable (ExcludableItems)`: structure of fields to include and exclude
+- `source_model (Type[Model])`: source model from which relation started
+- `current_relation_str (str)`: joined related parts into one string
+- `item (Dict)`: dictionary of already populated nested models, otherwise empty dict
+- `row (sqlalchemy.engine.result.ResultProxy)`: raw result row from the database
+- `related_models (Union[Dict, List])`: list or dict of related models
+
+**Returns**:
+
+`(Dict)`: dictionary with keys corresponding to model fields names
+and values are database values
+
+<a name="models.model_row.ModelRow.populate_through_instance"></a>
+#### populate\_through\_instance
+
+```python
+ | @classmethod
+ | populate_through_instance(cls, row: sqlalchemy.engine.ResultProxy, through_name: str, related: str, excludable: ExcludableItems) -> "ModelRow"
+```
+
+Initialize the through model from db row.
+Excluded all relation fields and other exclude/include set in excludable.
+
+**Arguments**:
+
+- `row (sqlalchemy.engine.ResultProxy)`: loaded row from database
+- `through_name (str)`: name of the through field
+- `related (str)`: name of the relation
+- `excludable (ExcludableItems)`: structure of fields to include and exclude
+
+**Returns**:
+
+`("ModelRow")`: initialized through model without relation
+
+<a name="models.model_row.ModelRow.extract_prefixed_table_columns"></a>
+#### extract\_prefixed\_table\_columns
+
+```python
+ | @classmethod
+ | extract_prefixed_table_columns(cls, item: dict, row: sqlalchemy.engine.result.ResultProxy, table_prefix: str, excludable: ExcludableItems) -> Dict
+```
+
+Extracts own fields from raw sql result, using a given prefix.
+Prefix changes depending on the table's position in a join.
+
+If the table is a main table, there is no prefix.
+All joined tables have prefixes to allow duplicate column names,
+as well as duplicated joins to the same table from multiple different tables.
+
+Extracted fields populates the related dict later used to construct a Model.
+
+Used in Model.from_row and PrefetchQuery._populate_rows methods.
+
+**Arguments**:
+
+- `excludable (ExcludableItems)`: structure of fields to include and exclude
+- `item (Dict)`: dictionary of already populated nested models, otherwise empty dict
+- `row (sqlalchemy.engine.result.ResultProxy)`: raw result row from the database
+- `table_prefix (str)`: prefix of the table from AliasManager
+each pair of tables have own prefix (two of them depending on direction) -
+used in joins to allow multiple joins to the same table.
+
+**Returns**:
+
+`(Dict)`: dictionary with keys corresponding to model fields names
+and values are database values
+

docs/api/models/model.md

@@ -5,122 +5,14 @@
 ## Model Objects
 
 ```python
-class Model(NewBaseModel)
+class Model(ModelRow)
 ```
 
-<a name="models.model.Model.from_row"></a>
-#### from\_row
-
-```python
- | @classmethod
- | from_row(cls: Type[T], row: sqlalchemy.engine.ResultProxy, select_related: List = None, related_models: Any = None, previous_model: Type[T] = None, source_model: Type[T] = None, related_name: str = None, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None, current_relation_str: str = None) -> Optional[T]
-```
-
-Model method to convert raw sql row from database into ormar.Model instance.
-Traverses nested models if they were specified in select_related for query.
-
-Called recurrently and returns model instance if it's present in the row.
-Note that it's processing one row at a time, so if there are duplicates of
-parent row that needs to be joined/combined
-(like parent row in sql join with 2+ child rows)
-instances populated in this method are later combined in the QuerySet.
-Other method working directly on raw database results is in prefetch_query,
-where rows are populated in a different way as they do not have
-nested models in result.
-
-**Arguments**:
-
-- `current_relation_str (str)`: name of the relation field
-- `source_model (Type[Model])`: model on which relation was defined
-- `row (sqlalchemy.engine.result.ResultProxy)`: raw result row from the database
-- `select_related (List)`: list of names of related models fetched from database
-- `related_models (Union[List, Dict])`: list or dict of related models
-- `previous_model (Model class)`: internal param for nested models to specify table_prefix
-- `related_name (str)`: internal parameter - name of current nested model
-- `fields (Optional[Union[Dict, Set]])`: fields and related model fields to include
-if provided only those are included
-- `exclude_fields (Optional[Union[Dict, Set]])`: fields and related model fields to exclude
-excludes the fields even if they are provided in fields
-
-**Returns**:
-
-`(Optional[Model])`: returns model if model is populated from database
-
-<a name="models.model.Model.populate_nested_models_from_row"></a>
-#### populate\_nested\_models\_from\_row
-
-```python
- | @classmethod
- | populate_nested_models_from_row(cls, item: dict, row: sqlalchemy.engine.ResultProxy, related_models: Any, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None, current_relation_str: str = None, source_model: Type[T] = None) -> dict
-```
-
-Traverses structure of related models and populates the nested models
-from the database row.
-Related models can be a list if only directly related models are to be
-populated, converted to dict if related models also have their own related
-models to be populated.
-
-Recurrently calls from_row method on nested instances and create nested
-instances. In the end those instances are added to the final model dictionary.
-
-**Arguments**:
-
-- `source_model (Type[Model])`: source model from which relation started
-- `current_relation_str (str)`: joined related parts into one string
-- `item (Dict)`: dictionary of already populated nested models, otherwise empty dict
-- `row (sqlalchemy.engine.result.ResultProxy)`: raw result row from the database
-- `related_models (Union[Dict, List])`: list or dict of related models
-- `fields (Optional[Union[Dict, Set]])`: fields and related model fields to include -
-if provided only those are included
-- `exclude_fields (Optional[Union[Dict, Set]])`: fields and related model fields to exclude
-excludes the fields even if they are provided in fields
-
-**Returns**:
-
-`(Dict)`: dictionary with keys corresponding to model fields names
-and values are database values
-
-<a name="models.model.Model.extract_prefixed_table_columns"></a>
-#### extract\_prefixed\_table\_columns
-
-```python
- | @classmethod
- | extract_prefixed_table_columns(cls, item: dict, row: sqlalchemy.engine.result.ResultProxy, table_prefix: str, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None) -> dict
-```
-
-Extracts own fields from raw sql result, using a given prefix.
-Prefix changes depending on the table's position in a join.
-
-If the table is a main table, there is no prefix.
-All joined tables have prefixes to allow duplicate column names,
-as well as duplicated joins to the same table from multiple different tables.
-
-Extracted fields populates the related dict later used to construct a Model.
-
-Used in Model.from_row and PrefetchQuery._populate_rows methods.
-
-**Arguments**:
-
-- `item (Dict)`: dictionary of already populated nested models, otherwise empty dict
-- `row (sqlalchemy.engine.result.ResultProxy)`: raw result row from the database
-- `table_prefix (str)`: prefix of the table from AliasManager
-each pair of tables have own prefix (two of them depending on direction) -
-used in joins to allow multiple joins to the same table.
-- `fields (Optional[Union[Dict, Set]])`: fields and related model fields to include -
-if provided only those are included
-- `exclude_fields (Optional[Union[Dict, Set]])`: fields and related model fields to exclude
-excludes the fields even if they are provided in fields
-
-**Returns**:
-
-`(Dict)`: dictionary with keys corresponding to model fields names
-and values are database values
-
 <a name="models.model.Model.upsert"></a>
 #### upsert
 
 ```python
- | async upsert(**kwargs: Any) -> T
+ | async upsert(**kwargs: Any) -> "Model"
 ```
 
 Performs either a save or an update depending on the presence of the pk.
@@ -139,7 +31,7 @@ For save kwargs are ignored, used only in update if provided.
 #### save
 
 ```python
- | async save() -> T
+ | async save() -> "Model"
 ```
 
 Performs a save of given Model instance.
@@ -203,7 +95,7 @@ number of updated instances
 
 ```python
  | @staticmethod
- | async _update_and_follow(rel: T, follow: bool, visited: Set, update_count: int) -> Tuple[int, Set]
+ | async _update_and_follow(rel: "Model", follow: bool, visited: Set, update_count: int) -> Tuple[int, Set]
 ```
 
 Internal method used in save_related to follow related models and update numbers
@@ -227,7 +119,7 @@ number of updated instances
 #### update
 
 ```python
- | async update(**kwargs: Any) -> T
+ | async update(**kwargs: Any) -> "Model"
 ```
 
 Performs update of Model instance in the database.
@@ -274,7 +166,7 @@ or update and the Model will be saved in database again.
 #### load
 
 ```python
- | async load() -> T
+ | async load() -> "Model"
 ```
 
 Allow to refresh existing Models fields from database.
@@ -289,3 +181,40 @@ Does NOT refresh the related models fields if they were loaded before.
 
 `(Model)`: reloaded Model
 
+<a name="models.model.Model.load_all"></a>
+#### load\_all
+
+```python
+ | async load_all(follow: bool = False, exclude: Union[List, str, Set, Dict] = None) -> "Model"
+```
+
+Allow to refresh existing Models fields from database.
+Performs refresh of the related models fields.
+
+By default loads only self and the directly related ones.
+
+If follow=True is set it loads also related models of related models.
+
+To not get stuck in an infinite loop as related models also keep a relation
+to parent model visited models set is kept.
+
+That way already visited models that are nested are loaded, but the load do not
+follow them inside. So Model A -> Model B -> Model C -> Model A -> Model X
+will load second Model A but will never follow into Model X.
+Nested relations of those kind need to be loaded manually.
+
+**Raises**:
+
+- `NoMatch`: If given pk is not found in database.
+
+**Arguments**:
+
+- `exclude ()`: 
+- `follow (bool)`: flag to trigger deep save -
+by default only directly related models are saved
+with follow=True also related models of related models are saved
+
+**Returns**:
+
+`(Model)`: reloaded Model
+

docs/api/models/new-basemodel.md

@@ -146,7 +146,7 @@ Raises exception if model is abstract or has ForwardRefs in relation fields.
 #### \_extract\_related\_model\_instead\_of\_field
 
 ```python
- | _extract_related_model_instead_of_field(item: str) -> Optional[Union["T", Sequence["T"]]]
+ | _extract_related_model_instead_of_field(item: str) -> Optional[Union["Model", Sequence["Model"]]]
 ```
 
 Retrieves the related model/models from RelationshipManager.
@@ -276,7 +276,7 @@ cause some dialect require different treatment
 #### remove
 
 ```python
- | remove(parent: "T", name: str) -> None
+ | remove(parent: "Model", name: str) -> None
 ```
 
 Removes child from relation with given name in RelationshipManager

docs/api/query-set/join.md

@@ -22,11 +22,25 @@ Shortcut for ormar's model AliasManager stored on Meta.
 
 `(AliasManager)`: alias manager from model's Meta
 
-<a name="queryset.join.SqlJoin.on_clause"></a>
-#### on\_clause
+<a name="queryset.join.SqlJoin.to_table"></a>
+#### to\_table
 
 ```python
- | on_clause(previous_alias: str, from_clause: str, to_clause: str) -> text
+ | @property
+ | to_table() -> str
+```
+
+Shortcut to table name of the next model
+
+**Returns**:
+
+`(str)`: name of the target table
+
+<a name="queryset.join.SqlJoin._on_clause"></a>
+#### \_on\_clause
+
+```python
+ | _on_clause(previous_alias: str, from_clause: str, to_clause: str) -> text
 ```
 
 Receives aliases and names of both ends of the join and combines them
@@ -99,11 +113,11 @@ Updated are:
 - `related_name (str)`: name of the relation to follow
 - `remainder (Any)`: deeper tables if there are more nested joins
 
-<a name="queryset.join.SqlJoin.process_m2m_through_table"></a>
-#### process\_m2m\_through\_table
+<a name="queryset.join.SqlJoin._process_m2m_through_table"></a>
+#### \_process\_m2m\_through\_table
 
 ```python
- | process_m2m_through_table() -> None
+ | _process_m2m_through_table() -> None
 ```
 
 Process Through table of the ManyToMany relation so that source table is
@@ -119,11 +133,11 @@ Replaces needed parameters like:
 
 To point to through model
 
-<a name="queryset.join.SqlJoin.process_m2m_related_name_change"></a>
-#### process\_m2m\_related\_name\_change
+<a name="queryset.join.SqlJoin._process_m2m_related_name_change"></a>
+#### \_process\_m2m\_related\_name\_change
 
 ```python
- | process_m2m_related_name_change(reverse: bool = False) -> str
+ | _process_m2m_related_name_change(reverse: bool = False) -> str
 ```
 
 Extracts relation name to link join through the Through model declared on
@@ -158,74 +172,21 @@ Updates the used aliases list directly.
 
 Process order_by causes for non m2m relations.
 
-<a name="queryset.join.SqlJoin._replace_many_to_many_order_by_columns"></a>
-#### \_replace\_many\_to\_many\_order\_by\_columns
+<a name="queryset.join.SqlJoin._get_order_bys"></a>
+#### \_get\_order\_bys
 
 ```python
- | _replace_many_to_many_order_by_columns(part: str, new_part: str) -> None
-```
-
-Substitutes the name of the relation with actual model name in m2m order bys.
-
-**Arguments**:
-
-- `part (str)`: name of the field with relation
-- `new_part (str)`: name of the target model
-
-<a name="queryset.join.SqlJoin._check_if_condition_apply"></a>
-#### \_check\_if\_condition\_apply
-
-```python
- | @staticmethod
- | _check_if_condition_apply(condition: List, part: str) -> bool
-```
-
-Checks filter conditions to find if they apply to current join.
-
-**Arguments**:
-
-- `condition (List[str])`: list of parts of condition split by '__'
-- `part (str)`: name of the current relation join.
-
-**Returns**:
-
-`(bool)`: result of the check
-
-<a name="queryset.join.SqlJoin.set_aliased_order_by"></a>
-#### set\_aliased\_order\_by
-
-```python
- | set_aliased_order_by(condition: List[str], to_table: str) -> None
-```
-
-Substitute hyphens ('-') with descending order.
-Construct actual sqlalchemy text clause using aliased table and column name.
-
-**Arguments**:
-
-- `condition (List[str])`: list of parts of a current condition split by '__'
-- `to_table (sqlalchemy.sql.elements.quoted_name)`: target table
-
-<a name="queryset.join.SqlJoin.get_order_bys"></a>
-#### get\_order\_bys
-
-```python
- | get_order_bys(to_table: str, pkname_alias: str) -> None
+ | _get_order_bys() -> None
 ```
 
 Triggers construction of order bys if they are given.
 Otherwise by default each table is sorted by a primary key column asc.
 
-**Arguments**:
-
-- `to_table (sqlalchemy.sql.elements.quoted_name)`: target table
-- `pkname_alias (str)`: alias of the primary key column
-
-<a name="queryset.join.SqlJoin.get_to_and_from_keys"></a>
-#### get\_to\_and\_from\_keys
+<a name="queryset.join.SqlJoin._get_to_and_from_keys"></a>
+#### \_get\_to\_and\_from\_keys
 
 ```python
- | get_to_and_from_keys() -> Tuple[str, str]
+ | _get_to_and_from_keys() -> Tuple[str, str]
 ```
 
 Based on the relation type, name of the relation and previous models and parts

docs/api/query-set/prefetch-query.md

@@ -1,26 +1,6 @@
 <a name="queryset.prefetch_query"></a>
 # queryset.prefetch\_query
 
-<a name="queryset.prefetch_query.add_relation_field_to_fields"></a>
-#### add\_relation\_field\_to\_fields
-
-```python
-add_relation_field_to_fields(fields: Union[Set[Any], Dict[Any, Any], None], related_field_name: str) -> Union[Set[Any], Dict[Any, Any], None]
-```
-
-Adds related field into fields to include as otherwise it would be skipped.
-Related field is added only if fields are already populated.
-Empty fields implies all fields.
-
-**Arguments**:
-
-- `fields (Dict)`: Union[Set[Any], Dict[Any, Any], None]
-- `related_field_name (str)`: name of the field with relation
-
-**Returns**:
-
-`(Union[Set[Any], Dict[Any, Any], None])`: updated fields dict
-
 <a name="queryset.prefetch_query.sort_models"></a>
 #### sort\_models
 
@@ -232,7 +212,7 @@ on each of the parent models from list.
 #### \_extract\_related\_models
 
 ```python
- | async _extract_related_models(related: str, target_model: Type["Model"], prefetch_dict: Dict, select_dict: Dict, fields: Union[Set[Any], Dict[Any, Any], None], exclude_fields: Union[Set[Any], Dict[Any, Any], None], orders_by: Dict) -> None
+ | async _extract_related_models(related: str, target_model: Type["Model"], prefetch_dict: Dict, select_dict: Dict, excludable: "ExcludableItems", orders_by: Dict) -> None
 ```
 
 Constructs queries with required ids and extracts data with fields that should
@@ -261,7 +241,7 @@ Calls itself recurrently to extract deeper nested relations of related model.
 #### \_run\_prefetch\_query
 
 ```python
- | async _run_prefetch_query(target_field: Type["BaseField"], fields: Union[Set[Any], Dict[Any, Any], None], exclude_fields: Union[Set[Any], Dict[Any, Any], None], filter_clauses: List) -> Tuple[str, List]
+ | async _run_prefetch_query(target_field: Type["BaseField"], excludable: "ExcludableItems", filter_clauses: List, related_field_name: str) -> Tuple[str, str, List]
 ```
 
 Actually runs the queries against the database and populates the raw response
@@ -273,8 +253,6 @@ models.
 **Arguments**:
 
 - `target_field (Type["BaseField"])`: ormar field with relation definition
-- `fields (Union[Set[Any], Dict[Any, Any], None])`: fields to include
-- `exclude_fields (Union[Set[Any], Dict[Any, Any], None])`: fields to exclude
 - `filter_clauses (List[sqlalchemy.sql.elements.TextClause])`: list of clauses, actually one clause with ids of relation
 
 **Returns**:
@@ -320,7 +298,7 @@ Updates models that are already loaded, usually children of children.
 #### \_populate\_rows
 
 ```python
- | _populate_rows(rows: List, target_field: Type["ForeignKeyField"], parent_model: Type["Model"], table_prefix: str, fields: Union[Set[Any], Dict[Any, Any], None], exclude_fields: Union[Set[Any], Dict[Any, Any], None], prefetch_dict: Dict, orders_by: Dict) -> None
+ | _populate_rows(rows: List, target_field: Type["ForeignKeyField"], parent_model: Type["Model"], table_prefix: str, exclude_prefix: str, excludable: "ExcludableItems", prefetch_dict: Dict, orders_by: Dict) -> None
 ```
 
 Instantiates children models extracted from given relation.
@@ -334,12 +312,11 @@ and set on the parent model after sorting if needed.
 
 **Arguments**:
 
+- `excludable (ExcludableItems)`: structure of fields to include and exclude
 - `rows (List[sqlalchemy.engine.result.RowProxy])`: raw sql response from the prefetch query
 - `target_field (Type["BaseField"])`: field with relation definition from parent model
 - `parent_model (Type[Model])`: model with relation definition
 - `table_prefix (str)`: prefix of the target table from current relation
-- `fields (Union[Set[Any], Dict[Any, Any], None])`: fields to include
-- `exclude_fields (Union[Set[Any], Dict[Any, Any], None])`: fields to exclude
 - `prefetch_dict (Dict)`: dictionaries of related models to prefetch
 - `orders_by (Dict)`: dictionary of order by clauses by model
 

docs/api/query-set/query-set.md

@@ -38,6 +38,16 @@ Shortcut to model class set on QuerySet.
 
 `(Type[Model])`: model class
 
+<a name="queryset.queryset.QuerySet.rebuild_self"></a>
+#### rebuild\_self
+
+```python
+ | rebuild_self(filter_clauses: List = None, exclude_clauses: List = None, select_related: List = None, limit_count: int = None, offset: int = None, excludable: "ExcludableItems" = None, order_bys: List = None, prefetch_related: List = None, limit_raw_sql: bool = None, proxy_source_model: Optional[Type["Model"]] = None) -> "QuerySet"
+```
+
+Method that returns new instance of queryset based on passed params,
+all not passed params are taken from current values.
+
 <a name="queryset.queryset.QuerySet._prefetch_related_models"></a>
 #### \_prefetch\_related\_models
 
@@ -252,7 +262,7 @@ To chain related `Models` relation use double underscores between names.
 #### fields
 
 ```python
- | fields(columns: Union[List, str, Set, Dict]) -> "QuerySet"
+ | fields(columns: Union[List, str, Set, Dict], _is_exclude: bool = False) -> "QuerySet"
 ```
 
 With `fields()` you can select subset of model columns to limit the data load.
@@ -293,6 +303,7 @@ To include whole nested model specify model related field name and ellipsis.
 
 **Arguments**:
 
+- `_is_exclude (bool)`: flag if it's exclude or include operation
 - `columns (Union[List, str, Set, Dict])`: columns to include
 
 **Returns**:

docs/api/query-set/query.md

@@ -17,38 +17,6 @@ class Query()
 
 Initialize empty order_by dict to be populated later during the query call
 
-<a name="queryset.query.Query.prefixed_pk_name"></a>
-#### prefixed\_pk\_name
-
-```python
- | @property
- | prefixed_pk_name() -> str
-```
-
-Shortcut for extracting prefixed with alias primary key column name from main
-model
-
-**Returns**:
-
-`(str)`: alias of pk column prefix with table name.
-
-<a name="queryset.query.Query.alias"></a>
-#### alias
-
-```python
- | alias(name: str) -> str
-```
-
-Shortcut to extracting column alias from given master model.
-
-**Arguments**:
-
-- `name (str)`: name of column
-
-**Returns**:
-
-`(str)`: alias of given column name
-
 <a name="queryset.query.Query.apply_order_bys_for_primary_model"></a>
 #### apply\_order\_bys\_for\_primary\_model
 

docs/api/query-set/utils.md

@@ -154,7 +154,7 @@ with all children models under their relation keys.
 #### get\_relationship\_alias\_model\_and\_str
 
 ```python
-get_relationship_alias_model_and_str(source_model: Type["Model"], related_parts: List) -> Tuple[str, Type["Model"], str]
+get_relationship_alias_model_and_str(source_model: Type["Model"], related_parts: List) -> Tuple[str, Type["Model"], str, bool]
 ```
 
 Walks the relation to retrieve the actual model on which the clause should be

docs/api/relations/alias-manager.md

@@ -120,7 +120,7 @@ Adds alias to the dictionary of aliases under given key.
 #### resolve\_relation\_alias
 
 ```python
- | resolve_relation_alias(from_model: Type["Model"], relation_name: str) -> str
+ | resolve_relation_alias(from_model: Union[Type["Model"], Type["ModelRow"]], relation_name: str) -> str
 ```
 
 Given model and relation name returns the alias for this relation.
@@ -134,3 +134,24 @@ Given model and relation name returns the alias for this relation.
 
 `(str)`: alias of the relation
 
+<a name="relations.alias_manager.AliasManager.resolve_relation_alias_after_complex"></a>
+#### resolve\_relation\_alias\_after\_complex
+
+```python
+ | resolve_relation_alias_after_complex(source_model: Union[Type["Model"], Type["ModelRow"]], relation_str: str, relation_field: Type["ForeignKeyField"]) -> str
+```
+
+Given source model and relation string returns the alias for this complex
+relation if it exists, otherwise fallback to normal relation from a relation
+field definition.
+
+**Arguments**:
+
+- `relation_field (Type["ForeignKeyField"])`: field with direct relation definition
+- `source_model (source Model)`: model with query starts
+- `relation_str (str)`: string with relation joins defined
+
+**Returns**:
+
+`(str)`: alias of the relation
+

docs/api/relations/queryset-proxy.md

@@ -5,7 +5,7 @@
 ## QuerysetProxy Objects
 
 ```python
-class QuerysetProxy(ormar.QuerySetProtocol)
+class QuerysetProxy()
 ```
 
 Exposes QuerySet methods on relations, but also handles creating and removing
@@ -43,7 +43,7 @@ Set's the queryset. Initialized in RelationProxy.
 #### \_assign\_child\_to\_parent
 
 ```python
- | _assign_child_to_parent(child: Optional["T"]) -> None
+ | _assign_child_to_parent(child: Optional["Model"]) -> None
 ```
 
 Registers child in parents RelationManager.
@@ -56,7 +56,7 @@ Registers child in parents RelationManager.
 #### \_register\_related
 
 ```python
- | _register_related(child: Union["T", Sequence[Optional["T"]]]) -> None
+ | _register_related(child: Union["Model", Sequence[Optional["Model"]]]) -> None
 ```
 
 Registers child/ children in parents RelationManager.
@@ -78,20 +78,35 @@ Cleans the current list of the related models.
 #### create\_through\_instance
 
 ```python
- | async create_through_instance(child: "T") -> None
+ | async create_through_instance(child: "Model", **kwargs: Any) -> None
 ```
 
 Crete a through model instance in the database for m2m relations.
 
 **Arguments**:
 
+- `kwargs (Any)`: dict of additional keyword arguments for through instance
+- `child (Model)`: child model instance
+
+<a name="relations.querysetproxy.QuerysetProxy.update_through_instance"></a>
+#### update\_through\_instance
+
+```python
+ | async update_through_instance(child: "Model", **kwargs: Any) -> None
+```
+
+Updates a through model instance in the database for m2m relations.
+
+**Arguments**:
+
+- `kwargs (Any)`: dict of additional keyword arguments for through instance
 - `child (Model)`: child model instance
 
 <a name="relations.querysetproxy.QuerysetProxy.delete_through_instance"></a>
 #### delete\_through\_instance
 
 ```python
- | async delete_through_instance(child: "T") -> None
+ | async delete_through_instance(child: "Model") -> None
 ```
 
 Removes through model instance from the database for m2m relations.
@@ -256,6 +271,27 @@ Actual call delegated to QuerySet.
 
 `(Model)`: created model
 
+<a name="relations.querysetproxy.QuerysetProxy.update"></a>
+#### update
+
+```python
+ | async update(each: bool = False, **kwargs: Any) -> int
+```
+
+Updates the model table after applying the filters from kwargs.
+
+You have to either pass a filter to narrow down a query or explicitly pass
+each=True flag to affect whole table.
+
+**Arguments**:
+
+- `each (bool)`: flag if whole table should be affected if no filter is passed
+- `kwargs (Any)`: fields names and proper value types
+
+**Returns**:
+
+`(int)`: number of updated rows
+
 <a name="relations.querysetproxy.QuerysetProxy.get_or_create"></a>
 #### get\_or\_create
 

docs/api/relations/relation-manager.md

@@ -10,37 +10,6 @@ class RelationsManager()
 
 Manages relations on a Model, each Model has it's own instance.
 
-<a name="relations.relation_manager.RelationsManager._get_relation_type"></a>
-#### \_get\_relation\_type
-
-```python
- | _get_relation_type(field: Type[BaseField]) -> RelationType
-```
-
-Returns type of the relation declared on a field.
-
-**Arguments**:
-
-- `field (Type[BaseField])`: field with relation declaration
-
-**Returns**:
-
-`(RelationType)`: type of the relation defined on field
-
-<a name="relations.relation_manager.RelationsManager._add_relation"></a>
-#### \_add\_relation
-
-```python
- | _add_relation(field: Type[BaseField]) -> None
-```
-
-Registers relation in the manager.
-Adds Relation instance under field.name.
-
-**Arguments**:
-
-- `field (Type[BaseField])`: field with relation declaration
-
 <a name="relations.relation_manager.RelationsManager.__contains__"></a>
 #### \_\_contains\_\_
 
@@ -62,7 +31,7 @@ Checks if relation with given name is already registered.
 #### get
 
 ```python
- | get(name: str) -> Optional[Union["T", Sequence["T"]]]
+ | get(name: str) -> Optional[Union["Model", Sequence["Model"]]]
 ```
 
 Returns the related model/models if relation is set.
@@ -76,23 +45,6 @@ Actual call is delegated to Relation instance registered under relation name.
 
 `(Optional[Union[Model, List[Model]])`: related model or list of related models if set
 
-<a name="relations.relation_manager.RelationsManager._get"></a>
-#### \_get
-
-```python
- | _get(name: str) -> Optional[Relation]
-```
-
-Returns the actual relation and not the related model(s).
-
-**Arguments**:
-
-- `name (str)`: name of the relation
-
-**Returns**:
-
-`(ormar.relations.relation.Relation)`: Relation instance
-
 <a name="relations.relation_manager.RelationsManager.add"></a>
 #### add
 
@@ -148,3 +100,51 @@ of relation from which you want to remove the parent.
 - `parent (Model)`: parent Model
 - `name (str)`: name of the relation
 
+<a name="relations.relation_manager.RelationsManager._get"></a>
+#### \_get
+
+```python
+ | _get(name: str) -> Optional[Relation]
+```
+
+Returns the actual relation and not the related model(s).
+
+**Arguments**:
+
+- `name (str)`: name of the relation
+
+**Returns**:
+
+`(ormar.relations.relation.Relation)`: Relation instance
+
+<a name="relations.relation_manager.RelationsManager._get_relation_type"></a>
+#### \_get\_relation\_type
+
+```python
+ | _get_relation_type(field: Type["BaseField"]) -> RelationType
+```
+
+Returns type of the relation declared on a field.
+
+**Arguments**:
+
+- `field (Type[BaseField])`: field with relation declaration
+
+**Returns**:
+
+`(RelationType)`: type of the relation defined on field
+
+<a name="relations.relation_manager.RelationsManager._add_relation"></a>
+#### \_add\_relation
+
+```python
+ | _add_relation(field: Type["BaseField"]) -> None
+```
+
+Registers relation in the manager.
+Adds Relation instance under field.name.
+
+**Arguments**:
+
+- `field (Type[BaseField])`: field with relation declaration
+

docs/api/relations/relation-proxy.md

@@ -131,7 +131,7 @@ will be deleted, and not only removed from relation).
 #### add
 
 ```python
- | async add(item: "Model") -> None
+ | async add(item: "Model", **kwargs: Any) -> None
 ```
 
 Adds child model to relation.
@@ -140,5 +140,6 @@ For ManyToMany relations through instance is automatically created.
 
 **Arguments**:
 
+- `kwargs (Any)`: dict of additional keyword arguments for through instance
 - `item (Model)`: child to add to relation
 

docs/api/relations/relation.md

@@ -27,7 +27,7 @@ Keeps related Models and handles adding/removing of the children.
 #### \_\_init\_\_
 
 ```python
- | __init__(manager: "RelationsManager", type_: RelationType, field_name: str, to: Type["T"], through: Type["T"] = None) -> None
+ | __init__(manager: "RelationsManager", type_: RelationType, field_name: str, to: Type["Model"], through: Type["Model"] = None) -> None
 ```
 
 Initialize the Relation and keep the related models either as instances of
@@ -73,7 +73,7 @@ Find child model in RelationProxy if exists.
 #### add
 
 ```python
- | add(child: "T") -> None
+ | add(child: "Model") -> None
 ```
 
 Adds child Model to relation, either sets child as related model or adds
@@ -101,7 +101,7 @@ it from the list in RelationProxy depending on relation type.
 #### get
 
 ```python
- | get() -> Optional[Union[List["T"], "T"]]
+ | get() -> Optional[Union[List["Model"], "Model"]]
 ```
 
 Return the related model or models from RelationProxy.

docs/index.md

@@ -306,7 +306,7 @@ async def joins():
     # visit: https://collerek.github.io/ormar/relations/
 
     # to read more about joins and subqueries
-    # visit: https://collerek.github.io/ormar/queries/delete/
+    # visit: https://collerek.github.io/ormar/queries/joins-and-subqueries/
 
 
 async def filter_and_sort():

docs/models/methods.md

@@ -27,6 +27,39 @@ await track.album.load()
 track.album.name # will return 'Malibu'
 ```
 
+## load_all
+
+`load_all(follow: bool = False, exclude: Union[List, str, Set, Dict] = None) -> Model`
+
+Method works like `load()` but also goes through all relations of the `Model` on which the method is called, 
+and reloads them from database.
+
+By default the `load_all` method loads only models that are directly related (one step away) to the model on which the method is called.
+
+But you can specify the `follow=True` parameter to traverse through nested models and load all of them in the relation tree.
+
+!!!warning
+    To avoid circular updates with `follow=True` set, `load_all` keeps a set of already visited Models, 
+    and won't perform nested `loads` on Models that were already visited.
+    
+    So if you have a diamond or circular relations types you need to perform the loads in a manual way.
+    
+    ```python
+    # in example like this the second Street (coming from City) won't be load_all, so ZipCode won't be reloaded
+    Street -> District -> City -> Street -> ZipCode
+    ```
+
+Method accepts also optional exclude parameter that works exactly the same as exclude_fields method in `QuerySet`.
+That way you can remove fields from related models being refreshed or skip whole related models.
+
+Method performs one database query so it's more efficient than nested calls to `load()` and `all()` on related models.
+
+!!!tip
+    To read more about `exclude` read [exclude_fields][exclude_fields]
+
+!!!warning
+    All relations are cleared on `load_all()`, so if you exclude some nested models they will be empty after call.
+
 ## save
 
 `save() -> self`
@@ -128,3 +161,4 @@ But you can specify the `follow=True` parameter to traverse through nested model
 [alembic]: https://alembic.sqlalchemy.org/en/latest/tutorial.html
 [save status]:  ../models/index/#model-save-status
 [Internals]:  #internals
+[exclude_fields]: ../queries/select-columns.md#exclude_fields

docs/relations/index.md

@@ -52,7 +52,7 @@ class Department(ormar.Model):
 
 To define many-to-many relation use `ManyToMany` field.
 
-```python hl_lines="25-26"
+```python hl_lines="18"
 class Category(ormar.Model):
     class Meta:
         tablename = "categories"
@@ -62,13 +62,6 @@ class Category(ormar.Model):
     id: int = ormar.Integer(primary_key=True)
     name: str = ormar.String(max_length=40)
 
-# note: you need to specify through model
-class PostCategory(ormar.Model):
-    class Meta:
-        tablename = "posts_categories"
-        database = database
-        metadata = metadata
-
 class Post(ormar.Model):
     class Meta:
         tablename = "posts"
@@ -77,9 +70,7 @@ class Post(ormar.Model):
 
     id: int = ormar.Integer(primary_key=True)
     title: str = ormar.String(max_length=200)
-    categories: Optional[Union[Category, List[Category]]] = ormar.ManyToMany(
-        Category, through=PostCategory
-    )
+    categories: Optional[List[Category]] = ormar.ManyToMany(Category)
 ```
 
 
@@ -92,7 +83,52 @@ class Post(ormar.Model):
 
     It allows you to use `await post.categories.all()` but also `await category.posts.all()` to fetch data related only to specific post, category etc.
 
-##Self-reference and postponed references
+## Through fields
+
+As part of the `ManyToMany` relation you can define a through model, that can contain additional 
+fields that you can use to filter, order etc. Fields defined like this are exposed on the reverse
+side of the current query for m2m models. 
+
+So if you query from model `A` to model `B`, only model `B` has through field exposed.
+Which kind of make sense, since it's a one through model/field for each of related models.
+
+```python hl_lines="10-15"
+class Category(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "categories"
+
+    id = ormar.Integer(primary_key=True)
+    name = ormar.String(max_length=40)
+
+# you can specify additional fields on through model
+class PostCategory(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "posts_x_categories"
+
+    id: int = ormar.Integer(primary_key=True)
+    sort_order: int = ormar.Integer(nullable=True)
+    param_name: str = ormar.String(default="Name", max_length=200)
+
+
+class Post(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    title: str = ormar.String(max_length=200)
+    categories = ormar.ManyToMany(Category, through=PostCategory)
+```
+
+!!!tip
+    To read more about many-to-many relations and through fields visit [many-to-many][many-to-many] section
+
+
+!!!tip
+    ManyToMany allows you to query the related models with [queryset-proxy][queryset-proxy].
+    
+    It allows you to use `await post.categories.all()` but also `await category.posts.all()` to fetch data related only to specific post, category etc.
+
+## Self-reference and postponed references
 
 In order to create auto-relation or create two models that reference each other in at least two
 different relations (remember the reverse side is auto-registered for you), you need to use

docs/relations/many-to-many.md

@@ -1,6 +1,6 @@
 # ManyToMany
 
-`ManyToMany(to, through)` has required parameters `to` and `through` that takes target and relation `Model` classes.  
+`ManyToMany(to, through)` has required parameters `to` and optional `through` that takes target and relation `Model` classes.  
 
 Sqlalchemy column and Type are automatically taken from target `Model`.
 
@@ -9,7 +9,7 @@ Sqlalchemy column and Type are automatically taken from target `Model`.
 
 ## Defining Models
 
-```Python hl_lines="32 49-50"
+```Python hl_lines="40"
 --8<-- "../docs_src/relations/docs002.py"
 ```
 
@@ -20,8 +20,154 @@ post = await Post.objects.create(title="Hello, M2M", author=guido)
 news = await Category.objects.create(name="News")
 ```
 
+## Through Model
+
+Optionally if you want to add additional fields you can explicitly create and pass
+the through model class.
+
+```Python hl_lines="14-20 29"
+--8<-- "../docs_src/relations/docs004.py"
+```
+
+!!!warning
+    Note that even of you do not provide through model it's going to be created for you automatically and 
+    still has to be included in example in `alembic` migrations. 
+
+!!!tip
+      Note that you need to provide `through` model if you want to 
+      customize the `Through` model name or the database table name of this model.
+
+If you do not provide the Through field it will be generated for you. 
+
+The default naming convention is:
+
+*  for class name it's union of both classes name (parent+other) so in example above 
+   it would be `PostCategory`
+*  for table name it similar but with underscore in between and s in the end of class 
+   lowercase name, in example above would be `posts_categorys`
+   
+## Through Fields
+
+The through field is auto added to the reverse side of the relation. 
+
+The exposed field is named as lowercase `Through` class name.
+
+The exposed field **explicitly has no relations loaded** as the relation is already populated in `ManyToMany` field,
+so it's useful only when additional fields are provided on `Through` model.
+
+In a sample model setup as following:
+
+```Python hl_lines="14-20 29"
+--8<-- "../docs_src/relations/docs004.py"
+```
+
+the through field can be used as a normal model field in most of the QuerySet operations.
+
+Note that through field is attached only to related side of the query so:
+
+```python
+post = await Post.objects.select_related("categories").get()
+# source model has no through field
+assert post.postcategory is None
+# related models have through field
+assert post.categories[0].postcategory is not None
+
+# same is applicable for reversed query
+category = await Category.objects.select_related("posts").get()
+assert category.postcategory is None
+assert category.posts[0].postcategory is not None
+```
+
+Through field can be used for filtering the data.
+```python
+post = (
+        await Post.objects.select_related("categories")
+        .filter(postcategory__sort_order__gt=1)
+        .get()
+        )
+```
+
+!!!tip
+    Note that despite that the actual instance is not populated on source model,
+    in queries, order by statements etc you can access through model from both sides.
+    So below query has exactly the same effect (note access through `categories`)
+    
+    ```python
+    post = (
+        await Post.objects.select_related("categories")
+        .filter(categories__postcategory__sort_order__gt=1)
+        .get()
+        )
+    ```
+
+Through model can be used in order by queries.
+```python
+post = (
+        await Post.objects.select_related("categories")
+        .order_by("-postcategory__sort_order")
+        .get()
+    )
+```
+
+You can also select subset of the columns in a normal `QuerySet` way with `fields` 
+and `exclude_fields`.
+
+```python
+post2 = (
+        await Post.objects.select_related("categories")
+        .exclude_fields("postcategory__param_name")
+        .get()
+        )
+```
+
+!!!warning
+    Note that because through fields explicitly nullifies all relation fields, as relation
+    is populated in ManyToMany field, you should not use the standard model methods like
+    `save()` and `update()` before re-loading the field from database.
+
+If you want to modify the through field in place remember to reload it from database.
+Otherwise you will set relations to None so effectively make the field useless!
+
+```python
+# always reload the field before modification
+await post2.categories[0].postcategory.load()
+# only then update the field
+await post2.categories[0].postcategory.update(sort_order=3)
+```
+Note that reloading the model effectively reloads the relations as `pk_only` models 
+(only primary key is set) so they are not fully populated, but it's enough to preserve 
+the relation on update.
+
+!!!warning
+    If you use i.e. `fastapi` the partially loaded related models on through field might cause
+    `pydantic` validation errors (that's the primary reason why they are not populated by default).
+    So either you need to exclude the related fields in your response, or fully load the related
+    models. In example above it would mean:
+    ```python
+    await post2.categories[0].postcategory.post.load()
+    await post2.categories[0].postcategory.category.load()
+    ```
+    Alternatively you can use `load_all()`:
+    ```python
+    await post2.categories[0].postcategory.load_all()
+    ```
+
+**Preferred way of update is through queryset proxy `update()` method**
+
+```python
+# filter the desired related model with through field and update only through field params
+await post2.categories.filter(name='Test category').update(postcategory={"sort_order": 3})
+```
+
+
+## Relation methods
+
 ### add
 
+`add(item: Model, **kwargs)`
+
+Allows you to add model to ManyToMany relation. 
+
 ```python
 # Add a category to a post.
 await post.categories.add(news)
@@ -30,10 +176,24 @@ await news.posts.add(post)
 ```
 
 !!!warning
-    In all not None cases the primary key value for related model **has to exist in database**.
+    In all not `None` cases the primary key value for related model **has to exist in database**.
     
     Otherwise an IntegrityError will be raised by your database driver library.
 
+If you declare your models with a Through model with additional fields, you can populate them
+during adding child model to relation.
+
+In order to do so, pass keyword arguments with field names and values to `add()` call.
+
+Note that this works only for `ManyToMany` relations.
+
+```python
+post = await Post(title="Test post").save()
+category = await Category(name="Test category").save()
+# apart from model pass arguments referencing through model fields
+await post.categories.add(category, sort_order=1, param_name='test')
+```
+
 ### remove
 
 Removal of the related model one by one.

docs/relations/queryset-proxy.md

@@ -104,6 +104,29 @@ assert len(await post.categories.all()) == 2
 !!!tip
     Read more in queries documentation [create][create]
 
+For `ManyToMany` relations there is an additional functionality of passing parameters
+that will be used to create a through model if you declared additional fields on explicitly 
+provided Through model.
+
+Given sample like this:
+
+```Python hl_lines="14-20, 29"
+--8<-- "../docs_src/relations/docs004.py"
+```
+
+You can populate fields on through model in the `create()` call in a following way:
+
+```python
+
+post = await Post(title="Test post").save()
+await post.categories.create(
+    name="Test category1",
+    # in arguments pass a dictionary with name of the through field and keys
+    # corresponding to through model fields
+    postcategory={"sort_order": 1, "param_name": "volume"},
+)
+```
+
 ### get_or_create
 
 `get_or_create(**kwargs) -> Model`
@@ -122,6 +145,29 @@ Updates the model, or in case there is no match in database creates a new one.
 !!!tip
     Read more in queries documentation [update_or_create][update_or_create]
 
+### update
+
+`update(**kwargs, each:bool = False) -> int`
+
+Updates the related model with provided keyword arguments, return number of updated rows.
+
+!!!tip
+    Read more in queries documentation [update][update]
+
+Note that for `ManyToMany` relations update can also accept an argument with through field
+name and a dictionary of fields.
+
+```Python hl_lines="14-20 29"
+--8<-- "../docs_src/relations/docs004.py"
+```
+
+In example above you can update attributes of `postcategory` in a following call:
+```python
+await post.categories.filter(name="Test category3").update(
+            postcategory={"sort_order": 4}
+        )
+```
+
 ## Filtering and sorting
 
 ### filter
@@ -251,6 +297,7 @@ Returns a bool value to confirm if there are rows matching the given criteria (a
 [create]: ../queries/create.md#create
 [get_or_create]: ../queries/read.md#get_or_create
 [update_or_create]: ../queries/update.md#update_or_create
+[update]: ../queries/update.md#update
 [filter]: ../queries/filter-and-sort.md#filter
 [exclude]: ../queries/filter-and-sort.md#exclude
 [select_related]: ../queries/joins-and-subqueries.md#select_related

docs/releases.md

@@ -1,9 +1,50 @@
+# 0.9.6
+
+##Important
+* `Through` model for `ManyToMany` relations now **becomes optional**. It's not a breaking change
+  since if you provide it everything works just fine as it used to. So if you don't want or need any additional 
+  fields on `Through` model you can skip it. Note that it's going to be created for you automatically and 
+  still has to be included in example in `alembic` migrations. 
+  If you want to delete existing one check the default naming convention to adjust your existing database structure.
+
+  Note that you still need to provide it if you want to 
+  customize the `Through` model name or the database table name.
+
+## Features
+* Add `update` method to `QuerysetProxy` so now it's possible to update related models directly from parent model
+  in `ManyToMany` relations and in reverse `ForeignKey` relations. Note that update like in `QuerySet` `update` returns number of
+  updated models and **does not update related models in place** on parent model. To get the refreshed data on parent model you need to refresh
+  the related models (i.e. `await model_instance.related.all()`)
+* Add `load_all(follow=False, exclude=None)` model method that allows to load current instance of the model
+  with all related models in one call. By default it loads only directly related models but setting
+  `follow=True` causes traversing the tree (avoiding loops). You can also pass `exclude` parameter
+  that works the same as `QuerySet.exclude_fields()` method.
+* Added possibility to add more fields on `Through` model for `ManyToMany` relationships:
+    * name of the through model field is the lowercase name of the Through class
+    * you can pass additional fields when calling `add(child, **kwargs)` on relation (on `QuerysetProxy`)
+    * you can pass additional fields when calling `create(**kwargs)` on relation (on `QuerysetProxy`)
+        when one of the keyword arguments should be the through model name with a dict of values
+    * you can order by on through model fields
+    * you can filter on through model fields
+    * you can include and exclude fields on through models
+    * through models are attached only to related models (i.e. if you query from A to B -> only on B)
+    * note that through models are explicitly loaded without relations -> relation is already populated in ManyToMany field. 
+    * note that just like before you cannot declare the relation fields on through model, they will be populated for you by `ormar`,
+      but now if you try to do so `ModelDefinitionError` will be thrown
+    * check the updated ManyToMany relation docs for more information
+
+# Other
+* Updated docs and api docs
+* Refactors and optimisations mainly related to filters, exclusions and order bys
+
+
 # 0.9.5
 
 ## Fixes
 * Fix creation of `pydantic` FieldInfo after update of `pydantic` to version >=1.8
 * Pin required dependency versions to avoid such situations in the future
 
+
 # 0.9.4
 
 ## Fixes

docs_src/relations/docs002.py

@@ -29,15 +29,6 @@ class Category(ormar.Model):
     name: str = ormar.String(max_length=40)
 
 
-class PostCategory(ormar.Model):
-    class Meta:
-        tablename = "posts_categories"
-        database = database
-        metadata = metadata
-
-    # If there are no additional columns id will be created automatically as Integer
-
-
 class Post(ormar.Model):
     class Meta:
         tablename = "posts"
@@ -46,7 +37,5 @@ class Post(ormar.Model):
 
     id: int = ormar.Integer(primary_key=True)
     title: str = ormar.String(max_length=200)
-    categories: Optional[Union[Category, List[Category]]] = ormar.ManyToMany(
-        Category, through=PostCategory
-    )
+    categories: Optional[List[Category]] = ormar.ManyToMany(Category)
     author: Optional[Author] = ormar.ForeignKey(Author)

docs_src/relations/docs004.py

@@ -0,0 +1,29 @@
+class BaseMeta(ormar.ModelMeta):
+    database = database
+    metadata = metadata
+
+
+class Category(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "categories"
+
+    id = ormar.Integer(primary_key=True)
+    name = ormar.String(max_length=40)
+
+
+class PostCategory(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "posts_x_categories"
+
+    id: int = ormar.Integer(primary_key=True)
+    sort_order: int = ormar.Integer(nullable=True)
+    param_name: str = ormar.String(default="Name", max_length=200)
+
+
+class Post(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    title: str = ormar.String(max_length=200)
+    categories = ormar.ManyToMany(Category, through=PostCategory)

mkdocs.yml

@@ -53,9 +53,11 @@ nav:
             - Relation Mixin: api/models/mixins/relation-mixin.md
             - Save Prepare Mixin: api/models/mixins/save-prepare-mixin.md
         - api/models/model.md
+        - Model Row: api/models/model-row.md
         - New BaseModel: api/models/new-basemodel.md
         - Model Table Proxy: api/models/model-table-proxy.md
         - Model Metaclass: api/models/model-metaclass.md
+        - Excludable Items: api/models/excludable-items.md
       - Fields:
           - Base Field: api/fields/base-field.md
           - Model Fields: api/fields/model-fields.md

ormar/__init__.py

@@ -54,9 +54,9 @@ from ormar.fields import (
     UUID,
     UniqueColumns,
 )  # noqa: I100
-from ormar.models import Model
+from ormar.models import ExcludableItems, Model
 from ormar.models.metaclass import ModelMeta
-from ormar.queryset import QuerySet
+from ormar.queryset import OrderAction, QuerySet
 from ormar.relations import RelationType
 from ormar.signals import Signal
 
@@ -106,4 +106,6 @@ __all__ = [
     "BaseField",
     "ManyToManyField",
     "ForeignKeyField",
+    "OrderAction",
+    "ExcludableItems",
 ]

ormar/fields/__init__.py

@@ -21,6 +21,7 @@ from ormar.fields.model_fields import (
     Time,
     UUID,
 )
+from ormar.fields.through_field import Through, ThroughField
 
 __all__ = [
     "Decimal",
@@ -41,4 +42,6 @@ __all__ = [
     "BaseField",
     "UniqueColumns",
     "ForeignKeyField",
+    "ThroughField",
+    "Through",
 ]

ormar/fields/base.py

@@ -37,9 +37,13 @@ class BaseField(FieldInfo):
     index: bool
     unique: bool
     pydantic_only: bool
-    virtual: bool = False
     choices: typing.Sequence
 
+    virtual: bool = False  # ManyToManyFields and reverse ForeignKeyFields
+    is_multi: bool = False  # ManyToManyField
+    is_relation: bool = False  # ForeignKeyField + subclasses
+    is_through: bool = False  # ThroughFields
+
     owner: Type["Model"]
     to: Type["Model"]
     through: Type["Model"]
@@ -63,7 +67,7 @@ class BaseField(FieldInfo):
         :return: result of the check
         :rtype: bool
         """
-        return not issubclass(cls, ormar.fields.ManyToManyField) and not cls.virtual
+        return not cls.is_multi and not cls.virtual
 
     @classmethod
     def get_alias(cls) -> str:

ormar/fields/foreign_key.py

@@ -48,7 +48,7 @@ def create_dummy_instance(fk: Type["Model"], pk: Any = None) -> "Model":
         **{
             k: create_dummy_instance(v.to)
             for k, v in fk.Meta.model_fields.items()
-            if isinstance(v, ForeignKeyField) and not v.nullable and not v.virtual
+            if v.is_relation and not v.nullable and not v.virtual
         },
     }
     return fk(**init_dict)
@@ -73,7 +73,9 @@ def create_dummy_model(
         "".join(choices(string.ascii_uppercase, k=2)) + uuid.uuid4().hex[:4]
     ).lower()
     fields = {f"{pk_field.name}": (pk_field.__type__, None)}
-    dummy_model = create_model(   # type: ignore
+
+    dummy_model = create_model(  # type: ignore
+
         f"PkOnly{base_model.get_name(lower=False)}{alias}",
         __module__=base_model.__module__,
         **fields,  # type: ignore
@@ -217,6 +219,7 @@ def ForeignKey(  # noqa CFQ002
         ondelete=ondelete,
         owner=owner,
         self_reference=self_reference,
+        is_relation=True,
     )
 
     return type("ForeignKey", (ForeignKeyField, BaseField), namespace)
@@ -457,3 +460,24 @@ class ForeignKeyField(BaseField):
             value.__class__.__name__, cls._construct_model_from_pk
         )(value, child, to_register)
         return model
+
+    @classmethod
+    def get_relation_name(cls) -> str:  # pragma: no cover
+        """
+        Returns name of the relation, which can be a own name or through model
+        names for m2m models
+
+        :return: result of the check
+        :rtype: bool
+        """
+        return cls.name
+
+    @classmethod
+    def get_source_model(cls) -> Type["Model"]:  # pragma: no cover
+        """
+        Returns model from which the relation comes -> either owner or through model
+
+        :return: source model
+        :rtype: Type["Model"]
+        """
+        return cls.owner

ormar/fields/many_to_many.py

@@ -1,8 +1,9 @@
 import sys
-from typing import Any, List, Optional, TYPE_CHECKING, Tuple, Type, Union
+from typing import Any, List, Optional, TYPE_CHECKING, Tuple, Type, Union, cast
 
 from pydantic.typing import ForwardRef, evaluate_forwardref
 import ormar  # noqa: I100
+from ormar import ModelDefinitionError
 from ormar.fields import BaseField
 from ormar.fields.foreign_key import ForeignKeyField
 
@@ -17,6 +18,21 @@ if TYPE_CHECKING:  # pragma no cover
 REF_PREFIX = "#/components/schemas/"
 
 
+def forbid_through_relations(through: Type["Model"]) -> None:
+    """
+    Verifies if the through model does not have relations.
+
+    :param through: through Model to be checked
+    :type through: Type['Model]
+    """
+    if any(field.is_relation for field in through.Meta.model_fields.values()):
+        raise ModelDefinitionError(
+            f"Through Models cannot have explicit relations "
+            f"defined. Remove the relations from Model "
+            f"{through.get_name(lower=False)}"
+        )
+
+
 def populate_m2m_params_based_on_to_model(
     to: Type["Model"], nullable: bool
 ) -> Tuple[Any, Any]:
@@ -43,7 +59,7 @@ def populate_m2m_params_based_on_to_model(
 
 def ManyToMany(
     to: "ToType",
-    through: "ToType",
+    through: Optional["ToType"] = None,
     *,
     name: str = None,
     unique: bool = False,
@@ -77,6 +93,8 @@ def ManyToMany(
     nullable = kwargs.pop("nullable", True)
     owner = kwargs.pop("owner", None)
     self_reference = kwargs.pop("self_reference", False)
+    if through is not None and through.__class__ != ForwardRef:
+        forbid_through_relations(cast(Type["Model"], through))
 
     if to.__class__ == ForwardRef:
         __type__ = to if not nullable else Optional[to]
@@ -103,6 +121,8 @@ def ManyToMany(
         server_default=None,
         owner=owner,
         self_reference=self_reference,
+        is_relation=True,
+        is_multi=True,
     )
 
     return type("ManyToMany", (ManyToManyField, BaseField), namespace)
@@ -187,3 +207,45 @@ class ManyToManyField(ForeignKeyField, ormar.QuerySetProtocol, ormar.RelationPro
                 globalns,
                 localns or None,
             )
+            forbid_through_relations(cls.through)
+
+    @classmethod
+    def get_relation_name(cls) -> str:
+        """
+        Returns name of the relation, which can be a own name or through model
+        names for m2m models
+
+        :return: result of the check
+        :rtype: bool
+        """
+        if cls.self_reference and cls.name == cls.self_reference_primary:
+            return cls.default_source_field_name()
+        return cls.default_target_field_name()
+
+    @classmethod
+    def get_source_model(cls) -> Type["Model"]:
+        """
+        Returns model from which the relation comes -> either owner or through model
+
+        :return: source model
+        :rtype: Type["Model"]
+        """
+        return cls.through
+
+    @classmethod
+    def create_default_through_model(cls) -> None:
+        """
+        Creates default empty through model if no additional fields are required.
+        """
+        owner_name = cls.owner.get_name(lower=False)
+        to_name = cls.to.get_name(lower=False)
+        class_name = f"{owner_name}{to_name}"
+        table_name = f"{owner_name.lower()}s_{to_name.lower()}s"
+        new_meta_namespace = {
+            "tablename": table_name,
+            "database": cls.owner.Meta.database,
+            "metadata": cls.owner.Meta.metadata,
+        }
+        new_meta = type("Meta", (), new_meta_namespace)
+        through_model = type(class_name, (ormar.Model,), {"Meta": new_meta})
+        cls.through = cast(Type["Model"], through_model)

ormar/fields/through_field.py

@@ -0,0 +1,66 @@
+import sys
+from typing import Any, TYPE_CHECKING, Type, Union
+
+from ormar.fields.base import BaseField
+from ormar.fields.foreign_key import ForeignKeyField
+
+if TYPE_CHECKING:  # pragma no cover
+    from ormar import Model
+    from pydantic.typing import ForwardRef
+
+    if sys.version_info < (3, 7):
+        ToType = Type[Model]
+    else:
+        ToType = Union[Type[Model], ForwardRef]
+
+
+def Through(  # noqa CFQ002
+    to: "ToType", *, name: str = None, related_name: str = None, **kwargs: Any,
+) -> Any:
+    """
+    Despite a name it's a function that returns constructed ThroughField.
+    It's a special field populated only for m2m relations.
+    Accepts number of relation setting parameters as well as all BaseField ones.
+
+    :param to: target related ormar Model
+    :type to: Model class
+    :param name: name of the database field - later called alias
+    :type name: str
+    :param related_name: name of reversed FK relation populated for you on to model
+    :type related_name: str
+    It is for reversed FK and auto generated FK on through model in Many2Many relations.
+    :param kwargs: all other args to be populated by BaseField
+    :type kwargs: Any
+    :return: ormar ForeignKeyField with relation to selected model
+    :rtype: ForeignKeyField
+    """
+    nullable = kwargs.pop("nullable", False)
+    owner = kwargs.pop("owner", None)
+    namespace = dict(
+        __type__=to,
+        to=to,
+        through=None,
+        alias=name,
+        name=kwargs.pop("real_name", None),
+        related_name=related_name,
+        virtual=True,
+        owner=owner,
+        nullable=nullable,
+        unique=False,
+        column_type=None,
+        primary_key=False,
+        index=False,
+        pydantic_only=False,
+        default=None,
+        server_default=None,
+        is_relation=True,
+        is_through=True,
+    )
+
+    return type("Through", (ThroughField, BaseField), namespace)
+
+
+class ThroughField(ForeignKeyField):
+    """
+    Field class used to access ManyToMany model through model.
+    """

ormar/models/__init__.py

@@ -5,6 +5,8 @@ ass well as vast number of helper functions for pydantic, sqlalchemy and relatio
 """
 
 from ormar.models.newbasemodel import NewBaseModel  # noqa I100
+from ormar.models.model_row import ModelRow  # noqa I100
 from ormar.models.model import Model  # noqa I100
+from ormar.models.excludable import ExcludableItems  # noqa I100
 
-__all__ = ["NewBaseModel", "Model"]
+__all__ = ["NewBaseModel", "Model", "ModelRow", "ExcludableItems"]

ormar/models/excludable.py

@@ -0,0 +1,271 @@
+from dataclasses import dataclass, field
+from typing import Dict, List, Set, TYPE_CHECKING, Tuple, Type, Union
+
+from ormar.queryset.utils import get_relationship_alias_model_and_str
+
+if TYPE_CHECKING:  # pragma: no cover
+    from ormar import Model
+
+
+@dataclass
+class Excludable:
+    """
+    Class that keeps sets of fields to exclude and include
+    """
+
+    include: Set = field(default_factory=set)
+    exclude: Set = field(default_factory=set)
+
+    def get_copy(self) -> "Excludable":
+        """
+        Return copy of self to avoid in place modifications
+        :return: copy of self with copied sets
+        :rtype: ormar.models.excludable.Excludable
+        """
+        _copy = self.__class__()
+        _copy.include = {x for x in self.include}
+        _copy.exclude = {x for x in self.exclude}
+        return _copy
+
+    def set_values(self, value: Set, is_exclude: bool) -> None:
+        """
+        Appends the data to include/exclude sets.
+
+        :param value: set of values to add
+        :type value: set
+        :param is_exclude: flag if values are to be excluded or included
+        :type is_exclude: bool
+        """
+        prop = "exclude" if is_exclude else "include"
+        current_value = getattr(self, prop)
+        current_value.update(value)
+        setattr(self, prop, current_value)
+
+    def is_included(self, key: str) -> bool:
+        """
+        Check if field in included (in set or set is {...})
+        :param key: key to check
+        :type key: str
+        :return: result of the check
+        :rtype: bool
+        """
+        return (... in self.include or key in self.include) if self.include else True
+
+    def is_excluded(self, key: str) -> bool:
+        """
+        Check if field in excluded (in set or set is {...})
+        :param key: key to check
+        :type key: str
+        :return: result of the check
+        :rtype: bool
+        """
+        return (... in self.exclude or key in self.exclude) if self.exclude else False
+
+
+class ExcludableItems:
+    """
+    Keeps a dictionary of Excludables by alias + model_name keys
+    to allow quick lookup by nested models without need to travers
+    deeply nested dictionaries and passing include/exclude around
+    """
+
+    def __init__(self) -> None:
+        self.items: Dict[str, Excludable] = dict()
+
+    @classmethod
+    def from_excludable(cls, other: "ExcludableItems") -> "ExcludableItems":
+        """
+        Copy passed ExcludableItems to avoid inplace modifications.
+
+        :param other: other excludable items to be copied
+        :type other: ormar.models.excludable.ExcludableItems
+        :return: copy of other
+        :rtype: ormar.models.excludable.ExcludableItems
+        """
+        new_excludable = cls()
+        for key, value in other.items.items():
+            new_excludable.items[key] = value.get_copy()
+        return new_excludable
+
+    def get(self, model_cls: Type["Model"], alias: str = "") -> Excludable:
+        """
+        Return Excludable for given model and alias.
+
+        :param model_cls: target model to check
+        :type model_cls: ormar.models.metaclass.ModelMetaclass
+        :param alias: table alias from relation manager
+        :type alias: str
+        :return: Excludable for given model and alias
+        :rtype: ormar.models.excludable.Excludable
+        """
+        key = f"{alias + '_' if alias else ''}{model_cls.get_name(lower=True)}"
+        excludable = self.items.get(key)
+        if not excludable:
+            excludable = Excludable()
+            self.items[key] = excludable
+        return excludable
+
+    def build(
+        self,
+        items: Union[List[str], str, Tuple[str], Set[str], Dict],
+        model_cls: Type["Model"],
+        is_exclude: bool = False,
+    ) -> None:
+        """
+        Receives the one of the types of items and parses them as to achieve
+        a end situation with one excludable per alias/model in relation.
+
+        Each excludable has two sets of values - one to include, one to exclude.
+
+        :param items: values to be included or excluded
+        :type items: Union[List[str], str, Tuple[str], Set[str], Dict]
+        :param model_cls: source model from which relations are constructed
+        :type model_cls: ormar.models.metaclass.ModelMetaclass
+        :param is_exclude: flag if items should be included or excluded
+        :type is_exclude: bool
+        """
+        if isinstance(items, str):
+            items = {items}
+
+        if isinstance(items, Dict):
+            self._traverse_dict(
+                values=items,
+                source_model=model_cls,
+                model_cls=model_cls,
+                is_exclude=is_exclude,
+            )
+
+        else:
+            items = set(items)
+            nested_items = set(x for x in items if "__" in x)
+            items.difference_update(nested_items)
+            self._set_excludes(
+                items=items,
+                model_name=model_cls.get_name(lower=True),
+                is_exclude=is_exclude,
+            )
+            if nested_items:
+                self._traverse_list(
+                    values=nested_items, model_cls=model_cls, is_exclude=is_exclude
+                )
+
+    def _set_excludes(
+        self, items: Set, model_name: str, is_exclude: bool, alias: str = ""
+    ) -> None:
+        """
+        Sets set of values to be included or excluded for given key and model.
+
+        :param items: items to include/exclude
+        :type items: set
+        :param model_name: name of model to construct key
+        :type model_name: str
+        :param is_exclude: flag if values should be included or excluded
+        :type is_exclude: bool
+        :param alias:
+        :type alias: str
+        """
+        key = f"{alias + '_' if alias else ''}{model_name}"
+        excludable = self.items.get(key)
+        if not excludable:
+            excludable = Excludable()
+        excludable.set_values(value=items, is_exclude=is_exclude)
+        self.items[key] = excludable
+
+    def _traverse_dict(  # noqa: CFQ002
+        self,
+        values: Dict,
+        source_model: Type["Model"],
+        model_cls: Type["Model"],
+        is_exclude: bool,
+        related_items: List = None,
+        alias: str = "",
+    ) -> None:
+        """
+        Goes through dict of nested values and construct/update Excludables.
+
+        :param values: items to include/exclude
+        :type values: Dict
+        :param source_model: source model from which relations are constructed
+        :type source_model: ormar.models.metaclass.ModelMetaclass
+        :param model_cls: model from which current relation is constructed
+        :type model_cls: ormar.models.metaclass.ModelMetaclass
+        :param is_exclude: flag if values should be included or excluded
+        :type is_exclude: bool
+        :param related_items: list of names of related fields chain
+        :type related_items: List
+        :param alias: alias of relation
+        :type alias: str
+        """
+        self_fields = set()
+        related_items = related_items[:] if related_items else []
+        for key, value in values.items():
+            if value is ...:
+                self_fields.add(key)
+            elif isinstance(value, set):
+                (
+                    table_prefix,
+                    target_model,
+                    _,
+                    _,
+                ) = get_relationship_alias_model_and_str(
+                    source_model=source_model, related_parts=related_items + [key]
+                )
+                self._set_excludes(
+                    items=value,
+                    model_name=target_model.get_name(),
+                    is_exclude=is_exclude,
+                    alias=table_prefix,
+                )
+            else:
+                # dict
+                related_items.append(key)
+                (
+                    table_prefix,
+                    target_model,
+                    _,
+                    _,
+                ) = get_relationship_alias_model_and_str(
+                    source_model=source_model, related_parts=related_items
+                )
+                self._traverse_dict(
+                    values=value,
+                    source_model=source_model,
+                    model_cls=target_model,
+                    is_exclude=is_exclude,
+                    related_items=related_items,
+                    alias=table_prefix,
+                )
+        if self_fields:
+            self._set_excludes(
+                items=self_fields,
+                model_name=model_cls.get_name(),
+                is_exclude=is_exclude,
+                alias=alias,
+            )
+
+    def _traverse_list(
+        self, values: Set[str], model_cls: Type["Model"], is_exclude: bool
+    ) -> None:
+        """
+        Goes through list of values and construct/update Excludables.
+
+        :param values: items to include/exclude
+        :type values: set
+        :param model_cls: model from which current relation is constructed
+        :type model_cls: ormar.models.metaclass.ModelMetaclass
+        :param is_exclude: flag if values should be included or excluded
+        :type is_exclude: bool
+        """
+        # here we have only nested related keys
+        for key in values:
+            key_split = key.split("__")
+            related_items, field_name = key_split[:-1], key_split[-1]
+            (table_prefix, target_model, _, _) = get_relationship_alias_model_and_str(
+                source_model=model_cls, related_parts=related_items
+            )
+            self._set_excludes(
+                items={field_name},
+                model_name=target_model.get_name(),
+                is_exclude=is_exclude,
+                alias=table_prefix,
+            )

ormar/models/helpers/models.py

@@ -1,3 +1,4 @@
+import collections
 import itertools
 import sqlite3
 from typing import Any, Dict, List, TYPE_CHECKING, Tuple, Type
@@ -21,7 +22,7 @@ def is_field_an_forward_ref(field: Type["BaseField"]) -> bool:
     :return: result of the check
     :rtype: bool
     """
-    return issubclass(field, ormar.ForeignKeyField) and (
+    return field.is_relation and (
         field.to.__class__ == ForwardRef or field.through.__class__ == ForwardRef
     )
 
@@ -123,7 +124,7 @@ def extract_annotations_and_default_vals(attrs: Dict) -> Tuple[Dict, Dict]:
     return attrs, model_fields
 
 
-def group_related_list(list_: List) -> Dict:
+def group_related_list(list_: List) -> collections.OrderedDict:
     """
     Translates the list of related strings into a dictionary.
     That way nested models are grouped to traverse them in a right order
@@ -152,7 +153,9 @@ def group_related_list(list_: List) -> Dict:
             result_dict[key] = group_related_list(new)
         else:
             result_dict.setdefault(key, []).extend(new)
-    return {k: v for k, v in sorted(result_dict.items(), key=lambda item: len(item[1]))}
+    return collections.OrderedDict(
+        sorted(result_dict.items(), key=lambda item: len(item[1]))
+    )
 
 
 def meta_field_not_set(model: Type["Model"], field_name: str) -> bool:

ormar/models/helpers/pydantic.py

@@ -6,14 +6,15 @@ from pydantic.fields import ModelField
 from pydantic.utils import lenient_issubclass
 
 import ormar  # noqa: I100, I202
-from ormar.fields import BaseField, ManyToManyField
+from ormar.fields import BaseField
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
+    from ormar.fields import ManyToManyField
 
 
 def create_pydantic_field(
-    field_name: str, model: Type["Model"], model_field: Type[ManyToManyField]
+    field_name: str, model: Type["Model"], model_field: Type["ManyToManyField"]
 ) -> None:
     """
     Registers pydantic field on through model that leads to passed model
@@ -59,7 +60,7 @@ def get_pydantic_field(field_name: str, model: Type["Model"]) -> "ModelField":
 
 
 def populate_default_pydantic_field_value(
-    ormar_field: Type[BaseField], field_name: str, attrs: dict
+    ormar_field: Type["BaseField"], field_name: str, attrs: dict
 ) -> dict:
     """
     Grabs current value of the ormar Field in class namespace

ormar/models/helpers/related_names_validation.py

@@ -25,7 +25,7 @@ def validate_related_names_in_relations(  # noqa CCR001
     """
     already_registered: Dict[str, List[Optional[str]]] = dict()
     for field in model_fields.values():
-        if issubclass(field, ormar.ForeignKeyField):
+        if field.is_relation:
             to_name = (
                 field.to.get_name()
                 if not field.to.__class__ == ForwardRef

ormar/models/helpers/relations.py

@@ -1,14 +1,14 @@
-from typing import TYPE_CHECKING, Type
+from typing import TYPE_CHECKING, Type, cast
 
 import ormar
 from ormar import ForeignKey, ManyToMany
-from ormar.fields import ManyToManyField
-from ormar.fields.foreign_key import ForeignKeyField
+from ormar.fields import Through
 from ormar.models.helpers.sqlalchemy import adjust_through_many_to_many_model
 from ormar.relations import AliasManager
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
+    from ormar.fields import ManyToManyField, ForeignKeyField
 
 alias_manager = AliasManager()
 
@@ -32,7 +32,7 @@ def register_relation_on_build(field: Type["ForeignKeyField"]) -> None:
     )
 
 
-def register_many_to_many_relation_on_build(field: Type[ManyToManyField]) -> None:
+def register_many_to_many_relation_on_build(field: Type["ManyToManyField"]) -> None:
     """
     Registers connection between through model and both sides of the m2m relation.
     Registration include also reverse relation side to be able to join both sides.
@@ -81,11 +81,10 @@ def expand_reverse_relationships(model: Type["Model"]) -> None:
     :param model: model on which relation should be checked and registered
     :type model: Model class
     """
-    for model_field in model.Meta.model_fields.values():
-        if (
-            issubclass(model_field, ForeignKeyField)
-            and not model_field.has_unresolved_forward_refs()
-        ):
+    model_fields = list(model.Meta.model_fields.values())
+    for model_field in model_fields:
+        if model_field.is_relation and not model_field.has_unresolved_forward_refs():
+            model_field = cast(Type["ForeignKeyField"], model_field)
             expand_reverse_relationship(model_field=model_field)
 
 
@@ -101,7 +100,7 @@ def register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None:
     :type model_field: relation Field
     """
     related_name = model_field.get_related_name()
-    if issubclass(model_field, ManyToManyField):
+    if model_field.is_multi:
         model_field.to.Meta.model_fields[related_name] = ManyToMany(
             model_field.owner,
             through=model_field.through,
@@ -113,6 +112,8 @@ def register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None:
             self_reference_primary=model_field.self_reference_primary,
         )
         # register foreign keys on through model
+        model_field = cast(Type["ManyToManyField"], model_field)
+        register_through_shortcut_fields(model_field=model_field)
         adjust_through_many_to_many_model(model_field=model_field)
     else:
         model_field.to.Meta.model_fields[related_name] = ForeignKey(
@@ -125,7 +126,35 @@ def register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None:
         )
 
 
-def register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None:
+def register_through_shortcut_fields(model_field: Type["ManyToManyField"]) -> None:
+    """
+    Registers m2m relation through shortcut on both ends of the relation.
+
+    :param model_field: relation field defined in parent model
+    :type model_field: ManyToManyField
+    """
+    through_model = model_field.through
+    through_name = through_model.get_name(lower=True)
+    related_name = model_field.get_related_name()
+
+    model_field.owner.Meta.model_fields[through_name] = Through(
+        through_model,
+        real_name=through_name,
+        virtual=True,
+        related_name=model_field.name,
+        owner=model_field.owner,
+    )
+
+    model_field.to.Meta.model_fields[through_name] = Through(
+        through_model,
+        real_name=through_name,
+        virtual=True,
+        related_name=related_name,
+        owner=model_field.to,
+    )
+
+
+def register_relation_in_alias_manager(field: Type["ForeignKeyField"]) -> None:
     """
     Registers the relation (and reverse relation) in alias manager.
     The m2m relations require registration of through model between
@@ -138,11 +167,12 @@ def register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None:
     :param field: relation field
     :type field: ForeignKey or ManyToManyField class
     """
-    if issubclass(field, ManyToManyField):
+    if field.is_multi:
         if field.has_unresolved_forward_refs():
             return
+        field = cast(Type["ManyToManyField"], field)
         register_many_to_many_relation_on_build(field=field)
-    elif issubclass(field, ForeignKeyField):
+    elif field.is_relation and not field.is_through:
         if field.has_unresolved_forward_refs():
             return
         register_relation_on_build(field=field)

ormar/models/helpers/sqlalchemy.py

@@ -154,13 +154,11 @@ def sqlalchemy_columns_from_model_fields(
     pkname = None
     for field_name, field in model_fields.items():
         field.owner = new_model
+        if field.is_multi and not field.through:
+            field.create_default_through_model()
         if field.primary_key:
             pkname = check_pk_column_validity(field_name, field, pkname)
-        if (
-            not field.pydantic_only
-            and not field.virtual
-            and not issubclass(field, ormar.ManyToManyField)
-        ):
+        if not field.pydantic_only and not field.virtual and not field.is_multi:
             columns.append(field.get_column(field.get_alias()))
     return pkname, columns
 

ormar/models/metaclass.py

@@ -209,13 +209,14 @@ def update_attrs_from_base_meta(  # noqa: CCR001
                 setattr(attrs["Meta"], param, parent_value)
 
 
-def copy_and_replace_m2m_through_model(
+def copy_and_replace_m2m_through_model(  # noqa: CFQ002
     field: Type[ManyToManyField],
     field_name: str,
     table_name: str,
     parent_fields: Dict,
     attrs: Dict,
     meta: ModelMeta,
+    base_class: Type["Model"],
 ) -> None:
     """
     Clones class with Through model for m2m relations, appends child name to the name
@@ -229,6 +230,8 @@ def copy_and_replace_m2m_through_model(
 
     Removes the original sqlalchemy table from metadata if it was not removed.
 
+    :param base_class: base class model
+    :type base_class: Type["Model"]
     :param field: field with relations definition
     :type field: Type[ManyToManyField]
     :param field_name: name of the relation field
@@ -249,6 +252,10 @@ def copy_and_replace_m2m_through_model(
     copy_field.related_name = related_name  # type: ignore
 
     through_class = field.through
+    if not through_class:
+        field.owner = base_class
+        field.create_default_through_model()
+        through_class = field.through
     new_meta: ormar.ModelMeta = type(  # type: ignore
         "Meta", (), dict(through_class.Meta.__dict__),
     )
@@ -262,7 +269,7 @@ def copy_and_replace_m2m_through_model(
     new_meta.model_fields = {
         name: field
         for name, field in new_meta.model_fields.items()
-        if not issubclass(field, ForeignKeyField)
+        if not field.is_relation
     }
     _, columns = sqlalchemy_columns_from_model_fields(
         new_meta.model_fields, copy_through
@@ -329,7 +336,8 @@ def copy_data_from_parent_model(  # noqa: CCR001
             else attrs.get("__name__", "").lower() + "s"
         )
         for field_name, field in base_class.Meta.model_fields.items():
-            if issubclass(field, ManyToManyField):
+            if field.is_multi:
+                field = cast(Type["ManyToManyField"], field)
                 copy_and_replace_m2m_through_model(
                     field=field,
                     field_name=field_name,
@@ -337,9 +345,10 @@ def copy_data_from_parent_model(  # noqa: CCR001
                     parent_fields=parent_fields,
                     attrs=attrs,
                     meta=meta,
+                    base_class=base_class,  # type: ignore
                 )
 
-            elif issubclass(field, ForeignKeyField) and field.related_name:
+            elif field.is_relation and field.related_name:
                 copy_field = type(  # type: ignore
                     field.__name__, (ForeignKeyField, BaseField), dict(field.__dict__)
                 )

ormar/models/mixins/excludable_mixin.py

@@ -4,14 +4,15 @@ from typing import (
     Dict,
     List,
     Mapping,
-    Optional,
     Set,
     TYPE_CHECKING,
     Type,
     TypeVar,
     Union,
+    cast,
 )
 
+from ormar.models.excludable import ExcludableItems
 from ormar.models.mixins.relation_mixin import RelationMixin
 from ormar.queryset.utils import translate_list_to_dict, update
 
@@ -31,6 +32,7 @@ class ExcludableMixin(RelationMixin):
 
     if TYPE_CHECKING:  # pragma: no cover
         from ormar import Model
+        from ormar.models import ModelRow
 
     @staticmethod
     def get_child(
@@ -50,87 +52,11 @@ class ExcludableMixin(RelationMixin):
             return items.get(key, {})
         return items
 
-    @staticmethod
-    def get_excluded(
-        exclude: Union[Set, Dict, None], key: str = None
-    ) -> Union[Set, Dict, None]:
-        """
-        Proxy to ExcludableMixin.get_child for exclusions.
-
-        :param exclude:  bag of items to exclude
-        :type exclude: Union[Set, Dict, None]
-        :param key: name of the child to extract
-        :type key: str
-        :return: child extracted from items if exists
-        :rtype: Union[Set, Dict, None]
-        """
-        return ExcludableMixin.get_child(items=exclude, key=key)
-
-    @staticmethod
-    def get_included(
-        include: Union[Set, Dict, None], key: str = None
-    ) -> Union[Set, Dict, None]:
-        """
-        Proxy to ExcludableMixin.get_child for inclusions.
-
-        :param include:  bag of items to include
-        :type include: Union[Set, Dict, None]
-        :param key: name of the child to extract
-        :type key: str
-        :return: child extracted from items if exists
-        :rtype: Union[Set, Dict, None]
-        """
-        return ExcludableMixin.get_child(items=include, key=key)
-
-    @staticmethod
-    def is_excluded(exclude: Union[Set, Dict, None], key: str = None) -> bool:
-        """
-        Checks if given key should be excluded on model/ dict.
-
-        :param exclude:  bag of items to exclude
-        :type exclude: Union[Set, Dict, None]
-        :param key: name of the child to extract
-        :type key: str
-        :return: child extracted from items if exists
-        :rtype: Union[Set, Dict, None]
-        """
-        if exclude is None:
-            return False
-        if exclude is Ellipsis:  # pragma: nocover
-            return True
-        to_exclude = ExcludableMixin.get_excluded(exclude=exclude, key=key)
-        if isinstance(to_exclude, Set):
-            return key in to_exclude
-        if to_exclude is ...:
-            return True
-        return False
-
-    @staticmethod
-    def is_included(include: Union[Set, Dict, None], key: str = None) -> bool:
-        """
-        Checks if given key should be included on model/ dict.
-
-        :param include:  bag of items to include
-        :type include: Union[Set, Dict, None]
-        :param key: name of the child to extract
-        :type key: str
-        :return: child extracted from items if exists
-        :rtype: Union[Set, Dict, None]
-        """
-        if include is None:
-            return True
-        if include is Ellipsis:
-            return True
-        to_include = ExcludableMixin.get_included(include=include, key=key)
-        if isinstance(to_include, Set):
-            return key in to_include
-        if to_include is ...:
-            return True
-        return False
-
     @staticmethod
     def _populate_pk_column(
-        model: Type["Model"], columns: List[str], use_alias: bool = False,
+        model: Union[Type["Model"], Type["ModelRow"]],
+        columns: List[str],
+        use_alias: bool = False,
     ) -> List[str]:
         """
         Adds primary key column/alias (depends on use_alias flag) to list of
@@ -157,9 +83,9 @@ class ExcludableMixin(RelationMixin):
     @classmethod
     def own_table_columns(
         cls,
-        model: Type["Model"],
-        fields: Optional[Union[Set, Dict]],
-        exclude_fields: Optional[Union[Set, Dict]],
+        model: Union[Type["Model"], Type["ModelRow"]],
+        excludable: ExcludableItems,
+        alias: str = "",
         use_alias: bool = False,
     ) -> List[str]:
         """
@@ -171,17 +97,18 @@ class ExcludableMixin(RelationMixin):
 
         Primary key field is always added and cannot be excluded (will be added anyway).
 
+        :param alias: relation prefix
+        :type alias: str
+        :param excludable: structure of fields to include and exclude
+        :type excludable: ExcludableItems
         :param model: model on columns are selected
         :type model: Type["Model"]
-        :param fields: set/dict of fields to include
-        :type fields: Optional[Union[Set, Dict]]
-        :param exclude_fields: set/dict of fields to exclude
-        :type exclude_fields: Optional[Union[Set, Dict]]
         :param use_alias: flag if aliases or field names should be used
         :type use_alias: bool
         :return: list of column field names or aliases
         :rtype: List[str]
         """
+        model_excludable = excludable.get(model_cls=model, alias=alias)  # type: ignore
         columns = [
             model.get_column_name_from_alias(col.name) if not use_alias else col.name
             for col in model.Meta.table.columns
@@ -190,17 +117,17 @@ class ExcludableMixin(RelationMixin):
             model.get_column_name_from_alias(col.name)
             for col in model.Meta.table.columns
         ]
-        if fields:
+        if model_excludable.include:
             columns = [
                 col
                 for col, name in zip(columns, field_names)
-                if model.is_included(fields, name)
+                if model_excludable.is_included(name)
             ]
-        if exclude_fields:
+        if model_excludable.exclude:
             columns = [
                 col
                 for col, name in zip(columns, field_names)
-                if not model.is_excluded(exclude_fields, name)
+                if not model_excludable.is_excluded(name)
             ]
 
         # always has to return pk column for ormar to work
@@ -241,11 +168,7 @@ class ExcludableMixin(RelationMixin):
         return exclude
 
     @classmethod
-    def get_names_to_exclude(
-        cls,
-        fields: Optional[Union[Dict, Set]] = None,
-        exclude_fields: Optional[Union[Dict, Set]] = None,
-    ) -> Set:
+    def get_names_to_exclude(cls, excludable: ExcludableItems, alias: str) -> Set:
         """
         Returns a set of models field names that should be explicitly excluded
         during model initialization.
@@ -256,33 +179,27 @@ class ExcludableMixin(RelationMixin):
         Used in parsing data from database rows that construct Models by initializing
         them with dicts constructed from those db rows.
 
-        :param fields: set/dict of fields to include
-        :type fields: Optional[Union[Set, Dict]]
-        :param exclude_fields: set/dict of fields to exclude
-        :type exclude_fields: Optional[Union[Set, Dict]]
+        :param alias: alias of current relation
+        :type alias: str
+        :param excludable: structure of fields to include and exclude
+        :type excludable: ExcludableItems
         :return: set of field names that should be excluded
         :rtype: Set
         """
+        model = cast(Type["Model"], cls)
+        model_excludable = excludable.get(model_cls=model, alias=alias)
         fields_names = cls.extract_db_own_fields()
-        if fields and fields is not Ellipsis:
-            fields_to_keep = {name for name in fields if name in fields_names}
+        if model_excludable.include:
+            fields_to_keep = model_excludable.include.intersection(fields_names)
         else:
             fields_to_keep = fields_names
 
         fields_to_exclude = fields_names - fields_to_keep
 
-        if isinstance(exclude_fields, Set):
+        if model_excludable.exclude:
             fields_to_exclude = fields_to_exclude.union(
-                {name for name in exclude_fields if name in fields_names}
+                model_excludable.exclude.intersection(fields_names)
             )
-        elif isinstance(exclude_fields, Dict):
-            new_to_exclude = {
-                name
-                for name in exclude_fields
-                if name in fields_names and exclude_fields[name] is Ellipsis
-            }
-            fields_to_exclude = fields_to_exclude.union(new_to_exclude)
-
         fields_to_exclude = fields_to_exclude - {cls.Meta.pkname}
 
         return fields_to_exclude

ormar/models/mixins/merge_mixin.py

@@ -1,5 +1,5 @@
 from collections import OrderedDict
-from typing import List, Sequence, TYPE_CHECKING
+from typing import List, TYPE_CHECKING
 
 import ormar
 
@@ -17,7 +17,7 @@ class MergeModelMixin:
     """
 
     @classmethod
-    def merge_instances_list(cls, result_rows: Sequence["Model"]) -> Sequence["Model"]:
+    def merge_instances_list(cls, result_rows: List["Model"]) -> List["Model"]:
         """
         Merges a list of models into list of unique models.
 

ormar/models/mixins/prefetch_mixin.py

@@ -1,9 +1,10 @@
-from typing import Callable, Dict, List, TYPE_CHECKING, Tuple, Type
+from typing import Callable, Dict, List, TYPE_CHECKING, Tuple, Type, cast
 
-import ormar
-from ormar.fields.foreign_key import ForeignKeyField
 from ormar.models.mixins.relation_mixin import RelationMixin
 
+if TYPE_CHECKING:  # pragma: no cover
+    from ormar.fields import ForeignKeyField, ManyToManyField
+
 
 class PrefetchQueryMixin(RelationMixin):
     """
@@ -39,7 +40,8 @@ class PrefetchQueryMixin(RelationMixin):
         if reverse:
             field_name = parent_model.Meta.model_fields[related].get_related_name()
             field = target_model.Meta.model_fields[field_name]
-            if issubclass(field, ormar.fields.ManyToManyField):
+            if field.is_multi:
+                field = cast(Type["ManyToManyField"], field)
                 field_name = field.default_target_field_name()
                 sub_field = field.through.Meta.model_fields[field_name]
                 return field.through, sub_field.get_alias()
@@ -87,7 +89,7 @@ class PrefetchQueryMixin(RelationMixin):
         :return: name of the field
         :rtype: str
         """
-        if issubclass(target_field, ormar.fields.ManyToManyField):
+        if target_field.is_multi:
             return cls.get_name()
         if target_field.virtual:
             return target_field.get_related_name()

ormar/models/mixins/relation_mixin.py

@@ -1,7 +1,13 @@
 import inspect
-from typing import List, Optional, Set, TYPE_CHECKING
-
-from ormar.fields.foreign_key import ForeignKeyField
+from typing import (
+    Callable,
+    List,
+    Optional,
+    Set,
+    TYPE_CHECKING,
+    Type,
+    Union,
+)
 
 
 class RelationMixin:
@@ -10,11 +16,12 @@ class RelationMixin:
     """
 
     if TYPE_CHECKING:  # pragma no cover
-        from ormar import ModelMeta
+        from ormar import ModelMeta, Model
 
         Meta: ModelMeta
         _related_names: Optional[Set]
         _related_fields: Optional[List]
+        get_name: Callable
 
     @classmethod
     def extract_db_own_fields(cls) -> Set:
@@ -43,27 +50,42 @@ class RelationMixin:
             return cls._related_fields
 
         related_fields = []
-        for name in cls.extract_related_names():
+        for name in cls.extract_related_names().union(cls.extract_through_names()):
             related_fields.append(cls.Meta.model_fields[name])
         cls._related_fields = related_fields
 
         return related_fields
 
     @classmethod
-    def extract_related_names(cls) -> Set:
+    def extract_through_names(cls) -> Set:
+        """
+        Extracts related fields through names which are shortcuts to through models.
+
+        :return: set of related through fields names
+        :rtype: Set
+        """
+        related_fields = set()
+        for name in cls.extract_related_names():
+            field = cls.Meta.model_fields[name]
+            if field.is_multi:
+                related_fields.add(field.through.get_name(lower=True))
+        return related_fields
+
+    @classmethod
+    def extract_related_names(cls) -> Set[str]:
         """
         Returns List of fields names for all relations declared on a model.
         List is cached in cls._related_names for quicker access.
 
-        :return: list of related fields names
-        :rtype: List
+        :return: set of related fields names
+        :rtype: Set
         """
         if isinstance(cls._related_names, Set):
             return cls._related_names
 
         related_names = set()
         for name, field in cls.Meta.model_fields.items():
-            if inspect.isclass(field) and issubclass(field, ForeignKeyField):
+            if inspect.isclass(field) and field.is_relation and not field.is_through:
                 related_names.add(name)
         cls._related_names = related_names
 
@@ -105,3 +127,61 @@ class RelationMixin:
             name for name in related_names if cls.Meta.model_fields[name].nullable
         }
         return related_names
+
+    @classmethod
+    def _iterate_related_models(
+        cls,
+        visited: Set[str] = None,
+        source_visited: Set[str] = None,
+        source_relation: str = None,
+        source_model: Union[Type["Model"], Type["RelationMixin"]] = None,
+    ) -> List[str]:
+        """
+        Iterates related models recursively to extract relation strings of
+        nested not visited models.
+
+        :param visited: set of already visited models
+        :type visited: Set[str]
+        :param source_relation: name of the current relation
+        :type source_relation: str
+        :param source_model: model from which relation comes in nested relations
+        :type source_model: Type["Model"]
+        :return: list of relation strings to be passed to select_related
+        :rtype: List[str]
+        """
+        source_visited = source_visited or set()
+        if not source_model:
+            source_visited = cls._populate_source_model_prefixes()
+        relations = cls.extract_related_names()
+        processed_relations = []
+        for relation in relations:
+            target_model = cls.Meta.model_fields[relation].to
+            if source_model and target_model == source_model:
+                continue
+            if target_model not in source_visited or not source_model:
+                deep_relations = target_model._iterate_related_models(
+                    visited=visited,
+                    source_visited=source_visited,
+                    source_relation=relation,
+                    source_model=cls,
+                )
+                processed_relations.extend(deep_relations)
+            else:
+                processed_relations.append(relation)
+        if processed_relations:
+            final_relations = [
+                f"{source_relation + '__' if source_relation else ''}{relation}"
+                for relation in processed_relations
+            ]
+        else:
+            final_relations = [source_relation] if source_relation else []
+        return final_relations
+
+    @classmethod
+    def _populate_source_model_prefixes(cls) -> Set:
+        relations = cls.extract_related_names()
+        visited = {cls}
+        for relation in relations:
+            target_model = cls.Meta.model_fields[relation].to
+            visited.add(target_model)
+        return visited

ormar/models/model.py

@@ -2,23 +2,18 @@ from typing import (
     Any,
     Dict,
     List,
-    Optional,
     Set,
     TYPE_CHECKING,
     Tuple,
-    Type,
     TypeVar,
     Union,
 )
 
-import sqlalchemy
-
 import ormar.queryset  # noqa I100
 from ormar.exceptions import ModelPersistenceError, NoMatch
-from ormar.fields.many_to_many import ManyToManyField
 from ormar.models import NewBaseModel  # noqa I100
-from ormar.models.helpers.models import group_related_list
 from ormar.models.metaclass import ModelMeta
+from ormar.models.model_row import ModelRow
 
 if TYPE_CHECKING:  # pragma nocover
     from ormar import QuerySet
@@ -26,7 +21,7 @@ if TYPE_CHECKING:  # pragma nocover
 T = TypeVar("T", bound="Model")
 
 
-class Model(NewBaseModel):
+class Model(ModelRow):
     __abstract__ = False
     if TYPE_CHECKING:  # pragma nocover
         Meta: ModelMeta
@@ -36,247 +31,6 @@ class Model(NewBaseModel):
         _repr = {k: getattr(self, k) for k, v in self.Meta.model_fields.items()}
         return f"{self.__class__.__name__}({str(_repr)})"
 
-    @classmethod
-    def from_row(  # noqa CCR001
-        cls: Type[T],
-        row: sqlalchemy.engine.ResultProxy,
-        select_related: List = None,
-        related_models: Any = None,
-        previous_model: Type[T] = None,
-        source_model: Type[T] = None,
-        related_name: str = None,
-        fields: Optional[Union[Dict, Set]] = None,
-        exclude_fields: Optional[Union[Dict, Set]] = None,
-        current_relation_str: str = None,
-    ) -> Optional[T]:
-        """
-        Model method to convert raw sql row from database into ormar.Model instance.
-        Traverses nested models if they were specified in select_related for query.
-
-        Called recurrently and returns model instance if it's present in the row.
-        Note that it's processing one row at a time, so if there are duplicates of
-        parent row that needs to be joined/combined
-        (like parent row in sql join with 2+ child rows)
-        instances populated in this method are later combined in the QuerySet.
-        Other method working directly on raw database results is in prefetch_query,
-        where rows are populated in a different way as they do not have
-        nested models in result.
-
-        :param current_relation_str: name of the relation field
-        :type current_relation_str: str
-        :param source_model: model on which relation was defined
-        :type source_model: Type[Model]
-        :param row: raw result row from the database
-        :type row: sqlalchemy.engine.result.ResultProxy
-        :param select_related: list of names of related models fetched from database
-        :type select_related: List
-        :param related_models: list or dict of related models
-        :type related_models: Union[List, Dict]
-        :param previous_model: internal param for nested models to specify table_prefix
-        :type previous_model: Model class
-        :param related_name: internal parameter - name of current nested model
-        :type related_name: str
-        :param fields: fields and related model fields to include
-        if provided only those are included
-        :type fields: Optional[Union[Dict, Set]]
-        :param exclude_fields: fields and related model fields to exclude
-        excludes the fields even if they are provided in fields
-        :type exclude_fields: Optional[Union[Dict, Set]]
-        :return: returns model if model is populated from database
-        :rtype: Optional[Model]
-        """
-        item: Dict[str, Any] = {}
-        select_related = select_related or []
-        related_models = related_models or []
-        table_prefix = ""
-
-        if select_related:
-            source_model = cls
-            related_models = group_related_list(select_related)
-
-        rel_name2 = related_name
-
-        if (
-            previous_model
-            and related_name
-            and issubclass(
-                previous_model.Meta.model_fields[related_name], ManyToManyField
-            )
-        ):
-            through_field = previous_model.Meta.model_fields[related_name]
-            if (
-                through_field.self_reference
-                and related_name == through_field.self_reference_primary
-            ):
-                rel_name2 = through_field.default_source_field_name()  # type: ignore
-            else:
-                rel_name2 = through_field.default_target_field_name()  # type: ignore
-            previous_model = through_field.through  # type: ignore
-
-        if previous_model and rel_name2:
-            if current_relation_str and "__" in current_relation_str and source_model:
-                table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
-                    from_model=source_model, relation_name=current_relation_str
-                )
-            if not table_prefix:
-                table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
-                    from_model=previous_model, relation_name=rel_name2
-                )
-
-        item = cls.populate_nested_models_from_row(
-            item=item,
-            row=row,
-            related_models=related_models,
-            fields=fields,
-            exclude_fields=exclude_fields,
-            current_relation_str=current_relation_str,
-            source_model=source_model,
-        )
-        item = cls.extract_prefixed_table_columns(
-            item=item,
-            row=row,
-            table_prefix=table_prefix,
-            fields=fields,
-            exclude_fields=exclude_fields,
-        )
-
-        instance: Optional[T] = None
-        if item.get(cls.Meta.pkname, None) is not None:
-            item["__excluded__"] = cls.get_names_to_exclude(
-                fields=fields, exclude_fields=exclude_fields
-            )
-            instance = cls(**item)
-            instance.set_save_status(True)
-        return instance
-
-    @classmethod
-    def populate_nested_models_from_row(  # noqa: CFQ002
-        cls,
-        item: dict,
-        row: sqlalchemy.engine.ResultProxy,
-        related_models: Any,
-        fields: Optional[Union[Dict, Set]] = None,
-        exclude_fields: Optional[Union[Dict, Set]] = None,
-        current_relation_str: str = None,
-        source_model: Type[T] = None,
-    ) -> dict:
-        """
-        Traverses structure of related models and populates the nested models
-        from the database row.
-        Related models can be a list if only directly related models are to be
-        populated, converted to dict if related models also have their own related
-        models to be populated.
-
-        Recurrently calls from_row method on nested instances and create nested
-        instances. In the end those instances are added to the final model dictionary.
-
-        :param source_model: source model from which relation started
-        :type source_model: Type[Model]
-        :param current_relation_str: joined related parts into one string
-        :type current_relation_str: str
-        :param item: dictionary of already populated nested models, otherwise empty dict
-        :type item: Dict
-        :param row: raw result row from the database
-        :type row: sqlalchemy.engine.result.ResultProxy
-        :param related_models: list or dict of related models
-        :type related_models: Union[Dict, List]
-        :param fields: fields and related model fields to include -
-        if provided only those are included
-        :type fields: Optional[Union[Dict, Set]]
-        :param exclude_fields: fields and related model fields to exclude
-        excludes the fields even if they are provided in fields
-        :type exclude_fields: Optional[Union[Dict, Set]]
-        :return: dictionary with keys corresponding to model fields names
-        and values are database values
-        :rtype: Dict
-        """
-
-        for related in related_models:
-            relation_str = (
-                "__".join([current_relation_str, related])
-                if current_relation_str
-                else related
-            )
-            fields = cls.get_included(fields, related)
-            exclude_fields = cls.get_excluded(exclude_fields, related)
-            model_cls = cls.Meta.model_fields[related].to
-
-            remainder = None
-            if isinstance(related_models, dict) and related_models[related]:
-                remainder = related_models[related]
-            child = model_cls.from_row(
-                row,
-                related_models=remainder,
-                previous_model=cls,
-                related_name=related,
-                fields=fields,
-                exclude_fields=exclude_fields,
-                current_relation_str=relation_str,
-                source_model=source_model,
-            )
-            item[model_cls.get_column_name_from_alias(related)] = child
-
-        return item
-
-    @classmethod
-    def extract_prefixed_table_columns(  # noqa CCR001
-        cls,
-        item: dict,
-        row: sqlalchemy.engine.result.ResultProxy,
-        table_prefix: str,
-        fields: Optional[Union[Dict, Set]] = None,
-        exclude_fields: Optional[Union[Dict, Set]] = None,
-    ) -> dict:
-        """
-        Extracts own fields from raw sql result, using a given prefix.
-        Prefix changes depending on the table's position in a join.
-
-        If the table is a main table, there is no prefix.
-        All joined tables have prefixes to allow duplicate column names,
-        as well as duplicated joins to the same table from multiple different tables.
-
-        Extracted fields populates the related dict later used to construct a Model.
-
-        Used in Model.from_row and PrefetchQuery._populate_rows methods.
-
-        :param item: dictionary of already populated nested models, otherwise empty dict
-        :type item: Dict
-        :param row: raw result row from the database
-        :type row: sqlalchemy.engine.result.ResultProxy
-        :param table_prefix: prefix of the table from AliasManager
-        each pair of tables have own prefix (two of them depending on direction) -
-        used in joins to allow multiple joins to the same table.
-        :type table_prefix: str
-        :param fields: fields and related model fields to include -
-        if provided only those are included
-        :type fields: Optional[Union[Dict, Set]]
-        :param exclude_fields: fields and related model fields to exclude
-        excludes the fields even if they are provided in fields
-        :type exclude_fields: Optional[Union[Dict, Set]]
-        :return: dictionary with keys corresponding to model fields names
-        and values are database values
-        :rtype: Dict
-        """
-        # databases does not keep aliases in Record for postgres, change to raw row
-        source = row._row if cls.db_backend_name() == "postgresql" else row
-
-        selected_columns = cls.own_table_columns(
-            model=cls,
-            fields=fields or {},
-            exclude_fields=exclude_fields or {},
-            use_alias=False,
-        )
-
-        for column in cls.Meta.table.columns:
-            alias = cls.get_column_name_from_alias(column.name)
-            if alias not in item and alias in selected_columns:
-                prefixed_name = (
-                    f'{table_prefix + "_" if table_prefix else ""}{column.name}'
-                )
-                item[alias] = source[prefixed_name]
-
-        return item
-
     async def upsert(self: T, **kwargs: Any) -> T:
         """
         Performs either a save or an update depending on the presence of the pk.
@@ -387,8 +141,9 @@ class Model(NewBaseModel):
             visited.add(self.__class__)
 
         for related in self.extract_related_names():
-            if self.Meta.model_fields[related].virtual or issubclass(
-                self.Meta.model_fields[related], ManyToManyField
+            if (
+                self.Meta.model_fields[related].virtual
+                or self.Meta.model_fields[related].is_multi
             ):
                 for rel in getattr(self, related):
                     update_count, visited = await self._update_and_follow(
@@ -408,7 +163,7 @@ class Model(NewBaseModel):
 
     @staticmethod
     async def _update_and_follow(
-        rel: T, follow: bool, visited: Set, update_count: int
+        rel: "Model", follow: bool, visited: Set, update_count: int
     ) -> Tuple[int, Set]:
         """
         Internal method used in save_related to follow related models and update numbers
@@ -473,7 +228,7 @@ class Model(NewBaseModel):
         await self.signals.post_update.send(sender=self.__class__, instance=self)
         return self
 
-    async def delete(self: T) -> int:
+    async def delete(self) -> int:
         """
         Removes the Model instance from the database.
 
@@ -516,3 +271,44 @@ class Model(NewBaseModel):
         self.update_from_dict(kwargs)
         self.set_save_status(True)
         return self
+
+    async def load_all(
+        self: T, follow: bool = False, exclude: Union[List, str, Set, Dict] = None
+    ) -> T:
+        """
+        Allow to refresh existing Models fields from database.
+        Performs refresh of the related models fields.
+
+        By default loads only self and the directly related ones.
+
+        If follow=True is set it loads also related models of related models.
+
+        To not get stuck in an infinite loop as related models also keep a relation
+        to parent model visited models set is kept.
+
+        That way already visited models that are nested are loaded, but the load do not
+        follow them inside. So Model A -> Model B -> Model C -> Model A -> Model X
+        will load second Model A but will never follow into Model X.
+        Nested relations of those kind need to be loaded manually.
+
+        :raises NoMatch: If given pk is not found in database.
+
+        :param exclude: related models to exclude
+        :type exclude: Union[List, str, Set, Dict]
+        :param follow: flag to trigger deep save -
+        by default only directly related models are saved
+        with follow=True also related models of related models are saved
+        :type follow: bool
+        :return: reloaded Model
+        :rtype: Model
+        """
+        relations = list(self.extract_related_names())
+        if follow:
+            relations = self._iterate_related_models()
+        queryset = self.__class__.objects
+        if exclude:
+            queryset = queryset.exclude_fields(exclude)
+        instance = await queryset.select_related(relations).get(pk=self.pk)
+        self._orm.clear()
+        self.update_from_dict(instance.dict())
+        return self

ormar/models/model_row.py

@@ -0,0 +1,299 @@
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    TYPE_CHECKING,
+    Type,
+    cast,
+)
+
+import sqlalchemy
+
+from ormar.models import NewBaseModel  # noqa: I202
+from ormar.models.excludable import ExcludableItems
+from ormar.models.helpers.models import group_related_list
+
+if TYPE_CHECKING:  # pragma: no cover
+    from ormar.fields import ForeignKeyField
+    from ormar.models import Model
+
+
+class ModelRow(NewBaseModel):
+    @classmethod
+    def from_row(  # noqa: CFQ002
+        cls,
+        row: sqlalchemy.engine.ResultProxy,
+        source_model: Type["Model"],
+        select_related: List = None,
+        related_models: Any = None,
+        related_field: Type["ForeignKeyField"] = None,
+        excludable: ExcludableItems = None,
+        current_relation_str: str = "",
+        proxy_source_model: Optional[Type["Model"]] = None,
+        used_prefixes: List[str] = None,
+    ) -> Optional["Model"]:
+        """
+        Model method to convert raw sql row from database into ormar.Model instance.
+        Traverses nested models if they were specified in select_related for query.
+
+        Called recurrently and returns model instance if it's present in the row.
+        Note that it's processing one row at a time, so if there are duplicates of
+        parent row that needs to be joined/combined
+        (like parent row in sql join with 2+ child rows)
+        instances populated in this method are later combined in the QuerySet.
+        Other method working directly on raw database results is in prefetch_query,
+        where rows are populated in a different way as they do not have
+        nested models in result.
+
+        :param used_prefixes: list of already extracted prefixes
+        :type used_prefixes: List[str]
+        :param proxy_source_model: source model from which querysetproxy is constructed
+        :type proxy_source_model: Optional[Type["ModelRow"]]
+        :param excludable: structure of fields to include and exclude
+        :type excludable: ExcludableItems
+        :param current_relation_str: name of the relation field
+        :type current_relation_str: str
+        :param source_model: model on which relation was defined
+        :type source_model: Type[Model]
+        :param row: raw result row from the database
+        :type row: sqlalchemy.engine.result.ResultProxy
+        :param select_related: list of names of related models fetched from database
+        :type select_related: List
+        :param related_models: list or dict of related models
+        :type related_models: Union[List, Dict]
+        :param related_field: field with relation declaration
+        :type related_field: Type[ForeignKeyField]
+        :return: returns model if model is populated from database
+        :rtype: Optional[Model]
+        """
+        item: Dict[str, Any] = {}
+        select_related = select_related or []
+        related_models = related_models or []
+        table_prefix = ""
+        used_prefixes = used_prefixes if used_prefixes is not None else []
+        excludable = excludable or ExcludableItems()
+
+        if select_related:
+            related_models = group_related_list(select_related)
+
+        if related_field:
+            if related_field.is_multi:
+                previous_model = related_field.through
+            else:
+                previous_model = related_field.owner
+            table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
+                from_model=previous_model, relation_name=related_field.name
+            )
+            if not table_prefix or table_prefix in used_prefixes:
+                manager = cls.Meta.alias_manager
+                table_prefix = manager.resolve_relation_alias_after_complex(
+                    source_model=source_model,
+                    relation_str=current_relation_str,
+                    relation_field=related_field,
+                )
+            used_prefixes.append(table_prefix)
+
+        item = cls._populate_nested_models_from_row(
+            item=item,
+            row=row,
+            related_models=related_models,
+            excludable=excludable,
+            current_relation_str=current_relation_str,
+            source_model=source_model,  # type: ignore
+            proxy_source_model=proxy_source_model,  # type: ignore
+            table_prefix=table_prefix,
+            used_prefixes=used_prefixes,
+        )
+        item = cls.extract_prefixed_table_columns(
+            item=item, row=row, table_prefix=table_prefix, excludable=excludable
+        )
+
+        instance: Optional["Model"] = None
+        if item.get(cls.Meta.pkname, None) is not None:
+            item["__excluded__"] = cls.get_names_to_exclude(
+                excludable=excludable, alias=table_prefix
+            )
+            instance = cast("Model", cls(**item))
+            instance.set_save_status(True)
+        return instance
+
+    @classmethod
+    def _populate_nested_models_from_row(  # noqa: CFQ002
+        cls,
+        item: dict,
+        row: sqlalchemy.engine.ResultProxy,
+        source_model: Type["Model"],
+        related_models: Any,
+        excludable: ExcludableItems,
+        table_prefix: str,
+        used_prefixes: List[str],
+        current_relation_str: str = None,
+        proxy_source_model: Type["Model"] = None,
+    ) -> dict:
+        """
+        Traverses structure of related models and populates the nested models
+        from the database row.
+        Related models can be a list if only directly related models are to be
+        populated, converted to dict if related models also have their own related
+        models to be populated.
+
+        Recurrently calls from_row method on nested instances and create nested
+        instances. In the end those instances are added to the final model dictionary.
+
+        :param proxy_source_model: source model from which querysetproxy is constructed
+        :type proxy_source_model: Optional[Type["ModelRow"]]
+        :param excludable: structure of fields to include and exclude
+        :type excludable: ExcludableItems
+        :param source_model: source model from which relation started
+        :type source_model: Type[Model]
+        :param current_relation_str: joined related parts into one string
+        :type current_relation_str: str
+        :param item: dictionary of already populated nested models, otherwise empty dict
+        :type item: Dict
+        :param row: raw result row from the database
+        :type row: sqlalchemy.engine.result.ResultProxy
+        :param related_models: list or dict of related models
+        :type related_models: Union[Dict, List]
+        :return: dictionary with keys corresponding to model fields names
+        and values are database values
+        :rtype: Dict
+        """
+
+        for related in related_models:
+            field = cls.Meta.model_fields[related]
+            field = cast(Type["ForeignKeyField"], field)
+            model_cls = field.to
+            model_excludable = excludable.get(
+                model_cls=cast(Type["Model"], cls), alias=table_prefix
+            )
+            if model_excludable.is_excluded(related):
+                return item
+
+            relation_str = (
+                "__".join([current_relation_str, related])
+                if current_relation_str
+                else related
+            )
+            remainder = None
+            if isinstance(related_models, dict) and related_models[related]:
+                remainder = related_models[related]
+            child = model_cls.from_row(
+                row,
+                related_models=remainder,
+                related_field=field,
+                excludable=excludable,
+                current_relation_str=relation_str,
+                source_model=source_model,
+                proxy_source_model=proxy_source_model,
+                used_prefixes=used_prefixes,
+            )
+            item[model_cls.get_column_name_from_alias(related)] = child
+            if field.is_multi and child:
+                through_name = cls.Meta.model_fields[related].through.get_name()
+                through_child = cls.populate_through_instance(
+                    row=row,
+                    related=related,
+                    through_name=through_name,
+                    excludable=excludable,
+                )
+
+                if child.__class__ != proxy_source_model:
+                    setattr(child, through_name, through_child)
+                else:
+                    item[through_name] = through_child
+                child.set_save_status(True)
+
+        return item
+
+    @classmethod
+    def populate_through_instance(
+        cls,
+        row: sqlalchemy.engine.ResultProxy,
+        through_name: str,
+        related: str,
+        excludable: ExcludableItems,
+    ) -> "ModelRow":
+        """
+        Initialize the through model from db row.
+        Excluded all relation fields and other exclude/include set in excludable.
+
+        :param row: loaded row from database
+        :type row: sqlalchemy.engine.ResultProxy
+        :param through_name: name of the through field
+        :type through_name: str
+        :param related: name of the relation
+        :type related: str
+        :param excludable: structure of fields to include and exclude
+        :type excludable: ExcludableItems
+        :return: initialized through model without relation
+        :rtype: "ModelRow"
+        """
+        model_cls = cls.Meta.model_fields[through_name].to
+        table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
+            from_model=cls, relation_name=related
+        )
+        # remove relations on through field
+        model_excludable = excludable.get(model_cls=model_cls, alias=table_prefix)
+        model_excludable.set_values(
+            value=model_cls.extract_related_names(), is_exclude=True
+        )
+        child_dict = model_cls.extract_prefixed_table_columns(
+            item={}, row=row, excludable=excludable, table_prefix=table_prefix
+        )
+        child_dict["__excluded__"] = model_cls.get_names_to_exclude(
+            excludable=excludable, alias=table_prefix
+        )
+        child = model_cls(**child_dict)  # type: ignore
+        return child
+
+    @classmethod
+    def extract_prefixed_table_columns(
+        cls,
+        item: dict,
+        row: sqlalchemy.engine.result.ResultProxy,
+        table_prefix: str,
+        excludable: ExcludableItems,
+    ) -> Dict:
+        """
+        Extracts own fields from raw sql result, using a given prefix.
+        Prefix changes depending on the table's position in a join.
+
+        If the table is a main table, there is no prefix.
+        All joined tables have prefixes to allow duplicate column names,
+        as well as duplicated joins to the same table from multiple different tables.
+
+        Extracted fields populates the related dict later used to construct a Model.
+
+        Used in Model.from_row and PrefetchQuery._populate_rows methods.
+
+        :param excludable: structure of fields to include and exclude
+        :type excludable: ExcludableItems
+        :param item: dictionary of already populated nested models, otherwise empty dict
+        :type item: Dict
+        :param row: raw result row from the database
+        :type row: sqlalchemy.engine.result.ResultProxy
+        :param table_prefix: prefix of the table from AliasManager
+        each pair of tables have own prefix (two of them depending on direction) -
+        used in joins to allow multiple joins to the same table.
+        :type table_prefix: str
+        :return: dictionary with keys corresponding to model fields names
+        and values are database values
+        :rtype: Dict
+        """
+        # databases does not keep aliases in Record for postgres, change to raw row
+        source = row._row if cls.db_backend_name() == "postgresql" else row
+
+        selected_columns = cls.own_table_columns(
+            model=cls, excludable=excludable, alias=table_prefix, use_alias=False,
+        )
+
+        for column in cls.Meta.table.columns:
+            alias = cls.get_column_name_from_alias(column.name)
+            if alias not in item and alias in selected_columns:
+                prefixed_name = (
+                    f'{table_prefix + "_" if table_prefix else ""}{column.name}'
+                )
+                item[alias] = source[prefixed_name]
+
+        return item

ormar/models/newbasemodel.py

@@ -13,7 +13,6 @@ from typing import (
     Set,
     TYPE_CHECKING,
     Type,
-    TypeVar,
     Union,
     cast,
 )
@@ -46,11 +45,9 @@ from ormar.relations.alias_manager import AliasManager
 from ormar.relations.relation_manager import RelationsManager
 
 if TYPE_CHECKING:  # pragma no cover
-    from ormar import Model
+    from ormar.models import Model
     from ormar.signals import SignalEmitter
 
-    T = TypeVar("T", bound=Model)
-
     IntStr = Union[int, str]
     DictStrAny = Dict[str, Any]
     AbstractSetIntStr = AbstractSet[IntStr]
@@ -129,7 +126,9 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
         object.__setattr__(
             self,
             "_orm",
-            RelationsManager(related_fields=self.extract_related_fields(), owner=self,),
+            RelationsManager(
+                related_fields=self.extract_related_fields(), owner=cast("Model", self),
+            ),
         )
 
         pk_only = kwargs.pop("__pk_only__", False)
@@ -172,7 +171,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
         object.__setattr__(self, "__fields_set__", fields_set)
 
         # register the columns models after initialization
-        for related in self.extract_related_names():
+        for related in self.extract_related_names().union(self.extract_through_names()):
             self.Meta.model_fields[related].expand_relationship(
                 new_kwargs.get(related), self, to_register=True,
             )
@@ -267,6 +266,10 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
             return object.__getattribute__(
                 self, "_extract_related_model_instead_of_field"
             )(item)
+        if item in object.__getattribute__(self, "extract_through_names")():
+            return object.__getattribute__(
+                self, "_extract_related_model_instead_of_field"
+            )(item)
         if item in object.__getattribute__(self, "Meta").property_fields:
             value = object.__getattribute__(self, item)
             return value() if callable(value) else value
@@ -294,7 +297,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
 
     def _extract_related_model_instead_of_field(
         self, item: str
-    ) -> Optional[Union["T", Sequence["T"]]]:
+    ) -> Optional[Union["Model", Sequence["Model"]]]:
         """
         Retrieves the related model/models from RelationshipManager.
 
@@ -304,7 +307,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
         :rtype: Optional[Union[Model, List[Model]]]
         """
         if item in self._orm:
-            return self._orm.get(item)
+            return self._orm.get(item)  # type: ignore
         return None  # pragma no cover
 
     def __eq__(self, other: object) -> bool:
@@ -391,7 +394,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
         cause some dialect require different treatment"""
         return cls.Meta.database._backend._dialect.name
 
-    def remove(self, parent: "T", name: str) -> None:
+    def remove(self, parent: "Model", name: str) -> None:
         """Removes child from relation with given name in RelationshipManager"""
         self._orm.remove_parent(self, parent, name)
 
@@ -751,9 +754,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
         :return: value of pk if set
         :rtype: Optional[int]
         """
-        if target_field.virtual or issubclass(
-            target_field, ormar.fields.ManyToManyField
-        ):
+        if target_field.virtual or target_field.is_multi:
             return self.pk
         related_name = target_field.name
         related_model = getattr(self, related_name)

ormar/models/quick_access_views.py

@@ -34,10 +34,12 @@ quick_access_set = {
     "_skip_ellipsis",
     "_update_and_follow",
     "_update_excluded_with_related_not_required",
+    "_verify_model_can_be_initialized",
     "copy",
     "delete",
     "dict",
     "extract_related_names",
+    "extract_through_names",
     "update_from_dict",
     "get_column_alias",
     "get_column_name_from_alias",

ormar/protocols/queryset_protocol.py

@@ -52,6 +52,9 @@ class QuerySetProtocol(Protocol):  # pragma: nocover
     async def create(self, **kwargs: Any) -> "Model":
         ...
 
+    async def update(self, each: bool = False, **kwargs: Any) -> int:
+        ...
+
     async def get_or_create(self, **kwargs: Any) -> "Model":
         ...
 

ormar/queryset/__init__.py

@@ -1,10 +1,19 @@
 """
 Contains QuerySet and different Query classes to allow for constructing of sql queries.
 """
+from ormar.queryset.actions import FilterAction, OrderAction
 from ormar.queryset.filter_query import FilterQuery
 from ormar.queryset.limit_query import LimitQuery
 from ormar.queryset.offset_query import OffsetQuery
 from ormar.queryset.order_query import OrderQuery
 from ormar.queryset.queryset import QuerySet
 
-__all__ = ["QuerySet", "FilterQuery", "LimitQuery", "OffsetQuery", "OrderQuery"]
+__all__ = [
+    "QuerySet",
+    "FilterQuery",
+    "LimitQuery",
+    "OffsetQuery",
+    "OrderQuery",
+    "FilterAction",
+    "OrderAction",
+]

ormar/queryset/actions/__init__.py

@@ -0,0 +1,4 @@
+from ormar.queryset.actions.filter_action import FilterAction
+from ormar.queryset.actions.order_action import OrderAction
+
+__all__ = ["FilterAction", "OrderAction"]

ormar/queryset/actions/filter_action.py

@@ -1,11 +1,11 @@
-from typing import Any, Dict, List, TYPE_CHECKING, Type
+from typing import Any, Dict, TYPE_CHECKING, Type
 
 import sqlalchemy
 from sqlalchemy import text
 
 import ormar  # noqa: I100, I202
 from ormar.exceptions import QueryDefinitionError
-from ormar.queryset.utils import get_relationship_alias_model_and_str
+from ormar.queryset.actions.query_action import QueryAction
 
 if TYPE_CHECKING:  # pragma: nocover
     from ormar import Model
@@ -28,7 +28,7 @@ FILTER_OPERATORS = {
 ESCAPE_CHARACTERS = ["%", "_"]
 
 
-class FilterAction:
+class FilterAction(QueryAction):
     """
     Filter Actions is populated by queryset when filter() is called.
 
@@ -39,7 +39,21 @@ class FilterAction:
     """
 
     def __init__(self, filter_str: str, value: Any, model_cls: Type["Model"]) -> None:
-        parts = filter_str.split("__")
+        super().__init__(query_str=filter_str, model_cls=model_cls)
+        self.filter_value = value
+        self._escape_characters_in_clause()
+        self.is_source_model_filter = False
+        if self.source_model == self.target_model and "__" not in self.related_str:
+            self.is_source_model_filter = True
+
+    def has_escaped_characters(self) -> bool:
+        """Check if value is a string that contains characters to escape"""
+        return isinstance(self.filter_value, str) and any(
+            c for c in ESCAPE_CHARACTERS if c in self.filter_value
+        )
+
+    def _split_value_into_parts(self, query_str: str) -> None:
+        parts = query_str.split("__")
         if parts[-1] in FILTER_OPERATORS:
             self.operator = parts[-1]
             self.field_name = parts[-2]
@@ -49,59 +63,6 @@ class FilterAction:
             self.field_name = parts[-1]
             self.related_parts = parts[:-1]
 
-        self.filter_value = value
-        self.table_prefix = ""
-        self.source_model = model_cls
-        self.target_model = model_cls
-        self._determine_filter_target_table()
-        self._escape_characters_in_clause()
-
-    @property
-    def table(self) -> sqlalchemy.Table:
-        """Shortcut to sqlalchemy Table of filtered target model"""
-        return self.target_model.Meta.table
-
-    @property
-    def column(self) -> sqlalchemy.Column:
-        """Shortcut to sqlalchemy column of filtered target model"""
-        aliased_name = self.target_model.get_column_alias(self.field_name)
-        return self.target_model.Meta.table.columns[aliased_name]
-
-    def has_escaped_characters(self) -> bool:
-        """Check if value is a string that contains characters to escape"""
-        return isinstance(self.filter_value, str) and any(
-            c for c in ESCAPE_CHARACTERS if c in self.filter_value
-        )
-
-    def update_select_related(self, select_related: List[str]) -> List[str]:
-        """
-        Updates list of select related with related part included in the filter key.
-        That way If you want to just filter by relation you do not have to provide
-        select_related separately.
-
-        :param select_related: list of relation join strings
-        :type select_related: List[str]
-        :return: list of relation joins with implied joins from filter added
-        :rtype: List[str]
-        """
-        select_related = select_related[:]
-        if self.related_str and not any(
-            rel.startswith(self.related_str) for rel in select_related
-        ):
-            select_related.append(self.related_str)
-        return select_related
-
-    def _determine_filter_target_table(self) -> None:
-        """
-        Walks the relation to retrieve the actual model on which the clause should be
-        constructed, extracts alias based on last relation leading to target model.
-        """
-        (
-            self.table_prefix,
-            self.target_model,
-            self.related_str,
-        ) = get_relationship_alias_model_and_str(self.source_model, self.related_parts)
-
     def _escape_characters_in_clause(self) -> None:
         """
         Escapes the special characters ["%", "_"] if needed.
@@ -149,7 +110,7 @@ class FilterAction:
         sufix = "%" if "end" not in self.operator else ""
         self.filter_value = f"{prefix}{self.filter_value}{sufix}"
 
-    def get_text_clause(self,) -> sqlalchemy.sql.expression.TextClause:
+    def get_text_clause(self) -> sqlalchemy.sql.expression.TextClause:
         """
         Escapes characters if it's required.
         Substitutes values of the models if value is a ormar Model with its pk value.

ormar/queryset/actions/order_action.py

@@ -0,0 +1,68 @@
+from typing import TYPE_CHECKING, Type
+
+import sqlalchemy
+from sqlalchemy import text
+
+from ormar.queryset.actions.query_action import QueryAction  # noqa: I100, I202
+
+if TYPE_CHECKING:  # pragma: nocover
+    from ormar import Model
+
+
+class OrderAction(QueryAction):
+    """
+    Order Actions is populated by queryset when order_by() is called.
+
+    All required params are extracted but kept raw until actual filter clause value
+    is required -> then the action is converted into text() clause.
+
+    Extracted in order to easily change table prefixes on complex relations.
+    """
+
+    def __init__(
+        self, order_str: str, model_cls: Type["Model"], alias: str = None
+    ) -> None:
+        self.direction: str = ""
+        super().__init__(query_str=order_str, model_cls=model_cls)
+        self.is_source_model_order = False
+        if alias:
+            self.table_prefix = alias
+        if self.source_model == self.target_model and "__" not in self.related_str:
+            self.is_source_model_order = True
+
+    @property
+    def field_alias(self) -> str:
+        return self.target_model.get_column_alias(self.field_name)
+
+    def get_text_clause(self) -> sqlalchemy.sql.expression.TextClause:
+        """
+        Escapes characters if it's required.
+        Substitutes values of the models if value is a ormar Model with its pk value.
+        Compiles the clause.
+
+        :return: complied and escaped clause
+        :rtype: sqlalchemy.sql.elements.TextClause
+        """
+        prefix = f"{self.table_prefix}_" if self.table_prefix else ""
+        return text(f"{prefix}{self.table}" f".{self.field_alias} {self.direction}")
+
+    def _split_value_into_parts(self, order_str: str) -> None:
+        if order_str.startswith("-"):
+            self.direction = "desc"
+            order_str = order_str[1:]
+        parts = order_str.split("__")
+        self.field_name = parts[-1]
+        self.related_parts = parts[:-1]
+
+    def check_if_filter_apply(self, target_model: Type["Model"], alias: str) -> bool:
+        """
+        Checks filter conditions to find if they apply to current join.
+
+        :param target_model: model which is now processed
+        :type target_model: Type["Model"]
+        :param alias: prefix of the relation
+        :type alias: str
+        :return: result of the check
+        :rtype: bool
+        """
+        return target_model == self.target_model and alias == self.table_prefix

ormar/queryset/actions/query_action.py

@@ -0,0 +1,93 @@
+import abc
+from typing import Any, List, TYPE_CHECKING, Type
+
+import sqlalchemy
+
+from ormar.queryset.utils import get_relationship_alias_model_and_str  # noqa: I202
+
+if TYPE_CHECKING:  # pragma: nocover
+    from ormar import Model
+
+
+class QueryAction(abc.ABC):
+    """
+    Base QueryAction class with common params for Filter and Order actions.
+    """
+
+    def __init__(self, query_str: str, model_cls: Type["Model"]) -> None:
+        self.query_str = query_str
+        self.field_name: str = ""
+        self.related_parts: List[str] = []
+        self.related_str: str = ""
+
+        self.table_prefix = ""
+        self.source_model = model_cls
+        self.target_model = model_cls
+        self.is_through = False
+
+        self._split_value_into_parts(query_str)
+        self._determine_filter_target_table()
+
+    def __eq__(self, other: object) -> bool:  # pragma: no cover
+        if not isinstance(other, QueryAction):
+            return False
+        return self.query_str == other.query_str
+
+    def __hash__(self) -> Any:
+        return hash((self.table_prefix, self.query_str))
+
+    @abc.abstractmethod
+    def _split_value_into_parts(self, query_str: str) -> None:  # pragma: no cover
+        """
+        Splits string into related parts and field_name
+        :param query_str: query action string to split (i..e filter or order by)
+        :type query_str: str
+        """
+        pass
+
+    @abc.abstractmethod
+    def get_text_clause(
+        self,
+    ) -> sqlalchemy.sql.expression.TextClause:  # pragma: no cover
+        pass
+
+    @property
+    def table(self) -> sqlalchemy.Table:
+        """Shortcut to sqlalchemy Table of filtered target model"""
+        return self.target_model.Meta.table
+
+    @property
+    def column(self) -> sqlalchemy.Column:
+        """Shortcut to sqlalchemy column of filtered target model"""
+        aliased_name = self.target_model.get_column_alias(self.field_name)
+        return self.target_model.Meta.table.columns[aliased_name]
+
+    def update_select_related(self, select_related: List[str]) -> List[str]:
+        """
+        Updates list of select related with related part included in the filter key.
+        That way If you want to just filter by relation you do not have to provide
+        select_related separately.
+
+        :param select_related: list of relation join strings
+        :type select_related: List[str]
+        :return: list of relation joins with implied joins from filter added
+        :rtype: List[str]
+        """
+        select_related = select_related[:]
+        if self.related_str and not any(
+            rel.startswith(self.related_str) for rel in select_related
+        ):
+            select_related.append(self.related_str)
+        return select_related
+
+    def _determine_filter_target_table(self) -> None:
+        """
+        Walks the relation to retrieve the actual model on which the clause should be
+        constructed, extracts alias based on last relation leading to target model.
+        """
+        (
+            self.table_prefix,
+            self.target_model,
+            self.related_str,
+            self.is_through,
+        ) = get_relationship_alias_model_and_str(self.source_model, self.related_parts)

ormar/queryset/clause.py

@@ -3,7 +3,7 @@ from dataclasses import dataclass
 from typing import Any, List, TYPE_CHECKING, Tuple, Type
 
 import ormar  # noqa I100
-from ormar.queryset.filter_action import FilterAction
+from ormar.queryset.actions.filter_action import FilterAction
 from ormar.queryset.utils import get_relationship_alias_model_and_str
 
 if TYPE_CHECKING:  # pragma no cover
@@ -16,6 +16,7 @@ class Prefix:
     table_prefix: str
     model_cls: Type["Model"]
     relation_str: str
+    is_through: bool
 
     @property
     def alias_key(self) -> str:

ormar/queryset/filter_query.py

@@ -1,7 +1,7 @@
 from typing import List
 
 import sqlalchemy
-from ormar.queryset.filter_action import FilterAction
+from ormar.queryset.actions.filter_action import FilterAction
 
 
 class FilterQuery:

ormar/queryset/join.py

@@ -1,25 +1,24 @@
 from collections import OrderedDict
 from typing import (
     Any,
-    Dict,
     List,
     Optional,
-    Set,
     TYPE_CHECKING,
     Tuple,
     Type,
-    Union,
 )
 
 import sqlalchemy
 from sqlalchemy import text
 
-from ormar.exceptions import RelationshipInstanceError  # noqa I100
-from ormar.fields import BaseField, ManyToManyField  # noqa I100
+import ormar  # noqa I100
+from ormar.exceptions import RelationshipInstanceError
 from ormar.relations import AliasManager
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
+    from ormar.queryset import OrderAction
+    from ormar.models.excludable import ExcludableItems
 
 
 class SqlJoin:
@@ -28,9 +27,8 @@ class SqlJoin:
         used_aliases: List,
         select_from: sqlalchemy.sql.select,
         columns: List[sqlalchemy.Column],
-        fields: Optional[Union[Set, Dict]],
-        exclude_fields: Optional[Union[Set, Dict]],
-        order_columns: Optional[List],
+        excludable: "ExcludableItems",
+        order_columns: Optional[List["OrderAction"]],
         sorted_orders: OrderedDict,
         main_model: Type["Model"],
         relation_name: str,
@@ -43,8 +41,7 @@ class SqlJoin:
         self.related_models = related_models or []
         self.select_from = select_from
         self.columns = columns
-        self.fields = fields
-        self.exclude_fields = exclude_fields
+        self.excludable = excludable
         self.order_columns = order_columns
         self.sorted_orders = sorted_orders
         self.main_model = main_model
@@ -90,7 +87,18 @@ class SqlJoin:
         """
         return self.main_model.Meta.alias_manager
 
-    def on_clause(self, previous_alias: str, from_clause: str, to_clause: str,) -> text:
+    @property
+    def to_table(self) -> str:
+        """
+        Shortcut to table name of the next model
+        :return: name of the target table
+        :rtype: str
+        """
+        return self.next_model.Meta.table.name
+
+    def _on_clause(
+        self, previous_alias: str, from_clause: str, to_clause: str,
+    ) -> text:
         """
         Receives aliases and names of both ends of the join and combines them
         into one text clause used in joins.
@@ -118,8 +126,8 @@ class SqlJoin:
         :return: list of used aliases, select from, list of aliased columns, sort orders
         :rtype: Tuple[List[str], Join, List[TextClause], collections.OrderedDict]
         """
-        if issubclass(self.target_field, ManyToManyField):
-            self.process_m2m_through_table()
+        if self.target_field.is_multi:
+            self._process_m2m_through_table()
 
         self.next_model = self.target_field.to
         self._forward_join()
@@ -188,10 +196,7 @@ class SqlJoin:
             used_aliases=self.used_aliases,
             select_from=self.select_from,
             columns=self.columns,
-            fields=self.main_model.get_excluded(self.fields, related_name),
-            exclude_fields=self.main_model.get_excluded(
-                self.exclude_fields, related_name
-            ),
+            excludable=self.excludable,
             order_columns=self.order_columns,
             sorted_orders=self.sorted_orders,
             main_model=self.next_model,
@@ -208,7 +213,7 @@ class SqlJoin:
             self.sorted_orders,
         ) = sql_join.build_join()
 
-    def process_m2m_through_table(self) -> None:
+    def _process_m2m_through_table(self) -> None:
         """
         Process Through table of the ManyToMany relation so that source table is
         linked to the through table (one additional join)
@@ -223,8 +228,7 @@ class SqlJoin:
 
         To point to through model
         """
-        new_part = self.process_m2m_related_name_change()
-        self._replace_many_to_many_order_by_columns(self.relation_name, new_part)
+        new_part = self._process_m2m_related_name_change()
 
         self.next_model = self.target_field.through
         self._forward_join()
@@ -233,7 +237,7 @@ class SqlJoin:
         self.own_alias = self.next_alias
         self.target_field = self.next_model.Meta.model_fields[self.relation_name]
 
-    def process_m2m_related_name_change(self, reverse: bool = False) -> str:
+    def _process_m2m_related_name_change(self, reverse: bool = False) -> str:
         """
         Extracts relation name to link join through the Through model declared on
         relation field.
@@ -273,29 +277,26 @@ class SqlJoin:
         Process order_by causes for non m2m relations.
 
         """
-        to_table = self.next_model.Meta.table.name
-        to_key, from_key = self.get_to_and_from_keys()
+        to_key, from_key = self._get_to_and_from_keys()
 
-        on_clause = self.on_clause(
+        on_clause = self._on_clause(
             previous_alias=self.own_alias,
             from_clause=f"{self.target_field.owner.Meta.tablename}.{from_key}",
-            to_clause=f"{to_table}.{to_key}",
+            to_clause=f"{self.to_table}.{to_key}",
+        )
+        target_table = self.alias_manager.prefixed_table_name(
+            self.next_alias, self.to_table
         )
-        target_table = self.alias_manager.prefixed_table_name(self.next_alias, to_table)
         self.select_from = sqlalchemy.sql.outerjoin(
             self.select_from, target_table, on_clause
         )
 
-        pkname_alias = self.next_model.get_column_alias(self.next_model.Meta.pkname)
-        if not issubclass(self.target_field, ManyToManyField):
-            self.get_order_bys(
-                to_table=to_table, pkname_alias=pkname_alias,
-            )
+        self._get_order_bys()
 
         self_related_fields = self.next_model.own_table_columns(
             model=self.next_model,
-            fields=self.fields,
-            exclude_fields=self.exclude_fields,
+            excludable=self.excludable,
+            alias=self.next_alias,
             use_alias=True,
         )
         self.columns.extend(
@@ -305,88 +306,35 @@ class SqlJoin:
         )
         self.used_aliases.append(self.next_alias)
 
-    def _replace_many_to_many_order_by_columns(self, part: str, new_part: str) -> None:
-        """
-        Substitutes the name of the relation with actual model name in m2m order bys.
-
-        :param part: name of the field with relation
-        :type part: str
-        :param new_part: name of the target model
-        :type new_part: str
-        """
-        if self.order_columns:
-            split_order_columns = [
-                x.split("__") for x in self.order_columns if "__" in x
-            ]
-            for condition in split_order_columns:
-                if self._check_if_condition_apply(condition, part):
-                    condition[-2] = condition[-2].replace(part, new_part)
-            self.order_columns = [x for x in self.order_columns if "__" not in x] + [
-                "__".join(x) for x in split_order_columns
-            ]
-
-    @staticmethod
-    def _check_if_condition_apply(condition: List, part: str) -> bool:
-        """
-        Checks filter conditions to find if they apply to current join.
-
-        :param condition: list of parts of condition split by '__'
-        :type condition: List[str]
-        :param part: name of the current relation join.
-        :type part: str
-        :return: result of the check
-        :rtype: bool
-        """
-        return len(condition) >= 2 and (
-            condition[-2] == part or condition[-2][1:] == part
+    def _set_default_primary_key_order_by(self) -> None:
+        clause = ormar.OrderAction(
+            order_str=self.next_model.Meta.pkname,
+            model_cls=self.next_model,
+            alias=self.next_alias,
         )
+        self.sorted_orders[clause] = clause.get_text_clause()
 
-    def set_aliased_order_by(self, condition: List[str], to_table: str,) -> None:
-        """
-        Substitute hyphens ('-') with descending order.
-        Construct actual sqlalchemy text clause using aliased table and column name.
-
-        :param condition: list of parts of a current condition split by '__'
-        :type condition: List[str]
-        :param to_table: target table
-        :type to_table: sqlalchemy.sql.elements.quoted_name
-        """
-        direction = f"{'desc' if condition[0][0] == '-' else ''}"
-        column_alias = self.next_model.get_column_alias(condition[-1])
-        order = text(f"{self.next_alias}_{to_table}.{column_alias} {direction}")
-        self.sorted_orders["__".join(condition)] = order
-
-    def get_order_bys(self, to_table: str, pkname_alias: str,) -> None:  # noqa: CCR001
+    def _get_order_bys(self) -> None:  # noqa: CCR001
         """
         Triggers construction of order bys if they are given.
         Otherwise by default each table is sorted by a primary key column asc.
-
-        :param to_table: target table
-        :type to_table: sqlalchemy.sql.elements.quoted_name
-        :param pkname_alias: alias of the primary key column
-        :type pkname_alias: str
         """
         alias = self.next_alias
         if self.order_columns:
             current_table_sorted = False
-            split_order_columns = [
-                x.split("__") for x in self.order_columns if "__" in x
-            ]
-            for condition in split_order_columns:
-                if self._check_if_condition_apply(condition, self.relation_name):
+            for condition in self.order_columns:
+                if condition.check_if_filter_apply(
+                    target_model=self.next_model, alias=alias
+                ):
                     current_table_sorted = True
-                    self.set_aliased_order_by(
-                        condition=condition, to_table=to_table,
-                    )
-            if not current_table_sorted:
-                order = text(f"{alias}_{to_table}.{pkname_alias}")
-                self.sorted_orders[f"{alias}.{pkname_alias}"] = order
+                    self.sorted_orders[condition] = condition.get_text_clause()
+            if not current_table_sorted and not self.target_field.is_multi:
+                self._set_default_primary_key_order_by()
 
-        else:
-            order = text(f"{alias}_{to_table}.{pkname_alias}")
-            self.sorted_orders[f"{alias}.{pkname_alias}"] = order
+        elif not self.target_field.is_multi:
+            self._set_default_primary_key_order_by()
 
-    def get_to_and_from_keys(self) -> Tuple[str, str]:
+    def _get_to_and_from_keys(self) -> Tuple[str, str]:
         """
         Based on the relation type, name of the relation and previous models and parts
         stored in JoinParameters it resolves the current to and from keys, which are
@@ -395,8 +343,8 @@ class SqlJoin:
         :return: to key and from key
         :rtype: Tuple[str, str]
         """
-        if issubclass(self.target_field, ManyToManyField):
-            to_key = self.process_m2m_related_name_change(reverse=True)
+        if self.target_field.is_multi:
+            to_key = self._process_m2m_related_name_change(reverse=True)
             from_key = self.main_model.get_column_alias(self.main_model.Meta.pkname)
 
         elif self.target_field.virtual:

ormar/queryset/prefetch_query.py

@@ -1,49 +1,24 @@
 from typing import (
-    Any,
     Dict,
     List,
-    Optional,
     Sequence,
     Set,
     TYPE_CHECKING,
     Tuple,
     Type,
-    Union,
     cast,
 )
 
 import ormar
-from ormar.fields import BaseField, ManyToManyField
-from ormar.fields.foreign_key import ForeignKeyField
 from ormar.queryset.clause import QueryClause
 from ormar.queryset.query import Query
 from ormar.queryset.utils import extract_models_to_dict_of_lists, translate_list_to_dict
 
 if TYPE_CHECKING:  # pragma: no cover
     from ormar import Model
-
-
-def add_relation_field_to_fields(
-    fields: Union[Set[Any], Dict[Any, Any], None], related_field_name: str
-) -> Union[Set[Any], Dict[Any, Any], None]:
-    """
-    Adds related field into fields to include as otherwise it would be skipped.
-    Related field is added only if fields are already populated.
-    Empty fields implies all fields.
-
-    :param fields: Union[Set[Any], Dict[Any, Any], None]
-    :type fields: Dict
-    :param related_field_name: name of the field with relation
-    :type related_field_name: str
-    :return: updated fields dict
-    :rtype: Union[Set[Any], Dict[Any, Any], None]
-    """
-    if fields and related_field_name not in fields:
-        if isinstance(fields, dict):
-            fields[related_field_name] = ...
-        elif isinstance(fields, set):
-            fields.add(related_field_name)
-    return fields
+    from ormar.fields import ForeignKeyField, BaseField
+    from ormar.queryset import OrderAction
+    from ormar.models.excludable import ExcludableItems
 
 
 def sort_models(models: List["Model"], orders_by: Dict) -> List["Model"]:
@@ -125,24 +100,25 @@ class PrefetchQuery:
     def __init__(  # noqa: CFQ002
         self,
         model_cls: Type["Model"],
-        fields: Optional[Union[Dict, Set]],
-        exclude_fields: Optional[Union[Dict, Set]],
+        excludable: "ExcludableItems",
         prefetch_related: List,
         select_related: List,
-        orders_by: List,
+        orders_by: List["OrderAction"],
     ) -> None:
 
         self.model = model_cls
         self.database = self.model.Meta.database
         self._prefetch_related = prefetch_related
         self._select_related = select_related
-        self._exclude_columns = exclude_fields
-        self._columns = fields
+        self.excludable = excludable
         self.already_extracted: Dict = dict()
         self.models: Dict = {}
         self.select_dict = translate_list_to_dict(self._select_related)
         self.orders_by = orders_by or []
-        self.order_dict = translate_list_to_dict(self.orders_by, is_order=True)
+        # TODO: refactor OrderActions to use it instead of strings from it
+        self.order_dict = translate_list_to_dict(
+            [x.query_str for x in self.orders_by], is_order=True
+        )
 
     async def prefetch_related(
         self, models: Sequence["Model"], rows: List
@@ -316,7 +292,7 @@ class PrefetchQuery:
 
         for related in related_to_extract:
             target_field = model.Meta.model_fields[related]
-            target_field = cast(Type[ForeignKeyField], target_field)
+            target_field = cast(Type["ForeignKeyField"], target_field)
             target_model = target_field.to.get_name()
             model_id = model.get_relation_model_id(target_field=target_field)
 
@@ -363,8 +339,6 @@ class PrefetchQuery:
         select_dict = translate_list_to_dict(self._select_related)
         prefetch_dict = translate_list_to_dict(self._prefetch_related)
         target_model = self.model
-        fields = self._columns
-        exclude_fields = self._exclude_columns
         orders_by = self.order_dict
         for related in prefetch_dict.keys():
             await self._extract_related_models(
@@ -372,8 +346,7 @@ class PrefetchQuery:
                 target_model=target_model,
                 prefetch_dict=prefetch_dict.get(related, {}),
                 select_dict=select_dict.get(related, {}),
-                fields=fields,
-                exclude_fields=exclude_fields,
+                excludable=self.excludable,
                 orders_by=orders_by.get(related, {}),
             )
         final_models = []
@@ -391,8 +364,7 @@ class PrefetchQuery:
         target_model: Type["Model"],
         prefetch_dict: Dict,
         select_dict: Dict,
-        fields: Union[Set[Any], Dict[Any, Any], None],
-        exclude_fields: Union[Set[Any], Dict[Any, Any], None],
+        excludable: "ExcludableItems",
         orders_by: Dict,
     ) -> None:
         """
@@ -421,12 +393,10 @@ class PrefetchQuery:
         :return: None
         :rtype: None
         """
-        fields = target_model.get_included(fields, related)
-        exclude_fields = target_model.get_excluded(exclude_fields, related)
         target_field = target_model.Meta.model_fields[related]
-        target_field = cast(Type[ForeignKeyField], target_field)
+        target_field = cast(Type["ForeignKeyField"], target_field)
         reverse = False
-        if target_field.virtual or issubclass(target_field, ManyToManyField):
+        if target_field.virtual or target_field.is_multi:
             reverse = True
 
         parent_model = target_model
@@ -447,18 +417,16 @@ class PrefetchQuery:
             related_field_name = parent_model.get_related_field_name(
                 target_field=target_field
             )
-            fields = add_relation_field_to_fields(
-                fields=fields, related_field_name=related_field_name
-            )
-            table_prefix, rows = await self._run_prefetch_query(
+            table_prefix, exclude_prefix, rows = await self._run_prefetch_query(
                 target_field=target_field,
-                fields=fields,
-                exclude_fields=exclude_fields,
+                excludable=excludable,
                 filter_clauses=filter_clauses,
+                related_field_name=related_field_name,
             )
         else:
             rows = []
             table_prefix = ""
+            exclude_prefix = ""
 
         if prefetch_dict and prefetch_dict is not Ellipsis:
             for subrelated in prefetch_dict.keys():
@@ -469,8 +437,7 @@ class PrefetchQuery:
                     select_dict=self._get_select_related_if_apply(
                         subrelated, select_dict
                     ),
-                    fields=fields,
-                    exclude_fields=exclude_fields,
+                    excludable=excludable,
                     orders_by=self._get_select_related_if_apply(subrelated, orders_by),
                 )
 
@@ -480,8 +447,8 @@ class PrefetchQuery:
                 parent_model=parent_model,
                 target_field=target_field,
                 table_prefix=table_prefix,
-                fields=fields,
-                exclude_fields=exclude_fields,
+                exclude_prefix=exclude_prefix,
+                excludable=excludable,
                 prefetch_dict=prefetch_dict,
                 orders_by=orders_by,
             )
@@ -495,10 +462,10 @@ class PrefetchQuery:
     async def _run_prefetch_query(
         self,
         target_field: Type["BaseField"],
-        fields: Union[Set[Any], Dict[Any, Any], None],
-        exclude_fields: Union[Set[Any], Dict[Any, Any], None],
+        excludable: "ExcludableItems",
         filter_clauses: List,
-    ) -> Tuple[str, List]:
+        related_field_name: str,
+    ) -> Tuple[str, str, List]:
         """
         Actually runs the queries against the database and populates the raw response
         for given related model.
@@ -508,10 +475,6 @@ class PrefetchQuery:
 
         :param target_field: ormar field with relation definition
         :type target_field: Type["BaseField"]
-        :param fields: fields to include
-        :type fields: Union[Set[Any], Dict[Any, Any], None]
-        :param exclude_fields: fields to exclude
-        :type exclude_fields: Union[Set[Any], Dict[Any, Any], None]
         :param filter_clauses: list of clauses, actually one clause with ids of relation
         :type filter_clauses: List[sqlalchemy.sql.elements.TextClause]
         :return: table prefix and raw rows from sql response
@@ -522,14 +485,24 @@ class PrefetchQuery:
         select_related = []
         query_target = target_model
         table_prefix = ""
-        if issubclass(target_field, ManyToManyField):
+        exclude_prefix = target_field.to.Meta.alias_manager.resolve_relation_alias(
+            from_model=target_field.owner, relation_name=target_field.name
+        )
+        if target_field.is_multi:
             query_target = target_field.through
             select_related = [target_name]
             table_prefix = target_field.to.Meta.alias_manager.resolve_relation_alias(
                 from_model=query_target, relation_name=target_name
             )
+            exclude_prefix = table_prefix
             self.already_extracted.setdefault(target_name, {})["prefix"] = table_prefix
 
+        model_excludable = excludable.get(model_cls=target_model, alias=exclude_prefix)
+        if model_excludable.include and not model_excludable.is_included(
+            related_field_name
+        ):
+            model_excludable.set_values({related_field_name}, is_exclude=False)
+
         qry = Query(
             model_cls=query_target,
             select_related=select_related,
@@ -537,8 +510,7 @@ class PrefetchQuery:
             exclude_clauses=[],
             offset=None,
             limit_count=None,
-            fields=fields,
-            exclude_fields=exclude_fields,
+            excludable=excludable,
             order_bys=None,
             limit_raw_sql=False,
         )
@@ -546,7 +518,7 @@ class PrefetchQuery:
         # print(expr.compile(compile_kwargs={"literal_binds": True}))
         rows = await self.database.fetch_all(expr)
         self.already_extracted.setdefault(target_name, {}).update({"raw": rows})
-        return table_prefix, rows
+        return table_prefix, exclude_prefix, rows
 
     @staticmethod
     def _get_select_related_if_apply(related: str, select_dict: Dict) -> Dict:
@@ -592,8 +564,8 @@ class PrefetchQuery:
         target_field: Type["ForeignKeyField"],
         parent_model: Type["Model"],
         table_prefix: str,
-        fields: Union[Set[Any], Dict[Any, Any], None],
-        exclude_fields: Union[Set[Any], Dict[Any, Any], None],
+        exclude_prefix: str,
+        excludable: "ExcludableItems",
         prefetch_dict: Dict,
         orders_by: Dict,
     ) -> None:
@@ -607,6 +579,8 @@ class PrefetchQuery:
         already_extracted dictionary. Later those instances will be fetched by ids
         and set on the parent model after sorting if needed.
 
+        :param excludable: structure of fields to include and exclude
+        :type excludable: ExcludableItems
         :param rows: raw sql response from the prefetch query
         :type rows: List[sqlalchemy.engine.result.RowProxy]
         :param target_field: field with relation definition from parent model
@@ -615,10 +589,6 @@ class PrefetchQuery:
         :type parent_model: Type[Model]
         :param table_prefix: prefix of the target table from current relation
         :type table_prefix: str
-        :param fields: fields to include
-        :type fields: Union[Set[Any], Dict[Any, Any], None]
-        :param exclude_fields: fields to exclude
-        :type exclude_fields: Union[Set[Any], Dict[Any, Any], None]
         :param prefetch_dict: dictionaries of related models to prefetch
         :type prefetch_dict: Dict
         :param orders_by: dictionary of order by clauses by model
@@ -628,14 +598,10 @@ class PrefetchQuery:
         for row in rows:
             field_name = parent_model.get_related_field_name(target_field=target_field)
             item = target_model.extract_prefixed_table_columns(
-                item={},
-                row=row,
-                table_prefix=table_prefix,
-                fields=fields,
-                exclude_fields=exclude_fields,
+                item={}, row=row, table_prefix=table_prefix, excludable=excludable,
             )
             item["__excluded__"] = target_model.get_names_to_exclude(
-                fields=fields, exclude_fields=exclude_fields
+                excludable=excludable, alias=exclude_prefix
             )
             instance = target_model(**item)
             instance = self._populate_nested_related(

ormar/queryset/query.py

@@ -1,6 +1,5 @@
-import copy
 from collections import OrderedDict
-from typing import Dict, List, Optional, Set, TYPE_CHECKING, Tuple, Type, Union
+from typing import List, Optional, TYPE_CHECKING, Tuple, Type
 
 import sqlalchemy
 from sqlalchemy import text
@@ -8,11 +7,13 @@ from sqlalchemy import text
 import ormar  # noqa I100
 from ormar.models.helpers.models import group_related_list
 from ormar.queryset import FilterQuery, LimitQuery, OffsetQuery, OrderQuery
-from ormar.queryset.filter_action import FilterAction
+from ormar.queryset.actions.filter_action import FilterAction
 from ormar.queryset.join import SqlJoin
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
+    from ormar.queryset import OrderAction
+    from ormar.models.excludable import ExcludableItems
 
 
 class Query:
@@ -24,9 +25,8 @@ class Query:
         select_related: List,
         limit_count: Optional[int],
         offset: Optional[int],
-        fields: Optional[Union[Dict, Set]],
-        exclude_fields: Optional[Union[Dict, Set]],
-        order_bys: Optional[List],
+        excludable: "ExcludableItems",
+        order_bys: Optional[List["OrderAction"]],
         limit_raw_sql: bool,
     ) -> None:
         self.query_offset = offset
@@ -34,8 +34,7 @@ class Query:
         self._select_related = select_related[:]
         self.filter_clauses = filter_clauses[:]
         self.exclude_clauses = exclude_clauses[:]
-        self.fields = copy.deepcopy(fields) if fields else {}
-        self.exclude_fields = copy.deepcopy(exclude_fields) if exclude_fields else {}
+        self.excludable = excludable
 
         self.model_cls = model_cls
         self.table = self.model_cls.Meta.table
@@ -45,7 +44,7 @@ class Query:
         self.select_from: List[str] = []
         self.columns = [sqlalchemy.Column]
         self.order_columns = order_bys
-        self.sorted_orders: OrderedDict = OrderedDict()
+        self.sorted_orders: OrderedDict[OrderAction, text] = OrderedDict()
         self._init_sorted_orders()
 
         self.limit_raw_sql = limit_raw_sql
@@ -58,28 +57,6 @@ class Query:
             for clause in self.order_columns:
                 self.sorted_orders[clause] = None
 
-    @property
-    def prefixed_pk_name(self) -> str:
-        """
-        Shortcut for extracting prefixed with alias primary key column name from main
-        model
-        :return: alias of pk column prefix with table name.
-        :rtype: str
-        """
-        pkname_alias = self.model_cls.get_column_alias(self.model_cls.Meta.pkname)
-        return f"{self.table.name}.{pkname_alias}"
-
-    def alias(self, name: str) -> str:
-        """
-        Shortcut to extracting column alias from given master model.
-
-        :param name: name of column
-        :type name: str
-        :return: alias of given column name
-        :rtype: str
-        """
-        return self.model_cls.get_column_alias(name)
-
     def apply_order_bys_for_primary_model(self) -> None:  # noqa: CCR001
         """
         Applies order_by queries on main model when it's used as a subquery.
@@ -88,16 +65,13 @@ class Query:
         """
         if self.order_columns:
             for clause in self.order_columns:
-                if "__" not in clause:
-                    text_clause = (
-                        text(f"{self.table.name}.{self.alias(clause[1:])} desc")
-                        if clause.startswith("-")
-                        else text(f"{self.table.name}.{self.alias(clause)}")
-                    )
-                    self.sorted_orders[clause] = text_clause
+                if clause.is_source_model_order:
+                    self.sorted_orders[clause] = clause.get_text_clause()
         else:
-            order = text(self.prefixed_pk_name)
-            self.sorted_orders[self.prefixed_pk_name] = order
+            clause = ormar.OrderAction(
+                order_str=self.model_cls.Meta.pkname, model_cls=self.model_cls
+            )
+            self.sorted_orders[clause] = clause.get_text_clause()
 
     def _pagination_query_required(self) -> bool:
         """
@@ -128,10 +102,7 @@ class Query:
         :rtype: sqlalchemy.sql.selectable.Select
         """
         self_related_fields = self.model_cls.own_table_columns(
-            model=self.model_cls,
-            fields=self.fields,
-            exclude_fields=self.exclude_fields,
-            use_alias=True,
+            model=self.model_cls, excludable=self.excludable, use_alias=True,
         )
         self.columns = self.model_cls.Meta.alias_manager.prefixed_columns(
             "", self.table, self_related_fields
@@ -145,8 +116,6 @@ class Query:
         related_models = group_related_list(self._select_related)
 
         for related in related_models:
-            fields = self.model_cls.get_included(self.fields, related)
-            exclude_fields = self.model_cls.get_excluded(self.exclude_fields, related)
             remainder = None
             if isinstance(related_models, dict) and related_models[related]:
                 remainder = related_models[related]
@@ -154,8 +123,7 @@ class Query:
                 used_aliases=self.used_aliases,
                 select_from=self.select_from,
                 columns=self.columns,
-                fields=fields,
-                exclude_fields=exclude_fields,
+                excludable=self.excludable,
                 order_columns=self.order_columns,
                 sorted_orders=self.sorted_orders,
                 main_model=self.model_cls,
@@ -201,14 +169,16 @@ class Query:
         filters_to_use = [
             filter_clause
             for filter_clause in self.filter_clauses
-            if filter_clause.table_prefix == ""
+            if filter_clause.is_source_model_filter
         ]
         excludes_to_use = [
             filter_clause
             for filter_clause in self.exclude_clauses
-            if filter_clause.table_prefix == ""
+            if filter_clause.is_source_model_filter
         ]
-        sorts_to_use = {k: v for k, v in self.sorted_orders.items() if "__" not in k}
+        sorts_to_use = {
+            k: v for k, v in self.sorted_orders.items() if k.is_source_model_order
+        }
         expr = FilterQuery(filter_clauses=filters_to_use).apply(expr)
         expr = FilterQuery(filter_clauses=excludes_to_use, exclude=True).apply(expr)
         expr = OrderQuery(sorted_orders=sorts_to_use).apply(expr)
@@ -253,5 +223,3 @@ class Query:
         self.select_from = []
         self.columns = []
         self.used_aliases = []
-        self.fields = {}
-        self.exclude_fields = {}

ormar/queryset/queryset.py

@@ -1,4 +1,15 @@
-from typing import Any, Dict, List, Optional, Sequence, Set, TYPE_CHECKING, Type, Union
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    Sequence,
+    Set,
+    TYPE_CHECKING,
+    Type,
+    Union,
+    cast,
+)
 
 import databases
 import sqlalchemy
@@ -8,15 +19,16 @@ import ormar  # noqa I100
 from ormar import MultipleMatches, NoMatch
 from ormar.exceptions import ModelError, ModelPersistenceError, QueryDefinitionError
 from ormar.queryset import FilterQuery
+from ormar.queryset.actions.order_action import OrderAction
 from ormar.queryset.clause import QueryClause
 from ormar.queryset.prefetch_query import PrefetchQuery
 from ormar.queryset.query import Query
-from ormar.queryset.utils import update, update_dict_from_list
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
     from ormar.models.metaclass import ModelMeta
     from ormar.relations.querysetproxy import QuerysetProxy
+    from ormar.models.excludable import ExcludableItems
 
 
 class QuerySet:
@@ -26,18 +38,19 @@ class QuerySet:
 
     def __init__(  # noqa CFQ002
         self,
-        model_cls: Type["Model"] = None,
+        model_cls: Optional[Type["Model"]] = None,
         filter_clauses: List = None,
         exclude_clauses: List = None,
         select_related: List = None,
         limit_count: int = None,
         offset: int = None,
-        columns: Dict = None,
-        exclude_columns: Dict = None,
+        excludable: "ExcludableItems" = None,
         order_bys: List = None,
         prefetch_related: List = None,
         limit_raw_sql: bool = False,
+        proxy_source_model: Optional[Type["Model"]] = None,
     ) -> None:
+        self.proxy_source_model = proxy_source_model
         self.model_cls = model_cls
         self.filter_clauses = [] if filter_clauses is None else filter_clauses
         self.exclude_clauses = [] if exclude_clauses is None else exclude_clauses
@@ -45,8 +58,7 @@ class QuerySet:
         self._prefetch_related = [] if prefetch_related is None else prefetch_related
         self.limit_count = limit_count
         self.query_offset = offset
-        self._columns = columns or {}
-        self._exclude_columns = exclude_columns or {}
+        self._excludable = excludable or ormar.ExcludableItems()
         self.order_bys = order_bys or []
         self.limit_sql_raw = limit_raw_sql
 
@@ -62,7 +74,7 @@ class QuerySet:
                     f"ForwardRefs. \nBefore using the model you "
                     f"need to call update_forward_refs()."
                 )
-        if issubclass(owner, ormar.Model):
+            owner = cast(Type["Model"], owner)
             return self.__class__(model_cls=owner)
         return self.__class__()  # pragma: no cover
 
@@ -90,9 +102,54 @@ class QuerySet:
             raise ValueError("Model class of QuerySet is not initialized")
         return self.model_cls
 
+    def rebuild_self(  # noqa: CFQ002
+        self,
+        filter_clauses: List = None,
+        exclude_clauses: List = None,
+        select_related: List = None,
+        limit_count: int = None,
+        offset: int = None,
+        excludable: "ExcludableItems" = None,
+        order_bys: List = None,
+        prefetch_related: List = None,
+        limit_raw_sql: bool = None,
+        proxy_source_model: Optional[Type["Model"]] = None,
+    ) -> "QuerySet":
+        """
+        Method that returns new instance of queryset based on passed params,
+        all not passed params are taken from current values.
+        """
+        overwrites = {
+            "select_related": "_select_related",
+            "offset": "query_offset",
+            "excludable": "_excludable",
+            "prefetch_related": "_prefetch_related",
+            "limit_raw_sql": "limit_sql_raw",
+        }
+        passed_args = locals()
+
+        def replace_if_none(arg_name: str) -> Any:
+            if passed_args.get(arg_name) is None:
+                return getattr(self, overwrites.get(arg_name, arg_name))
+            return passed_args.get(arg_name)
+
+        return self.__class__(
+            model_cls=self.model_cls,
+            filter_clauses=replace_if_none("filter_clauses"),
+            exclude_clauses=replace_if_none("exclude_clauses"),
+            select_related=replace_if_none("select_related"),
+            limit_count=replace_if_none("limit_count"),
+            offset=replace_if_none("offset"),
+            excludable=replace_if_none("excludable"),
+            order_bys=replace_if_none("order_bys"),
+            prefetch_related=replace_if_none("prefetch_related"),
+            limit_raw_sql=replace_if_none("limit_raw_sql"),
+            proxy_source_model=replace_if_none("proxy_source_model"),
+        )
+
     async def _prefetch_related_models(
-        self, models: Sequence[Optional["Model"]], rows: List
-    ) -> Sequence[Optional["Model"]]:
+        self, models: List[Optional["Model"]], rows: List
+    ) -> List[Optional["Model"]]:
         """
         Performs prefetch query for selected models names.
 
@@ -105,15 +162,14 @@ class QuerySet:
         """
         query = PrefetchQuery(
             model_cls=self.model,
-            fields=self._columns,
-            exclude_fields=self._exclude_columns,
+            excludable=self._excludable,
             prefetch_related=self._prefetch_related,
             select_related=self._select_related,
             orders_by=self.order_bys,
         )
         return await query.prefetch_related(models=models, rows=rows)  # type: ignore
 
-    def _process_query_result_rows(self, rows: List) -> Sequence[Optional["Model"]]:
+    def _process_query_result_rows(self, rows: List) -> List[Optional["Model"]]:
         """
         Process database rows and initialize ormar Model from each of the rows.
 
@@ -126,8 +182,9 @@ class QuerySet:
             self.model.from_row(
                 row=row,
                 select_related=self._select_related,
-                fields=self._columns,
-                exclude_fields=self._exclude_columns,
+                excludable=self._excludable,
+                source_model=self.model,
+                proxy_source_model=self.proxy_source_model,
             )
             for row in rows
         ]
@@ -191,8 +248,7 @@ class QuerySet:
             exclude_clauses=self.exclude_clauses,
             offset=offset or self.query_offset,
             limit_count=limit or self.limit_count,
-            fields=self._columns,
-            exclude_fields=self._exclude_columns,
+            excludable=self._excludable,
             order_bys=order_bys or self.order_bys,
             limit_raw_sql=self.limit_sql_raw,
         )
@@ -241,18 +297,10 @@ class QuerySet:
             exclude_clauses = self.exclude_clauses
             filter_clauses = filter_clauses
 
-        return self.__class__(
-            model_cls=self.model,
+        return self.rebuild_self(
             filter_clauses=filter_clauses,
             exclude_clauses=exclude_clauses,
             select_related=select_related,
-            limit_count=self.limit_count,
-            offset=self.query_offset,
-            columns=self._columns,
-            exclude_columns=self._exclude_columns,
-            order_bys=self.order_bys,
-            prefetch_related=self._prefetch_related,
-            limit_raw_sql=self.limit_sql_raw,
         )
 
     def exclude(self, **kwargs: Any) -> "QuerySet":  # noqa: A003
@@ -296,20 +344,8 @@ class QuerySet:
         if not isinstance(related, list):
             related = [related]
 
-        related = list(set(list(self._select_related) + related))
-        return self.__class__(
-            model_cls=self.model,
-            filter_clauses=self.filter_clauses,
-            exclude_clauses=self.exclude_clauses,
-            select_related=related,
-            limit_count=self.limit_count,
-            offset=self.query_offset,
-            columns=self._columns,
-            exclude_columns=self._exclude_columns,
-            order_bys=self.order_bys,
-            prefetch_related=self._prefetch_related,
-            limit_raw_sql=self.limit_sql_raw,
-        )
+        related = sorted(list(set(list(self._select_related) + related)))
+        return self.rebuild_self(select_related=related,)
 
     def prefetch_related(self, related: Union[List, str]) -> "QuerySet":
         """
@@ -333,21 +369,11 @@ class QuerySet:
             related = [related]
 
         related = list(set(list(self._prefetch_related) + related))
-        return self.__class__(
-            model_cls=self.model,
-            filter_clauses=self.filter_clauses,
-            exclude_clauses=self.exclude_clauses,
-            select_related=self._select_related,
-            limit_count=self.limit_count,
-            offset=self.query_offset,
-            columns=self._columns,
-            exclude_columns=self._exclude_columns,
-            order_bys=self.order_bys,
-            prefetch_related=related,
-            limit_raw_sql=self.limit_sql_raw,
-        )
+        return self.rebuild_self(prefetch_related=related,)
 
-    def fields(self, columns: Union[List, str, Set, Dict]) -> "QuerySet":
+    def fields(
+        self, columns: Union[List, str, Set, Dict], _is_exclude: bool = False
+    ) -> "QuerySet":
         """
         With `fields()` you can select subset of model columns to limit the data load.
 
@@ -385,34 +411,22 @@ class QuerySet:
 
         To include whole nested model specify model related field name and ellipsis.
 
+        :param _is_exclude: flag if it's exclude or include operation
+        :type _is_exclude: bool
         :param columns: columns to include
         :type columns: Union[List, str, Set, Dict]
         :return: QuerySet
         :rtype: QuerySet
         """
-        if isinstance(columns, str):
-            columns = [columns]
-
-        current_included = self._columns
-        if not isinstance(columns, dict):
-            current_included = update_dict_from_list(current_included, columns)
-        else:
-            current_included = update(current_included, columns)
-
-        return self.__class__(
-            model_cls=self.model,
-            filter_clauses=self.filter_clauses,
-            exclude_clauses=self.exclude_clauses,
-            select_related=self._select_related,
-            limit_count=self.limit_count,
-            offset=self.query_offset,
-            columns=current_included,
-            exclude_columns=self._exclude_columns,
-            order_bys=self.order_bys,
-            prefetch_related=self._prefetch_related,
-            limit_raw_sql=self.limit_sql_raw,
+        excludable = ormar.ExcludableItems.from_excludable(self._excludable)
+        excludable.build(
+            items=columns,
+            model_cls=self.model_cls,  # type: ignore
+            is_exclude=_is_exclude,
         )
 
+        return self.rebuild_self(excludable=excludable,)
+
     def exclude_fields(self, columns: Union[List, str, Set, Dict]) -> "QuerySet":
         """
         With `exclude_fields()` you can select subset of model columns that will
@@ -440,28 +454,7 @@ class QuerySet:
         :return: QuerySet
         :rtype: QuerySet
         """
-        if isinstance(columns, str):
-            columns = [columns]
-
-        current_excluded = self._exclude_columns
-        if not isinstance(columns, dict):
-            current_excluded = update_dict_from_list(current_excluded, columns)
-        else:
-            current_excluded = update(current_excluded, columns)
-
-        return self.__class__(
-            model_cls=self.model,
-            filter_clauses=self.filter_clauses,
-            exclude_clauses=self.exclude_clauses,
-            select_related=self._select_related,
-            limit_count=self.limit_count,
-            offset=self.query_offset,
-            columns=self._columns,
-            exclude_columns=current_excluded,
-            order_bys=self.order_bys,
-            prefetch_related=self._prefetch_related,
-            limit_raw_sql=self.limit_sql_raw,
-        )
+        return self.fields(columns=columns, _is_exclude=True)
 
     def order_by(self, columns: Union[List, str]) -> "QuerySet":
         """
@@ -498,20 +491,13 @@ class QuerySet:
         if not isinstance(columns, list):
             columns = [columns]
 
-        order_bys = self.order_bys + [x for x in columns if x not in self.order_bys]
-        return self.__class__(
-            model_cls=self.model,
-            filter_clauses=self.filter_clauses,
-            exclude_clauses=self.exclude_clauses,
-            select_related=self._select_related,
-            limit_count=self.limit_count,
-            offset=self.query_offset,
-            columns=self._columns,
-            exclude_columns=self._exclude_columns,
-            order_bys=order_bys,
-            prefetch_related=self._prefetch_related,
-            limit_raw_sql=self.limit_sql_raw,
-        )
+        orders_by = [
+            OrderAction(order_str=x, model_cls=self.model_cls)  # type: ignore
+            for x in columns
+        ]
+
+        order_bys = self.order_bys + [x for x in orders_by if x not in self.order_bys]
+        return self.rebuild_self(order_bys=order_bys,)
 
     async def exists(self) -> bool:
         """
@@ -551,17 +537,19 @@ class QuerySet:
         :return: number of updated rows
         :rtype: int
         """
+        if not each and not self.filter_clauses:
+            raise QueryDefinitionError(
+                "You cannot update without filtering the queryset first. "
+                "If you want to update all rows use update(each=True, **kwargs)"
+            )
+
         self_fields = self.model.extract_db_own_fields().union(
             self.model.extract_related_names()
         )
         updates = {k: v for k, v in kwargs.items() if k in self_fields}
         updates = self.model.validate_choices(updates)
         updates = self.model.translate_columns_to_aliases(updates)
-        if not each and not self.filter_clauses:
-            raise QueryDefinitionError(
-                "You cannot update without filtering the queryset first. "
-                "If you want to update all rows use update(each=True, **kwargs)"
-            )
+
         expr = FilterQuery(filter_clauses=self.filter_clauses).apply(
             self.table.update().values(**updates)
         )
@@ -610,19 +598,7 @@ class QuerySet:
 
         limit_count = page_size
         query_offset = (page - 1) * page_size
-        return self.__class__(
-            model_cls=self.model,
-            filter_clauses=self.filter_clauses,
-            exclude_clauses=self.exclude_clauses,
-            select_related=self._select_related,
-            limit_count=limit_count,
-            offset=query_offset,
-            columns=self._columns,
-            exclude_columns=self._exclude_columns,
-            order_bys=self.order_bys,
-            prefetch_related=self._prefetch_related,
-            limit_raw_sql=self.limit_sql_raw,
-        )
+        return self.rebuild_self(limit_count=limit_count, offset=query_offset,)
 
     def limit(self, limit_count: int, limit_raw_sql: bool = None) -> "QuerySet":
         """
@@ -639,19 +615,7 @@ class QuerySet:
         :rtype: QuerySet
         """
         limit_raw_sql = self.limit_sql_raw if limit_raw_sql is None else limit_raw_sql
-        return self.__class__(
-            model_cls=self.model,
-            filter_clauses=self.filter_clauses,
-            exclude_clauses=self.exclude_clauses,
-            select_related=self._select_related,
-            limit_count=limit_count,
-            offset=self.query_offset,
-            columns=self._columns,
-            exclude_columns=self._exclude_columns,
-            order_bys=self.order_bys,
-            prefetch_related=self._prefetch_related,
-            limit_raw_sql=limit_raw_sql,
-        )
+        return self.rebuild_self(limit_count=limit_count, limit_raw_sql=limit_raw_sql,)
 
     def offset(self, offset: int, limit_raw_sql: bool = None) -> "QuerySet":
         """
@@ -668,19 +632,7 @@ class QuerySet:
         :rtype: QuerySet
         """
         limit_raw_sql = self.limit_sql_raw if limit_raw_sql is None else limit_raw_sql
-        return self.__class__(
-            model_cls=self.model,
-            filter_clauses=self.filter_clauses,
-            exclude_clauses=self.exclude_clauses,
-            select_related=self._select_related,
-            limit_count=self.limit_count,
-            offset=offset,
-            columns=self._columns,
-            exclude_columns=self._exclude_columns,
-            order_bys=self.order_bys,
-            prefetch_related=self._prefetch_related,
-            limit_raw_sql=limit_raw_sql,
-        )
+        return self.rebuild_self(offset=offset, limit_raw_sql=limit_raw_sql,)
 
     async def first(self, **kwargs: Any) -> "Model":
         """
@@ -697,7 +649,14 @@ class QuerySet:
             return await self.filter(**kwargs).first()
 
         expr = self.build_select_expression(
-            limit=1, order_bys=[f"{self.model.Meta.pkname}"] + self.order_bys
+            limit=1,
+            order_bys=[
+                OrderAction(
+                    order_str=f"{self.model.Meta.pkname}",
+                    model_cls=self.model_cls,  # type: ignore
+                )
+            ]
+            + self.order_bys,
         )
         rows = await self.database.fetch_all(expr)
         processed_rows = self._process_query_result_rows(rows)
@@ -726,7 +685,14 @@ class QuerySet:
 
         if not self.filter_clauses:
             expr = self.build_select_expression(
-                limit=1, order_bys=[f"-{self.model.Meta.pkname}"] + self.order_bys
+                limit=1,
+                order_bys=[
+                    OrderAction(
+                        order_str=f"-{self.model.Meta.pkname}",
+                        model_cls=self.model_cls,  # type: ignore
+                    )
+                ]
+                + self.order_bys,
             )
         else:
             expr = self.build_select_expression()

ormar/queryset/utils.py

@@ -12,8 +12,6 @@ from typing import (
     Union,
 )
 
-from ormar.fields import ManyToManyField
-
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
 
@@ -219,7 +217,7 @@ def extract_models_to_dict_of_lists(
 
 def get_relationship_alias_model_and_str(
     source_model: Type["Model"], related_parts: List
-) -> Tuple[str, Type["Model"], str]:
+) -> Tuple[str, Type["Model"], str, bool]:
     """
     Walks the relation to retrieve the actual model on which the clause should be
     constructed, extracts alias based on last relation leading to target model.
@@ -231,19 +229,37 @@ def get_relationship_alias_model_and_str(
     :rtype: Tuple[str, Type["Model"], str]
     """
     table_prefix = ""
-    model_cls = source_model
-    previous_model = model_cls
-    manager = model_cls.Meta.alias_manager
-    for relation in related_parts:
-        related_field = model_cls.Meta.model_fields[relation]
-        if issubclass(related_field, ManyToManyField):
+    is_through = False
+    target_model = source_model
+    previous_model = target_model
+    previous_models = [target_model]
+    manager = target_model.Meta.alias_manager
+    for relation in related_parts[:]:
+        related_field = target_model.Meta.model_fields[relation]
+
+        if related_field.is_through:
+            # through is always last - cannot go further
+            is_through = True
+            related_parts.remove(relation)
+            through_field = related_field.owner.Meta.model_fields[
+                related_field.related_name or ""
+            ]
+            if len(previous_models) > 1 and previous_models[-2] == through_field.to:
+                previous_model = through_field.to
+                relation = through_field.related_name
+            else:
+                relation = related_field.related_name
+
+        if related_field.is_multi:
             previous_model = related_field.through
             relation = related_field.default_target_field_name()  # type: ignore
         table_prefix = manager.resolve_relation_alias(
             from_model=previous_model, relation_name=relation
         )
-        model_cls = related_field.to
-        previous_model = model_cls
+        target_model = related_field.to
+        previous_model = target_model
+        if not is_through:
+            previous_models.append(previous_model)
     relation_str = "__".join(related_parts)
 
-    return table_prefix, model_cls, relation_str
+    return table_prefix, target_model, relation_str, is_through

ormar/relations/alias_manager.py

@@ -1,13 +1,15 @@
 import string
 import uuid
 from random import choices
-from typing import Any, Dict, List, TYPE_CHECKING, Type
+from typing import Any, Dict, List, TYPE_CHECKING, Type, Union
 
 import sqlalchemy
 from sqlalchemy import text
 
 if TYPE_CHECKING:  # pragma: no cover
     from ormar import Model
+    from ormar.models import ModelRow
+    from ormar.fields import ForeignKeyField
 
 
 def get_table_alias() -> str:
@@ -133,7 +135,7 @@ class AliasManager:
         return alias
 
     def resolve_relation_alias(
-        self, from_model: Type["Model"], relation_name: str
+        self, from_model: Union[Type["Model"], Type["ModelRow"]], relation_name: str
     ) -> str:
         """
         Given model and relation name returns the alias for this relation.
@@ -147,3 +149,35 @@ class AliasManager:
         """
         alias = self._aliases_new.get(f"{from_model.get_name()}_{relation_name}", "")
         return alias
+
+    def resolve_relation_alias_after_complex(
+        self,
+        source_model: Union[Type["Model"], Type["ModelRow"]],
+        relation_str: str,
+        relation_field: Type["ForeignKeyField"],
+    ) -> str:
+        """
+        Given source model and relation string returns the alias for this complex
+        relation if it exists, otherwise fallback to normal relation from a relation
+        field definition.
+
+        :param relation_field: field with direct relation definition
+        :type relation_field: Type["ForeignKeyField"]
+        :param source_model: model with query starts
+        :type source_model: source Model
+        :param relation_str: string with relation joins defined
+        :type relation_str: str
+        :return: alias of the relation
+        :rtype: str
+        """
+        alias = ""
+        if relation_str and "__" in relation_str:
+            alias = self.resolve_relation_alias(
+                from_model=source_model, relation_name=relation_str
+            )
+        if not alias:
+            alias = self.resolve_relation_alias(
+                from_model=relation_field.get_source_model(),
+                relation_name=relation_field.get_relation_name(),
+            )
+        return alias

ormar/relations/querysetproxy.py

@@ -1,4 +1,5 @@
-from typing import (
+from _weakref import CallableProxyType
+from typing import (  # noqa: I100, I201
     Any,
     Dict,
     List,
@@ -7,12 +8,12 @@ from typing import (
     Sequence,
     Set,
     TYPE_CHECKING,
-    TypeVar,
     Union,
+    cast,
 )
 
 import ormar
-from ormar.exceptions import ModelPersistenceError
+from ormar.exceptions import ModelPersistenceError, QueryDefinitionError
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar.relations import Relation
@@ -20,10 +21,8 @@ if TYPE_CHECKING:  # pragma no cover
     from ormar.queryset import QuerySet
     from ormar import RelationType
 
-    T = TypeVar("T", bound=Model)
 
-
-class QuerysetProxy(ormar.QuerySetProtocol):
+class QuerysetProxy:
     """
     Exposes QuerySet methods on relations, but also handles creating and removing
     of through Models for m2m relations.
@@ -38,12 +37,17 @@ class QuerysetProxy(ormar.QuerySetProtocol):
         self.relation: Relation = relation
         self._queryset: Optional["QuerySet"] = qryset
         self.type_: "RelationType" = type_
-        self._owner: "Model" = self.relation.manager.owner
+        self._owner: Union[CallableProxyType, "Model"] = self.relation.manager.owner
         self.related_field_name = self._owner.Meta.model_fields[
             self.relation.field_name
         ].get_related_name()
         self.related_field = self.relation.to.Meta.model_fields[self.related_field_name]
         self.owner_pk_value = self._owner.pk
+        self.through_model_name = (
+            self.related_field.through.get_name()
+            if self.type_ == ormar.RelationType.MULTIPLE
+            else ""
+        )
 
     @property
     def queryset(self) -> "QuerySet":
@@ -65,7 +69,7 @@ class QuerysetProxy(ormar.QuerySetProtocol):
         """
         self._queryset = value
 
-    def _assign_child_to_parent(self, child: Optional["T"]) -> None:
+    def _assign_child_to_parent(self, child: Optional["Model"]) -> None:
         """
         Registers child in parents RelationManager.
 
@@ -77,7 +81,9 @@ class QuerysetProxy(ormar.QuerySetProtocol):
             rel_name = self.relation.field_name
             setattr(owner, rel_name, child)
 
-    def _register_related(self, child: Union["T", Sequence[Optional["T"]]]) -> None:
+    def _register_related(
+        self, child: Union["Model", Sequence[Optional["Model"]]]
+    ) -> None:
         """
         Registers child/ children in parents RelationManager.
 
@@ -89,6 +95,7 @@ class QuerysetProxy(ormar.QuerySetProtocol):
                 self._assign_child_to_parent(subchild)
         else:
             assert isinstance(child, ormar.Model)
+            child = cast("Model", child)
             self._assign_child_to_parent(child)
 
     def _clean_items_on_load(self) -> None:
@@ -99,17 +106,20 @@ class QuerysetProxy(ormar.QuerySetProtocol):
             for item in self.relation.related_models[:]:
                 self.relation.remove(item)
 
-    async def create_through_instance(self, child: "T") -> None:
+    async def create_through_instance(self, child: "Model", **kwargs: Any) -> None:
         """
         Crete a through model instance in the database for m2m relations.
 
+        :param kwargs: dict of additional keyword arguments for through instance
+        :type kwargs: Any
         :param child: child model instance
         :type child: Model
         """
         model_cls = self.relation.through
         owner_column = self.related_field.default_target_field_name()  # type: ignore
         child_column = self.related_field.default_source_field_name()  # type: ignore
-        kwargs = {owner_column: self._owner.pk, child_column: child.pk}
+        rel_kwargs = {owner_column: self._owner.pk, child_column: child.pk}
+        final_kwargs = {**rel_kwargs, **kwargs}
         if child.pk is None:
             raise ModelPersistenceError(
                 f"You cannot save {child.get_name()} "
@@ -117,18 +127,34 @@ class QuerysetProxy(ormar.QuerySetProtocol):
                 f"Save the child model first."
             )
         expr = model_cls.Meta.table.insert()
-        expr = expr.values(**kwargs)
+        expr = expr.values(**final_kwargs)
         # print("\n", expr.compile(compile_kwargs={"literal_binds": True}))
         await model_cls.Meta.database.execute(expr)
 
-    async def delete_through_instance(self, child: "T") -> None:
+    async def update_through_instance(self, child: "Model", **kwargs: Any) -> None:
+        """
+        Updates a through model instance in the database for m2m relations.
+
+        :param kwargs: dict of additional keyword arguments for through instance
+        :type kwargs: Any
+        :param child: child model instance
+        :type child: Model
+        """
+        model_cls = self.relation.through
+        owner_column = self.related_field.default_target_field_name()  # type: ignore
+        child_column = self.related_field.default_source_field_name()  # type: ignore
+        rel_kwargs = {owner_column: self._owner.pk, child_column: child.pk}
+        through_model = await model_cls.objects.get(**rel_kwargs)
+        await through_model.update(**kwargs)
+
+    async def delete_through_instance(self, child: "Model") -> None:
         """
         Removes through model instance from the database for m2m relations.
 
         :param child: child model instance
         :type child: Model
         """
-        queryset = ormar.QuerySet(model_cls=self.relation.through)
+        queryset = ormar.QuerySet(model_cls=self.relation.through)  # type: ignore
         owner_column = self.related_field.default_target_field_name()  # type: ignore
         child_column = self.related_field.default_source_field_name()  # type: ignore
         kwargs = {owner_column: self._owner, child_column: child}
@@ -176,10 +202,10 @@ class QuerysetProxy(ormar.QuerySetProtocol):
         :rtype: int
         """
         if self.type_ == ormar.RelationType.MULTIPLE:
-            queryset = ormar.QuerySet(model_cls=self.relation.through)
+            queryset = ormar.QuerySet(model_cls=self.relation.through)  # type: ignore
             owner_column = self._owner.get_name()
         else:
-            queryset = ormar.QuerySet(model_cls=self.relation.to)
+            queryset = ormar.QuerySet(model_cls=self.relation.to)  # type: ignore
             owner_column = self.related_field.name
         kwargs = {owner_column: self._owner}
         self._clean_items_on_load()
@@ -270,14 +296,47 @@ class QuerysetProxy(ormar.QuerySetProtocol):
         :return: created model
         :rtype: Model
         """
+        through_kwargs = kwargs.pop(self.through_model_name, {})
         if self.type_ == ormar.RelationType.REVERSE:
             kwargs[self.related_field.name] = self._owner
         created = await self.queryset.create(**kwargs)
         self._register_related(created)
         if self.type_ == ormar.RelationType.MULTIPLE:
-            await self.create_through_instance(created)
+            await self.create_through_instance(created, **through_kwargs)
         return created
 
+    async def update(self, each: bool = False, **kwargs: Any) -> int:
+        """
+        Updates the model table after applying the filters from kwargs.
+
+        You have to either pass a filter to narrow down a query or explicitly pass
+        each=True flag to affect whole table.
+
+        :param each: flag if whole table should be affected if no filter is passed
+        :type each: bool
+        :param kwargs: fields names and proper value types
+        :type kwargs: Any
+        :return: number of updated rows
+        :rtype: int
+        """
+        # queryset proxy always have one filter for pk of parent model
+        if not each and len(self.queryset.filter_clauses) == 1:
+            raise QueryDefinitionError(
+                "You cannot update without filtering the queryset first. "
+                "If you want to update all rows use update(each=True, **kwargs)"
+            )
+
+        through_kwargs = kwargs.pop(self.through_model_name, {})
+        children = await self.queryset.all()
+        for child in children:
+            await child.update(**kwargs)  # type: ignore
+            if self.type_ == ormar.RelationType.MULTIPLE and through_kwargs:
+                await self.update_through_instance(
+                    child=child,  # type: ignore
+                    **through_kwargs,
+                )
+        return len(children)
+
     async def get_or_create(self, **kwargs: Any) -> "Model":
         """
         Combination of create and get methods.

ormar/relations/relation.py

@@ -1,17 +1,13 @@
 from enum import Enum
-from typing import List, Optional, Set, TYPE_CHECKING, Type, TypeVar, Union
+from typing import List, Optional, Set, TYPE_CHECKING, Type, Union
 
 import ormar  # noqa I100
 from ormar.exceptions import RelationshipInstanceError  # noqa I100
-from ormar.fields.foreign_key import ForeignKeyField  # noqa I100
 from ormar.relations.relation_proxy import RelationProxy
 
 if TYPE_CHECKING:  # pragma no cover
-    from ormar import Model
     from ormar.relations import RelationsManager
-    from ormar.models import NewBaseModel
-
-    T = TypeVar("T", bound=Model)
+    from ormar.models import Model, NewBaseModel
 
 
 class RelationType(Enum):
@@ -26,6 +22,7 @@ class RelationType(Enum):
     PRIMARY = 1
     REVERSE = 2
     MULTIPLE = 3
+    THROUGH = 4
 
 
 class Relation:
@@ -38,8 +35,8 @@ class Relation:
         manager: "RelationsManager",
         type_: RelationType,
         field_name: str,
-        to: Type["T"],
-        through: Type["T"] = None,
+        to: Type["Model"],
+        through: Type["Model"] = None,
     ) -> None:
         """
         Initialize the Relation and keep the related models either as instances of
@@ -62,17 +59,25 @@ class Relation:
         self._owner: "Model" = manager.owner
         self._type: RelationType = type_
         self._to_remove: Set = set()
-        self.to: Type["T"] = to
-        self._through: Optional[Type["T"]] = through
+        self.to: Type["Model"] = to
+        self._through = through
         self.field_name: str = field_name
-        self.related_models: Optional[Union[RelationProxy, "T"]] = (
+        self.related_models: Optional[Union[RelationProxy, "Model"]] = (
             RelationProxy(relation=self, type_=type_, field_name=field_name)
             if type_ in (RelationType.REVERSE, RelationType.MULTIPLE)
             else None
         )
 
+    def clear(self) -> None:
+        if self._type in (RelationType.PRIMARY, RelationType.THROUGH):
+            self.related_models = None
+            self._owner.__dict__[self.field_name] = None
+        elif self.related_models is not None:
+            self.related_models._clear()
+            self._owner.__dict__[self.field_name] = None
+
     @property
-    def through(self) -> Type["T"]:
+    def through(self) -> Type["Model"]:
         if not self._through:  # pragma: no cover
             raise RelationshipInstanceError("Relation does not have through model!")
         return self._through
@@ -119,7 +124,7 @@ class Relation:
                 self._to_remove.add(ind)
         return None
 
-    def add(self, child: "T") -> None:
+    def add(self, child: "Model") -> None:
         """
         Adds child Model to relation, either sets child as related model or adds
         it to the list in RelationProxy depending on relation type.
@@ -128,7 +133,7 @@ class Relation:
         :type child: Model
         """
         relation_name = self.field_name
-        if self._type == RelationType.PRIMARY:
+        if self._type in (RelationType.PRIMARY, RelationType.THROUGH):
             self.related_models = child
             self._owner.__dict__[relation_name] = child
         else:
@@ -160,7 +165,7 @@ class Relation:
                 self.related_models.pop(position)  # type: ignore
                 del self._owner.__dict__[relation_name][position]
 
-    def get(self) -> Optional[Union[List["T"], "T"]]:
+    def get(self) -> Optional[Union[List["Model"], "Model"]]:
         """
         Return the related model or models from RelationProxy.
 

ormar/relations/relation_manager.py

@@ -1,17 +1,12 @@
-from typing import Dict, List, Optional, Sequence, TYPE_CHECKING, Type, TypeVar, Union
+from typing import Dict, List, Optional, Sequence, TYPE_CHECKING, Type, Union
 from weakref import proxy
 
-from ormar.fields import BaseField
-from ormar.fields.foreign_key import ForeignKeyField
-from ormar.fields.many_to_many import ManyToManyField
 from ormar.relations.relation import Relation, RelationType
 from ormar.relations.utils import get_relations_sides_and_names
 
 if TYPE_CHECKING:  # pragma no cover
-    from ormar import Model
-    from ormar.models import NewBaseModel
-
-    T = TypeVar("T", bound=Model)
+    from ormar.models import NewBaseModel, Model
+    from ormar.fields import ForeignKeyField, BaseField
 
 
 class RelationsManager:
@@ -21,8 +16,8 @@ class RelationsManager:
 
     def __init__(
         self,
-        related_fields: List[Type[ForeignKeyField]] = None,
-        owner: "NewBaseModel" = None,
+        related_fields: List[Type["ForeignKeyField"]] = None,
+        owner: Optional["Model"] = None,
     ) -> None:
         self.owner = proxy(owner)
         self._related_fields = related_fields or []
@@ -31,35 +26,6 @@ class RelationsManager:
         for field in self._related_fields:
             self._add_relation(field)
 
-    def _get_relation_type(self, field: Type[BaseField]) -> RelationType:
-        """
-        Returns type of the relation declared on a field.
-
-        :param field: field with relation declaration
-        :type field: Type[BaseField]
-        :return: type of the relation defined on field
-        :rtype: RelationType
-        """
-        if issubclass(field, ManyToManyField):
-            return RelationType.MULTIPLE
-        return RelationType.PRIMARY if not field.virtual else RelationType.REVERSE
-
-    def _add_relation(self, field: Type[BaseField]) -> None:
-        """
-        Registers relation in the manager.
-        Adds Relation instance under field.name.
-
-        :param field: field with relation declaration
-        :type field: Type[BaseField]
-        """
-        self._relations[field.name] = Relation(
-            manager=self,
-            type_=self._get_relation_type(field),
-            field_name=field.name,
-            to=field.to,
-            through=getattr(field, "through", None),
-        )
-
     def __contains__(self, item: str) -> bool:
         """
         Checks if relation with given name is already registered.
@@ -71,7 +37,11 @@ class RelationsManager:
         """
         return item in self._related_names
 
-    def get(self, name: str) -> Optional[Union["T", Sequence["T"]]]:
+    def clear(self) -> None:
+        for relation in self._relations.values():
+            relation.clear()
+
+    def get(self, name: str) -> Optional[Union["Model", Sequence["Model"]]]:
         """
         Returns the related model/models if relation is set.
         Actual call is delegated to Relation instance registered under relation name.
@@ -86,20 +56,6 @@ class RelationsManager:
             return relation.get()
         return None  # pragma nocover
 
-    def _get(self, name: str) -> Optional[Relation]:
-        """
-        Returns the actual relation and not the related model(s).
-
-        :param name: name of the relation
-        :type name: str
-        :return: Relation instance
-        :rtype: ormar.relations.relation.Relation
-        """
-        relation = self._relations.get(name, None)
-        if relation is not None:
-            return relation
-        return None
-
     @staticmethod
     def add(parent: "Model", child: "Model", field: Type["ForeignKeyField"],) -> None:
         """
@@ -167,3 +123,48 @@ class RelationsManager:
         relation_name = item.Meta.model_fields[name].get_related_name()
         item._orm.remove(name, parent)
         parent._orm.remove(relation_name, item)
+
+    def _get(self, name: str) -> Optional[Relation]:
+        """
+        Returns the actual relation and not the related model(s).
+
+        :param name: name of the relation
+        :type name: str
+        :return: Relation instance
+        :rtype: ormar.relations.relation.Relation
+        """
+        relation = self._relations.get(name, None)
+        if relation is not None:
+            return relation
+        return None
+
+    def _get_relation_type(self, field: Type["BaseField"]) -> RelationType:
+        """
+        Returns type of the relation declared on a field.
+
+        :param field: field with relation declaration
+        :type field: Type[BaseField]
+        :return: type of the relation defined on field
+        :rtype: RelationType
+        """
+        if field.is_multi:
+            return RelationType.MULTIPLE
+        if field.is_through:
+            return RelationType.THROUGH
+        return RelationType.PRIMARY if not field.virtual else RelationType.REVERSE
+
+    def _add_relation(self, field: Type["BaseField"]) -> None:
+        """
+        Registers relation in the manager.
+        Adds Relation instance under field.name.
+
+        :param field: field with relation declaration
+        :type field: Type[BaseField]
+        """
+        self._relations[field.name] = Relation(
+            manager=self,
+            type_=self._get_relation_type(field),
+            field_name=field.name,
+            to=field.to,
+            through=getattr(field, "through", None),
+        )

ormar/relations/relation_proxy.py

@@ -27,7 +27,9 @@ class RelationProxy(list):
         self.type_: "RelationType" = type_
         self.field_name = field_name
         self._owner: "Model" = self.relation.manager.owner
-        self.queryset_proxy = QuerysetProxy(relation=self.relation, type_=type_)
+        self.queryset_proxy: QuerysetProxy = QuerysetProxy(
+            relation=self.relation, type_=type_
+        )
         self._related_field_name: Optional[str] = None
 
     @property
@@ -73,6 +75,9 @@ class RelationProxy(list):
         self._initialize_queryset()
         return getattr(self.queryset_proxy, item)
 
+    def _clear(self) -> None:
+        super().clear()
+
     def _initialize_queryset(self) -> None:
         """
         Initializes the QuerySetProxy if not yet initialized.
@@ -117,7 +122,9 @@ class RelationProxy(list):
         self._check_if_model_saved()
         kwargs = {f"{related_field.get_alias()}__{pkname}": self._owner.pk}
         queryset = (
-            ormar.QuerySet(model_cls=self.relation.to)
+            ormar.QuerySet(
+                model_cls=self.relation.to, proxy_source_model=self._owner.__class__
+            )
             .select_related(related_field.name)
             .filter(**kwargs)
         )
@@ -163,19 +170,21 @@ class RelationProxy(list):
             else:
                 await item.delete()
 
-    async def add(self, item: "Model") -> None:
+    async def add(self, item: "Model", **kwargs: Any) -> None:
         """
         Adds child model to relation.
 
         For ManyToMany relations through instance is automatically created.
 
+        :param kwargs: dict of additional keyword arguments for through instance
+        :type kwargs: Any
         :param item: child to add to relation
         :type item: Model
         """
         relation_name = self.related_field_name
         self._check_if_model_saved()
         if self.type_ == ormar.RelationType.MULTIPLE:
-            await self.queryset_proxy.create_through_instance(item)
+            await self.queryset_proxy.create_through_instance(item, **kwargs)
             setattr(item, relation_name, self._owner)
         else:
             setattr(item, relation_name, self._owner)

pydoc-markdown.yml

@@ -21,9 +21,15 @@ renderer:
         - title: Model
           contents:
             - models.model.*
+        - title: Model Row
+          contents:
+              - models.model_row.*
         - title: New BaseModel
           contents:
             - models.newbasemodel.*
+        - title: Excludable Items
+          contents:
+              - models.excludable.*
         - title: Model Table Proxy
           contents:
             - models.modelproxy.*

tests/test_aliases.py

@@ -1,4 +1,4 @@
-from typing import Optional, Union, List
+from typing import List, Optional
 
 import databases
 import pytest
@@ -23,13 +23,6 @@ class Child(ormar.Model):
     born_year: int = ormar.Integer(name="year_born", nullable=True)
 
 
-class ArtistChildren(ormar.Model):
-    class Meta:
-        tablename = "children_x_artists"
-        metadata = metadata
-        database = database
-
-
 class Artist(ormar.Model):
     class Meta:
         tablename = "artists"
@@ -40,9 +33,7 @@ class Artist(ormar.Model):
     first_name: str = ormar.String(name="fname", max_length=100)
     last_name: str = ormar.String(name="lname", max_length=100)
     born_year: int = ormar.Integer(name="year")
-    children: Optional[Union[Child, List[Child]]] = ormar.ManyToMany(
-        Child, through=ArtistChildren
-    )
+    children: Optional[List[Child]] = ormar.ManyToMany(Child)
 
 
 class Album(ormar.Model):

tests/test_excludable_items.py

@@ -0,0 +1,218 @@
+from typing import List, Optional
+
+import databases
+import sqlalchemy
+
+import ormar
+from ormar.models.excludable import ExcludableItems
+from tests.settings import DATABASE_URL
+
+database = databases.Database(DATABASE_URL, force_rollback=True)
+metadata = sqlalchemy.MetaData()
+
+
+class BaseMeta(ormar.ModelMeta):
+    database = database
+    metadata = metadata
+
+
+class NickNames(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "nicks"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100, nullable=False, name="hq_name")
+    is_lame: bool = ormar.Boolean(nullable=True)
+
+
+class NicksHq(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "nicks_x_hq"
+
+
+class HQ(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100, nullable=False, name="hq_name")
+    nicks: List[NickNames] = ormar.ManyToMany(NickNames, through=NicksHq)
+
+
+class Company(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "companies"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100, nullable=False, name="company_name")
+    founded: int = ormar.Integer(nullable=True)
+    hq: HQ = ormar.ForeignKey(HQ)
+
+
+class Car(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    manufacturer: Optional[Company] = ormar.ForeignKey(Company)
+    name: str = ormar.String(max_length=100)
+    year: int = ormar.Integer(nullable=True)
+    gearbox_type: str = ormar.String(max_length=20, nullable=True)
+    gears: int = ormar.Integer(nullable=True)
+    aircon_type: str = ormar.String(max_length=20, nullable=True)
+
+
+def compare_results(excludable):
+    car_excludable = excludable.get(Car)
+    assert car_excludable.exclude == {"year", "gearbox_type", "gears", "aircon_type"}
+    assert car_excludable.include == set()
+
+    assert car_excludable.is_excluded("year")
+
+    alias = Company.Meta.alias_manager.resolve_relation_alias(Car, "manufacturer")
+    manu_excludable = excludable.get(Company, alias=alias)
+    assert manu_excludable.exclude == {"founded"}
+    assert manu_excludable.include == set()
+
+    assert manu_excludable.is_excluded("founded")
+
+
+def compare_results_include(excludable):
+    manager = Company.Meta.alias_manager
+    car_excludable = excludable.get(Car)
+    assert car_excludable.include == {"id", "name"}
+    assert car_excludable.exclude == set()
+
+    assert car_excludable.is_included("name")
+    assert not car_excludable.is_included("gears")
+
+    alias = manager.resolve_relation_alias(Car, "manufacturer")
+    manu_excludable = excludable.get(Company, alias=alias)
+    assert manu_excludable.include == {"name"}
+    assert manu_excludable.exclude == set()
+
+    assert manu_excludable.is_included("name")
+    assert not manu_excludable.is_included("founded")
+
+    alias = manager.resolve_relation_alias(Company, "hq")
+    hq_excludable = excludable.get(HQ, alias=alias)
+    assert hq_excludable.include == {"name"}
+    assert hq_excludable.exclude == set()
+
+    alias = manager.resolve_relation_alias(NicksHq, "nicknames")
+    nick_excludable = excludable.get(NickNames, alias=alias)
+    assert nick_excludable.include == {"name"}
+    assert nick_excludable.exclude == set()
+
+
+def test_excluding_fields_from_list():
+    fields = [
+        "gearbox_type",
+        "gears",
+        "aircon_type",
+        "year",
+        "manufacturer__founded",
+    ]
+    excludable = ExcludableItems()
+    excludable.build(items=fields, model_cls=Car, is_exclude=True)
+    compare_results(excludable)
+
+
+def test_excluding_fields_from_dict():
+    fields = {
+        "gearbox_type": ...,
+        "gears": ...,
+        "aircon_type": ...,
+        "year": ...,
+        "manufacturer": {"founded": ...},
+    }
+    excludable = ExcludableItems()
+    excludable.build(items=fields, model_cls=Car, is_exclude=True)
+    compare_results(excludable)
+
+
+def test_excluding_fields_from_dict_with_set():
+    fields = {
+        "gearbox_type": ...,
+        "gears": ...,
+        "aircon_type": ...,
+        "year": ...,
+        "manufacturer": {"founded"},
+    }
+    excludable = ExcludableItems()
+    excludable.build(items=fields, model_cls=Car, is_exclude=True)
+    compare_results(excludable)
+
+
+def test_gradual_build_from_lists():
+    fields_col = [
+        "year",
+        ["gearbox_type", "gears"],
+        "aircon_type",
+        ["manufacturer__founded"],
+    ]
+    excludable = ExcludableItems()
+    for fields in fields_col:
+        excludable.build(items=fields, model_cls=Car, is_exclude=True)
+    compare_results(excludable)
+
+
+def test_nested_includes():
+    fields = [
+        "id",
+        "name",
+        "manufacturer__name",
+        "manufacturer__hq__name",
+        "manufacturer__hq__nicks__name",
+    ]
+    excludable = ExcludableItems()
+    excludable.build(items=fields, model_cls=Car, is_exclude=False)
+    compare_results_include(excludable)
+
+
+def test_nested_includes_from_dict():
+    fields = {
+        "id": ...,
+        "name": ...,
+        "manufacturer": {"name": ..., "hq": {"name": ..., "nicks": {"name": ...}},},
+    }
+    excludable = ExcludableItems()
+    excludable.build(items=fields, model_cls=Car, is_exclude=False)
+    compare_results_include(excludable)
+
+
+def test_nested_includes_from_dict_with_set():
+    fields = {
+        "id": ...,
+        "name": ...,
+        "manufacturer": {"name": ..., "hq": {"name": ..., "nicks": {"name"}},},
+    }
+    excludable = ExcludableItems()
+    excludable.build(items=fields, model_cls=Car, is_exclude=False)
+    compare_results_include(excludable)
+
+
+def test_includes_and_excludes_combo():
+    fields_inc1 = ["id", "name", "year", "gearbox_type", "gears"]
+    fields_inc2 = {"manufacturer": {"name"}}
+    fields_exc1 = {"manufacturer__founded"}
+    fields_exc2 = "aircon_type"
+    excludable = ExcludableItems()
+    excludable.build(items=fields_inc1, model_cls=Car, is_exclude=False)
+    excludable.build(items=fields_inc2, model_cls=Car, is_exclude=False)
+    excludable.build(items=fields_exc1, model_cls=Car, is_exclude=True)
+    excludable.build(items=fields_exc2, model_cls=Car, is_exclude=True)
+
+    car_excludable = excludable.get(Car)
+    assert car_excludable.include == {"id", "name", "year", "gearbox_type", "gears"}
+    assert car_excludable.exclude == {"aircon_type"}
+
+    assert car_excludable.is_excluded("aircon_type")
+    assert car_excludable.is_included("name")
+
+    alias = Company.Meta.alias_manager.resolve_relation_alias(Car, "manufacturer")
+    manu_excludable = excludable.get(Company, alias=alias)
+    assert manu_excludable.include == {"name"}
+    assert manu_excludable.exclude == {"founded"}
+
+    assert manu_excludable.is_excluded("founded")

tests/test_excluding_fields_in_fastapi.py

@@ -135,26 +135,22 @@ async def create_user3(user: User2):
 
 @app.post("/users4/")
 async def create_user4(user: User2):
-    user = await user.save()
-    return user.dict(exclude={"password"})
+    return (await user.save()).dict(exclude={"password"})
 
 
 @app.post("/random/", response_model=RandomModel)
 async def create_user5(user: RandomModel):
-    user = await user.save()
-    return user
+    return await user.save()
 
 
 @app.post("/random2/", response_model=RandomModel)
 async def create_user6(user: RandomModel):
-    user = await user.save()
-    return user.dict()
+    return await user.save()
 
 
 @app.post("/random3/", response_model=RandomModel, response_model_exclude={"full_name"})
 async def create_user7(user: RandomModel):
-    user = await user.save()
-    return user.dict()
+    return await user.save()
 
 
 def test_excluding_fields_in_endpoints():

tests/test_fastapi_docs.py

@@ -42,18 +42,13 @@ class Category(ormar.Model):
     name: str = ormar.String(max_length=100)
 
 
-class ItemsXCategories(ormar.Model):
-    class Meta(LocalMeta):
-        tablename = "items_x_categories"
-
-
 class Item(ormar.Model):
     class Meta(LocalMeta):
         pass
 
     id: int = ormar.Integer(primary_key=True)
     name: str = ormar.String(max_length=100)
-    categories = ormar.ManyToMany(Category, through=ItemsXCategories)
+    categories = ormar.ManyToMany(Category)
 
 
 @pytest.fixture(autouse=True, scope="module")

tests/test_inheritance_concrete.py

@@ -121,11 +121,11 @@ class Bus(Car):
     max_persons: int = ormar.Integer()
 
 
-class PersonsCar(ormar.Model):
-    class Meta:
-        tablename = "cars_x_persons"
-        metadata = metadata
-        database = db
+# class PersonsCar(ormar.Model):
+#     class Meta:
+#         tablename = "cars_x_persons"
+#         metadata = metadata
+#         database = db
 
 
 class Car2(ormar.Model):
@@ -138,7 +138,9 @@ class Car2(ormar.Model):
     name: str = ormar.String(max_length=50)
     owner: Person = ormar.ForeignKey(Person, related_name="owned")
     co_owners: List[Person] = ormar.ManyToMany(
-        Person, through=PersonsCar, related_name="coowned"
+        Person,
+        # through=PersonsCar,
+        related_name="coowned",
     )
     created_date: datetime.datetime = ormar.DateTime(default=datetime.datetime.now)
 

tests/test_load_all.py

@@ -0,0 +1,171 @@
+from typing import List
+
+import databases
+import pytest
+import sqlalchemy
+
+import ormar
+from tests.settings import DATABASE_URL
+
+database = databases.Database(DATABASE_URL, force_rollback=True)
+metadata = sqlalchemy.MetaData()
+
+
+class BaseMeta(ormar.ModelMeta):
+    database = database
+    metadata = metadata
+
+
+class Language(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "languages"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    level: str = ormar.String(max_length=150, default="Beginner")
+
+
+class CringeLevel(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "levels"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    language = ormar.ForeignKey(Language)
+
+
+class NickName(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "nicks"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100, nullable=False, name="hq_name")
+    is_lame: bool = ormar.Boolean(nullable=True)
+    level: CringeLevel = ormar.ForeignKey(CringeLevel)
+
+
+class HQ(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "hqs"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100, nullable=False, name="hq_name")
+    nicks: List[NickName] = ormar.ManyToMany(NickName)
+
+
+class Company(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "companies"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100, nullable=False, name="company_name")
+    founded: int = ormar.Integer(nullable=True)
+    hq: HQ = ormar.ForeignKey(HQ, related_name="companies")
+
+
+@pytest.fixture(autouse=True, scope="module")
+def create_test_database():
+    engine = sqlalchemy.create_engine(DATABASE_URL)
+    metadata.drop_all(engine)
+    metadata.create_all(engine)
+    yield
+    metadata.drop_all(engine)
+
+
+@pytest.mark.asyncio
+async def test_load_all_fk_rel():
+    async with database:
+        async with database.transaction(force_rollback=True):
+            hq = await HQ.objects.create(name="Main")
+            company = await Company.objects.create(name="Banzai", founded=1988, hq=hq)
+
+            hq = await HQ.objects.get(name="Main")
+            await hq.load_all()
+
+            assert hq.companies[0] == company
+            assert hq.companies[0].name == "Banzai"
+            assert hq.companies[0].founded == 1988
+
+
+@pytest.mark.asyncio
+async def test_load_all_many_to_many():
+    async with database:
+        async with database.transaction(force_rollback=True):
+            nick1 = await NickName.objects.create(name="BazingaO", is_lame=False)
+            nick2 = await NickName.objects.create(name="Bazinga20", is_lame=True)
+            hq = await HQ.objects.create(name="Main")
+            await hq.nicks.add(nick1)
+            await hq.nicks.add(nick2)
+
+            hq = await HQ.objects.get(name="Main")
+            await hq.load_all()
+
+            assert hq.nicks[0] == nick1
+            assert hq.nicks[0].name == "BazingaO"
+
+            assert hq.nicks[1] == nick2
+            assert hq.nicks[1].name == "Bazinga20"
+
+
+@pytest.mark.asyncio
+async def test_loading_reversed_relation():
+    async with database:
+        async with database.transaction(force_rollback=True):
+            hq = await HQ.objects.create(name="Main")
+            await Company.objects.create(name="Banzai", founded=1988, hq=hq)
+
+            company = await Company.objects.get(name="Banzai")
+            await company.load_all()
+
+            assert company.hq == hq
+
+
+@pytest.mark.asyncio
+async def test_loading_nested():
+    async with database:
+        async with database.transaction(force_rollback=True):
+            language = await Language.objects.create(name="English")
+            level = await CringeLevel.objects.create(name="High", language=language)
+            level2 = await CringeLevel.objects.create(name="Low", language=language)
+            nick1 = await NickName.objects.create(
+                name="BazingaO", is_lame=False, level=level
+            )
+            nick2 = await NickName.objects.create(
+                name="Bazinga20", is_lame=True, level=level2
+            )
+            hq = await HQ.objects.create(name="Main")
+            await hq.nicks.add(nick1)
+            await hq.nicks.add(nick2)
+
+            hq = await HQ.objects.get(name="Main")
+            await hq.load_all(follow=True)
+
+            assert hq.nicks[0] == nick1
+            assert hq.nicks[0].name == "BazingaO"
+            assert hq.nicks[0].level.name == "High"
+            assert hq.nicks[0].level.language.name == "English"
+
+            assert hq.nicks[1] == nick2
+            assert hq.nicks[1].name == "Bazinga20"
+            assert hq.nicks[1].level.name == "Low"
+            assert hq.nicks[1].level.language.name == "English"
+
+            await hq.load_all(follow=True, exclude="nicks__level__language")
+            assert len(hq.nicks) == 2
+            assert hq.nicks[0].level.language is None
+            assert hq.nicks[1].level.language is None
+
+            await hq.load_all(follow=True, exclude="nicks__level__language__level")
+            assert len(hq.nicks) == 2
+            assert hq.nicks[0].level.language is not None
+            assert hq.nicks[0].level.language.level is None
+            assert hq.nicks[1].level.language is not None
+            assert hq.nicks[1].level.language.level is None
+
+            await hq.load_all(follow=True, exclude="nicks__level")
+            assert len(hq.nicks) == 2
+            assert hq.nicks[0].level is None
+            assert hq.nicks[1].level is None
+
+            await hq.load_all(follow=True, exclude="nicks")
+            assert len(hq.nicks) == 0

tests/test_m2m_through_fields.py

@@ -1,6 +1,9 @@
+from typing import Any, Sequence, cast
+
 import databases
 import pytest
 import sqlalchemy
+from pydantic.typing import ForwardRef
 
 import ormar
 from tests.settings import DATABASE_URL
@@ -18,8 +21,8 @@ class Category(ormar.Model):
     class Meta(BaseMeta):
         tablename = "categories"
 
-    id: int = ormar.Integer(primary_key=True)
-    name: str = ormar.String(max_length=40)
+    id = ormar.Integer(primary_key=True)
+    name = ormar.String(max_length=40)
 
 
 class PostCategory(ormar.Model):
@@ -28,6 +31,15 @@ class PostCategory(ormar.Model):
 
     id: int = ormar.Integer(primary_key=True)
     sort_order: int = ormar.Integer(nullable=True)
+    param_name: str = ormar.String(default="Name", max_length=200)
+
+
+class Blog(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    title: str = ormar.String(max_length=200)
 
 
 class Post(ormar.Model):
@@ -37,30 +49,329 @@ class Post(ormar.Model):
     id: int = ormar.Integer(primary_key=True)
     title: str = ormar.String(max_length=200)
     categories = ormar.ManyToMany(Category, through=PostCategory)
+    blog = ormar.ForeignKey(Blog)
 
 
-#
-# @pytest.fixture(autouse=True, scope="module")
-# async def create_test_database():
-#     engine = sqlalchemy.create_engine(DATABASE_URL)
-#     metadata.create_all(engine)
-#     yield
-#     metadata.drop_all(engine)
-#
-#
-# @pytest.mark.asyncio
-# async def test_setting_fields_on_through_model():
-#     async with database:
-#         # TODO: check/ modify following
-#         # loading the data into model instance of though model?
-#         # <- attach to other side? both sides? access by through, or add to fields?
-#         # creating while adding to relation (kwargs in add?)
-#         # creating in query (dividing kwargs between final and through)
-#         # updating in query
-#         # sorting in filter (special __through__<field_name> notation?)
-#         # ordering by in order_by
-#         # accessing from instance (both sides?)
-#         # modifying from instance (both sides?)
-#         # including/excluding in fields?
-#         # allowing to change fk fields names in through model?
-#         pass
+@pytest.fixture(autouse=True, scope="module")
+def create_test_database():
+    engine = sqlalchemy.create_engine(DATABASE_URL)
+    metadata.drop_all(engine)
+    metadata.create_all(engine)
+    yield
+    metadata.drop_all(engine)
+
+
+class PostCategory2(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "posts_x_categories2"
+
+    id: int = ormar.Integer(primary_key=True)
+    sort_order: int = ormar.Integer(nullable=True)
+
+
+class Post2(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    title: str = ormar.String(max_length=200)
+    categories = ormar.ManyToMany(Category, through=ForwardRef("PostCategory2"))
+
+
+@pytest.mark.asyncio
+async def test_forward_ref_is_updated():
+    async with database:
+        assert Post2.Meta.requires_ref_update
+        Post2.update_forward_refs()
+
+        assert Post2.Meta.model_fields["postcategory2"].to == PostCategory2
+
+
+@pytest.mark.asyncio
+async def test_setting_fields_on_through_model():
+    async with database:
+        post = await Post(title="Test post").save()
+        category = await Category(name="Test category").save()
+        await post.categories.add(category)
+
+        assert hasattr(post.categories[0], "postcategory")
+        assert post.categories[0].postcategory is None
+
+
+@pytest.mark.asyncio
+async def test_setting_additional_fields_on_through_model_in_add():
+    async with database:
+        post = await Post(title="Test post").save()
+        category = await Category(name="Test category").save()
+        await post.categories.add(category, sort_order=1)
+        postcat = await PostCategory.objects.get()
+        assert postcat.sort_order == 1
+
+
+@pytest.mark.asyncio
+async def test_setting_additional_fields_on_through_model_in_create():
+    async with database:
+        post = await Post(title="Test post").save()
+        await post.categories.create(
+            name="Test category2", postcategory={"sort_order": 2}
+        )
+        postcat = await PostCategory.objects.get()
+        assert postcat.sort_order == 2
+
+
+@pytest.mark.asyncio
+async def test_getting_additional_fields_from_queryset() -> Any:
+    async with database:
+        post = await Post(title="Test post").save()
+        await post.categories.create(
+            name="Test category1", postcategory={"sort_order": 1}
+        )
+        await post.categories.create(
+            name="Test category2", postcategory={"sort_order": 2}
+        )
+
+        await post.categories.all()
+        assert post.postcategory is None
+        assert post.categories[0].postcategory.sort_order == 1
+        assert post.categories[1].postcategory.sort_order == 2
+
+        post2 = await Post.objects.select_related("categories").get(
+            categories__name="Test category2"
+        )
+        assert post2.categories[0].postcategory.sort_order == 2
+
+
+@pytest.mark.asyncio
+async def test_only_one_side_has_through() -> Any:
+    async with database:
+        post = await Post(title="Test post").save()
+        await post.categories.create(
+            name="Test category1", postcategory={"sort_order": 1}
+        )
+        await post.categories.create(
+            name="Test category2", postcategory={"sort_order": 2}
+        )
+
+        post2 = await Post.objects.select_related("categories").get()
+        assert post2.postcategory is None
+        assert post2.categories[0].postcategory is not None
+
+        await post2.categories.all()
+        assert post2.postcategory is None
+        assert post2.categories[0].postcategory is not None
+
+        categories = await Category.objects.select_related("posts").all()
+        categories = cast(Sequence[Category], categories)
+        assert categories[0].postcategory is None
+        assert categories[0].posts[0].postcategory is not None
+
+
+@pytest.mark.asyncio
+async def test_filtering_by_through_model() -> Any:
+    async with database:
+        post = await Post(title="Test post").save()
+        await post.categories.create(
+            name="Test category1",
+            postcategory={"sort_order": 1, "param_name": "volume"},
+        )
+        await post.categories.create(
+            name="Test category2", postcategory={"sort_order": 2, "param_name": "area"}
+        )
+
+        post2 = (
+            await Post.objects.select_related("categories")
+            .filter(postcategory__sort_order__gt=1)
+            .get()
+        )
+        assert len(post2.categories) == 1
+        assert post2.categories[0].postcategory.sort_order == 2
+
+        post3 = await Post.objects.filter(
+            categories__postcategory__param_name="volume"
+        ).get()
+        assert len(post3.categories) == 1
+        assert post3.categories[0].postcategory.param_name == "volume"
+
+
+@pytest.mark.asyncio
+async def test_deep_filtering_by_through_model() -> Any:
+    async with database:
+        blog = await Blog(title="My Blog").save()
+        post = await Post(title="Test post", blog=blog).save()
+
+        await post.categories.create(
+            name="Test category1",
+            postcategory={"sort_order": 1, "param_name": "volume"},
+        )
+        await post.categories.create(
+            name="Test category2", postcategory={"sort_order": 2, "param_name": "area"}
+        )
+
+        blog2 = (
+            await Blog.objects.select_related("posts__categories")
+            .filter(posts__postcategory__sort_order__gt=1)
+            .get()
+        )
+        assert len(blog2.posts) == 1
+        assert len(blog2.posts[0].categories) == 1
+        assert blog2.posts[0].categories[0].postcategory.sort_order == 2
+
+        blog3 = await Blog.objects.filter(
+            posts__categories__postcategory__param_name="volume"
+        ).get()
+        assert len(blog3.posts) == 1
+        assert len(blog3.posts[0].categories) == 1
+        assert blog3.posts[0].categories[0].postcategory.param_name == "volume"
+
+
+@pytest.mark.asyncio
+async def test_ordering_by_through_model() -> Any:
+    async with database:
+        post = await Post(title="Test post").save()
+        await post.categories.create(
+            name="Test category1",
+            postcategory={"sort_order": 2, "param_name": "volume"},
+        )
+        await post.categories.create(
+            name="Test category2", postcategory={"sort_order": 1, "param_name": "area"}
+        )
+        await post.categories.create(
+            name="Test category3",
+            postcategory={"sort_order": 3, "param_name": "velocity"},
+        )
+
+        post2 = (
+            await Post.objects.select_related("categories")
+            .order_by("-postcategory__sort_order")
+            .get()
+        )
+        assert len(post2.categories) == 3
+        assert post2.categories[0].name == "Test category3"
+        assert post2.categories[2].name == "Test category2"
+
+        post3 = (
+            await Post.objects.select_related("categories")
+            .order_by("categories__postcategory__param_name")
+            .get()
+        )
+        assert len(post3.categories) == 3
+        assert post3.categories[0].postcategory.param_name == "area"
+        assert post3.categories[2].postcategory.param_name == "volume"
+
+
+@pytest.mark.asyncio
+async def test_update_through_models_from_queryset_on_through() -> Any:
+    async with database:
+        post = await Post(title="Test post").save()
+        await post.categories.create(
+            name="Test category1",
+            postcategory={"sort_order": 2, "param_name": "volume"},
+        )
+        await post.categories.create(
+            name="Test category2", postcategory={"sort_order": 1, "param_name": "area"}
+        )
+        await post.categories.create(
+            name="Test category3",
+            postcategory={"sort_order": 3, "param_name": "velocity"},
+        )
+
+        await PostCategory.objects.filter(param_name="volume", post=post.id).update(
+            sort_order=4
+        )
+        post2 = (
+            await Post.objects.select_related("categories")
+            .order_by("-postcategory__sort_order")
+            .get()
+        )
+        assert len(post2.categories) == 3
+        assert post2.categories[0].postcategory.param_name == "volume"
+        assert post2.categories[2].postcategory.param_name == "area"
+
+
+@pytest.mark.asyncio
+async def test_update_through_model_after_load() -> Any:
+    async with database:
+        post = await Post(title="Test post").save()
+        await post.categories.create(
+            name="Test category1",
+            postcategory={"sort_order": 2, "param_name": "volume"},
+        )
+        post2 = await Post.objects.select_related("categories").get()
+        assert len(post2.categories) == 1
+
+        await post2.categories[0].postcategory.load()
+        await post2.categories[0].postcategory.update(sort_order=3)
+
+        post3 = await Post.objects.select_related("categories").get()
+        assert len(post3.categories) == 1
+        assert post3.categories[0].postcategory.sort_order == 3
+
+
+@pytest.mark.asyncio
+async def test_update_through_from_related() -> Any:
+    async with database:
+        post = await Post(title="Test post").save()
+        await post.categories.create(
+            name="Test category1",
+            postcategory={"sort_order": 2, "param_name": "volume"},
+        )
+        await post.categories.create(
+            name="Test category2", postcategory={"sort_order": 1, "param_name": "area"}
+        )
+        await post.categories.create(
+            name="Test category3",
+            postcategory={"sort_order": 3, "param_name": "velocity"},
+        )
+
+        await post.categories.filter(name="Test category3").update(
+            postcategory={"sort_order": 4}
+        )
+
+        post2 = (
+            await Post.objects.select_related("categories")
+            .order_by("postcategory__sort_order")
+            .get()
+        )
+        assert len(post2.categories) == 3
+        assert post2.categories[2].postcategory.sort_order == 4
+
+
+@pytest.mark.asyncio
+async def test_excluding_fields_on_through_model() -> Any:
+    async with database:
+        post = await Post(title="Test post").save()
+        await post.categories.create(
+            name="Test category1",
+            postcategory={"sort_order": 2, "param_name": "volume"},
+        )
+        await post.categories.create(
+            name="Test category2", postcategory={"sort_order": 1, "param_name": "area"}
+        )
+        await post.categories.create(
+            name="Test category3",
+            postcategory={"sort_order": 3, "param_name": "velocity"},
+        )
+
+        post2 = (
+            await Post.objects.select_related("categories")
+            .exclude_fields("postcategory__param_name")
+            .order_by("postcategory__sort_order")
+            .get()
+        )
+        assert len(post2.categories) == 3
+        assert post2.categories[0].postcategory.param_name is None
+        assert post2.categories[0].postcategory.sort_order == 1
+
+        assert post2.categories[2].postcategory.param_name is None
+        assert post2.categories[2].postcategory.sort_order == 3
+
+        post3 = (
+            await Post.objects.select_related("categories")
+            .fields({"postcategory": ..., "title": ...})
+            .exclude_fields({"postcategory": {"param_name", "sort_order"}})
+            .get()
+        )
+        assert len(post3.categories) == 3
+        for category in post3.categories:
+            assert category.postcategory.param_name is None
+            assert category.postcategory.sort_order is None

tests/test_many_to_many.py

@@ -1,5 +1,5 @@
 import asyncio
-from typing import List, Union, Optional
+from typing import List, Optional
 
 import databases
 import pytest
@@ -34,13 +34,6 @@ class Category(ormar.Model):
     name: str = ormar.String(max_length=40)
 
 
-class PostCategory(ormar.Model):
-    class Meta:
-        tablename = "posts_categories"
-        database = database
-        metadata = metadata
-
-
 class Post(ormar.Model):
     class Meta:
         tablename = "posts"
@@ -49,9 +42,7 @@ class Post(ormar.Model):
 
     id: int = ormar.Integer(primary_key=True)
     title: str = ormar.String(max_length=200)
-    categories: Optional[Union[Category, List[Category]]] = ormar.ManyToMany(
-        Category, through=PostCategory
-    )
+    categories: Optional[List[Category]] = ormar.ManyToMany(Category)
     author: Optional[Author] = ormar.ForeignKey(Author)
 
 
@@ -74,6 +65,7 @@ async def create_test_database():
 async def cleanup():
     yield
     async with database:
+        PostCategory = Post.Meta.model_fields["categories"].through
         await PostCategory.objects.delete(each=True)
         await Post.objects.delete(each=True)
         await Category.objects.delete(each=True)

tests/test_more_same_table_joins.py

@@ -108,3 +108,17 @@ async def test_model_multiple_instances_of_same_table_in_schema():
         assert len(classes[0].dict().get("students")) == 2
         assert classes[0].teachers[0].category.department.name == "Law Department"
         assert classes[0].students[0].category.department.name == "Math Department"
+
+
+@pytest.mark.asyncio
+async def test_load_all_multiple_instances_of_same_table_in_schema():
+    async with database:
+        await create_data()
+        math_class = await SchoolClass.objects.get(name="Math")
+        assert math_class.name == "Math"
+
+        await math_class.load_all(follow=True)
+        assert math_class.students[0].name == "Jane"
+        assert len(math_class.dict().get("students")) == 2
+        assert math_class.teachers[0].category.department.name == "Law Department"
+        assert math_class.students[0].category.department.name == "Math Department"

tests/test_order_by.py

@@ -85,13 +85,6 @@ class Car(ormar.Model):
     factory: Optional[Factory] = ormar.ForeignKey(Factory)
 
 
-class UsersCar(ormar.Model):
-    class Meta:
-        tablename = "cars_x_users"
-        metadata = metadata
-        database = database
-
-
 class User(ormar.Model):
     class Meta:
         tablename = "users"
@@ -100,7 +93,7 @@ class User(ormar.Model):
 
     id: int = ormar.Integer(primary_key=True)
     name: str = ormar.String(max_length=100)
-    cars: List[Car] = ormar.ManyToMany(Car, through=UsersCar)
+    cars: List[Car] = ormar.ManyToMany(Car)
 
 
 @pytest.fixture(autouse=True, scope="module")

tests/test_queryproxy_on_m2m_models.py

@@ -6,6 +6,7 @@ import pytest
 import sqlalchemy
 
 import ormar
+from ormar.exceptions import QueryDefinitionError
 from tests.settings import DATABASE_URL
 
 database = databases.Database(DATABASE_URL, force_rollback=True)
@@ -180,3 +181,42 @@ async def test_queryset_methods():
             assert len(categories) == 3 == len(post.categories)
             for cat in post.categories:
                 assert cat.subject.name is not None
+
+
+@pytest.mark.asyncio
+async def test_queryset_update():
+    async with database:
+        async with database.transaction(force_rollback=True):
+            guido = await Author.objects.create(
+                first_name="Guido", last_name="Van Rossum"
+            )
+            subject = await Subject(name="Random").save()
+            post = await Post.objects.create(title="Hello, M2M", author=guido)
+            await post.categories.create(name="News", sort_order=1, subject=subject)
+            await post.categories.create(name="Breaking", sort_order=3, subject=subject)
+
+            await post.categories.order_by("sort_order").all()
+            assert len(post.categories) == 2
+            assert post.categories[0].sort_order == 1
+            assert post.categories[0].name == "News"
+            assert post.categories[1].sort_order == 3
+            assert post.categories[1].name == "Breaking"
+
+            updated = await post.categories.update(each=True, name="Test")
+            assert updated == 2
+
+            await post.categories.order_by("sort_order").all()
+            assert len(post.categories) == 2
+            assert post.categories[0].name == "Test"
+            assert post.categories[1].name == "Test"
+
+            updated = await post.categories.filter(sort_order=3).update(name="Test 2")
+            assert updated == 1
+
+            await post.categories.order_by("sort_order").all()
+            assert len(post.categories) == 2
+            assert post.categories[0].name == "Test"
+            assert post.categories[1].name == "Test 2"
+
+            with pytest.raises(QueryDefinitionError):
+                await post.categories.update(name="Test WRONG")

tests/test_queryset_utils.py

@@ -8,11 +8,6 @@ from ormar.queryset.utils import translate_list_to_dict, update_dict_from_list,
 from tests.settings import DATABASE_URL
 
 
-def test_empty_excludable():
-    assert ExcludableMixin.is_included(None, "key")  # all fields included if empty
-    assert not ExcludableMixin.is_excluded(None, "key")  # none field excluded if empty
-
-
 def test_list_to_dict_translation():
     tet_list = ["aa", "bb", "cc__aa", "cc__bb", "cc__aa__xx", "cc__aa__yy"]
     test = translate_list_to_dict(tet_list)

tests/test_selecting_subset_of_columns.py

@@ -204,8 +204,8 @@ async def test_selecting_subset():
             all_cars_dummy = (
                 await Car.objects.select_related("manufacturer")
                 .fields(["id", "name", "year", "gearbox_type", "gears", "aircon_type"])
-                .fields({"manufacturer": ...})
-                .exclude_fields({"manufacturer": ...})
+                # .fields({"manufacturer": ...})
+                # .exclude_fields({"manufacturer": ...})
                 .fields({"manufacturer": {"name"}})
                 .exclude_fields({"manufacturer__founded"})
                 .all()

tests/test_through_relations_fail.py

@@ -0,0 +1,51 @@
+# type: ignore
+
+import databases
+import pytest
+import sqlalchemy
+
+import ormar
+from ormar import ModelDefinitionError
+from tests.settings import DATABASE_URL
+
+database = databases.Database(DATABASE_URL, force_rollback=True)
+metadata = sqlalchemy.MetaData()
+
+
+def test_through_with_relation_fails():
+    class BaseMeta(ormar.ModelMeta):
+        database = database
+        metadata = metadata
+
+    class Category(ormar.Model):
+        class Meta(BaseMeta):
+            tablename = "categories"
+
+        id = ormar.Integer(primary_key=True)
+        name = ormar.String(max_length=40)
+
+    class Blog(ormar.Model):
+        class Meta(BaseMeta):
+            pass
+
+        id: int = ormar.Integer(primary_key=True)
+        title: str = ormar.String(max_length=200)
+
+    class PostCategory(ormar.Model):
+        class Meta(BaseMeta):
+            tablename = "posts_x_categories"
+
+        id: int = ormar.Integer(primary_key=True)
+        sort_order: int = ormar.Integer(nullable=True)
+        param_name: str = ormar.String(default="Name", max_length=200)
+        blog = ormar.ForeignKey(Blog)
+
+    with pytest.raises(ModelDefinitionError):
+
+        class Post(ormar.Model):
+            class Meta(BaseMeta):
+                pass
+
+            id: int = ormar.Integer(primary_key=True)
+            title: str = ormar.String(max_length=200)
+            categories = ormar.ManyToMany(Category, through=PostCategory)

tests/test_wekref_exclusion.py

@@ -0,0 +1,147 @@
+from typing import List, Optional
+from uuid import UUID, uuid4
+
+import databases
+import pydantic
+import pytest
+import sqlalchemy
+from fastapi import FastAPI
+from starlette.testclient import TestClient
+
+import ormar
+from tests.settings import DATABASE_URL
+
+app = FastAPI()
+
+database = databases.Database(DATABASE_URL, force_rollback=True)
+metadata = sqlalchemy.MetaData()
+
+app.state.database = database
+
+
+@app.on_event("startup")
+async def startup() -> None:
+    database_ = app.state.database
+    if not database_.is_connected:
+        await database_.connect()
+
+
+@app.on_event("shutdown")
+async def shutdown() -> None:
+    database_ = app.state.database
+    if database_.is_connected:
+        await database_.disconnect()
+
+
+@pytest.fixture(autouse=True, scope="module")
+def create_test_database():
+    engine = sqlalchemy.create_engine(DATABASE_URL)
+    metadata.create_all(engine)
+    yield
+    metadata.drop_all(engine)
+
+
+class BaseMeta(ormar.ModelMeta):
+    database = database
+    metadata = metadata
+
+
+class OtherThing(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "other_things"
+
+    id: UUID = ormar.UUID(primary_key=True, default=uuid4)
+    name: str = ormar.Text(default="")
+    ot_contents: str = ormar.Text(default="")
+
+
+class Thing(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "things"
+
+    id: UUID = ormar.UUID(primary_key=True, default=uuid4)
+    name: str = ormar.Text(default="")
+    js: pydantic.Json = ormar.JSON(nullable=True)
+    other_thing: Optional[OtherThing] = ormar.ForeignKey(OtherThing, nullable=True)
+
+
+@app.post("/test/1")
+async def post_test_1():
+    # don't split initialization and attribute assignment
+    ot = await OtherThing(ot_contents="otc").save()
+    await Thing(other_thing=ot, name="t1").save()
+    await Thing(other_thing=ot, name="t2").save()
+    await Thing(other_thing=ot, name="t3").save()
+
+    # if you do not care about returned object you can even go with bulk_create
+    # all of them are created in one transaction
+    # things = [Thing(other_thing=ot, name='t1'),
+    #           Thing(other_thing=ot, name="t2"),
+    #           Thing(other_thing=ot, name="t3")]
+    # await Thing.objects.bulk_create(things)
+
+
+@app.get("/test/2", response_model=List[Thing])
+async def get_test_2():
+    # if you only query for one use get or first
+    ot = await OtherThing.objects.get()
+    ts = await ot.things.all()
+    # specifically null out the relation on things before return
+    for t in ts:
+        t.remove(ot, name="other_thing")
+    return ts
+
+
+@app.get("/test/3", response_model=List[Thing])
+async def get_test_3():
+    ot = await OtherThing.objects.select_related("things").get()
+    # exclude unwanted field while ot is still in scope
+    # in order not to pass it to fastapi
+    return [t.dict(exclude={"other_thing"}) for t in ot.things]
+
+
+@app.get("/test/4", response_model=List[Thing], response_model_exclude={"other_thing"})
+async def get_test_4():
+    ot = await OtherThing.objects.get()
+    # query from the active side
+    return await Thing.objects.all(other_thing=ot)
+
+
+@app.get("/get_ot/", response_model=OtherThing)
+async def get_ot():
+    return await OtherThing.objects.get()
+
+
+# more real life (usually) is not getting some random OT and get it's Things
+# but query for a specific one by some kind of id
+@app.get(
+    "/test/5/{thing_id}",
+    response_model=List[Thing],
+    response_model_exclude={"other_thing"},
+)
+async def get_test_5(thing_id: UUID):
+    return await Thing.objects.all(other_thing__id=thing_id)
+
+
+def test_endpoints():
+    client = TestClient(app)
+    with client:
+        resp = client.post("/test/1")
+        assert resp.status_code == 200
+
+        resp2 = client.get("/test/2")
+        assert resp2.status_code == 200
+        assert len(resp2.json()) == 3
+
+        resp3 = client.get("/test/3")
+        assert resp3.status_code == 200
+        assert len(resp3.json()) == 3
+
+        resp4 = client.get("/test/4")
+        assert resp4.status_code == 200
+        assert len(resp4.json()) == 3
+
+        ot = OtherThing(**client.get("/get_ot/").json())
+        resp5 = client.get(f"/test/5/{ot.id}")
+        assert resp5.status_code == 200
+        assert len(resp5.json()) == 3