Merge pull request #87 from collerek/forwardref

Processing of Forwardrefs

.coveragerc

@@ -0,0 +1,7 @@
+[run]
+source = ormar, tests
+omit = ./tests/test.db, *py.typed*
+data_file = .coverage
+
+[report]
+omit = ./tests/test.db, *py.typed*

.github/workflows/test-package.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.6, 3.7, 3.8, 3.9]
       fail-fast: false
     
     services:

docs/api/fields/base-field.md

@@ -217,7 +217,7 @@ primary_key, index, unique, nullable, default and server_default.
 
 ```python
  | @classmethod
- | expand_relationship(cls, value: Any, child: Union["Model", "NewBaseModel"], to_register: bool = True, relation_name: str = None) -> Any
+ | expand_relationship(cls, value: Any, child: Union["Model", "NewBaseModel"], to_register: bool = True) -> Any
 ```
 
 Function overwritten for relations, in basic field the value is returned as is.
@@ -236,3 +236,66 @@ dict (from Model) or actual instance/list of a "Model".
 
 `(Any)`: returns untouched value for normal fields, expands only for relations
 
+<a name="fields.base.BaseField.set_self_reference_flag"></a>
+#### set\_self\_reference\_flag
+
+```python
+ | @classmethod
+ | set_self_reference_flag(cls) -> None
+```
+
+Sets `self_reference` to True if field to and owner are same model.
+
+**Returns**:
+
+`(None)`: None
+
+<a name="fields.base.BaseField.has_unresolved_forward_refs"></a>
+#### has\_unresolved\_forward\_refs
+
+```python
+ | @classmethod
+ | has_unresolved_forward_refs(cls) -> bool
+```
+
+Verifies if the filed has any ForwardRefs that require updating before the
+model can be used.
+
+**Returns**:
+
+`(bool)`: result of the check
+
+<a name="fields.base.BaseField.evaluate_forward_ref"></a>
+#### evaluate\_forward\_ref
+
+```python
+ | @classmethod
+ | evaluate_forward_ref(cls, globalns: Any, localns: Any) -> None
+```
+
+Evaluates the ForwardRef to actual Field based on global and local namespaces
+
+**Arguments**:
+
+- `globalns (Any)`: global namespace
+- `localns (Any)`: local namespace
+
+**Returns**:
+
+`(None)`: None
+
+<a name="fields.base.BaseField.get_related_name"></a>
+#### get\_related\_name
+
+```python
+ | @classmethod
+ | get_related_name(cls) -> str
+```
+
+Returns name to use for reverse relation.
+It's either set as `related_name` or by default it's owner model. get_name + 's'
+
+**Returns**:
+
+`(str)`: name of the related_name or default related name.
+

docs/api/fields/foreign-key.md

@@ -46,6 +46,29 @@ Populates only pk field and set it to desired type.
 
 `(pydantic.BaseModel)`: constructed dummy model
 
+<a name="fields.foreign_key.populate_fk_params_based_on_to_model"></a>
+#### populate\_fk\_params\_based\_on\_to\_model
+
+```python
+populate_fk_params_based_on_to_model(to: Type["Model"], nullable: bool, onupdate: str = None, ondelete: str = None) -> Tuple[Any, List, Any]
+```
+
+Based on target to model to which relation leads to populates the type of the
+pydantic field to use, ForeignKey constraint and type of the target column field.
+
+**Arguments**:
+
+- `to (Model class)`: target related ormar Model
+- `nullable (bool)`: marks field as optional/ required
+- `onupdate (str)`: parameter passed to sqlalchemy.ForeignKey.
+How to treat child rows on update of parent (the one where FK is defined) model.
+- `ondelete (str)`: parameter passed to sqlalchemy.ForeignKey.
+How to treat child rows on delete of parent (the one where FK is defined) model.
+
+**Returns**:
+
+`(Tuple[Any, List, Any])`: tuple with target pydantic type, list of fk constraints and target col type
+
 <a name="fields.foreign_key.UniqueColumns"></a>
 ## UniqueColumns Objects
 
@@ -71,7 +94,7 @@ to produce sqlalchemy.ForeignKeys
 #### ForeignKey
 
 ```python
-ForeignKey(to: Type["Model"], *, name: str = None, unique: bool = False, nullable: bool = True, related_name: str = None, virtual: bool = False, onupdate: str = None, ondelete: str = None, **kwargs: Any, ,) -> Any
+ForeignKey(to: "ToType", *, name: str = None, unique: bool = False, nullable: bool = True, related_name: str = None, virtual: bool = False, onupdate: str = None, ondelete: str = None, **kwargs: Any, ,) -> Any
 ```
 
 Despite a name it's a function that returns constructed ForeignKeyField.
@@ -107,12 +130,62 @@ class ForeignKeyField(BaseField)
 
 Actual class returned from ForeignKey function call and stored in model_fields.
 
+<a name="fields.foreign_key.ForeignKeyField.get_source_related_name"></a>
+#### get\_source\_related\_name
+
+```python
+ | @classmethod
+ | get_source_related_name(cls) -> str
+```
+
+Returns name to use for source relation name.
+For FK it's the same, differs for m2m fields.
+It's either set as `related_name` or by default it's owner model. get_name + 's'
+
+**Returns**:
+
+`(str)`: name of the related_name or default related name.
+
+<a name="fields.foreign_key.ForeignKeyField.get_related_name"></a>
+#### get\_related\_name
+
+```python
+ | @classmethod
+ | get_related_name(cls) -> str
+```
+
+Returns name to use for reverse relation.
+It's either set as `related_name` or by default it's owner model. get_name + 's'
+
+**Returns**:
+
+`(str)`: name of the related_name or default related name.
+
+<a name="fields.foreign_key.ForeignKeyField.evaluate_forward_ref"></a>
+#### evaluate\_forward\_ref
+
+```python
+ | @classmethod
+ | evaluate_forward_ref(cls, globalns: Any, localns: Any) -> None
+```
+
+Evaluates the ForwardRef to actual Field based on global and local namespaces
+
+**Arguments**:
+
+- `globalns (Any)`: global namespace
+- `localns (Any)`: local namespace
+
+**Returns**:
+
+`(None)`: None
+
 <a name="fields.foreign_key.ForeignKeyField._extract_model_from_sequence"></a>
 #### \_extract\_model\_from\_sequence
 
 ```python
  | @classmethod
- | _extract_model_from_sequence(cls, value: List, child: "Model", to_register: bool, relation_name: str) -> List["Model"]
+ | _extract_model_from_sequence(cls, value: List, child: "Model", to_register: bool) -> List["Model"]
 ```
 
 Takes a list of Models and registers them on parent.
@@ -135,7 +208,7 @@ Used in reverse FK relations.
 
 ```python
  | @classmethod
- | _register_existing_model(cls, value: "Model", child: "Model", to_register: bool, relation_name: str) -> "Model"
+ | _register_existing_model(cls, value: "Model", child: "Model", to_register: bool) -> "Model"
 ```
 
 Takes already created instance and registers it for parent.
@@ -158,7 +231,7 @@ Used in reverse FK relations and normal FK for single models.
 
 ```python
  | @classmethod
- | _construct_model_from_dict(cls, value: dict, child: "Model", to_register: bool, relation_name: str) -> "Model"
+ | _construct_model_from_dict(cls, value: dict, child: "Model", to_register: bool) -> "Model"
 ```
 
 Takes a dictionary, creates a instance and registers it for parent.
@@ -182,7 +255,7 @@ Used in normal FK for dictionaries.
 
 ```python
  | @classmethod
- | _construct_model_from_pk(cls, value: Any, child: "Model", to_register: bool, relation_name: str) -> "Model"
+ | _construct_model_from_pk(cls, value: Any, child: "Model", to_register: bool) -> "Model"
 ```
 
 Takes a pk value, creates a dummy instance and registers it for parent.
@@ -205,7 +278,7 @@ Used in normal FK for dictionaries.
 
 ```python
  | @classmethod
- | register_relation(cls, model: "Model", child: "Model", relation_name: str) -> None
+ | register_relation(cls, model: "Model", child: "Model") -> None
 ```
 
 Registers relation between parent and child in relation manager.
@@ -219,12 +292,27 @@ Used in Metaclass and sometimes some relations are missing
 - `model (Model class)`: parent model (with relation definition)
 - `child (Model class)`: child model
 
+<a name="fields.foreign_key.ForeignKeyField.has_unresolved_forward_refs"></a>
+#### has\_unresolved\_forward\_refs
+
+```python
+ | @classmethod
+ | has_unresolved_forward_refs(cls) -> bool
+```
+
+Verifies if the filed has any ForwardRefs that require updating before the
+model can be used.
+
+**Returns**:
+
+`(bool)`: result of the check
+
 <a name="fields.foreign_key.ForeignKeyField.expand_relationship"></a>
 #### expand\_relationship
 
 ```python
  | @classmethod
- | expand_relationship(cls, value: Any, child: Union["Model", "NewBaseModel"], to_register: bool = True, relation_name: str = None) -> Optional[Union["Model", List["Model"]]]
+ | expand_relationship(cls, value: Any, child: Union["Model", "NewBaseModel"], to_register: bool = True) -> Optional[Union["Model", List["Model"]]]
 ```
 
 For relations the child model is first constructed (if needed),

docs/api/fields/many-to-many.md

@@ -1,11 +1,30 @@
 <a name="fields.many_to_many"></a>
 # fields.many\_to\_many
 
+<a name="fields.many_to_many.populate_m2m_params_based_on_to_model"></a>
+#### populate\_m2m\_params\_based\_on\_to\_model
+
+```python
+populate_m2m_params_based_on_to_model(to: Type["Model"], nullable: bool) -> Tuple[Any, Any]
+```
+
+Based on target to model to which relation leads to populates the type of the
+pydantic field to use and type of the target column field.
+
+**Arguments**:
+
+- `to (Model class)`: target related ormar Model
+- `nullable (bool)`: marks field as optional/ required
+
+**Returns**:
+
+`(tuple with target pydantic type and target col type)`: Tuple[List, Any]
+
 <a name="fields.many_to_many.ManyToMany"></a>
 #### ManyToMany
 
 ```python
-ManyToMany(to: Type["Model"], through: Type["Model"], *, name: str = None, unique: bool = False, virtual: bool = False, **kwargs: Any) -> Any
+ManyToMany(to: "ToType", through: "ToType", *, name: str = None, unique: bool = False, virtual: bool = False, **kwargs: Any, ,) -> Any
 ```
 
 Despite a name it's a function that returns constructed ManyToManyField.
@@ -37,6 +56,22 @@ class ManyToManyField(ForeignKeyField,  ormar.QuerySetProtocol,  ormar.RelationP
 
 Actual class returned from ManyToMany function call and stored in model_fields.
 
+<a name="fields.many_to_many.ManyToManyField.get_source_related_name"></a>
+#### get\_source\_related\_name
+
+```python
+ | @classmethod
+ | get_source_related_name(cls) -> str
+```
+
+Returns name to use for source relation name.
+For FK it's the same, differs for m2m fields.
+It's either set as `related_name` or by default it's field name.
+
+**Returns**:
+
+`(str)`: name of the related_name or default related name.
+
 <a name="fields.many_to_many.ManyToManyField.default_target_field_name"></a>
 #### default\_target\_field\_name
 
@@ -51,3 +86,51 @@ Returns default target model name on through model.
 
 `(str)`: name of the field
 
+<a name="fields.many_to_many.ManyToManyField.default_source_field_name"></a>
+#### default\_source\_field\_name
+
+```python
+ | @classmethod
+ | default_source_field_name(cls) -> str
+```
+
+Returns default target model name on through model.
+
+**Returns**:
+
+`(str)`: name of the field
+
+<a name="fields.many_to_many.ManyToManyField.has_unresolved_forward_refs"></a>
+#### has\_unresolved\_forward\_refs
+
+```python
+ | @classmethod
+ | has_unresolved_forward_refs(cls) -> bool
+```
+
+Verifies if the filed has any ForwardRefs that require updating before the
+model can be used.
+
+**Returns**:
+
+`(bool)`: result of the check
+
+<a name="fields.many_to_many.ManyToManyField.evaluate_forward_ref"></a>
+#### evaluate\_forward\_ref
+
+```python
+ | @classmethod
+ | evaluate_forward_ref(cls, globalns: Any, localns: Any) -> None
+```
+
+Evaluates the ForwardRef to actual Field based on global and local namespaces
+
+**Arguments**:
+
+- `globalns (Any)`: global namespace
+- `localns (Any)`: local namespace
+
+**Returns**:
+
+`(None)`: None
+

docs/api/models/helpers/models.md

@@ -1,6 +1,24 @@
 <a name="models.helpers.models"></a>
 # models.helpers.models
 
+<a name="models.helpers.models.is_field_an_forward_ref"></a>
+#### is\_field\_an\_forward\_ref
+
+```python
+is_field_an_forward_ref(field: Type["BaseField"]) -> bool
+```
+
+Checks if field is a relation field and whether any of the referenced models
+are ForwardRefs that needs to be updated before proceeding.
+
+**Arguments**:
+
+- `field (Type[BaseField])`: model field to verify
+
+**Returns**:
+
+`(bool)`: result of the check
+
 <a name="models.helpers.models.populate_default_options_values"></a>
 #### populate\_default\_options\_values
 
@@ -62,3 +80,28 @@ Also related_names have to be unique for given related model.
 - `model_fields (Dict[str, ormar.Field])`: dictionary of declared ormar model fields
 - `new_model (Model class)`: 
 
+<a name="models.helpers.models.group_related_list"></a>
+#### group\_related\_list
+
+```python
+group_related_list(list_: List) -> Dict
+```
+
+Translates the list of related strings into a dictionary.
+That way nested models are grouped to traverse them in a right order
+and to avoid repetition.
+
+Sample: ["people__houses", "people__cars__models", "people__cars__colors"]
+will become:
+{'people': {'houses': [], 'cars': ['models', 'colors']}}
+
+Result dictionary is sorted by length of the values and by key
+
+**Arguments**:
+
+- `list_ (List[str])`: list of related models used in select related
+
+**Returns**:
+
+`(Dict[str, List])`: list converted to dictionary to avoid repetition and group nested models
+

docs/api/models/helpers/relations.md

@@ -5,7 +5,7 @@
 #### register\_relation\_on\_build
 
 ```python
-register_relation_on_build(new_model: Type["Model"], field_name: str) -> None
+register_relation_on_build(field: Type["ForeignKeyField"]) -> None
 ```
 
 Registers ForeignKey relation in alias_manager to set a table_prefix.
@@ -17,14 +17,13 @@ aliases for proper sql joins.
 
 **Arguments**:
 
-- `new_model (Model class)`: constructed model
+- `field (ForeignKey class)`: relation field
-- `field_name (str)`: name of the related field
 
 <a name="models.helpers.relations.register_many_to_many_relation_on_build"></a>
 #### register\_many\_to\_many\_relation\_on\_build
 
 ```python
-register_many_to_many_relation_on_build(new_model: Type["Model"], field: Type[ManyToManyField], field_name: str) -> None
+register_many_to_many_relation_on_build(field: Type[ManyToManyField]) -> None
 ```
 
 Registers connection between through model and both sides of the m2m relation.
@@ -38,10 +37,25 @@ By default relation name is a model.name.lower().
 
 **Arguments**:
 
-- `field_name (str)`: name of the relation key
-- `new_model (Model class)`: model on which m2m field is declared
 - `field (ManyToManyField class)`: relation field
 
+<a name="models.helpers.relations.expand_reverse_relationship"></a>
+#### expand\_reverse\_relationship
+
+```python
+expand_reverse_relationship(model_field: Type["ForeignKeyField"]) -> None
+```
+
+If the reverse relation has not been set before it's set here.
+
+**Arguments**:
+
+- `model_field ()`: 
+
+**Returns**:
+
+`(None)`: None
+
 <a name="models.helpers.relations.expand_reverse_relationships"></a>
 #### expand\_reverse\_relationships
 
@@ -62,7 +76,7 @@ If the reverse relation has not been set before it's set here.
 #### register\_reverse\_model\_fields
 
 ```python
-register_reverse_model_fields(model: Type["Model"], child: Type["Model"], related_name: str, model_field: Type["ForeignKeyField"]) -> None
+register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None
 ```
 
 Registers reverse ForeignKey field on related model.
@@ -73,16 +87,13 @@ Autogenerated reverse fields also set related_name to the original field name.
 
 **Arguments**:
 
-- `model (Model class)`: related model on which reverse field should be defined
-- `child (Model class)`: parent model with relation definition
-- `related_name (str)`: name by which reverse key should be registered
 - `model_field (relation Field)`: original relation ForeignKey field
 
 <a name="models.helpers.relations.register_relation_in_alias_manager"></a>
 #### register\_relation\_in\_alias\_manager
 
 ```python
-register_relation_in_alias_manager(new_model: Type["Model"], field: Type[ForeignKeyField], field_name: str) -> None
+register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None
 ```
 
 Registers the relation (and reverse relation) in alias manager.
@@ -95,15 +106,13 @@ fk - register_relation_on_build
 
 **Arguments**:
 
-- `new_model (Model class)`: model on which relation field is declared
 - `field (ForeignKey or ManyToManyField class)`: relation field
-- `field_name (str)`: name of the relation key
 
 <a name="models.helpers.relations.verify_related_name_dont_duplicate"></a>
 #### verify\_related\_name\_dont\_duplicate
 
 ```python
-verify_related_name_dont_duplicate(child: Type["Model"], parent_model: Type["Model"], related_name: str) -> None
+verify_related_name_dont_duplicate(related_name: str, model_field: Type["ForeignKeyField"]) -> None
 ```
 
 Verifies whether the used related_name (regardless of the fact if user defined or
@@ -117,9 +126,8 @@ model
 
 **Arguments**:
 
-- `child (ormar.models.metaclass.ModelMetaclass)`: related Model class
-- `parent_model (ormar.models.metaclass.ModelMetaclass)`: parent Model class
 - `related_name ()`: 
+- `model_field (relation Field)`: original relation ForeignKey field
 
 **Returns**:
 
@@ -129,7 +137,7 @@ model
 #### reverse\_field\_not\_already\_registered
 
 ```python
-reverse_field_not_already_registered(child: Type["Model"], child_model_name: str, parent_model: Type["Model"]) -> bool
+reverse_field_not_already_registered(model_field: Type["ForeignKeyField"]) -> bool
 ```
 
 Checks if child is already registered in parents pydantic fields.
@@ -141,9 +149,7 @@ related model
 
 **Arguments**:
 
-- `child (ormar.models.metaclass.ModelMetaclass)`: related Model class
+- `model_field (relation Field)`: original relation ForeignKey field
-- `child_model_name (str)`: related_name of the child if provided
-- `parent_model (ormar.models.metaclass.ModelMetaclass)`: parent Model class
 
 **Returns**:
 

docs/api/models/helpers/sqlalchemy.md

@@ -5,7 +5,7 @@
 #### adjust\_through\_many\_to\_many\_model
 
 ```python
-adjust_through_many_to_many_model(model: Type["Model"], child: Type["Model"], model_field: Type[ManyToManyField]) -> None
+adjust_through_many_to_many_model(model_field: Type[ManyToManyField]) -> None
 ```
 
 Registers m2m relation on through model.
@@ -15,23 +15,22 @@ Sets pydantic fields with child and parent model types.
 
 **Arguments**:
 
-- `model (Model class)`: model on which relation is declared
-- `child (Model class)`: model to which m2m relation leads
 - `model_field (ManyToManyField)`: relation field defined in parent model
 
 <a name="models.helpers.sqlalchemy.create_and_append_m2m_fk"></a>
 #### create\_and\_append\_m2m\_fk
 
 ```python
-create_and_append_m2m_fk(model: Type["Model"], model_field: Type[ManyToManyField]) -> None
+create_and_append_m2m_fk(model: Type["Model"], model_field: Type[ManyToManyField], field_name: str) -> None
 ```
 
-Registers sqlalchemy Column with sqlalchemy.ForeignKey leadning to the model.
+Registers sqlalchemy Column with sqlalchemy.ForeignKey leading to the model.
 
 Newly created field is added to m2m relation through model Meta columns and table.
 
 **Arguments**:
 
+- `field_name (str)`: name of the column to create
 - `model (Model class)`: Model class to which FK should be created
 - `model_field (ManyToManyField field)`: field with ManyToMany relation
 
@@ -83,6 +82,8 @@ cannot be pydantic_only.
 Append fields to columns if it's not pydantic_only,
 virtual ForeignKey or ManyToMany field.
 
+Sets `owner` on each model_field as reference to newly created Model.
+
 **Raises**:
 
 - `ModelDefinitionError`: if validation of related_names fail,
@@ -125,6 +126,23 @@ Each model has to have pk.
 
 `(ormar.models.metaclass.ModelMetaclass)`: Model with populated pkname and columns in Meta
 
+<a name="models.helpers.sqlalchemy.check_for_null_type_columns_from_forward_refs"></a>
+#### check\_for\_null\_type\_columns\_from\_forward\_refs
+
+```python
+check_for_null_type_columns_from_forward_refs(meta: "ModelMeta") -> bool
+```
+
+Check is any column is of NUllType() meaning it's empty column from ForwardRef
+
+**Arguments**:
+
+- `meta (Model class Meta)`: Meta class of the Model without sqlalchemy table constructed
+
+**Returns**:
+
+`(bool)`: result of the check
+
 <a name="models.helpers.sqlalchemy.populate_meta_sqlalchemy_table_if_required"></a>
 #### populate\_meta\_sqlalchemy\_table\_if\_required
 
@@ -143,3 +161,21 @@ It populates name, metadata, columns and constraints.
 
 `(Model class)`: class with populated Meta.table
 
+<a name="models.helpers.sqlalchemy.update_column_definition"></a>
+#### update\_column\_definition
+
+```python
+update_column_definition(model: Union[Type["Model"], Type["NewBaseModel"]], field: Type[ForeignKeyField]) -> None
+```
+
+Updates a column with a new type column based on updated parameters in FK fields.
+
+**Arguments**:
+
+- `model (Type["Model"])`: model on which columns needs to be updated
+- `field (Type[ForeignKeyField])`: field with column definition that requires update
+
+**Returns**:
+
+`(None)`: None
+

docs/api/models/mixins/prefetch-query-mixin.md

@@ -59,7 +59,7 @@ or field name specified by related parameter.
 
 ```python
  | @classmethod
- | get_related_field_name(cls, target_field: Type["BaseField"]) -> str
+ | get_related_field_name(cls, target_field: Type["ForeignKeyField"]) -> str
 ```
 
 Returns name of the relation field that should be used in prefetch query.

docs/api/models/model-metaclass.md

@@ -1,6 +1,17 @@
 <a name="models.metaclass"></a>
 # models.metaclass
 
+<a name="models.metaclass.ModelMeta"></a>
+## ModelMeta Objects
+
+```python
+class ModelMeta()
+```
+
+Class used for type hinting.
+Users can subclass this one for convenience but it's not required.
+The only requirement is that ormar.Model has to have inner class with name Meta.
+
 <a name="models.metaclass.check_if_field_has_choices"></a>
 #### check\_if\_field\_has\_choices
 
@@ -143,7 +154,7 @@ as well as model.Meta.model_fields definitions from parents.
 **Arguments**:
 
 - `attrs (Dict)`: new namespace for class being constructed
-- `new_attrs (Dict)`: part of the namespace extracted from parent class
+- `new_attrs (Dict)`: related of the namespace extracted from parent class
 - `model_fields (Dict[str, BaseField])`: ormar fields in defined in current class
 - `new_model_fields (Dict[str, BaseField])`: ormar fields defined in parent classes
 - `new_fields (Set[str])`: set of new fields names
@@ -270,18 +281,6 @@ If the class is a ormar.Model it is skipped.
 
 `(Tuple[Dict, Dict])`: updated attrs and model_fields
 
-<a name="models.metaclass.ModelMeta"></a>
-## ModelMeta Objects
-
-```python
-class ModelMeta()
-```
-
-Class used for type hinting.
-Users can subclass this one for convenience but it's not required.
-The only requirement is that ormar.Model has to have inner class with name Meta.
-
-
 <a name="models.metaclass.ModelMetaclass"></a>
 ## ModelMetaclass Objects
 

docs/api/models/model.md

@@ -1,29 +1,6 @@
 <a name="models.model"></a>
 # models.model
 
-<a name="models.model.group_related_list"></a>
-#### group\_related\_list
-
-```python
-group_related_list(list_: List) -> Dict
-```
-
-Translates the list of related strings into a dictionary.
-That way nested models are grouped to traverse them in a right order
-and to avoid repetition.
-
-Sample: ["people__houses", "people__cars__models", "people__cars__colors"]
-will become:
-{'people': {'houses': [], 'cars': ['models', 'colors']}}
-
-**Arguments**:
-
-- `list_ (List[str])`: list of related models used in select related
-
-**Returns**:
-
-`(Dict[str, List])`: list converted to dictionary to avoid repetition and group nested models
-
 <a name="models.model.Model"></a>
 ## Model Objects
 
@@ -36,7 +13,7 @@ class Model(NewBaseModel)
 
 ```python
  | @classmethod
- | from_row(cls: Type[T], row: sqlalchemy.engine.ResultProxy, select_related: List = None, related_models: Any = None, previous_model: Type[T] = None, related_name: str = None, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None) -> Optional[T]
+ | from_row(cls: Type[T], row: sqlalchemy.engine.ResultProxy, select_related: List = None, related_models: Any = None, previous_model: Type[T] = None, source_model: Type[T] = None, related_name: str = None, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None, current_relation_str: str = None) -> Optional[T]
 ```
 
 Model method to convert raw sql row from database into ormar.Model instance.
@@ -72,7 +49,7 @@ excludes the fields even if they are provided in fields
 
 ```python
  | @classmethod
- | populate_nested_models_from_row(cls, item: dict, row: sqlalchemy.engine.ResultProxy, related_models: Any, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None) -> dict
+ | populate_nested_models_from_row(cls, item: dict, row: sqlalchemy.engine.ResultProxy, related_models: Any, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None, current_relation_str: str = None, source_model: Type[T] = None) -> dict
 ```
 
 Traverses structure of related models and populates the nested models
@@ -86,6 +63,8 @@ instances. In the end those instances are added to the final model dictionary.
 
 **Arguments**:
 
+- `source_model (Type[Model])`: source model from which relation started
+- `current_relation_str (str)`: joined related parts into one string
 - `item (Dict)`: dictionary of already populated nested models, otherwise empty dict
 - `row (sqlalchemy.engine.result.ResultProxy)`: raw result row from the database
 - `related_models (Union[Dict, List])`: list or dict of related models
@@ -114,7 +93,7 @@ If the table is a main table, there is no prefix.
 All joined tables have prefixes to allow duplicate column names,
 as well as duplicated joins to the same table from multiple different tables.
 
-Extracted fields populates the item dict later used to construct a Model.
+Extracted fields populates the related dict later used to construct a Model.
 
 Used in Model.from_row and PrefetchQuery._populate_rows methods.
 

docs/api/models/new-basemodel.md

@@ -48,7 +48,8 @@ them with their default values if default is set.
 
 **Raises**:
 
-- `ModelError`: if abstract model is initialized or unknown field is passed
+- `ModelError`: if abstract model is initialized, model has ForwardRefs
+that has not been updated or unknown field is passed
 
 **Arguments**:
 
@@ -128,6 +129,19 @@ Json fields are converted if needed.
 
 `(Any)`: value of the attribute
 
+<a name="models.newbasemodel.NewBaseModel._verify_model_can_be_initialized"></a>
+#### \_verify\_model\_can\_be\_initialized
+
+```python
+ | _verify_model_can_be_initialized() -> None
+```
+
+Raises exception if model is abstract or has ForwardRefs in relation fields.
+
+**Returns**:
+
+`(None)`: None
+
 <a name="models.newbasemodel.NewBaseModel._extract_related_model_instead_of_field"></a>
 #### \_extract\_related\_model\_instead\_of\_field
 
@@ -299,6 +313,34 @@ present in fastapi responses.
 
 `(Set[str])`: set of property fields names
 
+<a name="models.newbasemodel.NewBaseModel.update_forward_refs"></a>
+#### update\_forward\_refs
+
+```python
+ | @classmethod
+ | update_forward_refs(cls, **localns: Any) -> None
+```
+
+Processes fields that are ForwardRef and need to be evaluated into actual
+models.
+
+Expands relationships, register relation in alias manager and substitutes
+sqlalchemy columns with new ones with proper column type (null before).
+
+Populates Meta table of the Model which is left empty before.
+
+Sets self_reference flag on models that links to themselves.
+
+Calls the pydantic method to evaluate pydantic fields.
+
+**Arguments**:
+
+- `localns (Any)`: local namespace
+
+**Returns**:
+
+`(None)`: None
+
 <a name="models.newbasemodel.NewBaseModel._get_related_not_excluded_fields"></a>
 #### \_get\_related\_not\_excluded\_fields
 

docs/api/query-set/clause.md

@@ -8,13 +8,13 @@
 class QueryClause()
 ```
 
-Constructs where clauses from strings passed as arguments
+Constructs FilterActions from strings passed as arguments
 
-<a name="queryset.clause.QueryClause.filter"></a>
+<a name="queryset.clause.QueryClause.prepare_filter"></a>
-#### filter
+#### prepare\_filter
 
 ```python
- | filter(**kwargs: Any) -> Tuple[List[sqlalchemy.sql.expression.TextClause], List[str]]
+ | prepare_filter(**kwargs: Any) -> Tuple[List[FilterAction], List[str]]
 ```
 
 Main external access point that processes the clauses into sqlalchemy text
@@ -33,7 +33,7 @@ mentioned in select_related strings but not included in select_related.
 #### \_populate\_filter\_clauses
 
 ```python
- | _populate_filter_clauses(**kwargs: Any) -> Tuple[List[sqlalchemy.sql.expression.TextClause], List[str]]
+ | _populate_filter_clauses(**kwargs: Any) -> Tuple[List[FilterAction], List[str]]
 ```
 
 Iterates all clauses and extracts used operator and field from related
@@ -48,114 +48,59 @@ is determined and the final clause is escaped if needed and compiled.
 
 `(Tuple[List[sqlalchemy.sql.elements.TextClause], List[str]])`: Tuple with list of where clauses and updated select_related list
 
-<a name="queryset.clause.QueryClause._process_column_clause_for_operator_and_value"></a>
+<a name="queryset.clause.QueryClause._register_complex_duplicates"></a>
-#### \_process\_column\_clause\_for\_operator\_and\_value
+#### \_register\_complex\_duplicates
 
 ```python
- | _process_column_clause_for_operator_and_value(value: Any, op: str, column: sqlalchemy.Column, table: sqlalchemy.Table, table_prefix: str) -> sqlalchemy.sql.expression.TextClause
+ | _register_complex_duplicates(select_related: List[str]) -> None
 ```
 
-Escapes characters if it's required.
+Checks if duplicate aliases are presented which can happen in self relation
-Substitutes values of the models if value is a ormar Model with its pk value.
+or when two joins end with the same pair of models.
-Compiles the clause.
+
+If there are duplicates, the all duplicated joins are registered as source
+model and whole relation key (not just last relation name).
 
 **Arguments**:
 
-- `value (Any)`: value of the filter
+- `select_related (List[str])`: list of relation strings
-- `op (str)`: filter operator
-- `column (sqlalchemy.sql.schema.Column)`: column on which filter should be applied
-- `table (sqlalchemy.sql.schema.Table)`: table on which filter should be applied
-- `table_prefix (str)`: prefix from AliasManager
 
 **Returns**:
 
-`(sqlalchemy.sql.elements.TextClause)`: complied and escaped clause
+`(None)`: None
 
-<a name="queryset.clause.QueryClause._determine_filter_target_table"></a>
+<a name="queryset.clause.QueryClause._parse_related_prefixes"></a>
-#### \_determine\_filter\_target\_table
+#### \_parse\_related\_prefixes
 
 ```python
- | _determine_filter_target_table(related_parts: List[str], select_related: List[str]) -> Tuple[List[str], str, Type["Model"]]
+ | _parse_related_prefixes(select_related: List[str]) -> List[Prefix]
 ```
 
-Adds related strings to select_related list otherwise the clause would fail as
+Walks all relation strings and parses the target models and prefixes.
-the required columns would not be present. That means that select_related
-list is filled with missing values present in filters.
-
-Walks the relation to retrieve the actual model on which the clause should be
-constructed, extracts alias based on last relation leading to target model.
 
 **Arguments**:
 
-- `related_parts (List[str])`: list of split parts of related string
+- `select_related (List[str])`: list of relation strings
-- `select_related (List[str])`: list of related models
 
 **Returns**:
 
-`(Tuple[List[str], str, Type[Model]])`: list of related models, table_prefix, final model class
+`(List[Prefix])`: list of parsed prefixes
 
-<a name="queryset.clause.QueryClause._compile_clause"></a>
+<a name="queryset.clause.QueryClause._switch_filter_action_prefixes"></a>
-#### \_compile\_clause
+#### \_switch\_filter\_action\_prefixes
 
 ```python
- | _compile_clause(clause: sqlalchemy.sql.expression.BinaryExpression, column: sqlalchemy.Column, table: sqlalchemy.Table, table_prefix: str, modifiers: Dict) -> sqlalchemy.sql.expression.TextClause
+ | _switch_filter_action_prefixes(filter_clauses: List[FilterAction]) -> List[FilterAction]
 ```
 
-Compiles the clause to str using appropriate database dialect, replace columns
+Substitutes aliases for filter action if the complex key (whole relation str) is
-names with aliased names and converts it back to TextClause.
+present in alias_manager.
 
 **Arguments**:
 
-- `clause (sqlalchemy.sql.elements.BinaryExpression)`: original not compiled clause
+- `filter_clauses (List[FilterAction])`: raw list of actions
-- `column (sqlalchemy.sql.schema.Column)`: column on which filter should be applied
-- `table (sqlalchemy.sql.schema.Table)`: table on which filter should be applied
-- `table_prefix (str)`: prefix from AliasManager
-- `modifiers (Dict[str, NoneType])`: sqlalchemy modifiers - used only to escape chars here
 
 **Returns**:
 
-`(sqlalchemy.sql.elements.TextClause)`: compiled and escaped clause
+`(List[FilterAction])`: list of actions with aliases changed if needed
-
-<a name="queryset.clause.QueryClause._escape_characters_in_clause"></a>
-#### \_escape\_characters\_in\_clause
-
-```python
- | @staticmethod
- | _escape_characters_in_clause(op: str, value: Any) -> Tuple[Any, bool]
-```
-
-Escapes the special characters ["%", "_"] if needed.
-Adds `%` for `like` queries.
-
-**Raises**:
-
-- `QueryDefinitionError`: if contains or icontains is used with
-ormar model instance
-
-**Arguments**:
-
-- `op (str)`: operator used in query
-- `value (Any)`: value of the filter
-
-**Returns**:
-
-`(Tuple[Any, bool])`: escaped value and flag if escaping is needed
-
-<a name="queryset.clause.QueryClause._extract_operator_field_and_related"></a>
-#### \_extract\_operator\_field\_and\_related
-
-```python
- | @staticmethod
- | _extract_operator_field_and_related(parts: List[str]) -> Tuple[str, str, Optional[List]]
-```
-
-Splits filter query key and extracts required parts.
-
-**Arguments**:
-
-- `parts (List[str])`: split filter query key
-
-**Returns**:
-
-`(Tuple[str, str, Optional[List]])`: operator, field_name, list of related parts
 

docs/api/query-set/join.md

@@ -1,15 +1,6 @@
 <a name="queryset.join"></a>
 # queryset.join
 
-<a name="queryset.join.JoinParameters"></a>
-## JoinParameters Objects
-
-```python
-class JoinParameters(NamedTuple)
-```
-
-Named tuple that holds set of parameters passed during join construction.
-
 <a name="queryset.join.SqlJoin"></a>
 ## SqlJoin Objects
 
@@ -21,15 +12,11 @@ class SqlJoin()
 #### alias\_manager
 
 ```python
- | @staticmethod
+ | @property
- | alias_manager(model_cls: Type["Model"]) -> AliasManager
+ | alias_manager() -> AliasManager
 ```
 
-Shortcut for ormars model AliasManager stored on Meta.
+Shortcut for ormar's model AliasManager stored on Meta.
-
-**Arguments**:
-
-- `model_cls (Type[Model])`: ormar Model class
 
 **Returns**:
 
@@ -39,8 +26,7 @@ Shortcut for ormars model AliasManager stored on Meta.
 #### on\_clause
 
 ```python
- | @staticmethod
+ | on_clause(previous_alias: str, from_clause: str, to_clause: str) -> text
- | on_clause(previous_alias: str, alias: str, from_clause: str, to_clause: str) -> text
 ```
 
 Receives aliases and names of both ends of the join and combines them
@@ -49,7 +35,6 @@ into one text clause used in joins.
 **Arguments**:
 
 - `previous_alias (str)`: alias of previous table
-- `alias (str)`: alias of current table
 - `from_clause (str)`: from table name
 - `to_clause (str)`: to table name
 
@@ -57,32 +42,11 @@ into one text clause used in joins.
 
 `(sqlalchemy.text)`: clause combining all strings
 
-<a name="queryset.join.SqlJoin.update_inclusions"></a>
-#### update\_inclusions
-
-```python
- | @staticmethod
- | update_inclusions(model_cls: Type["Model"], fields: Optional[Union[Set, Dict]], exclude_fields: Optional[Union[Set, Dict]], nested_name: str) -> Tuple[Optional[Union[Dict, Set]], Optional[Union[Dict, Set]]]
-```
-
-Extract nested fields and exclude_fields if applicable.
-
-**Arguments**:
-
-- `model_cls (Type["Model"])`: ormar model class
-- `fields (Optional[Union[Set, Dict]])`: fields to include
-- `exclude_fields (Optional[Union[Set, Dict]])`: fields to exclude
-- `nested_name (str)`: name of the nested field
-
-**Returns**:
-
-`(Tuple[Optional[Union[Dict, Set]], Optional[Union[Dict, Set]]])`: updated exclude and include fields from nested objects
-
 <a name="queryset.join.SqlJoin.build_join"></a>
 #### build\_join
 
 ```python
- | build_join(item: str, join_parameters: JoinParameters) -> Tuple[List, sqlalchemy.sql.select, List, OrderedDict]
+ | build_join() -> Tuple[List, sqlalchemy.sql.select, List, OrderedDict]
 ```
 
 Main external access point for building a join.
@@ -90,42 +54,96 @@ Splits the join definition, updates fields and exclude_fields if needed,
 handles switching to through models for m2m relations, returns updated lists of
 used_aliases and sort_orders.
 
-**Arguments**:
-
-- `item (str)`: string with join definition
-- `join_parameters (JoinParameters)`: parameters from previous/ current join
-
 **Returns**:
 
 `(Tuple[List[str], Join, List[TextClause], collections.OrderedDict])`: list of used aliases, select from, list of aliased columns, sort orders
 
-<a name="queryset.join.SqlJoin._build_join_parameters"></a>
+<a name="queryset.join.SqlJoin._forward_join"></a>
-#### \_build\_join\_parameters
+#### \_forward\_join
 
 ```python
- | _build_join_parameters(part: str, join_params: JoinParameters, fields: Optional[Union[Set, Dict]], exclude_fields: Optional[Union[Set, Dict]], is_multi: bool = False) -> JoinParameters
+ | _forward_join() -> None
 ```
 
-Updates used_aliases to not join multiple times to the same table.
+Process actual join.
-Updates join parameters with new values.
+Registers complex relation join on encountering of the duplicated alias.
+
+<a name="queryset.join.SqlJoin._process_following_joins"></a>
+#### \_process\_following\_joins
+
+```python
+ | _process_following_joins() -> None
+```
+
+Iterates through nested models to create subsequent joins.
+
+<a name="queryset.join.SqlJoin._process_deeper_join"></a>
+#### \_process\_deeper\_join
+
+```python
+ | _process_deeper_join(related_name: str, remainder: Any) -> None
+```
+
+Creates nested recurrent instance of SqlJoin for each nested join table,
+updating needed return params here as a side effect.
+
+Updated are:
+
+* self.used_aliases,
+* self.select_from,
+* self.columns,
+* self.sorted_orders,
 
 **Arguments**:
 
-- `part (str)`: part of the join str definition
+- `related_name (str)`: name of the relation to follow
-- `join_params (JoinParameters)`: parameters from previous/ current join
+- `remainder (Any)`: deeper tables if there are more nested joins
-- `fields (Optional[Union[Set, Dict]])`: fields to include
+
-- `exclude_fields (Optional[Union[Set, Dict]])`: fields to exclude
+<a name="queryset.join.SqlJoin.process_m2m_through_table"></a>
-- `is_multi (bool)`: flag if the relation is m2m
+#### process\_m2m\_through\_table
+
+```python
+ | process_m2m_through_table() -> None
+```
+
+Process Through table of the ManyToMany relation so that source table is
+linked to the through table (one additional join)
+
+Replaces needed parameters like:
+
+* self.next_model,
+* self.next_alias,
+* self.relation_name,
+* self.own_alias,
+* self.target_field
+
+To point to through model
+
+<a name="queryset.join.SqlJoin.process_m2m_related_name_change"></a>
+#### process\_m2m\_related\_name\_change
+
+```python
+ | process_m2m_related_name_change(reverse: bool = False) -> str
+```
+
+Extracts relation name to link join through the Through model declared on
+relation field.
+
+Changes the same names in order_by queries if they are present.
+
+**Arguments**:
+
+- `reverse (bool)`: flag if it's on_clause lookup - use reverse fields
 
 **Returns**:
 
-`(ormar.queryset.join.JoinParameters)`: updated join parameters
+`(str)`: new relation name switched to through model field
 
 <a name="queryset.join.SqlJoin._process_join"></a>
 #### \_process\_join
 
 ```python
- | _process_join(join_params: JoinParameters, is_multi: bool, model_cls: Type["Model"], part: str, alias: str, fields: Optional[Union[Set, Dict]], exclude_fields: Optional[Union[Set, Dict]]) -> None
+ | _process_join() -> None
 ```
 
 Resolves to and from column names and table names.
@@ -140,21 +158,11 @@ Updates the used aliases list directly.
 
 Process order_by causes for non m2m relations.
 
-**Arguments**:
+<a name="queryset.join.SqlJoin._replace_many_to_many_order_by_columns"></a>
-
+#### \_replace\_many\_to\_many\_order\_by\_columns
-- `join_params (JoinParameters)`: parameters from previous/ current join
-- `is_multi (bool)`: flag if it's m2m relation
-- `model_cls (ormar.models.metaclass.ModelMetaclass)`: 
-- `part (str)`: name of the field used in join
-- `alias (str)`: alias of the current join
-- `fields (Optional[Union[Set, Dict]])`: fields to include
-- `exclude_fields (Optional[Union[Set, Dict]])`: fields to exclude
-
-<a name="queryset.join.SqlJoin._switch_many_to_many_order_columns"></a>
-#### \_switch\_many\_to\_many\_order\_columns
 
 ```python
- | _switch_many_to_many_order_columns(part: str, new_part: str) -> None
+ | _replace_many_to_many_order_by_columns(part: str, new_part: str) -> None
 ```
 
 Substitutes the name of the relation with actual model name in m2m order bys.
@@ -187,7 +195,7 @@ Checks filter conditions to find if they apply to current join.
 #### set\_aliased\_order\_by
 
 ```python
- | set_aliased_order_by(condition: List[str], alias: str, to_table: str, model_cls: Type["Model"]) -> None
+ | set_aliased_order_by(condition: List[str], to_table: str) -> None
 ```
 
 Substitute hyphens ('-') with descending order.
@@ -196,15 +204,13 @@ Construct actual sqlalchemy text clause using aliased table and column name.
 **Arguments**:
 
 - `condition (List[str])`: list of parts of a current condition split by '__'
-- `alias (str)`: alias of the table in current join
 - `to_table (sqlalchemy.sql.elements.quoted_name)`: target table
-- `model_cls (ormar.models.metaclass.ModelMetaclass)`: ormar model class
 
 <a name="queryset.join.SqlJoin.get_order_bys"></a>
 #### get\_order\_bys
 
 ```python
- | get_order_bys(alias: str, to_table: str, pkname_alias: str, part: str, model_cls: Type["Model"]) -> None
+ | get_order_bys(to_table: str, pkname_alias: str) -> None
 ```
 
 Triggers construction of order bys if they are given.
@@ -212,30 +218,19 @@ Otherwise by default each table is sorted by a primary key column asc.
 
 **Arguments**:
 
-- `alias (str)`: alias of current table in join
 - `to_table (sqlalchemy.sql.elements.quoted_name)`: target table
 - `pkname_alias (str)`: alias of the primary key column
-- `part (str)`: name of the current relation join
-- `model_cls (Type[Model])`: ormar model class
 
 <a name="queryset.join.SqlJoin.get_to_and_from_keys"></a>
 #### get\_to\_and\_from\_keys
 
 ```python
- | @staticmethod
+ | get_to_and_from_keys() -> Tuple[str, str]
- | get_to_and_from_keys(join_params: JoinParameters, is_multi: bool, model_cls: Type["Model"], part: str) -> Tuple[str, str]
 ```
 
 Based on the relation type, name of the relation and previous models and parts
 stored in JoinParameters it resolves the current to and from keys, which are
-different for ManyToMany relation, ForeignKey and reverse part of relations.
+different for ManyToMany relation, ForeignKey and reverse related of relations.
-
-**Arguments**:
-
-- `join_params (JoinParameters)`: parameters from previous/ current join
-- `is_multi (bool)`: flag if the relation is of m2m type
-- `model_cls (Type[Model])`: ormar model class
-- `part (str)`: name of the current relation join
 
 **Returns**:
 

docs/api/query-set/prefetch-query.md

@@ -289,7 +289,7 @@ models.
  | _get_select_related_if_apply(related: str, select_dict: Dict) -> Dict
 ```
 
-Extract nested part of select_related dictionary to extract models nested
+Extract nested related of select_related dictionary to extract models nested
 deeper on related model and already loaded in select related query.
 
 **Arguments**:
@@ -299,7 +299,7 @@ deeper on related model and already loaded in select related query.
 
 **Returns**:
 
-`(Dict)`: dictionary with nested part of select related
+`(Dict)`: dictionary with nested related of select related
 
 <a name="queryset.prefetch_query.PrefetchQuery._update_already_loaded_rows"></a>
 #### \_update\_already\_loaded\_rows
@@ -320,7 +320,7 @@ Updates models that are already loaded, usually children of children.
 #### \_populate\_rows
 
 ```python
- | _populate_rows(rows: List, target_field: Type["BaseField"], parent_model: Type["Model"], table_prefix: str, fields: Union[Set[Any], Dict[Any, Any], None], exclude_fields: Union[Set[Any], Dict[Any, Any], None], prefetch_dict: Dict, orders_by: Dict) -> None
+ | _populate_rows(rows: List, target_field: Type["ForeignKeyField"], parent_model: Type["Model"], table_prefix: str, fields: Union[Set[Any], Dict[Any, Any], None], exclude_fields: Union[Set[Any], Dict[Any, Any], None], prefetch_dict: Dict, orders_by: Dict) -> None
 ```
 
 Instantiates children models extracted from given relation.

docs/api/query-set/query-set.md

@@ -444,6 +444,25 @@ each=True flag to affect whole table.
 
 `(int)`: number of deleted rows
 
+<a name="queryset.queryset.QuerySet.paginate"></a>
+#### paginate
+
+```python
+ | paginate(page: int, page_size: int = 20) -> "QuerySet"
+```
+
+You can paginate the result which is a combination of offset and limit clauses.
+Limit is set to page size and offset is set to (page-1) * page_size.
+
+**Arguments**:
+
+- `page_size (int)`: numbers of items per page
+- `page (int)`: page number
+
+**Returns**:
+
+`(QuerySet)`: QuerySet
+
 <a name="queryset.queryset.QuerySet.limit"></a>
 #### limit
 

docs/api/query-set/utils.md

@@ -150,3 +150,22 @@ with all children models under their relation keys.
 
 `(Dict)`: dictionary of lists f related models
 
+<a name="queryset.utils.get_relationship_alias_model_and_str"></a>
+#### get\_relationship\_alias\_model\_and\_str
+
+```python
+get_relationship_alias_model_and_str(source_model: Type["Model"], related_parts: List) -> Tuple[str, Type["Model"], str]
+```
+
+Walks the relation to retrieve the actual model on which the clause should be
+constructed, extracts alias based on last relation leading to target model.
+
+**Arguments**:
+
+- `related_parts (Union[List, List[str]])`: list of related names extracted from string
+- `source_model (Type[Model])`: model from which relation starts
+
+**Returns**:
+
+`(Tuple[str, Type["Model"], str])`: table prefix, target model and relation string
+

docs/api/relations/alias-manager.md

@@ -74,7 +74,7 @@ Creates text clause with table name with aliased name.
 #### add\_relation\_type
 
 ```python
- | add_relation_type(source_model: Type["Model"], relation_name: str, reverse_name: str = None, is_multi: bool = False) -> None
+ | add_relation_type(source_model: Type["Model"], relation_name: str, reverse_name: str = None) -> None
 ```
 
 Registers the relations defined in ormar models.
@@ -94,12 +94,28 @@ on one model as well as from multiple different models in one join.
 - `source_model (source Model)`: model with relation defined
 - `relation_name (str)`: name of the relation to define
 - `reverse_name (Optional[str])`: name of related_name fo given relation for m2m relations
-- `is_multi (bool)`: flag if relation being registered is a through m2m model
 
 **Returns**:
 
 `(None)`: none
 
+<a name="relations.alias_manager.AliasManager.add_alias"></a>
+#### add\_alias
+
+```python
+ | add_alias(alias_key: str) -> str
+```
+
+Adds alias to the dictionary of aliases under given key.
+
+**Arguments**:
+
+- `alias_key (str)`: key of relation to generate alias for
+
+**Returns**:
+
+`(str)`: generated alias
+
 <a name="relations.alias_manager.AliasManager.resolve_relation_alias"></a>
 #### resolve\_relation\_alias
 

docs/api/relations/queryset-proxy.md

@@ -416,6 +416,27 @@ Actual call delegated to QuerySet.
 
 `(QuerysetProxy)`: QuerysetProxy
 
+<a name="relations.querysetproxy.QuerysetProxy.paginate"></a>
+#### paginate
+
+```python
+ | paginate(page: int, page_size: int = 20) -> "QuerysetProxy"
+```
+
+You can paginate the result which is a combination of offset and limit clauses.
+Limit is set to page size and offset is set to (page-1) * page_size.
+
+Actual call delegated to QuerySet.
+
+**Arguments**:
+
+- `page_size (int)`: numbers of items per page
+- `page (int)`: page number
+
+**Returns**:
+
+`(QuerySet)`: QuerySet
+
 <a name="relations.querysetproxy.QuerysetProxy.limit"></a>
 #### limit
 

docs/api/relations/relation-manager.md

@@ -98,7 +98,7 @@ Returns the actual relation and not the related model(s).
 
 ```python
  | @staticmethod
- | add(parent: "Model", child: "Model", child_name: str, virtual: bool, relation_name: str) -> None
+ | add(parent: "Model", child: "Model", field: Type["ForeignKeyField"]) -> None
 ```
 
 Adds relation on both sides -> meaning on both child and parent models.
@@ -112,9 +112,7 @@ on both ends.
 
 - `parent (Model)`: parent model on which relation should be registered
 - `child (Model)`: child model to register
-- `child_name (str)`: potential child name used if related name is not set
+- `field (ForeignKeyField)`: field with relation definition
-- `virtual (bool)`: 
-- `relation_name (str)`: name of the relation
 
 <a name="relations.relation_manager.RelationsManager.remove"></a>
 #### remove

docs/api/relations/relation-proxy.md

@@ -114,7 +114,7 @@ to the parent model only, without need for user to filter them.
  | async remove(item: "Model", keep_reversed: bool = True) -> None
 ```
 
-Removes the item from relation with parent.
+Removes the related from relation with parent.
 
 Through models are automatically deleted for m2m relations.
 

docs/api/relations/utils.md

@@ -5,7 +5,7 @@
 #### get\_relations\_sides\_and\_names
 
 ```python
-get_relations_sides_and_names(to_field: Type[BaseField], parent: "Model", child: "Model", child_name: str, virtual: bool, relation_name: str) -> Tuple["Model", "Model", str, str]
+get_relations_sides_and_names(to_field: Type[ForeignKeyField], parent: "Model", child: "Model") -> Tuple["Model", "Model", str, str]
 ```
 
 Determines the names of child and parent relations names, as well as
@@ -13,12 +13,9 @@ changes one of the sides of the relation into weakref.proxy to model.
 
 **Arguments**:
 
-- `to_field (BaseField)`: field with relation definition
+- `to_field (ForeignKeyField)`: field with relation definition
 - `parent (Model)`: parent model
 - `child (Model)`: child model
-- `child_name (str)`: name of the child
-- `virtual (bool)`: flag if relation is virtual
-- `relation_name ()`: 
 
 **Returns**:
 

docs/fields/common-parameters.md

@@ -121,6 +121,6 @@ Prevents insertion of value not present in the choices list.
 Used in pydantic only.
 
 [relations]: ../relations/index.md
-[queries]: ../queries.md
+[queries]: ../queries/index.md
 [pydantic]: https://pydantic-docs.helpmanual.io/usage/types/#constrained-types
 [server default]: https://docs.sqlalchemy.org/en/13/core/defaults.html#server-invoked-ddl-explicit-default-expressions

docs/models/index.md

@@ -76,7 +76,7 @@ Since it can be a function you can set `default=datetime.datetime.now` and get c
     response with `include`/`exclude` and `response_model_include`/`response_model_exclude` accordingly.
 
 ```python
-# <==part of code removed for clarity==>
+# <==related of code removed for clarity==>
 class User(ormar.Model):
     class Meta:
         tablename: str = "users2"
@@ -93,14 +93,14 @@ class User(ormar.Model):
         pydantic_only=True, default=datetime.datetime.now
     )
 
-# <==part of code removed for clarity==>
+# <==related of code removed for clarity==>
 app =FastAPI()
 
 @app.post("/users/")
 async def create_user(user: User):
     return await user.save()
 
-# <==part of code removed for clarity==>
+# <==related of code removed for clarity==>
 
 def test_excluding_fields_in_endpoints():
     client = TestClient(app)
@@ -127,7 +127,7 @@ def test_excluding_fields_in_endpoints():
         assert response.json().get("timestamp") == str(timestamp).replace(" ", "T")
 
 
-# <==part of code removed for clarity==>
+# <==related of code removed for clarity==>
 ```
 
 #### Property fields
@@ -190,7 +190,7 @@ in the response from `fastapi` and `dict()` and `json()` methods. You cannot pas
     ```
     
 ```python
-# <==part of code removed for clarity==>
+# <==related of code removed for clarity==>
 def gen_pass():  # note: NOT production ready 
     choices = string.ascii_letters + string.digits + "!@#$%^&*()"
     return "".join(random.choice(choices) for _ in range(20))
@@ -215,7 +215,7 @@ class RandomModel(ormar.Model):
     def full_name(self) -> str:
         return " ".join([self.first_name, self.last_name])
 
-# <==part of code removed for clarity==>
+# <==related of code removed for clarity==>
 app =FastAPI()
 
 # explicitly exclude property_field in this endpoint
@@ -223,7 +223,7 @@ app =FastAPI()
 async def create_user(user: RandomModel):
     return await user.save()
 
-# <==part of code removed for clarity==>
+# <==related of code removed for clarity==>
 
 def test_excluding_property_field_in_endpoints2():
     client = TestClient(app)
@@ -241,7 +241,7 @@ def test_excluding_property_field_in_endpoints2():
         # despite being decorated with property_field if you explictly exclude it it will be gone
         assert response.json().get("full_name") is None
 
-# <==part of code removed for clarity==>
+# <==related of code removed for clarity==>
 ```
 
 #### Fields names vs Column names
@@ -423,7 +423,7 @@ You can check if model is saved with `ModelInstance.saved` property
 
 [fields]: ../fields/field-types.md
 [relations]: ../relations/index.md
-[queries]: ../queries.md
+[queries]: ../queries/index.md
 [pydantic]: https://pydantic-docs.helpmanual.io/
 [sqlalchemy-core]: https://docs.sqlalchemy.org/en/latest/core/
 [sqlalchemy-metadata]: https://docs.sqlalchemy.org/en/13/core/metadata.html

docs/models/methods.md

@@ -65,7 +65,7 @@ await track.update(name='The Bird Strikes Again')
 
 `upsert(**kwargs) -> self`
 
-It's an proxy to either `save()` or `update(**kwargs)` methods described above.
+It's a proxy to either `save()` or `update(**kwargs)` methods described above.
 
 If the primary key is set -> the `update` method will be called.
 
@@ -118,7 +118,7 @@ But you can specify the `follow=True` parameter to traverse through nested model
 
 [fields]: ../fields.md
 [relations]: ../relations/index.md
-[queries]: ../queries.md
+[queries]: ../queries/index.md
 [pydantic]: https://pydantic-docs.helpmanual.io/
 [sqlalchemy-core]: https://docs.sqlalchemy.org/en/latest/core/
 [sqlalchemy-metadata]: https://docs.sqlalchemy.org/en/13/core/metadata.html

docs/queries.md

@@ -1,718 +0,0 @@
-# Queries
-
-## QuerySet
-
-Each Model is auto registered with a `QuerySet` that represents the underlaying query and it's options.
-
-Most of the methods are also available through many to many relation interface.
-
-!!!info
-    To see which one are supported and how to construct relations visit [relations][relations].
-
-Given the Models like this
-
-```Python 
---8<-- "../docs_src/queries/docs001.py"
-```
-
-we can demonstrate available methods to fetch and save the data into the database.
-
-
-### create
-
-`create(**kwargs): -> Model` 
-
-Creates the model instance, saves it in a database and returns the updates model 
-(with pk populated if not passed and autoincrement is set).
-
-The allowed kwargs are `Model` fields names and proper value types. 
-
-```python
-malibu = await Album.objects.create(name="Malibu")
-await Track.objects.create(album=malibu, title="The Bird", position=1)
-```
-
-The alternative is a split creation and persistence of the `Model`.
-```python
-malibu = Album(name="Malibu")
-await malibu.save()
-```
-
-!!!tip
-    Check other `Model` methods in [models][models]
-
-### get
-
-`get(**kwargs): -> Model` 
-
-Get's the first row from the db meeting the criteria set by kwargs.
-
-If no criteria set it will return the last row in db sorted by pk.
-
-Passing a criteria is actually calling filter(**kwargs) method described below.
-
-```python
-track = await Track.objects.get(name='The Bird')
-# note that above is equivalent to await Track.objects.filter(name='The Bird').get()
-track2 = track = await Track.objects.get()
-track == track2 # True since it's the only row in db in our example
-```
-
-!!!warning
-    If no row meets the criteria `NoMatch` exception is raised.
-    
-    If there are multiple rows meeting the criteria the `MultipleMatches` exception is raised.
-
-### get_or_create
-
-`get_or_create(**kwargs) -> Model`
-
-Combination of create and get methods.
-
-Tries to get a row meeting the criteria and if `NoMatch` exception is raised it creates a new one with given kwargs.
-
-```python
-album = await Album.objects.get_or_create(name='The Cat')
-# object is created as it does not exist
-album2 = await Album.objects.get_or_create(name='The Cat')
-assert album == album2
-# return True as the same db row is returned
-```
-
-!!!warning
-    Despite being a equivalent row from database the `album` and `album2` in example above are 2 different python objects!
-    Updating one of them will not refresh the second one until you excplicitly load() the fresh data from db.
-
-!!!note
-    Note that if you want to create a new object you either have to pass pk column value or pk column has to be set as autoincrement
-
-### first
-
-`first(): -> Model` 
-
-Gets the first row from the db ordered by primary key column ascending.
-
-
-### update
-
-`update(each: bool = False, **kwargs) -> int`
-
-QuerySet level update is used to update multiple records with the same value at once.
-
-You either have to filter the QuerySet first or provide a `each=True` flag to update whole table.
-
-If you do not provide this flag or a filter a `QueryDefinitionError` will be raised.
-
-Return number of rows updated.
-
-```Python hl_lines="26-28"
---8<-- "../docs_src/queries/docs002.py"
-```
-
-!!!warning
-    Queryset needs to be filtered before updating to prevent accidental overwrite.
-    
-    To update whole database table `each=True` needs to be provided as a safety switch
-    
-
-### update_or_create
-
-`update_or_create(**kwargs) -> Model`
-
-Updates the model, or in case there is no match in database creates a new one.
-
-```Python hl_lines="26-32"
---8<-- "../docs_src/queries/docs003.py"
-```
-
-!!!note
-    Note that if you want to create a new object you either have to pass pk column value or pk column has to be set as autoincrement
-
-
-### bulk_create
-
-`bulk_create(objects: List["Model"]) -> None`
-
-Allows you to create multiple objects at once.
-
-A valid list of `Model` objects needs to be passed.
-
-```python hl_lines="21-27"
---8<-- "../docs_src/queries/docs004.py"
-```
-
-### bulk_update
-
-`bulk_update(objects: List["Model"], columns: List[str] = None) -> None`
-
-Allows to update multiple instance at once.
-
-All `Models` passed need to have primary key column populated.
-
-You can also select which fields to update by passing `columns` list as a list of string names.
-
-```python hl_lines="8"
-# continuing the example from bulk_create
-# update objects
-for todo in todoes:
-    todo.completed = False
-
-# perform update of all objects at once
-# objects need to have pk column set, otherwise exception is raised
-await ToDo.objects.bulk_update(todoes)
-
-completed = await ToDo.objects.filter(completed=False).all()
-assert len(completed) == 3
-```
-
-### delete
-
-`delete(each: bool = False, **kwargs) -> int`
-
-QuerySet level delete is used to delete multiple records at once.
-
-You either have to filter the QuerySet first or provide a `each=True` flag to delete whole table.
-
-If you do not provide this flag or a filter a `QueryDefinitionError` will be raised.
-
-Return number of rows deleted.
-
-```python hl_lines="26-30"
---8<-- "../docs_src/queries/docs005.py"
-```
-
-### all
-
-`all(**kwargs) -> List[Optional["Model"]]`
-
-Returns all rows from a database for given model for set filter options.
-
-Passing kwargs is a shortcut and equals to calling `filter(**kwrags).all()`.
-
-If there are no rows meeting the criteria an empty list is returned.
-
-```python
-tracks = await Track.objects.select_related("album").all(title='Sample')
-# will return a list of all Tracks with title Sample
-
-tracks = await Track.objects.all()
-# will return a list of all Tracks in database
-
-```
-
-### filter
-
-`filter(**kwargs) -> QuerySet`
-
-Allows you to filter by any `Model` attribute/field 
-as well as to fetch instances, with a filter across an FK relationship.
-
-```python
-track = Track.objects.filter(name="The Bird").get()
-# will return a track with name equal to 'The Bird'
- 
-tracks = Track.objects.filter(album__name="Fantasies").all()
-# will return all tracks where the columns album name = 'Fantasies'
-```
-
-You can use special filter suffix to change the filter operands:
-
-*  exact - like `album__name__exact='Malibu'` (exact match)
-*  iexact - like `album__name__iexact='malibu'` (exact match case insensitive)
-*  contains - like `album__name__contains='Mal'` (sql like)
-*  icontains - like `album__name__icontains='mal'` (sql like case insensitive)
-*  in - like `album__name__in=['Malibu', 'Barclay']` (sql in)
-*  gt - like `position__gt=3` (sql >)
-*  gte - like `position__gte=3` (sql >=)
-*  lt - like `position__lt=3` (sql <)
-*  lte - like `position__lte=3` (sql <=)
-*  startswith - like `album__name__startswith='Mal'` (exact start match)
-*  istartswith - like `album__name__istartswith='mal'` (exact start match case insensitive)
-*  endswith - like `album__name__endswith='ibu'` (exact end match)
-*  iendswith - like `album__name__iendswith='IBU'` (exact end match case insensitive)
-
-!!!note
-    All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together
-    
-    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
-    
-    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
-
-### exclude
-
-`exclude(**kwargs) -> QuerySet`
-
-Works exactly the same as filter and all modifiers (suffixes) are the same, but returns a not condition.
-
-So if you use `filter(name='John')` which equals to `where name = 'John'` in SQL, 
-the `exclude(name='John')` equals to `where name <> 'John'`
-
-Note that all conditions are joined so if you pass multiple values it becomes a union of conditions.
-
-`exclude(name='John', age>=35)` will become `where not (name='John' and age>=35)`
-
-```python
-notes = await Track.objects.exclude(position_gt=3).all()
-# returns all tracks with position < 3
-```
-
-### select_related
-
-`select_related(related: Union[List, str]) -> QuerySet`
-
-Allows to prefetch related models during the same query. 
-
-**With `select_related` always only one query is run against the database**, meaning that one 
-(sometimes complicated) join is generated and later nested models are processed in python.
-
-To fetch related model use `ForeignKey` names.
-
-To chain related `Models` relation use double underscores between names.
-
-!!!note
-    If you are coming from `django` note that `ormar` `select_related` differs -> in `django` you can `select_related`
-    only singe relation types, while in `ormar` you can select related across `ForeignKey` relation, 
-    reverse side of `ForeignKey` (so virtual auto generated keys) and `ManyToMany` fields (so all relations as of current version).
-
-!!!tip
-    To control which model fields to select use `fields()` and `exclude_fields()` `QuerySet` methods.
-    
-!!!tip
-    To control order of models (both main or nested) use `order_by()` method.
-
-```python
-album = await Album.objects.select_related("tracks").all()
-# will return album will all columns tracks
-```
-
-You can provide a string or a list of strings
-
-```python
-classes = await SchoolClass.objects.select_related(
-["teachers__category", "students"]).all()
-# will return classes with teachers and teachers categories
-# as well as classes students
-```
-
-Exactly the same behavior is for Many2Many fields, where you put the names of Many2Many fields and the final `Models` are fetched for you.
-
-!!!warning
-    If you set `ForeignKey` field as not nullable (so required) during 
-    all queries the not nullable `Models` will be auto prefetched, even if you do not include them in select_related.
-
-!!!note
-    All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together
-    
-    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
-    
-    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
-
-### prefetch_related
-
-`prefetch_related(related: Union[List, str]) -> QuerySet`
-
-Allows to prefetch related models during query - but opposite to `select_related` each 
-subsequent model is fetched in a separate database query. 
-
-**With `prefetch_related` always one query per Model is run against the database**, 
-meaning that you will have multiple queries executed one after another.
-
-To fetch related model use `ForeignKey` names.
-
-To chain related `Models` relation use double underscores between names.
-
-!!!tip
-    To control which model fields to select use `fields()` and `exclude_fields()` `QuerySet` methods.
-    
-!!!tip
-    To control order of models (both main or nested) use `order_by()` method.
-
-```python
-album = await Album.objects.prefetch_related("tracks").all()
-# will return album will all columns tracks
-```
-
-You can provide a string or a list of strings
-
-```python
-classes = await SchoolClass.objects.prefetch_related(
-["teachers__category", "students"]).all()
-# will return classes with teachers and teachers categories
-# as well as classes students
-```
-
-Exactly the same behavior is for Many2Many fields, where you put the names of Many2Many fields and the final `Models` are fetched for you.
-
-!!!warning
-    If you set `ForeignKey` field as not nullable (so required) during 
-    all queries the not nullable `Models` will be auto prefetched, even if you do not include them in select_related.
-
-!!!note
-    All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together
-    
-    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
-    
-    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
-
-### select_related vs prefetch_related
-
-Which should you use -> `select_related` or `prefetch_related`?
-
-Well, it really depends on your data. The best answer is try yourself and see which one performs faster/better in your system constraints.
-
-What to keep in mind:
-
-#### Performance
-
-**Number of queries**:
-`select_related` always executes one query against the database, while `prefetch_related` executes multiple queries. 
-Usually the query (I/O) operation is the slowest one but it does not have to be.
-
-**Number of rows**:
-Imagine that you have 10 000 object in one table A and each of those objects have 3 children in table B, 
-and subsequently each object in table B has 2 children in table C. Something like this:
-
-```
-                     Model C
-                   /
-           Model B - Model C
-         / 
-Model A  - Model B - Model C
-       \           \ 
-        \            Model C
-         \
-           Model B - Model C
-                   \ 
-                     Model C
-```
-
-That means that `select_related` will always return 60 000 rows (10 000 * 3 * 2) later compacted to 10 000 models. 
-
-How many rows will return `prefetch_related`?
-
-Well, that depends, if each of models B and C is unique it will return 10 000 rows in first query, 30 000 rows 
-(each of 3 children of A in table B are unique) in second query and 60 000 rows (each of 2 children of model B 
-in table C are unique) in 3rd query. 
-
-In this case `select_related` seems like a better choice, not only it will run one query comparing to 3 of 
-`prefetch_related` but will also return 60 000 rows comparing to 100 000 of `prefetch_related` (10+30+60k).
-
-But what if each Model A has exactly the same 3 models B and each models C has exactly same models C? `select_related`
-will still return 60 000 rows, while `prefetch_related` will return 10 000 for model A, 3 rows for model B and 2 rows for Model C.
-So in total 10 006 rows. Now depending on the structure of models (i.e. if it has long Text() fields etc.) `prefetch_related`
-might be faster despite it needs to perform three separate queries instead of one. 
-
-#### Memory
-
-`ormar` is a mini ORM meaning that it does not keep a registry of already loaded models.
- 
-That means that in `select_related` example above you will always have 10 000 Models A, 30 000 Models B 
-(even if the unique number of rows in db is 3 - processing of `select_related` spawns **new** child models for each parent model).
-And 60 000 Models C. 
-
-If the same Model B is shared by rows 1, 10, 100 etc. and you update one of those, the rest of rows
-that share the same child will **not** be updated on the spot. 
-If you persist your changes into the database the change **will be available only after reload 
-(either each child separately or the whole query again)**. 
-That means that `select_related` will use more memory as each child is instantiated as a new object - obviously using it's own space.
-
-!!!note
-    This might change in future versions if we decide to introduce caching.
-
-!!!warning
-    By default all children (or event the same models loaded 2+ times) are completely independent, distinct python objects, despite that they represent the same row in db.
-    
-    They will evaluate to True when compared, so in example above: 
-    
-    ```python
-    # will return True if child1 of both rows is the same child db row 
-    row1.child1 == row100.child1
-    
-    # same here:
-    model1 = await Model.get(pk=1)
-    model2 = await Model.get(pk=1) # same pk = same row in db
-    # will return `True`
-    model1 == model2
-    ``` 
-    
-    but 
-    
-    ```python
-    # will return False (note that id is a python `builtin` function not ormar one).
-    id(row1.child1) == (ro100.child1)
-    
-    # from above - will also return False
-    id(model1) == id(model2)
-    ``` 
-
-
-On the contrary - with `prefetch_related` each unique distinct child model is instantiated 
-only once and the same child models is shared across all parent models. 
-That means that in `prefetch_related` example above if there are 3 distinct models in table B and 2 in table C, 
-there will be only 5 children nested models shared between all model A instances. That also means that if you update
-any attribute it will be updated on all parents as they share the same child object.
-
-### limit
-
-`limit(limit_count: int, limit_raw_sql: bool = None) -> QuerySet`
-
-You can limit the results to desired number of parent models.
-
-To limit the actual number of database query rows instead of number of main models
-use the `limit_raw_sql` parameter flag, and set it to `True`.
-
-```python
-tracks = await Track.objects.limit(1).all()
-# will return just one Track
-```
-
-!!!note
-    All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together
-    
-    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
-    
-    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
-
-### offset
-
-`offset(offset: int, limit_raw_sql: bool = None) -> QuerySet`
-
-You can also offset the results by desired number of main models.
-
-To offset the actual number of database query rows instead of number of main models
-use the `limit_raw_sql` parameter flag, and set it to `True`.
-
-```python
-tracks = await Track.objects.offset(1).limit(1).all()
-# will return just one Track, but this time the second one
-```
-
-!!!note
-    All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together
-    
-    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
-    
-    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
-
-
-### count
-
-`count() -> int`
-
-Returns number of rows matching the given criteria (applied with `filter` and `exclude`)
-
-```python
-# returns count of rows in db
-no_of_books = await Book.objects.count()
-```
-
-### exists
-
-`exists() -> bool`
-
-Returns a bool value to confirm if there are rows matching the given criteria (applied with `filter` and `exclude`)
-
-```python
-# returns a boolean value if given row exists
-has_sample = await Book.objects.filter(title='Sample').exists()
-```
-
-### fields
-
-`fields(columns: Union[List, str, set, dict]) -> QuerySet`
-
-With `fields()` you can select subset of model columns to limit the data load.
-
-!!!note
-    Note that `fields()` and `exclude_fields()` works both for main models (on normal queries like `get`, `all` etc.) 
-    as well as `select_related` and `prefetch_related` models (with nested notation).
-
-Given a sample data like following:
-
-```python
---8<-- "../docs_src/queries/docs006.py"
-```
-
-You can select specified fields by passing a `str, List[str], Set[str] or dict` with nested definition.
-
-To include related models use notation `{related_name}__{column}[__{optional_next} etc.]`.
-
-```python hl_lines="1"
-all_cars = await Car.objects.select_related('manufacturer').fields(['id', 'name', 'manufacturer__name']).all()
-for car in all_cars:
-    # excluded columns will yield None
-    assert all(getattr(car, x) is None for x in ['year', 'gearbox_type', 'gears', 'aircon_type'])
-    # included column on related models will be available, pk column is always included
-    # even if you do not include it in fields list
-    assert car.manufacturer.name == 'Toyota'
-    # also in the nested related models - you cannot exclude pk - it's always auto added
-    assert car.manufacturer.founded is None
-```
-
-`fields()` can be called several times, building up the columns to select.
-
-If you include related models into `select_related()` call but you won't specify columns for those models in fields 
-- implies a list of all fields for those nested models.
-
-```python hl_lines="1"
-all_cars = await Car.objects.select_related('manufacturer').fields('id').fields(
-    ['name']).all()
-# all fiels from company model are selected
-assert all_cars[0].manufacturer.name == 'Toyota'
-assert all_cars[0].manufacturer.founded == 1937
-```
-
-!!!warning
-    Mandatory fields cannot be excluded as it will raise `ValidationError`, to exclude a field it has to be nullable.
-
-You cannot exclude mandatory model columns - `manufacturer__name` in this example.
-
-```python
-await Car.objects.select_related('manufacturer').fields(['id', 'name', 'manufacturer__founded']).all()
-# will raise pydantic ValidationError as company.name is required
-```
-
-!!!tip
-    Pk column cannot be excluded - it's always auto added even if not explicitly included.
-
-You can also pass fields to include as dictionary or set.
-
-To mark a field as included in a dictionary use it's name as key and ellipsis as value.
-
-To traverse nested models use nested dictionaries.
-
-To include fields at last level instead of nested dictionary a set can be used.
-
-To include whole nested model specify model related field name and ellipsis.
-
-Below you can see examples that are equivalent:
-
-```python
---8<-- "../docs_src/queries/docs009.py"
-```
-
-!!!note
-    All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together
-    
-    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
-    
-    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
-
-### exclude_fields
-
-`exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet`
-
-With `exclude_fields()` you can select subset of model columns that will be excluded to limit the data load.
-
-It's the opposite of `fields()` method so check documentation above to see what options are available.
-
-Especially check above how you can pass also nested dictionaries and sets as a mask to exclude fields from whole hierarchy.
-
-!!!note
-    Note that `fields()` and `exclude_fields()` works both for main models (on normal queries like `get`, `all` etc.) 
-    as well as `select_related` and `prefetch_related` models (with nested notation).
-
-Below you can find few simple examples:
-
-```python hl_lines="47 48 60 61 67"
---8<-- "../docs_src/queries/docs008.py"
-```
-
-!!!warning
-    Mandatory fields cannot be excluded as it will raise `ValidationError`, to exclude a field it has to be nullable.
-
-!!!tip
-    Pk column cannot be excluded - it's always auto added even if explicitly excluded.
-
-
-!!!note
-    All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together
-    
-    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
-    
-    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
-
-
-### order_by
-
-`order_by(columns: Union[List, str]) -> QuerySet`
-
-With `order_by()` you can order the results from database based on your choice of fields.
-
-You can provide a string with field name or list of strings with different fields.
-
-Ordering in sql will be applied in order of names you provide in order_by.
-
-!!!tip
-    By default if you do not provide ordering `ormar` explicitly orders by all primary keys
-    
-!!!warning
-    If you are sorting by nested models that causes that the result rows are unsorted by the main model
-    `ormar` will combine those children rows into one main model.
-    
-    Sample raw database rows result (sort by child model desc):
-    ```
-    MODEL: 1 - Child Model - 3
-    MODEL: 2 - Child Model - 2
-    MODEL: 1 - Child Model - 1
-    ```
-    
-    will result in 2 rows of result:
-    ```
-    MODEL: 1 - Child Models: [3, 1] # encountered first in result, all children rows combined
-    MODEL: 2 - Child Modles: [2]
-    ```
-    
-    The main model will never duplicate in the result
-
-Given sample Models like following:
-    
-```python
---8<-- "../docs_src/queries/docs007.py"
-```
-
-To order by main model field just provide a field name
-
-```python
-toys = await Toy.objects.select_related("owner").order_by("name").all()
-assert [x.name.replace("Toy ", "") for x in toys] == [
-    str(x + 1) for x in range(6)
-]
-assert toys[0].owner == zeus
-assert toys[1].owner == aphrodite
-```
-
-To sort on nested models separate field names with dunder '__'.
-
-You can sort this way across all relation types -> `ForeignKey`, reverse virtual FK and `ManyToMany` fields.
-
-```python
-toys = await Toy.objects.select_related("owner").order_by("owner__name").all()
-assert toys[0].owner.name == toys[1].owner.name == "Aphrodite"
-assert toys[2].owner.name == toys[3].owner.name == "Hermes"
-assert toys[4].owner.name == toys[5].owner.name == "Zeus"
-```
-
-To sort in descending order provide a hyphen in front of the field name
-
-```python
-owner = (
-    await Owner.objects.select_related("toys")
-        .order_by("-toys__name")
-        .filter(name="Zeus")
-        .get()
-)
-assert owner.toys[0].name == "Toy 4"
-assert owner.toys[1].name == "Toy 1"
-```
-
-!!!note
-    All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together
-    
-    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
-    
-    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
-
-
-[models]: ./models/index.md
-[relations]: ./relations/index.md

docs/queries/aggregations.md

@@ -0,0 +1,94 @@
+# Aggregation functions
+
+Currently 2 aggregation functions are supported.
+
+
+* `count() -> int`
+* `exists() -> bool`
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.count()` method
+    * `QuerysetProxy.exists()` method
+  
+
+## count
+
+`count() -> int`
+
+Returns number of rows matching the given criteria (i.e. applied with `filter` and `exclude`)
+
+```python
+class Book(ormar.Model):
+    class Meta:
+        tablename = "books"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    title: str = ormar.String(max_length=200)
+    author: str = ormar.String(max_length=100)
+    genre: str = ormar.String(
+        max_length=100,
+        default="Fiction",
+        choices=["Fiction", "Adventure", "Historic", "Fantasy"],
+    )
+```
+
+```python
+# returns count of rows in db for Books model
+no_of_books = await Book.objects.count()
+```
+
+## exists
+
+`exists() -> bool`
+
+Returns a bool value to confirm if there are rows matching the given criteria (applied with `filter` and `exclude`)
+
+```python
+class Book(ormar.Model):
+    class Meta:
+        tablename = "books"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    title: str = ormar.String(max_length=200)
+    author: str = ormar.String(max_length=100)
+    genre: str = ormar.String(
+        max_length=100,
+        default="Fiction",
+        choices=["Fiction", "Adventure", "Historic", "Fantasy"],
+    )
+```
+
+```python
+# returns a boolean value if given row exists
+has_sample = await Book.objects.filter(title='Sample').exists()
+```
+
+## QuerysetProxy methods
+
+When access directly the related `ManyToMany` field as well as `ReverseForeignKey`
+returns the list of related models.
+
+But at the same time it exposes a subset of QuerySet API, so you can filter, create,
+select related etc related models directly from parent model.
+
+### count
+
+Works exactly the same as [count](./#count) function above but allows you to select columns from related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+### exists
+
+Works exactly the same as [exists](./#exists) function above but allows you to select columns from related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+

docs/queries/create.md

@@ -0,0 +1,185 @@
+# Insert data into database
+
+Following methods allow you to insert data into the database.
+
+* `create(**kwargs) -> Model`
+* `get_or_create(**kwargs) -> Model`
+* `update_or_create(**kwargs) -> Model`
+* `bulk_create(objects: List[Model]) -> None`
+
+
+* `Model`
+      * `Model.save()` method
+      * `Model.upsert()` method
+      * `Model.save_related()` method
+
+
+* `QuerysetProxy`
+      * `QuerysetProxy.create(**kwargs)` method
+      * `QuerysetProxy.get_or_create(**kwargs)` method
+      * `QuerysetProxy.update_or_create(**kwargs)` method
+
+## create
+
+`create(**kwargs): -> Model`
+
+Creates the model instance, saves it in a database and returns the updates model
+(with pk populated if not passed and autoincrement is set).
+
+The allowed kwargs are `Model` fields names and proper value types.
+
+```python
+class Album(ormar.Model):
+    class Meta:
+        tablename = "album"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+```
+
+```python
+malibu = await Album.objects.create(name="Malibu")
+await Track.objects.create(album=malibu, title="The Bird", position=1)
+```
+
+The alternative is a split creation and persistence of the `Model`.
+
+```python
+malibu = Album(name="Malibu")
+await malibu.save()
+```
+
+!!!tip 
+        Check other `Model` methods in [models][models]
+
+## get_or_create
+
+`get_or_create(**kwargs) -> Model`
+
+Combination of create and get methods.
+
+Tries to get a row meeting the criteria and if `NoMatch` exception is raised it creates
+a new one with given kwargs.
+
+```python
+class Album(ormar.Model):
+    class Meta:
+        tablename = "album"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+```
+
+```python
+album = await Album.objects.get_or_create(name='The Cat')
+# object is created as it does not exist
+album2 = await Album.objects.get_or_create(name='The Cat')
+assert album == album2
+# return True as the same db row is returned
+```
+
+!!!warning 
+    Despite being a equivalent row from database the `album` and `album2` in
+    example above are 2 different python objects!
+    Updating one of them will not refresh the second one until you excplicitly load() the
+    fresh data from db.
+
+!!!note 
+    Note that if you want to create a new object you either have to pass pk column
+    value or pk column has to be set as autoincrement
+
+## update_or_create
+
+`update_or_create(**kwargs) -> Model`
+
+Updates the model, or in case there is no match in database creates a new one.
+
+```Python hl_lines="26-32"
+--8<-- "../docs_src/queries/docs003.py"
+```
+
+!!!note 
+    Note that if you want to create a new object you either have to pass pk column
+    value or pk column has to be set as autoincrement
+
+## bulk_create
+
+`bulk_create(objects: List["Model"]) -> None`
+
+Allows you to create multiple objects at once.
+
+A valid list of `Model` objects needs to be passed.
+
+```python hl_lines="21-27"
+--8<-- "../docs_src/queries/docs004.py"
+```
+
+## Model methods
+
+Each model instance have a set of methods to `save`, `update` or `load` itself.
+
+###save
+
+You can create new models by using `QuerySet.create()` method or by initializing your model as a normal pydantic model 
+and later calling `save()` method.
+
+!!!tip
+    Read more about `save()` method in [models-save][models-save]
+
+###upsert
+
+It's a proxy to either `save()` or `update(**kwargs)` methods of a Model.
+If the pk is not set the `save()` method will be called.
+
+!!!tip
+    Read more about `upsert()` method in [models-upsert][models-upsert]
+
+###save_related
+
+Method goes through all relations of the `Model` on which the method is called, 
+and calls `upsert()` method on each model that is **not** saved. 
+
+!!!tip
+    Read more about `save_related()` method in [models-save-related][models-save-related]
+
+## QuerysetProxy methods
+
+When access directly the related `ManyToMany` field as well as `ReverseForeignKey` returns the list of related models.
+
+But at the same time it exposes subset of QuerySet API, so you can filter, create, select related etc related models directly from parent model.
+
+### create
+
+Works exactly the same as [create](./#create) function above but allows you to create related objects
+from other side of the relation.
+
+!!!tip
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+
+### get_or_create
+
+Works exactly the same as [get_or_create](./#get_or_create) function above but allows you to query or create related objects
+from other side of the relation.
+
+!!!tip
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+
+### update_or_create
+
+Works exactly the same as [update_or_create](./#update_or_create) function above but allows you to update or create related objects
+from other side of the relation.
+
+!!!tip
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+[models]: ../models/methods.md
+[models-save]: ../models/methods.md#save
+[models-upsert]: ../models/methods.md#upsert
+[models-save-related]: ../models/methods.md#save_related
+[querysetproxy]: ../relations/queryset-proxy.md

docs/queries/delete.md

@@ -0,0 +1,151 @@
+# Delete data from database
+
+Following methods allow you to delete data from the database.
+
+* `delete(each: bool = False, **kwargs) -> int`
+
+
+* `Model`
+    * `Model.delete()` method
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.remove()` method
+    * `QuerysetProxy.clear()` method
+
+## delete
+
+`delete(each: bool = False, **kwargs) -> int`
+
+QuerySet level delete is used to delete multiple records at once.
+
+You either have to filter the QuerySet first or provide a `each=True` flag to delete
+whole table.
+
+If you do not provide this flag or a filter a `QueryDefinitionError` will be raised.
+
+Return number of rows deleted.
+
+```python hl_lines="26-30"
+--8<-- "../docs_src/queries/docs005.py"
+```
+
+## Model methods
+
+Each model instance have a set of methods to `save`, `update` or `load` itself.
+
+### delete
+
+You can delete model instance by calling `delete()` method on it.
+
+!!!tip
+    Read more about `delete()` method in [models methods](../models/methods.md#delete)
+
+## QuerysetProxy methods
+
+When access directly the related `ManyToMany` field as well as `ReverseForeignKey`
+returns the list of related models.
+
+But at the same time it exposes subset of QuerySet API, so you can filter, create,
+select related etc related models directly from parent model.
+
+### remove
+
+Removal of the related model one by one.
+
+Removes the relation in the database.
+
+If you specify the keep_reversed flag to `False` `ormar` will also delete the related model from the database.
+
+```python
+class Album(ormar.Model):
+    class Meta:
+        tablename = "albums"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    is_best_seller: bool = ormar.Boolean(default=False)
+
+class Track(ormar.Model):
+    class Meta:
+        tablename = "tracks"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    album: Optional[Album] = ormar.ForeignKey(Album)
+    title: str = ormar.String(max_length=100)
+    position: int = ormar.Integer()
+    play_count: int = ormar.Integer(nullable=True)
+```
+
+```python
+album = await Album(name="Malibu").save()
+track1 = await Track(
+    album=album, title="The Bird", position=1, play_count=30, 
+).save()
+# remove through proxy from reverse side of relation
+await album.tracks.remove(track1, keep_reversed=False)
+
+# the track was also deleted
+tracks = await Track.objects.all()
+assert len(tracks) == 0
+```
+
+### clear
+
+Removal of all related models in one call.
+
+Removes also the relation in the database.
+
+If you specify the keep_reversed flag to `False` `ormar` will also delete the related model from the database.
+
+```python
+class Album(ormar.Model):
+    class Meta:
+        tablename = "albums"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    is_best_seller: bool = ormar.Boolean(default=False)
+
+class Track(ormar.Model):
+    class Meta:
+        tablename = "tracks"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    album: Optional[Album] = ormar.ForeignKey(Album)
+    title: str = ormar.String(max_length=100)
+    position: int = ormar.Integer()
+    play_count: int = ormar.Integer(nullable=True)
+```
+
+```python
+album = await Album(name="Malibu").save()
+track1 = await Track(
+    album=album, 
+    title="The Bird", 
+    position=1, 
+    play_count=30, 
+).save()
+track2 = await Track(
+    album=album,
+    title="Heart don't stand a chance",
+    position=2,
+    play_count=20,
+).save()
+
+# removes the relation only -> clears foreign keys on tracks
+await album.tracks.clear()
+
+# removes also the tracks
+await album.tracks.clear(keep_reversed=False)
+```
+
+[querysetproxy]: ../relations/queryset-proxy.md

docs/queries/filter-and-sort.md

@@ -0,0 +1,337 @@
+# Filtering and sorting data
+
+You can use following methods to filter the data (sql where clause).
+
+* `filter(**kwargs) -> QuerySet`
+* `exclude(**kwargs) -> QuerySet`
+* `get(**kwargs) -> Model`
+* `get_or_create(**kwargs) -> Model`
+* `all(**kwargs) -> List[Optional[Model]]`
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.filter(**kwargs)` method
+    * `QuerysetProxy.exclude(**kwargs)` method
+    * `QuerysetProxy.get(**kwargs)` method
+    * `QuerysetProxy.get_or_create(**kwargs)` method
+    * `QuerysetProxy.all(**kwargs)` method
+
+And following methods to sort the data (sql order by clause).
+
+* `order_by(columns:Union[List, str]) -> QuerySet`
+* `QuerysetProxy`
+    * `QuerysetProxy.order_by(columns:Union[List, str])` method
+
+## Filtering
+
+### filter
+
+`filter(**kwargs) -> QuerySet`
+
+Allows you to filter by any `Model` attribute/field as well as to fetch instances, with
+a filter across an FK relationship.
+
+```python
+class Album(ormar.Model):
+    class Meta:
+        tablename = "albums"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    is_best_seller: bool = ormar.Boolean(default=False)
+
+class Track(ormar.Model):
+    class Meta:
+        tablename = "tracks"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    album: Optional[Album] = ormar.ForeignKey(Album)
+    name: str = ormar.String(max_length=100)
+    position: int = ormar.Integer()
+    play_count: int = ormar.Integer(nullable=True)
+```
+
+```python
+track = Track.objects.filter(name="The Bird").get()
+# will return a track with name equal to 'The Bird'
+
+tracks = Track.objects.filter(album__name="Fantasies").all()
+# will return all tracks where the columns album name = 'Fantasies'
+```
+
+You can use special filter suffix to change the filter operands:
+
+* exact - like `album__name__exact='Malibu'` (exact match)
+* iexact - like `album__name__iexact='malibu'` (exact match case insensitive)
+* contains - like `album__name__contains='Mal'` (sql like)
+* icontains - like `album__name__icontains='mal'` (sql like case insensitive)
+* in - like `album__name__in=['Malibu', 'Barclay']` (sql in)
+* gt - like `position__gt=3` (sql >)
+* gte - like `position__gte=3` (sql >=)
+* lt - like `position__lt=3` (sql <)
+* lte - like `position__lte=3` (sql <=)
+* startswith - like `album__name__startswith='Mal'` (exact start match)
+* istartswith - like `album__name__istartswith='mal'` (exact start match case
+  insensitive)
+* endswith - like `album__name__endswith='ibu'` (exact end match)
+* iendswith - like `album__name__iendswith='IBU'` (exact end match case insensitive)
+
+!!!note 
+    All methods that do not return the rows explicitly returns a QueySet instance so
+    you can chain them together
+
+    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
+    
+    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
+
+!!!warning 
+    Note that you do not have to specify the `%` wildcard in contains and other
+    filters, it's added for you. If you include `%` in your search value it will be escaped
+    and treated as literal percentage sign inside the text.
+
+### exclude
+
+`exclude(**kwargs) -> QuerySet`
+
+Works exactly the same as filter and all modifiers (suffixes) are the same, but returns
+a not condition.
+
+So if you use `filter(name='John')` which equals to `where name = 'John'` in SQL,
+the `exclude(name='John')` equals to `where name <> 'John'`
+
+Note that all conditions are joined so if you pass multiple values it becomes a union of
+conditions.
+
+`exclude(name='John', age>=35)` will become `where not (name='John' and age>=35)`
+
+```python
+class Album(ormar.Model):
+    class Meta:
+        tablename = "albums"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    is_best_seller: bool = ormar.Boolean(default=False)
+
+class Track(ormar.Model):
+    class Meta:
+        tablename = "tracks"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    album: Optional[Album] = ormar.ForeignKey(Album)
+    name: str = ormar.String(max_length=100)
+    position: int = ormar.Integer()
+    play_count: int = ormar.Integer(nullable=True)
+```
+
+```python
+notes = await Track.objects.exclude(position_gt=3).all()
+# returns all tracks with position < 3
+```
+
+## get
+
+`get(**kwargs) -> Model`
+
+Get's the first row from the db meeting the criteria set by kwargs.
+
+When any kwargs are passed it's a shortcut equivalent to calling `filter(**kwargs).get()`
+
+!!!tip
+    To read more about `filter` go to [filter](./#filter).
+    
+    To read more about `get` go to [read/get](../read/#get)
+
+## get_or_create
+
+`get_or_create(**kwargs) -> Model`
+
+Combination of create and get methods.
+
+When any kwargs are passed it's a shortcut equivalent to calling `filter(**kwargs).get_or_create()`
+
+!!!tip
+    To read more about `filter` go to [filter](./#filter).
+    
+    To read more about `get_or_create` go to [read/get_or_create](../read/#get_or_create)
+
+!!!warning
+    When given item does not exist you need to pass kwargs for all required fields of the
+    model, including but not limited to primary_key column (unless it's autoincrement).
+
+## all
+
+`all(**kwargs) -> List[Optional["Model"]]`
+
+Returns all rows from a database for given model for set filter options.
+
+When any kwargs are passed it's a shortcut equivalent to calling `filter(**kwargs).all()`
+
+!!!tip
+    To read more about `filter` go to [filter](./#filter).
+    
+    To read more about `all` go to [read/all](../read/#all)
+
+### QuerysetProxy methods
+
+When access directly the related `ManyToMany` field as well as `ReverseForeignKey`
+returns the list of related models.
+
+But at the same time it exposes subset of QuerySet API, so you can filter, create,
+select related etc related models directly from parent model.
+
+#### filter
+
+Works exactly the same as [filter](./#filter) function above but allows you to filter related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+
+#### exclude
+
+Works exactly the same as [exclude](./#exclude) function above but allows you to filter related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+
+#### get
+
+Works exactly the same as [get](./#get) function above but allows you to filter related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+#### get_or_create
+
+Works exactly the same as [get_or_create](./#get_or_create) function above but allows
+you to filter related objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+#### all
+
+Works exactly the same as [all](./#all) function above but allows you to filter related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+## Sorting
+
+### order_by
+
+`order_by(columns: Union[List, str]) -> QuerySet`
+
+With `order_by()` you can order the results from database based on your choice of
+fields.
+
+You can provide a string with field name or list of strings with different fields.
+
+Ordering in sql will be applied in order of names you provide in order_by.
+
+!!!tip 
+    By default if you do not provide ordering `ormar` explicitly orders by all
+    primary keys
+
+!!!warning 
+    If you are sorting by nested models that causes that the result rows are
+    unsorted by the main model
+    `ormar` will combine those children rows into one main model.
+
+    Sample raw database rows result (sort by child model desc):
+    ```
+    MODEL: 1 - Child Model - 3
+    MODEL: 2 - Child Model - 2
+    MODEL: 1 - Child Model - 1
+    ```
+    
+    will result in 2 rows of result:
+    ```
+    MODEL: 1 - Child Models: [3, 1] # encountered first in result, all children rows combined
+    MODEL: 2 - Child Modles: [2]
+    ```
+    
+    The main model will never duplicate in the result
+
+Given sample Models like following:
+
+```python
+--8 < -- "../../docs_src/queries/docs007.py"
+```
+
+To order by main model field just provide a field name
+
+```python
+toys = await Toy.objects.select_related("owner").order_by("name").all()
+assert [x.name.replace("Toy ", "") for x in toys] == [
+    str(x + 1) for x in range(6)
+]
+assert toys[0].owner == zeus
+assert toys[1].owner == aphrodite
+```
+
+To sort on nested models separate field names with dunder '__'.
+
+You can sort this way across all relation types -> `ForeignKey`, reverse virtual FK
+and `ManyToMany` fields.
+
+```python
+toys = await Toy.objects.select_related("owner").order_by("owner__name").all()
+assert toys[0].owner.name == toys[1].owner.name == "Aphrodite"
+assert toys[2].owner.name == toys[3].owner.name == "Hermes"
+assert toys[4].owner.name == toys[5].owner.name == "Zeus"
+```
+
+To sort in descending order provide a hyphen in front of the field name
+
+```python
+owner = (
+    await Owner.objects.select_related("toys")
+        .order_by("-toys__name")
+        .filter(name="Zeus")
+        .get()
+)
+assert owner.toys[0].name == "Toy 4"
+assert owner.toys[1].name == "Toy 1"
+```
+
+!!!note 
+    All methods that do not return the rows explicitly returns a QueySet instance so
+    you can chain them together
+
+    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
+    
+    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
+
+### QuerysetProxy methods
+
+When access directly the related `ManyToMany` field as well as `ReverseForeignKey`
+returns the list of related models.
+
+But at the same time it exposes subset of QuerySet API, so you can filter, create,
+select related etc related models directly from parent model.
+
+#### order_by
+
+Works exactly the same as [order_by](./#order_by) function above but allows you to sort related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+
+[querysetproxy]: ../relations/queryset-proxy.md

docs/queries/index.md

@@ -0,0 +1,157 @@
+# Querying database with ormar
+
+## QuerySet
+
+Each Model is auto registered with a `QuerySet` that represents the underlying query,
+and it's options.
+
+Most of the methods are also available through many to many relations and on reverse
+foreign key relations through `QuerysetProxy` interface.
+
+!!!info 
+    To see which relations are supported and how to construct relations
+    visit [relations][relations].
+
+For simplicity available methods to fetch and save the data into the database are
+divided into categories according to the function they fulfill.
+
+Note that some functions/methods are in multiple categories.
+
+For complicity also Models and relations methods are listed.
+
+To read more about any specific section or function please refer to the details subpage.
+
+###[Insert data into database](./create.md)
+
+* `create(**kwargs) -> Model`
+* `get_or_create(**kwargs) -> Model`
+* `update_or_create(**kwargs) -> Model`
+* `bulk_create(objects: List[Model]) -> None`
+
+
+* `Model`
+    * `Model.save()` method
+    * `Model.upsert()` method
+    * `Model.save_related()` method
+
+  
+* `QuerysetProxy`
+    * `QuerysetProxy.create(**kwargs)` method
+    * `QuerysetProxy.get_or_create(**kwargs)` method
+    * `QuerysetProxy.update_or_create(**kwargs)` method
+
+### [Read data from database](./read.md)
+
+* `get(**kwargs) -> Model`
+* `get_or_create(**kwargs) -> Model`
+* `first() -> Model`
+* `all(**kwargs) -> List[Optional[Model]]`
+
+
+* `Model`
+    * `Model.load()` method
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.get(**kwargs)` method
+    * `QuerysetProxy.get_or_create(**kwargs)` method
+    * `QuerysetProxy.first()` method
+    * `QuerysetProxy.all(**kwargs)` method
+
+### [Update data in database](./update.md)
+
+* `update(each: bool = False, **kwargs) -> int`
+* `update_or_create(**kwargs) -> Model`
+* `bulk_update(objects: List[Model], columns: List[str] = None) -> None`
+
+
+* `Model`
+    * `Model.update()` method
+    * `Model.upsert()` method
+    * `Model.save_related()` method
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.update_or_create(**kwargs)` method
+
+### [Delete data from database](./delete.md)
+
+* `delete(each: bool = False, **kwargs) -> int`
+
+
+* `Model`
+    * `Model.delete()` method
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.remove()` method
+    * `QuerysetProxy.clear()` method
+
+### [Joins and subqueries](./joins-and-subqueries.md)
+
+* `select_related(related: Union[List, str]) -> QuerySet`
+* `prefetch_related(related: Union[List, str]) -> QuerySet`
+
+
+* `Model`
+    * `Model.load()` method
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.select_related(related: Union[List, str])` method
+    * `QuerysetProxy.prefetch_related(related: Union[List, str])` method
+
+### [Filtering and sorting](./filter-and-sort.md)
+
+* `filter(**kwargs) -> QuerySet`
+* `exclude(**kwargs) -> QuerySet`
+* `order_by(columns:Union[List, str]) -> QuerySet`
+* `get(**kwargs) -> Model`
+* `get_or_create(**kwargs) -> Model`
+* `all(**kwargs) -> List[Optional[Model]]`
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.filter(**kwargs)` method
+    * `QuerysetProxy.exclude(**kwargs)` method
+    * `QuerysetProxy.order_by(columns:Union[List, str])` method
+    * `QuerysetProxy.get(**kwargs)` method
+    * `QuerysetProxy.get_or_create(**kwargs)` method
+    * `QuerysetProxy.all(**kwargs)` method
+
+### [Selecting columns](./select-columns.md)
+
+* `fields(columns: Union[List, str, set, dict]) -> QuerySet`
+* `exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet`
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.fields(columns: Union[List, str, set, dict])` method
+    * `QuerysetProxy.exclude_fields(columns: Union[List, str, set, dict])` method
+
+### [Pagination and rows number](./pagination-and-rows-number.md)
+
+* `paginate(page: int) -> QuerySet`
+* `limit(limit_count: int) -> QuerySet`
+* `offset(offset: int) -> QuerySet`
+* `get() -> Model`
+* `first() -> Model`
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.paginate(page: int)` method
+    * `QuerysetProxy.limit(limit_count: int)` method
+    * `QuerysetProxy.offset(offset: int)` method
+
+### [Aggregated functions](./aggregations.md)
+
+* `count() -> int`
+* `exists() -> bool`
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.count()` method
+    * `QuerysetProxy.exists()` method
+  
+
+[relations]: ../relations/index.md

docs/queries/joins-and-subqueries.md

@@ -0,0 +1,416 @@
+# Joins and subqueries
+
+To join one table to another, so load also related models you can use following methods.
+
+* `select_related(related: Union[List, str]) -> QuerySet`
+* `prefetch_related(related: Union[List, str]) -> QuerySet`
+
+
+* `Model`
+    * `Model.load()` method
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.select_related(related: Union[List, str])` method
+    * `QuerysetProxy.prefetch_related(related: Union[List, str])` method
+
+## select_related
+
+`select_related(related: Union[List, str]) -> QuerySet`
+
+Allows to prefetch related models during the same query.
+
+**With `select_related` always only one query is run against the database**, meaning
+that one (sometimes complicated) join is generated and later nested models are processed in
+python.
+
+To fetch related model use `ForeignKey` names.
+
+To chain related `Models` relation use double underscores between names.
+
+!!!note 
+    If you are coming from `django` note that `ormar` `select_related` differs ->
+    in `django` you can `select_related`
+    only singe relation types, while in `ormar` you can select related across `ForeignKey`
+    relation, reverse side of `ForeignKey` (so virtual auto generated keys) and `ManyToMany`
+    fields (so all relations as of current version).
+
+!!!tip 
+    To control which model fields to select use `fields()`
+    and `exclude_fields()` `QuerySet` methods.
+
+!!!tip 
+    To control order of models (both main or nested) use `order_by()` method.
+
+```python
+class Album(ormar.Model):
+    class Meta:
+        tablename = "albums"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    is_best_seller: bool = ormar.Boolean(default=False)
+
+class Track(ormar.Model):
+    class Meta:
+        tablename = "tracks"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    album: Optional[Album] = ormar.ForeignKey(Album)
+    title: str = ormar.String(max_length=100)
+    position: int = ormar.Integer()
+    play_count: int = ormar.Integer(nullable=True)
+```
+
+```python
+album = await Album.objects.select_related("tracks").all()
+# will return album will all columns tracks
+```
+
+You can provide a string or a list of strings
+
+```python
+class SchoolClass(ormar.Model):
+    class Meta:
+        tablename = "schoolclasses"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    department: Optional[Department] = ormar.ForeignKey(Department, nullable=False)
+
+
+class Category(ormar.Model):
+    class Meta:
+        tablename = "categories"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+
+
+class Student(ormar.Model):
+    class Meta:
+        tablename = "students"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    schoolclass: Optional[SchoolClass] = ormar.ForeignKey(SchoolClass)
+    category: Optional[Category] = ormar.ForeignKey(Category, nullable=True)
+
+
+class Teacher(ormar.Model):
+    class Meta:
+        tablename = "teachers"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    schoolclass: Optional[SchoolClass] = ormar.ForeignKey(SchoolClass)
+    category: Optional[Category] = ormar.ForeignKey(Category, nullable=True)
+```
+
+```python
+classes = await SchoolClass.objects.select_related(
+    ["teachers__category", "students"]).all()
+# will return classes with teachers and teachers categories
+# as well as classes students
+```
+
+Exactly the same behavior is for Many2Many fields, where you put the names of Many2Many
+fields and the final `Models` are fetched for you.
+
+!!!warning 
+    If you set `ForeignKey` field as not nullable (so required) during all
+    queries the not nullable `Models` will be auto prefetched, even if you do not include
+    them in select_related.
+
+!!!note 
+    All methods that do not return the rows explicitly returns a QueySet instance so
+    you can chain them together
+
+    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
+    
+    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
+
+## prefetch_related
+
+`prefetch_related(related: Union[List, str]) -> QuerySet`
+
+Allows to prefetch related models during query - but opposite to `select_related` each
+subsequent model is fetched in a separate database query.
+
+**With `prefetch_related` always one query per Model is run against the database**,
+meaning that you will have multiple queries executed one after another.
+
+To fetch related model use `ForeignKey` names.
+
+To chain related `Models` relation use double underscores between names.
+
+!!!tip 
+    To control which model fields to select use `fields()`
+    and `exclude_fields()` `QuerySet` methods.
+
+!!!tip 
+    To control order of models (both main or nested) use `order_by()` method.
+
+```python
+class Album(ormar.Model):
+    class Meta:
+        tablename = "albums"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    is_best_seller: bool = ormar.Boolean(default=False)
+
+class Track(ormar.Model):
+    class Meta:
+        tablename = "tracks"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    album: Optional[Album] = ormar.ForeignKey(Album)
+    title: str = ormar.String(max_length=100)
+    position: int = ormar.Integer()
+    play_count: int = ormar.Integer(nullable=True)
+```
+
+```python
+album = await Album.objects.prefetch_related("tracks").all()
+# will return album will all columns tracks
+```
+
+You can provide a string, or a list of strings
+
+```python
+class SchoolClass(ormar.Model):
+    class Meta:
+        tablename = "schoolclasses"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    department: Optional[Department] = ormar.ForeignKey(Department, nullable=False)
+
+
+class Category(ormar.Model):
+    class Meta:
+        tablename = "categories"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+
+
+class Student(ormar.Model):
+    class Meta:
+        tablename = "students"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    schoolclass: Optional[SchoolClass] = ormar.ForeignKey(SchoolClass)
+    category: Optional[Category] = ormar.ForeignKey(Category, nullable=True)
+
+
+class Teacher(ormar.Model):
+    class Meta:
+        tablename = "teachers"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    schoolclass: Optional[SchoolClass] = ormar.ForeignKey(SchoolClass)
+    category: Optional[Category] = ormar.ForeignKey(Category, nullable=True)
+```
+
+```python
+classes = await SchoolClass.objects.prefetch_related(
+    ["teachers__category", "students"]).all()
+# will return classes with teachers and teachers categories
+# as well as classes students
+```
+
+Exactly the same behavior is for Many2Many fields, where you put the names of Many2Many
+fields and the final `Models` are fetched for you.
+
+!!!warning 
+    If you set `ForeignKey` field as not nullable (so required) during all
+    queries the not nullable `Models` will be auto prefetched, even if you do not include
+    them in select_related.
+
+!!!note 
+    All methods that do not return the rows explicitly returns a QueySet instance so
+    you can chain them together
+
+    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
+    
+    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
+
+## select_related vs prefetch_related
+
+Which should you use -> `select_related` or `prefetch_related`?
+
+Well, it really depends on your data. The best answer is try yourself and see which one
+performs faster/better in your system constraints.
+
+What to keep in mind:
+
+### Performance
+
+**Number of queries**:
+`select_related` always executes one query against the database,
+while `prefetch_related` executes multiple queries. Usually the query (I/O) operation is
+the slowest one but it does not have to be.
+
+**Number of rows**:
+Imagine that you have 10 000 object in one table A and each of those objects have 3
+children in table B, and subsequently each object in table B has 2 children in table C.
+Something like this:
+
+```
+                     Model C
+                   /
+           Model B - Model C
+         / 
+Model A  - Model B - Model C
+       \           \ 
+        \            Model C
+         \
+           Model B - Model C
+                   \ 
+                     Model C
+```
+
+That means that `select_related` will always return 60 000 rows (10 000 * 3 * 2) later
+compacted to 10 000 models.
+
+How many rows will return `prefetch_related`?
+
+Well, that depends, if each of models B and C is unique it will return 10 000 rows in
+first query, 30 000 rows
+(each of 3 children of A in table B are unique) in second query and 60 000 rows (each of
+2 children of model B in table C are unique) in 3rd query.
+
+In this case `select_related` seems like a better choice, not only it will run one query
+comparing to 3 of
+`prefetch_related` but will also return 60 000 rows comparing to 100 000
+of `prefetch_related` (10+30+60k).
+
+But what if each Model A has exactly the same 3 models B and each models C has exactly
+same models C? `select_related`
+will still return 60 000 rows, while `prefetch_related` will return 10 000 for model A,
+3 rows for model B and 2 rows for Model C. So in total 10 006 rows. Now depending on the
+structure of models (i.e. if it has long Text() fields etc.) `prefetch_related`
+might be faster despite it needs to perform three separate queries instead of one.
+
+#### Memory
+
+`ormar` is a mini ORM meaning that it does not keep a registry of already loaded models.
+
+That means that in `select_related` example above you will always have 10 000 Models A,
+30 000 Models B
+(even if the unique number of rows in db is 3 - processing of `select_related` spawns **
+new** child models for each parent model). And 60 000 Models C.
+
+If the same Model B is shared by rows 1, 10, 100 etc. and you update one of those, the
+rest of rows that share the same child will **not** be updated on the spot. If you
+persist your changes into the database the change **will be available only after reload
+(either each child separately or the whole query again)**. That means
+that `select_related` will use more memory as each child is instantiated as a new object
+- obviously using it's own space.
+
+!!!note 
+    This might change in future versions if we decide to introduce caching.
+
+!!!warning 
+    By default all children (or event the same models loaded 2+ times) are
+    completely independent, distinct python objects, despite that they represent the same
+    row in db.
+
+    They will evaluate to True when compared, so in example above: 
+    
+    ```python
+    # will return True if child1 of both rows is the same child db row 
+    row1.child1 == row100.child1
+    
+    # same here:
+    model1 = await Model.get(pk=1)
+    model2 = await Model.get(pk=1) # same pk = same row in db
+    # will return `True`
+    model1 == model2
+    ``` 
+    
+    but 
+    
+    ```python
+    # will return False (note that id is a python `builtin` function not ormar one).
+    id(row1.child1) == (ro100.child1)
+    
+    # from above - will also return False
+    id(model1) == id(model2)
+    ``` 
+
+On the contrary - with `prefetch_related` each unique distinct child model is
+instantiated only once and the same child models is shared across all parent models.
+That means that in `prefetch_related` example above if there are 3 distinct models in
+table B and 2 in table C, there will be only 5 children nested models shared between all
+model A instances. That also means that if you update any attribute it will be updated
+on all parents as they share the same child object.
+
+
+## Model methods
+
+Each model instance have a set of methods to `save`, `update` or `load` itself.
+
+### load
+
+You can load the `ForeignKey` related model by calling `load()` method.
+
+`load()` can be used to refresh the model from the database (if it was changed by some other process).
+
+!!!tip
+    Read more about `load()` method in [models methods](../models/methods.md#load)
+
+## QuerysetProxy methods
+
+When access directly the related `ManyToMany` field as well as `ReverseForeignKey`
+returns the list of related models.
+
+But at the same time it exposes subset of QuerySet API, so you can filter, create,
+select related etc related models directly from parent model.
+
+### select_related
+
+Works exactly the same as [select_related](./#select_related) function above but allows you to fetch related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+### prefetch_related
+
+Works exactly the same as [prefetch_related](./#prefetch_related) function above but allows you to fetch related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+
+[querysetproxy]: ../relations/queryset-proxy.md

docs/queries/pagination-and-rows-number.md

@@ -0,0 +1,169 @@
+#Pagination and rows number
+
+Following methods allow you to paginate and limit number of rows in queries. 
+
+* `paginate(page: int) -> QuerySet`
+* `limit(limit_count: int) -> QuerySet`
+* `offset(offset: int) -> QuerySet`
+* `get() -> Model`
+* `first() -> Model`
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.paginate(page: int)` method
+    * `QuerysetProxy.limit(limit_count: int)` method
+    * `QuerysetProxy.offset(offset: int)` method
+
+## paginate
+
+`paginate(page: int, page_size: int = 20) -> QuerySet`
+
+Combines the `offset` and `limit` methods based on page number and size
+
+```python
+class Track(ormar.Model):
+    class Meta:
+        tablename = "track"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    album: Optional[Album] = ormar.ForeignKey(Album)
+    name: str = ormar.String(max_length=100)
+    position: int = ormar.Integer()
+```
+
+```python
+tracks = await Track.objects.paginate(3).all()
+# will return 20 tracks starting at row 41 
+# (with default page size of 20)
+```
+
+Note that `paginate(2)` is equivalent to `offset(20).limit(20)`
+
+## limit
+
+`limit(limit_count: int, limit_raw_sql: bool = None) -> QuerySet`
+
+You can limit the results to desired number of parent models.
+
+To limit the actual number of database query rows instead of number of main models
+use the `limit_raw_sql` parameter flag, and set it to `True`.
+
+```python
+class Track(ormar.Model):
+    class Meta:
+        tablename = "track"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    album: Optional[Album] = ormar.ForeignKey(Album)
+    name: str = ormar.String(max_length=100)
+    position: int = ormar.Integer()
+```
+
+```python
+tracks = await Track.objects.limit(1).all()
+# will return just one Track
+```
+
+!!!note
+    All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together
+    
+    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
+    
+    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
+
+## offset
+
+`offset(offset: int, limit_raw_sql: bool = None) -> QuerySet`
+
+You can also offset the results by desired number of main models.
+
+To offset the actual number of database query rows instead of number of main models
+use the `limit_raw_sql` parameter flag, and set it to `True`.
+
+```python
+class Track(ormar.Model):
+    class Meta:
+        tablename = "track"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    album: Optional[Album] = ormar.ForeignKey(Album)
+    name: str = ormar.String(max_length=100)
+    position: int = ormar.Integer()
+```
+
+```python
+tracks = await Track.objects.offset(1).limit(1).all()
+# will return just one Track, but this time the second one
+```
+
+!!!note
+    All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together
+    
+    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
+    
+    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
+
+
+
+## get
+
+`get(**kwargs) -> Model` 
+
+Get's the first row from the db meeting the criteria set by kwargs.
+
+If no criteria is set it will return the last row in db sorted by pk.
+(The criteria cannot be set also with filter/exclude).
+
+!!!tip
+    To read more about `get` visit [read/get](./read/#get)
+
+
+## first
+
+`first() -> Model` 
+
+Gets the first row from the db ordered by primary key column ascending.
+
+!!!tip
+    To read more about `first` visit [read/first](./read/#first)
+
+
+## QuerysetProxy methods
+
+When access directly the related `ManyToMany` field as well as `ReverseForeignKey`
+returns the list of related models.
+
+But at the same time it exposes subset of QuerySet API, so you can filter, create,
+select related etc related models directly from parent model.
+
+### paginate
+
+Works exactly the same as [paginate](./#paginate) function above but allows you to paginate related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+### limit
+
+Works exactly the same as [limit](./#limit) function above but allows you to paginate related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+### offset
+
+Works exactly the same as [offset](./#offset) function above but allows you to paginate related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+[querysetproxy]: ../relations/queryset-proxy.md

docs/queries/read.md

@@ -0,0 +1,218 @@
+# Read data from database
+
+Following methods allow you to load data from the database.
+
+* `get(**kwargs) -> Model`
+* `get_or_create(**kwargs) -> Model`
+* `first() -> Model`
+* `all(**kwargs) -> List[Optional[Model]]`
+
+
+* `Model`
+    * `Model.load()` method
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.get(**kwargs)` method
+    * `QuerysetProxy.get_or_create(**kwargs)` method
+    * `QuerysetProxy.first()` method
+    * `QuerysetProxy.all(**kwargs)` method
+
+## get
+
+`get(**kwargs) -> Model`
+
+Get's the first row from the db meeting the criteria set by kwargs.
+
+If no criteria set it will return the last row in db sorted by pk column.
+
+Passing a criteria is actually calling filter(**kwargs) method described below.
+
+```python
+class Track(ormar.Model):
+    class Meta:
+        tablename = "track"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    album: Optional[Album] = ormar.ForeignKey(Album)
+    name: str = ormar.String(max_length=100)
+    position: int = ormar.Integer()
+```
+
+```python
+track = await Track.objects.get(name='The Bird')
+# note that above is equivalent to await Track.objects.filter(name='The Bird').get()
+track2 = track = await Track.objects.get()
+track == track2
+# True since it's the only row in db in our example
+# and get without arguments return first row by pk column desc 
+```
+
+!!!warning 
+    If no row meets the criteria `NoMatch` exception is raised.
+
+    If there are multiple rows meeting the criteria the `MultipleMatches` exception is raised.
+
+## get_or_create
+
+`get_or_create(**kwargs) -> Model`
+
+Combination of create and get methods.
+
+Tries to get a row meeting the criteria and if `NoMatch` exception is raised it creates
+a new one with given kwargs.
+
+```python
+class Album(ormar.Model):
+    class Meta:
+        tablename = "album"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+```
+
+```python
+album = await Album.objects.get_or_create(name='The Cat')
+# object is created as it does not exist
+album2 = await Album.objects.get_or_create(name='The Cat')
+assert album == album2
+# return True as the same db row is returned
+```
+
+!!!warning 
+    Despite being an equivalent row from database the `album` and `album2` in
+    example above are 2 different python objects!
+    Updating one of them will not refresh the second one until you excplicitly load() the
+    fresh data from db.
+
+!!!note 
+    Note that if you want to create a new object you either have to pass pk column
+    value or pk column has to be set as autoincrement
+
+## first
+
+`first() -> Model`
+
+Gets the first row from the db ordered by primary key column ascending.
+
+```python
+class Album(ormar.Model):
+    class Meta:
+        tablename = "album"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+```
+
+```python
+await Album.objects.create(name='The Cat')
+await Album.objects.create(name='The Dog')
+album = await Album.objects.first()
+# first row by primary_key column asc
+assert album.name == 'The Cat'
+```
+
+## all
+
+`all(**kwargs) -> List[Optional["Model"]]`
+
+Returns all rows from a database for given model for set filter options.
+
+Passing kwargs is a shortcut and equals to calling `filter(**kwrags).all()`.
+
+If there are no rows meeting the criteria an empty list is returned.
+
+```python
+class Album(ormar.Model):
+    class Meta:
+        tablename = "album"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+
+
+class Track(ormar.Model):
+    class Meta:
+        tablename = "track"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    album: Optional[Album] = ormar.ForeignKey(Album)
+    title: str = ormar.String(max_length=100)
+    position: int = ormar.Integer()
+```
+
+```python
+tracks = await Track.objects.select_related("album").all(album__title='Sample')
+# will return a list of all Tracks for album Sample
+# for more on joins visit joining and subqueries section
+
+tracks = await Track.objects.all()
+# will return a list of all Tracks in database
+
+```
+
+## Model methods
+
+Each model instance have a set of methods to `save`, `update` or `load` itself.
+
+### load
+
+You can load the `ForeignKey` related model by calling `load()` method.
+
+`load()` can be used to refresh the model from the database (if it was changed by some other process).
+
+!!!tip
+    Read more about `load()` method in [models methods](../models/methods.md#load)
+
+## QuerysetProxy methods
+
+When access directly the related `ManyToMany` field as well as `ReverseForeignKey`
+returns the list of related models.
+
+But at the same time it exposes subset of QuerySet API, so you can filter, create,
+select related etc related models directly from parent model.
+
+### get
+
+Works exactly the same as [get](./#get) function above but allows you to fetch related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+### get_or_create
+
+Works exactly the same as [get_or_create](./#get_or_create) function above but allows
+you to query or create related objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+### first
+
+Works exactly the same as [first](./#first) function above but allows you to query
+related objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+### all
+
+Works exactly the same as [all](./#all) function above but allows you to query related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+
+[querysetproxy]: ../relations/queryset-proxy.md

docs/queries/select-columns.md

@@ -0,0 +1,315 @@
+# Selecting subset of columns
+
+To select only chosen columns of your model you can use following functions.
+
+* `fields(columns: Union[List, str, set, dict]) -> QuerySet`
+* `exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet`
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.fields(columns: Union[List, str, set, dict])` method
+    * `QuerysetProxy.exclude_fields(columns: Union[List, str, set, dict])` method
+
+## fields
+
+`fields(columns: Union[List, str, set, dict]) -> QuerySet`
+
+With `fields()` you can select subset of model columns to limit the data load.
+
+!!!note 
+    Note that `fields()` and `exclude_fields()` works both for main models (on
+    normal queries like `get`, `all` etc.)
+    as well as `select_related` and `prefetch_related` models (with nested notation).
+
+Given a sample data like following:
+
+```python
+import databases
+import sqlalchemy
+
+import ormar
+from tests.settings import DATABASE_URL
+
+database = databases.Database(DATABASE_URL, force_rollback=True)
+metadata = sqlalchemy.MetaData()
+
+
+class Company(ormar.Model):
+    class Meta:
+        tablename = "companies"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    founded: int = ormar.Integer(nullable=True)
+
+
+class Car(ormar.Model):
+    class Meta:
+        tablename = "cars"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    manufacturer = ormar.ForeignKey(Company)
+    name: str = ormar.String(max_length=100)
+    year: int = ormar.Integer(nullable=True)
+    gearbox_type: str = ormar.String(max_length=20, nullable=True)
+    gears: int = ormar.Integer(nullable=True)
+    aircon_type: str = ormar.String(max_length=20, nullable=True)
+
+
+# build some sample data
+toyota = await Company.objects.create(name="Toyota", founded=1937)
+await Car.objects.create(manufacturer=toyota, name="Corolla", year=2020, gearbox_type='Manual', gears=5,
+                         aircon_type='Manual')
+await Car.objects.create(manufacturer=toyota, name="Yaris", year=2019, gearbox_type='Manual', gears=5,
+                         aircon_type='Manual')
+await Car.objects.create(manufacturer=toyota, name="Supreme", year=2020, gearbox_type='Auto', gears=6,
+                         aircon_type='Auto')
+
+
+```
+
+You can select specified fields by passing a `str, List[str], Set[str] or dict` with
+nested definition.
+
+To include related models use
+notation `{related_name}__{column}[__{optional_next} etc.]`.
+
+```python hl_lines="1"
+all_cars = await Car.objects.select_related('manufacturer').fields(['id', 'name', 'manufacturer__name']).all()
+for car in all_cars:
+    # excluded columns will yield None
+    assert all(getattr(car, x) is None for x in ['year', 'gearbox_type', 'gears', 'aircon_type'])
+    # included column on related models will be available, pk column is always included
+    # even if you do not include it in fields list
+    assert car.manufacturer.name == 'Toyota'
+    # also in the nested related models - you cannot exclude pk - it's always auto added
+    assert car.manufacturer.founded is None
+```
+
+`fields()` can be called several times, building up the columns to select.
+
+If you include related models into `select_related()` call but you won't specify columns
+for those models in fields
+
+- implies a list of all fields for those nested models.
+
+```python hl_lines="1"
+all_cars = await Car.objects.select_related('manufacturer').fields('id').fields(
+    ['name']).all()
+# all fiels from company model are selected
+assert all_cars[0].manufacturer.name == 'Toyota'
+assert all_cars[0].manufacturer.founded == 1937
+```
+
+!!!warning 
+    Mandatory fields cannot be excluded as it will raise `ValidationError`, to
+    exclude a field it has to be nullable.
+
+You cannot exclude mandatory model columns - `manufacturer__name` in this example.
+
+```python
+await Car.objects.select_related('manufacturer').fields(
+    ['id', 'name', 'manufacturer__founded']).all()
+# will raise pydantic ValidationError as company.name is required
+```
+
+!!!tip 
+    Pk column cannot be excluded - it's always auto added even if not explicitly
+    included.
+
+You can also pass fields to include as dictionary or set.
+
+To mark a field as included in a dictionary use it's name as key and ellipsis as value.
+
+To traverse nested models use nested dictionaries.
+
+To include fields at last level instead of nested dictionary a set can be used.
+
+To include whole nested model specify model related field name and ellipsis.
+
+Below you can see examples that are equivalent:
+
+```python
+# 1. like in example above
+await Car.objects.select_related('manufacturer').fields(['id', 'name', 'manufacturer__name']).all()
+
+# 2. to mark a field as required use ellipsis
+await Car.objects.select_related('manufacturer').fields({'id': ...,
+                                                         'name': ...,
+                                                         'manufacturer': {
+                                                             'name': ...}
+                                                         }).all()
+
+# 3. to include whole nested model use ellipsis
+await Car.objects.select_related('manufacturer').fields({'id': ...,
+                                                         'name': ...,
+                                                         'manufacturer': ...
+                                                         }).all()
+
+# 4. to specify fields at last nesting level you can also use set - equivalent to 2. above
+await Car.objects.select_related('manufacturer').fields({'id': ...,
+                                                         'name': ...,
+                                                         'manufacturer': {'name'}
+                                                         }).all()
+
+# 5. of course set can have multiple fields
+await Car.objects.select_related('manufacturer').fields({'id': ...,
+                                                         'name': ...,
+                                                         'manufacturer': {'name', 'founded'}
+                                                         }).all()
+
+# 6. you can include all nested fields but it will be equivalent of 3. above which is shorter
+await Car.objects.select_related('manufacturer').fields({'id': ...,
+                                                         'name': ...,
+                                                         'manufacturer': {'id', 'name', 'founded'}
+                                                         }).all()
+
+```
+
+!!!note 
+    All methods that do not return the rows explicitly returns a QueySet instance so
+    you can chain them together
+
+    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
+    
+    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
+
+## exclude_fields
+
+`exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet`
+
+With `exclude_fields()` you can select subset of model columns that will be excluded to
+limit the data load.
+
+It's the opposite of `fields()` method so check documentation above to see what options
+are available.
+
+Especially check above how you can pass also nested dictionaries and sets as a mask to
+exclude fields from whole hierarchy.
+
+!!!note 
+    Note that `fields()` and `exclude_fields()` works both for main models (on
+    normal queries like `get`, `all` etc.)
+    as well as `select_related` and `prefetch_related` models (with nested notation).
+
+Below you can find few simple examples:
+
+```python hl_lines="47 48 60 61 67"
+import databases
+import sqlalchemy
+
+import ormar
+from tests.settings import DATABASE_URL
+
+database = databases.Database(DATABASE_URL, force_rollback=True)
+metadata = sqlalchemy.MetaData()
+
+
+class Company(ormar.Model):
+    class Meta:
+        tablename = "companies"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    founded: int = ormar.Integer(nullable=True)
+
+
+class Car(ormar.Model):
+    class Meta:
+        tablename = "cars"
+        metadata = metadata
+        database = database
+
+    id: int = ormar.Integer(primary_key=True)
+    manufacturer = ormar.ForeignKey(Company)
+    name: str = ormar.String(max_length=100)
+    year: int = ormar.Integer(nullable=True)
+    gearbox_type: str = ormar.String(max_length=20, nullable=True)
+    gears: int = ormar.Integer(nullable=True)
+    aircon_type: str = ormar.String(max_length=20, nullable=True)
+
+
+# build some sample data
+toyota = await Company.objects.create(name="Toyota", founded=1937)
+await Car.objects.create(manufacturer=toyota, name="Corolla", year=2020, gearbox_type='Manual', gears=5,
+                         aircon_type='Manual')
+await Car.objects.create(manufacturer=toyota, name="Yaris", year=2019, gearbox_type='Manual', gears=5,
+                         aircon_type='Manual')
+await Car.objects.create(manufacturer=toyota, name="Supreme", year=2020, gearbox_type='Auto', gears=6,
+                         aircon_type='Auto')
+
+# select manufacturer but only name - to include related models use notation {model_name}__{column}
+all_cars = await Car.objects.select_related('manufacturer').exclude_fields(
+    ['year', 'gearbox_type', 'gears', 'aircon_type', 'company__founded']).all()
+for car in all_cars:
+    # excluded columns will yield None
+    assert all(getattr(car, x) is None for x in ['year', 'gearbox_type', 'gears', 'aircon_type'])
+    # included column on related models will be available, pk column is always included
+    # even if you do not include it in fields list
+    assert car.manufacturer.name == 'Toyota'
+    # also in the nested related models - you cannot exclude pk - it's always auto added
+    assert car.manufacturer.founded is None
+
+# fields() can be called several times, building up the columns to select
+# models selected in select_related but with no columns in fields list implies all fields
+all_cars = await Car.objects.select_related('manufacturer').exclude_fields('year').exclude_fields(
+    ['gear', 'gearbox_type']).all()
+# all fiels from company model are selected
+assert all_cars[0].manufacturer.name == 'Toyota'
+assert all_cars[0].manufacturer.founded == 1937
+
+# cannot exclude mandatory model columns - company__name in this example - note usage of dict/set this time
+await Car.objects.select_related('manufacturer').exclude_fields([{'company': {'name'}}]).all()
+# will raise pydantic ValidationError as company.name is required
+
+```
+
+!!!warning 
+    Mandatory fields cannot be excluded as it will raise `ValidationError`, to
+    exclude a field it has to be nullable.
+
+!!!tip 
+    Pk column cannot be excluded - it's always auto added even if explicitly
+    excluded.
+
+!!!note 
+    All methods that do not return the rows explicitly returns a QueySet instance so
+    you can chain them together
+
+    So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained.
+    
+    Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()`
+
+
+## QuerysetProxy methods
+
+When access directly the related `ManyToMany` field as well as `ReverseForeignKey`
+returns the list of related models.
+
+But at the same time it exposes subset of QuerySet API, so you can filter, create,
+select related etc related models directly from parent model.
+
+### fields
+
+Works exactly the same as [fields](./#fields) function above but allows you to select columns from related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+### exclude_fields
+
+Works exactly the same as [exclude_fields](./#exclude_fields) function above but allows you to select columns from related
+objects from other side of the relation.
+
+!!!tip 
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+
+[querysetproxy]: ../relations/queryset-proxy.md

docs/queries/update.md

@@ -0,0 +1,126 @@
+# Update data in database
+
+Following methods and functions allow updating existing data in the database.
+
+* `update(each: bool = False, **kwargs) -> int`
+* `update_or_create(**kwargs) -> Model`
+* `bulk_update(objects: List[Model], columns: List[str] = None) -> None`
+
+
+* `Model`
+    * `Model.update()` method
+    * `Model.upsert()` method
+    * `Model.save_related()` method
+
+
+* `QuerysetProxy`
+    * `QuerysetProxy.update_or_create(**kwargs)` method
+
+## update
+
+`update(each: bool = False, **kwargs) -> int`
+
+QuerySet level update is used to update multiple records with the same value at once.
+
+You either have to filter the QuerySet first or provide a `each=True` flag to update
+whole table.
+
+If you do not provide this flag or a filter a `QueryDefinitionError` will be raised.
+
+Return number of rows updated.
+
+```Python hl_lines="26-28"
+--8<-- "../docs_src/queries/docs002.py"
+```
+
+!!!warning 
+    Queryset needs to be filtered before updating to prevent accidental overwrite.
+
+    To update whole database table `each=True` needs to be provided as a safety switch
+
+## update_or_create
+
+`update_or_create(**kwargs) -> Model`
+
+Updates the model, or in case there is no match in database creates a new one.
+
+```Python hl_lines="26-32"
+--8<-- "../docs_src/queries/docs003.py"
+```
+
+!!!note 
+    Note that if you want to create a new object you either have to pass pk column
+    value or pk column has to be set as autoincrement
+
+## bulk_update
+
+`bulk_update(objects: List["Model"], columns: List[str] = None) -> None`
+
+Allows to update multiple instance at once.
+
+All `Models` passed need to have primary key column populated.
+
+You can also select which fields to update by passing `columns` list as a list of string
+names.
+
+```python hl_lines="8"
+# continuing the example from bulk_create
+# update objects
+for todo in todoes:
+    todo.completed = False
+
+# perform update of all objects at once
+# objects need to have pk column set, otherwise exception is raised
+await ToDo.objects.bulk_update(todoes)
+
+completed = await ToDo.objects.filter(completed=False).all()
+assert len(completed) == 3
+```
+
+## Model methods
+
+Each model instance have a set of methods to `save`, `update` or `load` itself.
+
+###update
+
+You can update models by updating your model attributes (fields) and calling `update()` method.
+
+If you try to update a model without a primary key set a `ModelPersistenceError` exception will be thrown.
+
+
+!!!tip
+    Read more about `update()` method in [models-update](../models/methods.md#update)
+
+###upsert
+
+It's a proxy to either `save()` or `update(**kwargs)` methods of a Model.
+If the pk is set the `update()` method will be called.
+
+!!!tip
+    Read more about `upsert()` method in [models-upsert][models-upsert]
+
+###save_related
+
+Method goes through all relations of the `Model` on which the method is called, 
+and calls `upsert()` method on each model that is **not** saved. 
+
+!!!tip
+    Read more about `save_related()` method in [models-save-related][models-save-related]
+
+## QuerysetProxy methods
+
+When access directly the related `ManyToMany` field as well as `ReverseForeignKey` returns the list of related models.
+
+But at the same time it exposes subset of QuerySet API, so you can filter, create, select related etc related models directly from parent model.
+
+### update_or_create
+
+Works exactly the same as [update_or_create](./#update_or_create) function above but allows you to update or create related objects
+from other side of the relation.
+
+!!!tip
+    To read more about `QuerysetProxy` visit [querysetproxy][querysetproxy] section
+
+[querysetproxy]: ../relations/queryset-proxy.md
+[models-upsert]: ../models/methods.md#upsert
+[models-save-related]: ../models/methods.md#save_related

docs/relations/index.md

@@ -92,7 +92,34 @@ class Post(ormar.Model):
 
     It allows you to use `await post.categories.all()` but also `await category.posts.all()` to fetch data related only to specific post, category etc.
 
+##Self-reference and postponed references
+
+In order to create auto-relation or create two models that reference each other in at least two
+different relations (remember the reverse side is auto-registered for you), you need to use
+`ForwardRef` from `typing` module.
+
+```python hl_lines="1 11 14"
+PersonRef = ForwardRef("Person")
+
+
+class Person(ormar.Model):
+    class Meta(ModelMeta):
+        metadata = metadata
+        database = db
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    supervisor: PersonRef = ormar.ForeignKey(PersonRef, related_name="employees")
+
+
+Person.update_forward_refs()
+```
+
+!!!tip
+    To read more about self-reference and postponed relations visit [postponed-annotations][postponed-annotations] section
+
 
 [foreign-keys]: ./foreign-key.md
 [many-to-many]: ./many-to-many.md
-[queryset-proxy]: ./queryset-proxy.md
+[queryset-proxy]: ./queryset-proxy.md
+[postponed-annotations]: ./postponed-annotations.md

docs/relations/postponed-annotations.md

@@ -0,0 +1,171 @@
+# Postponed annotations
+
+## Self-referencing Models
+
+When you want to reference the same model during declaration to create a 
+relation you need to declare the referenced model as a `ForwardRef`, as during the declaration
+the class is not yet ready and python by default won't let you reference it.
+
+Although you might be tempted to use __future__ annotations or simply quote the name with `""` it won't work
+as `ormar` is designed to work with explicitly declared `ForwardRef`.
+
+First, you need to import the required ref from typing.
+```python
+from typing import ForwardRef
+```
+
+But note that before python 3.7 it used to be internal, so for python <= 3.6 you need
+
+```python
+from typing import _ForwardRef as ForwardRef
+```
+
+or since `pydantic` is required by `ormar` it can handle this switch for you. 
+In that case you can simply import ForwardRef from pydantic regardless of your python version.
+
+```python
+from pydantic.typing import ForwardRef
+```
+
+Now we need a sample model and a reference to the same model, 
+which will be used to creat a self referencing relation.
+
+```python
+# create the forwardref to model Person
+PersonRef = ForwardRef("Person")
+
+
+class Person(ormar.Model):
+    class Meta(ModelMeta):
+        metadata = metadata
+        database = db
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    # use the forwardref as to parameter
+    supervisor: PersonRef = ormar.ForeignKey(PersonRef, related_name="employees")
+
+```
+
+That's so simple. But before you can use the model you need to manually update the references
+so that they lead to the actual models.
+
+!!!warning
+    If you try to use the model without updated references, `ModelError` exception will be raised.
+    So in our example above any call like following will cause exception
+    ```python
+    # creation of model - exception
+    await Person.objects.create(name="Test")
+    # initialization of model - exception
+    Person2(name="Test")
+    # usage of model's QuerySet - exception
+    await Person2.objects.get()
+    ```
+
+To update the references call the `update_forward_refs` method on **each model** 
+with forward references, only **after all related models were declared.**
+
+So in order to make our previous example work we need just one extra line.
+
+```python hl_lines="14"
+PersonRef = ForwardRef("Person")
+
+
+class Person(ormar.Model):
+    class Meta(ModelMeta):
+        metadata = metadata
+        database = db
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    supervisor: PersonRef = ormar.ForeignKey(PersonRef, related_name="employees")
+
+
+Person.update_forward_refs()
+
+```
+
+Of course the same can be done with ManyToMany relations in exactly same way, both for to
+and through parameters.
+
+```python
+# declare the reference
+ChildRef = ForwardRef("Child")
+
+class ChildFriend(ormar.Model):
+    class Meta(ModelMeta):
+        metadata = metadata
+        database = db
+
+class Child(ormar.Model):
+    class Meta(ModelMeta):
+        metadata = metadata
+        database = db
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    # use it in relation
+    friends = ormar.ManyToMany(ChildRef, through=ChildFriend,
+                               related_name="also_friends")
+
+
+Child.update_forward_refs()
+```
+
+## Cross model relations
+
+The same mechanism and logic as for self-reference model can be used to link multiple different
+models between each other.
+
+Of course `ormar` links both sides of relation for you, 
+creating a reverse relation with specified (or default) `related_name`.
+
+But if you need two (or more) relations between any two models, that for whatever reason
+should be stored on both sides (so one relation is declared on one model, 
+and other on the second model), you need to use `ForwardRef` to achieve that.
+
+Look at the following simple example.
+
+```python
+# teacher is not yet defined
+TeacherRef = ForwardRef("Teacher")
+
+
+class Student(ormar.Model):
+    class Meta(ModelMeta):
+        metadata = metadata
+        database = db
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    # so we use reference instead of actual model
+    primary_teacher: TeacherRef = ormar.ForeignKey(TeacherRef,
+                                                   related_name="own_students")
+
+
+class StudentTeacher(ormar.Model):
+    class Meta(ModelMeta):
+        tablename = 'students_x_teachers'
+        metadata = metadata
+        database = db
+
+
+class Teacher(ormar.Model):
+    class Meta(ModelMeta):
+        metadata = metadata
+        database = db
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    # we need students for other relation hence the order
+    students = ormar.ManyToMany(Student, through=StudentTeacher,
+                                related_name="teachers")
+
+# now the Teacher model is already defined we can update references
+Student.update_forward_refs()
+
+```
+
+!!!warning
+    Remember that `related_name` needs to be unique across related models regardless 
+    of how many relations are defined. 

docs/relations/queryset-proxy.md

@@ -30,7 +30,9 @@ But at the same time it exposes subset of QuerySet API, so you can filter, creat
     Note that value returned by `create` or created in `get_or_create` and `update_or_create` 
     if model does not exist will be added to relation list (not clearing it).
 
-## get
+## Read data from database
+
+### get
 
 `get(**kwargs): -> Model` 
 
@@ -52,7 +54,16 @@ assert post.categories[0] == news
 !!!tip
     Read more in queries documentation [get][get]
 
-## all
+### get_or_create
+
+`get_or_create(**kwargs) -> Model`
+
+Tries to get a row meeting the criteria and if NoMatch exception is raised it creates a new one with given kwargs.
+
+!!!tip
+    Read more in queries documentation [get_or_create][get_or_create]
+
+### all
 
 `all(**kwargs) -> List[Optional["Model"]]`
 
@@ -73,7 +84,9 @@ assert news_posts[0].author == guido
 !!!tip
     Read more in queries documentation [all][all]
 
-## create
+## Insert/ update data into database
+
+### create
 
 `create(**kwargs): -> Model` 
 
@@ -91,113 +104,162 @@ assert len(await post.categories.all()) == 2
 !!!tip
     Read more in queries documentation [create][create]
 
-
+### get_or_create
-## get_or_create
 
 `get_or_create(**kwargs) -> Model`
 
+Tries to get a row meeting the criteria and if NoMatch exception is raised it creates a new one with given kwargs.
+
 !!!tip
     Read more in queries documentation [get_or_create][get_or_create]
 
-## update_or_create
+### update_or_create
 
 `update_or_create(**kwargs) -> Model`
 
+Updates the model, or in case there is no match in database creates a new one.
+
 !!!tip
     Read more in queries documentation [update_or_create][update_or_create]
 
-## filter
+## Filtering and sorting
+
+### filter
 
 `filter(**kwargs) -> QuerySet`
 
+Allows you to filter by any Model attribute/field as well as to fetch instances, with a filter across an FK relationship.
+
 !!!tip
     Read more in queries documentation [filter][filter]
 
-## exclude
+### exclude
 
 `exclude(**kwargs) -> QuerySet`
 
+Works exactly the same as filter and all modifiers (suffixes) are the same, but returns a not condition.
+
 !!!tip
     Read more in queries documentation [exclude][exclude]
 
-## select_related
+### order_by
-
-`select_related(related: Union[List, str]) -> QuerySet`
-
-!!!tip
-    Read more in queries documentation [select_related][select_related]
-
-## prefetch_related
-
-`prefetch_related(related: Union[List, str]) -> QuerySet`
-
-!!!tip
-    Read more in queries documentation [prefetch_related][prefetch_related]
-
-## limit
-
-`limit(limit_count: int) -> QuerySet`
-
-!!!tip
-    Read more in queries documentation [limit][limit]
-
-## offset
-
-`offset(offset: int) -> QuerySet`
-
-!!!tip
-    Read more in queries documentation [offset][offset]
-
-## count
-
-`count() -> int`
-
-!!!tip
-    Read more in queries documentation [count][count]
-
-## exists
-
-`exists() -> bool`
-
-!!!tip
-    Read more in queries documentation [exists][exists]
-
-## fields
-
-`fields(columns: Union[List, str, set, dict]) -> QuerySet`
-
-!!!tip
-    Read more in queries documentation [fields][fields]
-
-## exclude_fields
-
-`exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet`
-
-!!!tip
-    Read more in queries documentation [exclude_fields][exclude_fields]
-
-## order_by
 
 `order_by(columns:Union[List, str]) -> QuerySet`
 
+With order_by() you can order the results from database based on your choice of fields.
+
 !!!tip
     Read more in queries documentation [order_by][order_by]
 
+## Joins and subqueries
 
-[queries]: ../queries.md
+### select_related
-[get]: ../queries.md#get
+
-[all]: ../queries.md#all
+`select_related(related: Union[List, str]) -> QuerySet`
-[create]: ../queries.md#create
+
-[get_or_create]: ../queries.md#get_or_create
+Allows to prefetch related models during the same query.
-[update_or_create]: ../queries.md#update_or_create
+
-[filter]: ../queries.md#filter
+With select_related always only one query is run against the database, meaning that one (sometimes complicated) join is generated and later nested models are processed in python.
-[exclude]: ../queries.md#exclude
+
-[select_related]: ../queries.md#select_related
+!!!tip
-[prefetch_related]: ../queries.md#prefetch_related
+    Read more in queries documentation [select_related][select_related]
-[limit]: ../queries.md#limit
+
-[offset]: ../queries.md#offset
+### prefetch_related
-[count]: ../queries.md#count
+
-[exists]: ../queries.md#exists
+`prefetch_related(related: Union[List, str]) -> QuerySet`
-[fields]: ../queries.md#fields
+
-[exclude_fields]: ../queries.md#exclude_fields
+Allows to prefetch related models during query - but opposite to select_related each subsequent model is fetched in a separate database query.
-[order_by]: ../queries.md#order_by
+
+With prefetch_related always one query per Model is run against the database, meaning that you will have multiple queries executed one after another.
+
+!!!tip
+    Read more in queries documentation [prefetch_related][prefetch_related]
+
+## Pagination and rows number
+
+### paginate
+
+`paginate(page: int, page_size: int = 20) -> QuerySet`
+
+Combines the offset and limit methods based on page number and size.
+
+!!!tip
+    Read more in queries documentation [paginate][paginate]
+
+### limit
+
+`limit(limit_count: int) -> QuerySet`
+
+You can limit the results to desired number of parent models.
+
+!!!tip
+    Read more in queries documentation [limit][limit]
+
+### offset
+
+`offset(offset: int) -> QuerySet`
+
+You can offset the results by desired number of main models.
+
+!!!tip
+    Read more in queries documentation [offset][offset]
+
+## Selecting subset of columns
+
+### fields
+
+`fields(columns: Union[List, str, set, dict]) -> QuerySet`
+
+With fields() you can select subset of model columns to limit the data load.
+
+!!!tip
+    Read more in queries documentation [fields][fields]
+
+### exclude_fields
+
+`exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet`
+
+With exclude_fields() you can select subset of model columns that will be excluded to limit the data load.
+
+!!!tip
+    Read more in queries documentation [exclude_fields][exclude_fields]
+
+## Aggregated functions
+
+### count
+
+`count() -> int`
+
+Returns number of rows matching the given criteria (i.e. applied with filter and exclude)
+
+!!!tip
+    Read more in queries documentation [count][count]
+
+### exists
+
+`exists() -> bool`
+
+Returns a bool value to confirm if there are rows matching the given criteria (applied with filter and exclude)
+
+!!!tip
+    Read more in queries documentation [exists][exists]
+
+
+[queries]: ../queries/index.md
+[get]: ../queries/read.md#get
+[all]: ../queries/read.md#all
+[create]: ../queries/create.md#create
+[get_or_create]: ../queries/read.md#get_or_create
+[update_or_create]: ../queries/update.md#update_or_create
+[filter]: ../queries/filter-and-sort.md#filter
+[exclude]: ../queries/filter-and-sort.md#exclude
+[select_related]: ../queries/joins-and-subqueries.md#select_related
+[prefetch_related]: ../queries/joins-and-subqueries.md#prefetch_related
+[limit]: ../queries/pagination-and-rows-number.md#limit
+[offset]: ../queries/pagination-and-rows-number.md#offset
+[paginate]: ../queries/pagination-and-rows-number.md#paginate
+[count]: ../queries/aggregations.md#count
+[exists]: ../queries/aggregations.md#exists
+[fields]: ../queries/select-columns.md#fields
+[exclude_fields]: ../queries/select-columns.md#exclude_fields
+[order_by]: ../queries/filter-and-sort.md#order_by

docs/releases.md

@@ -1,3 +1,29 @@
+# 0.8.1
+
+## Features
+
+* Introduce processing of `ForwardRef` in relations. 
+  Now you can create self-referencing models - both `ForeignKey` and `ManyToMany` relations. 
+  `ForwardRef` can be used both for `to` and `through` `Models`.
+* Introduce the possibility to perform two **same relation** joins in one query, so to process complex relations like:
+  ```
+      B = X = Y
+    //
+   A 
+    \
+      C = X = Y <= before you could link from X to Y only once in one query
+                   unless two different relation were used 
+                   (two relation fields with different names)
+  ```
+* Introduce the `paginate` method that allows to limit/offset by `page` and `page_size`. 
+  Available for `QuerySet` and `QuerysetProxy`.
+
+## Other
+
+* Refactoring and performance optimization in queries and joins.
+* Add python 3.9 to tests and pypi setup.
+* Update API docs and docs -> i.e. split of queries documentation.
+
 # 0.8.0
 
 ## Breaking

mkdocs.yml

@@ -14,10 +14,21 @@ nav:
     - Fields types: fields/field-types.md
   - Relations:
     - relations/index.md
+    - relations/postponed-annotations.md
     - relations/foreign-key.md
     - relations/many-to-many.md
     - relations/queryset-proxy.md
-  - Queries: queries.md
+  - Queries:
+    - queries/index.md
+    - queries/create.md
+    - queries/read.md
+    - queries/update.md
+    - queries/delete.md
+    - queries/joins-and-subqueries.md
+    - queries/filter-and-sort.md
+    - queries/select-columns.md
+    - queries/pagination-and-rows-number.md
+    - queries/aggregations.md
   - Signals: signals.md
   - Use with Fastapi: fastapi.md
   - Use with mypy: mypy.md

ormar/__init__.py

@@ -65,7 +65,7 @@ class UndefinedType:  # pragma no cover
 
 Undefined = UndefinedType()
 
-__version__ = "0.8.0"
+__version__ = "0.8.1"
 __all__ = [
     "Integer",
     "BigInteger",

ormar/fields/base.py

@@ -40,8 +40,12 @@ class BaseField(FieldInfo):
     pydantic_only: bool
     virtual: bool = False
     choices: typing.Sequence
+
+    owner: Type["Model"]
     to: Type["Model"]
     through: Type["Model"]
+    self_reference: bool = False
+    self_reference_primary: Optional[str] = None
 
     default: Any
     server_default: Any
@@ -244,7 +248,6 @@ class BaseField(FieldInfo):
         value: Any,
         child: Union["Model", "NewBaseModel"],
         to_register: bool = True,
-        relation_name: str = None,
     ) -> Any:
         """
         Function overwritten for relations, in basic field the value is returned as is.
@@ -263,3 +266,50 @@ class BaseField(FieldInfo):
         :rtype: Any
         """
         return value
+
+    @classmethod
+    def set_self_reference_flag(cls) -> None:
+        """
+        Sets `self_reference` to True if field to and owner are same model.
+        :return: None
+        :rtype: None
+        """
+        if cls.owner is not None and (
+            cls.owner == cls.to or cls.owner.Meta == cls.to.Meta
+        ):
+            cls.self_reference = True
+            cls.self_reference_primary = cls.name
+
+    @classmethod
+    def has_unresolved_forward_refs(cls) -> bool:
+        """
+        Verifies if the filed has any ForwardRefs that require updating before the
+        model can be used.
+
+        :return: result of the check
+        :rtype: bool
+        """
+        return False
+
+    @classmethod
+    def evaluate_forward_ref(cls, globalns: Any, localns: Any) -> None:
+        """
+        Evaluates the ForwardRef to actual Field based on global and local namespaces
+
+        :param globalns: global namespace
+        :type globalns: Any
+        :param localns: local namespace
+        :type localns: Any
+        :return: None
+        :rtype: None
+        """
+
+    @classmethod
+    def get_related_name(cls) -> str:
+        """
+        Returns name to use for reverse relation.
+        It's either set as `related_name` or by default it's owner model. get_name + 's'
+        :return: name of the related_name or default related name.
+        :rtype: str
+        """
+        return ""  # pragma: no cover

ormar/fields/foreign_key.py

@@ -1,8 +1,10 @@
+import sys
 import uuid
 from dataclasses import dataclass
-from typing import Any, List, Optional, TYPE_CHECKING, Type, Union
+from typing import Any, List, Optional, TYPE_CHECKING, Tuple, Type, Union
 
 from pydantic import BaseModel, create_model
+from pydantic.typing import ForwardRef, evaluate_forwardref
 from sqlalchemy import UniqueConstraint
 
 import ormar  # noqa I101
@@ -13,6 +15,11 @@ if TYPE_CHECKING:  # pragma no cover
     from ormar.models import Model, NewBaseModel
     from ormar.fields import ManyToManyField
 
+    if sys.version_info < (3, 7):
+        ToType = Type["Model"]
+    else:
+        ToType = Union[Type["Model"], "ForwardRef"]
+
 
 def create_dummy_instance(fk: Type["Model"], pk: Any = None) -> "Model":
     """
@@ -66,6 +73,43 @@ def create_dummy_model(
     return dummy_model
 
 
+def populate_fk_params_based_on_to_model(
+    to: Type["Model"], nullable: bool, onupdate: str = None, ondelete: str = None,
+) -> Tuple[Any, List, Any]:
+    """
+    Based on target to model to which relation leads to populates the type of the
+    pydantic field to use, ForeignKey constraint and type of the target column field.
+
+    :param to: target related ormar Model
+    :type to: Model class
+    :param nullable: marks field as optional/ required
+    :type nullable: bool
+    :param onupdate: parameter passed to sqlalchemy.ForeignKey.
+    How to treat child rows on update of parent (the one where FK is defined) model.
+    :type onupdate: str
+    :param ondelete: parameter passed to sqlalchemy.ForeignKey.
+    How to treat child rows on delete of parent (the one where FK is defined) model.
+    :type ondelete: str
+    :return: tuple with target pydantic type, list of fk constraints and target col type
+    :rtype: Tuple[Any, List, Any]
+    """
+    fk_string = to.Meta.tablename + "." + to.get_column_alias(to.Meta.pkname)
+    to_field = to.Meta.model_fields[to.Meta.pkname]
+    pk_only_model = create_dummy_model(to, to_field)
+    __type__ = (
+        Union[to_field.__type__, to, pk_only_model]
+        if not nullable
+        else Optional[Union[to_field.__type__, to, pk_only_model]]
+    )
+    constraints = [
+        ForeignKeyConstraint(
+            name=fk_string, ondelete=ondelete, onupdate=onupdate  # type: ignore
+        )
+    ]
+    column_type = to_field.column_type
+    return __type__, constraints, column_type
+
+
 class UniqueColumns(UniqueConstraint):
     """
     Subclass of sqlalchemy.UniqueConstraint.
@@ -86,7 +130,7 @@ class ForeignKeyConstraint:
 
 
 def ForeignKey(  # noqa CFQ002
-    to: Type["Model"],
+    to: "ToType",
     *,
     name: str = None,
     unique: bool = False,
@@ -127,27 +171,32 @@ def ForeignKey(  # noqa CFQ002
     :return: ormar ForeignKeyField with relation to selected model
     :rtype: ForeignKeyField
     """
-    fk_string = to.Meta.tablename + "." + to.get_column_alias(to.Meta.pkname)
+
-    to_field = to.Meta.model_fields[to.Meta.pkname]
+    owner = kwargs.pop("owner", None)
-    pk_only_model = create_dummy_model(to, to_field)
+    self_reference = kwargs.pop("self_reference", False)
-    __type__ = (
+
-        Union[to_field.__type__, to, pk_only_model]
+    if to.__class__ == ForwardRef:
-        if not nullable
+        __type__ = to if not nullable else Optional[to]
-        else Optional[Union[to_field.__type__, to, pk_only_model]]
+        constraints: List = []
-    )
+        column_type = None
+    else:
+        __type__, constraints, column_type = populate_fk_params_based_on_to_model(
+            to=to,  # type: ignore
+            nullable=nullable,
+            ondelete=ondelete,
+            onupdate=onupdate,
+        )
+
     namespace = dict(
         __type__=__type__,
         to=to,
+        through=None,
         alias=name,
         name=kwargs.pop("real_name", None),
         nullable=nullable,
-        constraints=[
+        constraints=constraints,
-            ForeignKeyConstraint(
-                name=fk_string, ondelete=ondelete, onupdate=onupdate  # type: ignore
-            )
-        ],
         unique=unique,
-        column_type=to_field.column_type,
+        column_type=column_type,
         related_name=related_name,
         virtual=virtual,
         primary_key=False,
@@ -155,6 +204,10 @@ def ForeignKey(  # noqa CFQ002
         pydantic_only=False,
         default=None,
         server_default=None,
+        onupdate=onupdate,
+        ondelete=ondelete,
+        owner=owner,
+        self_reference=self_reference,
     )
 
     return type("ForeignKey", (ForeignKeyField, BaseField), namespace)
@@ -169,10 +222,62 @@ class ForeignKeyField(BaseField):
     name: str
     related_name: str
     virtual: bool
+    ondelete: str
+    onupdate: str
+
+    @classmethod
+    def get_source_related_name(cls) -> str:
+        """
+        Returns name to use for source relation name.
+        For FK it's the same, differs for m2m fields.
+        It's either set as `related_name` or by default it's owner model. get_name + 's'
+        :return: name of the related_name or default related name.
+        :rtype: str
+        """
+        return cls.get_related_name()
+
+    @classmethod
+    def get_related_name(cls) -> str:
+        """
+        Returns name to use for reverse relation.
+        It's either set as `related_name` or by default it's owner model. get_name + 's'
+        :return: name of the related_name or default related name.
+        :rtype: str
+        """
+        return cls.related_name or cls.owner.get_name() + "s"
+
+    @classmethod
+    def evaluate_forward_ref(cls, globalns: Any, localns: Any) -> None:
+        """
+        Evaluates the ForwardRef to actual Field based on global and local namespaces
+
+        :param globalns: global namespace
+        :type globalns: Any
+        :param localns: local namespace
+        :type localns: Any
+        :return: None
+        :rtype: None
+        """
+        if cls.to.__class__ == ForwardRef:
+            cls.to = evaluate_forwardref(
+                cls.to,  # type: ignore
+                globalns,
+                localns or None,
+            )
+            (
+                cls.__type__,
+                cls.constraints,
+                cls.column_type,
+            ) = populate_fk_params_based_on_to_model(
+                to=cls.to,
+                nullable=cls.nullable,
+                ondelete=cls.ondelete,
+                onupdate=cls.onupdate,
+            )
 
     @classmethod
     def _extract_model_from_sequence(
-        cls, value: List, child: "Model", to_register: bool, relation_name: str
+        cls, value: List, child: "Model", to_register: bool,
     ) -> List["Model"]:
         """
         Takes a list of Models and registers them on parent.
@@ -191,17 +296,14 @@ class ForeignKeyField(BaseField):
         """
         return [
             cls.expand_relationship(  # type: ignore
-                value=val,
+                value=val, child=child, to_register=to_register,
-                child=child,
-                to_register=to_register,
-                relation_name=relation_name,
             )
             for val in value
         ]
 
     @classmethod
     def _register_existing_model(
-        cls, value: "Model", child: "Model", to_register: bool, relation_name: str
+        cls, value: "Model", child: "Model", to_register: bool,
     ) -> "Model":
         """
         Takes already created instance and registers it for parent.
@@ -219,12 +321,12 @@ class ForeignKeyField(BaseField):
         :rtype: Model
         """
         if to_register:
-            cls.register_relation(model=value, child=child, relation_name=relation_name)
+            cls.register_relation(model=value, child=child)
         return value
 
     @classmethod
     def _construct_model_from_dict(
-        cls, value: dict, child: "Model", to_register: bool, relation_name: str
+        cls, value: dict, child: "Model", to_register: bool
     ) -> "Model":
         """
         Takes a dictionary, creates a instance and registers it for parent.
@@ -246,12 +348,12 @@ class ForeignKeyField(BaseField):
             value["__pk_only__"] = True
         model = cls.to(**value)
         if to_register:
-            cls.register_relation(model=model, child=child, relation_name=relation_name)
+            cls.register_relation(model=model, child=child)
         return model
 
     @classmethod
     def _construct_model_from_pk(
-        cls, value: Any, child: "Model", to_register: bool, relation_name: str
+        cls, value: Any, child: "Model", to_register: bool
     ) -> "Model":
         """
         Takes a pk value, creates a dummy instance and registers it for parent.
@@ -278,13 +380,11 @@ class ForeignKeyField(BaseField):
             )
         model = create_dummy_instance(fk=cls.to, pk=value)
         if to_register:
-            cls.register_relation(model=model, child=child, relation_name=relation_name)
+            cls.register_relation(model=model, child=child)
         return model
 
     @classmethod
-    def register_relation(
+    def register_relation(cls, model: "Model", child: "Model") -> None:
-        cls, model: "Model", child: "Model", relation_name: str
-    ) -> None:
         """
         Registers relation between parent and child in relation manager.
         Relation manager is kep on each model (different instance).
@@ -298,20 +398,26 @@ class ForeignKeyField(BaseField):
         :type child: Model class
         """
         model._orm.add(
-            parent=model,
+            parent=model, child=child, field=cls,
-            child=child,
-            child_name=cls.related_name or child.get_name() + "s",
-            virtual=cls.virtual,
-            relation_name=relation_name,
         )
 
+    @classmethod
+    def has_unresolved_forward_refs(cls) -> bool:
+        """
+        Verifies if the filed has any ForwardRefs that require updating before the
+        model can be used.
+
+        :return: result of the check
+        :rtype: bool
+        """
+        return cls.to.__class__ == ForwardRef
+
     @classmethod
     def expand_relationship(
         cls,
         value: Any,
         child: Union["Model", "NewBaseModel"],
         to_register: bool = True,
-        relation_name: str = None,
     ) -> Optional[Union["Model", List["Model"]]]:
         """
         For relations the child model is first constructed (if needed),
@@ -340,5 +446,5 @@ class ForeignKeyField(BaseField):
 
         model = constructors.get(  # type: ignore
             value.__class__.__name__, cls._construct_model_from_pk
-        )(value, child, to_register, relation_name)
+        )(value, child, to_register)
         return model

ormar/fields/many_to_many.py

@@ -1,23 +1,54 @@
-from typing import Any, List, Optional, TYPE_CHECKING, Type, Union
+import sys
+from typing import Any, List, Optional, TYPE_CHECKING, Tuple, Type, Union
 
-import ormar
+from pydantic.typing import ForwardRef, evaluate_forwardref
+import ormar  # noqa: I100
 from ormar.fields import BaseField
 from ormar.fields.foreign_key import ForeignKeyField
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar.models import Model
 
+    if sys.version_info < (3, 7):
+        ToType = Type["Model"]
+    else:
+        ToType = Union[Type["Model"], "ForwardRef"]
+
 REF_PREFIX = "#/components/schemas/"
 
 
+def populate_m2m_params_based_on_to_model(
+    to: Type["Model"], nullable: bool
+) -> Tuple[Any, Any]:
+    """
+    Based on target to model to which relation leads to populates the type of the
+    pydantic field to use and type of the target column field.
+
+    :param to: target related ormar Model
+    :type to: Model class
+    :param nullable: marks field as optional/ required
+    :type nullable: bool
+    :return: Tuple[List, Any]
+    :rtype: tuple with target pydantic type and target col type
+    """
+    to_field = to.Meta.model_fields[to.Meta.pkname]
+    __type__ = (
+        Union[to_field.__type__, to, List[to]]  # type: ignore
+        if not nullable
+        else Optional[Union[to_field.__type__, to, List[to]]]  # type: ignore
+    )
+    column_type = to_field.column_type
+    return __type__, column_type
+
+
 def ManyToMany(
-    to: Type["Model"],
+    to: "ToType",
-    through: Type["Model"],
+    through: "ToType",
     *,
     name: str = None,
     unique: bool = False,
     virtual: bool = False,
-    **kwargs: Any
+    **kwargs: Any,
 ) -> Any:
     """
     Despite a name it's a function that returns constructed ManyToManyField.
@@ -42,23 +73,27 @@ def ManyToMany(
     :return: ormar ManyToManyField with m2m relation to selected model
     :rtype: ManyToManyField
     """
-    to_field = to.Meta.model_fields[to.Meta.pkname]
     related_name = kwargs.pop("related_name", None)
     nullable = kwargs.pop("nullable", True)
-    __type__ = (
+    owner = kwargs.pop("owner", None)
-        Union[to_field.__type__, to, List[to]]  # type: ignore
+    self_reference = kwargs.pop("self_reference", False)
-        if not nullable
+
-        else Optional[Union[to_field.__type__, to, List[to]]]  # type: ignore
+    if to.__class__ == ForwardRef:
-    )
+        __type__ = to if not nullable else Optional[to]
+        column_type = None
+    else:
+        __type__, column_type = populate_m2m_params_based_on_to_model(
+            to=to, nullable=nullable  # type: ignore
+        )
     namespace = dict(
         __type__=__type__,
         to=to,
         through=through,
         alias=name,
         name=name,
-        nullable=True,
+        nullable=nullable,
         unique=unique,
-        column_type=to_field.column_type,
+        column_type=column_type,
         related_name=related_name,
         virtual=virtual,
         primary_key=False,
@@ -66,6 +101,8 @@ def ManyToMany(
         pydantic_only=False,
         default=None,
         server_default=None,
+        owner=owner,
+        self_reference=self_reference,
     )
 
     return type("ManyToMany", (ManyToManyField, BaseField), namespace)
@@ -76,7 +113,19 @@ class ManyToManyField(ForeignKeyField, ormar.QuerySetProtocol, ormar.RelationPro
     Actual class returned from ManyToMany function call and stored in model_fields.
     """
 
-    through: Type["Model"]
+    @classmethod
+    def get_source_related_name(cls) -> str:
+        """
+        Returns name to use for source relation name.
+        For FK it's the same, differs for m2m fields.
+        It's either set as `related_name` or by default it's field name.
+        :return: name of the related_name or default related name.
+        :rtype: str
+        """
+        return (
+            cls.through.Meta.model_fields[cls.default_source_field_name()].related_name
+            or cls.name
+        )
 
     @classmethod
     def default_target_field_name(cls) -> str:
@@ -85,4 +134,56 @@ class ManyToManyField(ForeignKeyField, ormar.QuerySetProtocol, ormar.RelationPro
         :return: name of the field
         :rtype: str
         """
-        return cls.to.get_name()
+        prefix = "from_" if cls.self_reference else ""
+        return f"{prefix}{cls.to.get_name()}"
+
+    @classmethod
+    def default_source_field_name(cls) -> str:
+        """
+        Returns default target model name on through model.
+        :return: name of the field
+        :rtype: str
+        """
+        prefix = "to_" if cls.self_reference else ""
+        return f"{prefix}{cls.owner.get_name()}"
+
+    @classmethod
+    def has_unresolved_forward_refs(cls) -> bool:
+        """
+        Verifies if the filed has any ForwardRefs that require updating before the
+        model can be used.
+
+        :return: result of the check
+        :rtype: bool
+        """
+        return cls.to.__class__ == ForwardRef or cls.through.__class__ == ForwardRef
+
+    @classmethod
+    def evaluate_forward_ref(cls, globalns: Any, localns: Any) -> None:
+        """
+        Evaluates the ForwardRef to actual Field based on global and local namespaces
+
+        :param globalns: global namespace
+        :type globalns: Any
+        :param localns: local namespace
+        :type localns: Any
+        :return: None
+        :rtype: None
+        """
+        if cls.to.__class__ == ForwardRef:
+            cls.to = evaluate_forwardref(
+                cls.to,  # type: ignore
+                globalns,
+                localns or None,
+            )
+
+            (cls.__type__, cls.column_type,) = populate_m2m_params_based_on_to_model(
+                to=cls.to, nullable=cls.nullable,
+            )
+
+        if cls.through.__class__ == ForwardRef:
+            cls.through = evaluate_forwardref(
+                cls.through,  # type: ignore
+                globalns,
+                localns or None,
+            )

ormar/models/helpers/models.py

@@ -1,11 +1,29 @@
-from typing import Dict, List, Optional, TYPE_CHECKING, Tuple, Type
+import itertools
+from typing import Any, Dict, List, Optional, TYPE_CHECKING, Tuple, Type
 
-import ormar
+import ormar  # noqa: I100
 from ormar.fields.foreign_key import ForeignKeyField
 from ormar.models.helpers.pydantic import populate_pydantic_default_values
+from pydantic.typing import ForwardRef
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
+    from ormar.fields import BaseField
+
+
+def is_field_an_forward_ref(field: Type["BaseField"]) -> bool:
+    """
+    Checks if field is a relation field and whether any of the referenced models
+    are ForwardRefs that needs to be updated before proceeding.
+
+    :param field: model field to verify
+    :type field: Type[BaseField]
+    :return: result of the check
+    :rtype: bool
+    """
+    return issubclass(field, ForeignKeyField) and (
+        field.to.__class__ == ForwardRef or field.through.__class__ == ForwardRef
+    )
 
 
 def populate_default_options_values(
@@ -33,6 +51,13 @@ def populate_default_options_values(
     if not hasattr(new_model.Meta, "abstract"):
         new_model.Meta.abstract = False
 
+    if any(
+        is_field_an_forward_ref(field) for field in new_model.Meta.model_fields.values()
+    ):
+        new_model.Meta.requires_ref_update = True
+    else:
+        new_model.Meta.requires_ref_update = False
+
 
 def extract_annotations_and_default_vals(attrs: Dict) -> Tuple[Dict, Dict]:
     """
@@ -51,7 +76,7 @@ def extract_annotations_and_default_vals(attrs: Dict) -> Tuple[Dict, Dict]:
 
 
 # cannot be in relations helpers due to cyclical import
-def validate_related_names_in_relations(
+def validate_related_names_in_relations(  # noqa CCR001
     model_fields: Dict, new_model: Type["Model"]
 ) -> None:
     """
@@ -70,7 +95,12 @@ def validate_related_names_in_relations(
     already_registered: Dict[str, List[Optional[str]]] = dict()
     for field in model_fields.values():
         if issubclass(field, ForeignKeyField):
-            previous_related_names = already_registered.setdefault(field.to, [])
+            to_name = (
+                field.to.get_name()
+                if not field.to.__class__ == ForwardRef
+                else str(field.to)
+            )
+            previous_related_names = already_registered.setdefault(to_name, [])
             if field.related_name in previous_related_names:
                 raise ormar.ModelDefinitionError(
                     f"Multiple fields declared on {new_model.get_name(lower=False)} "
@@ -80,3 +110,35 @@ def validate_related_names_in_relations(
                     f"\nTip: provide different related_name for FK and/or M2M fields"
                 )
             previous_related_names.append(field.related_name)
+
+
+def group_related_list(list_: List) -> Dict:
+    """
+    Translates the list of related strings into a dictionary.
+    That way nested models are grouped to traverse them in a right order
+    and to avoid repetition.
+
+    Sample: ["people__houses", "people__cars__models", "people__cars__colors"]
+    will become:
+    {'people': {'houses': [], 'cars': ['models', 'colors']}}
+
+    Result dictionary is sorted by length of the values and by key
+
+    :param list_: list of related models used in select related
+    :type list_: List[str]
+    :return: list converted to dictionary to avoid repetition and group nested models
+    :rtype: Dict[str, List]
+    """
+    result_dict: Dict[str, Any] = dict()
+    list_.sort(key=lambda x: x.split("__")[0])
+    grouped = itertools.groupby(list_, key=lambda x: x.split("__")[0])
+    for key, group in grouped:
+        group_list = list(group)
+        new = sorted(
+            ["__".join(x.split("__")[1:]) for x in group_list if len(x.split("__")) > 1]
+        )
+        if any("__" in x for x in new):
+            result_dict[key] = group_related_list(new)
+        else:
+            result_dict.setdefault(key, []).extend(new)
+    return {k: v for k, v in sorted(result_dict.items(), key=lambda item: len(item[1]))}

ormar/models/helpers/relations.py

@@ -13,7 +13,7 @@ if TYPE_CHECKING:  # pragma no cover
 alias_manager = AliasManager()
 
 
-def register_relation_on_build(new_model: Type["Model"], field_name: str) -> None:
+def register_relation_on_build(field: Type["ForeignKeyField"]) -> None:
     """
     Registers ForeignKey relation in alias_manager to set a table_prefix.
     Registration include also reverse relation side to be able to join both sides.
@@ -22,17 +22,17 @@ def register_relation_on_build(new_model: Type["Model"], field_name: str) -> Non
     relations between two Models that needs to have different
     aliases for proper sql joins.
 
-    :param new_model: constructed model
+    :param field: relation field
-    :type new_model: Model class
+    :type field: ForeignKey class
-    :param field_name: name of the related field
-    :type field_name: str
     """
-    alias_manager.add_relation_type(new_model, field_name)
+    alias_manager.add_relation_type(
+        source_model=field.owner,
+        relation_name=field.name,
+        reverse_name=field.get_source_related_name(),
+    )
 
 
-def register_many_to_many_relation_on_build(
+def register_many_to_many_relation_on_build(field: Type[ManyToManyField]) -> None:
-    new_model: Type["Model"], field: Type[ManyToManyField], field_name: str
-) -> None:
     """
     Registers connection between through model and both sides of the m2m relation.
     Registration include also reverse relation side to be able to join both sides.
@@ -43,24 +43,34 @@ def register_many_to_many_relation_on_build(
 
     By default relation name is a model.name.lower().
 
-    :param field_name: name of the relation key
-    :type field_name: str
-    :param new_model: model on which m2m field is declared
-    :type new_model: Model class
     :param field: relation field
     :type field: ManyToManyField class
     """
     alias_manager.add_relation_type(
-        field.through, new_model.get_name(), is_multi=True, reverse_name=field_name
+        source_model=field.through,
+        relation_name=field.default_source_field_name(),
+        reverse_name=field.get_source_related_name(),
     )
     alias_manager.add_relation_type(
-        field.through,
+        source_model=field.through,
-        field.to.get_name(),
+        relation_name=field.default_target_field_name(),
-        is_multi=True,
+        reverse_name=field.get_related_name(),
-        reverse_name=field.related_name or new_model.get_name() + "s",
     )
 
 
+def expand_reverse_relationship(model_field: Type["ForeignKeyField"]) -> None:
+    """
+    If the reverse relation has not been set before it's set here.
+
+    :param model_field:
+    :type model_field:
+    :return: None
+    :rtype: None
+    """
+    if reverse_field_not_already_registered(model_field=model_field):
+        register_reverse_model_fields(model_field=model_field)
+
+
 def expand_reverse_relationships(model: Type["Model"]) -> None:
     """
     Iterates through model_fields of given model and verifies if all reverse
@@ -72,24 +82,14 @@ def expand_reverse_relationships(model: Type["Model"]) -> None:
     :type model: Model class
     """
     for model_field in model.Meta.model_fields.values():
-        if issubclass(model_field, ForeignKeyField):
+        if (
-            child_model_name = model_field.related_name or model.get_name() + "s"
+            issubclass(model_field, ForeignKeyField)
-            parent_model = model_field.to
+            and not model_field.has_unresolved_forward_refs()
-            child = model
+        ):
-            if reverse_field_not_already_registered(
+            expand_reverse_relationship(model_field=model_field)
-                child, child_model_name, parent_model
-            ):
-                register_reverse_model_fields(
-                    parent_model, child, child_model_name, model_field
-                )
 
 
-def register_reverse_model_fields(
+def register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None:
-    model: Type["Model"],
-    child: Type["Model"],
-    related_name: str,
-    model_field: Type["ForeignKeyField"],
-) -> None:
     """
     Registers reverse ForeignKey field on related model.
     By default it's name.lower()+'s' of the model on which relation is defined.
@@ -97,34 +97,35 @@ def register_reverse_model_fields(
     But if the related_model name is provided it's registered with that name.
     Autogenerated reverse fields also set related_name to the original field name.
 
-    :param model: related model on which reverse field should be defined
-    :type model: Model class
-    :param child: parent model with relation definition
-    :type child: Model class
-    :param related_name: name by which reverse key should be registered
-    :type related_name: str
     :param model_field: original relation ForeignKey field
     :type model_field: relation Field
     """
+    related_name = model_field.get_related_name()
     if issubclass(model_field, ManyToManyField):
-        model.Meta.model_fields[related_name] = ManyToMany(
+        model_field.to.Meta.model_fields[related_name] = ManyToMany(
-            child,
+            model_field.owner,
             through=model_field.through,
             name=related_name,
             virtual=True,
             related_name=model_field.name,
+            owner=model_field.to,
+            self_reference=model_field.self_reference,
+            self_reference_primary=model_field.self_reference_primary,
         )
         # register foreign keys on through model
-        adjust_through_many_to_many_model(model, child, model_field)
+        adjust_through_many_to_many_model(model_field=model_field)
     else:
-        model.Meta.model_fields[related_name] = ForeignKey(
+        model_field.to.Meta.model_fields[related_name] = ForeignKey(
-            child, real_name=related_name, virtual=True, related_name=model_field.name,
+            model_field.owner,
+            real_name=related_name,
+            virtual=True,
+            related_name=model_field.name,
+            owner=model_field.to,
+            self_reference=model_field.self_reference,
         )
 
 
-def register_relation_in_alias_manager(
+def register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None:
-    new_model: Type["Model"], field: Type[ForeignKeyField], field_name: str
-) -> None:
     """
     Registers the relation (and reverse relation) in alias manager.
     The m2m relations require registration of through model between
@@ -134,23 +135,21 @@ def register_relation_in_alias_manager(
     m2m - register_many_to_many_relation_on_build
     fk - register_relation_on_build
 
-    :param new_model: model on which relation field is declared
-    :type new_model: Model class
     :param field: relation field
     :type field: ForeignKey or ManyToManyField class
-    :param field_name: name of the relation key
-    :type field_name: str
     """
     if issubclass(field, ManyToManyField):
-        register_many_to_many_relation_on_build(
+        if field.has_unresolved_forward_refs():
-            new_model=new_model, field=field, field_name=field_name
+            return
-        )
+        register_many_to_many_relation_on_build(field=field)
     elif issubclass(field, ForeignKeyField):
-        register_relation_on_build(new_model=new_model, field_name=field_name)
+        if field.has_unresolved_forward_refs():
+            return
+        register_relation_on_build(field=field)
 
 
 def verify_related_name_dont_duplicate(
-    child: Type["Model"], parent_model: Type["Model"], related_name: str,
+    related_name: str, model_field: Type["ForeignKeyField"]
 ) -> None:
     """
     Verifies whether the used related_name (regardless of the fact if user defined or
@@ -159,59 +158,51 @@ def verify_related_name_dont_duplicate(
 
     :raises ModelDefinitionError: if name is already used but lead to different related
     model
-    :param child: related Model class
-    :type child: ormar.models.metaclass.ModelMetaclass
-    :param parent_model: parent Model class
-    :type parent_model: ormar.models.metaclass.ModelMetaclass
     :param related_name:
     :type related_name:
+    :param model_field: original relation ForeignKey field
+    :type model_field: relation Field
     :return: None
     :rtype: None
     """
-    if parent_model.Meta.model_fields.get(related_name):
+    fk_field = model_field.to.Meta.model_fields.get(related_name)
-        fk_field = parent_model.Meta.model_fields.get(related_name)
+    if not fk_field:  # pragma: no cover
-        if not fk_field:  # pragma: no cover
+        return
-            return
+    if fk_field.to != model_field.owner and fk_field.to.Meta != model_field.owner.Meta:
-        if fk_field.to != child and fk_field.to.Meta != child.Meta:
+        raise ormar.ModelDefinitionError(
-            raise ormar.ModelDefinitionError(
+            f"Relation with related_name "
-                f"Relation with related_name "
+            f"'{related_name}' "
-                f"'{related_name}' "
+            f"leading to model "
-                f"leading to model "
+            f"{model_field.to.get_name(lower=False)} "
-                f"{parent_model.get_name(lower=False)} "
+            f"cannot be used on model "
-                f"cannot be used on model "
+            f"{model_field.owner.get_name(lower=False)} "
-                f"{child.get_name(lower=False)} "
+            f"because it's already used by model "
-                f"because it's already used by model "
+            f"{fk_field.to.get_name(lower=False)}"
-                f"{fk_field.to.get_name(lower=False)}"
+        )
-            )
 
 
-def reverse_field_not_already_registered(
+def reverse_field_not_already_registered(model_field: Type["ForeignKeyField"]) -> bool:
-    child: Type["Model"], child_model_name: str, parent_model: Type["Model"]
-) -> bool:
     """
     Checks if child is already registered in parents pydantic fields.
 
     :raises ModelDefinitionError: if related name is already used but lead to different
     related model
-    :param child: related Model class
+    :param model_field: original relation ForeignKey field
-    :type child: ormar.models.metaclass.ModelMetaclass
+    :type model_field: relation Field
-    :param child_model_name: related_name of the child if provided
-    :type child_model_name: str
-    :param parent_model: parent Model class
-    :type parent_model: ormar.models.metaclass.ModelMetaclass
     :return: result of the check
     :rtype: bool
     """
-    check_result = child_model_name not in parent_model.Meta.model_fields
+    related_name = model_field.get_related_name()
-    check_result2 = child.get_name() not in parent_model.Meta.model_fields
+    check_result = related_name not in model_field.to.Meta.model_fields
+    check_result2 = model_field.owner.get_name() not in model_field.to.Meta.model_fields
 
     if not check_result:
         verify_related_name_dont_duplicate(
-            child=child, parent_model=parent_model, related_name=child_model_name
+            related_name=related_name, model_field=model_field
         )
     if not check_result2:
         verify_related_name_dont_duplicate(
-            child=child, parent_model=parent_model, related_name=child.get_name()
+            related_name=model_field.owner.get_name(), model_field=model_field
         )
 
     return check_result and check_result2

ormar/models/helpers/sqlalchemy.py

@@ -1,56 +1,67 @@
 import copy
 import logging
-from typing import Dict, List, Optional, TYPE_CHECKING, Tuple, Type
+from typing import Dict, List, Optional, TYPE_CHECKING, Tuple, Type, Union
 
 import sqlalchemy
 
 from ormar import ForeignKey, Integer, ModelDefinitionError  # noqa: I202
 from ormar.fields import BaseField, ManyToManyField
+from ormar.fields.foreign_key import ForeignKeyField
 from ormar.models.helpers.models import validate_related_names_in_relations
 from ormar.models.helpers.pydantic import create_pydantic_field
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model, ModelMeta
+    from ormar.models import NewBaseModel
 
 
-def adjust_through_many_to_many_model(
+def adjust_through_many_to_many_model(model_field: Type[ManyToManyField]) -> None:
-    model: Type["Model"], child: Type["Model"], model_field: Type[ManyToManyField]
-) -> None:
     """
     Registers m2m relation on through model.
     Sets ormar.ForeignKey from through model to both child and parent models.
     Sets sqlalchemy.ForeignKey to both child and parent models.
     Sets pydantic fields with child and parent model types.
 
-    :param model: model on which relation is declared
-    :type model: Model class
-    :param child: model to which m2m relation leads
-    :type child: Model class
     :param model_field: relation field defined in parent model
     :type model_field: ManyToManyField
     """
-    model_field.through.Meta.model_fields[model.get_name()] = ForeignKey(
+    parent_name = model_field.default_target_field_name()
-        model, real_name=model.get_name(), ondelete="CASCADE"
+    child_name = model_field.default_source_field_name()
+
+    model_field.through.Meta.model_fields[parent_name] = ForeignKey(
+        model_field.to,
+        real_name=parent_name,
+        ondelete="CASCADE",
+        owner=model_field.through,
     )
-    model_field.through.Meta.model_fields[child.get_name()] = ForeignKey(
+    model_field.through.Meta.model_fields[child_name] = ForeignKey(
-        child, real_name=child.get_name(), ondelete="CASCADE"
+        model_field.owner,
+        real_name=child_name,
+        ondelete="CASCADE",
+        owner=model_field.through,
     )
 
-    create_and_append_m2m_fk(model, model_field)
+    create_and_append_m2m_fk(
-    create_and_append_m2m_fk(child, model_field)
+        model=model_field.to, model_field=model_field, field_name=parent_name
+    )
+    create_and_append_m2m_fk(
+        model=model_field.owner, model_field=model_field, field_name=child_name
+    )
 
-    create_pydantic_field(model.get_name(), model, model_field)
+    create_pydantic_field(parent_name, model_field.to, model_field)
-    create_pydantic_field(child.get_name(), child, model_field)
+    create_pydantic_field(child_name, model_field.owner, model_field)
 
 
 def create_and_append_m2m_fk(
-    model: Type["Model"], model_field: Type[ManyToManyField]
+    model: Type["Model"], model_field: Type[ManyToManyField], field_name: str
 ) -> None:
     """
-    Registers sqlalchemy Column with sqlalchemy.ForeignKey leadning to the model.
+    Registers sqlalchemy Column with sqlalchemy.ForeignKey leading to the model.
 
     Newly created field is added to m2m relation through model Meta columns and table.
 
+    :param field_name: name of the column to create
+    :type field_name: str
     :param model: Model class to which FK should be created
     :type model: Model class
     :param model_field: field with ManyToMany relation
@@ -63,7 +74,7 @@ def create_and_append_m2m_fk(
             "ManyToMany relation cannot lead to field without pk"
         )
     column = sqlalchemy.Column(
-        model.get_name(),
+        field_name,
         pk_column.type,
         sqlalchemy.schema.ForeignKey(
             model.Meta.tablename + "." + pk_alias,
@@ -72,7 +83,6 @@ def create_and_append_m2m_fk(
         ),
     )
     model_field.through.Meta.columns.append(column)
-    # breakpoint()
     model_field.through.Meta.table.append_column(copy.deepcopy(column))
 
 
@@ -121,6 +131,8 @@ def sqlalchemy_columns_from_model_fields(
     Append fields to columns if it's not pydantic_only,
     virtual ForeignKey or ManyToMany field.
 
+    Sets `owner` on each model_field as reference to newly created Model.
+
     :raises ModelDefinitionError: if validation of related_names fail,
     or pkname validation fails.
     :param model_fields: dictionary of declared ormar model fields
@@ -140,6 +152,7 @@ def sqlalchemy_columns_from_model_fields(
     columns = []
     pkname = None
     for field_name, field in model_fields.items():
+        field.owner = new_model
         if field.primary_key:
             pkname = check_pk_column_validity(field_name, field, pkname)
         if (
@@ -194,6 +207,20 @@ def populate_meta_tablename_columns_and_pk(
     return new_model
 
 
+def check_for_null_type_columns_from_forward_refs(meta: "ModelMeta") -> bool:
+    """
+    Check is any column is of NUllType() meaning it's empty column from ForwardRef
+
+    :param meta: Meta class of the Model without sqlalchemy table constructed
+    :type meta: Model class Meta
+    :return: result of the check
+    :rtype: bool
+    """
+    return not any(
+        isinstance(col.type, sqlalchemy.sql.sqltypes.NullType) for col in meta.columns
+    )
+
+
 def populate_meta_sqlalchemy_table_if_required(meta: "ModelMeta") -> None:
     """
     Constructs sqlalchemy table out of columns and parameters set on Meta class.
@@ -204,10 +231,33 @@ def populate_meta_sqlalchemy_table_if_required(meta: "ModelMeta") -> None:
     :return: class with populated Meta.table
     :rtype: Model class
     """
-    if not hasattr(meta, "table"):
+    if not hasattr(meta, "table") and check_for_null_type_columns_from_forward_refs(
+        meta
+    ):
         meta.table = sqlalchemy.Table(
             meta.tablename,
             meta.metadata,
             *[copy.deepcopy(col) for col in meta.columns],
             *meta.constraints,
         )
+
+
+def update_column_definition(
+    model: Union[Type["Model"], Type["NewBaseModel"]], field: Type[ForeignKeyField]
+) -> None:
+    """
+    Updates a column with a new type column based on updated parameters in FK fields.
+
+    :param model: model on which columns needs to be updated
+    :type model: Type["Model"]
+    :param field: field with column definition that requires update
+    :type field: Type[ForeignKeyField]
+    :return: None
+    :rtype: None
+    """
+    columns = model.Meta.columns
+    for ind, column in enumerate(columns):
+        if column.name == field.get_alias():
+            new_column = field.get_column(field.get_alias())
+            columns[ind] = new_column
+            break

ormar/models/metaclass.py

@@ -66,6 +66,7 @@ class ModelMeta:
     property_fields: Set
     signals: SignalEmitter
     abstract: bool
+    requires_ref_update: bool
 
 
 def check_if_field_has_choices(field: Type[BaseField]) -> bool:
@@ -220,7 +221,7 @@ def update_attrs_and_fields(
 
     :param attrs: new namespace for class being constructed
     :type attrs: Dict
-    :param new_attrs: part of the namespace extracted from parent class
+    :param new_attrs: related of the namespace extracted from parent class
     :type new_attrs: Dict
     :param model_fields: ormar fields in defined in current class
     :type model_fields: Dict[str, BaseField]
@@ -585,8 +586,8 @@ class ModelMetaclass(pydantic.main.ModelMetaclass):
                 new_model = populate_meta_tablename_columns_and_pk(name, new_model)
                 populate_meta_sqlalchemy_table_if_required(new_model.Meta)
                 expand_reverse_relationships(new_model)
-                for field_name, field in new_model.Meta.model_fields.items():
+                for field in new_model.Meta.model_fields.values():
-                    register_relation_in_alias_manager(new_model, field, field_name)
+                    register_relation_in_alias_manager(field=field)
 
                 if new_model.Meta.pkname not in attrs["__annotations__"]:
                     field_name = new_model.Meta.pkname

ormar/models/mixins/prefetch_mixin.py

@@ -1,7 +1,7 @@
 from typing import Callable, Dict, List, TYPE_CHECKING, Tuple, Type
 
 import ormar
-from ormar.fields import BaseField
+from ormar.fields.foreign_key import ForeignKeyField
 from ormar.models.mixins.relation_mixin import RelationMixin
 
 
@@ -37,10 +37,7 @@ class PrefetchQueryMixin(RelationMixin):
         :rtype: Tuple[Type[Model], str]
         """
         if reverse:
-            field_name = (
+            field_name = parent_model.Meta.model_fields[related].get_related_name()
-                parent_model.Meta.model_fields[related].related_name
-                or parent_model.get_name() + "s"
-            )
             field = target_model.Meta.model_fields[field_name]
             if issubclass(field, ormar.fields.ManyToManyField):
                 field_name = field.default_target_field_name()
@@ -79,7 +76,7 @@ class PrefetchQueryMixin(RelationMixin):
         return column.get_alias() if use_raw else column.name
 
     @classmethod
-    def get_related_field_name(cls, target_field: Type["BaseField"]) -> str:
+    def get_related_field_name(cls, target_field: Type["ForeignKeyField"]) -> str:
         """
         Returns name of the relation field that should be used in prefetch query.
         This field is later used to register relation in prefetch query,
@@ -93,7 +90,7 @@ class PrefetchQueryMixin(RelationMixin):
         if issubclass(target_field, ormar.fields.ManyToManyField):
             return cls.get_name()
         if target_field.virtual:
-            return target_field.related_name or cls.get_name() + "s"
+            return target_field.get_related_name()
         return target_field.to.Meta.pkname
 
     @classmethod

ormar/models/model.py

@@ -1,4 +1,3 @@
-import itertools
 from typing import (
     Any,
     Dict,
@@ -18,38 +17,9 @@ import ormar.queryset  # noqa I100
 from ormar.exceptions import ModelPersistenceError, NoMatch
 from ormar.fields.many_to_many import ManyToManyField
 from ormar.models import NewBaseModel  # noqa I100
+from ormar.models.helpers.models import group_related_list
 from ormar.models.metaclass import ModelMeta
 
-
-def group_related_list(list_: List) -> Dict:
-    """
-    Translates the list of related strings into a dictionary.
-    That way nested models are grouped to traverse them in a right order
-    and to avoid repetition.
-
-    Sample: ["people__houses", "people__cars__models", "people__cars__colors"]
-    will become:
-    {'people': {'houses': [], 'cars': ['models', 'colors']}}
-
-    :param list_: list of related models used in select related
-    :type list_: List[str]
-    :return: list converted to dictionary to avoid repetition and group nested models
-    :rtype: Dict[str, List]
-    """
-    test_dict: Dict[str, Any] = dict()
-    grouped = itertools.groupby(list_, key=lambda x: x.split("__")[0])
-    for key, group in grouped:
-        group_list = list(group)
-        new = [
-            "__".join(x.split("__")[1:]) for x in group_list if len(x.split("__")) > 1
-        ]
-        if any("__" in x for x in new):
-            test_dict[key] = group_related_list(new)
-        else:
-            test_dict[key] = new
-    return test_dict
-
-
 if TYPE_CHECKING:  # pragma nocover
     from ormar import QuerySet
 
@@ -73,9 +43,11 @@ class Model(NewBaseModel):
         select_related: List = None,
         related_models: Any = None,
         previous_model: Type[T] = None,
+        source_model: Type[T] = None,
         related_name: str = None,
         fields: Optional[Union[Dict, Set]] = None,
         exclude_fields: Optional[Union[Dict, Set]] = None,
+        current_relation_str: str = None,
     ) -> Optional[T]:
         """
         Model method to convert raw sql row from database into ormar.Model instance.
@@ -112,7 +84,10 @@ class Model(NewBaseModel):
         item: Dict[str, Any] = {}
         select_related = select_related or []
         related_models = related_models or []
+        table_prefix = ""
+
         if select_related:
+            source_model = cls
             related_models = group_related_list(select_related)
 
         rel_name2 = related_name
@@ -125,15 +100,24 @@ class Model(NewBaseModel):
             )
         ):
             through_field = previous_model.Meta.model_fields[related_name]
-            rel_name2 = through_field.default_target_field_name()  # type: ignore
+            if (
+                through_field.self_reference
+                and related_name == through_field.self_reference_primary
+            ):
+                rel_name2 = through_field.default_source_field_name()  # type: ignore
+            else:
+                rel_name2 = through_field.default_target_field_name()  # type: ignore
             previous_model = through_field.through  # type: ignore
 
         if previous_model and rel_name2:
-            table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
+            if current_relation_str and "__" in current_relation_str and source_model:
-                previous_model, rel_name2
+                table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
-            )
+                    from_model=source_model, relation_name=current_relation_str
-        else:
+                )
-            table_prefix = ""
+            if not table_prefix:
+                table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
+                    from_model=previous_model, relation_name=rel_name2
+                )
 
         item = cls.populate_nested_models_from_row(
             item=item,
@@ -141,6 +125,8 @@ class Model(NewBaseModel):
             related_models=related_models,
             fields=fields,
             exclude_fields=exclude_fields,
+            current_relation_str=current_relation_str,
+            source_model=source_model,
         )
         item = cls.extract_prefixed_table_columns(
             item=item,
@@ -157,8 +143,6 @@ class Model(NewBaseModel):
             )
             instance = cls(**item)
             instance.set_save_status(True)
-        else:
-            instance = None
         return instance
 
     @classmethod
@@ -169,6 +153,8 @@ class Model(NewBaseModel):
         related_models: Any,
         fields: Optional[Union[Dict, Set]] = None,
         exclude_fields: Optional[Union[Dict, Set]] = None,
+        current_relation_str: str = None,
+        source_model: Type[T] = None,
     ) -> dict:
         """
         Traverses structure of related models and populates the nested models
@@ -180,6 +166,10 @@ class Model(NewBaseModel):
         Recurrently calls from_row method on nested instances and create nested
         instances. In the end those instances are added to the final model dictionary.
 
+        :param source_model: source model from which relation started
+        :type source_model: Type[Model]
+        :param current_relation_str: joined related parts into one string
+        :type current_relation_str: str
         :param item: dictionary of already populated nested models, otherwise empty dict
         :type item: Dict
         :param row: raw result row from the database
@@ -196,35 +186,31 @@ class Model(NewBaseModel):
         and values are database values
         :rtype: Dict
         """
+
         for related in related_models:
+            relation_str = (
+                "__".join([current_relation_str, related])
+                if current_relation_str
+                else related
+            )
+            fields = cls.get_included(fields, related)
+            exclude_fields = cls.get_excluded(exclude_fields, related)
+            model_cls = cls.Meta.model_fields[related].to
+
+            remainder = None
             if isinstance(related_models, dict) and related_models[related]:
-                first_part, remainder = related, related_models[related]
+                remainder = related_models[related]
-                model_cls = cls.Meta.model_fields[first_part].to
+            child = model_cls.from_row(
-
+                row,
-                fields = cls.get_included(fields, first_part)
+                related_models=remainder,
-                exclude_fields = cls.get_excluded(exclude_fields, first_part)
+                previous_model=cls,
-
+                related_name=related,
-                child = model_cls.from_row(
+                fields=fields,
-                    row,
+                exclude_fields=exclude_fields,
-                    related_models=remainder,
+                current_relation_str=relation_str,
-                    previous_model=cls,
+                source_model=source_model,
-                    related_name=related,
+            )
-                    fields=fields,
+            item[model_cls.get_column_name_from_alias(related)] = child
-                    exclude_fields=exclude_fields,
-                )
-                item[model_cls.get_column_name_from_alias(first_part)] = child
-            else:
-                model_cls = cls.Meta.model_fields[related].to
-                fields = cls.get_included(fields, related)
-                exclude_fields = cls.get_excluded(exclude_fields, related)
-                child = model_cls.from_row(
-                    row,
-                    previous_model=cls,
-                    related_name=related,
-                    fields=fields,
-                    exclude_fields=exclude_fields,
-                )
-                item[model_cls.get_column_name_from_alias(related)] = child
 
         return item
 
@@ -245,7 +231,7 @@ class Model(NewBaseModel):
         All joined tables have prefixes to allow duplicate column names,
         as well as duplicated joins to the same table from multiple different tables.
 
-        Extracted fields populates the item dict later used to construct a Model.
+        Extracted fields populates the related dict later used to construct a Model.
 
         Used in Model.from_row and PrefetchQuery._populate_rows methods.
 

ormar/models/newbasemodel.py

@@ -1,7 +1,4 @@
-try:
+import sys
-    import orjson as json
-except ImportError:  # pragma: no cover
-    import json  # type: ignore
 import uuid
 from typing import (
     AbstractSet,
@@ -18,16 +15,30 @@ from typing import (
     Type,
     TypeVar,
     Union,
+    cast,
 )
 
+try:
+    import orjson as json
+except ImportError:  # pragma: no cover
+    import json  # type: ignore
+
+
 import databases
 import pydantic
 import sqlalchemy
 from pydantic import BaseModel
 
 import ormar  # noqa I100
-from ormar.exceptions import ModelError
+from ormar.exceptions import ModelError, ModelPersistenceError
 from ormar.fields import BaseField
+from ormar.fields.foreign_key import ForeignKeyField
+from ormar.models.helpers import register_relation_in_alias_manager
+from ormar.models.helpers.relations import expand_reverse_relationship
+from ormar.models.helpers.sqlalchemy import (
+    populate_meta_sqlalchemy_table_if_required,
+    update_column_definition,
+)
 from ormar.models.metaclass import ModelMeta, ModelMetaclass
 from ormar.models.modelproxy import ModelTableProxy
 from ormar.queryset.utils import translate_list_to_dict
@@ -103,14 +114,14 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
         should be explicitly set to None, as otherwise pydantic will try to populate
         them with their default values if default is set.
 
-        :raises ModelError: if abstract model is initialized or unknown field is passed
+        :raises ModelError: if abstract model is initialized, model has ForwardRefs
+         that has not been updated or unknown field is passed
         :param args: ignored args
         :type args: Any
         :param kwargs: keyword arguments - all fields values and some special params
         :type kwargs: Any
         """
-        if self.Meta.abstract:
+        self._verify_model_can_be_initialized()
-            raise ModelError(f"You cannot initialize abstract model {self.get_name()}")
         object.__setattr__(self, "_orm_id", uuid.uuid4().hex)
         object.__setattr__(self, "_orm_saved", False)
         object.__setattr__(self, "_pk_column", None)
@@ -133,7 +144,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
                 k: self._convert_json(
                     k,
                     self.Meta.model_fields[k].expand_relationship(
-                        v, self, to_register=False, relation_name=k
+                        v, self, to_register=False,
                     ),
                     "dumps",
                 )
@@ -162,7 +173,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
         # register the columns models after initialization
         for related in self.extract_related_names():
             self.Meta.model_fields[related].expand_relationship(
-                new_kwargs.get(related), self, to_register=True, relation_name=related
+                new_kwargs.get(related), self, to_register=True,
             )
 
     def __setattr__(self, name: str, value: Any) -> None:  # noqa CCR001
@@ -199,7 +210,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
             self.set_save_status(False)
         elif name in self._orm:
             model = self.Meta.model_fields[name].expand_relationship(
-                value=value, child=self, relation_name=name
+                value=value, child=self
             )
             if isinstance(self.__dict__.get(name), list):
                 # virtual foreign key or many to many
@@ -265,6 +276,22 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
             return value
         return object.__getattribute__(self, item)  # pragma: no cover
 
+    def _verify_model_can_be_initialized(self) -> None:
+        """
+        Raises exception if model is abstract or has ForwardRefs in relation fields.
+
+        :return: None
+        :rtype: None
+        """
+        if self.Meta.abstract:
+            raise ModelError(f"You cannot initialize abstract model {self.get_name()}")
+        if self.Meta.requires_ref_update:
+            raise ModelError(
+                f"Model {self.get_name()} has not updated "
+                f"ForwardRefs. \nBefore using the model you "
+                f"need to call update_forward_refs()."
+            )
+
     def _extract_related_model_instead_of_field(
         self, item: str
     ) -> Optional[Union["T", Sequence["T"]]]:
@@ -398,6 +425,41 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
             props = {prop for prop in props if prop not in exclude}
         return props
 
+    @classmethod
+    def update_forward_refs(cls, **localns: Any) -> None:
+        """
+        Processes fields that are ForwardRef and need to be evaluated into actual
+        models.
+
+        Expands relationships, register relation in alias manager and substitutes
+        sqlalchemy columns with new ones with proper column type (null before).
+
+        Populates Meta table of the Model which is left empty before.
+
+        Sets self_reference flag on models that links to themselves.
+
+        Calls the pydantic method to evaluate pydantic fields.
+
+        :param localns: local namespace
+        :type localns: Any
+        :return: None
+        :rtype: None
+        """
+        globalns = sys.modules[cls.__module__].__dict__.copy()
+        globalns.setdefault(cls.__name__, cls)
+        fields_to_check = cls.Meta.model_fields.copy()
+        for field in fields_to_check.values():
+            if field.has_unresolved_forward_refs():
+                field = cast(Type[ForeignKeyField], field)
+                field.evaluate_forward_ref(globalns=globalns, localns=localns)
+                field.set_self_reference_flag()
+                expand_reverse_relationship(model_field=field)
+                register_relation_in_alias_manager(field=field)
+                update_column_definition(model=cls, field=field)
+        populate_meta_sqlalchemy_table_if_required(meta=cls.Meta)
+        super().update_forward_refs(**localns)
+        cls.Meta.requires_ref_update = False
+
     def _get_related_not_excluded_fields(
         self, include: Optional[Dict], exclude: Optional[Dict],
     ) -> List:
@@ -669,9 +731,15 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
             if self.get_column_alias(k) in self.Meta.table.columns
         }
         for field in self._extract_db_related_names():
-            target_pk_name = self.Meta.model_fields[field].to.Meta.pkname
+            relation_field = self.Meta.model_fields[field]
+            target_pk_name = relation_field.to.Meta.pkname
             target_field = getattr(self, field)
             self_fields[field] = getattr(target_field, target_pk_name, None)
+            if not relation_field.nullable and not self_fields[field]:
+                raise ModelPersistenceError(
+                    f"You cannot save {relation_field.to.get_name()} "
+                    f"model without pk set!"
+                )
         return self_fields
 
     def get_relation_model_id(self, target_field: Type["BaseField"]) -> Optional[int]:

ormar/queryset/clause.py

@@ -1,36 +1,31 @@
-from typing import Any, Dict, List, Optional, TYPE_CHECKING, Tuple, Type
+import itertools
-
+from dataclasses import dataclass
-import sqlalchemy
+from typing import Any, List, TYPE_CHECKING, Tuple, Type
-from sqlalchemy import text
 
 import ormar  # noqa I100
-from ormar.exceptions import QueryDefinitionError
+from ormar.queryset.filter_action import FilterAction
-from ormar.fields.many_to_many import ManyToManyField
+from ormar.queryset.utils import get_relationship_alias_model_and_str
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
 
-FILTER_OPERATORS = {
+
-    "exact": "__eq__",
+@dataclass
-    "iexact": "ilike",
+class Prefix:
-    "contains": "like",
+    source_model: Type["Model"]
-    "icontains": "ilike",
+    table_prefix: str
-    "startswith": "like",
+    model_cls: Type["Model"]
-    "istartswith": "ilike",
+    relation_str: str
-    "endswith": "like",
+
-    "iendswith": "ilike",
+    @property
-    "in": "in_",
+    def alias_key(self) -> str:
-    "gt": "__gt__",
+        source_model_name = self.source_model.get_name()
-    "gte": "__ge__",
+        return f"{source_model_name}_" f"{self.relation_str}"
-    "lt": "__lt__",
-    "lte": "__le__",
-}
-ESCAPE_CHARACTERS = ["%", "_"]
 
 
 class QueryClause:
     """
-    Constructs where clauses from strings passed as arguments
+    Constructs FilterActions from strings passed as arguments
     """
 
     def __init__(
@@ -43,9 +38,9 @@ class QueryClause:
         self.model_cls = model_cls
         self.table = self.model_cls.Meta.table
 
-    def filter(  # noqa: A003
+    def prepare_filter(  # noqa: A003
         self, **kwargs: Any
-    ) -> Tuple[List[sqlalchemy.sql.expression.TextClause], List[str]]:
+    ) -> Tuple[List[FilterAction], List[str]]:
         """
         Main external access point that processes the clauses into sqlalchemy text
         clauses and updates select_related list with implicit related tables
@@ -66,7 +61,7 @@ class QueryClause:
 
     def _populate_filter_clauses(
         self, **kwargs: Any
-    ) -> Tuple[List[sqlalchemy.sql.expression.TextClause], List[str]]:
+    ) -> Tuple[List[FilterAction], List[str]]:
         """
         Iterates all clauses and extracts used operator and field from related
         models if needed. Based on the chain of related names the target table
@@ -81,227 +76,84 @@ class QueryClause:
         select_related = list(self._select_related)
 
         for key, value in kwargs.items():
-            table_prefix = ""
+            filter_action = FilterAction(
-            if "__" in key:
+                filter_str=key, value=value, model_cls=self.model_cls
-                parts = key.split("__")
-
-                (
-                    op,
-                    field_name,
-                    related_parts,
-                ) = self._extract_operator_field_and_related(parts)
-
-                model_cls = self.model_cls
-                if related_parts:
-                    (
-                        select_related,
-                        table_prefix,
-                        model_cls,
-                    ) = self._determine_filter_target_table(
-                        related_parts, select_related
-                    )
-
-                table = model_cls.Meta.table
-                column = model_cls.Meta.table.columns[field_name]
-
-            else:
-                op = "exact"
-                column = self.table.columns[self.model_cls.get_column_alias(key)]
-                table = self.table
-
-            clause = self._process_column_clause_for_operator_and_value(
-                value, op, column, table, table_prefix
             )
-            filter_clauses.append(clause)
+            select_related = filter_action.update_select_related(
+                select_related=select_related
+            )
+
+            filter_clauses.append(filter_action)
+
+        self._register_complex_duplicates(select_related)
+        filter_clauses = self._switch_filter_action_prefixes(
+            filter_clauses=filter_clauses
+        )
         return filter_clauses, select_related
 
-    def _process_column_clause_for_operator_and_value(
+    def _register_complex_duplicates(self, select_related: List[str]) -> None:
-        self,
-        value: Any,
-        op: str,
-        column: sqlalchemy.Column,
-        table: sqlalchemy.Table,
-        table_prefix: str,
-    ) -> sqlalchemy.sql.expression.TextClause:
         """
-        Escapes characters if it's required.
+        Checks if duplicate aliases are presented which can happen in self relation
-        Substitutes values of the models if value is a ormar Model with its pk value.
+        or when two joins end with the same pair of models.
-        Compiles the clause.
 
-        :param value: value of the filter
+        If there are duplicates, the all duplicated joins are registered as source
-        :type value: Any
+        model and whole relation key (not just last relation name).
-        :param op: filter operator
-        :type op: str
-        :param column: column on which filter should be applied
-        :type column: sqlalchemy.sql.schema.Column
-        :param table: table on which filter should be applied
-        :type table: sqlalchemy.sql.schema.Table
-        :param table_prefix: prefix from AliasManager
-        :type table_prefix: str
-        :return: complied and escaped clause
-        :rtype: sqlalchemy.sql.elements.TextClause
-        """
-        value, has_escaped_character = self._escape_characters_in_clause(op, value)
 
-        if isinstance(value, ormar.Model):
+        :param select_related: list of relation strings
-            value = value.pk
-
-        op_attr = FILTER_OPERATORS[op]
-        clause = getattr(column, op_attr)(value)
-        clause = self._compile_clause(
-            clause,
-            column,
-            table,
-            table_prefix,
-            modifiers={"escape": "\\" if has_escaped_character else None},
-        )
-        return clause
-
-    def _determine_filter_target_table(
-        self, related_parts: List[str], select_related: List[str]
-    ) -> Tuple[List[str], str, Type["Model"]]:
-        """
-        Adds related strings to select_related list otherwise the clause would fail as
-        the required columns would not be present. That means that select_related
-        list is filled with missing values present in filters.
-
-        Walks the relation to retrieve the actual model on which the clause should be
-        constructed, extracts alias based on last relation leading to target model.
-
-        :param related_parts: list of split parts of related string
-        :type related_parts: List[str]
-        :param select_related: list of related models
         :type select_related: List[str]
-        :return: list of related models, table_prefix, final model class
+        :return: None
-        :rtype: Tuple[List[str], str, Type[Model]]
+        :rtype: None
         """
-        table_prefix = ""
+        prefixes = self._parse_related_prefixes(select_related=select_related)
-        model_cls = self.model_cls
-        select_related = [relation for relation in select_related]
 
-        # Add any implied select_related
+        manager = self.model_cls.Meta.alias_manager
-        related_str = "__".join(related_parts)
+        filtered_prefixes = sorted(prefixes, key=lambda x: x.table_prefix)
-        if related_str not in select_related:
+        grouped = itertools.groupby(filtered_prefixes, key=lambda x: x.table_prefix)
-            select_related.append(related_str)
+        for _, group in grouped:
-
+            sorted_group = sorted(
-        # Walk the relationships to the actual model class
+                group, key=lambda x: len(x.relation_str), reverse=True
-        # against which the comparison is being made.
-        previous_model = model_cls
-        for part in related_parts:
-            part2 = part
-            if issubclass(model_cls.Meta.model_fields[part], ManyToManyField):
-                through_field = model_cls.Meta.model_fields[part]
-                previous_model = through_field.through
-                part2 = through_field.default_target_field_name()  # type: ignore
-            manager = model_cls.Meta.alias_manager
-            table_prefix = manager.resolve_relation_alias(previous_model, part2)
-            model_cls = model_cls.Meta.model_fields[part].to
-            previous_model = model_cls
-        return select_related, table_prefix, model_cls
-
-    def _compile_clause(
-        self,
-        clause: sqlalchemy.sql.expression.BinaryExpression,
-        column: sqlalchemy.Column,
-        table: sqlalchemy.Table,
-        table_prefix: str,
-        modifiers: Dict,
-    ) -> sqlalchemy.sql.expression.TextClause:
-        """
-        Compiles the clause to str using appropriate database dialect, replace columns
-        names with aliased names and converts it back to TextClause.
-
-        :param clause: original not compiled clause
-        :type clause: sqlalchemy.sql.elements.BinaryExpression
-        :param column: column on which filter should be applied
-        :type column: sqlalchemy.sql.schema.Column
-        :param table: table on which filter should be applied
-        :type table: sqlalchemy.sql.schema.Table
-        :param table_prefix: prefix from AliasManager
-        :type table_prefix: str
-        :param modifiers: sqlalchemy modifiers - used only to escape chars here
-        :type modifiers: Dict[str, NoneType]
-        :return: compiled and escaped clause
-        :rtype: sqlalchemy.sql.elements.TextClause
-        """
-        for modifier, modifier_value in modifiers.items():
-            clause.modifiers[modifier] = modifier_value
-
-        clause_text = str(
-            clause.compile(
-                dialect=self.model_cls.Meta.database._backend._dialect,
-                compile_kwargs={"literal_binds": True},
             )
-        )
+            for prefix in sorted_group[:-1]:
-        alias = f"{table_prefix}_" if table_prefix else ""
+                if prefix.alias_key not in manager:
-        aliased_name = f"{alias}{table.name}.{column.name}"
+                    manager.add_alias(alias_key=prefix.alias_key)
-        clause_text = clause_text.replace(f"{table.name}.{column.name}", aliased_name)
-        clause = text(clause_text)
-        return clause
 
-    @staticmethod
+    def _parse_related_prefixes(self, select_related: List[str]) -> List[Prefix]:
-    def _escape_characters_in_clause(op: str, value: Any) -> Tuple[Any, bool]:
         """
-        Escapes the special characters ["%", "_"] if needed.
+        Walks all relation strings and parses the target models and prefixes.
-        Adds `%` for `like` queries.
 
-        :raises QueryDefinitionError: if contains or icontains is used with
+        :param select_related: list of relation strings
-        ormar model instance
+        :type select_related: List[str]
-        :param op: operator used in query
+        :return: list of parsed prefixes
-        :type op: str
+        :rtype: List[Prefix]
-        :param value: value of the filter
-        :type value: Any
-        :return: escaped value and flag if escaping is needed
-        :rtype: Tuple[Any, bool]
         """
-        has_escaped_character = False
+        prefixes: List[Prefix] = []
-
+        for related in select_related:
-        if op not in [
+            prefix = Prefix(
-            "contains",
+                self.model_cls,
-            "icontains",
+                *get_relationship_alias_model_and_str(
-            "startswith",
+                    self.model_cls, related.split("__")
-            "istartswith",
+                ),
-            "endswith",
-            "iendswith",
-        ]:
-            return value, has_escaped_character
-
-        if isinstance(value, ormar.Model):
-            raise QueryDefinitionError(
-                "You cannot use contains and icontains with instance of the Model"
             )
+            prefixes.append(prefix)
+        return prefixes
 
-        has_escaped_character = any(c for c in ESCAPE_CHARACTERS if c in value)
+    def _switch_filter_action_prefixes(
-
+        self, filter_clauses: List[FilterAction]
-        if has_escaped_character:
+    ) -> List[FilterAction]:
-            # enable escape modifier
-            for char in ESCAPE_CHARACTERS:
-                value = value.replace(char, f"\\{char}")
-        prefix = "%" if "start" not in op else ""
-        sufix = "%" if "end" not in op else ""
-        value = f"{prefix}{value}{sufix}"
-
-        return value, has_escaped_character
-
-    @staticmethod
-    def _extract_operator_field_and_related(
-        parts: List[str],
-    ) -> Tuple[str, str, Optional[List]]:
         """
-        Splits filter query key and extracts required parts.
+        Substitutes aliases for filter action if the complex key (whole relation str) is
+        present in alias_manager.
 
-        :param parts: split filter query key
+        :param filter_clauses: raw list of actions
-        :type parts: List[str]
+        :type filter_clauses: List[FilterAction]
-        :return: operator, field_name, list of related parts
+        :return: list of actions with aliases changed if needed
-        :rtype: Tuple[str, str, Optional[List]]
+        :rtype: List[FilterAction]
         """
-        if parts[-1] in FILTER_OPERATORS:
+        manager = self.model_cls.Meta.alias_manager
-            op = parts[-1]
+        for action in filter_clauses:
-            field_name = parts[-2]
+            new_alias = manager.resolve_relation_alias(
-            related_parts = parts[:-2]
+                self.model_cls, action.related_str
-        else:
+            )
-            op = "exact"
+            if "__" in action.related_str and new_alias:
-            field_name = parts[-1]
+                action.table_prefix = new_alias
-            related_parts = parts[:-1]
+        return filter_clauses
-
-        return op, field_name, related_parts

ormar/queryset/filter_action.py

@@ -0,0 +1,201 @@
+from typing import Any, Dict, List, TYPE_CHECKING, Type
+
+import sqlalchemy
+from sqlalchemy import text
+
+import ormar  # noqa: I100, I202
+from ormar.exceptions import QueryDefinitionError
+from ormar.queryset.utils import get_relationship_alias_model_and_str
+
+if TYPE_CHECKING:  # pragma: nocover
+    from ormar import Model
+
+FILTER_OPERATORS = {
+    "exact": "__eq__",
+    "iexact": "ilike",
+    "contains": "like",
+    "icontains": "ilike",
+    "startswith": "like",
+    "istartswith": "ilike",
+    "endswith": "like",
+    "iendswith": "ilike",
+    "in": "in_",
+    "gt": "__gt__",
+    "gte": "__ge__",
+    "lt": "__lt__",
+    "lte": "__le__",
+}
+ESCAPE_CHARACTERS = ["%", "_"]
+
+
+class FilterAction:
+    """
+    Filter Actions is populated by queryset when filter() is called.
+
+    All required params are extracted but kept raw until actual filter clause value
+    is required -> then the action is converted into text() clause.
+
+    Extracted in order to easily change table prefixes on complex relations.
+    """
+
+    def __init__(self, filter_str: str, value: Any, model_cls: Type["Model"]) -> None:
+        parts = filter_str.split("__")
+        if parts[-1] in FILTER_OPERATORS:
+            self.operator = parts[-1]
+            self.field_name = parts[-2]
+            self.related_parts = parts[:-2]
+        else:
+            self.operator = "exact"
+            self.field_name = parts[-1]
+            self.related_parts = parts[:-1]
+
+        self.filter_value = value
+        self.table_prefix = ""
+        self.source_model = model_cls
+        self.target_model = model_cls
+        self._determine_filter_target_table()
+        self._escape_characters_in_clause()
+
+    @property
+    def table(self) -> sqlalchemy.Table:
+        """Shortcut to sqlalchemy Table of filtered target model"""
+        return self.target_model.Meta.table
+
+    @property
+    def column(self) -> sqlalchemy.Column:
+        """Shortcut to sqlalchemy column of filtered target model"""
+        aliased_name = self.target_model.get_column_alias(self.field_name)
+        return self.target_model.Meta.table.columns[aliased_name]
+
+    def has_escaped_characters(self) -> bool:
+        """Check if value is a string that contains characters to escape"""
+        return isinstance(self.filter_value, str) and any(
+            c for c in ESCAPE_CHARACTERS if c in self.filter_value
+        )
+
+    def update_select_related(self, select_related: List[str]) -> List[str]:
+        """
+        Updates list of select related with related part included in the filter key.
+        That way If you want to just filter by relation you do not have to provide
+        select_related separately.
+
+        :param select_related: list of relation join strings
+        :type select_related: List[str]
+        :return: list of relation joins with implied joins from filter added
+        :rtype: List[str]
+        """
+        select_related = select_related[:]
+        if self.related_str and not any(
+            rel.startswith(self.related_str) for rel in select_related
+        ):
+            select_related.append(self.related_str)
+        return select_related
+
+    def _determine_filter_target_table(self) -> None:
+        """
+        Walks the relation to retrieve the actual model on which the clause should be
+        constructed, extracts alias based on last relation leading to target model.
+        """
+        (
+            self.table_prefix,
+            self.target_model,
+            self.related_str,
+        ) = get_relationship_alias_model_and_str(self.source_model, self.related_parts)
+
+    def _escape_characters_in_clause(self) -> None:
+        """
+        Escapes the special characters ["%", "_"] if needed.
+        Adds `%` for `like` queries.
+
+        :raises QueryDefinitionError: if contains or icontains is used with
+        ormar model instance
+        :return: escaped value and flag if escaping is needed
+        :rtype: Tuple[Any, bool]
+        """
+        self.has_escaped_character = False
+        if self.operator in [
+            "contains",
+            "icontains",
+            "startswith",
+            "istartswith",
+            "endswith",
+            "iendswith",
+        ]:
+            if isinstance(self.filter_value, ormar.Model):
+                raise QueryDefinitionError(
+                    "You cannot use contains and icontains with instance of the Model"
+                )
+            self.has_escaped_character = self.has_escaped_characters()
+            if self.has_escaped_character:
+                self._escape_chars()
+            self._prefix_suffix_quote()
+
+    def _escape_chars(self) -> None:
+        """Actually replaces chars to escape in value"""
+        for char in ESCAPE_CHARACTERS:
+            self.filter_value = self.filter_value.replace(char, f"\\{char}")
+
+    def _prefix_suffix_quote(self) -> None:
+        """
+        Adds % to the beginning of the value if operator checks for containment and not
+        starts with.
+
+        Adds % to the end of the value if operator checks for containment and not
+        end with.
+        :return:
+        :rtype:
+        """
+        prefix = "%" if "start" not in self.operator else ""
+        sufix = "%" if "end" not in self.operator else ""
+        self.filter_value = f"{prefix}{self.filter_value}{sufix}"
+
+    def get_text_clause(self,) -> sqlalchemy.sql.expression.TextClause:
+        """
+        Escapes characters if it's required.
+        Substitutes values of the models if value is a ormar Model with its pk value.
+        Compiles the clause.
+
+        :return: complied and escaped clause
+        :rtype: sqlalchemy.sql.elements.TextClause
+        """
+
+        if isinstance(self.filter_value, ormar.Model):
+            self.filter_value = self.filter_value.pk
+
+        op_attr = FILTER_OPERATORS[self.operator]
+        clause = getattr(self.column, op_attr)(self.filter_value)
+        clause = self._compile_clause(
+            clause, modifiers={"escape": "\\" if self.has_escaped_character else None},
+        )
+        return clause
+
+    def _compile_clause(
+        self, clause: sqlalchemy.sql.expression.BinaryExpression, modifiers: Dict,
+    ) -> sqlalchemy.sql.expression.TextClause:
+        """
+        Compiles the clause to str using appropriate database dialect, replace columns
+        names with aliased names and converts it back to TextClause.
+
+        :param clause: original not compiled clause
+        :type clause: sqlalchemy.sql.elements.BinaryExpression
+        :param modifiers: sqlalchemy modifiers - used only to escape chars here
+        :type modifiers: Dict[str, NoneType]
+        :return: compiled and escaped clause
+        :rtype: sqlalchemy.sql.elements.TextClause
+        """
+        for modifier, modifier_value in modifiers.items():
+            clause.modifiers[modifier] = modifier_value
+
+        clause_text = str(
+            clause.compile(
+                dialect=self.target_model.Meta.database._backend._dialect,
+                compile_kwargs={"literal_binds": True},
+            )
+        )
+        alias = f"{self.table_prefix}_" if self.table_prefix else ""
+        aliased_name = f"{alias}{self.table.name}.{self.column.name}"
+        clause_text = clause_text.replace(
+            f"{self.table.name}.{self.column.name}", aliased_name
+        )
+        clause = text(clause_text)
+        return clause

ormar/queryset/filter_query.py

@@ -1,6 +1,7 @@
 from typing import List
 
 import sqlalchemy
+from ormar.queryset.filter_action import FilterAction
 
 
 class FilterQuery:
@@ -8,7 +9,9 @@ class FilterQuery:
     Modifies the select query with given list of where/filter clauses.
     """
 
-    def __init__(self, filter_clauses: List, exclude: bool = False) -> None:
+    def __init__(
+        self, filter_clauses: List[FilterAction], exclude: bool = False
+    ) -> None:
         self.exclude = exclude
         self.filter_clauses = filter_clauses
 
@@ -23,9 +26,11 @@ class FilterQuery:
         """
         if self.filter_clauses:
             if len(self.filter_clauses) == 1:
-                clause = self.filter_clauses[0]
+                clause = self.filter_clauses[0].get_text_clause()
             else:
-                clause = sqlalchemy.sql.and_(*self.filter_clauses)
+                clause = sqlalchemy.sql.and_(
+                    *[x.get_text_clause() for x in self.filter_clauses]
+                )
             clause = sqlalchemy.sql.not_(clause) if self.exclude else clause
             expr = expr.where(clause)
         return expr

ormar/queryset/join.py

@@ -1,8 +1,8 @@
 from collections import OrderedDict
 from typing import (
+    Any,
     Dict,
     List,
-    NamedTuple,
     Optional,
     Set,
     TYPE_CHECKING,
@@ -14,24 +14,14 @@ from typing import (
 import sqlalchemy
 from sqlalchemy import text
 
-from ormar.fields import ManyToManyField  # noqa I100
+from ormar.exceptions import RelationshipInstanceError  # noqa I100
+from ormar.fields import BaseField, ManyToManyField  # noqa I100
 from ormar.relations import AliasManager
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
 
 
-class JoinParameters(NamedTuple):
-    """
-    Named tuple that holds set of parameters passed during join construction.
-    """
-
-    prev_model: Type["Model"]
-    previous_alias: str
-    from_table: str
-    model_cls: Type["Model"]
-
-
 class SqlJoin:
     def __init__(  # noqa:  CFQ002
         self,
@@ -42,39 +32,71 @@ class SqlJoin:
         exclude_fields: Optional[Union[Set, Dict]],
         order_columns: Optional[List],
         sorted_orders: OrderedDict,
+        main_model: Type["Model"],
+        relation_name: str,
+        relation_str: str,
+        related_models: Any = None,
+        own_alias: str = "",
+        source_model: Type["Model"] = None,
     ) -> None:
-        self.used_aliases = used_aliases
+        self.relation_name = relation_name
+        self.related_models = related_models or []
         self.select_from = select_from
         self.columns = columns
         self.fields = fields
         self.exclude_fields = exclude_fields
         self.order_columns = order_columns
         self.sorted_orders = sorted_orders
+        self.main_model = main_model
+        self.own_alias = own_alias
+        self.used_aliases = used_aliases
+        self.target_field = self.main_model.Meta.model_fields[self.relation_name]
 
-    @staticmethod
+        self._next_model: Optional[Type["Model"]] = None
-    def alias_manager(model_cls: Type["Model"]) -> AliasManager:
+        self._next_alias: Optional[str] = None
+
+        self.relation_str = relation_str
+        self.source_model = source_model
+
+    @property
+    def next_model(self) -> Type["Model"]:
+        if not self._next_model:  # pragma: nocover
+            raise RelationshipInstanceError(
+                "Cannot link to related table if relation.to model is not set."
+            )
+        return self._next_model
+
+    @next_model.setter
+    def next_model(self, value: Type["Model"]) -> None:
+        self._next_model = value
+
+    @property
+    def next_alias(self) -> str:
+        if not self._next_alias:  # pragma: nocover
+            raise RelationshipInstanceError("Alias for given relation not found.")
+        return self._next_alias
+
+    @next_alias.setter
+    def next_alias(self, value: str) -> None:
+        self._next_alias = value
+
+    @property
+    def alias_manager(self) -> AliasManager:
         """
-        Shortcut for ormars model AliasManager stored on Meta.
+        Shortcut for ormar's model AliasManager stored on Meta.
 
-        :param model_cls: ormar Model class
-        :type model_cls: Type[Model]
         :return: alias manager from model's Meta
         :rtype: AliasManager
         """
-        return model_cls.Meta.alias_manager
+        return self.main_model.Meta.alias_manager
 
-    @staticmethod
+    def on_clause(self, previous_alias: str, from_clause: str, to_clause: str,) -> text:
-    def on_clause(
-        previous_alias: str, alias: str, from_clause: str, to_clause: str,
-    ) -> text:
         """
         Receives aliases and names of both ends of the join and combines them
         into one text clause used in joins.
 
         :param previous_alias: alias of previous table
         :type previous_alias: str
-        :param alias: alias of current table
-        :type alias: str
         :param from_clause: from table name
         :type from_clause: str
         :param to_clause: to table name
@@ -82,91 +104,27 @@ class SqlJoin:
         :return: clause combining all strings
         :rtype: sqlalchemy.text
         """
-        left_part = f"{alias}_{to_clause}"
+        left_part = f"{self.next_alias}_{to_clause}"
         right_part = f"{previous_alias + '_' if previous_alias else ''}{from_clause}"
         return text(f"{left_part}={right_part}")
 
-    @staticmethod
+    def build_join(self) -> Tuple[List, sqlalchemy.sql.select, List, OrderedDict]:
-    def update_inclusions(
-        model_cls: Type["Model"],
-        fields: Optional[Union[Set, Dict]],
-        exclude_fields: Optional[Union[Set, Dict]],
-        nested_name: str,
-    ) -> Tuple[Optional[Union[Dict, Set]], Optional[Union[Dict, Set]]]:
-        """
-        Extract nested fields and exclude_fields if applicable.
-
-        :param model_cls: ormar model class
-        :type model_cls: Type["Model"]
-        :param fields: fields to include
-        :type fields: Optional[Union[Set, Dict]]
-        :param exclude_fields: fields to exclude
-        :type exclude_fields: Optional[Union[Set, Dict]]
-        :param nested_name: name of the nested field
-        :type nested_name: str
-        :return: updated exclude and include fields from nested objects
-        :rtype: Tuple[Optional[Union[Dict, Set]], Optional[Union[Dict, Set]]]
-        """
-        fields = model_cls.get_included(fields, nested_name)
-        exclude_fields = model_cls.get_excluded(exclude_fields, nested_name)
-        return fields, exclude_fields
-
-    def build_join(  # noqa:  CCR001
-        self, item: str, join_parameters: JoinParameters
-    ) -> Tuple[List, sqlalchemy.sql.select, List, OrderedDict]:
         """
         Main external access point for building a join.
         Splits the join definition, updates fields and exclude_fields if needed,
         handles switching to through models for m2m relations, returns updated lists of
         used_aliases and sort_orders.
 
-        :param item: string with join definition
-        :type item: str
-        :param join_parameters: parameters from previous/ current join
-        :type join_parameters: JoinParameters
         :return: list of used aliases, select from, list of aliased columns, sort orders
         :rtype: Tuple[List[str], Join, List[TextClause], collections.OrderedDict]
         """
-        fields = self.fields
+        if issubclass(self.target_field, ManyToManyField):
-        exclude_fields = self.exclude_fields
+            self.process_m2m_through_table()
 
-        for index, part in enumerate(item.split("__")):
+        self.next_model = self.target_field.to
-            if issubclass(
+        self._forward_join()
-                join_parameters.model_cls.Meta.model_fields[part], ManyToManyField
-            ):
-                _fields = join_parameters.model_cls.Meta.model_fields
-                new_part = _fields[part].to.get_name()
-                self._switch_many_to_many_order_columns(part, new_part)
-                if index > 0:  # nested joins
-                    fields, exclude_fields = SqlJoin.update_inclusions(
-                        model_cls=join_parameters.model_cls,
-                        fields=fields,
-                        exclude_fields=exclude_fields,
-                        nested_name=part,
-                    )
 
-                join_parameters = self._build_join_parameters(
+        self._process_following_joins()
-                    part=part,
-                    join_params=join_parameters,
-                    is_multi=True,
-                    fields=fields,
-                    exclude_fields=exclude_fields,
-                )
-                part = new_part
-
-            if index > 0:  # nested joins
-                fields, exclude_fields = SqlJoin.update_inclusions(
-                    model_cls=join_parameters.model_cls,
-                    fields=fields,
-                    exclude_fields=exclude_fields,
-                    nested_name=part,
-                )
-            join_parameters = self._build_join_parameters(
-                part=part,
-                join_params=join_parameters,
-                fields=fields,
-                exclude_fields=exclude_fields,
-            )
 
         return (
             self.used_aliases,
@@ -175,66 +133,132 @@ class SqlJoin:
             self.sorted_orders,
         )
 
-    def _build_join_parameters(
+    def _forward_join(self) -> None:
-        self,
-        part: str,
-        join_params: JoinParameters,
-        fields: Optional[Union[Set, Dict]],
-        exclude_fields: Optional[Union[Set, Dict]],
-        is_multi: bool = False,
-    ) -> JoinParameters:
         """
-        Updates used_aliases to not join multiple times to the same table.
+        Process actual join.
-        Updates join parameters with new values.
+        Registers complex relation join on encountering of the duplicated alias.
-
-        :param part: part of the join str definition
-        :type part: str
-        :param join_params: parameters from previous/ current join
-        :type join_params: JoinParameters
-        :param fields: fields to include
-        :type fields: Optional[Union[Set, Dict]]
-        :param exclude_fields: fields to exclude
-        :type exclude_fields: Optional[Union[Set, Dict]]
-        :param is_multi: flag if the relation is m2m
-        :type is_multi: bool
-        :return: updated join parameters
-        :rtype: ormar.queryset.join.JoinParameters
         """
-        if is_multi:
+        self.next_alias = self.alias_manager.resolve_relation_alias(
-            model_cls = join_params.model_cls.Meta.model_fields[part].through
+            from_model=self.target_field.owner, relation_name=self.relation_name
-        else:
-            model_cls = join_params.model_cls.Meta.model_fields[part].to
-        to_table = model_cls.Meta.table.name
-
-        alias = model_cls.Meta.alias_manager.resolve_relation_alias(
-            join_params.prev_model, part
         )
-        if alias not in self.used_aliases:
+        if self.next_alias not in self.used_aliases:
-            self._process_join(
+            self._process_join()
-                join_params=join_params,
+        else:
-                is_multi=is_multi,
+            if "__" in self.relation_str and self.source_model:
-                model_cls=model_cls,
+                relation_key = f"{self.source_model.get_name()}_{self.relation_str}"
-                part=part,
+                if relation_key not in self.alias_manager:
-                alias=alias,
+                    self.next_alias = self.alias_manager.add_alias(
-                fields=fields,
+                        alias_key=relation_key
-                exclude_fields=exclude_fields,
+                    )
-            )
+                else:
+                    self.next_alias = self.alias_manager[relation_key]
+                self._process_join()
 
-        previous_alias = alias
+    def _process_following_joins(self) -> None:
-        from_table = to_table
+        """
-        prev_model = model_cls
+        Iterates through nested models to create subsequent joins.
-        return JoinParameters(prev_model, previous_alias, from_table, model_cls)
+        """
+        for related_name in self.related_models:
+            remainder = None
+            if (
+                isinstance(self.related_models, dict)
+                and self.related_models[related_name]
+            ):
+                remainder = self.related_models[related_name]
+            self._process_deeper_join(related_name=related_name, remainder=remainder)
 
-    def _process_join(  # noqa: CFQ002
+    def _process_deeper_join(self, related_name: str, remainder: Any) -> None:
-        self,
+        """
-        join_params: JoinParameters,
+        Creates nested recurrent instance of SqlJoin for each nested join table,
-        is_multi: bool,
+        updating needed return params here as a side effect.
-        model_cls: Type["Model"],
+
-        part: str,
+        Updated are:
-        alias: str,
+
-        fields: Optional[Union[Set, Dict]],
+        * self.used_aliases,
-        exclude_fields: Optional[Union[Set, Dict]],
+        * self.select_from,
-    ) -> None:
+        * self.columns,
+        * self.sorted_orders,
+
+        :param related_name: name of the relation to follow
+        :type related_name: str
+        :param remainder: deeper tables if there are more nested joins
+        :type remainder: Any
+        """
+        sql_join = SqlJoin(
+            used_aliases=self.used_aliases,
+            select_from=self.select_from,
+            columns=self.columns,
+            fields=self.main_model.get_excluded(self.fields, related_name),
+            exclude_fields=self.main_model.get_excluded(
+                self.exclude_fields, related_name
+            ),
+            order_columns=self.order_columns,
+            sorted_orders=self.sorted_orders,
+            main_model=self.next_model,
+            relation_name=related_name,
+            related_models=remainder,
+            relation_str="__".join([self.relation_str, related_name]),
+            own_alias=self.next_alias,
+            source_model=self.source_model or self.main_model,
+        )
+        (
+            self.used_aliases,
+            self.select_from,
+            self.columns,
+            self.sorted_orders,
+        ) = sql_join.build_join()
+
+    def process_m2m_through_table(self) -> None:
+        """
+        Process Through table of the ManyToMany relation so that source table is
+        linked to the through table (one additional join)
+
+        Replaces needed parameters like:
+
+        * self.next_model,
+        * self.next_alias,
+        * self.relation_name,
+        * self.own_alias,
+        * self.target_field
+
+        To point to through model
+        """
+        new_part = self.process_m2m_related_name_change()
+        self._replace_many_to_many_order_by_columns(self.relation_name, new_part)
+
+        self.next_model = self.target_field.through
+        self._forward_join()
+
+        self.relation_name = new_part
+        self.own_alias = self.next_alias
+        self.target_field = self.next_model.Meta.model_fields[self.relation_name]
+
+    def process_m2m_related_name_change(self, reverse: bool = False) -> str:
+        """
+        Extracts relation name to link join through the Through model declared on
+        relation field.
+
+        Changes the same names in order_by queries if they are present.
+
+        :param reverse: flag if it's on_clause lookup - use reverse fields
+        :type reverse: bool
+        :return: new relation name switched to through model field
+        :rtype: str
+        """
+        target_field = self.target_field
+        is_primary_self_ref = (
+            target_field.self_reference
+            and self.relation_name == target_field.self_reference_primary
+        )
+        if (is_primary_self_ref and not reverse) or (
+            not is_primary_self_ref and reverse
+        ):
+            new_part = target_field.default_source_field_name()  # type: ignore
+        else:
+            new_part = target_field.default_target_field_name()  # type: ignore
+        return new_part
+
+    def _process_join(self,) -> None:  # noqa: CFQ002
         """
         Resolves to and from column names and table names.
 
@@ -248,63 +272,40 @@ class SqlJoin:
 
         Process order_by causes for non m2m relations.
 
-        :param join_params: parameters from previous/ current join
-        :type join_params: JoinParameters
-        :param is_multi: flag if it's m2m relation
-        :type is_multi: bool
-        :param model_cls:
-        :type model_cls: ormar.models.metaclass.ModelMetaclass
-        :param part: name of the field used in join
-        :type part: str
-        :param alias: alias of the current join
-        :type alias: str
-        :param fields: fields to include
-        :type fields: Optional[Union[Set, Dict]]
-        :param exclude_fields: fields to exclude
-        :type exclude_fields: Optional[Union[Set, Dict]]
         """
-        to_table = model_cls.Meta.table.name
+        to_table = self.next_model.Meta.table.name
-        to_key, from_key = self.get_to_and_from_keys(
+        to_key, from_key = self.get_to_and_from_keys()
-            join_params, is_multi, model_cls, part
-        )
 
         on_clause = self.on_clause(
-            previous_alias=join_params.previous_alias,
+            previous_alias=self.own_alias,
-            alias=alias,
+            from_clause=f"{self.target_field.owner.Meta.tablename}.{from_key}",
-            from_clause=f"{join_params.from_table}.{from_key}",
             to_clause=f"{to_table}.{to_key}",
         )
-        target_table = self.alias_manager(model_cls).prefixed_table_name(
+        target_table = self.alias_manager.prefixed_table_name(self.next_alias, to_table)
-            alias, to_table
-        )
         self.select_from = sqlalchemy.sql.outerjoin(
             self.select_from, target_table, on_clause
         )
 
-        pkname_alias = model_cls.get_column_alias(model_cls.Meta.pkname)
+        pkname_alias = self.next_model.get_column_alias(self.next_model.Meta.pkname)
-        if not is_multi:
+        if not issubclass(self.target_field, ManyToManyField):
             self.get_order_bys(
-                alias=alias,
+                to_table=to_table, pkname_alias=pkname_alias,
-                to_table=to_table,
-                pkname_alias=pkname_alias,
-                part=part,
-                model_cls=model_cls,
             )
 
-        self_related_fields = model_cls.own_table_columns(
+        self_related_fields = self.next_model.own_table_columns(
-            model=model_cls,
+            model=self.next_model,
-            fields=fields,
+            fields=self.fields,
-            exclude_fields=exclude_fields,
+            exclude_fields=self.exclude_fields,
             use_alias=True,
         )
         self.columns.extend(
-            self.alias_manager(model_cls).prefixed_columns(
+            self.alias_manager.prefixed_columns(
-                alias, model_cls.Meta.table, self_related_fields
+                self.next_alias, self.next_model.Meta.table, self_related_fields
             )
         )
-        self.used_aliases.append(alias)
+        self.used_aliases.append(self.next_alias)
 
-    def _switch_many_to_many_order_columns(self, part: str, new_part: str) -> None:
+    def _replace_many_to_many_order_by_columns(self, part: str, new_part: str) -> None:
         """
         Substitutes the name of the relation with actual model name in m2m order bys.
 
@@ -318,7 +319,7 @@ class SqlJoin:
                 x.split("__") for x in self.order_columns if "__" in x
             ]
             for condition in split_order_columns:
-                if condition[-2] == part or condition[-2][1:] == part:
+                if self._check_if_condition_apply(condition, part):
                     condition[-2] = condition[-2].replace(part, new_part)
             self.order_columns = [x for x in self.order_columns if "__" not in x] + [
                 "__".join(x) for x in split_order_columns
@@ -340,63 +341,42 @@ class SqlJoin:
             condition[-2] == part or condition[-2][1:] == part
         )
 
-    def set_aliased_order_by(
+    def set_aliased_order_by(self, condition: List[str], to_table: str,) -> None:
-        self, condition: List[str], alias: str, to_table: str, model_cls: Type["Model"],
-    ) -> None:
         """
         Substitute hyphens ('-') with descending order.
         Construct actual sqlalchemy text clause using aliased table and column name.
 
         :param condition: list of parts of a current condition split by '__'
         :type condition: List[str]
-        :param alias: alias of the table in current join
-        :type alias: str
         :param to_table: target table
         :type to_table: sqlalchemy.sql.elements.quoted_name
-        :param model_cls: ormar model class
-        :type model_cls: ormar.models.metaclass.ModelMetaclass
         """
         direction = f"{'desc' if condition[0][0] == '-' else ''}"
-        column_alias = model_cls.get_column_alias(condition[-1])
+        column_alias = self.next_model.get_column_alias(condition[-1])
-        order = text(f"{alias}_{to_table}.{column_alias} {direction}")
+        order = text(f"{self.next_alias}_{to_table}.{column_alias} {direction}")
         self.sorted_orders["__".join(condition)] = order
 
-    def get_order_bys(  # noqa: CCR001
+    def get_order_bys(self, to_table: str, pkname_alias: str,) -> None:  # noqa: CCR001
-        self,
-        alias: str,
-        to_table: str,
-        pkname_alias: str,
-        part: str,
-        model_cls: Type["Model"],
-    ) -> None:
         """
         Triggers construction of order bys if they are given.
         Otherwise by default each table is sorted by a primary key column asc.
 
-        :param alias: alias of current table in join
-        :type alias: str
         :param to_table: target table
         :type to_table: sqlalchemy.sql.elements.quoted_name
         :param pkname_alias: alias of the primary key column
         :type pkname_alias: str
-        :param part: name of the current relation join
-        :type part: str
-        :param model_cls: ormar model class
-        :type model_cls: Type[Model]
         """
+        alias = self.next_alias
         if self.order_columns:
             current_table_sorted = False
             split_order_columns = [
                 x.split("__") for x in self.order_columns if "__" in x
             ]
             for condition in split_order_columns:
-                if self._check_if_condition_apply(condition, part):
+                if self._check_if_condition_apply(condition, self.relation_name):
                     current_table_sorted = True
                     self.set_aliased_order_by(
-                        condition=condition,
+                        condition=condition, to_table=to_table,
-                        alias=alias,
-                        to_table=to_table,
-                        model_cls=model_cls,
                     )
             if not current_table_sorted:
                 order = text(f"{alias}_{to_table}.{pkname_alias}")
@@ -406,46 +386,28 @@ class SqlJoin:
             order = text(f"{alias}_{to_table}.{pkname_alias}")
             self.sorted_orders[f"{alias}.{pkname_alias}"] = order
 
-    @staticmethod
+    def get_to_and_from_keys(self) -> Tuple[str, str]:
-    def get_to_and_from_keys(
-        join_params: JoinParameters,
-        is_multi: bool,
-        model_cls: Type["Model"],
-        part: str,
-    ) -> Tuple[str, str]:
         """
         Based on the relation type, name of the relation and previous models and parts
         stored in JoinParameters it resolves the current to and from keys, which are
-        different for ManyToMany relation, ForeignKey and reverse part of relations.
+        different for ManyToMany relation, ForeignKey and reverse related of relations.
 
-        :param join_params: parameters from previous/ current join
-        :type join_params: JoinParameters
-        :param is_multi: flag if the relation is of m2m type
-        :type is_multi: bool
-        :param model_cls: ormar model class
-        :type model_cls: Type[Model]
-        :param part: name of the current relation join
-        :type part: str
         :return: to key and from key
         :rtype: Tuple[str, str]
         """
-        if is_multi:
+        if issubclass(self.target_field, ManyToManyField):
-            to_field = join_params.prev_model.get_name()
+            to_key = self.process_m2m_related_name_change(reverse=True)
-            to_key = model_cls.get_column_alias(to_field)
+            from_key = self.main_model.get_column_alias(self.main_model.Meta.pkname)
-            from_key = join_params.prev_model.get_column_alias(
+
-                join_params.prev_model.Meta.pkname
+        elif self.target_field.virtual:
-            )
+            to_field = self.target_field.get_related_name()
-        elif join_params.prev_model.Meta.model_fields[part].virtual:
+            to_key = self.target_field.to.get_column_alias(to_field)
-            to_field = (
+            from_key = self.main_model.get_column_alias(self.main_model.Meta.pkname)
-                join_params.prev_model.Meta.model_fields[part].related_name
+
-                or join_params.prev_model.get_name() + "s"
-            )
-            to_key = model_cls.get_column_alias(to_field)
-            from_key = join_params.prev_model.get_column_alias(
-                join_params.prev_model.Meta.pkname
-            )
         else:
-            to_key = model_cls.get_column_alias(model_cls.Meta.pkname)
+            to_key = self.target_field.to.get_column_alias(
-            from_key = join_params.prev_model.get_column_alias(part)
+                self.target_field.to.Meta.pkname
+            )
+            from_key = self.main_model.get_column_alias(self.relation_name)
 
         return to_key, from_key

ormar/queryset/prefetch_query.py

@@ -9,10 +9,12 @@ from typing import (
     Tuple,
     Type,
     Union,
+    cast,
 )
 
 import ormar
 from ormar.fields import BaseField, ManyToManyField
+from ormar.fields.foreign_key import ForeignKeyField
 from ormar.queryset.clause import QueryClause
 from ormar.queryset.query import Query
 from ormar.queryset.utils import extract_models_to_dict_of_lists, translate_list_to_dict
@@ -288,7 +290,7 @@ class PrefetchQuery:
                 model_cls=clause_target, select_related=[], filter_clauses=[],
             )
             kwargs = {f"{filter_column}__in": ids}
-            filter_clauses, _ = qryclause.filter(**kwargs)
+            filter_clauses, _ = qryclause.prepare_filter(**kwargs)
             return filter_clauses
         return []
 
@@ -314,6 +316,7 @@ class PrefetchQuery:
 
         for related in related_to_extract:
             target_field = model.Meta.model_fields[related]
+            target_field = cast(Type[ForeignKeyField], target_field)
             target_model = target_field.to.get_name()
             model_id = model.get_relation_model_id(target_field=target_field)
 
@@ -421,6 +424,7 @@ class PrefetchQuery:
         fields = target_model.get_included(fields, related)
         exclude_fields = target_model.get_excluded(exclude_fields, related)
         target_field = target_model.Meta.model_fields[related]
+        target_field = cast(Type[ForeignKeyField], target_field)
         reverse = False
         if target_field.virtual or issubclass(target_field, ManyToManyField):
             reverse = True
@@ -522,7 +526,7 @@ class PrefetchQuery:
             query_target = target_field.through
             select_related = [target_name]
             table_prefix = target_field.to.Meta.alias_manager.resolve_relation_alias(
-                query_target, target_name
+                from_model=query_target, relation_name=target_name
             )
             self.already_extracted.setdefault(target_name, {})["prefix"] = table_prefix
 
@@ -547,14 +551,14 @@ class PrefetchQuery:
     @staticmethod
     def _get_select_related_if_apply(related: str, select_dict: Dict) -> Dict:
         """
-        Extract nested part of select_related dictionary to extract models nested
+        Extract nested related of select_related dictionary to extract models nested
         deeper on related model and already loaded in select related query.
 
         :param related: name of the relation
         :type related: str
         :param select_dict: dictionary of select related models in main query
         :type select_dict: Dict
-        :return: dictionary with nested part of select related
+        :return: dictionary with nested related of select related
         :rtype: Dict
         """
         return (
@@ -585,7 +589,7 @@ class PrefetchQuery:
     def _populate_rows(  # noqa: CFQ002
         self,
         rows: List,
-        target_field: Type["BaseField"],
+        target_field: Type["ForeignKeyField"],
         parent_model: Type["Model"],
         table_prefix: str,
         fields: Union[Set[Any], Dict[Any, Any], None],

ormar/queryset/query.py

@@ -6,8 +6,10 @@ import sqlalchemy
 from sqlalchemy import text
 
 import ormar  # noqa I100
+from ormar.models.helpers.models import group_related_list
 from ormar.queryset import FilterQuery, LimitQuery, OffsetQuery, OrderQuery
-from ormar.queryset.join import JoinParameters, SqlJoin
+from ormar.queryset.filter_action import FilterAction
+from ormar.queryset.join import SqlJoin
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
@@ -17,8 +19,8 @@ class Query:
     def __init__(  # noqa CFQ002
         self,
         model_cls: Type["Model"],
-        filter_clauses: List,
+        filter_clauses: List[FilterAction],
-        exclude_clauses: List,
+        exclude_clauses: List[FilterAction],
         select_related: List,
         limit_count: Optional[int],
         offset: Optional[int],
@@ -140,14 +142,14 @@ class Query:
         else:
             self.select_from = self.table
 
-        self._select_related.sort(key=lambda item: (item, -len(item)))
+        related_models = group_related_list(self._select_related)
 
-        for item in self._select_related:
+        for related in related_models:
-            join_parameters = JoinParameters(
+            fields = self.model_cls.get_included(self.fields, related)
-                self.model_cls, "", self.table.name, self.model_cls
+            exclude_fields = self.model_cls.get_excluded(self.exclude_fields, related)
-            )
+            remainder = None
-            fields = self.model_cls.get_included(self.fields, item)
+            if isinstance(related_models, dict) and related_models[related]:
-            exclude_fields = self.model_cls.get_excluded(self.exclude_fields, item)
+                remainder = related_models[related]
             sql_join = SqlJoin(
                 used_aliases=self.used_aliases,
                 select_from=self.select_from,
@@ -156,6 +158,10 @@ class Query:
                 exclude_fields=exclude_fields,
                 order_columns=self.order_columns,
                 sorted_orders=self.sorted_orders,
+                main_model=self.model_cls,
+                relation_name=related,
+                relation_str=related,
+                related_models=remainder,
             )
 
             (
@@ -163,14 +169,14 @@ class Query:
                 self.select_from,
                 self.columns,
                 self.sorted_orders,
-            ) = sql_join.build_join(item, join_parameters)
+            ) = sql_join.build_join()
 
         expr = sqlalchemy.sql.select(self.columns)
         expr = expr.select_from(self.select_from)
 
         expr = self._apply_expression_modifiers(expr)
 
-        # print(expr.compile(compile_kwargs={"literal_binds": True}))
+        # print("\n", expr.compile(compile_kwargs={"literal_binds": True}))
         self._reset_query_parameters()
 
         return expr
@@ -195,12 +201,12 @@ class Query:
         filters_to_use = [
             filter_clause
             for filter_clause in self.filter_clauses
-            if filter_clause.text.startswith(f"{self.table.name}.")
+            if filter_clause.table_prefix == ""
         ]
         excludes_to_use = [
             filter_clause
             for filter_clause in self.exclude_clauses
-            if filter_clause.text.startswith(f"{self.table.name}.")
+            if filter_clause.table_prefix == ""
         ]
         sorts_to_use = {k: v for k, v in self.sorted_orders.items() if "__" not in k}
         expr = FilterQuery(filter_clauses=filters_to_use).apply(expr)

ormar/queryset/queryset.py

@@ -6,7 +6,7 @@ from sqlalchemy import bindparam
 
 import ormar  # noqa I100
 from ormar import MultipleMatches, NoMatch
-from ormar.exceptions import ModelPersistenceError, QueryDefinitionError
+from ormar.exceptions import ModelError, ModelPersistenceError, QueryDefinitionError
 from ormar.queryset import FilterQuery
 from ormar.queryset.clause import QueryClause
 from ormar.queryset.prefetch_query import PrefetchQuery
@@ -55,6 +55,13 @@ class QuerySet:
         instance: Optional[Union["QuerySet", "QuerysetProxy"]],
         owner: Union[Type["Model"], Type["QuerysetProxy"]],
     ) -> "QuerySet":
+        if issubclass(owner, ormar.Model):
+            if owner.Meta.requires_ref_update:
+                raise ModelError(
+                    f"Model {owner.get_name()} has not updated "
+                    f"ForwardRefs. \nBefore using the model you "
+                    f"need to call update_forward_refs()."
+                )
         if issubclass(owner, ormar.Model):
             return self.__class__(model_cls=owner)
         return self.__class__()  # pragma: no cover
@@ -226,7 +233,7 @@ class QuerySet:
             select_related=self._select_related,
             filter_clauses=self.filter_clauses,
         )
-        filter_clauses, select_related = qryclause.filter(**kwargs)
+        filter_clauses, select_related = qryclause.prepare_filter(**kwargs)
         if _exclude:
             exclude_clauses = filter_clauses
             filter_clauses = self.filter_clauses
@@ -585,6 +592,37 @@ class QuerySet:
         )
         return await self.database.execute(expr)
 
+    def paginate(self, page: int, page_size: int = 20) -> "QuerySet":
+        """
+        You can paginate the result which is a combination of offset and limit clauses.
+        Limit is set to page size and offset is set to (page-1) * page_size.
+
+        :param page_size: numbers of items per page
+        :type page_size: int
+        :param page: page number
+        :type page: int
+        :return: QuerySet
+        :rtype: QuerySet
+        """
+        if page < 1 or page_size < 1:
+            raise QueryDefinitionError("Page size and page have to be greater than 0.")
+
+        limit_count = page_size
+        query_offset = (page - 1) * page_size
+        return self.__class__(
+            model_cls=self.model,
+            filter_clauses=self.filter_clauses,
+            exclude_clauses=self.exclude_clauses,
+            select_related=self._select_related,
+            limit_count=limit_count,
+            offset=query_offset,
+            columns=self._columns,
+            exclude_columns=self._exclude_columns,
+            order_bys=self.order_bys,
+            prefetch_related=self._prefetch_related,
+            limit_raw_sql=self.limit_sql_raw,
+        )
+
     def limit(self, limit_count: int, limit_raw_sql: bool = None) -> "QuerySet":
         """
         You can limit the results to desired number of parent models.

ormar/queryset/utils.py

@@ -7,10 +7,13 @@ from typing import (
     Sequence,
     Set,
     TYPE_CHECKING,
+    Tuple,
     Type,
     Union,
 )
 
+from ormar.fields import ManyToManyField
+
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
 
@@ -212,3 +215,35 @@ def extract_models_to_dict_of_lists(
     for model in models:
         extract_nested_models(model, model_type, select_dict, extracted)
     return extracted
+
+
+def get_relationship_alias_model_and_str(
+    source_model: Type["Model"], related_parts: List
+) -> Tuple[str, Type["Model"], str]:
+    """
+    Walks the relation to retrieve the actual model on which the clause should be
+    constructed, extracts alias based on last relation leading to target model.
+    :param related_parts: list of related names extracted from string
+    :type related_parts: Union[List, List[str]]
+    :param source_model: model from which relation starts
+    :type source_model: Type[Model]
+    :return: table prefix, target model and relation string
+    :rtype: Tuple[str, Type["Model"], str]
+    """
+    table_prefix = ""
+    model_cls = source_model
+    previous_model = model_cls
+    manager = model_cls.Meta.alias_manager
+    for relation in related_parts:
+        related_field = model_cls.Meta.model_fields[relation]
+        if issubclass(related_field, ManyToManyField):
+            previous_model = related_field.through
+            relation = related_field.default_target_field_name()  # type: ignore
+        table_prefix = manager.resolve_relation_alias(
+            from_model=previous_model, relation_name=relation
+        )
+        model_cls = related_field.to
+        previous_model = model_cls
+    relation_str = "__".join(related_parts)
+
+    return table_prefix, model_cls, relation_str

ormar/relations/alias_manager.py

@@ -1,7 +1,7 @@
 import string
 import uuid
 from random import choices
-from typing import Dict, List, TYPE_CHECKING, Type
+from typing import Any, Dict, List, TYPE_CHECKING, Type
 
 import sqlalchemy
 from sqlalchemy import text
@@ -31,9 +31,14 @@ class AliasManager:
     """
 
     def __init__(self) -> None:
-        self._aliases: Dict[str, str] = dict()
         self._aliases_new: Dict[str, str] = dict()
 
+    def __contains__(self, item: str) -> bool:
+        return self._aliases_new.__contains__(item)
+
+    def __getitem__(self, key: str) -> Any:
+        return self._aliases_new.__getitem__(key)
+
     @staticmethod
     def prefixed_columns(
         alias: str, table: sqlalchemy.Table, fields: List = None
@@ -80,11 +85,7 @@ class AliasManager:
         return text(f"{name} {alias}_{name}")
 
     def add_relation_type(
-        self,
+        self, source_model: Type["Model"], relation_name: str, reverse_name: str = None,
-        source_model: Type["Model"],
-        relation_name: str,
-        reverse_name: str = None,
-        is_multi: bool = False,
     ) -> None:
         """
         Registers the relations defined in ormar models.
@@ -105,23 +106,31 @@ class AliasManager:
         :type relation_name: str
         :param reverse_name: name of related_name fo given relation for m2m relations
         :type reverse_name: Optional[str]
-        :param is_multi: flag if relation being registered is a through m2m model
-        :type is_multi: bool
         :return: none
         :rtype: None
         """
         parent_key = f"{source_model.get_name()}_{relation_name}"
         if parent_key not in self._aliases_new:
-            self._aliases_new[parent_key] = get_table_alias()
+            self.add_alias(parent_key)
+
         to_field = source_model.Meta.model_fields[relation_name]
         child_model = to_field.to
-        related_name = to_field.related_name
+        child_key = f"{child_model.get_name()}_{reverse_name}"
-        if not related_name:
-            related_name = reverse_name if is_multi else source_model.get_name() + "s"
-
-        child_key = f"{child_model.get_name()}_{related_name}"
         if child_key not in self._aliases_new:
-            self._aliases_new[child_key] = get_table_alias()
+            self.add_alias(child_key)
+
+    def add_alias(self, alias_key: str) -> str:
+        """
+        Adds alias to the dictionary of aliases under given key.
+
+        :param alias_key: key of relation to generate alias for
+        :type alias_key: str
+        :return: generated alias
+        :rtype: str
+        """
+        alias = get_table_alias()
+        self._aliases_new[alias_key] = alias
+        return alias
 
     def resolve_relation_alias(
         self, from_model: Type["Model"], relation_name: str

ormar/relations/querysetproxy.py

@@ -12,6 +12,7 @@ from typing import (
 )
 
 import ormar
+from ormar.exceptions import ModelPersistenceError
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar.relations import Relation
@@ -38,10 +39,9 @@ class QuerysetProxy(ormar.QuerySetProtocol):
         self._queryset: Optional["QuerySet"] = qryset
         self.type_: "RelationType" = type_
         self._owner: "Model" = self.relation.manager.owner
-        self.related_field_name = (
+        self.related_field_name = self._owner.Meta.model_fields[
-            self._owner.Meta.model_fields[self.relation.field_name].related_name
+            self.relation.field_name
-            or self._owner.get_name() + "s"
+        ].get_related_name()
-        )
         self.related_field = self.relation.to.Meta.model_fields[self.related_field_name]
         self.owner_pk_value = self._owner.pk
 
@@ -106,11 +106,20 @@ class QuerysetProxy(ormar.QuerySetProtocol):
         :param child: child model instance
         :type child: Model
         """
-        queryset = ormar.QuerySet(model_cls=self.relation.through)
+        model_cls = self.relation.through
-        owner_column = self._owner.get_name()
+        owner_column = self.related_field.default_target_field_name()  # type: ignore
-        child_column = child.get_name()
+        child_column = self.related_field.default_source_field_name()  # type: ignore
-        kwargs = {owner_column: self._owner, child_column: child}
+        kwargs = {owner_column: self._owner.pk, child_column: child.pk}
-        await queryset.create(**kwargs)
+        if child.pk is None:
+            raise ModelPersistenceError(
+                f"You cannot save {child.get_name()} "
+                f"model without primary key set! \n"
+                f"Save the child model first."
+            )
+        expr = model_cls.Meta.table.insert()
+        expr = expr.values(**kwargs)
+        # print("\n", expr.compile(compile_kwargs={"literal_binds": True}))
+        await model_cls.Meta.database.execute(expr)
 
     async def delete_through_instance(self, child: "T") -> None:
         """
@@ -120,8 +129,8 @@ class QuerysetProxy(ormar.QuerySetProtocol):
         :type child: Model
         """
         queryset = ormar.QuerySet(model_cls=self.relation.through)
-        owner_column = self._owner.get_name()
+        owner_column = self.related_field.default_target_field_name()  # type: ignore
-        child_column = child.get_name()
+        child_column = self.related_field.default_source_field_name()  # type: ignore
         kwargs = {owner_column: self._owner, child_column: child}
         link_instance = await queryset.filter(**kwargs).get()  # type: ignore
         await link_instance.delete()
@@ -406,6 +415,23 @@ class QuerysetProxy(ormar.QuerySetProtocol):
         queryset = self.queryset.prefetch_related(related)
         return self.__class__(relation=self.relation, type_=self.type_, qryset=queryset)
 
+    def paginate(self, page: int, page_size: int = 20) -> "QuerysetProxy":
+        """
+        You can paginate the result which is a combination of offset and limit clauses.
+        Limit is set to page size and offset is set to (page-1) * page_size.
+
+        Actual call delegated to QuerySet.
+
+        :param page_size: numbers of items per page
+        :type page_size: int
+        :param page: page number
+        :type page: int
+        :return: QuerySet
+        :rtype: QuerySet
+        """
+        queryset = self.queryset.paginate(page=page, page_size=page_size)
+        return self.__class__(relation=self.relation, type_=self.type_, qryset=queryset)
+
     def limit(self, limit_count: int) -> "QuerysetProxy":
         """
         You can limit the results to desired number of parent models.

ormar/relations/relation.py

@@ -63,7 +63,7 @@ class Relation:
         self._type: RelationType = type_
         self._to_remove: Set = set()
         self.to: Type["T"] = to
-        self.through: Optional[Type["T"]] = through
+        self._through: Optional[Type["T"]] = through
         self.field_name = field_name
         self.related_models: Optional[Union[RelationProxy, "T"]] = (
             RelationProxy(relation=self, type_=type_, field_name=field_name)
@@ -71,6 +71,12 @@ class Relation:
             else None
         )
 
+    @property
+    def through(self) -> Type["T"]:
+        if not self._through:  # pragma: no cover
+            raise RelationshipInstanceError("Relation does not have through model!")
+        return self._through
+
     def _clean_related(self) -> None:
         """
         Removes dead weakrefs from RelationProxy.

ormar/relations/relation_manager.py

@@ -101,13 +101,7 @@ class RelationsManager:
         return None
 
     @staticmethod
-    def add(
+    def add(parent: "Model", child: "Model", field: Type["ForeignKeyField"],) -> None:
-        parent: "Model",
-        child: "Model",
-        child_name: str,
-        virtual: bool,
-        relation_name: str,
-    ) -> None:
         """
         Adds relation on both sides -> meaning on both child and parent models.
         One side of the relation is always weakref proxy to avoid circular refs.
@@ -120,25 +114,19 @@ class RelationsManager:
         :type parent: Model
         :param child: child model to register
         :type child: Model
-        :param child_name: potential child name used if related name is not set
+        :param field: field with relation definition
-        :type child_name: str
+        :type field: ForeignKeyField
-        :param virtual:
-        :type virtual: bool
-        :param relation_name: name of the relation
-        :type relation_name: str
         """
-        to_field: Type[BaseField] = child.Meta.model_fields[relation_name]
-        # print('comming', child_name, relation_name)
         (parent, child, child_name, to_name,) = get_relations_sides_and_names(
-            to_field, parent, child, child_name, virtual, relation_name
+            field, parent, child
         )
 
-        # print('adding', parent.get_name(), child.get_name(), child_name)
+        # print('adding parent', parent.get_name(), child.get_name(), child_name)
         parent_relation = parent._orm._get(child_name)
         if parent_relation:
             parent_relation.add(child)  # type: ignore
 
-        # print('adding', child.get_name(), parent.get_name(), child_name)
+        # print('adding child', child.get_name(), parent.get_name(), to_name)
         child_relation = child._orm._get(to_name)
         if child_relation:
             child_relation.add(parent)
@@ -176,8 +164,6 @@ class RelationsManager:
         :param name: name of the relation
         :type name: str
         """
-        relation_name = (
+        relation_name = item.Meta.model_fields[name].get_related_name()
-            item.Meta.model_fields[name].related_name or item.get_name() + "s"
-        )
         item._orm.remove(name, parent)
         parent._orm.remove(relation_name, item)

ormar/relations/relation_proxy.py

@@ -42,9 +42,8 @@ class RelationProxy(list):
         if self._related_field_name:
             return self._related_field_name
         owner_field = self._owner.Meta.model_fields[self.field_name]
-        self._related_field_name = (
+        self._related_field_name = owner_field.get_related_name()
-            owner_field.related_name or self._owner.get_name() + "s"
+
-        )
         return self._related_field_name
 
     def __getattribute__(self, item: str) -> Any:
@@ -128,7 +127,7 @@ class RelationProxy(list):
         self, item: "Model", keep_reversed: bool = True
     ) -> None:
         """
-        Removes the item from relation with parent.
+        Removes the related from relation with parent.
 
         Through models are automatically deleted for m2m relations.
 

ormar/relations/utils.py

@@ -1,48 +1,33 @@
 from typing import TYPE_CHECKING, Tuple, Type
 from weakref import proxy
 
-from ormar.fields import BaseField
+from ormar.fields.foreign_key import ForeignKeyField
-from ormar.fields.many_to_many import ManyToManyField
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
 
 
 def get_relations_sides_and_names(
-    to_field: Type[BaseField],
+    to_field: Type[ForeignKeyField], parent: "Model", child: "Model",
-    parent: "Model",
-    child: "Model",
-    child_name: str,
-    virtual: bool,
-    relation_name: str,
 ) -> Tuple["Model", "Model", str, str]:
     """
     Determines the names of child and parent relations names, as well as
     changes one of the sides of the relation into weakref.proxy to model.
 
     :param to_field: field with relation definition
-    :type to_field: BaseField
+    :type to_field: ForeignKeyField
     :param parent: parent model
     :type parent: Model
     :param child: child model
     :type child: Model
-    :param child_name: name of the child
-    :type child_name: str
-    :param virtual: flag if relation is virtual
-    :type virtual: bool
-    :param relation_name:
-    :type relation_name:
     :return: parent, child, child_name, to_name
     :rtype: Tuple["Model", "Model", str, str]
     """
     to_name = to_field.name
-    if issubclass(to_field, ManyToManyField):
+    child_name = to_field.get_related_name()
-        child_name = to_field.related_name or child.get_name() + "s"
+    if to_field.virtual:
-        child = proxy(child)
+        child_name, to_name = to_name, child_name
-    elif virtual:
-        child_name, to_name = to_name, child_name or child.get_name()
         child, parent = parent, proxy(child)
     else:
-        child_name = child_name or child.get_name() + "s"
         child = proxy(child)
     return parent, child, child_name, to_name

setup.py

@@ -61,7 +61,7 @@ setup(
         "orjson": ["orjson"]
     },
     classifiers=[
-        "Development Status :: 3 - Alpha",
+        "Development Status :: 4 - Beta",
         "Environment :: Web Environment",
         "Intended Audience :: Developers",
         "License :: OSI Approved :: MIT License",
@@ -72,6 +72,7 @@ setup(
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3 :: Only",
     ],
 )

tests/test_forward_cross_refs.py

@@ -0,0 +1,116 @@
+# type: ignore
+
+import databases
+import pytest
+import sqlalchemy as sa
+from pydantic.typing import ForwardRef
+from sqlalchemy import create_engine
+
+import ormar
+from ormar import ModelMeta
+from tests.settings import DATABASE_URL
+
+metadata = sa.MetaData()
+db = databases.Database(DATABASE_URL)
+engine = create_engine(DATABASE_URL)
+
+TeacherRef = ForwardRef("Teacher")
+
+
+class Student(ormar.Model):
+    class Meta(ModelMeta):
+        metadata = metadata
+        database = db
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    primary_teacher: TeacherRef = ormar.ForeignKey(
+        TeacherRef, related_name="own_students"
+    )
+
+
+class StudentTeacher(ormar.Model):
+    class Meta(ModelMeta):
+        tablename = "students_x_teachers"
+        metadata = metadata
+        database = db
+
+
+class Teacher(ormar.Model):
+    class Meta(ModelMeta):
+        metadata = metadata
+        database = db
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    students = ormar.ManyToMany(
+        Student, through=StudentTeacher, related_name="teachers"
+    )
+
+
+Student.update_forward_refs()
+
+
+@pytest.fixture(autouse=True, scope="module")
+def create_test_database():
+    metadata.create_all(engine)
+    yield
+    metadata.drop_all(engine)
+
+
+@pytest.mark.asyncio
+async def test_double_relations():
+    async with db:
+        async with db.transaction(force_rollback=True):
+            t1 = await Teacher.objects.create(name="Mr. Jones")
+            t2 = await Teacher.objects.create(name="Ms. Smith")
+            t3 = await Teacher.objects.create(name="Mr. Quibble")
+
+            s1 = await Student.objects.create(name="Joe", primary_teacher=t1)
+            s2 = await Student.objects.create(name="Sam", primary_teacher=t1)
+            s3 = await Student.objects.create(name="Kate", primary_teacher=t2)
+            s4 = await Student.objects.create(name="Zoe", primary_teacher=t2)
+            s5 = await Student.objects.create(name="John", primary_teacher=t3)
+            s6 = await Student.objects.create(name="Anna", primary_teacher=t3)
+
+            for t in [t1, t2, t3]:
+                for s in [s1, s2, s3, s4, s5, s6]:
+                    await t.students.add(s)
+
+            jones = (
+                await Teacher.objects.select_related(["students", "own_students"])
+                .order_by(["students__name", "own_students__name"])
+                .get(name="Mr. Jones")
+            )
+            assert len(jones.students) == 6
+            assert jones.students[0].name == "Anna"
+            assert jones.students[5].name == "Zoe"
+            assert len(jones.own_students) == 2
+            assert jones.own_students[0].name == "Joe"
+            assert jones.own_students[1].name == "Sam"
+
+            smith = (
+                await Teacher.objects.select_related(["students", "own_students"])
+                .filter(students__name__contains="a")
+                .order_by(["students__name", "own_students__name"])
+                .get(name="Ms. Smith")
+            )
+            assert len(smith.students) == 3
+            assert smith.students[0].name == "Anna"
+            assert smith.students[2].name == "Sam"
+            assert len(smith.own_students) == 2
+            assert smith.own_students[0].name == "Kate"
+            assert smith.own_students[1].name == "Zoe"
+
+            quibble = (
+                await Teacher.objects.select_related(["students", "own_students"])
+                .filter(students__name__startswith="J")
+                .order_by(["-students__name", "own_students__name"])
+                .get(name="Mr. Quibble")
+            )
+            assert len(quibble.students) == 2
+            assert quibble.students[1].name == "Joe"
+            assert quibble.students[0].name == "John"
+            assert len(quibble.own_students) == 2
+            assert quibble.own_students[1].name == "John"
+            assert quibble.own_students[0].name == "Anna"

tests/test_forward_refs.py

@@ -0,0 +1,304 @@
+# type: ignore
+from typing import List
+
+import databases
+import pytest
+import sqlalchemy as sa
+from pydantic.typing import ForwardRef
+from sqlalchemy import create_engine
+
+import ormar
+from ormar import ModelMeta
+from ormar.exceptions import ModelError
+from tests.settings import DATABASE_URL
+
+metadata = sa.MetaData()
+db = databases.Database(DATABASE_URL)
+engine = create_engine(DATABASE_URL)
+
+PersonRef = ForwardRef("Person")
+
+
+class Person(ormar.Model):
+    class Meta(ModelMeta):
+        metadata = metadata
+        database = db
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    supervisor: PersonRef = ormar.ForeignKey(PersonRef, related_name="employees")
+
+
+Person.update_forward_refs()
+
+GameRef = ForwardRef("Game")
+ChildRef = ForwardRef("Child")
+ChildFriendRef = ForwardRef("ChildFriend")
+
+
+class Child(ormar.Model):
+    class Meta(ModelMeta):
+        metadata = metadata
+        database = db
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    favourite_game: GameRef = ormar.ForeignKey(GameRef, related_name="liked_by")
+    least_favourite_game: GameRef = ormar.ForeignKey(
+        GameRef, related_name="not_liked_by"
+    )
+    friends = ormar.ManyToMany(
+        ChildRef, through=ChildFriendRef, related_name="also_friends"
+    )
+
+
+class ChildFriend(ormar.Model):
+    class Meta(ModelMeta):
+        metadata = metadata
+        database = db
+
+
+class Game(ormar.Model):
+    class Meta(ModelMeta):
+        metadata = metadata
+        database = db
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+
+
+Child.update_forward_refs()
+
+
+@pytest.fixture(autouse=True, scope="module")
+def create_test_database():
+    metadata.create_all(engine)
+    yield
+    metadata.drop_all(engine)
+
+
+@pytest.fixture(scope="function")
+async def cleanup():
+    yield
+    async with db:
+        await ChildFriend.objects.delete(each=True)
+        await Child.objects.delete(each=True)
+        await Game.objects.delete(each=True)
+        await Person.objects.delete(each=True)
+
+
+@pytest.mark.asyncio
+async def test_not_updated_model_raises_errors():
+    Person2Ref = ForwardRef("Person2")
+
+    class Person2(ormar.Model):
+        class Meta(ModelMeta):
+            metadata = metadata
+            database = db
+
+        id: int = ormar.Integer(primary_key=True)
+        name: str = ormar.String(max_length=100)
+        supervisor: Person2Ref = ormar.ForeignKey(Person2Ref, related_name="employees")
+
+    with pytest.raises(ModelError):
+        await Person2.objects.create(name="Test")
+
+    with pytest.raises(ModelError):
+        Person2(name="Test")
+
+    with pytest.raises(ModelError):
+        await Person2.objects.get()
+
+
+@pytest.mark.asyncio
+async def test_not_updated_model_m2m_raises_errors():
+    Person3Ref = ForwardRef("Person3")
+
+    class PersonFriend(ormar.Model):
+        class Meta(ModelMeta):
+            metadata = metadata
+            database = db
+
+    class Person3(ormar.Model):
+        class Meta(ModelMeta):
+            metadata = metadata
+            database = db
+
+        id: int = ormar.Integer(primary_key=True)
+        name: str = ormar.String(max_length=100)
+        supervisors: Person3Ref = ormar.ManyToMany(
+            Person3Ref, through=PersonFriend, related_name="employees"
+        )
+
+    with pytest.raises(ModelError):
+        await Person3.objects.create(name="Test")
+
+    with pytest.raises(ModelError):
+        Person3(name="Test")
+
+    with pytest.raises(ModelError):
+        await Person3.objects.get()
+
+
+@pytest.mark.asyncio
+async def test_not_updated_model_m2m_through_raises_errors():
+    PersonPetRef = ForwardRef("PersonPet")
+
+    class Pet(ormar.Model):
+        class Meta(ModelMeta):
+            metadata = metadata
+            database = db
+
+        id: int = ormar.Integer(primary_key=True)
+        name: str = ormar.String(max_length=100)
+
+    class Person4(ormar.Model):
+        class Meta(ModelMeta):
+            metadata = metadata
+            database = db
+
+        id: int = ormar.Integer(primary_key=True)
+        name: str = ormar.String(max_length=100)
+        pets: List[Pet] = ormar.ManyToMany(
+            Pet, through=PersonPetRef, related_name="owners"
+        )
+
+    class PersonPet(ormar.Model):
+        class Meta(ModelMeta):
+            metadata = metadata
+            database = db
+
+    with pytest.raises(ModelError):
+        await Person4.objects.create(name="Test")
+
+    with pytest.raises(ModelError):
+        Person4(name="Test")
+
+    with pytest.raises(ModelError):
+        await Person4.objects.get()
+
+
+def test_proper_field_init():
+    assert "supervisor" in Person.Meta.model_fields
+    assert Person.Meta.model_fields["supervisor"].to == Person
+
+    assert "supervisor" in Person.__fields__
+    assert Person.__fields__["supervisor"].type_ == Person
+
+    assert "supervisor" in Person.Meta.table.columns
+    assert isinstance(
+        Person.Meta.table.columns["supervisor"].type, sa.sql.sqltypes.Integer
+    )
+    assert len(Person.Meta.table.columns["supervisor"].foreign_keys) > 0
+
+    assert "person_supervisor" in Person.Meta.alias_manager._aliases_new
+
+
+@pytest.mark.asyncio
+async def test_self_relation():
+    async with db:
+        async with db.transaction(force_rollback=True):
+            sam = await Person.objects.create(name="Sam")
+            joe = await Person(name="Joe", supervisor=sam).save()
+            assert joe.supervisor.name == "Sam"
+
+            joe_check = await Person.objects.select_related("supervisor").get(
+                name="Joe"
+            )
+            assert joe_check.supervisor.name == "Sam"
+
+            sam_check = await Person.objects.select_related("employees").get(name="Sam")
+            assert sam_check.name == "Sam"
+            assert sam_check.employees[0].name == "Joe"
+
+
+@pytest.mark.asyncio
+async def test_other_forwardref_relation(cleanup):
+    async with db:
+        async with db.transaction(force_rollback=True):
+            checkers = await Game.objects.create(name="checkers")
+            uno = await Game(name="Uno").save()
+
+            await Child(
+                name="Billy", favourite_game=uno, least_favourite_game=checkers
+            ).save()
+            await Child(
+                name="Kate", favourite_game=checkers, least_favourite_game=uno
+            ).save()
+
+            billy_check = await Child.objects.select_related(
+                ["favourite_game", "least_favourite_game"]
+            ).get(name="Billy")
+            assert billy_check.favourite_game == uno
+            assert billy_check.least_favourite_game == checkers
+
+            uno_check = await Game.objects.select_related(
+                ["liked_by", "not_liked_by"]
+            ).get(name="Uno")
+            assert uno_check.liked_by[0].name == "Billy"
+            assert uno_check.not_liked_by[0].name == "Kate"
+
+
+@pytest.mark.asyncio
+async def test_m2m_self_forwardref_relation(cleanup):
+    async with db:
+        async with db.transaction(force_rollback=True):
+            checkers = await Game.objects.create(name="Checkers")
+            uno = await Game(name="Uno").save()
+            jenga = await Game(name="Jenga").save()
+
+            billy = await Child(
+                name="Billy", favourite_game=uno, least_favourite_game=checkers
+            ).save()
+            kate = await Child(
+                name="Kate", favourite_game=checkers, least_favourite_game=uno
+            ).save()
+            steve = await Child(
+                name="Steve", favourite_game=jenga, least_favourite_game=uno
+            ).save()
+
+            await billy.friends.add(kate)
+            await billy.friends.add(steve)
+
+            billy_check = await Child.objects.select_related(
+                [
+                    "friends",
+                    "favourite_game",
+                    "least_favourite_game",
+                    "friends__favourite_game",
+                    "friends__least_favourite_game",
+                ]
+            ).get(name="Billy")
+            assert len(billy_check.friends) == 2
+            assert billy_check.friends[0].name == "Kate"
+            assert billy_check.friends[0].favourite_game.name == "Checkers"
+            assert billy_check.friends[0].least_favourite_game.name == "Uno"
+            assert billy_check.friends[1].name == "Steve"
+            assert billy_check.friends[1].favourite_game.name == "Jenga"
+            assert billy_check.friends[1].least_favourite_game.name == "Uno"
+            assert billy_check.favourite_game.name == "Uno"
+
+            kate_check = await Child.objects.select_related(["also_friends"]).get(
+                name="Kate"
+            )
+
+            assert len(kate_check.also_friends) == 1
+            assert kate_check.also_friends[0].name == "Billy"
+
+            billy_check = (
+                await Child.objects.select_related(
+                    [
+                        "friends",
+                        "favourite_game",
+                        "least_favourite_game",
+                        "friends__favourite_game",
+                        "friends__least_favourite_game",
+                    ]
+                )
+                .filter(friends__favourite_game__name="Checkers")
+                .get(name="Billy")
+            )
+            assert len(billy_check.friends) == 1
+            assert billy_check.friends[0].name == "Kate"
+            assert billy_check.friends[0].favourite_game.name == "Checkers"
+            assert billy_check.friends[0].least_favourite_game.name == "Uno"

tests/test_many_to_many.py

@@ -80,6 +80,17 @@ async def cleanup():
         await Author.objects.delete(each=True)
 
 
+@pytest.mark.asyncio
+async def test_not_saved_raises_error(cleanup):
+    async with database:
+        guido = await Author(first_name="Guido", last_name="Van Rossum").save()
+        post = await Post.objects.create(title="Hello, M2M", author=guido)
+        news = Category(name="News")
+
+        with pytest.raises(ModelPersistenceError):
+            await post.categories.add(news)
+
+
 @pytest.mark.asyncio
 async def test_assigning_related_objects(cleanup):
     async with database:

tests/test_models_helpers.py

@@ -0,0 +1,17 @@
+from ormar.models.helpers.models import group_related_list
+
+
+def test_group_related_list():
+    given = [
+        "friends__least_favourite_game",
+        "least_favourite_game",
+        "friends",
+        "favourite_game",
+        "friends__favourite_game",
+    ]
+    expected = {
+        "least_favourite_game": [],
+        "favourite_game": [],
+        "friends": ["favourite_game", "least_favourite_game"],
+    }
+    assert group_related_list(given) == expected

tests/test_more_same_table_joins.py

@@ -101,15 +101,10 @@ async def test_model_multiple_instances_of_same_table_in_schema():
     async with database:
         await create_data()
         classes = await SchoolClass.objects.select_related(
-            ["teachers__category__department", "students"]
+            ["teachers__category__department", "students__category__department"]
         ).all()
         assert classes[0].name == "Math"
         assert classes[0].students[0].name == "Jane"
         assert len(classes[0].dict().get("students")) == 2
         assert classes[0].teachers[0].category.department.name == "Law Department"
-
-        assert classes[0].students[0].category.pk is not None
-        assert classes[0].students[0].category.name is None
-        await classes[0].students[0].category.load()
-        await classes[0].students[0].category.department.load()
         assert classes[0].students[0].category.department.name == "Math Department"

tests/test_order_by.py

@@ -280,7 +280,7 @@ async def test_sort_order_on_many_to_many():
         assert users[1].cars[3].name == "Buggy"
 
         users = (
-            await User.objects.select_related(["cars", "cars__factory"])
+            await User.objects.select_related(["cars__factory"])
             .order_by(["-cars__factory__name", "cars__name"])
             .all()
         )

tests/test_pagination.py

@@ -0,0 +1,110 @@
+import databases
+import pytest
+import sqlalchemy
+
+import ormar
+from ormar import ModelMeta
+from ormar.exceptions import QueryDefinitionError
+from tests.settings import DATABASE_URL
+
+database = databases.Database(DATABASE_URL, force_rollback=True)
+metadata = sqlalchemy.MetaData()
+
+
+class BaseMeta(ModelMeta):
+    metadata = metadata
+    database = database
+
+
+class Car(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+
+
+class UsersCar(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "cars_x_users"
+
+
+class User(ormar.Model):
+    class Meta(BaseMeta):
+        pass
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    cars = ormar.ManyToMany(Car, through=UsersCar)
+
+
+@pytest.fixture(autouse=True, scope="module")
+def create_test_database():
+    engine = sqlalchemy.create_engine(DATABASE_URL)
+    metadata.drop_all(engine)
+    metadata.create_all(engine)
+    yield
+    metadata.drop_all(engine)
+
+
+@pytest.mark.asyncio
+async def test_pagination_errors():
+    async with database:
+        async with database.transaction(force_rollback=True):
+            with pytest.raises(QueryDefinitionError):
+                await Car.objects.paginate(0).all()
+
+            with pytest.raises(QueryDefinitionError):
+                await Car.objects.paginate(1, page_size=0).all()
+
+
+@pytest.mark.asyncio
+async def test_pagination_on_single_model():
+    async with database:
+        async with database.transaction(force_rollback=True):
+            for i in range(20):
+                await Car(name=f"{i}").save()
+
+            cars_page1 = await Car.objects.paginate(1, page_size=5).all()
+            assert len(cars_page1) == 5
+            assert cars_page1[0].name == "0"
+            assert cars_page1[4].name == "4"
+            cars_page2 = await Car.objects.paginate(2, page_size=5).all()
+            assert len(cars_page2) == 5
+            assert cars_page2[0].name == "5"
+            assert cars_page2[4].name == "9"
+
+            all_cars = await Car.objects.paginate(1).all()
+            assert len(all_cars) == 20
+
+            half_cars = await Car.objects.paginate(2, page_size=10).all()
+            assert len(half_cars) == 10
+            assert half_cars[0].name == "10"
+
+
+@pytest.mark.asyncio
+async def test_proxy_pagination():
+    async with database:
+        async with database.transaction(force_rollback=True):
+            user = await User(name="Jon").save()
+
+            for i in range(20):
+                c = await Car(name=f"{i}").save()
+                await user.cars.add(c)
+
+            await user.cars.paginate(1, page_size=5).all()
+            assert len(user.cars) == 5
+            assert user.cars[0].name == "0"
+            assert user.cars[4].name == "4"
+
+            await user.cars.paginate(2, page_size=5).all()
+            assert len(user.cars) == 5
+            assert user.cars[0].name == "5"
+            assert user.cars[4].name == "9"
+
+            await user.cars.paginate(1).all()
+            assert len(user.cars) == 20
+
+            await user.cars.paginate(2, page_size=10).all()
+            assert len(user.cars) == 10
+            assert user.cars[0].name == "10"

tests/test_save_related.py

@@ -21,7 +21,7 @@ class CringeLevel(ormar.Model):
     name: str = ormar.String(max_length=100)
 
 
-class NickNames(ormar.Model):
+class NickName(ormar.Model):
     class Meta:
         tablename = "nicks"
         metadata = metadata
@@ -48,7 +48,7 @@ class HQ(ormar.Model):
 
     id: int = ormar.Integer(primary_key=True)
     name: str = ormar.String(max_length=100, nullable=False, name="hq_name")
-    nicks: List[NickNames] = ormar.ManyToMany(NickNames, through=NicksHq)
+    nicks: List[NickName] = ormar.ManyToMany(NickName, through=NicksHq)
 
 
 class Company(ormar.Model):
@@ -96,8 +96,8 @@ async def test_saving_related_fk_rel():
 async def test_saving_many_to_many():
     async with database:
         async with database.transaction(force_rollback=True):
-            nick1 = await NickNames.objects.create(name="BazingaO", is_lame=False)
+            nick1 = await NickName.objects.create(name="BazingaO", is_lame=False)
-            nick2 = await NickNames.objects.create(name="Bazinga20", is_lame=True)
+            nick2 = await NickName.objects.create(name="Bazinga20", is_lame=True)
 
             hq = await HQ.objects.create(name="Main")
             assert hq.saved
@@ -168,10 +168,10 @@ async def test_saving_nested():
         async with database.transaction(force_rollback=True):
             level = await CringeLevel.objects.create(name="High")
             level2 = await CringeLevel.objects.create(name="Low")
-            nick1 = await NickNames.objects.create(
+            nick1 = await NickName.objects.create(
                 name="BazingaO", is_lame=False, level=level
             )
-            nick2 = await NickNames.objects.create(
+            nick2 = await NickName.objects.create(
                 name="Bazinga20", is_lame=True, level=level2
             )
 

tests/test_saving_related.py

@@ -6,6 +6,7 @@ import sqlalchemy as sa
 from sqlalchemy import create_engine
 
 import ormar
+from ormar.exceptions import ModelPersistenceError
 from tests.settings import DATABASE_URL
 
 metadata = sa.MetaData()
@@ -61,3 +62,15 @@ async def test_model_relationship():
             assert ws.id == 1
             assert ws.topic == "Topic 2"
             assert ws.category.name == "Foo"
+
+
+@pytest.mark.asyncio
+async def test_model_relationship_with_not_saved():
+    async with db:
+        async with db.transaction(force_rollback=True):
+            cat = Category(name="Foo", code=123)
+            with pytest.raises(ModelPersistenceError):
+                await Workshop(topic="Topic 1", category=cat).save()
+
+            with pytest.raises(ModelPersistenceError):
+                await Workshop.objects.create(topic="Topic 1", category=cat)

tests/test_selecting_subset_of_columns.py

@@ -116,9 +116,7 @@ async def test_selecting_subset():
             )
 
             all_cars = (
-                await Car.objects.select_related(
+                await Car.objects.select_related(["manufacturer__hq__nicks"])
-                    ["manufacturer", "manufacturer__hq", "manufacturer__hq__nicks"]
-                )
                 .fields(
                     [
                         "id",
@@ -132,9 +130,7 @@ async def test_selecting_subset():
             )
 
             all_cars2 = (
-                await Car.objects.select_related(
+                await Car.objects.select_related(["manufacturer__hq__nicks"])
-                    ["manufacturer", "manufacturer__hq", "manufacturer__hq__nicks"]
-                )
                 .fields(
                     {
                         "id": ...,
@@ -149,9 +145,7 @@ async def test_selecting_subset():
             )
 
             all_cars3 = (
-                await Car.objects.select_related(
+                await Car.objects.select_related(["manufacturer__hq__nicks"])
-                    ["manufacturer", "manufacturer__hq", "manufacturer__hq__nicks"]
-                )
                 .fields(
                     {
                         "id": ...,