From b710ed9780833d6398d77b4a02206d8284b70d4d Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 26 Jan 2021 17:29:40 +0100 Subject: [PATCH] add tests for cross model forward references, add docs for processing forwardrefs, wip on refactoring queries into separate pages based on functionality --- docs/api/fields/base-field.md | 65 +- docs/api/fields/foreign-key.md | 102 ++- docs/api/fields/many-to-many.md | 85 ++- docs/api/models/helpers/models.md | 43 ++ docs/api/models/helpers/relations.md | 46 +- docs/api/models/helpers/sqlalchemy.md | 46 +- .../api/models/mixins/prefetch-query-mixin.md | 2 +- docs/api/models/model-metaclass.md | 25 +- docs/api/models/model.md | 31 +- docs/api/models/new-basemodel.md | 44 +- docs/api/query-set/clause.md | 111 +-- docs/api/query-set/join.md | 167 ++-- docs/api/query-set/prefetch-query.md | 6 +- docs/api/query-set/utils.md | 19 + docs/api/relations/alias-manager.md | 20 +- docs/api/relations/relation-manager.md | 6 +- docs/api/relations/relation-proxy.md | 2 +- docs/api/relations/utils.md | 7 +- docs/queries.md | 718 ------------------ docs/queries/aggregations.md | 28 + docs/queries/create.md | 83 ++ docs/queries/delete.md | 23 + docs/queries/filter-and-sort.md | 151 ++++ docs/queries/index.md | 156 ++++ docs/queries/joins-and-subqueries.md | 223 ++++++ docs/queries/pagination-and-rows-number.md | 94 +++ docs/queries/read.md | 80 ++ docs/queries/select-columns.md | 126 +++ docs/queries/update.md | 71 ++ docs/relations/index.md | 29 +- docs/relations/postponed-annotations.md | 171 +++++ docs/releases.md | 18 + mkdocs.yml | 13 +- ormar/fields/foreign_key.py | 2 +- ormar/fields/many_to_many.py | 16 +- ormar/models/model.py | 5 +- ormar/queryset/queryset.py | 3 - tests/test_forward_cross_refs.py | 114 +++ tests/test_forward_refs.py | 107 ++- 39 files changed, 2054 insertions(+), 1004 deletions(-) create mode 100644 docs/queries/aggregations.md create mode 100644 docs/queries/create.md create mode 100644 docs/queries/delete.md create mode 100644 docs/queries/filter-and-sort.md create mode 100644 docs/queries/index.md create mode 100644 docs/queries/joins-and-subqueries.md create mode 100644 docs/queries/pagination-and-rows-number.md create mode 100644 docs/queries/read.md create mode 100644 docs/queries/select-columns.md create mode 100644 docs/queries/update.md create mode 100644 docs/relations/postponed-annotations.md create mode 100644 tests/test_forward_cross_refs.py diff --git a/docs/api/fields/base-field.md b/docs/api/fields/base-field.md index 7bd2d25..ac7ea01 100644 --- a/docs/api/fields/base-field.md +++ b/docs/api/fields/base-field.md @@ -217,7 +217,7 @@ primary_key, index, unique, nullable, default and server_default. ```python | @classmethod - | expand_relationship(cls, value: Any, child: Union["Model", "NewBaseModel"], to_register: bool = True, relation_name: str = None) -> Any + | expand_relationship(cls, value: Any, child: Union["Model", "NewBaseModel"], to_register: bool = True) -> Any ``` Function overwritten for relations, in basic field the value is returned as is. @@ -236,3 +236,66 @@ dict (from Model) or actual instance/list of a "Model". `(Any)`: returns untouched value for normal fields, expands only for relations + +#### set\_self\_reference\_flag + +```python + | @classmethod + | set_self_reference_flag(cls) -> None +``` + +Sets `self_reference` to True if field to and owner are same model. + +**Returns**: + +`(None)`: None + + +#### has\_unresolved\_forward\_refs + +```python + | @classmethod + | has_unresolved_forward_refs(cls) -> bool +``` + +Verifies if the filed has any ForwardRefs that require updating before the +model can be used. + +**Returns**: + +`(bool)`: result of the check + + +#### evaluate\_forward\_ref + +```python + | @classmethod + | evaluate_forward_ref(cls, globalns: Any, localns: Any) -> None +``` + +Evaluates the ForwardRef to actual Field based on global and local namespaces + +**Arguments**: + +- `globalns (Any)`: global namespace +- `localns (Any)`: local namespace + +**Returns**: + +`(None)`: None + + +#### get\_related\_name + +```python + | @classmethod + | get_related_name(cls) -> str +``` + +Returns name to use for reverse relation. +It's either set as `related_name` or by default it's owner model. get_name + 's' + +**Returns**: + +`(str)`: name of the related_name or default related name. + diff --git a/docs/api/fields/foreign-key.md b/docs/api/fields/foreign-key.md index e6875dc..3f74f1c 100644 --- a/docs/api/fields/foreign-key.md +++ b/docs/api/fields/foreign-key.md @@ -46,6 +46,29 @@ Populates only pk field and set it to desired type. `(pydantic.BaseModel)`: constructed dummy model + +#### populate\_fk\_params\_based\_on\_to\_model + +```python +populate_fk_params_based_on_to_model(to: Type["Model"], nullable: bool, onupdate: str = None, ondelete: str = None) -> Tuple[Any, List, Any] +``` + +Based on target to model to which relation leads to populates the type of the +pydantic field to use, ForeignKey constraint and type of the target column field. + +**Arguments**: + +- `to (Model class)`: target related ormar Model +- `nullable (bool)`: marks field as optional/ required +- `onupdate (str)`: parameter passed to sqlalchemy.ForeignKey. +How to treat child rows on update of parent (the one where FK is defined) model. +- `ondelete (str)`: parameter passed to sqlalchemy.ForeignKey. +How to treat child rows on delete of parent (the one where FK is defined) model. + +**Returns**: + +`(Tuple[Any, List, Any])`: tuple with target pydantic type, list of fk constraints and target col type + ## UniqueColumns Objects @@ -71,7 +94,7 @@ to produce sqlalchemy.ForeignKeys #### ForeignKey ```python -ForeignKey(to: Type["Model"], *, name: str = None, unique: bool = False, nullable: bool = True, related_name: str = None, virtual: bool = False, onupdate: str = None, ondelete: str = None, **kwargs: Any, ,) -> Any +ForeignKey(to: Union[Type["Model"], "ForwardRef"], *, name: str = None, unique: bool = False, nullable: bool = True, related_name: str = None, virtual: bool = False, onupdate: str = None, ondelete: str = None, **kwargs: Any, ,) -> Any ``` Despite a name it's a function that returns constructed ForeignKeyField. @@ -107,12 +130,62 @@ class ForeignKeyField(BaseField) Actual class returned from ForeignKey function call and stored in model_fields. + +#### get\_source\_related\_name + +```python + | @classmethod + | get_source_related_name(cls) -> str +``` + +Returns name to use for source relation name. +For FK it's the same, differs for m2m fields. +It's either set as `related_name` or by default it's owner model. get_name + 's' + +**Returns**: + +`(str)`: name of the related_name or default related name. + + +#### get\_related\_name + +```python + | @classmethod + | get_related_name(cls) -> str +``` + +Returns name to use for reverse relation. +It's either set as `related_name` or by default it's owner model. get_name + 's' + +**Returns**: + +`(str)`: name of the related_name or default related name. + + +#### evaluate\_forward\_ref + +```python + | @classmethod + | evaluate_forward_ref(cls, globalns: Any, localns: Any) -> None +``` + +Evaluates the ForwardRef to actual Field based on global and local namespaces + +**Arguments**: + +- `globalns (Any)`: global namespace +- `localns (Any)`: local namespace + +**Returns**: + +`(None)`: None + #### \_extract\_model\_from\_sequence ```python | @classmethod - | _extract_model_from_sequence(cls, value: List, child: "Model", to_register: bool, relation_name: str) -> List["Model"] + | _extract_model_from_sequence(cls, value: List, child: "Model", to_register: bool) -> List["Model"] ``` Takes a list of Models and registers them on parent. @@ -135,7 +208,7 @@ Used in reverse FK relations. ```python | @classmethod - | _register_existing_model(cls, value: "Model", child: "Model", to_register: bool, relation_name: str) -> "Model" + | _register_existing_model(cls, value: "Model", child: "Model", to_register: bool) -> "Model" ``` Takes already created instance and registers it for parent. @@ -158,7 +231,7 @@ Used in reverse FK relations and normal FK for single models. ```python | @classmethod - | _construct_model_from_dict(cls, value: dict, child: "Model", to_register: bool, relation_name: str) -> "Model" + | _construct_model_from_dict(cls, value: dict, child: "Model", to_register: bool) -> "Model" ``` Takes a dictionary, creates a instance and registers it for parent. @@ -182,7 +255,7 @@ Used in normal FK for dictionaries. ```python | @classmethod - | _construct_model_from_pk(cls, value: Any, child: "Model", to_register: bool, relation_name: str) -> "Model" + | _construct_model_from_pk(cls, value: Any, child: "Model", to_register: bool) -> "Model" ``` Takes a pk value, creates a dummy instance and registers it for parent. @@ -205,7 +278,7 @@ Used in normal FK for dictionaries. ```python | @classmethod - | register_relation(cls, model: "Model", child: "Model", relation_name: str) -> None + | register_relation(cls, model: "Model", child: "Model") -> None ``` Registers relation between parent and child in relation manager. @@ -219,12 +292,27 @@ Used in Metaclass and sometimes some relations are missing - `model (Model class)`: parent model (with relation definition) - `child (Model class)`: child model + +#### has\_unresolved\_forward\_refs + +```python + | @classmethod + | has_unresolved_forward_refs(cls) -> bool +``` + +Verifies if the filed has any ForwardRefs that require updating before the +model can be used. + +**Returns**: + +`(bool)`: result of the check + #### expand\_relationship ```python | @classmethod - | expand_relationship(cls, value: Any, child: Union["Model", "NewBaseModel"], to_register: bool = True, relation_name: str = None) -> Optional[Union["Model", List["Model"]]] + | expand_relationship(cls, value: Any, child: Union["Model", "NewBaseModel"], to_register: bool = True) -> Optional[Union["Model", List["Model"]]] ``` For relations the child model is first constructed (if needed), diff --git a/docs/api/fields/many-to-many.md b/docs/api/fields/many-to-many.md index 259b66f..0b12763 100644 --- a/docs/api/fields/many-to-many.md +++ b/docs/api/fields/many-to-many.md @@ -1,11 +1,30 @@ # fields.many\_to\_many + +#### populate\_m2m\_params\_based\_on\_to\_model + +```python +populate_m2m_params_based_on_to_model(to: Type["Model"], nullable: bool) -> Tuple[Any, Any] +``` + +Based on target to model to which relation leads to populates the type of the +pydantic field to use and type of the target column field. + +**Arguments**: + +- `to (Model class)`: target related ormar Model +- `nullable (bool)`: marks field as optional/ required + +**Returns**: + +`(tuple with target pydantic type and target col type)`: Tuple[List, Any] + #### ManyToMany ```python -ManyToMany(to: Type["Model"], through: Type["Model"], *, name: str = None, unique: bool = False, virtual: bool = False, **kwargs: Any) -> Any +ManyToMany(to: Union[Type["Model"], ForwardRef], through: Union[Type["Model"], ForwardRef], *, name: str = None, unique: bool = False, virtual: bool = False, **kwargs: Any, ,) -> Any ``` Despite a name it's a function that returns constructed ManyToManyField. @@ -37,6 +56,22 @@ class ManyToManyField(ForeignKeyField, ormar.QuerySetProtocol, ormar.RelationP Actual class returned from ManyToMany function call and stored in model_fields. + +#### get\_source\_related\_name + +```python + | @classmethod + | get_source_related_name(cls) -> str +``` + +Returns name to use for source relation name. +For FK it's the same, differs for m2m fields. +It's either set as `related_name` or by default it's field name. + +**Returns**: + +`(str)`: name of the related_name or default related name. + #### default\_target\_field\_name @@ -51,3 +86,51 @@ Returns default target model name on through model. `(str)`: name of the field + +#### default\_source\_field\_name + +```python + | @classmethod + | default_source_field_name(cls) -> str +``` + +Returns default target model name on through model. + +**Returns**: + +`(str)`: name of the field + + +#### has\_unresolved\_forward\_refs + +```python + | @classmethod + | has_unresolved_forward_refs(cls) -> bool +``` + +Verifies if the filed has any ForwardRefs that require updating before the +model can be used. + +**Returns**: + +`(bool)`: result of the check + + +#### evaluate\_forward\_ref + +```python + | @classmethod + | evaluate_forward_ref(cls, globalns: Any, localns: Any) -> None +``` + +Evaluates the ForwardRef to actual Field based on global and local namespaces + +**Arguments**: + +- `globalns (Any)`: global namespace +- `localns (Any)`: local namespace + +**Returns**: + +`(None)`: None + diff --git a/docs/api/models/helpers/models.md b/docs/api/models/helpers/models.md index ddeb85d..ee3b100 100644 --- a/docs/api/models/helpers/models.md +++ b/docs/api/models/helpers/models.md @@ -1,6 +1,24 @@ # models.helpers.models + +#### is\_field\_an\_forward\_ref + +```python +is_field_an_forward_ref(field: Type["BaseField"]) -> bool +``` + +Checks if field is a relation field and whether any of the referenced models +are ForwardRefs that needs to be updated before proceeding. + +**Arguments**: + +- `field (Type[BaseField])`: model field to verify + +**Returns**: + +`(bool)`: result of the check + #### populate\_default\_options\_values @@ -62,3 +80,28 @@ Also related_names have to be unique for given related model. - `model_fields (Dict[str, ormar.Field])`: dictionary of declared ormar model fields - `new_model (Model class)`: + +#### group\_related\_list + +```python +group_related_list(list_: List) -> Dict +``` + +Translates the list of related strings into a dictionary. +That way nested models are grouped to traverse them in a right order +and to avoid repetition. + +Sample: ["people__houses", "people__cars__models", "people__cars__colors"] +will become: +{'people': {'houses': [], 'cars': ['models', 'colors']}} + +Result dictionary is sorted by length of the values and by key + +**Arguments**: + +- `list_ (List[str])`: list of related models used in select related + +**Returns**: + +`(Dict[str, List])`: list converted to dictionary to avoid repetition and group nested models + diff --git a/docs/api/models/helpers/relations.md b/docs/api/models/helpers/relations.md index 5e51ec4..d470756 100644 --- a/docs/api/models/helpers/relations.md +++ b/docs/api/models/helpers/relations.md @@ -5,7 +5,7 @@ #### register\_relation\_on\_build ```python -register_relation_on_build(new_model: Type["Model"], field_name: str) -> None +register_relation_on_build(field: Type["ForeignKeyField"]) -> None ``` Registers ForeignKey relation in alias_manager to set a table_prefix. @@ -17,14 +17,13 @@ aliases for proper sql joins. **Arguments**: -- `new_model (Model class)`: constructed model -- `field_name (str)`: name of the related field +- `field (ForeignKey class)`: relation field #### register\_many\_to\_many\_relation\_on\_build ```python -register_many_to_many_relation_on_build(new_model: Type["Model"], field: Type[ManyToManyField], field_name: str) -> None +register_many_to_many_relation_on_build(field: Type[ManyToManyField]) -> None ``` Registers connection between through model and both sides of the m2m relation. @@ -38,10 +37,25 @@ By default relation name is a model.name.lower(). **Arguments**: -- `field_name (str)`: name of the relation key -- `new_model (Model class)`: model on which m2m field is declared - `field (ManyToManyField class)`: relation field + +#### expand\_reverse\_relationship + +```python +expand_reverse_relationship(model_field: Type["ForeignKeyField"]) -> None +``` + +If the reverse relation has not been set before it's set here. + +**Arguments**: + +- `model_field ()`: + +**Returns**: + +`(None)`: None + #### expand\_reverse\_relationships @@ -62,7 +76,7 @@ If the reverse relation has not been set before it's set here. #### register\_reverse\_model\_fields ```python -register_reverse_model_fields(model: Type["Model"], child: Type["Model"], related_name: str, model_field: Type["ForeignKeyField"]) -> None +register_reverse_model_fields(model_field: Type["ForeignKeyField"]) -> None ``` Registers reverse ForeignKey field on related model. @@ -73,16 +87,13 @@ Autogenerated reverse fields also set related_name to the original field name. **Arguments**: -- `model (Model class)`: related model on which reverse field should be defined -- `child (Model class)`: parent model with relation definition -- `related_name (str)`: name by which reverse key should be registered - `model_field (relation Field)`: original relation ForeignKey field #### register\_relation\_in\_alias\_manager ```python -register_relation_in_alias_manager(new_model: Type["Model"], field: Type[ForeignKeyField], field_name: str) -> None +register_relation_in_alias_manager(field: Type[ForeignKeyField]) -> None ``` Registers the relation (and reverse relation) in alias manager. @@ -95,15 +106,13 @@ fk - register_relation_on_build **Arguments**: -- `new_model (Model class)`: model on which relation field is declared - `field (ForeignKey or ManyToManyField class)`: relation field -- `field_name (str)`: name of the relation key #### verify\_related\_name\_dont\_duplicate ```python -verify_related_name_dont_duplicate(child: Type["Model"], parent_model: Type["Model"], related_name: str) -> None +verify_related_name_dont_duplicate(related_name: str, model_field: Type["ForeignKeyField"]) -> None ``` Verifies whether the used related_name (regardless of the fact if user defined or @@ -117,9 +126,8 @@ model **Arguments**: -- `child (ormar.models.metaclass.ModelMetaclass)`: related Model class -- `parent_model (ormar.models.metaclass.ModelMetaclass)`: parent Model class - `related_name ()`: +- `model_field (relation Field)`: original relation ForeignKey field **Returns**: @@ -129,7 +137,7 @@ model #### reverse\_field\_not\_already\_registered ```python -reverse_field_not_already_registered(child: Type["Model"], child_model_name: str, parent_model: Type["Model"]) -> bool +reverse_field_not_already_registered(model_field: Type["ForeignKeyField"]) -> bool ``` Checks if child is already registered in parents pydantic fields. @@ -141,9 +149,7 @@ related model **Arguments**: -- `child (ormar.models.metaclass.ModelMetaclass)`: related Model class -- `child_model_name (str)`: related_name of the child if provided -- `parent_model (ormar.models.metaclass.ModelMetaclass)`: parent Model class +- `model_field (relation Field)`: original relation ForeignKey field **Returns**: diff --git a/docs/api/models/helpers/sqlalchemy.md b/docs/api/models/helpers/sqlalchemy.md index 87b6d0e..02c43c1 100644 --- a/docs/api/models/helpers/sqlalchemy.md +++ b/docs/api/models/helpers/sqlalchemy.md @@ -5,7 +5,7 @@ #### adjust\_through\_many\_to\_many\_model ```python -adjust_through_many_to_many_model(model: Type["Model"], child: Type["Model"], model_field: Type[ManyToManyField]) -> None +adjust_through_many_to_many_model(model_field: Type[ManyToManyField]) -> None ``` Registers m2m relation on through model. @@ -15,23 +15,22 @@ Sets pydantic fields with child and parent model types. **Arguments**: -- `model (Model class)`: model on which relation is declared -- `child (Model class)`: model to which m2m relation leads - `model_field (ManyToManyField)`: relation field defined in parent model #### create\_and\_append\_m2m\_fk ```python -create_and_append_m2m_fk(model: Type["Model"], model_field: Type[ManyToManyField]) -> None +create_and_append_m2m_fk(model: Type["Model"], model_field: Type[ManyToManyField], field_name: str) -> None ``` -Registers sqlalchemy Column with sqlalchemy.ForeignKey leadning to the model. +Registers sqlalchemy Column with sqlalchemy.ForeignKey leading to the model. Newly created field is added to m2m relation through model Meta columns and table. **Arguments**: +- `field_name (str)`: name of the column to create - `model (Model class)`: Model class to which FK should be created - `model_field (ManyToManyField field)`: field with ManyToMany relation @@ -83,6 +82,8 @@ cannot be pydantic_only. Append fields to columns if it's not pydantic_only, virtual ForeignKey or ManyToMany field. +Sets `owner` on each model_field as reference to newly created Model. + **Raises**: - `ModelDefinitionError`: if validation of related_names fail, @@ -125,6 +126,23 @@ Each model has to have pk. `(ormar.models.metaclass.ModelMetaclass)`: Model with populated pkname and columns in Meta + +#### check\_for\_null\_type\_columns\_from\_forward\_refs + +```python +check_for_null_type_columns_from_forward_refs(meta: "ModelMeta") -> bool +``` + +Check is any column is of NUllType() meaning it's empty column from ForwardRef + +**Arguments**: + +- `meta (Model class Meta)`: Meta class of the Model without sqlalchemy table constructed + +**Returns**: + +`(bool)`: result of the check + #### populate\_meta\_sqlalchemy\_table\_if\_required @@ -143,3 +161,21 @@ It populates name, metadata, columns and constraints. `(Model class)`: class with populated Meta.table + +#### update\_column\_definition + +```python +update_column_definition(model: Union[Type["Model"], Type["NewBaseModel"]], field: Type[ForeignKeyField]) -> None +``` + +Updates a column with a new type column based on updated parameters in FK fields. + +**Arguments**: + +- `model (Type["Model"])`: model on which columns needs to be updated +- `field (Type[ForeignKeyField])`: field with column definition that requires update + +**Returns**: + +`(None)`: None + diff --git a/docs/api/models/mixins/prefetch-query-mixin.md b/docs/api/models/mixins/prefetch-query-mixin.md index 05d8b8b..b5eb0f7 100644 --- a/docs/api/models/mixins/prefetch-query-mixin.md +++ b/docs/api/models/mixins/prefetch-query-mixin.md @@ -59,7 +59,7 @@ or field name specified by related parameter. ```python | @classmethod - | get_related_field_name(cls, target_field: Type["BaseField"]) -> str + | get_related_field_name(cls, target_field: Type["ForeignKeyField"]) -> str ``` Returns name of the relation field that should be used in prefetch query. diff --git a/docs/api/models/model-metaclass.md b/docs/api/models/model-metaclass.md index 949828e..c31abe9 100644 --- a/docs/api/models/model-metaclass.md +++ b/docs/api/models/model-metaclass.md @@ -1,6 +1,17 @@ # models.metaclass + +## ModelMeta Objects + +```python +class ModelMeta() +``` + +Class used for type hinting. +Users can subclass this one for convenience but it's not required. +The only requirement is that ormar.Model has to have inner class with name Meta. + #### check\_if\_field\_has\_choices @@ -143,7 +154,7 @@ as well as model.Meta.model_fields definitions from parents. **Arguments**: - `attrs (Dict)`: new namespace for class being constructed -- `new_attrs (Dict)`: part of the namespace extracted from parent class +- `new_attrs (Dict)`: related of the namespace extracted from parent class - `model_fields (Dict[str, BaseField])`: ormar fields in defined in current class - `new_model_fields (Dict[str, BaseField])`: ormar fields defined in parent classes - `new_fields (Set[str])`: set of new fields names @@ -270,18 +281,6 @@ If the class is a ormar.Model it is skipped. `(Tuple[Dict, Dict])`: updated attrs and model_fields - -## ModelMeta Objects - -```python -class ModelMeta() -``` - -Class used for type hinting. -Users can subclass this one for convenience but it's not required. -The only requirement is that ormar.Model has to have inner class with name Meta. - - ## ModelMetaclass Objects diff --git a/docs/api/models/model.md b/docs/api/models/model.md index c770017..9bf3a90 100644 --- a/docs/api/models/model.md +++ b/docs/api/models/model.md @@ -1,29 +1,6 @@ # models.model - -#### group\_related\_list - -```python -group_related_list(list_: List) -> Dict -``` - -Translates the list of related strings into a dictionary. -That way nested models are grouped to traverse them in a right order -and to avoid repetition. - -Sample: ["people__houses", "people__cars__models", "people__cars__colors"] -will become: -{'people': {'houses': [], 'cars': ['models', 'colors']}} - -**Arguments**: - -- `list_ (List[str])`: list of related models used in select related - -**Returns**: - -`(Dict[str, List])`: list converted to dictionary to avoid repetition and group nested models - ## Model Objects @@ -36,7 +13,7 @@ class Model(NewBaseModel) ```python | @classmethod - | from_row(cls: Type[T], row: sqlalchemy.engine.ResultProxy, select_related: List = None, related_models: Any = None, previous_model: Type[T] = None, related_name: str = None, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None) -> Optional[T] + | from_row(cls: Type[T], row: sqlalchemy.engine.ResultProxy, select_related: List = None, related_models: Any = None, previous_model: Type[T] = None, source_model: Type[T] = None, related_name: str = None, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None, current_relation_str: str = None) -> Optional[T] ``` Model method to convert raw sql row from database into ormar.Model instance. @@ -72,7 +49,7 @@ excludes the fields even if they are provided in fields ```python | @classmethod - | populate_nested_models_from_row(cls, item: dict, row: sqlalchemy.engine.ResultProxy, related_models: Any, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None) -> dict + | populate_nested_models_from_row(cls, item: dict, row: sqlalchemy.engine.ResultProxy, related_models: Any, fields: Optional[Union[Dict, Set]] = None, exclude_fields: Optional[Union[Dict, Set]] = None, current_relation_str: str = None, source_model: Type[T] = None) -> dict ``` Traverses structure of related models and populates the nested models @@ -86,6 +63,8 @@ instances. In the end those instances are added to the final model dictionary. **Arguments**: +- `source_model (Type[Model])`: source model from which relation started +- `current_relation_str (str)`: joined related parts into one string - `item (Dict)`: dictionary of already populated nested models, otherwise empty dict - `row (sqlalchemy.engine.result.ResultProxy)`: raw result row from the database - `related_models (Union[Dict, List])`: list or dict of related models @@ -114,7 +93,7 @@ If the table is a main table, there is no prefix. All joined tables have prefixes to allow duplicate column names, as well as duplicated joins to the same table from multiple different tables. -Extracted fields populates the item dict later used to construct a Model. +Extracted fields populates the related dict later used to construct a Model. Used in Model.from_row and PrefetchQuery._populate_rows methods. diff --git a/docs/api/models/new-basemodel.md b/docs/api/models/new-basemodel.md index 5cec0ad..6499cf0 100644 --- a/docs/api/models/new-basemodel.md +++ b/docs/api/models/new-basemodel.md @@ -48,7 +48,8 @@ them with their default values if default is set. **Raises**: -- `ModelError`: if abstract model is initialized or unknown field is passed +- `ModelError`: if abstract model is initialized, model has ForwardRefs +that has not been updated or unknown field is passed **Arguments**: @@ -128,6 +129,19 @@ Json fields are converted if needed. `(Any)`: value of the attribute + +#### \_verify\_model\_can\_be\_initialized + +```python + | _verify_model_can_be_initialized() -> None +``` + +Raises exception if model is abstract or has ForwardRefs in relation fields. + +**Returns**: + +`(None)`: None + #### \_extract\_related\_model\_instead\_of\_field @@ -299,6 +313,34 @@ present in fastapi responses. `(Set[str])`: set of property fields names + +#### update\_forward\_refs + +```python + | @classmethod + | update_forward_refs(cls, **localns: Any) -> None +``` + +Processes fields that are ForwardRef and need to be evaluated into actual +models. + +Expands relationships, register relation in alias manager and substitutes +sqlalchemy columns with new ones with proper column type (null before). + +Populates Meta table of the Model which is left empty before. + +Sets self_reference flag on models that links to themselves. + +Calls the pydantic method to evaluate pydantic fields. + +**Arguments**: + +- `localns (Any)`: local namespace + +**Returns**: + +`(None)`: None + #### \_get\_related\_not\_excluded\_fields diff --git a/docs/api/query-set/clause.md b/docs/api/query-set/clause.md index f43a6c2..53bd55f 100644 --- a/docs/api/query-set/clause.md +++ b/docs/api/query-set/clause.md @@ -8,13 +8,13 @@ class QueryClause() ``` -Constructs where clauses from strings passed as arguments +Constructs FilterActions from strings passed as arguments - -#### filter + +#### prepare\_filter ```python - | filter(**kwargs: Any) -> Tuple[List[sqlalchemy.sql.expression.TextClause], List[str]] + | prepare_filter(**kwargs: Any) -> Tuple[List[FilterAction], List[str]] ``` Main external access point that processes the clauses into sqlalchemy text @@ -33,7 +33,7 @@ mentioned in select_related strings but not included in select_related. #### \_populate\_filter\_clauses ```python - | _populate_filter_clauses(**kwargs: Any) -> Tuple[List[sqlalchemy.sql.expression.TextClause], List[str]] + | _populate_filter_clauses(**kwargs: Any) -> Tuple[List[FilterAction], List[str]] ``` Iterates all clauses and extracts used operator and field from related @@ -48,114 +48,59 @@ is determined and the final clause is escaped if needed and compiled. `(Tuple[List[sqlalchemy.sql.elements.TextClause], List[str]])`: Tuple with list of where clauses and updated select_related list - -#### \_process\_column\_clause\_for\_operator\_and\_value + +#### \_register\_complex\_duplicates ```python - | _process_column_clause_for_operator_and_value(value: Any, op: str, column: sqlalchemy.Column, table: sqlalchemy.Table, table_prefix: str) -> sqlalchemy.sql.expression.TextClause + | _register_complex_duplicates(select_related: List[str]) -> None ``` -Escapes characters if it's required. -Substitutes values of the models if value is a ormar Model with its pk value. -Compiles the clause. +Checks if duplicate aliases are presented which can happen in self relation +or when two joins end with the same pair of models. + +If there are duplicates, the all duplicated joins are registered as source +model and whole relation key (not just last relation name). **Arguments**: -- `value (Any)`: value of the filter -- `op (str)`: filter operator -- `column (sqlalchemy.sql.schema.Column)`: column on which filter should be applied -- `table (sqlalchemy.sql.schema.Table)`: table on which filter should be applied -- `table_prefix (str)`: prefix from AliasManager +- `select_related (List[str])`: list of relation strings **Returns**: -`(sqlalchemy.sql.elements.TextClause)`: complied and escaped clause +`(None)`: None - -#### \_determine\_filter\_target\_table + +#### \_parse\_related\_prefixes ```python - | _determine_filter_target_table(related_parts: List[str], select_related: List[str]) -> Tuple[List[str], str, Type["Model"]] + | _parse_related_prefixes(select_related: List[str]) -> List[Prefix] ``` -Adds related strings to select_related list otherwise the clause would fail as -the required columns would not be present. That means that select_related -list is filled with missing values present in filters. - -Walks the relation to retrieve the actual model on which the clause should be -constructed, extracts alias based on last relation leading to target model. +Walks all relation strings and parses the target models and prefixes. **Arguments**: -- `related_parts (List[str])`: list of split parts of related string -- `select_related (List[str])`: list of related models +- `select_related (List[str])`: list of relation strings **Returns**: -`(Tuple[List[str], str, Type[Model]])`: list of related models, table_prefix, final model class +`(List[Prefix])`: list of parsed prefixes - -#### \_compile\_clause + +#### \_switch\_filter\_action\_prefixes ```python - | _compile_clause(clause: sqlalchemy.sql.expression.BinaryExpression, column: sqlalchemy.Column, table: sqlalchemy.Table, table_prefix: str, modifiers: Dict) -> sqlalchemy.sql.expression.TextClause + | _switch_filter_action_prefixes(filter_clauses: List[FilterAction]) -> List[FilterAction] ``` -Compiles the clause to str using appropriate database dialect, replace columns -names with aliased names and converts it back to TextClause. +Substitutes aliases for filter action if the complex key (whole relation str) is +present in alias_manager. **Arguments**: -- `clause (sqlalchemy.sql.elements.BinaryExpression)`: original not compiled clause -- `column (sqlalchemy.sql.schema.Column)`: column on which filter should be applied -- `table (sqlalchemy.sql.schema.Table)`: table on which filter should be applied -- `table_prefix (str)`: prefix from AliasManager -- `modifiers (Dict[str, NoneType])`: sqlalchemy modifiers - used only to escape chars here +- `filter_clauses (List[FilterAction])`: raw list of actions **Returns**: -`(sqlalchemy.sql.elements.TextClause)`: compiled and escaped clause - - -#### \_escape\_characters\_in\_clause - -```python - | @staticmethod - | _escape_characters_in_clause(op: str, value: Any) -> Tuple[Any, bool] -``` - -Escapes the special characters ["%", "_"] if needed. -Adds `%` for `like` queries. - -**Raises**: - -- `QueryDefinitionError`: if contains or icontains is used with -ormar model instance - -**Arguments**: - -- `op (str)`: operator used in query -- `value (Any)`: value of the filter - -**Returns**: - -`(Tuple[Any, bool])`: escaped value and flag if escaping is needed - - -#### \_extract\_operator\_field\_and\_related - -```python - | @staticmethod - | _extract_operator_field_and_related(parts: List[str]) -> Tuple[str, str, Optional[List]] -``` - -Splits filter query key and extracts required parts. - -**Arguments**: - -- `parts (List[str])`: split filter query key - -**Returns**: - -`(Tuple[str, str, Optional[List]])`: operator, field_name, list of related parts +`(List[FilterAction])`: list of actions with aliases changed if needed diff --git a/docs/api/query-set/join.md b/docs/api/query-set/join.md index 7a08704..213d4c0 100644 --- a/docs/api/query-set/join.md +++ b/docs/api/query-set/join.md @@ -1,15 +1,6 @@ # queryset.join - -## JoinParameters Objects - -```python -class JoinParameters(NamedTuple) -``` - -Named tuple that holds set of parameters passed during join construction. - ## SqlJoin Objects @@ -21,15 +12,11 @@ class SqlJoin() #### alias\_manager ```python - | @staticmethod - | alias_manager(model_cls: Type["Model"]) -> AliasManager + | @property + | alias_manager() -> AliasManager ``` -Shortcut for ormars model AliasManager stored on Meta. - -**Arguments**: - -- `model_cls (Type[Model])`: ormar Model class +Shortcut for ormar's model AliasManager stored on Meta. **Returns**: @@ -39,8 +26,7 @@ Shortcut for ormars model AliasManager stored on Meta. #### on\_clause ```python - | @staticmethod - | on_clause(previous_alias: str, alias: str, from_clause: str, to_clause: str) -> text + | on_clause(previous_alias: str, from_clause: str, to_clause: str) -> text ``` Receives aliases and names of both ends of the join and combines them @@ -49,7 +35,6 @@ into one text clause used in joins. **Arguments**: - `previous_alias (str)`: alias of previous table -- `alias (str)`: alias of current table - `from_clause (str)`: from table name - `to_clause (str)`: to table name @@ -57,32 +42,11 @@ into one text clause used in joins. `(sqlalchemy.text)`: clause combining all strings - -#### update\_inclusions - -```python - | @staticmethod - | update_inclusions(model_cls: Type["Model"], fields: Optional[Union[Set, Dict]], exclude_fields: Optional[Union[Set, Dict]], nested_name: str) -> Tuple[Optional[Union[Dict, Set]], Optional[Union[Dict, Set]]] -``` - -Extract nested fields and exclude_fields if applicable. - -**Arguments**: - -- `model_cls (Type["Model"])`: ormar model class -- `fields (Optional[Union[Set, Dict]])`: fields to include -- `exclude_fields (Optional[Union[Set, Dict]])`: fields to exclude -- `nested_name (str)`: name of the nested field - -**Returns**: - -`(Tuple[Optional[Union[Dict, Set]], Optional[Union[Dict, Set]]])`: updated exclude and include fields from nested objects - #### build\_join ```python - | build_join(item: str, join_parameters: JoinParameters) -> Tuple[List, sqlalchemy.sql.select, List, OrderedDict] + | build_join() -> Tuple[List, sqlalchemy.sql.select, List, OrderedDict] ``` Main external access point for building a join. @@ -90,42 +54,96 @@ Splits the join definition, updates fields and exclude_fields if needed, handles switching to through models for m2m relations, returns updated lists of used_aliases and sort_orders. -**Arguments**: - -- `item (str)`: string with join definition -- `join_parameters (JoinParameters)`: parameters from previous/ current join - **Returns**: `(Tuple[List[str], Join, List[TextClause], collections.OrderedDict])`: list of used aliases, select from, list of aliased columns, sort orders - -#### \_build\_join\_parameters + +#### \_forward\_join ```python - | _build_join_parameters(part: str, join_params: JoinParameters, fields: Optional[Union[Set, Dict]], exclude_fields: Optional[Union[Set, Dict]], is_multi: bool = False) -> JoinParameters + | _forward_join() -> None ``` -Updates used_aliases to not join multiple times to the same table. -Updates join parameters with new values. +Process actual join. +Registers complex relation join on encountering of the duplicated alias. + + +#### \_process\_following\_joins + +```python + | _process_following_joins() -> None +``` + +Iterates through nested models to create subsequent joins. + + +#### \_process\_deeper\_join + +```python + | _process_deeper_join(related_name: str, remainder: Any) -> None +``` + +Creates nested recurrent instance of SqlJoin for each nested join table, +updating needed return params here as a side effect. + +Updated are: + +* self.used_aliases, +* self.select_from, +* self.columns, +* self.sorted_orders, **Arguments**: -- `part (str)`: part of the join str definition -- `join_params (JoinParameters)`: parameters from previous/ current join -- `fields (Optional[Union[Set, Dict]])`: fields to include -- `exclude_fields (Optional[Union[Set, Dict]])`: fields to exclude -- `is_multi (bool)`: flag if the relation is m2m +- `related_name (str)`: name of the relation to follow +- `remainder (Any)`: deeper tables if there are more nested joins + + +#### process\_m2m\_through\_table + +```python + | process_m2m_through_table() -> None +``` + +Process Through table of the ManyToMany relation so that source table is +linked to the through table (one additional join) + +Replaces needed parameters like: + +* self.next_model, +* self.next_alias, +* self.relation_name, +* self.own_alias, +* self.target_field + +To point to through model + + +#### process\_m2m\_related\_name\_change + +```python + | process_m2m_related_name_change(reverse: bool = False) -> str +``` + +Extracts relation name to link join through the Through model declared on +relation field. + +Changes the same names in order_by queries if they are present. + +**Arguments**: + +- `reverse (bool)`: flag if it's on_clause lookup - use reverse fields **Returns**: -`(ormar.queryset.join.JoinParameters)`: updated join parameters +`(str)`: new relation name switched to through model field #### \_process\_join ```python - | _process_join(join_params: JoinParameters, is_multi: bool, model_cls: Type["Model"], part: str, alias: str, fields: Optional[Union[Set, Dict]], exclude_fields: Optional[Union[Set, Dict]]) -> None + | _process_join() -> None ``` Resolves to and from column names and table names. @@ -140,18 +158,8 @@ Updates the used aliases list directly. Process order_by causes for non m2m relations. -**Arguments**: - -- `join_params (JoinParameters)`: parameters from previous/ current join -- `is_multi (bool)`: flag if it's m2m relation -- `model_cls (ormar.models.metaclass.ModelMetaclass)`: -- `part (str)`: name of the field used in join -- `alias (str)`: alias of the current join -- `fields (Optional[Union[Set, Dict]])`: fields to include -- `exclude_fields (Optional[Union[Set, Dict]])`: fields to exclude - -#### \_switch\_many\_to\_many\_order\_columns +#### \_replace\_many\_to\_many\_order\_by\_columns ```python | _replace_many_to_many_order_by_columns(part: str, new_part: str) -> None @@ -187,7 +195,7 @@ Checks filter conditions to find if they apply to current join. #### set\_aliased\_order\_by ```python - | set_aliased_order_by(condition: List[str], alias: str, to_table: str, model_cls: Type["Model"]) -> None + | set_aliased_order_by(condition: List[str], to_table: str) -> None ``` Substitute hyphens ('-') with descending order. @@ -196,15 +204,13 @@ Construct actual sqlalchemy text clause using aliased table and column name. **Arguments**: - `condition (List[str])`: list of parts of a current condition split by '__' -- `alias (str)`: alias of the table in current join - `to_table (sqlalchemy.sql.elements.quoted_name)`: target table -- `model_cls (ormar.models.metaclass.ModelMetaclass)`: ormar model class #### get\_order\_bys ```python - | get_order_bys(alias: str, to_table: str, pkname_alias: str, part: str, model_cls: Type["Model"]) -> None + | get_order_bys(to_table: str, pkname_alias: str) -> None ``` Triggers construction of order bys if they are given. @@ -212,30 +218,19 @@ Otherwise by default each table is sorted by a primary key column asc. **Arguments**: -- `alias (str)`: alias of current table in join - `to_table (sqlalchemy.sql.elements.quoted_name)`: target table - `pkname_alias (str)`: alias of the primary key column -- `part (str)`: name of the current relation join -- `model_cls (Type[Model])`: ormar model class #### get\_to\_and\_from\_keys ```python - | @staticmethod - | get_to_and_from_keys(join_params: JoinParameters, is_multi: bool, model_cls: Type["Model"], part: str) -> Tuple[str, str] + | get_to_and_from_keys() -> Tuple[str, str] ``` Based on the relation type, name of the relation and previous models and parts stored in JoinParameters it resolves the current to and from keys, which are -different for ManyToMany relation, ForeignKey and reverse part of relations. - -**Arguments**: - -- `join_params (JoinParameters)`: parameters from previous/ current join -- `is_multi (bool)`: flag if the relation is of m2m type -- `model_cls (Type[Model])`: ormar model class -- `part (str)`: name of the current relation join +different for ManyToMany relation, ForeignKey and reverse related of relations. **Returns**: diff --git a/docs/api/query-set/prefetch-query.md b/docs/api/query-set/prefetch-query.md index d6ceea0..cc848f0 100644 --- a/docs/api/query-set/prefetch-query.md +++ b/docs/api/query-set/prefetch-query.md @@ -289,7 +289,7 @@ models. | _get_select_related_if_apply(related: str, select_dict: Dict) -> Dict ``` -Extract nested part of select_related dictionary to extract models nested +Extract nested related of select_related dictionary to extract models nested deeper on related model and already loaded in select related query. **Arguments**: @@ -299,7 +299,7 @@ deeper on related model and already loaded in select related query. **Returns**: -`(Dict)`: dictionary with nested part of select related +`(Dict)`: dictionary with nested related of select related #### \_update\_already\_loaded\_rows @@ -320,7 +320,7 @@ Updates models that are already loaded, usually children of children. #### \_populate\_rows ```python - | _populate_rows(rows: List, target_field: Type["BaseField"], parent_model: Type["Model"], table_prefix: str, fields: Union[Set[Any], Dict[Any, Any], None], exclude_fields: Union[Set[Any], Dict[Any, Any], None], prefetch_dict: Dict, orders_by: Dict) -> None + | _populate_rows(rows: List, target_field: Type["ForeignKeyField"], parent_model: Type["Model"], table_prefix: str, fields: Union[Set[Any], Dict[Any, Any], None], exclude_fields: Union[Set[Any], Dict[Any, Any], None], prefetch_dict: Dict, orders_by: Dict) -> None ``` Instantiates children models extracted from given relation. diff --git a/docs/api/query-set/utils.md b/docs/api/query-set/utils.md index 27989a3..0ba9471 100644 --- a/docs/api/query-set/utils.md +++ b/docs/api/query-set/utils.md @@ -150,3 +150,22 @@ with all children models under their relation keys. `(Dict)`: dictionary of lists f related models + +#### get\_relationship\_alias\_model\_and\_str + +```python +get_relationship_alias_model_and_str(source_model: Type["Model"], related_parts: List) -> Tuple[str, Type["Model"], str] +``` + +Walks the relation to retrieve the actual model on which the clause should be +constructed, extracts alias based on last relation leading to target model. + +**Arguments**: + +- `related_parts (Union[List, List[str]])`: list of related names extracted from string +- `source_model (Type[Model])`: model from which relation starts + +**Returns**: + +`(Tuple[str, Type["Model"], str])`: table prefix, target model and relation string + diff --git a/docs/api/relations/alias-manager.md b/docs/api/relations/alias-manager.md index 4190288..24016de 100644 --- a/docs/api/relations/alias-manager.md +++ b/docs/api/relations/alias-manager.md @@ -74,7 +74,7 @@ Creates text clause with table name with aliased name. #### add\_relation\_type ```python - | add_relation_type(source_model: Type["Model"], relation_name: str, reverse_name: str = None, is_multi: bool = False) -> None + | add_relation_type(source_model: Type["Model"], relation_name: str, reverse_name: str = None) -> None ``` Registers the relations defined in ormar models. @@ -94,12 +94,28 @@ on one model as well as from multiple different models in one join. - `source_model (source Model)`: model with relation defined - `relation_name (str)`: name of the relation to define - `reverse_name (Optional[str])`: name of related_name fo given relation for m2m relations -- `is_multi (bool)`: flag if relation being registered is a through m2m model **Returns**: `(None)`: none + +#### add\_alias + +```python + | add_alias(alias_key: str) -> str +``` + +Adds alias to the dictionary of aliases under given key. + +**Arguments**: + +- `alias_key (str)`: key of relation to generate alias for + +**Returns**: + +`(str)`: generated alias + #### resolve\_relation\_alias diff --git a/docs/api/relations/relation-manager.md b/docs/api/relations/relation-manager.md index 21f5947..57ad512 100644 --- a/docs/api/relations/relation-manager.md +++ b/docs/api/relations/relation-manager.md @@ -98,7 +98,7 @@ Returns the actual relation and not the related model(s). ```python | @staticmethod - | add(parent: "Model", child: "Model", child_name: str, virtual: bool, relation_name: str) -> None + | add(parent: "Model", child: "Model", field: Type["ForeignKeyField"]) -> None ``` Adds relation on both sides -> meaning on both child and parent models. @@ -112,9 +112,7 @@ on both ends. - `parent (Model)`: parent model on which relation should be registered - `child (Model)`: child model to register -- `child_name (str)`: potential child name used if related name is not set -- `virtual (bool)`: -- `relation_name (str)`: name of the relation +- `field (ForeignKeyField)`: field with relation definition #### remove diff --git a/docs/api/relations/relation-proxy.md b/docs/api/relations/relation-proxy.md index b2716f7..645bb2a 100644 --- a/docs/api/relations/relation-proxy.md +++ b/docs/api/relations/relation-proxy.md @@ -114,7 +114,7 @@ to the parent model only, without need for user to filter them. | async remove(item: "Model", keep_reversed: bool = True) -> None ``` -Removes the item from relation with parent. +Removes the related from relation with parent. Through models are automatically deleted for m2m relations. diff --git a/docs/api/relations/utils.md b/docs/api/relations/utils.md index cf3c945..a771d31 100644 --- a/docs/api/relations/utils.md +++ b/docs/api/relations/utils.md @@ -5,7 +5,7 @@ #### get\_relations\_sides\_and\_names ```python -get_relations_sides_and_names(to_field: Type[BaseField], parent: "Model", child: "Model", child_name: str, virtual: bool, relation_name: str) -> Tuple["Model", "Model", str, str] +get_relations_sides_and_names(to_field: Type[ForeignKeyField], parent: "Model", child: "Model") -> Tuple["Model", "Model", str, str] ``` Determines the names of child and parent relations names, as well as @@ -13,12 +13,9 @@ changes one of the sides of the relation into weakref.proxy to model. **Arguments**: -- `to_field (BaseField)`: field with relation definition +- `to_field (ForeignKeyField)`: field with relation definition - `parent (Model)`: parent model - `child (Model)`: child model -- `child_name (str)`: name of the child -- `virtual (bool)`: flag if relation is virtual -- `relation_name ()`: **Returns**: diff --git a/docs/queries.md b/docs/queries.md index 04efcd2..e69de29 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -1,718 +0,0 @@ -# Queries - -## QuerySet - -Each Model is auto registered with a `QuerySet` that represents the underlaying query and it's options. - -Most of the methods are also available through many to many relation interface. - -!!!info - To see which one are supported and how to construct relations visit [relations][relations]. - -Given the Models like this - -```Python ---8<-- "../docs_src/queries/docs001.py" -``` - -we can demonstrate available methods to fetch and save the data into the database. - - -### create - -`create(**kwargs): -> Model` - -Creates the model instance, saves it in a database and returns the updates model -(with pk populated if not passed and autoincrement is set). - -The allowed kwargs are `Model` fields names and proper value types. - -```python -malibu = await Album.objects.create(name="Malibu") -await Track.objects.create(album=malibu, title="The Bird", position=1) -``` - -The alternative is a split creation and persistence of the `Model`. -```python -malibu = Album(name="Malibu") -await malibu.save() -``` - -!!!tip - Check other `Model` methods in [models][models] - -### get - -`get(**kwargs): -> Model` - -Get's the first row from the db meeting the criteria set by kwargs. - -If no criteria set it will return the last row in db sorted by pk. - -Passing a criteria is actually calling filter(**kwargs) method described below. - -```python -track = await Track.objects.get(name='The Bird') -# note that above is equivalent to await Track.objects.filter(name='The Bird').get() -track2 = track = await Track.objects.get() -track == track2 # True since it's the only row in db in our example -``` - -!!!warning - If no row meets the criteria `NoMatch` exception is raised. - - If there are multiple rows meeting the criteria the `MultipleMatches` exception is raised. - -### get_or_create - -`get_or_create(**kwargs) -> Model` - -Combination of create and get methods. - -Tries to get a row meeting the criteria and if `NoMatch` exception is raised it creates a new one with given kwargs. - -```python -album = await Album.objects.get_or_create(name='The Cat') -# object is created as it does not exist -album2 = await Album.objects.get_or_create(name='The Cat') -assert album == album2 -# return True as the same db row is returned -``` - -!!!warning - Despite being a equivalent row from database the `album` and `album2` in example above are 2 different python objects! - Updating one of them will not refresh the second one until you excplicitly load() the fresh data from db. - -!!!note - Note that if you want to create a new object you either have to pass pk column value or pk column has to be set as autoincrement - -### first - -`first(): -> Model` - -Gets the first row from the db ordered by primary key column ascending. - - -### update - -`update(each: bool = False, **kwargs) -> int` - -QuerySet level update is used to update multiple records with the same value at once. - -You either have to filter the QuerySet first or provide a `each=True` flag to update whole table. - -If you do not provide this flag or a filter a `QueryDefinitionError` will be raised. - -Return number of rows updated. - -```Python hl_lines="26-28" ---8<-- "../docs_src/queries/docs002.py" -``` - -!!!warning - Queryset needs to be filtered before updating to prevent accidental overwrite. - - To update whole database table `each=True` needs to be provided as a safety switch - - -### update_or_create - -`update_or_create(**kwargs) -> Model` - -Updates the model, or in case there is no match in database creates a new one. - -```Python hl_lines="26-32" ---8<-- "../docs_src/queries/docs003.py" -``` - -!!!note - Note that if you want to create a new object you either have to pass pk column value or pk column has to be set as autoincrement - - -### bulk_create - -`bulk_create(objects: List["Model"]) -> None` - -Allows you to create multiple objects at once. - -A valid list of `Model` objects needs to be passed. - -```python hl_lines="21-27" ---8<-- "../docs_src/queries/docs004.py" -``` - -### bulk_update - -`bulk_update(objects: List["Model"], columns: List[str] = None) -> None` - -Allows to update multiple instance at once. - -All `Models` passed need to have primary key column populated. - -You can also select which fields to update by passing `columns` list as a list of string names. - -```python hl_lines="8" -# continuing the example from bulk_create -# update objects -for todo in todoes: - todo.completed = False - -# perform update of all objects at once -# objects need to have pk column set, otherwise exception is raised -await ToDo.objects.bulk_update(todoes) - -completed = await ToDo.objects.filter(completed=False).all() -assert len(completed) == 3 -``` - -### delete - -`delete(each: bool = False, **kwargs) -> int` - -QuerySet level delete is used to delete multiple records at once. - -You either have to filter the QuerySet first or provide a `each=True` flag to delete whole table. - -If you do not provide this flag or a filter a `QueryDefinitionError` will be raised. - -Return number of rows deleted. - -```python hl_lines="26-30" ---8<-- "../docs_src/queries/docs005.py" -``` - -### all - -`all(**kwargs) -> List[Optional["Model"]]` - -Returns all rows from a database for given model for set filter options. - -Passing kwargs is a shortcut and equals to calling `filter(**kwrags).all()`. - -If there are no rows meeting the criteria an empty list is returned. - -```python -tracks = await Track.objects.select_related("album").all(title='Sample') -# will return a list of all Tracks with title Sample - -tracks = await Track.objects.all() -# will return a list of all Tracks in database - -``` - -### filter - -`filter(**kwargs) -> QuerySet` - -Allows you to filter by any `Model` attribute/field -as well as to fetch instances, with a filter across an FK relationship. - -```python -track = Track.objects.filter(name="The Bird").get() -# will return a track with name equal to 'The Bird' - -tracks = Track.objects.filter(album__name="Fantasies").all() -# will return all tracks where the columns album name = 'Fantasies' -``` - -You can use special filter suffix to change the filter operands: - -* exact - like `album__name__exact='Malibu'` (exact match) -* iexact - like `album__name__iexact='malibu'` (exact match case insensitive) -* contains - like `album__name__contains='Mal'` (sql like) -* icontains - like `album__name__icontains='mal'` (sql like case insensitive) -* in - like `album__name__in=['Malibu', 'Barclay']` (sql in) -* gt - like `position__gt=3` (sql >) -* gte - like `position__gte=3` (sql >=) -* lt - like `position__lt=3` (sql <) -* lte - like `position__lte=3` (sql <=) -* startswith - like `album__name__startswith='Mal'` (exact start match) -* istartswith - like `album__name__istartswith='mal'` (exact start match case insensitive) -* endswith - like `album__name__endswith='ibu'` (exact end match) -* iendswith - like `album__name__iendswith='IBU'` (exact end match case insensitive) - -!!!note - All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together - - So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. - - Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` - -### exclude - -`exclude(**kwargs) -> QuerySet` - -Works exactly the same as filter and all modifiers (suffixes) are the same, but returns a not condition. - -So if you use `filter(name='John')` which equals to `where name = 'John'` in SQL, -the `exclude(name='John')` equals to `where name <> 'John'` - -Note that all conditions are joined so if you pass multiple values it becomes a union of conditions. - -`exclude(name='John', age>=35)` will become `where not (name='John' and age>=35)` - -```python -notes = await Track.objects.exclude(position_gt=3).all() -# returns all tracks with position < 3 -``` - -### select_related - -`select_related(related: Union[List, str]) -> QuerySet` - -Allows to prefetch related models during the same query. - -**With `select_related` always only one query is run against the database**, meaning that one -(sometimes complicated) join is generated and later nested models are processed in python. - -To fetch related model use `ForeignKey` names. - -To chain related `Models` relation use double underscores between names. - -!!!note - If you are coming from `django` note that `ormar` `select_related` differs -> in `django` you can `select_related` - only singe relation types, while in `ormar` you can select related across `ForeignKey` relation, - reverse side of `ForeignKey` (so virtual auto generated keys) and `ManyToMany` fields (so all relations as of current version). - -!!!tip - To control which model fields to select use `fields()` and `exclude_fields()` `QuerySet` methods. - -!!!tip - To control order of models (both main or nested) use `order_by()` method. - -```python -album = await Album.objects.select_related("tracks").all() -# will return album will all columns tracks -``` - -You can provide a string or a list of strings - -```python -classes = await SchoolClass.objects.select_related( -["teachers__category", "students"]).all() -# will return classes with teachers and teachers categories -# as well as classes students -``` - -Exactly the same behavior is for Many2Many fields, where you put the names of Many2Many fields and the final `Models` are fetched for you. - -!!!warning - If you set `ForeignKey` field as not nullable (so required) during - all queries the not nullable `Models` will be auto prefetched, even if you do not include them in select_related. - -!!!note - All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together - - So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. - - Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` - -### prefetch_related - -`prefetch_related(related: Union[List, str]) -> QuerySet` - -Allows to prefetch related models during query - but opposite to `select_related` each -subsequent model is fetched in a separate database query. - -**With `prefetch_related` always one query per Model is run against the database**, -meaning that you will have multiple queries executed one after another. - -To fetch related model use `ForeignKey` names. - -To chain related `Models` relation use double underscores between names. - -!!!tip - To control which model fields to select use `fields()` and `exclude_fields()` `QuerySet` methods. - -!!!tip - To control order of models (both main or nested) use `order_by()` method. - -```python -album = await Album.objects.prefetch_related("tracks").all() -# will return album will all columns tracks -``` - -You can provide a string or a list of strings - -```python -classes = await SchoolClass.objects.prefetch_related( -["teachers__category", "students"]).all() -# will return classes with teachers and teachers categories -# as well as classes students -``` - -Exactly the same behavior is for Many2Many fields, where you put the names of Many2Many fields and the final `Models` are fetched for you. - -!!!warning - If you set `ForeignKey` field as not nullable (so required) during - all queries the not nullable `Models` will be auto prefetched, even if you do not include them in select_related. - -!!!note - All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together - - So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. - - Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` - -### select_related vs prefetch_related - -Which should you use -> `select_related` or `prefetch_related`? - -Well, it really depends on your data. The best answer is try yourself and see which one performs faster/better in your system constraints. - -What to keep in mind: - -#### Performance - -**Number of queries**: -`select_related` always executes one query against the database, while `prefetch_related` executes multiple queries. -Usually the query (I/O) operation is the slowest one but it does not have to be. - -**Number of rows**: -Imagine that you have 10 000 object in one table A and each of those objects have 3 children in table B, -and subsequently each object in table B has 2 children in table C. Something like this: - -``` - Model C - / - Model B - Model C - / -Model A - Model B - Model C - \ \ - \ Model C - \ - Model B - Model C - \ - Model C -``` - -That means that `select_related` will always return 60 000 rows (10 000 * 3 * 2) later compacted to 10 000 models. - -How many rows will return `prefetch_related`? - -Well, that depends, if each of models B and C is unique it will return 10 000 rows in first query, 30 000 rows -(each of 3 children of A in table B are unique) in second query and 60 000 rows (each of 2 children of model B -in table C are unique) in 3rd query. - -In this case `select_related` seems like a better choice, not only it will run one query comparing to 3 of -`prefetch_related` but will also return 60 000 rows comparing to 100 000 of `prefetch_related` (10+30+60k). - -But what if each Model A has exactly the same 3 models B and each models C has exactly same models C? `select_related` -will still return 60 000 rows, while `prefetch_related` will return 10 000 for model A, 3 rows for model B and 2 rows for Model C. -So in total 10 006 rows. Now depending on the structure of models (i.e. if it has long Text() fields etc.) `prefetch_related` -might be faster despite it needs to perform three separate queries instead of one. - -#### Memory - -`ormar` is a mini ORM meaning that it does not keep a registry of already loaded models. - -That means that in `select_related` example above you will always have 10 000 Models A, 30 000 Models B -(even if the unique number of rows in db is 3 - processing of `select_related` spawns **new** child models for each parent model). -And 60 000 Models C. - -If the same Model B is shared by rows 1, 10, 100 etc. and you update one of those, the rest of rows -that share the same child will **not** be updated on the spot. -If you persist your changes into the database the change **will be available only after reload -(either each child separately or the whole query again)**. -That means that `select_related` will use more memory as each child is instantiated as a new object - obviously using it's own space. - -!!!note - This might change in future versions if we decide to introduce caching. - -!!!warning - By default all children (or event the same models loaded 2+ times) are completely independent, distinct python objects, despite that they represent the same row in db. - - They will evaluate to True when compared, so in example above: - - ```python - # will return True if child1 of both rows is the same child db row - row1.child1 == row100.child1 - - # same here: - model1 = await Model.get(pk=1) - model2 = await Model.get(pk=1) # same pk = same row in db - # will return `True` - model1 == model2 - ``` - - but - - ```python - # will return False (note that id is a python `builtin` function not ormar one). - id(row1.child1) == (ro100.child1) - - # from above - will also return False - id(model1) == id(model2) - ``` - - -On the contrary - with `prefetch_related` each unique distinct child model is instantiated -only once and the same child models is shared across all parent models. -That means that in `prefetch_related` example above if there are 3 distinct models in table B and 2 in table C, -there will be only 5 children nested models shared between all model A instances. That also means that if you update -any attribute it will be updated on all parents as they share the same child object. - -### limit - -`limit(limit_count: int, limit_raw_sql: bool = None) -> QuerySet` - -You can limit the results to desired number of parent models. - -To limit the actual number of database query rows instead of number of main models -use the `limit_raw_sql` parameter flag, and set it to `True`. - -```python -tracks = await Track.objects.limit(1).all() -# will return just one Track -``` - -!!!note - All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together - - So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. - - Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` - -### offset - -`offset(offset: int, limit_raw_sql: bool = None) -> QuerySet` - -You can also offset the results by desired number of main models. - -To offset the actual number of database query rows instead of number of main models -use the `limit_raw_sql` parameter flag, and set it to `True`. - -```python -tracks = await Track.objects.offset(1).limit(1).all() -# will return just one Track, but this time the second one -``` - -!!!note - All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together - - So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. - - Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` - - -### count - -`count() -> int` - -Returns number of rows matching the given criteria (applied with `filter` and `exclude`) - -```python -# returns count of rows in db -no_of_books = await Book.objects.count() -``` - -### exists - -`exists() -> bool` - -Returns a bool value to confirm if there are rows matching the given criteria (applied with `filter` and `exclude`) - -```python -# returns a boolean value if given row exists -has_sample = await Book.objects.filter(title='Sample').exists() -``` - -### fields - -`fields(columns: Union[List, str, set, dict]) -> QuerySet` - -With `fields()` you can select subset of model columns to limit the data load. - -!!!note - Note that `fields()` and `exclude_fields()` works both for main models (on normal queries like `get`, `all` etc.) - as well as `select_related` and `prefetch_related` models (with nested notation). - -Given a sample data like following: - -```python ---8<-- "../docs_src/queries/docs006.py" -``` - -You can select specified fields by passing a `str, List[str], Set[str] or dict` with nested definition. - -To include related models use notation `{related_name}__{column}[__{optional_next} etc.]`. - -```python hl_lines="1" -all_cars = await Car.objects.select_related('manufacturer').fields(['id', 'name', 'manufacturer__name']).all() -for car in all_cars: - # excluded columns will yield None - assert all(getattr(car, x) is None for x in ['year', 'gearbox_type', 'gears', 'aircon_type']) - # included column on related models will be available, pk column is always included - # even if you do not include it in fields list - assert car.manufacturer.name == 'Toyota' - # also in the nested related models - you cannot exclude pk - it's always auto added - assert car.manufacturer.founded is None -``` - -`fields()` can be called several times, building up the columns to select. - -If you include related models into `select_related()` call but you won't specify columns for those models in fields -- implies a list of all fields for those nested models. - -```python hl_lines="1" -all_cars = await Car.objects.select_related('manufacturer').fields('id').fields( - ['name']).all() -# all fiels from company model are selected -assert all_cars[0].manufacturer.name == 'Toyota' -assert all_cars[0].manufacturer.founded == 1937 -``` - -!!!warning - Mandatory fields cannot be excluded as it will raise `ValidationError`, to exclude a field it has to be nullable. - -You cannot exclude mandatory model columns - `manufacturer__name` in this example. - -```python -await Car.objects.select_related('manufacturer').fields(['id', 'name', 'manufacturer__founded']).all() -# will raise pydantic ValidationError as company.name is required -``` - -!!!tip - Pk column cannot be excluded - it's always auto added even if not explicitly included. - -You can also pass fields to include as dictionary or set. - -To mark a field as included in a dictionary use it's name as key and ellipsis as value. - -To traverse nested models use nested dictionaries. - -To include fields at last level instead of nested dictionary a set can be used. - -To include whole nested model specify model related field name and ellipsis. - -Below you can see examples that are equivalent: - -```python ---8<-- "../docs_src/queries/docs009.py" -``` - -!!!note - All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together - - So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. - - Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` - -### exclude_fields - -`exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet` - -With `exclude_fields()` you can select subset of model columns that will be excluded to limit the data load. - -It's the opposite of `fields()` method so check documentation above to see what options are available. - -Especially check above how you can pass also nested dictionaries and sets as a mask to exclude fields from whole hierarchy. - -!!!note - Note that `fields()` and `exclude_fields()` works both for main models (on normal queries like `get`, `all` etc.) - as well as `select_related` and `prefetch_related` models (with nested notation). - -Below you can find few simple examples: - -```python hl_lines="47 48 60 61 67" ---8<-- "../docs_src/queries/docs008.py" -``` - -!!!warning - Mandatory fields cannot be excluded as it will raise `ValidationError`, to exclude a field it has to be nullable. - -!!!tip - Pk column cannot be excluded - it's always auto added even if explicitly excluded. - - -!!!note - All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together - - So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. - - Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` - - -### order_by - -`order_by(columns: Union[List, str]) -> QuerySet` - -With `order_by()` you can order the results from database based on your choice of fields. - -You can provide a string with field name or list of strings with different fields. - -Ordering in sql will be applied in order of names you provide in order_by. - -!!!tip - By default if you do not provide ordering `ormar` explicitly orders by all primary keys - -!!!warning - If you are sorting by nested models that causes that the result rows are unsorted by the main model - `ormar` will combine those children rows into one main model. - - Sample raw database rows result (sort by child model desc): - ``` - MODEL: 1 - Child Model - 3 - MODEL: 2 - Child Model - 2 - MODEL: 1 - Child Model - 1 - ``` - - will result in 2 rows of result: - ``` - MODEL: 1 - Child Models: [3, 1] # encountered first in result, all children rows combined - MODEL: 2 - Child Modles: [2] - ``` - - The main model will never duplicate in the result - -Given sample Models like following: - -```python ---8<-- "../docs_src/queries/docs007.py" -``` - -To order by main model field just provide a field name - -```python -toys = await Toy.objects.select_related("owner").order_by("name").all() -assert [x.name.replace("Toy ", "") for x in toys] == [ - str(x + 1) for x in range(6) -] -assert toys[0].owner == zeus -assert toys[1].owner == aphrodite -``` - -To sort on nested models separate field names with dunder '__'. - -You can sort this way across all relation types -> `ForeignKey`, reverse virtual FK and `ManyToMany` fields. - -```python -toys = await Toy.objects.select_related("owner").order_by("owner__name").all() -assert toys[0].owner.name == toys[1].owner.name == "Aphrodite" -assert toys[2].owner.name == toys[3].owner.name == "Hermes" -assert toys[4].owner.name == toys[5].owner.name == "Zeus" -``` - -To sort in descending order provide a hyphen in front of the field name - -```python -owner = ( - await Owner.objects.select_related("toys") - .order_by("-toys__name") - .filter(name="Zeus") - .get() -) -assert owner.toys[0].name == "Toy 4" -assert owner.toys[1].name == "Toy 1" -``` - -!!!note - All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together - - So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. - - Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` - - -[models]: ./models/index.md -[relations]: ./relations/index.md \ No newline at end of file diff --git a/docs/queries/aggregations.md b/docs/queries/aggregations.md new file mode 100644 index 0000000..9732314 --- /dev/null +++ b/docs/queries/aggregations.md @@ -0,0 +1,28 @@ +# Aggregation functions + +`ormar` currently supports 2 aggregation functions: + +* `count() -> int` +* `exists() -> bool` + +## count + +`count() -> int` + +Returns number of rows matching the given criteria (i.e. applied with `filter` and `exclude`) + +```python +# returns count of rows in db for Books model +no_of_books = await Book.objects.count() +``` + +## exists + +`exists() -> bool` + +Returns a bool value to confirm if there are rows matching the given criteria (applied with `filter` and `exclude`) + +```python +# returns a boolean value if given row exists +has_sample = await Book.objects.filter(title='Sample').exists() +``` diff --git a/docs/queries/create.md b/docs/queries/create.md new file mode 100644 index 0000000..ca85137 --- /dev/null +++ b/docs/queries/create.md @@ -0,0 +1,83 @@ +# Create / Insert data into database + +* `create(**kwargs): -> Model` +* `get_or_create(**kwargs) -> Model` +* `update_or_create(**kwargs) -> Model` +* `bulk_create(objects: List[Model]) -> None` +* `Model.save()` method +* `Model.upsert()` method + +## create + +`create(**kwargs): -> Model` + +Creates the model instance, saves it in a database and returns the updates model +(with pk populated if not passed and autoincrement is set). + +The allowed kwargs are `Model` fields names and proper value types. + +```python +malibu = await Album.objects.create(name="Malibu") +await Track.objects.create(album=malibu, title="The Bird", position=1) +``` + +The alternative is a split creation and persistence of the `Model`. + +```python +malibu = Album(name="Malibu") +await malibu.save() +``` + +!!!tip Check other `Model` methods in [models][models] + +## get_or_create + +`get_or_create(**kwargs) -> Model` + +Combination of create and get methods. + +Tries to get a row meeting the criteria and if `NoMatch` exception is raised it creates +a new one with given kwargs. + +```python +album = await Album.objects.get_or_create(name='The Cat') +# object is created as it does not exist +album2 = await Album.objects.get_or_create(name='The Cat') +assert album == album2 +# return True as the same db row is returned +``` + +!!!warning Despite being a equivalent row from database the `album` and `album2` in +example above are 2 different python objects! +Updating one of them will not refresh the second one until you excplicitly load() the +fresh data from db. + +!!!note Note that if you want to create a new object you either have to pass pk column +value or pk column has to be set as autoincrement + +## update_or_create + +`update_or_create(**kwargs) -> Model` + +Updates the model, or in case there is no match in database creates a new one. + +```Python hl_lines="26-32" +--8<-- "../docs_src/queries/docs003.py" +``` + +!!!note Note that if you want to create a new object you either have to pass pk column +value or pk column has to be set as autoincrement + +## bulk_create + +`bulk_create(objects: List["Model"]) -> None` + +Allows you to create multiple objects at once. + +A valid list of `Model` objects needs to be passed. + +```python hl_lines="21-27" +--8<-- "../docs_src/queries/docs004.py" +``` + +## Model method \ No newline at end of file diff --git a/docs/queries/delete.md b/docs/queries/delete.md new file mode 100644 index 0000000..8d122a9 --- /dev/null +++ b/docs/queries/delete.md @@ -0,0 +1,23 @@ +# Delete/ remove data from database + +* `delete(each: bool = False, **kwargs) -> int` +* `Model.delete()` method + +## delete + +`delete(each: bool = False, **kwargs) -> int` + +QuerySet level delete is used to delete multiple records at once. + +You either have to filter the QuerySet first or provide a `each=True` flag to delete +whole table. + +If you do not provide this flag or a filter a `QueryDefinitionError` will be raised. + +Return number of rows deleted. + +```python hl_lines="26-30" +--8<-- "../docs_src/queries/docs005.py" +``` + +## Model method \ No newline at end of file diff --git a/docs/queries/filter-and-sort.md b/docs/queries/filter-and-sort.md new file mode 100644 index 0000000..2b93f9d --- /dev/null +++ b/docs/queries/filter-and-sort.md @@ -0,0 +1,151 @@ +# Filtering and sorting data + +* `filter(**kwargs) -> QuerySet` +* `exclude(**kwargs) -> QuerySet` +* `order_by(columns:Union[List, str]) -> QuerySet` + +## filter + +`filter(**kwargs) -> QuerySet` + +Allows you to filter by any `Model` attribute/field as well as to fetch instances, with +a filter across an FK relationship. + +```python +track = Track.objects.filter(name="The Bird").get() +# will return a track with name equal to 'The Bird' + +tracks = Track.objects.filter(album__name="Fantasies").all() +# will return all tracks where the columns album name = 'Fantasies' +``` + +You can use special filter suffix to change the filter operands: + +* exact - like `album__name__exact='Malibu'` (exact match) +* iexact - like `album__name__iexact='malibu'` (exact match case insensitive) +* contains - like `album__name__contains='Mal'` (sql like) +* icontains - like `album__name__icontains='mal'` (sql like case insensitive) +* in - like `album__name__in=['Malibu', 'Barclay']` (sql in) +* gt - like `position__gt=3` (sql >) +* gte - like `position__gte=3` (sql >=) +* lt - like `position__lt=3` (sql <) +* lte - like `position__lte=3` (sql <=) +* startswith - like `album__name__startswith='Mal'` (exact start match) +* istartswith - like `album__name__istartswith='mal'` (exact start match case + insensitive) +* endswith - like `album__name__endswith='ibu'` (exact end match) +* iendswith - like `album__name__iendswith='IBU'` (exact end match case insensitive) + +!!!note All methods that do not return the rows explicitly returns a QueySet instance so +you can chain them together + + So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. + + Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` + +!!!warning Note that you do not have to specify the `%` wildcard in contains and other +filters, it's added for you. If you include `%` in your search value it will be escaped +and treated as literal percentage sign inside the text. + +### exclude + +`exclude(**kwargs) -> QuerySet` + +Works exactly the same as filter and all modifiers (suffixes) are the same, but returns +a not condition. + +So if you use `filter(name='John')` which equals to `where name = 'John'` in SQL, +the `exclude(name='John')` equals to `where name <> 'John'` + +Note that all conditions are joined so if you pass multiple values it becomes a union of +conditions. + +`exclude(name='John', age>=35)` will become `where not (name='John' and age>=35)` + +```python +notes = await Track.objects.exclude(position_gt=3).all() +# returns all tracks with position < 3 +``` + +### order_by + +`order_by(columns: Union[List, str]) -> QuerySet` + +With `order_by()` you can order the results from database based on your choice of +fields. + +You can provide a string with field name or list of strings with different fields. + +Ordering in sql will be applied in order of names you provide in order_by. + +!!!tip By default if you do not provide ordering `ormar` explicitly orders by all +primary keys + +!!!warning If you are sorting by nested models that causes that the result rows are +unsorted by the main model +`ormar` will combine those children rows into one main model. + + Sample raw database rows result (sort by child model desc): + ``` + MODEL: 1 - Child Model - 3 + MODEL: 2 - Child Model - 2 + MODEL: 1 - Child Model - 1 + ``` + + will result in 2 rows of result: + ``` + MODEL: 1 - Child Models: [3, 1] # encountered first in result, all children rows combined + MODEL: 2 - Child Modles: [2] + ``` + + The main model will never duplicate in the result + +Given sample Models like following: + +```python +--8 < -- "../docs_src/queries/docs007.py" +``` + +To order by main model field just provide a field name + +```python +toys = await Toy.objects.select_related("owner").order_by("name").all() +assert [x.name.replace("Toy ", "") for x in toys] == [ + str(x + 1) for x in range(6) +] +assert toys[0].owner == zeus +assert toys[1].owner == aphrodite +``` + +To sort on nested models separate field names with dunder '__'. + +You can sort this way across all relation types -> `ForeignKey`, reverse virtual FK +and `ManyToMany` fields. + +```python +toys = await Toy.objects.select_related("owner").order_by("owner__name").all() +assert toys[0].owner.name == toys[1].owner.name == "Aphrodite" +assert toys[2].owner.name == toys[3].owner.name == "Hermes" +assert toys[4].owner.name == toys[5].owner.name == "Zeus" +``` + +To sort in descending order provide a hyphen in front of the field name + +```python +owner = ( + await Owner.objects.select_related("toys") + .order_by("-toys__name") + .filter(name="Zeus") + .get() +) +assert owner.toys[0].name == "Toy 4" +assert owner.toys[1].name == "Toy 1" +``` + +!!!note All methods that do not return the rows explicitly returns a QueySet instance so +you can chain them together + + So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. + + Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` + diff --git a/docs/queries/index.md b/docs/queries/index.md new file mode 100644 index 0000000..fa00eec --- /dev/null +++ b/docs/queries/index.md @@ -0,0 +1,156 @@ +# Querying database with ormar + +## QuerySet + +Each Model is auto registered with a `QuerySet` that represents the underlying query, +and it's options. + +Most of the methods are also available through many to many relations and on reverse +foreign key relations through `QuerysetProxy` interface. + +!!!info To see which one are supported and how to construct relations +visit [relations][relations]. + +For simplicity available methods to fetch and save the data into the database are +divided into categories according to the function they fulfill. + +Note that some functions/methods are in multiple categories. + +For complicity also Models and relations methods are listed. + +To read more about any specific section or function please refer to the details subpage. + +### Create + +* `create(**kwargs) -> Model` +* `get_or_create(**kwargs) -> Model` +* `update_or_create(**kwargs) -> Model` +* `bulk_create(objects: List[Model]) -> None` + + +* `Model` + * `Model.save()` method + * `Model.upsert()` method + * `Model.save_related()` method + + +* `QuerysetProxy` + * `QuerysetProxy.create(**kwargs)` method + * `QuerysetProxy.get_or_create(**kwargs)` method + * `QuerysetProxy.update_or_create(**kwargs)` method + +### Read + +* `get(**kwargs) -> Model` +* `get_or_create(**kwargs) -> Model` +* `first() -> Model` +* `all(**kwargs) -> List[Optional[Model]]` + + +* `Model` + * `Model.load()` method + + +* `QuerysetProxy` + * `QuerysetProxy.get(**kwargs)` method + * `QuerysetProxy.get_or_create(**kwargs)` method + * `QuerysetProxy.first()` method + * `QuerysetProxy.all(**kwargs)` method + +### Update + +* `update(each: bool = False, **kwargs) -> int` +* `update_or_create(**kwargs) -> Model` +* `bulk_update(objects: List[Model], columns: List[str] = None) -> None` + + +* `Model` + * `Model.update()` method + * `Model.upsert()` method + * `Model.save_related()` method + + +* `QuerysetProxy` + * `QuerysetProxy.update_or_create(**kwargs)` method + +### Delete + +* `delete(each: bool = False, **kwargs) -> int` + + +* `Model` + * `Model.delete()` method + + +* `QuerysetProxy` + * `QuerysetProxy.remove()` method + * `QuerysetProxy.clear()` method + +### Joins and subqueries + +* `select_related(related: Union[List, str]) -> QuerySet` +* `prefetch_related(related: Union[List, str]) -> QuerySet` + + +* `Model` + * `Model.load()` method + + +* `QuerysetProxy` + * `QuerysetProxy.select_related(related: Union[List, str])` method + * `QuerysetProxy.prefetch_related(related: Union[List, str])` method + +### Filtering and sorting + +* `filter(**kwargs) -> QuerySet` +* `exclude(**kwargs) -> QuerySet` +* `order_by(columns:Union[List, str]) -> QuerySet` +* `get(**kwargs) -> Model` +* `get_or_create(**kwargs) -> Model` +* `all(**kwargs) -> List[Optional[Model]]` + + +* `QuerysetProxy` + * `QuerysetProxy.filter(**kwargs)` method + * `QuerysetProxy.exclude(**kwargs)` method + * `QuerysetProxy.order_by(columns:Union[List, str])` method + * `QuerysetProxy.get(**kwargs)` method + * `QuerysetProxy.get_or_create(**kwargs)` method + * `QuerysetProxy.all(**kwargs)` method + +### Selecting columns + +* `fields(columns: Union[List, str, set, dict]) -> QuerySet` +* `exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet` + + +* `QuerysetProxy` + * `QuerysetProxy.fields(columns: Union[List, str, set, dict])` method + * `QuerysetProxy.exclude_fields(columns: Union[List, str, set, dict])` method + +### Pagination and rows number + +* `paginate(page: int) -> QuerySet` +* `limit(limit_count: int) -> QuerySet` +* `offset(offset: int) -> QuerySet` +* `get() -> Model` +* `first() -> Model` + + +* `QuerysetProxy` + * `QuerysetProxy.paginate(page: int)` method + * `QuerysetProxy.limit(limit_count: int)` method + * `QuerysetProxy.offset(offset: int)` method + +### Aggregated functions + +* `count() -> int` +* `exists() -> bool` + + +* `QuerysetProxy` + * `QuerysetProxy.count()` method + * `QuerysetProxy.exists()` method + + +[relations]: ./relations/index.md \ No newline at end of file diff --git a/docs/queries/joins-and-subqueries.md b/docs/queries/joins-and-subqueries.md new file mode 100644 index 0000000..a2635a7 --- /dev/null +++ b/docs/queries/joins-and-subqueries.md @@ -0,0 +1,223 @@ +# Joins and subqueries + + + +## select_related + +`select_related(related: Union[List, str]) -> QuerySet` + +Allows to prefetch related models during the same query. + +**With `select_related` always only one query is run against the database**, meaning +that one (sometimes complicated) join is generated and later nested models are processed in +python. + +To fetch related model use `ForeignKey` names. + +To chain related `Models` relation use double underscores between names. + +!!!note + If you are coming from `django` note that `ormar` `select_related` differs -> + in `django` you can `select_related` + only singe relation types, while in `ormar` you can select related across `ForeignKey` + relation, reverse side of `ForeignKey` (so virtual auto generated keys) and `ManyToMany` + fields (so all relations as of current version). + +!!!tip + To control which model fields to select use `fields()` + and `exclude_fields()` `QuerySet` methods. + +!!!tip + To control order of models (both main or nested) use `order_by()` method. + +```python +album = await Album.objects.select_related("tracks").all() +# will return album will all columns tracks +``` + +You can provide a string or a list of strings + +```python +classes = await SchoolClass.objects.select_related( + ["teachers__category", "students"]).all() +# will return classes with teachers and teachers categories +# as well as classes students +``` + +Exactly the same behavior is for Many2Many fields, where you put the names of Many2Many +fields and the final `Models` are fetched for you. + +!!!warning + If you set `ForeignKey` field as not nullable (so required) during all + queries the not nullable `Models` will be auto prefetched, even if you do not include + them in select_related. + +!!!note + All methods that do not return the rows explicitly returns a QueySet instance so + you can chain them together + + So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. + + Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` + +## prefetch_related + +`prefetch_related(related: Union[List, str]) -> QuerySet` + +Allows to prefetch related models during query - but opposite to `select_related` each +subsequent model is fetched in a separate database query. + +**With `prefetch_related` always one query per Model is run against the database**, +meaning that you will have multiple queries executed one after another. + +To fetch related model use `ForeignKey` names. + +To chain related `Models` relation use double underscores between names. + +!!!tip + To control which model fields to select use `fields()` + and `exclude_fields()` `QuerySet` methods. + +!!!tip + To control order of models (both main or nested) use `order_by()` method. + +```python +album = await Album.objects.prefetch_related("tracks").all() +# will return album will all columns tracks +``` + +You can provide a string or a list of strings + +```python +classes = await SchoolClass.objects.prefetch_related( + ["teachers__category", "students"]).all() +# will return classes with teachers and teachers categories +# as well as classes students +``` + +Exactly the same behavior is for Many2Many fields, where you put the names of Many2Many +fields and the final `Models` are fetched for you. + +!!!warning + If you set `ForeignKey` field as not nullable (so required) during all + queries the not nullable `Models` will be auto prefetched, even if you do not include + them in select_related. + +!!!note + All methods that do not return the rows explicitly returns a QueySet instance so + you can chain them together + + So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. + + Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` + +## select_related vs prefetch_related + +Which should you use -> `select_related` or `prefetch_related`? + +Well, it really depends on your data. The best answer is try yourself and see which one +performs faster/better in your system constraints. + +What to keep in mind: + +### Performance + +**Number of queries**: +`select_related` always executes one query against the database, +while `prefetch_related` executes multiple queries. Usually the query (I/O) operation is +the slowest one but it does not have to be. + +**Number of rows**: +Imagine that you have 10 000 object in one table A and each of those objects have 3 +children in table B, and subsequently each object in table B has 2 children in table C. +Something like this: + +``` + Model C + / + Model B - Model C + / +Model A - Model B - Model C + \ \ + \ Model C + \ + Model B - Model C + \ + Model C +``` + +That means that `select_related` will always return 60 000 rows (10 000 * 3 * 2) later +compacted to 10 000 models. + +How many rows will return `prefetch_related`? + +Well, that depends, if each of models B and C is unique it will return 10 000 rows in +first query, 30 000 rows +(each of 3 children of A in table B are unique) in second query and 60 000 rows (each of +2 children of model B in table C are unique) in 3rd query. + +In this case `select_related` seems like a better choice, not only it will run one query +comparing to 3 of +`prefetch_related` but will also return 60 000 rows comparing to 100 000 +of `prefetch_related` (10+30+60k). + +But what if each Model A has exactly the same 3 models B and each models C has exactly +same models C? `select_related` +will still return 60 000 rows, while `prefetch_related` will return 10 000 for model A, +3 rows for model B and 2 rows for Model C. So in total 10 006 rows. Now depending on the +structure of models (i.e. if it has long Text() fields etc.) `prefetch_related` +might be faster despite it needs to perform three separate queries instead of one. + +#### Memory + +`ormar` is a mini ORM meaning that it does not keep a registry of already loaded models. + +That means that in `select_related` example above you will always have 10 000 Models A, +30 000 Models B +(even if the unique number of rows in db is 3 - processing of `select_related` spawns ** +new** child models for each parent model). And 60 000 Models C. + +If the same Model B is shared by rows 1, 10, 100 etc. and you update one of those, the +rest of rows that share the same child will **not** be updated on the spot. If you +persist your changes into the database the change **will be available only after reload +(either each child separately or the whole query again)**. That means +that `select_related` will use more memory as each child is instantiated as a new object +- obviously using it's own space. + +!!!note + This might change in future versions if we decide to introduce caching. + +!!!warning + By default all children (or event the same models loaded 2+ times) are + completely independent, distinct python objects, despite that they represent the same + row in db. + + They will evaluate to True when compared, so in example above: + + ```python + # will return True if child1 of both rows is the same child db row + row1.child1 == row100.child1 + + # same here: + model1 = await Model.get(pk=1) + model2 = await Model.get(pk=1) # same pk = same row in db + # will return `True` + model1 == model2 + ``` + + but + + ```python + # will return False (note that id is a python `builtin` function not ormar one). + id(row1.child1) == (ro100.child1) + + # from above - will also return False + id(model1) == id(model2) + ``` + +On the contrary - with `prefetch_related` each unique distinct child model is +instantiated only once and the same child models is shared across all parent models. +That means that in `prefetch_related` example above if there are 3 distinct models in +table B and 2 in table C, there will be only 5 children nested models shared between all +model A instances. That also means that if you update any attribute it will be updated +on all parents as they share the same child object. diff --git a/docs/queries/pagination-and-rows-number.md b/docs/queries/pagination-and-rows-number.md new file mode 100644 index 0000000..888eaf4 --- /dev/null +++ b/docs/queries/pagination-and-rows-number.md @@ -0,0 +1,94 @@ +#Pagination and rows number + +* `paginate(page: int) -> QuerySet` +* `limit(limit_count: int) -> QuerySet` +* `offset(offset: int) -> QuerySet` +* `get(**kwargs): -> Model` +* `first(): -> Model` + + +## paginate + +`paginate(page: int, page_size: int = 20) -> QuerySet` + +Combines the `offset` and `limit` methods based on page number and size + +```python +tracks = await Track.objects.paginate(3).all() +# will return 20 tracks starting at row 41 +# (with default page size of 20) +``` + +Note that `paginate(2)` is equivalent to `offset(20).limit(20)` + +## limit + +`limit(limit_count: int, limit_raw_sql: bool = None) -> QuerySet` + +You can limit the results to desired number of parent models. + +To limit the actual number of database query rows instead of number of main models +use the `limit_raw_sql` parameter flag, and set it to `True`. + +```python +tracks = await Track.objects.limit(1).all() +# will return just one Track +``` + +!!!note + All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together + + So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. + + Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` + +## offset + +`offset(offset: int, limit_raw_sql: bool = None) -> QuerySet` + +You can also offset the results by desired number of main models. + +To offset the actual number of database query rows instead of number of main models +use the `limit_raw_sql` parameter flag, and set it to `True`. + +```python +tracks = await Track.objects.offset(1).limit(1).all() +# will return just one Track, but this time the second one +``` + +!!!note + All methods that do not return the rows explicitly returns a QueySet instance so you can chain them together + + So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. + + Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` + + + +## get + +`get(**kwargs): -> Model` + +Get's the first row from the db meeting the criteria set by kwargs. + +If no criteria set it will return the last row in db sorted by pk. + +Passing a criteria is actually calling filter(**kwargs) method described below. + +```python +track = await Track.objects.get(name='The Bird') +# note that above is equivalent to await Track.objects.filter(name='The Bird').get() +track2 = track = await Track.objects.get() +track == track2 # True since it's the only row in db in our example +``` + +!!!warning + If no row meets the criteria `NoMatch` exception is raised. + + If there are multiple rows meeting the criteria the `MultipleMatches` exception is raised. + +## first + +`first(): -> Model` + +Gets the first row from the db ordered by primary key column ascending. diff --git a/docs/queries/read.md b/docs/queries/read.md new file mode 100644 index 0000000..b761537 --- /dev/null +++ b/docs/queries/read.md @@ -0,0 +1,80 @@ +# Read/ Load data from database + +* `get(**kwargs): -> Model` +* `get_or_create(**kwargs) -> Model` +* `first(): -> Model` +* `all(**kwargs) -> List[Optional[Model]]` +* `Model.load() method` + +## get + +`get(**kwargs): -> Model` + +Get's the first row from the db meeting the criteria set by kwargs. + +If no criteria set it will return the last row in db sorted by pk. + +Passing a criteria is actually calling filter(**kwargs) method described below. + +```python +track = await Track.objects.get(name='The Bird') +# note that above is equivalent to await Track.objects.filter(name='The Bird').get() +track2 = track = await Track.objects.get() +track == track2 # True since it's the only row in db in our example +``` + +!!!warning If no row meets the criteria `NoMatch` exception is raised. + + If there are multiple rows meeting the criteria the `MultipleMatches` exception is raised. + +## get_or_create + +`get_or_create(**kwargs) -> Model` + +Combination of create and get methods. + +Tries to get a row meeting the criteria and if `NoMatch` exception is raised it creates +a new one with given kwargs. + +```python +album = await Album.objects.get_or_create(name='The Cat') +# object is created as it does not exist +album2 = await Album.objects.get_or_create(name='The Cat') +assert album == album2 +# return True as the same db row is returned +``` + +!!!warning Despite being a equivalent row from database the `album` and `album2` in +example above are 2 different python objects! +Updating one of them will not refresh the second one until you excplicitly load() the +fresh data from db. + +!!!note Note that if you want to create a new object you either have to pass pk column +value or pk column has to be set as autoincrement + +## first + +`first(): -> Model` + +Gets the first row from the db ordered by primary key column ascending. + +## all + +`all(**kwargs) -> List[Optional["Model"]]` + +Returns all rows from a database for given model for set filter options. + +Passing kwargs is a shortcut and equals to calling `filter(**kwrags).all()`. + +If there are no rows meeting the criteria an empty list is returned. + +```python +tracks = await Track.objects.select_related("album").all(title='Sample') +# will return a list of all Tracks with title Sample + +tracks = await Track.objects.all() +# will return a list of all Tracks in database + +``` + +## Model method diff --git a/docs/queries/select-columns.md b/docs/queries/select-columns.md new file mode 100644 index 0000000..abb56f5 --- /dev/null +++ b/docs/queries/select-columns.md @@ -0,0 +1,126 @@ +# Selecting subset of columns + +* `fields(columns: Union[List, str, set, dict]) -> QuerySet` +* `exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet` + +## fields + +`fields(columns: Union[List, str, set, dict]) -> QuerySet` + +With `fields()` you can select subset of model columns to limit the data load. + +!!!note Note that `fields()` and `exclude_fields()` works both for main models (on +normal queries like `get`, `all` etc.) +as well as `select_related` and `prefetch_related` models (with nested notation). + +Given a sample data like following: + +```python +--8 < -- "../docs_src/queries/docs006.py" +``` + +You can select specified fields by passing a `str, List[str], Set[str] or dict` with +nested definition. + +To include related models use +notation `{related_name}__{column}[__{optional_next} etc.]`. + +```python hl_lines="1" +all_cars = await Car.objects.select_related('manufacturer').fields(['id', 'name', 'manufacturer__name']).all() +for car in all_cars: + # excluded columns will yield None + assert all(getattr(car, x) is None for x in ['year', 'gearbox_type', 'gears', 'aircon_type']) + # included column on related models will be available, pk column is always included + # even if you do not include it in fields list + assert car.manufacturer.name == 'Toyota' + # also in the nested related models - you cannot exclude pk - it's always auto added + assert car.manufacturer.founded is None +``` + +`fields()` can be called several times, building up the columns to select. + +If you include related models into `select_related()` call but you won't specify columns +for those models in fields + +- implies a list of all fields for those nested models. + +```python hl_lines="1" +all_cars = await Car.objects.select_related('manufacturer').fields('id').fields( + ['name']).all() +# all fiels from company model are selected +assert all_cars[0].manufacturer.name == 'Toyota' +assert all_cars[0].manufacturer.founded == 1937 +``` + +!!!warning Mandatory fields cannot be excluded as it will raise `ValidationError`, to +exclude a field it has to be nullable. + +You cannot exclude mandatory model columns - `manufacturer__name` in this example. + +```python +await Car.objects.select_related('manufacturer').fields( + ['id', 'name', 'manufacturer__founded']).all() +# will raise pydantic ValidationError as company.name is required +``` + +!!!tip Pk column cannot be excluded - it's always auto added even if not explicitly +included. + +You can also pass fields to include as dictionary or set. + +To mark a field as included in a dictionary use it's name as key and ellipsis as value. + +To traverse nested models use nested dictionaries. + +To include fields at last level instead of nested dictionary a set can be used. + +To include whole nested model specify model related field name and ellipsis. + +Below you can see examples that are equivalent: + +```python +--8 < -- "../docs_src/queries/docs009.py" +``` + +!!!note All methods that do not return the rows explicitly returns a QueySet instance so +you can chain them together + + So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. + + Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` + +## exclude_fields + +`exclude_fields(columns: Union[List, str, set, dict]) -> QuerySet` + +With `exclude_fields()` you can select subset of model columns that will be excluded to +limit the data load. + +It's the opposite of `fields()` method so check documentation above to see what options +are available. + +Especially check above how you can pass also nested dictionaries and sets as a mask to +exclude fields from whole hierarchy. + +!!!note Note that `fields()` and `exclude_fields()` works both for main models (on +normal queries like `get`, `all` etc.) +as well as `select_related` and `prefetch_related` models (with nested notation). + +Below you can find few simple examples: + +```python hl_lines="47 48 60 61 67" +--8<-- "../docs_src/queries/docs008.py" +``` + +!!!warning Mandatory fields cannot be excluded as it will raise `ValidationError`, to +exclude a field it has to be nullable. + +!!!tip Pk column cannot be excluded - it's always auto added even if explicitly +excluded. + +!!!note All methods that do not return the rows explicitly returns a QueySet instance so +you can chain them together + + So operations like `filter()`, `select_related()`, `limit()` and `offset()` etc. can be chained. + + Something like `Track.object.select_related("album").filter(album__name="Malibu").offset(1).limit(1).all()` diff --git a/docs/queries/update.md b/docs/queries/update.md new file mode 100644 index 0000000..255b15f --- /dev/null +++ b/docs/queries/update.md @@ -0,0 +1,71 @@ +# Update + +* `update(each: bool = False, **kwargs) -> int` +* `update_or_create(**kwargs) -> Model` +* `bulk_update(objects: List[Model], columns: List[str] = None) -> None` +* `Model.update() method` +* `Model.upsert() method` +* `Model.save_related() method` + +## update + +`update(each: bool = False, **kwargs) -> int` + +QuerySet level update is used to update multiple records with the same value at once. + +You either have to filter the QuerySet first or provide a `each=True` flag to update +whole table. + +If you do not provide this flag or a filter a `QueryDefinitionError` will be raised. + +Return number of rows updated. + +```Python hl_lines="26-28" +--8<-- "../docs_src/queries/docs002.py" +``` + +!!!warning Queryset needs to be filtered before updating to prevent accidental +overwrite. + + To update whole database table `each=True` needs to be provided as a safety switch + +## update_or_create + +`update_or_create(**kwargs) -> Model` + +Updates the model, or in case there is no match in database creates a new one. + +```Python hl_lines="26-32" +--8<-- "../docs_src/queries/docs003.py" +``` + +!!!note Note that if you want to create a new object you either have to pass pk column +value or pk column has to be set as autoincrement + +## bulk_update + +`bulk_update(objects: List["Model"], columns: List[str] = None) -> None` + +Allows to update multiple instance at once. + +All `Models` passed need to have primary key column populated. + +You can also select which fields to update by passing `columns` list as a list of string +names. + +```python hl_lines="8" +# continuing the example from bulk_create +# update objects +for todo in todoes: + todo.completed = False + +# perform update of all objects at once +# objects need to have pk column set, otherwise exception is raised +await ToDo.objects.bulk_update(todoes) + +completed = await ToDo.objects.filter(completed=False).all() +assert len(completed) == 3 +``` + +## Model method + diff --git a/docs/relations/index.md b/docs/relations/index.md index 235eda0..0896c13 100644 --- a/docs/relations/index.md +++ b/docs/relations/index.md @@ -92,7 +92,34 @@ class Post(ormar.Model): It allows you to use `await post.categories.all()` but also `await category.posts.all()` to fetch data related only to specific post, category etc. +##Self-reference and postponed references + +In order to create auto-relation or create two models that reference each other in at least two +different relations (remember the reverse side is auto-registered for you), you need to use +`ForwardRef` from `typing` module. + +```python hl_lines="1 11 14" +PersonRef = ForwardRef("Person") + + +class Person(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + supervisor: PersonRef = ormar.ForeignKey(PersonRef, related_name="employees") + + +Person.update_forward_refs() +``` + +!!!tip + To read more about self-reference and postponed relations visit [postponed-annotations][postponed-annotations] section + [foreign-keys]: ./foreign-key.md [many-to-many]: ./many-to-many.md -[queryset-proxy]: ./queryset-proxy.md \ No newline at end of file +[queryset-proxy]: ./queryset-proxy.md +[postponed-annotations]: ./postponed-annotations.md \ No newline at end of file diff --git a/docs/relations/postponed-annotations.md b/docs/relations/postponed-annotations.md new file mode 100644 index 0000000..e156296 --- /dev/null +++ b/docs/relations/postponed-annotations.md @@ -0,0 +1,171 @@ +# Postponed annotations + +## Self-referencing Models + +When you want to reference the same model during declaration to create a +relation you need to declare the referenced model as a `ForwardRef`, as during the declaration +the class is not yet ready and python by default won't let you reference it. + +Although you might be tempted to use __future__ annotations or simply quote the name with `""` it won't work +as `ormar` is designed to work with explicitly declared `ForwardRef`. + +First, you need to import the required ref from typing. +```python +from typing import ForwardRef +``` + +But note that before python 3.7 it used to be internal, so for python <= 3.6 you need + +```python +from typing import _ForwardRef as ForwardRef +``` + +or since `pydantic` is required by `ormar` it can handle this switch for you. +In that case you can simply import ForwardRef from pydantic regardless of your python version. + +```python +from pydantic.typing import ForwardRef +``` + +Now we need a sample model and a reference to the same model, +which will be used to creat a self referencing relation. + +```python +# create the forwardref to model Person +PersonRef = ForwardRef("Person") + + +class Person(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + # use the forwardref as to parameter + supervisor: PersonRef = ormar.ForeignKey(PersonRef, related_name="employees") + +``` + +That's so simple. But before you can use the model you need to manually update the references +so that they lead to the actual models. + +!!!warning + If you try to use the model without updated references, `ModelError` exception will be raised. + So in our example above any call like following will cause exception + ```python + # creation of model - exception + await Person.objects.create(name="Test") + # initialization of model - exception + Person2(name="Test") + # usage of model's QuerySet - exception + await Person2.objects.get() + ``` + +To update the references call the `update_forward_refs` method on **each model** +with forward references, only **after all related models were declared.** + +So in order to make our previous example work we need just one extra line. + +```python hl_lines="14" +PersonRef = ForwardRef("Person") + + +class Person(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + supervisor: PersonRef = ormar.ForeignKey(PersonRef, related_name="employees") + + +Person.update_forward_refs() + +``` + +Of course the same can be done with ManyToMany relations in exactly same way, both for to +and through parameters. + +```python +# declare the reference +ChildRef = ForwardRef("Child") + +class ChildFriend(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db + +class Child(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + # use it in relation + friends = ormar.ManyToMany(ChildRef, through=ChildFriend, + related_name="also_friends") + + +Child.update_forward_refs() +``` + +## Cross model relations + +The same mechanism and logic as for self-reference model can be used to link multiple different +models between each other. + +Of course `ormar` links both sides of relation for you, +creating a reverse relation with specified (or default) `related_name`. + +But if you need two (or more) relations between any two models, that for whatever reason +should be stored on both sides (so one relation is declared on one model, +and other on the second model), you need to use `ForwardRef` to achieve that. + +Look at the following simple example. + +```python +# teacher is not yet defined +TeacherRef = ForwardRef("Teacher") + + +class Student(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + # so we use reference instead of actual model + primary_teacher: TeacherRef = ormar.ForeignKey(TeacherRef, + related_name="own_students") + + +class StudentTeacher(ormar.Model): + class Meta(ModelMeta): + tablename = 'students_x_teachers' + metadata = metadata + database = db + + +class Teacher(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + # we need students for other relation hence the order + students = ormar.ManyToMany(Student, through=StudentTeacher, + related_name="teachers") + +# now the Teacher model is already defined we can update references +Student.update_forward_refs() + +``` + +!!!warning + Remember that `related_name` needs to be unique across related models regardless + of how many relations are defined. \ No newline at end of file diff --git a/docs/releases.md b/docs/releases.md index ac840dc..3521ac9 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,3 +1,21 @@ +# 0.8.1 + +* Introduce processing of `ForwardRef` in relations. + Now you can create self-referencing models - both `ForeignKey` and `ManyToMany` relations. + `ForwardRef` can be used both for `to` and `through` `Models`. +* Introduce the possibility to perform two **same relation** joins in one query, so to process complex relations like: + ``` + B = X = Y + // + A + \ + C = X = Y <= before you could link from X to Y only once in one query + unless two different relation were used + (two relation fields with different names) + ``` +* Refactoring and performance optimization in queries and joins. +* Update API docs and docs. + # 0.8.0 ## Breaking diff --git a/mkdocs.yml b/mkdocs.yml index a57c320..c6379f1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,10 +14,21 @@ nav: - Fields types: fields/field-types.md - Relations: - relations/index.md + - relations/postponed-annotations.md - relations/foreign-key.md - relations/many-to-many.md - relations/queryset-proxy.md - - Queries: queries.md + - Queries: + - queries/index.md + - queries/create.md + - queries/read.md + - queries/update.md + - queries/delete.md + - queries/joins-and-subqueries.md + - queries/filter-and-sort.md + - queries/select-columns.md + - queries/pagination-and-rows-number.md + - queries/aggregations.md - Signals: signals.md - Use with Fastapi: fastapi.md - Use with mypy: mypy.md diff --git a/ormar/fields/foreign_key.py b/ormar/fields/foreign_key.py index 561bc73..3161a21 100644 --- a/ormar/fields/foreign_key.py +++ b/ormar/fields/foreign_key.py @@ -124,7 +124,7 @@ class ForeignKeyConstraint: def ForeignKey( # noqa CFQ002 - to: Union[Type["Model"]], + to: Union[Type["Model"], "ForwardRef"], *, name: str = None, unique: bool = False, diff --git a/ormar/fields/many_to_many.py b/ormar/fields/many_to_many.py index a596b52..a55c04e 100644 --- a/ormar/fields/many_to_many.py +++ b/ormar/fields/many_to_many.py @@ -36,8 +36,8 @@ def populate_m2m_params_based_on_to_model( def ManyToMany( - to: Type["Model"], - through: Type["Model"], + to: Union[Type["Model"], ForwardRef], + through: Union[Type["Model"], ForwardRef], *, name: str = None, unique: bool = False, @@ -77,7 +77,7 @@ def ManyToMany( column_type = None else: __type__, column_type = populate_m2m_params_based_on_to_model( - to=to, nullable=nullable + to=to, nullable=nullable # type: ignore ) namespace = dict( __type__=__type__, @@ -164,12 +164,20 @@ class ManyToManyField(ForeignKeyField, ormar.QuerySetProtocol, ormar.RelationPro :return: None :rtype: None """ - if cls.to.__class__ == ForwardRef or cls.through.__class__ == ForwardRef: + if cls.to.__class__ == ForwardRef: cls.to = evaluate_forwardref( cls.to, # type: ignore globalns, localns or None, ) + (cls.__type__, cls.column_type,) = populate_m2m_params_based_on_to_model( to=cls.to, nullable=cls.nullable, ) + + if cls.through.__class__ == ForwardRef: + cls.through = evaluate_forwardref( + cls.through, # type: ignore + globalns, + localns or None, + ) diff --git a/ormar/models/model.py b/ormar/models/model.py index c81a14b..4f1b0e3 100644 --- a/ormar/models/model.py +++ b/ormar/models/model.py @@ -110,7 +110,6 @@ class Model(NewBaseModel): previous_model = through_field.through # type: ignore if previous_model and rel_name2: - # TODO finish duplicated nested relation or remove this if current_relation_str and "__" in current_relation_str and source_model: table_prefix = cls.Meta.alias_manager.resolve_relation_alias( from_model=source_model, relation_name=current_relation_str @@ -167,6 +166,10 @@ class Model(NewBaseModel): Recurrently calls from_row method on nested instances and create nested instances. In the end those instances are added to the final model dictionary. + :param source_model: source model from which relation started + :type source_model: Type[Model] + :param current_relation_str: joined related parts into one string + :type current_relation_str: str :param item: dictionary of already populated nested models, otherwise empty dict :type item: Dict :param row: raw result row from the database diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index dff9635..2094be8 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -228,9 +228,6 @@ class QuerySet: :return: filtered QuerySet :rtype: QuerySet """ - # TODO: delay processing of filter clauses or switch to group one - # that keeps all aliases even if duplicated - now initialized too late - # in the join qryclause = QueryClause( model_cls=self.model, select_related=self._select_related, diff --git a/tests/test_forward_cross_refs.py b/tests/test_forward_cross_refs.py new file mode 100644 index 0000000..9b813aa --- /dev/null +++ b/tests/test_forward_cross_refs.py @@ -0,0 +1,114 @@ +# type: ignore + +import databases +import pytest +import sqlalchemy as sa +from pydantic.typing import ForwardRef +from sqlalchemy import create_engine + +import ormar +from ormar import ModelMeta +from tests.settings import DATABASE_URL + +metadata = sa.MetaData() +db = databases.Database(DATABASE_URL) +engine = create_engine(DATABASE_URL) + +TeacherRef = ForwardRef("Teacher") + + +class Student(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + primary_teacher: TeacherRef = ormar.ForeignKey( + TeacherRef, related_name="own_students" + ) + + +class StudentTeacher(ormar.Model): + class Meta(ModelMeta): + tablename = "students_x_teachers" + metadata = metadata + database = db + + +class Teacher(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + students = ormar.ManyToMany( + Student, through=StudentTeacher, related_name="teachers" + ) + + +Student.update_forward_refs() + + +@pytest.fixture(autouse=True, scope="module") +def create_test_database(): + metadata.create_all(engine) + yield + metadata.drop_all(engine) + + +@pytest.mark.asyncio +async def test_double_relations(): + t1 = await Teacher.objects.create(name="Mr. Jones") + t2 = await Teacher.objects.create(name="Ms. Smith") + t3 = await Teacher.objects.create(name="Mr. Quibble") + + s1 = await Student.objects.create(name="Joe", primary_teacher=t1) + s2 = await Student.objects.create(name="Sam", primary_teacher=t1) + s3 = await Student.objects.create(name="Kate", primary_teacher=t2) + s4 = await Student.objects.create(name="Zoe", primary_teacher=t2) + s5 = await Student.objects.create(name="John", primary_teacher=t3) + s6 = await Student.objects.create(name="Anna", primary_teacher=t3) + + for t in [t1, t2, t3]: + for s in [s1, s2, s3, s4, s5, s6]: + await t.students.add(s) + + jones = ( + await Teacher.objects.select_related(["students", "own_students"]) + .order_by(["students__name", "own_students__name"]) + .get(name="Mr. Jones") + ) + assert len(jones.students) == 6 + assert jones.students[0].name == "Anna" + assert jones.students[5].name == "Zoe" + assert len(jones.own_students) == 2 + assert jones.own_students[0].name == "Joe" + assert jones.own_students[1].name == "Sam" + + smith = ( + await Teacher.objects.select_related(["students", "own_students"]) + .filter(students__name__contains="a") + .order_by(["students__name", "own_students__name"]) + .get(name="Ms. Smith") + ) + assert len(smith.students) == 3 + assert smith.students[0].name == "Anna" + assert smith.students[2].name == "Sam" + assert len(smith.own_students) == 2 + assert smith.own_students[0].name == "Kate" + assert smith.own_students[1].name == "Zoe" + + quibble = ( + await Teacher.objects.select_related(["students", "own_students"]) + .filter(students__name__startswith="J") + .order_by(["-students__name", "own_students__name"]) + .get(name="Mr. Quibble") + ) + assert len(quibble.students) == 2 + assert quibble.students[1].name == "Joe" + assert quibble.students[0].name == "John" + assert len(quibble.own_students) == 2 + assert quibble.own_students[1].name == "John" + assert quibble.own_students[0].name == "Anna" diff --git a/tests/test_forward_refs.py b/tests/test_forward_refs.py index 60505b6..98393d5 100644 --- a/tests/test_forward_refs.py +++ b/tests/test_forward_refs.py @@ -1,4 +1,6 @@ # type: ignore +from typing import List + import databases import pytest import sqlalchemy @@ -15,7 +17,7 @@ metadata = sa.MetaData() db = databases.Database(DATABASE_URL) engine = create_engine(DATABASE_URL) -Person = ForwardRef("Person") +PersonRef = ForwardRef("Person") class Person(ormar.Model): @@ -25,19 +27,14 @@ class Person(ormar.Model): id: int = ormar.Integer(primary_key=True) name: str = ormar.String(max_length=100) - supervisor: Person = ormar.ForeignKey(Person, related_name="employees") + supervisor: PersonRef = ormar.ForeignKey(PersonRef, related_name="employees") Person.update_forward_refs() -Game = ForwardRef("Game") -Child = ForwardRef("Child") - - -class ChildFriend(ormar.Model): - class Meta(ModelMeta): - metadata = metadata - database = db +GameRef = ForwardRef("Game") +ChildRef = ForwardRef("Child") +ChildFriendRef = ForwardRef("ChildFriend") class Child(ormar.Model): @@ -47,9 +44,19 @@ class Child(ormar.Model): id: int = ormar.Integer(primary_key=True) name: str = ormar.String(max_length=100) - favourite_game: Game = ormar.ForeignKey(Game, related_name="liked_by") - least_favourite_game: Game = ormar.ForeignKey(Game, related_name="not_liked_by") - friends = ormar.ManyToMany(Child, through=ChildFriend, related_name="also_friends") + favourite_game: GameRef = ormar.ForeignKey(GameRef, related_name="liked_by") + least_favourite_game: GameRef = ormar.ForeignKey( + GameRef, related_name="not_liked_by" + ) + friends = ormar.ManyToMany( + ChildRef, through=ChildFriendRef, related_name="also_friends" + ) + + +class ChildFriend(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db class Game(ormar.Model): @@ -82,8 +89,8 @@ async def cleanup(): @pytest.mark.asyncio -async def test_not_uprated_model_raises_errors(): - Person2 = ForwardRef("Person2") +async def test_not_updated_model_raises_errors(): + Person2Ref = ForwardRef("Person2") class Person2(ormar.Model): class Meta(ModelMeta): @@ -92,7 +99,7 @@ async def test_not_uprated_model_raises_errors(): id: int = ormar.Integer(primary_key=True) name: str = ormar.String(max_length=100) - supervisor: Person2 = ormar.ForeignKey(Person2, related_name="employees") + supervisor: Person2Ref = ormar.ForeignKey(Person2Ref, related_name="employees") with pytest.raises(ModelError): await Person2.objects.create(name="Test") @@ -104,6 +111,74 @@ async def test_not_uprated_model_raises_errors(): await Person2.objects.get() +@pytest.mark.asyncio +async def test_not_updated_model_m2m_raises_errors(): + Person3Ref = ForwardRef("Person3") + + class PersonFriend(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db + + class Person3(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + supervisors: Person3Ref = ormar.ManyToMany( + Person3Ref, through=PersonFriend, related_name="employees" + ) + + with pytest.raises(ModelError): + await Person3.objects.create(name="Test") + + with pytest.raises(ModelError): + Person3(name="Test") + + with pytest.raises(ModelError): + await Person3.objects.get() + + +@pytest.mark.asyncio +async def test_not_updated_model_m2m_through_raises_errors(): + PersonPetRef = ForwardRef("PersonPet") + + class Pet(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + + class Person4(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + pets: List[Pet] = ormar.ManyToMany( + Pet, through=PersonPetRef, related_name="owners" + ) + + class PersonPet(ormar.Model): + class Meta(ModelMeta): + metadata = metadata + database = db + + with pytest.raises(ModelError): + await Person4.objects.create(name="Test") + + with pytest.raises(ModelError): + Person4(name="Test") + + with pytest.raises(ModelError): + await Person4.objects.get() + + def test_proper_field_init(): assert "supervisor" in Person.Meta.model_fields assert Person.Meta.model_fields["supervisor"].to == Person