From 3279ef7a853361960e303c78328c9704ab3301aa Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 5 Jan 2021 15:18:13 +0100 Subject: [PATCH] finish inheritance docs, remove original through model from metadta, add high level overview in api docs --- docs/api/index.md | 104 ++++++++++++++++++++++++++- docs/contributing.md | 6 ++ docs/models/inheritance.md | 142 ++++++++++++++++++++++++++++++++++++- mkdocs.yml | 6 +- ormar/models/metaclass.py | 106 +++++++++++++++++++-------- 5 files changed, 330 insertions(+), 34 deletions(-) diff --git a/docs/api/index.md b/docs/api/index.md index 2081c16..0a681d7 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -13,4 +13,106 @@ Note that this is a technical part of the documentation intended for `ormar` con Ormar is still under development, and the **internals can change at any moment**. You shouldn't rely even on the "public" methods if they are not documented in the - normal part of the docs. \ No newline at end of file + normal part of the docs. + +## High level overview + +Ormar is divided into packages for maintainability and ease of development. + +Below you can find a short description of the structure of the whole project and +individual packages. + +### Models + +Contains the actual `ormar.Model` class, which is based on: + +* `ormar.NewBaseModel` which in turns: + * inherits from `pydantic.BaseModel`, + * uses `ormar.ModelMetaclass` for all heavy lifting, relations declaration, + parsing `ormar` fields, creating `sqlalchemy` columns and tables etc. + * There is a lot of tasks during class creation so `ormar` is using a lot of + `helpers` methods separated by functionality: `pydantic`, `sqlachemy`, + `relations` & `models` located in `helpers` submodule. + * inherits from `ormar.ModelTableProxy` that combines `Mixins` providing a special + additional behavior for `ormar.Models` + * `AliasMixin` - handling of column aliases, which are names changed only in db + * `ExcludableMixin` - handling excluding and including fields in dict() and database calls + * `MergeModelMixin` - handling merging Models initialized from raw sql raws into Models that needs to be merged, + in example parent models in join query that duplicates in raw response. + * `PrefetchQueryMixin` - handling resolving relations and ids of models to extract during issuing + subsequent queries in prefetch_related + * `RelationMixin` - handling resolving relations names, related fields etc. + * `SavePrepareMixin` - handling converting related models to their pk values, translating ormar field + names into aliases etc. + +### Fields + +Contains `ormar.BaseField` that is a base for all fields. + +All basic types are declared in `model_fields`, while relation fields are located in: + +* `foreign_key`: `ForeignKey` relation, expanding relations meaning initializing nested models, + creating dummy models with pk only that skips validation etc. +* `many_to_many`: `ManyToMany` relation that do not have a lot of logic on its own. + +Related to fields is a `@property_field` decorator that is located in `decorators.property_field`. + +There is also a special UUID field declaration for `sqlalchemy` that is based on `CHAR` field type. + +### Query Set + +Package that handles almost all interactions with db (some small parts are in `ormar.Model` and in `ormar.QuerysetProxy`). + +Provides a `QuerySet` that is exposed on each Model as `objects` property. + +Have a vast number of methods to query, filter, create, update and delete database rows. + +* Actual construction of the queries is delegated to `Query` class + * which in tern uses `SqlJoin` to construct joins + * `Clause` to convert `filter` and `exclude` conditions into sql + * `FilterQuery` to apply filter clauses on query + * `OrderQuery` to apply order by clauses on query + * `LimitQuery` to apply limit clause on query + * `OffsetQuery` to apply offset clause on query +* For prefetch_related the same is done by `PrefetchQuery` +* Common helpers functions are extracted into `utils` + +### Relations + +Handles registering relations, adding/removing to relations as well as returning the +actual related models instead of relation fields declared on Models. + +* Each `ormar.Model` has its own `RelationManager` registered under `_orm` property. + * `RelationManager` handles `Relations` between two different models + * In case of reverse relations or m2m relations the `RelationProxy` is used which + is basically a list with some special methods that keeps a reference to a list of related models + * Also, for reverse relations and m2m relations `QuerySetProxy` is exposed, that is + used to query the already pre-filtered related models and handles Through models + instances for m2m relations, while delegating actual queries to `QuerySet` +* `AliasManager` handles registration of aliases for relations that are used in queries. + In order to be able to link multiple times to the same table in one query each link + has to have unique alias to properly identify columns and extract proper values. + Kind of global registry, aliases are randomly generated, so might differ on each run. +* Common helpers functions are extracted into `utils` + +### Signals + +Handles sending signals on particular events. + +* `SignalEmitter` is registered on each `ormar.Model`, that allows to register any number of +receiver functions that will be notified on each event. +* For now only combination of (pre, post) (save, update, delete) events are pre populated for user +although it's easy to register user `Signal`s. +* set of decorators is prepared, each corresponding to one of the builtin signals, +that can be used to mark functions/methods that should become receivers, those decorators +are located in `decorators.signals`. +* You can register same function to different `ormar.Models` but each Model has it's own +Emitter that is independednt and issued on events for given Model. +* Currently, there is no way to register global `Signal` triggered for all models. + +### Exceptions + +Gathers all exceptions specific to `ormar`. + +All `ormar` exceptions inherit from `AsyncOrmException`. + diff --git a/docs/contributing.md b/docs/contributing.md index 90c957a..4b67abc 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -60,3 +60,9 @@ mkdocs build # ... commit, push, and create your pull request ``` + +!!!tip + For more information on how and why ormar works the way it works + please see the [API documentation][API documentation] + +[API documentation]: ./api/index.md \ No newline at end of file diff --git a/docs/models/inheritance.md b/docs/models/inheritance.md index 019fa60..450e788 100644 --- a/docs/models/inheritance.md +++ b/docs/models/inheritance.md @@ -220,6 +220,8 @@ Therefore, you have two options: That might sound complicated but let's look at the following example: +### ForeignKey relations + ```python # normal model used in relation class Person(ormar.Model): @@ -320,4 +322,142 @@ Person.Meta.model_fields `owner: Person = ormar.ForeignKey(Person, related_name="owned")` and model fields for Person owned cars would become `owned_trucks` and `owned_buses`. - \ No newline at end of file + +### ManyToMany relations + +Similarly, you can inherit from Models that have ManyToMany relations declared but +there is one, but substantial difference - the Through model. + +Since in the future the Through model will be able to hold additional fields and now it links only two Tables +(`from` and `to` ones), each child that inherits the m2m relation field has to have separate +Through model. + +Of course, you can overwrite the relation in each Child model, but that requires additional +code and undermines the point of the whole inheritance. `Ormar` will handle this for you if +you agree with default naming convention, which you can always manually overwrite in +children if needed. + +Again, let's look at the example to easier grasp the concepts. + +We will modify the previous example described above to use m2m relation for co_owners. + +```python +# person remain the same as above +class Person(ormar.Model): + class Meta: + metadata = metadata + database = db + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + +# new through model between Person and Car2 +class PersonsCar(ormar.Model): + class Meta: + tablename = "cars_x_persons" + metadata = metadata + database = db + +# note how co_owners is now ManyToMany relation +class Car2(ormar.Model): + class Meta: + # parent class needs to be marked abstract + abstract = True + metadata = metadata + database = db + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=50) + # note the related_name - needs to be unique across Person + # model, regardless of how many different models leads to Person + owner: Person = ormar.ForeignKey(Person, related_name="owned") + co_owners: List[Person] = ormar.ManyToMany( + Person, through=PersonsCar, related_name="coowned" + ) + created_date: datetime.datetime = ormar.DateTime(default=datetime.datetime.now) + + +# child models define only additional Fields +class Truck2(Car2): + class Meta: + # note how you don't have to provide inherited Meta params + tablename = "trucks2" + + max_capacity: int = ormar.Integer() + + +class Bus2(Car2): + class Meta: + tablename = "buses2" + + max_persons: int = ormar.Integer() +``` + +`Ormar` automatically modifies related_name of the fields to include the **table** name +of the children models. The dafault name is original related_name + '_' + child table name. + +That way for class Truck2 the relation defined in +`owner: Person = ormar.ForeignKey(Person, related_name="owned")` becomes `owned_trucks2` + +You can verify the names by inspecting the list of fields present on `Person` model. + +```python +Person.Meta.model_fields +{ +# note how all relation fields need to be unique on Person +# regardless if autogenerated or manually overwritten +'id': , +'name': , +# note that we expanded on previous example so all 'old' fields are here +'trucks': , +'coowned_trucks': , +'buses': , +'coowned_buses': , +# newly defined related fields +'owned_trucks2': , +'coowned_trucks2': , +'owned_buses2': , +'coowned_buses2': +} +``` + +But that's not all. It's kind of internal to `ormar` but affects the data structure in the database, +so let's examine the through models for both `Bus2` and `Truck2` models. + +```python +Bus2.Meta.model_fields['co_owners'].through + +Bus2.Meta.model_fields['co_owners'].through.Meta.tablename +'cars_x_persons_buses2' + +Truck2.Meta.model_fields['co_owners'].through + +Truck2.Meta.model_fields['co_owners'].through.Meta.tablename +'cars_x_persons_trucks2' +``` + +As you can see above `ormar` cloned the Through model for each of the Child classes and added +Child **class** name at the end, while changing the table names of the cloned fields +the name of the **table** from the child is used. + +Note that original model is not only not used, the table for this model is removed from metadata: + +```python +Bus2.Meta.metadata.tables.keys() +dict_keys(['test_date_models', 'categories', 'subjects', 'persons', 'trucks', 'buses', + 'cars_x_persons_trucks2', 'trucks2', 'cars_x_persons_buses2', 'buses2']) +``` + +So be aware that if you introduce inheritance along the way and convert a model into +abstract parent model you may lose your data on through table if not careful. + +!!!note + Note that original table name and model name of the Through model is never used. + Only the cloned models tables are created and used. + +!!!warning + Note that each subclass of the Model that has `ManyToMany` relation defined generates + a new `Through` model, meaning also **new database table**. + + That means that each time you define a Child model you need to either manually create + the table in the database, or run a migration (with alembic). \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 126fc97..a57c320 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -73,9 +73,9 @@ nav: - Exceptions: api/exceptions.md repo_name: collerek/ormar repo_url: https://github.com/collerek/ormar -#google_analytics: -# - UA-72514911-3 -# - auto +google_analytics: + - UA-72514911-3 + - auto theme: name: material highlightjs: true diff --git a/ormar/models/metaclass.py b/ormar/models/metaclass.py index 17e844d..91340f0 100644 --- a/ormar/models/metaclass.py +++ b/ormar/models/metaclass.py @@ -304,6 +304,71 @@ def update_attrs_from_base_meta( # noqa: CCR001 setattr(attrs["Meta"], param, parent_value) +def copy_and_replace_m2m_through_model( + field: Type[ManyToManyField], + field_name: str, + table_name: str, + parent_fields: Dict, + attrs: Dict, + meta: ModelMeta, +) -> None: + """ + Clones class with Through model for m2m relations, appends child name to the name + of the cloned class. + + Clones non foreign keys fields from parent model, the same with database columns. + + Modifies related_name with appending child table name after '_' + + For table name, the table name of child is appended after '_'. + + Removes the original sqlalchemy table from metadata if it was not removed. + + :param field: field with relations definition + :type field: Type[ManyToManyField] + :param field_name: name of the relation field + :type field_name: str + :param table_name: name of the table + :type table_name: str + :param parent_fields: dictionary of fields to copy to new models from parent + :type parent_fields: Dict + :param attrs: new namespace for class being constructed + :type attrs: Dict + :param meta: metaclass of currently created model + :type meta: ModelMeta + """ + copy_field: Type[BaseField] = type( # type: ignore + field.__name__, (ManyToManyField, BaseField), dict(field.__dict__) + ) + related_name = field.related_name + "_" + table_name + copy_field.related_name = related_name # type: ignore + + through_class = field.through + new_meta: ormar.ModelMeta = type( # type: ignore + "Meta", (), dict(through_class.Meta.__dict__), + ) + new_meta.tablename += "_" + meta.tablename + # create new table with copied columns but remove foreign keys + # they will be populated later in expanding reverse relation + if hasattr(new_meta, "table"): + del new_meta.table + new_meta.columns = [col for col in new_meta.columns if not col.foreign_keys] + new_meta.model_fields = { + name: field + for name, field in new_meta.model_fields.items() + if not issubclass(field, ForeignKeyField) + } + populate_meta_sqlalchemy_table_if_required(new_meta) + copy_name = through_class.__name__ + attrs.get("__name__", "") + copy_through = type(copy_name, (ormar.Model,), {"Meta": new_meta}) + copy_field.through = copy_through + + parent_fields[field_name] = copy_field + + if through_class.Meta.table in through_class.Meta.metadata: + through_class.Meta.metadata.remove(through_class.Meta.table) + + def copy_data_from_parent_model( # noqa: CCR001 base_class: Type["Model"], curr_class: type, @@ -344,7 +409,7 @@ def copy_data_from_parent_model( # noqa: CCR001 attrs=attrs, model_fields=model_fields, ) - parent_fields = dict() + parent_fields: Dict = dict() meta = attrs.get("Meta") if not meta: # pragma: no cover raise ModelDefinitionError( @@ -357,41 +422,21 @@ def copy_data_from_parent_model( # noqa: CCR001 ) for field_name, field in base_class.Meta.model_fields.items(): if issubclass(field, ManyToManyField): - copy_field: Type[BaseField] = type( # type: ignore - field.__name__, (ManyToManyField, BaseField), dict(field.__dict__) + copy_and_replace_m2m_through_model( + field=field, + field_name=field_name, + table_name=table_name, + parent_fields=parent_fields, + attrs=attrs, + meta=meta, ) - related_name = field.related_name + "_" + table_name - copy_field.related_name = related_name - - through_class = field.through - new_meta: ormar.ModelMeta = type( # type: ignore - "Meta", (), dict(through_class.Meta.__dict__), - ) - new_meta.tablename += "_" + meta.tablename - # create new table with copied columns but remove foreign keys - # they will be populated later in expanding reverse relation - del new_meta.table - new_meta.columns = [ - col for col in new_meta.columns if not col.foreign_keys - ] - new_meta.model_fields = { - name: field - for name, field in new_meta.model_fields.items() - if not issubclass(field, ForeignKeyField) - } - populate_meta_sqlalchemy_table_if_required(new_meta) - copy_name = through_class.__name__ + attrs.get("__name__", "") - copy_through = type(copy_name, (ormar.Model,), {"Meta": new_meta}) - copy_field.through = copy_through - - parent_fields[field_name] = copy_field elif issubclass(field, ForeignKeyField) and field.related_name: copy_field = type( # type: ignore field.__name__, (ForeignKeyField, BaseField), dict(field.__dict__) ) related_name = field.related_name + "_" + table_name - copy_field.related_name = related_name + copy_field.related_name = related_name # type: ignore parent_fields[field_name] = copy_field else: parent_fields[field_name] = field @@ -518,6 +563,9 @@ class ModelMetaclass(pydantic.main.ModelMetaclass): attrs["__name__"] = name attrs, model_fields = extract_annotations_and_default_vals(attrs) for base in reversed(bases): + mod = base.__module__ + if mod.startswith("ormar.models.") or mod.startswith("pydantic."): + continue attrs, model_fields = extract_from_parents_definition( base_class=base, curr_class=mcs, attrs=attrs, model_fields=model_fields )