diff --git a/docs/releases.md b/docs/releases.md index 6cd8b4f..5e7ef64 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -5,6 +5,10 @@ * Add support for multi-column non-unique `IndexColumns` in `Meta.constraints` [#307](https://github.com/collerek/ormar/issues/307) * Add `sql_nullable` field attribute that allows to set different nullable setting for pydantic model and for underlying sql column [#308](https://github.com/collerek/ormar/issues/308) +## 馃悰 Fixes + +* Enable caching of relation map to increase performance [#337](https://github.com/collerek/ormar/issues/337) + # 0.10.18 ## 馃悰 Fixes diff --git a/ormar/__init__.py b/ormar/__init__.py index 54d926f..859159e 100644 --- a/ormar/__init__.py +++ b/ormar/__init__.py @@ -78,7 +78,7 @@ class UndefinedType: # pragma no cover Undefined = UndefinedType() -__version__ = "0.10.18" +__version__ = "0.10.19" __all__ = [ "Integer", "BigInteger", diff --git a/ormar/models/helpers/models.py b/ormar/models/helpers/models.py index bc56e1a..73d5b4e 100644 --- a/ormar/models/helpers/models.py +++ b/ormar/models/helpers/models.py @@ -75,6 +75,8 @@ def populate_default_options_values( if field.__type__ == bytes } + new_model.__relation_map__ = None + class Connection(sqlite3.Connection): def __init__(self, *args: Any, **kwargs: Any) -> None: # pragma: no cover diff --git a/ormar/models/mixins/relation_mixin.py b/ormar/models/mixins/relation_mixin.py index c4bd618..97df67e 100644 --- a/ormar/models/mixins/relation_mixin.py +++ b/ormar/models/mixins/relation_mixin.py @@ -20,6 +20,7 @@ class RelationMixin: from ormar import ModelMeta Meta: ModelMeta + __relation_map__: Optional[List[str]] _related_names: Optional[Set] _through_names: Optional[Set] _related_fields: Optional[List] @@ -120,7 +121,10 @@ class RelationMixin: @classmethod def _iterate_related_models( # noqa: CCR001 - cls, node_list: NodeList = None, source_relation: str = None + cls, + node_list: NodeList = None, + source_relation: str = None, + recurrent: bool = False, ) -> List[str]: """ Iterates related models recursively to extract relation strings of @@ -130,6 +134,8 @@ class RelationMixin: :rtype: List[str] """ if not node_list: + if cls.__relation_map__: + return cls.__relation_map__ node_list = NodeList() current_node = node_list.add(node_class=cls) else: @@ -145,11 +151,14 @@ class RelationMixin: parent_node=current_node, ) deep_relations = target_model._iterate_related_models( - source_relation=relation, node_list=node_list + source_relation=relation, node_list=node_list, recurrent=True ) processed_relations.extend(deep_relations) - return cls._get_final_relations(processed_relations, source_relation) + result = cls._get_final_relations(processed_relations, source_relation) + if not recurrent: + cls.__relation_map__ = result + return result @staticmethod def _get_final_relations( diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index a046552..04968d4 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -76,6 +76,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass __tablename__: str __metadata__: sqlalchemy.MetaData __database__: databases.Database + __relation_map__: Optional[List[str]] _orm_relationship_manager: AliasManager _orm: RelationsManager _orm_id: int diff --git a/tests/test_exclude_include_dict/test_complex_relation_tree_performance.py b/tests/test_exclude_include_dict/test_complex_relation_tree_performance.py new file mode 100644 index 0000000..8e8993a --- /dev/null +++ b/tests/test_exclude_include_dict/test_complex_relation_tree_performance.py @@ -0,0 +1,420 @@ +from datetime import datetime +from typing import List, Optional, Union + +import databases +import pydantic +import pytest +import sqlalchemy + +import ormar as orm + +from tests.settings import DATABASE_URL + +database = databases.Database(DATABASE_URL, force_rollback=True) +metadata = sqlalchemy.MetaData() + + +class MainMeta(orm.ModelMeta): + database = database + metadata = metadata + + +class ChagenlogRelease(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + + class Meta(MainMeta): + tablename = "changelog_release" + + +class CommitIssue(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + + class Meta(MainMeta): + tablename = "commit_issues" + + +class CommitLabel(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + + class Meta(MainMeta): + tablename = "commit_label" + + +class MergeRequestCommit(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + + class Meta(MainMeta): + tablename = "merge_request_commits" + + +class MergeRequestIssue(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + + class Meta(MainMeta): + tablename = "merge_request_issues" + + +class MergeRequestLabel(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + + class Meta(MainMeta): + tablename = "merge_request_labels" + + +class ProjectLabel(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + + class Meta(MainMeta): + tablename = "project_label" + + +class PushCommit(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + + class Meta(MainMeta): + tablename = "push_commit" + + +class PushLabel(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + + class Meta(MainMeta): + tablename = "push_label" + + +class TagCommit(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + + class Meta(MainMeta): + tablename = "tag_commits" + + +class TagIssue(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + + class Meta(MainMeta): + tablename = "tag_issue" + + +class TagLabel(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + + class Meta(MainMeta): + tablename = "tag_label" + + +class UserProject(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + access_level: int = orm.Integer(default=0) + + class Meta(MainMeta): + tablename = "user_project" + + +class Label(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + title: str = orm.String(max_length=100) + description: str = orm.Text(default="") + type: str = orm.String(max_length=100, default="") + + class Meta(MainMeta): + tablename = "labels" + + +class Project(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + name: str = orm.String(max_length=100) + description: str = orm.Text(default="") + git_url: str = orm.String(max_length=500, default="") + labels: Optional[Union[List[Label], Label]] = orm.ManyToMany( + Label, through=ProjectLabel, ondelete="CASCADE", onupdate="CASCADE" + ) + changelog_jira_tag: str = orm.String(max_length=100, default="") + change_type_jira_tag: str = orm.String(max_length=100, default="") + jira_prefix: str = orm.String(max_length=10, default="SAN") + type: str = orm.String(max_length=10, default="cs") + target_branch_name: str = orm.String(max_length=100, default="master") + header: str = orm.String(max_length=250, default="") + jira_url: str = orm.String(max_length=500,) + changelog_file: str = orm.String(max_length=250, default="") + version_file: str = orm.String(max_length=250, default="") + + class Meta(MainMeta): + tablename = "projects" + + +class Issue(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + summary: str = orm.Text(default="") + description: str = orm.Text(default="") + changelog: str = orm.Text(default="") + link: str = orm.String(max_length=500) + issue_type: str = orm.String(max_length=100) + key: str = orm.String(max_length=100) + change_type: str = orm.String(max_length=100, default="") + data: pydantic.Json = orm.JSON(default={}) + + class Meta(MainMeta): + tablename = "issues" + + +class User(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + username: str = orm.String(max_length=100, unique=True) + name: str = orm.String(max_length=200, default="") + + class Meta(MainMeta): + tablename = "users" + + +class Branch(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + name: str = orm.String(max_length=200) + description: str = orm.Text(default="") + automatic_tags: bool = orm.Boolean(default=False) + is_it_locked: bool = orm.Boolean(default=True) + prefix_tag: str = orm.String(max_length=50, default="") + postfix_tag: str = orm.String(max_length=50, default="") + project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE") + + class Meta(MainMeta): + tablename = "branches" + + +class Changelog(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + content: str = orm.Text(default="") + version: str = orm.Text(default="") + past_changelog: int = orm.Integer(default=0) + label: Label = orm.ForeignKey( + Label, nullable=True, ondelete="CASCADE", onupdate="CASCADE" + ) + project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE") + created_date: datetime = orm.DateTime(default=datetime.utcnow()) + + class Meta(MainMeta): + tablename = "changelogs" + + +class Commit(orm.Model): + id: str = orm.String(max_length=500, primary_key=True) + short_id: str = orm.String(max_length=500) + title: str = orm.String(max_length=500) + message: str = orm.Text(default="") + url = orm.String(max_length=500, default="") + author_name = orm.String(max_length=500, default="") + labels: Optional[Union[List[Label], Label]] = orm.ManyToMany( + Label, through=CommitLabel, ondelete="CASCADE", onupdate="CASCADE" + ) + issues: Optional[Union[List[Issue], Issue]] = orm.ManyToMany( + Issue, through=CommitIssue, ondelete="CASCADE", onupdate="CASCADE" + ) + + class Meta(MainMeta): + tablename = "commits" + + +class MergeRequest(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + idd: int = orm.Integer(default=0) + title: str = orm.String(max_length=500) + state: str = orm.String(max_length=100) + merge_status: str = orm.String(max_length=100) + description: str = orm.Text(default="") + source: Branch = orm.ForeignKey(Branch, related_name="source") + target: Branch = orm.ForeignKey(Branch, related_name="target") + labels: Optional[Union[List[Label], Label]] = orm.ManyToMany( + Label, through=MergeRequestLabel, ondelete="CASCADE", onupdate="CASCADE" + ) + commits: Optional[Union[List[Commit], Commit]] = orm.ManyToMany( + Commit, through=MergeRequestCommit, ondelete="CASCADE", onupdate="CASCADE" + ) + issues: Optional[Union[List[Issue], Issue]] = orm.ManyToMany( + Issue, through=MergeRequestIssue, ondelete="CASCADE", onupdate="CASCADE" + ) + project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE") + + class Meta(MainMeta): + tablename = "merge_requests" + + +class Push(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + branch: Branch = orm.ForeignKey( + Branch, nullable=True, ondelete="CASCADE", onupdate="CASCADE" + ) + has_locking_changes: bool = orm.Boolean(default=False) + sha: str = orm.String(max_length=200) + labels: Optional[Union[List[Label], Label]] = orm.ManyToMany( + Label, through=PushLabel, ondelete="CASCADE", onupdate="CASCADE" + ) + commits: Optional[Union[List[Commit], Commit]] = orm.ManyToMany( + Commit, + through=PushCommit, + through_relation_name="push", + through_reverse_relation_name="commit_id", + ondelete="CASCADE", + onupdate="CASCADE", + ) + author: User = orm.ForeignKey(User, ondelete="CASCADE", onupdate="CASCADE") + project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE") + + class Meta(MainMeta): + tablename = "pushes" + + +class Tag(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + name: str = orm.String(max_length=200) + ref: str = orm.String(max_length=200) + project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE") + title: str = orm.String(max_length=200, default="") + description: str = orm.Text(default="") + commits: Optional[Union[List[Commit], Commit]] = orm.ManyToMany( + Commit, + through=TagCommit, + through_relation_name="tag", + through_reverse_relation_name="commit_id", + ondelete="CASCADE", + onupdate="CASCADE", + ) + issues: Optional[Union[List[Issue], Issue]] = orm.ManyToMany( + Issue, through=TagIssue, ondelete="CASCADE", onupdate="CASCADE" + ) + labels: Optional[Union[List[Label], Label]] = orm.ManyToMany( + Label, through=TagLabel, ondelete="CASCADE", onupdate="CASCADE" + ) + user: User = orm.ForeignKey( + User, nullable=True, ondelete="CASCADE", onupdate="CASCADE" + ) + branch: Branch = orm.ForeignKey( + Branch, nullable=True, ondelete="CASCADE", onupdate="CASCADE" + ) + + class Meta(MainMeta): + tablename = "tags" + + +class Release(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + title: str = orm.String(max_length=200, default="") + description: str = orm.Text(default="") + tag: Tag = orm.ForeignKey(Tag, ondelete="CASCADE", onupdate="CASCADE") + changelogs: List[Changelog] = orm.ManyToMany( + Changelog, through=ChagenlogRelease, ondelete="CASCADE", onupdate="CASCADE" + ) + data: pydantic.Json = orm.JSON(default={}) + + class Meta(MainMeta): + tablename = "releases" + + +class Webhook(orm.Model): + id: int = orm.Integer(name="id", primary_key=True) + object_kind = orm.String(max_length=100) + project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE") + merge_request: MergeRequest = orm.ForeignKey( + MergeRequest, nullable=True, ondelete="CASCADE", onupdate="CASCADE" + ) + tag: Tag = orm.ForeignKey( + Tag, nullable=True, ondelete="CASCADE", onupdate="CASCADE" + ) + push: Push = orm.ForeignKey( + Push, nullable=True, ondelete="CASCADE", onupdate="CASCADE" + ) + created_at: datetime = orm.DateTime(default=datetime.now()) + data: pydantic.Json = orm.JSON(default={}) + status: int = orm.Integer(default=200) + error: str = orm.Text(default="") + + +@pytest.fixture(autouse=True, scope="module") +def create_test_database(): + engine = sqlalchemy.create_engine(DATABASE_URL) + metadata.drop_all(engine) + metadata.create_all(engine) + yield + metadata.drop_all(engine) + + +@pytest.mark.asyncio +async def test_very_complex_relation_map(): + async with database: + tags = [ + {"id": 18, "name": "name-18", "ref": "ref-18"}, + {"id": 17, "name": "name-17", "ref": "ref-17"}, + {"id": 12, "name": "name-12", "ref": "ref-12"}, + ] + payload = [ + { + "id": 9, + "title": "prueba-2321", + "description": "\n\n### [v.1.3.0.0] - 2021-08-19\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n Description: Se probara el flujo de changelog automatic. \n\n Changelog: Se agrega funci贸n para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n \n\n", + "data": {}, + }, + { + "id": 8, + "title": "prueba-123-prod", + "description": "\n\n### [v.1.2.0.0] - 2021-08-19\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n Description: Se probara el flujo de changelog automatic. \n\n Changelog: Se agrega funci贸n para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n \n\n", + "data": {}, + }, + { + "id": 6, + "title": "prueba-3-2", + "description": "\n\n### [v.1.1.0.0] - 2021-07-29\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n Description: Se probara el flujo de changelog automatic. \n\n Changelog: Se agrega funci贸n para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n \n\n", + "data": {}, + }, + ] + saved_tags = [] + for tag in tags: + saved_tags.append(await Tag(**tag).save()) + + for ind, pay in enumerate(payload): + await Release(**pay, tag=saved_tags[ind]).save() + + releases = await Release.objects.order_by(Release.id.desc()).all() + dicts = [release.dict() for release in releases] + + result = [ + { + "id": 9, + "title": "prueba-2321", + "description": "\n\n### [v.1.3.0.0] - 2021-08-19\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n Description: Se probara el flujo de changelog automatic. \n\n Changelog: Se agrega funci贸n para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n \n\n", + "data": {}, + "tag": { + "id": 18, + "taglabel": None, + "tagcommit": None, + "tagissue": None, + }, + "changelogs": [], + }, + { + "id": 8, + "title": "prueba-123-prod", + "description": "\n\n### [v.1.2.0.0] - 2021-08-19\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n Description: Se probara el flujo de changelog automatic. \n\n Changelog: Se agrega funci贸n para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n \n\n", + "data": {}, + "tag": { + "id": 17, + "taglabel": None, + "tagcommit": None, + "tagissue": None, + }, + "changelogs": [], + }, + { + "id": 6, + "title": "prueba-3-2", + "description": "\n\n### [v.1.1.0.0] - 2021-07-29\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n Description: Se probara el flujo de changelog automatic. \n\n Changelog: Se agrega funci贸n para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n \n\n", + "data": {}, + "tag": { + "id": 12, + "taglabel": None, + "tagcommit": None, + "tagissue": None, + }, + "changelogs": [], + }, + ] + + assert dicts == result