add caching of relation map to increase performance

docs/releases.md

@@ -5,6 +5,10 @@
 * Add support for multi-column non-unique `IndexColumns` in `Meta.constraints` [#307](https://github.com/collerek/ormar/issues/307)
 * Add `sql_nullable` field attribute that allows to set different nullable setting for pydantic model and for underlying sql column [#308](https://github.com/collerek/ormar/issues/308)
 
+## 🐛 Fixes
+
+* Enable caching of relation map to increase performance [#337](https://github.com/collerek/ormar/issues/337)
+
 # 0.10.18
 
 ## 🐛 Fixes

ormar/__init__.py

@@ -78,7 +78,7 @@ class UndefinedType:  # pragma no cover
 
 Undefined = UndefinedType()
 
-__version__ = "0.10.18"
+__version__ = "0.10.19"
 __all__ = [
     "Integer",
     "BigInteger",

ormar/models/helpers/models.py

@@ -75,6 +75,8 @@ def populate_default_options_values(
         if field.__type__ == bytes
     }
 
+    new_model.__relation_map__ = None
+
 
 class Connection(sqlite3.Connection):
     def __init__(self, *args: Any, **kwargs: Any) -> None:  # pragma: no cover

ormar/models/mixins/relation_mixin.py

@@ -20,6 +20,7 @@ class RelationMixin:
         from ormar import ModelMeta
 
         Meta: ModelMeta
+        __relation_map__: Optional[List[str]]
         _related_names: Optional[Set]
         _through_names: Optional[Set]
         _related_fields: Optional[List]
@@ -120,7 +121,10 @@ class RelationMixin:
 
     @classmethod
     def _iterate_related_models(  # noqa: CCR001
-        cls, node_list: NodeList = None, source_relation: str = None
+        cls,
+        node_list: NodeList = None,
+        source_relation: str = None,
+        recurrent: bool = False,
     ) -> List[str]:
         """
         Iterates related models recursively to extract relation strings of
@@ -130,6 +134,8 @@ class RelationMixin:
         :rtype: List[str]
         """
         if not node_list:
+            if cls.__relation_map__:
+                return cls.__relation_map__
             node_list = NodeList()
             current_node = node_list.add(node_class=cls)
         else:
@@ -145,11 +151,14 @@ class RelationMixin:
                     parent_node=current_node,
                 )
                 deep_relations = target_model._iterate_related_models(
-                    source_relation=relation, node_list=node_list
+                    source_relation=relation, node_list=node_list, recurrent=True
                 )
                 processed_relations.extend(deep_relations)
 
-        return cls._get_final_relations(processed_relations, source_relation)
+        result = cls._get_final_relations(processed_relations, source_relation)
+        if not recurrent:
+            cls.__relation_map__ = result
+        return result
 
     @staticmethod
     def _get_final_relations(

ormar/models/newbasemodel.py

@@ -76,6 +76,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
         __tablename__: str
         __metadata__: sqlalchemy.MetaData
         __database__: databases.Database
+        __relation_map__: Optional[List[str]]
         _orm_relationship_manager: AliasManager
         _orm: RelationsManager
         _orm_id: int

tests/test_exclude_include_dict/test_complex_relation_tree_performance.py

@@ -0,0 +1,420 @@
+from datetime import datetime
+from typing import List, Optional, Union
+
+import databases
+import pydantic
+import pytest
+import sqlalchemy
+
+import ormar as orm
+
+from tests.settings import DATABASE_URL
+
+database = databases.Database(DATABASE_URL, force_rollback=True)
+metadata = sqlalchemy.MetaData()
+
+
+class MainMeta(orm.ModelMeta):
+    database = database
+    metadata = metadata
+
+
+class ChagenlogRelease(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+
+    class Meta(MainMeta):
+        tablename = "changelog_release"
+
+
+class CommitIssue(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+
+    class Meta(MainMeta):
+        tablename = "commit_issues"
+
+
+class CommitLabel(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+
+    class Meta(MainMeta):
+        tablename = "commit_label"
+
+
+class MergeRequestCommit(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+
+    class Meta(MainMeta):
+        tablename = "merge_request_commits"
+
+
+class MergeRequestIssue(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+
+    class Meta(MainMeta):
+        tablename = "merge_request_issues"
+
+
+class MergeRequestLabel(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+
+    class Meta(MainMeta):
+        tablename = "merge_request_labels"
+
+
+class ProjectLabel(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+
+    class Meta(MainMeta):
+        tablename = "project_label"
+
+
+class PushCommit(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+
+    class Meta(MainMeta):
+        tablename = "push_commit"
+
+
+class PushLabel(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+
+    class Meta(MainMeta):
+        tablename = "push_label"
+
+
+class TagCommit(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+
+    class Meta(MainMeta):
+        tablename = "tag_commits"
+
+
+class TagIssue(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+
+    class Meta(MainMeta):
+        tablename = "tag_issue"
+
+
+class TagLabel(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+
+    class Meta(MainMeta):
+        tablename = "tag_label"
+
+
+class UserProject(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+    access_level: int = orm.Integer(default=0)
+
+    class Meta(MainMeta):
+        tablename = "user_project"
+
+
+class Label(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+    title: str = orm.String(max_length=100)
+    description: str = orm.Text(default="")
+    type: str = orm.String(max_length=100, default="")
+
+    class Meta(MainMeta):
+        tablename = "labels"
+
+
+class Project(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+    name: str = orm.String(max_length=100)
+    description: str = orm.Text(default="")
+    git_url: str = orm.String(max_length=500, default="")
+    labels: Optional[Union[List[Label], Label]] = orm.ManyToMany(
+        Label, through=ProjectLabel, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    changelog_jira_tag: str = orm.String(max_length=100, default="")
+    change_type_jira_tag: str = orm.String(max_length=100, default="")
+    jira_prefix: str = orm.String(max_length=10, default="SAN")
+    type: str = orm.String(max_length=10, default="cs")
+    target_branch_name: str = orm.String(max_length=100, default="master")
+    header: str = orm.String(max_length=250, default="")
+    jira_url: str = orm.String(max_length=500,)
+    changelog_file: str = orm.String(max_length=250, default="")
+    version_file: str = orm.String(max_length=250, default="")
+
+    class Meta(MainMeta):
+        tablename = "projects"
+
+
+class Issue(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+    summary: str = orm.Text(default="")
+    description: str = orm.Text(default="")
+    changelog: str = orm.Text(default="")
+    link: str = orm.String(max_length=500)
+    issue_type: str = orm.String(max_length=100)
+    key: str = orm.String(max_length=100)
+    change_type: str = orm.String(max_length=100, default="")
+    data: pydantic.Json = orm.JSON(default={})
+
+    class Meta(MainMeta):
+        tablename = "issues"
+
+
+class User(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+    username: str = orm.String(max_length=100, unique=True)
+    name: str = orm.String(max_length=200, default="")
+
+    class Meta(MainMeta):
+        tablename = "users"
+
+
+class Branch(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+    name: str = orm.String(max_length=200)
+    description: str = orm.Text(default="")
+    automatic_tags: bool = orm.Boolean(default=False)
+    is_it_locked: bool = orm.Boolean(default=True)
+    prefix_tag: str = orm.String(max_length=50, default="")
+    postfix_tag: str = orm.String(max_length=50, default="")
+    project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE")
+
+    class Meta(MainMeta):
+        tablename = "branches"
+
+
+class Changelog(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+    content: str = orm.Text(default="")
+    version: str = orm.Text(default="")
+    past_changelog: int = orm.Integer(default=0)
+    label: Label = orm.ForeignKey(
+        Label, nullable=True, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE")
+    created_date: datetime = orm.DateTime(default=datetime.utcnow())
+
+    class Meta(MainMeta):
+        tablename = "changelogs"
+
+
+class Commit(orm.Model):
+    id: str = orm.String(max_length=500, primary_key=True)
+    short_id: str = orm.String(max_length=500)
+    title: str = orm.String(max_length=500)
+    message: str = orm.Text(default="")
+    url = orm.String(max_length=500, default="")
+    author_name = orm.String(max_length=500, default="")
+    labels: Optional[Union[List[Label], Label]] = orm.ManyToMany(
+        Label, through=CommitLabel, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    issues: Optional[Union[List[Issue], Issue]] = orm.ManyToMany(
+        Issue, through=CommitIssue, ondelete="CASCADE", onupdate="CASCADE"
+    )
+
+    class Meta(MainMeta):
+        tablename = "commits"
+
+
+class MergeRequest(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+    idd: int = orm.Integer(default=0)
+    title: str = orm.String(max_length=500)
+    state: str = orm.String(max_length=100)
+    merge_status: str = orm.String(max_length=100)
+    description: str = orm.Text(default="")
+    source: Branch = orm.ForeignKey(Branch, related_name="source")
+    target: Branch = orm.ForeignKey(Branch, related_name="target")
+    labels: Optional[Union[List[Label], Label]] = orm.ManyToMany(
+        Label, through=MergeRequestLabel, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    commits: Optional[Union[List[Commit], Commit]] = orm.ManyToMany(
+        Commit, through=MergeRequestCommit, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    issues: Optional[Union[List[Issue], Issue]] = orm.ManyToMany(
+        Issue, through=MergeRequestIssue, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE")
+
+    class Meta(MainMeta):
+        tablename = "merge_requests"
+
+
+class Push(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+    branch: Branch = orm.ForeignKey(
+        Branch, nullable=True, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    has_locking_changes: bool = orm.Boolean(default=False)
+    sha: str = orm.String(max_length=200)
+    labels: Optional[Union[List[Label], Label]] = orm.ManyToMany(
+        Label, through=PushLabel, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    commits: Optional[Union[List[Commit], Commit]] = orm.ManyToMany(
+        Commit,
+        through=PushCommit,
+        through_relation_name="push",
+        through_reverse_relation_name="commit_id",
+        ondelete="CASCADE",
+        onupdate="CASCADE",
+    )
+    author: User = orm.ForeignKey(User, ondelete="CASCADE", onupdate="CASCADE")
+    project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE")
+
+    class Meta(MainMeta):
+        tablename = "pushes"
+
+
+class Tag(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+    name: str = orm.String(max_length=200)
+    ref: str = orm.String(max_length=200)
+    project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE")
+    title: str = orm.String(max_length=200, default="")
+    description: str = orm.Text(default="")
+    commits: Optional[Union[List[Commit], Commit]] = orm.ManyToMany(
+        Commit,
+        through=TagCommit,
+        through_relation_name="tag",
+        through_reverse_relation_name="commit_id",
+        ondelete="CASCADE",
+        onupdate="CASCADE",
+    )
+    issues: Optional[Union[List[Issue], Issue]] = orm.ManyToMany(
+        Issue, through=TagIssue, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    labels: Optional[Union[List[Label], Label]] = orm.ManyToMany(
+        Label, through=TagLabel, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    user: User = orm.ForeignKey(
+        User, nullable=True, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    branch: Branch = orm.ForeignKey(
+        Branch, nullable=True, ondelete="CASCADE", onupdate="CASCADE"
+    )
+
+    class Meta(MainMeta):
+        tablename = "tags"
+
+
+class Release(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+    title: str = orm.String(max_length=200, default="")
+    description: str = orm.Text(default="")
+    tag: Tag = orm.ForeignKey(Tag, ondelete="CASCADE", onupdate="CASCADE")
+    changelogs: List[Changelog] = orm.ManyToMany(
+        Changelog, through=ChagenlogRelease, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    data: pydantic.Json = orm.JSON(default={})
+
+    class Meta(MainMeta):
+        tablename = "releases"
+
+
+class Webhook(orm.Model):
+    id: int = orm.Integer(name="id", primary_key=True)
+    object_kind = orm.String(max_length=100)
+    project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE")
+    merge_request: MergeRequest = orm.ForeignKey(
+        MergeRequest, nullable=True, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    tag: Tag = orm.ForeignKey(
+        Tag, nullable=True, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    push: Push = orm.ForeignKey(
+        Push, nullable=True, ondelete="CASCADE", onupdate="CASCADE"
+    )
+    created_at: datetime = orm.DateTime(default=datetime.now())
+    data: pydantic.Json = orm.JSON(default={})
+    status: int = orm.Integer(default=200)
+    error: str = orm.Text(default="")
+
+
+@pytest.fixture(autouse=True, scope="module")
+def create_test_database():
+    engine = sqlalchemy.create_engine(DATABASE_URL)
+    metadata.drop_all(engine)
+    metadata.create_all(engine)
+    yield
+    metadata.drop_all(engine)
+
+
+@pytest.mark.asyncio
+async def test_very_complex_relation_map():
+    async with database:
+        tags = [
+            {"id": 18, "name": "name-18", "ref": "ref-18"},
+            {"id": 17, "name": "name-17", "ref": "ref-17"},
+            {"id": 12, "name": "name-12", "ref": "ref-12"},
+        ]
+        payload = [
+            {
+                "id": 9,
+                "title": "prueba-2321",
+                "description": "\n<!--- start changelog ver.v.1.3.0.0 -->\n### [v.1.3.0.0] - 2021-08-19\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic  Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n    Description: Se probara el flujo de changelog automatic.  \n\n    Changelog: Se agrega función para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n            \n<!--- end changelog ver.v.1.3.0.0 -->\n",
+                "data": {},
+            },
+            {
+                "id": 8,
+                "title": "prueba-123-prod",
+                "description": "\n<!--- start changelog ver.v.1.2.0.0 -->\n### [v.1.2.0.0] - 2021-08-19\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic  Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n    Description: Se probara el flujo de changelog automatic.  \n\n    Changelog: Se agrega función para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n            \n<!--- end changelog ver.v.1.2.0.0 -->\n",
+                "data": {},
+            },
+            {
+                "id": 6,
+                "title": "prueba-3-2",
+                "description": "\n<!--- start changelog ver.v.1.1.0.0 -->\n### [v.1.1.0.0] - 2021-07-29\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic  Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n    Description: Se probara el flujo de changelog automatic.  \n\n    Changelog: Se agrega función para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n            \n<!--- end changelog ver.v.1.1.0.0 -->\n",
+                "data": {},
+            },
+        ]
+        saved_tags = []
+        for tag in tags:
+            saved_tags.append(await Tag(**tag).save())
+
+        for ind, pay in enumerate(payload):
+            await Release(**pay, tag=saved_tags[ind]).save()
+
+        releases = await Release.objects.order_by(Release.id.desc()).all()
+        dicts = [release.dict() for release in releases]
+
+        result = [
+            {
+                "id": 9,
+                "title": "prueba-2321",
+                "description": "\n<!--- start changelog ver.v.1.3.0.0 -->\n### [v.1.3.0.0] - 2021-08-19\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic  Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n    Description: Se probara el flujo de changelog automatic.  \n\n    Changelog: Se agrega función para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n            \n<!--- end changelog ver.v.1.3.0.0 -->\n",
+                "data": {},
+                "tag": {
+                    "id": 18,
+                    "taglabel": None,
+                    "tagcommit": None,
+                    "tagissue": None,
+                },
+                "changelogs": [],
+            },
+            {
+                "id": 8,
+                "title": "prueba-123-prod",
+                "description": "\n<!--- start changelog ver.v.1.2.0.0 -->\n### [v.1.2.0.0] - 2021-08-19\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic  Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n    Description: Se probara el flujo de changelog automatic.  \n\n    Changelog: Se agrega función para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n            \n<!--- end changelog ver.v.1.2.0.0 -->\n",
+                "data": {},
+                "tag": {
+                    "id": 17,
+                    "taglabel": None,
+                    "tagcommit": None,
+                    "tagissue": None,
+                },
+                "changelogs": [],
+            },
+            {
+                "id": 6,
+                "title": "prueba-3-2",
+                "description": "\n<!--- start changelog ver.v.1.1.0.0 -->\n### [v.1.1.0.0] - 2021-07-29\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic  Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n    Description: Se probara el flujo de changelog automatic.  \n\n    Changelog: Se agrega función para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n            \n<!--- end changelog ver.v.1.1.0.0 -->\n",
+                "data": {},
+                "tag": {
+                    "id": 12,
+                    "taglabel": None,
+                    "tagcommit": None,
+                    "tagissue": None,
+                },
+                "changelogs": [],
+            },
+        ]
+
+        assert dicts == result