Prodream/ormar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add caching of relation map to increase performance

Browse Source
This commit is contained in:
collerek
2021-09-10 12:58:26 +02:00
parent 913de2dc44
commit beb43dd76c
6 changed files with 440 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/releases.md
View File

@ -5,6 +5,10 @@
* Add support for multi-column non-unique `IndexColumns` in `Meta.constraints` [#307](https://github.com/collerek/ormar/issues/307)
* Add `sql_nullable` field attribute that allows to set different nullable setting for pydantic model and for underlying sql column [#308](https://github.com/collerek/ormar/issues/308)
## 🐛 Fixes
* Enable caching of relation map to increase performance [#337](https://github.com/collerek/ormar/issues/337)
# 0.10.18
## 🐛 Fixes

2
ormar/__init__.py
View File

@ -78,7 +78,7 @@ class UndefinedType: # pragma no cover
Undefined = UndefinedType()
__version__ = "0.10.18"
__version__ = "0.10.19"
__all__ = [
"Integer",
"BigInteger",

2
ormar/models/helpers/models.py
View File

@ -75,6 +75,8 @@ def populate_default_options_values(
if field.__type__ == bytes
}
new_model.__relation_map__ = None
class Connection(sqlite3.Connection):
def __init__(self, *args: Any, **kwargs: Any) -> None: # pragma: no cover

15
ormar/models/mixins/relation_mixin.py
View File

@ -20,6 +20,7 @@ class RelationMixin:
from ormar import ModelMeta
Meta: ModelMeta
__relation_map__: Optional[List[str]]
_related_names: Optional[Set]
_through_names: Optional[Set]
_related_fields: Optional[List]
@ -120,7 +121,10 @@ class RelationMixin:
@classmethod
def _iterate_related_models( # noqa: CCR001
cls, node_list: NodeList = None, source_relation: str = None
cls,
node_list: NodeList = None,
source_relation: str = None,
recurrent: bool = False,
) -> List[str]:
"""
Iterates related models recursively to extract relation strings of
@ -130,6 +134,8 @@ class RelationMixin:
:rtype: List[str]
"""
if not node_list:
if cls.__relation_map__:
return cls.__relation_map__
node_list = NodeList()
current_node = node_list.add(node_class=cls)
else:
@ -145,11 +151,14 @@ class RelationMixin:
parent_node=current_node,
)
deep_relations = target_model._iterate_related_models(
source_relation=relation, node_list=node_list
source_relation=relation, node_list=node_list, recurrent=True
)
processed_relations.extend(deep_relations)
return cls._get_final_relations(processed_relations, source_relation)
result = cls._get_final_relations(processed_relations, source_relation)
if not recurrent:
cls.__relation_map__ = result
return result
@staticmethod
def _get_final_relations(

1
ormar/models/newbasemodel.py
View File

@ -76,6 +76,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass
__tablename__: str
__metadata__: sqlalchemy.MetaData
__database__: databases.Database
__relation_map__: Optional[List[str]]
_orm_relationship_manager: AliasManager
_orm: RelationsManager
_orm_id: int

420
tests/test_exclude_include_dict/test_complex_relation_tree_performance.py Normal file
View File

@ -0,0 +1,420 @@
from datetime import datetime
from typing import List, Optional, Union
import databases
import pydantic
import pytest
import sqlalchemy
import ormar as orm
from tests.settings import DATABASE_URL
database = databases.Database(DATABASE_URL, force_rollback=True)
metadata = sqlalchemy.MetaData()
class MainMeta(orm.ModelMeta):
database = database
metadata = metadata
class ChagenlogRelease(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
class Meta(MainMeta):
tablename = "changelog_release"
class CommitIssue(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
class Meta(MainMeta):
tablename = "commit_issues"
class CommitLabel(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
class Meta(MainMeta):
tablename = "commit_label"
class MergeRequestCommit(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
class Meta(MainMeta):
tablename = "merge_request_commits"
class MergeRequestIssue(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
class Meta(MainMeta):
tablename = "merge_request_issues"
class MergeRequestLabel(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
class Meta(MainMeta):
tablename = "merge_request_labels"
class ProjectLabel(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
class Meta(MainMeta):
tablename = "project_label"
class PushCommit(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
class Meta(MainMeta):
tablename = "push_commit"
class PushLabel(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
class Meta(MainMeta):
tablename = "push_label"
class TagCommit(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
class Meta(MainMeta):
tablename = "tag_commits"
class TagIssue(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
class Meta(MainMeta):
tablename = "tag_issue"
class TagLabel(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
class Meta(MainMeta):
tablename = "tag_label"
class UserProject(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
access_level: int = orm.Integer(default=0)
class Meta(MainMeta):
tablename = "user_project"
class Label(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
title: str = orm.String(max_length=100)
description: str = orm.Text(default="")
type: str = orm.String(max_length=100, default="")
class Meta(MainMeta):
tablename = "labels"
class Project(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
name: str = orm.String(max_length=100)
description: str = orm.Text(default="")
git_url: str = orm.String(max_length=500, default="")
labels: Optional[Union[List[Label], Label]] = orm.ManyToMany(
Label, through=ProjectLabel, ondelete="CASCADE", onupdate="CASCADE"
)
changelog_jira_tag: str = orm.String(max_length=100, default="")
change_type_jira_tag: str = orm.String(max_length=100, default="")
jira_prefix: str = orm.String(max_length=10, default="SAN")
type: str = orm.String(max_length=10, default="cs")
target_branch_name: str = orm.String(max_length=100, default="master")
header: str = orm.String(max_length=250, default="")
jira_url: str = orm.String(max_length=500,)
changelog_file: str = orm.String(max_length=250, default="")
version_file: str = orm.String(max_length=250, default="")
class Meta(MainMeta):
tablename = "projects"
class Issue(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
summary: str = orm.Text(default="")
description: str = orm.Text(default="")
changelog: str = orm.Text(default="")
link: str = orm.String(max_length=500)
issue_type: str = orm.String(max_length=100)
key: str = orm.String(max_length=100)
change_type: str = orm.String(max_length=100, default="")
data: pydantic.Json = orm.JSON(default={})
class Meta(MainMeta):
tablename = "issues"
class User(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
username: str = orm.String(max_length=100, unique=True)
name: str = orm.String(max_length=200, default="")
class Meta(MainMeta):
tablename = "users"
class Branch(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
name: str = orm.String(max_length=200)
description: str = orm.Text(default="")
automatic_tags: bool = orm.Boolean(default=False)
is_it_locked: bool = orm.Boolean(default=True)
prefix_tag: str = orm.String(max_length=50, default="")
postfix_tag: str = orm.String(max_length=50, default="")
project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE")
class Meta(MainMeta):
tablename = "branches"
class Changelog(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
content: str = orm.Text(default="")
version: str = orm.Text(default="")
past_changelog: int = orm.Integer(default=0)
label: Label = orm.ForeignKey(
Label, nullable=True, ondelete="CASCADE", onupdate="CASCADE"
)
project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE")
created_date: datetime = orm.DateTime(default=datetime.utcnow())
class Meta(MainMeta):
tablename = "changelogs"
class Commit(orm.Model):
id: str = orm.String(max_length=500, primary_key=True)
short_id: str = orm.String(max_length=500)
title: str = orm.String(max_length=500)
message: str = orm.Text(default="")
url = orm.String(max_length=500, default="")
author_name = orm.String(max_length=500, default="")
labels: Optional[Union[List[Label], Label]] = orm.ManyToMany(
Label, through=CommitLabel, ondelete="CASCADE", onupdate="CASCADE"
)
issues: Optional[Union[List[Issue], Issue]] = orm.ManyToMany(
Issue, through=CommitIssue, ondelete="CASCADE", onupdate="CASCADE"
)
class Meta(MainMeta):
tablename = "commits"
class MergeRequest(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
idd: int = orm.Integer(default=0)
title: str = orm.String(max_length=500)
state: str = orm.String(max_length=100)
merge_status: str = orm.String(max_length=100)
description: str = orm.Text(default="")
source: Branch = orm.ForeignKey(Branch, related_name="source")
target: Branch = orm.ForeignKey(Branch, related_name="target")
labels: Optional[Union[List[Label], Label]] = orm.ManyToMany(
Label, through=MergeRequestLabel, ondelete="CASCADE", onupdate="CASCADE"
)
commits: Optional[Union[List[Commit], Commit]] = orm.ManyToMany(
Commit, through=MergeRequestCommit, ondelete="CASCADE", onupdate="CASCADE"
)
issues: Optional[Union[List[Issue], Issue]] = orm.ManyToMany(
Issue, through=MergeRequestIssue, ondelete="CASCADE", onupdate="CASCADE"
)
project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE")
class Meta(MainMeta):
tablename = "merge_requests"
class Push(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
branch: Branch = orm.ForeignKey(
Branch, nullable=True, ondelete="CASCADE", onupdate="CASCADE"
)
has_locking_changes: bool = orm.Boolean(default=False)
sha: str = orm.String(max_length=200)
labels: Optional[Union[List[Label], Label]] = orm.ManyToMany(
Label, through=PushLabel, ondelete="CASCADE", onupdate="CASCADE"
)
commits: Optional[Union[List[Commit], Commit]] = orm.ManyToMany(
Commit,
through=PushCommit,
through_relation_name="push",
through_reverse_relation_name="commit_id",
ondelete="CASCADE",
onupdate="CASCADE",
)
author: User = orm.ForeignKey(User, ondelete="CASCADE", onupdate="CASCADE")
project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE")
class Meta(MainMeta):
tablename = "pushes"
class Tag(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
name: str = orm.String(max_length=200)
ref: str = orm.String(max_length=200)
project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE")
title: str = orm.String(max_length=200, default="")
description: str = orm.Text(default="")
commits: Optional[Union[List[Commit], Commit]] = orm.ManyToMany(
Commit,
through=TagCommit,
through_relation_name="tag",
through_reverse_relation_name="commit_id",
ondelete="CASCADE",
onupdate="CASCADE",
)
issues: Optional[Union[List[Issue], Issue]] = orm.ManyToMany(
Issue, through=TagIssue, ondelete="CASCADE", onupdate="CASCADE"
)
labels: Optional[Union[List[Label], Label]] = orm.ManyToMany(
Label, through=TagLabel, ondelete="CASCADE", onupdate="CASCADE"
)
user: User = orm.ForeignKey(
User, nullable=True, ondelete="CASCADE", onupdate="CASCADE"
)
branch: Branch = orm.ForeignKey(
Branch, nullable=True, ondelete="CASCADE", onupdate="CASCADE"
)
class Meta(MainMeta):
tablename = "tags"
class Release(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
title: str = orm.String(max_length=200, default="")
description: str = orm.Text(default="")
tag: Tag = orm.ForeignKey(Tag, ondelete="CASCADE", onupdate="CASCADE")
changelogs: List[Changelog] = orm.ManyToMany(
Changelog, through=ChagenlogRelease, ondelete="CASCADE", onupdate="CASCADE"
)
data: pydantic.Json = orm.JSON(default={})
class Meta(MainMeta):
tablename = "releases"
class Webhook(orm.Model):
id: int = orm.Integer(name="id", primary_key=True)
object_kind = orm.String(max_length=100)
project: Project = orm.ForeignKey(Project, ondelete="CASCADE", onupdate="CASCADE")
merge_request: MergeRequest = orm.ForeignKey(
MergeRequest, nullable=True, ondelete="CASCADE", onupdate="CASCADE"
)
tag: Tag = orm.ForeignKey(
Tag, nullable=True, ondelete="CASCADE", onupdate="CASCADE"
)
push: Push = orm.ForeignKey(
Push, nullable=True, ondelete="CASCADE", onupdate="CASCADE"
)
created_at: datetime = orm.DateTime(default=datetime.now())
data: pydantic.Json = orm.JSON(default={})
status: int = orm.Integer(default=200)
error: str = orm.Text(default="")
@pytest.fixture(autouse=True, scope="module")
def create_test_database():
engine = sqlalchemy.create_engine(DATABASE_URL)
metadata.drop_all(engine)
metadata.create_all(engine)
yield
metadata.drop_all(engine)
@pytest.mark.asyncio
async def test_very_complex_relation_map():
async with database:
tags = [
{"id": 18, "name": "name-18", "ref": "ref-18"},
{"id": 17, "name": "name-17", "ref": "ref-17"},
{"id": 12, "name": "name-12", "ref": "ref-12"},
]
payload = [
{
"id": 9,
"title": "prueba-2321",
"description": "\n<!--- start changelog ver.v.1.3.0.0 -->\n### [v.1.3.0.0] - 2021-08-19\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n Description: Se probara el flujo de changelog automatic. \n\n Changelog: Se agrega función para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n \n<!--- end changelog ver.v.1.3.0.0 -->\n",
"data": {},
},
{
"id": 8,
"title": "prueba-123-prod",
"description": "\n<!--- start changelog ver.v.1.2.0.0 -->\n### [v.1.2.0.0] - 2021-08-19\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n Description: Se probara el flujo de changelog automatic. \n\n Changelog: Se agrega función para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n \n<!--- end changelog ver.v.1.2.0.0 -->\n",
"data": {},
},
{
"id": 6,
"title": "prueba-3-2",
"description": "\n<!--- start changelog ver.v.1.1.0.0 -->\n### [v.1.1.0.0] - 2021-07-29\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n Description: Se probara el flujo de changelog automatic. \n\n Changelog: Se agrega función para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n \n<!--- end changelog ver.v.1.1.0.0 -->\n",
"data": {},
},
]
saved_tags = []
for tag in tags:
saved_tags.append(await Tag(**tag).save())
for ind, pay in enumerate(payload):
await Release(**pay, tag=saved_tags[ind]).save()
releases = await Release.objects.order_by(Release.id.desc()).all()
dicts = [release.dict() for release in releases]
result = [
{
"id": 9,
"title": "prueba-2321",
"description": "\n<!--- start changelog ver.v.1.3.0.0 -->\n### [v.1.3.0.0] - 2021-08-19\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n Description: Se probara el flujo de changelog automatic. \n\n Changelog: Se agrega función para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n \n<!--- end changelog ver.v.1.3.0.0 -->\n",
"data": {},
"tag": {
"id": 18,
"taglabel": None,
"tagcommit": None,
"tagissue": None,
},
"changelogs": [],
},
{
"id": 8,
"title": "prueba-123-prod",
"description": "\n<!--- start changelog ver.v.1.2.0.0 -->\n### [v.1.2.0.0] - 2021-08-19\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n Description: Se probara el flujo de changelog automatic. \n\n Changelog: Se agrega función para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n \n<!--- end changelog ver.v.1.2.0.0 -->\n",
"data": {},
"tag": {
"id": 17,
"taglabel": None,
"tagcommit": None,
"tagissue": None,
},
"changelogs": [],
},
{
"id": 6,
"title": "prueba-3-2",
"description": "\n<!--- start changelog ver.v.1.1.0.0 -->\n### [v.1.1.0.0] - 2021-07-29\n#### Resolved Issues\n\n#### Task\n\n- Probar flujo de changelog Automatic Jira: [SAN-86](https://htech.atlassian.net/browse/SAN-86)\n\n Description: Se probara el flujo de changelog automatic. \n\n Changelog: Se agrega función para extraer texto del campo changelog del dashboard de Sanval y ponerlo directamente en el changelog.md del repositorio. \n\n\n \n<!--- end changelog ver.v.1.1.0.0 -->\n",
"data": {},
"tag": {
"id": 12,
"taglabel": None,
"tagcommit": None,
"tagissue": None,
},
"changelogs": [],
},
]
assert dicts == result