some cleanup of unused relations code, introduced caching of related_names and props on model, set profiling

.gitignore

@@ -9,3 +9,4 @@ test.db
 dist
 /ormar.egg-info/
 site
+profile.py

ormar/models/modelproxy.py

@@ -12,11 +12,17 @@ from typing import (
     Union,
 )
 
-import ormar
 from ormar.exceptions import RelationshipInstanceError
+
+try:
+    import orjson as json
+except ImportError:  # pragma: nocover
+    import json # type: ignore
+
+import ormar
 from ormar.fields import BaseField, ManyToManyField
 from ormar.fields.foreign_key import ForeignKeyField
-from ormar.models.metaclass import ModelMeta, expand_reverse_relationships
+from ormar.models.metaclass import ModelMeta
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
@@ -30,6 +36,8 @@ Field = TypeVar("Field", bound=BaseField)
 class ModelTableProxy:
     if TYPE_CHECKING:  # pragma no cover
         Meta: ModelMeta
+        _related_names: Set
+        _related_names_hash: Union[str, bytes]
 
     def dict(self):  # noqa A003
         raise NotImplementedError  # pragma no cover
@@ -88,10 +96,17 @@ class ModelTableProxy:
 
     @classmethod
     def extract_related_names(cls) -> Set:
+
+        if isinstance(cls._related_names_hash, (str, bytes)):
+            return cls._related_names
+
         related_names = set()
         for name, field in cls.Meta.model_fields.items():
             if inspect.isclass(field) and issubclass(field, ForeignKeyField):
                 related_names.add(name)
+        cls._related_names_hash = json.dumps(list(cls.Meta.model_fields.keys()))
+        cls._related_names = related_names
+
         return related_names
 
     @classmethod
@@ -137,8 +152,7 @@ class ModelTableProxy:
     @staticmethod
     def resolve_relation_name(  # noqa CCR001
             item: Union["NewBaseModel", Type["NewBaseModel"]],
-        related: Union["NewBaseModel", Type["NewBaseModel"]],
+            related: Union["NewBaseModel", Type["NewBaseModel"]]
-        register_missing: bool = True,
     ) -> str:
         for name, field in item.Meta.model_fields.items():
             if issubclass(field, ForeignKeyField):
@@ -147,12 +161,6 @@ class ModelTableProxy:
                 # so we need to compare Meta too as this one is copied as is
                 if field.to == related.__class__ or field.to.Meta == related.Meta:
                     return name
-        # fallback for not registered relation
-        if register_missing:  # pragma nocover
-            expand_reverse_relationships(related.__class__)  # type: ignore
-            return ModelTableProxy.resolve_relation_name(
-                item, related, register_missing=False
-            )
 
         raise ValueError(
             f"No relation between {item.get_name()} and {related.get_name()}"
@@ -161,7 +169,7 @@ class ModelTableProxy:
     @staticmethod
     def resolve_relation_field(
             item: Union["Model", Type["Model"]], related: Union["Model", Type["Model"]]
-    ) -> Union[Type[BaseField], Type[ForeignKeyField]]:
+    ) -> Type[BaseField]:
         name = ModelTableProxy.resolve_relation_name(item, related)
         to_field = item.Meta.model_fields.get(name)
         if not to_field:  # pragma no cover

ormar/models/newbasemodel.py

@@ -9,7 +9,7 @@ from typing import (
     Mapping,
     Optional,
     Sequence,
-    TYPE_CHECKING,
+    Set, TYPE_CHECKING,
     Type,
     TypeVar,
     Union,
@@ -43,7 +43,7 @@ if TYPE_CHECKING:  # pragma no cover
 class NewBaseModel(
     pydantic.BaseModel, ModelTableProxy, Excludable, metaclass=ModelMetaclass
 ):
-    __slots__ = ("_orm_id", "_orm_saved", "_orm")
+    __slots__ = ("_orm_id", "_orm_saved", "_orm", "_related_names", "_related_names_hash", "_props")
 
     if TYPE_CHECKING:  # pragma no cover
         __model_fields__: Dict[str, Type[BaseField]]
@@ -56,6 +56,10 @@ class NewBaseModel(
         __database__: databases.Database
         _orm_relationship_manager: AliasManager
         _orm: RelationsManager
+        _orm_saved: bool
+        _related_names: Set
+        _related_names_hash: str
+        _props: List[str]
         Meta: ModelMeta
 
     # noinspection PyMissingConstructor
@@ -107,7 +111,7 @@ class NewBaseModel(
             )
 
     def __setattr__(self, name: str, value: Any) -> None:  # noqa CCR001
-        if name in ("_orm_id", "_orm_saved", "_orm"):
+        if name in ("_orm_id", "_orm_saved", "_orm", "_related_names", "_props"):
             object.__setattr__(self, name, value)
         elif name == "pk":
             object.__setattr__(self, self.Meta.pkname, value)
@@ -126,12 +130,12 @@ class NewBaseModel(
             super().__setattr__(name, value)
 
     def __getattribute__(self, item: str) -> Any:
-        if item in ("_orm_id", "_orm_saved", "_orm", "__fields__"):
+        if item in ("_orm_id", "_orm_saved", "_orm", "__fields__", "_related_names", "_props"):
             return object.__getattribute__(self, item)
-        if item != "extract_related_names" and item in self.extract_related_names():
-            return self._extract_related_model_instead_of_field(item)
         if item == "pk":
             return self.__dict__.get(self.Meta.pkname, None)
+        if item != "extract_related_names" and item in self.extract_related_names():
+            return self._extract_related_model_instead_of_field(item)
         if item != "__fields__" and item in self.__fields__:
             value = self.__dict__.get(item, None)
             value = self._convert_json(item, value, "loads")
@@ -186,12 +190,16 @@ class NewBaseModel(
         include: Union["AbstractSetIntStr", "MappingIntStrAny"] = None,
         exclude: Union["AbstractSetIntStr", "MappingIntStrAny"] = None,
     ) -> List[str]:
+        if isinstance(cls._props, list):
+            props = cls._props
+        else:
             props = [
                 prop
                 for prop in dir(cls)
                 if isinstance(getattr(cls, prop), property)
                 and prop not in ("__values__", "__fields__", "fields", "pk_column")
             ]
+            cls._props = props
         if include:
             props = [prop for prop in props if prop in include]
         if exclude:

ormar/relations/__init__.py

@@ -3,7 +3,6 @@ from ormar.relations.relation import Relation, RelationType
 from ormar.relations.relation_manager import RelationsManager
 from ormar.relations.utils import (
     get_relations_sides_and_names,
-    register_missing_relation,
 )
 
 __all__ = [
@@ -11,6 +10,5 @@ __all__ = [
     "Relation",
     "RelationsManager",
     "RelationType",
-    "register_missing_relation",
     "get_relations_sides_and_names",
 ]

ormar/relations/relation_manager.py

@@ -7,7 +7,6 @@ from ormar.fields.many_to_many import ManyToManyField
 from ormar.relations.relation import Relation, RelationType
 from ormar.relations.utils import (
     get_relations_sides_and_names,
-    register_missing_relation,
 )
 
 if TYPE_CHECKING:  # pragma no cover
@@ -42,8 +41,6 @@ class RelationsManager:
             to=field.to,
             through=getattr(field, "through", None),
         )
-        if field.name not in self._related_names:
-            self._related_names.append(field.name)
 
     def __contains__(self, item: str) -> bool:
         return item in self._related_names
@@ -69,8 +66,9 @@ class RelationsManager:
         )
 
         parent_relation = parent._orm._get(child_name)
-        if not parent_relation:
+        if parent_relation:
-            parent_relation = register_missing_relation(parent, child, child_name)
+            # print('missing', child_name)
+            # parent_relation = register_missing_relation(parent, child, child_name)
             parent_relation.add(child)  # type: ignore
 
         child_relation = child._orm._get(to_name)

ormar/relations/relation_proxy.py

@@ -72,6 +72,4 @@ class RelationProxy(list):
         if self.relation._type == ormar.RelationType.MULTIPLE:
             await self.queryset_proxy.create_through_instance(item)
         rel_name = item.resolve_relation_name(item, self._owner)
-        if rel_name not in item._orm:  # pragma nocover
-            item._orm._add_relation(item.Meta.model_fields[rel_name])
         setattr(item, rel_name, self._owner)

ormar/relations/utils.py

@@ -1,26 +1,13 @@
-from typing import Optional, TYPE_CHECKING, Tuple, Type
+from typing import TYPE_CHECKING, Tuple, Type
 from weakref import proxy
 
-import ormar
 from ormar.fields import BaseField
 from ormar.fields.many_to_many import ManyToManyField
-from ormar.relations import Relation
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
 
 
-def register_missing_relation(
-    parent: "Model", child: "Model", child_name: str
-) -> Optional[Relation]:
-    ormar.models.expand_reverse_relationships(child.__class__)
-    name = parent.resolve_relation_name(parent, child)
-    field = parent.Meta.model_fields[name]
-    parent._orm._add_relation(field)
-    parent_relation = parent._orm._get(child_name)
-    return parent_relation
-
-
 def get_relations_sides_and_names(
         to_field: Type[BaseField],
         parent: "Model",

requirements.txt

@@ -4,6 +4,7 @@ databases[mysql]
 pydantic
 sqlalchemy
 typing_extensions
+orjson
 
 # Async database drivers
 aiomysql
@@ -34,3 +35,6 @@ flake8-variables-names
 flake8-cognitive-complexity
 flake8-functions
 flake8-expression-complexity
+
+# Performance testing
+yappi

setup.py

@@ -42,7 +42,7 @@ setup(
     version=get_version(PACKAGE),
     url=URL,
     license="MIT",
-    description="An simple async ORM with fastapi in mind and pydantic validation.",
+    description="A simple async ORM with fastapi in mind and pydantic validation.",
     long_description=get_long_description(),
     long_description_content_type="text/markdown",
     keywords=['orm', 'sqlalchemy', 'fastapi', 'pydantic', 'databases', 'async', 'alembic'],
@@ -56,6 +56,7 @@ setup(
         "postgresql": ["asyncpg", "psycopg2"],
         "mysql": ["aiomysql", "pymysql"],
         "sqlite": ["aiosqlite"],
+        "orjson": ["orjson"]
     },
     classifiers=[
         "Development Status :: 3 - Alpha",

tests/test_new_annotation_style.py

@@ -1,374 +0,0 @@
-from typing import Optional
-
-import databases
-import pytest
-import sqlalchemy
-
-import ormar
-from ormar.exceptions import NoMatch, MultipleMatches, RelationshipInstanceError
-from tests.settings import DATABASE_URL
-
-database = databases.Database(DATABASE_URL, force_rollback=True)
-metadata = sqlalchemy.MetaData()
-
-
-class Album(ormar.Model):
-    class Meta:
-        tablename = "albums"
-        metadata = metadata
-        database = database
-
-    id: int = ormar.Integer(primary_key=True)
-    name: str = ormar.String(max_length=100)
-
-
-class Track(ormar.Model):
-    class Meta:
-        tablename = "tracks"
-        metadata = metadata
-        database = database
-
-    id: int = ormar.Integer(primary_key=True)
-    album: Optional[Album] = ormar.ForeignKey(Album)
-    title: str = ormar.String(max_length=100)
-    position: int = ormar.Integer()
-
-
-class Cover(ormar.Model):
-    class Meta:
-        tablename = "covers"
-        metadata = metadata
-        database = database
-
-    id: int = ormar.Integer(primary_key=True)
-    album: Optional[Album] = ormar.ForeignKey(Album, related_name="cover_pictures")
-    title: str = ormar.String(max_length=100)
-
-
-class Organisation(ormar.Model):
-    class Meta:
-        tablename = "org"
-        metadata = metadata
-        database = database
-
-    id: int = ormar.Integer(primary_key=True)
-    ident: str = ormar.String(max_length=100, choices=["ACME Ltd", "Other ltd"])
-
-
-class Team(ormar.Model):
-    class Meta:
-        tablename = "teams"
-        metadata = metadata
-        database = database
-
-    id: int = ormar.Integer(primary_key=True)
-    org: Optional[Organisation] = ormar.ForeignKey(Organisation)
-    name: str = ormar.String(max_length=100)
-
-
-class Member(ormar.Model):
-    class Meta:
-        tablename = "members"
-        metadata = metadata
-        database = database
-
-    id: int = ormar.Integer(primary_key=True)
-    team: Optional[Team] = ormar.ForeignKey(Team)
-    email: str = ormar.String(max_length=100)
-
-
-@pytest.fixture(autouse=True, scope="module")
-def create_test_database():
-    engine = sqlalchemy.create_engine(DATABASE_URL)
-    metadata.drop_all(engine)
-    metadata.create_all(engine)
-    yield
-    metadata.drop_all(engine)
-
-
-@pytest.mark.asyncio
-async def test_wrong_query_foreign_key_type():
-    async with database:
-        with pytest.raises(RelationshipInstanceError):
-            Track(title="The Error", album="wrong_pk_type")
-
-
-@pytest.mark.asyncio
-async def test_setting_explicitly_empty_relation():
-    async with database:
-        track = Track(album=None, title="The Bird", position=1)
-        assert track.album is None
-
-
-@pytest.mark.asyncio
-async def test_related_name():
-    async with database:
-        async with database.transaction(force_rollback=True):
-            album = await Album.objects.create(name="Vanilla")
-            await Cover.objects.create(album=album, title="The cover file")
-            assert len(album.cover_pictures) == 1
-
-
-@pytest.mark.asyncio
-async def test_model_crud():
-    async with database:
-        async with database.transaction(force_rollback=True):
-            album = Album(name="Jamaica")
-            await album.save()
-            track1 = Track(album=album, title="The Bird", position=1)
-            track2 = Track(album=album, title="Heart don't stand a chance", position=2)
-            track3 = Track(album=album, title="The Waters", position=3)
-            await track1.save()
-            await track2.save()
-            await track3.save()
-
-            track = await Track.objects.get(title="The Bird")
-            assert track.album.pk == album.pk
-            assert isinstance(track.album, ormar.Model)
-            assert track.album.name is None
-            await track.album.load()
-            assert track.album.name == "Jamaica"
-
-            assert len(album.tracks) == 3
-            assert album.tracks[1].title == "Heart don't stand a chance"
-
-            album1 = await Album.objects.get(name="Jamaica")
-            assert album1.pk == album.pk
-            assert album1.tracks == []
-
-            await Track.objects.create(
-                album={"id": track.album.pk}, title="The Bird2", position=4
-            )
-
-
-@pytest.mark.asyncio
-async def test_select_related():
-    async with database:
-        async with database.transaction(force_rollback=True):
-            album = Album(name="Malibu")
-            await album.save()
-            track1 = Track(album=album, title="The Bird", position=1)
-            track2 = Track(album=album, title="Heart don't stand a chance", position=2)
-            track3 = Track(album=album, title="The Waters", position=3)
-            await track1.save()
-            await track2.save()
-            await track3.save()
-
-            fantasies = Album(name="Fantasies")
-            await fantasies.save()
-            track4 = Track(album=fantasies, title="Help I'm Alive", position=1)
-            track5 = Track(album=fantasies, title="Sick Muse", position=2)
-            track6 = Track(album=fantasies, title="Satellite Mind", position=3)
-            await track4.save()
-            await track5.save()
-            await track6.save()
-
-            track = await Track.objects.select_related("album").get(title="The Bird")
-            assert track.album.name == "Malibu"
-
-            tracks = await Track.objects.select_related("album").all()
-            assert len(tracks) == 6
-
-
-@pytest.mark.asyncio
-async def test_model_removal_from_relations():
-    async with database:
-        async with database.transaction(force_rollback=True):
-            album = Album(name="Chichi")
-            await album.save()
-            track1 = Track(album=album, title="The Birdman", position=1)
-            track2 = Track(album=album, title="Superman", position=2)
-            track3 = Track(album=album, title="Wonder Woman", position=3)
-            await track1.save()
-            await track2.save()
-            await track3.save()
-
-            assert len(album.tracks) == 3
-            await album.tracks.remove(track1)
-            assert len(album.tracks) == 2
-            assert track1.album is None
-
-            await track1.update()
-            track1 = await Track.objects.get(title="The Birdman")
-            assert track1.album is None
-
-            await album.tracks.add(track1)
-            assert len(album.tracks) == 3
-            assert track1.album == album
-
-            await track1.update()
-            track1 = await Track.objects.select_related("album__tracks").get(
-                title="The Birdman"
-            )
-            album = await Album.objects.select_related("tracks").get(name="Chichi")
-            assert track1.album == album
-
-            track1.remove(album)
-            assert track1.album is None
-            assert len(album.tracks) == 2
-
-            track2.remove(album)
-            assert track2.album is None
-            assert len(album.tracks) == 1
-
-
-@pytest.mark.asyncio
-async def test_fk_filter():
-    async with database:
-        async with database.transaction(force_rollback=True):
-            malibu = Album(name="Malibu%")
-            await malibu.save()
-            await Track.objects.create(album=malibu, title="The Bird", position=1)
-            await Track.objects.create(
-                album=malibu, title="Heart don't stand a chance", position=2
-            )
-            await Track.objects.create(album=malibu, title="The Waters", position=3)
-
-            fantasies = await Album.objects.create(name="Fantasies")
-            await Track.objects.create(
-                album=fantasies, title="Help I'm Alive", position=1
-            )
-            await Track.objects.create(album=fantasies, title="Sick Muse", position=2)
-            await Track.objects.create(
-                album=fantasies, title="Satellite Mind", position=3
-            )
-
-            tracks = (
-                await Track.objects.select_related("album")
-                .filter(album__name="Fantasies")
-                .all()
-            )
-            assert len(tracks) == 3
-            for track in tracks:
-                assert track.album.name == "Fantasies"
-
-            tracks = (
-                await Track.objects.select_related("album")
-                .filter(album__name__icontains="fan")
-                .all()
-            )
-            assert len(tracks) == 3
-            for track in tracks:
-                assert track.album.name == "Fantasies"
-
-            tracks = await Track.objects.filter(album__name__contains="Fan").all()
-            assert len(tracks) == 3
-            for track in tracks:
-                assert track.album.name == "Fantasies"
-
-            tracks = await Track.objects.filter(album__name__contains="Malibu%").all()
-            assert len(tracks) == 3
-
-            tracks = (
-                await Track.objects.filter(album=malibu).select_related("album").all()
-            )
-            assert len(tracks) == 3
-            for track in tracks:
-                assert track.album.name == "Malibu%"
-
-            tracks = await Track.objects.select_related("album").all(album=malibu)
-            assert len(tracks) == 3
-            for track in tracks:
-                assert track.album.name == "Malibu%"
-
-
-@pytest.mark.asyncio
-async def test_multiple_fk():
-    async with database:
-        async with database.transaction(force_rollback=True):
-            acme = await Organisation.objects.create(ident="ACME Ltd")
-            red_team = await Team.objects.create(org=acme, name="Red Team")
-            blue_team = await Team.objects.create(org=acme, name="Blue Team")
-            await Member.objects.create(team=red_team, email="a@example.org")
-            await Member.objects.create(team=red_team, email="b@example.org")
-            await Member.objects.create(team=blue_team, email="c@example.org")
-            await Member.objects.create(team=blue_team, email="d@example.org")
-
-            other = await Organisation.objects.create(ident="Other ltd")
-            team = await Team.objects.create(org=other, name="Green Team")
-            await Member.objects.create(team=team, email="e@example.org")
-
-            members = (
-                await Member.objects.select_related("team__org")
-                .filter(team__org__ident="ACME Ltd")
-                .all()
-            )
-            assert len(members) == 4
-            for member in members:
-                assert member.team.org.ident == "ACME Ltd"
-
-
-@pytest.mark.asyncio
-async def test_wrong_choices():
-    async with database:
-        async with database.transaction(force_rollback=True):
-            with pytest.raises(ValueError):
-                await Organisation.objects.create(ident="Test 1")
-
-
-@pytest.mark.asyncio
-async def test_pk_filter():
-    async with database:
-        async with database.transaction(force_rollback=True):
-            fantasies = await Album.objects.create(name="Test")
-            track = await Track.objects.create(
-                album=fantasies, title="Test1", position=1
-            )
-            await Track.objects.create(album=fantasies, title="Test2", position=2)
-            await Track.objects.create(album=fantasies, title="Test3", position=3)
-            tracks = (
-                await Track.objects.select_related("album").filter(pk=track.pk).all()
-            )
-            assert len(tracks) == 1
-
-            tracks = (
-                await Track.objects.select_related("album")
-                .filter(position=2, album__name="Test")
-                .all()
-            )
-            assert len(tracks) == 1
-
-
-@pytest.mark.asyncio
-async def test_limit_and_offset():
-    async with database:
-        async with database.transaction(force_rollback=True):
-            fantasies = await Album.objects.create(name="Limitless")
-            await Track.objects.create(
-                id=None, album=fantasies, title="Sample", position=1
-            )
-            await Track.objects.create(album=fantasies, title="Sample2", position=2)
-            await Track.objects.create(album=fantasies, title="Sample3", position=3)
-
-            tracks = await Track.objects.limit(1).all()
-            assert len(tracks) == 1
-            assert tracks[0].title == "Sample"
-
-            tracks = await Track.objects.limit(1).offset(1).all()
-            assert len(tracks) == 1
-            assert tracks[0].title == "Sample2"
-
-
-@pytest.mark.asyncio
-async def test_get_exceptions():
-    async with database:
-        async with database.transaction(force_rollback=True):
-            fantasies = await Album.objects.create(name="Test")
-
-            with pytest.raises(NoMatch):
-                await Album.objects.get(name="Test2")
-
-            await Track.objects.create(album=fantasies, title="Test1", position=1)
-            await Track.objects.create(album=fantasies, title="Test2", position=2)
-            await Track.objects.create(album=fantasies, title="Test3", position=3)
-            with pytest.raises(MultipleMatches):
-                await Track.objects.select_related("album").get(album=fantasies)
-
-
-@pytest.mark.asyncio
-async def test_wrong_model_passed_as_fk():
-    async with database:
-        async with database.transaction(force_rollback=True):
-            with pytest.raises(RelationshipInstanceError):
-                org = await Organisation.objects.create(ident="ACME Ltd")
-                await Track.objects.create(album=org, title="Test1", position=1)