Draft 0.11.0 (#594)

* fix for #584

* fix for #580

* fix typing

* connect to db in test

* refactor test

* remove async mark

* connect client

* fix mypy

* fix mypy

* update deps

* check py3.10?

* remove py3.6, bump version

.github/workflows/test-package.yml

@@ -17,7 +17,7 @@ jobs:
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != 'collerek/ormar'
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9, "3.10"]
       fail-fast: false
     services:
       mysql:

docs/releases.md

@@ -1,3 +1,21 @@
+# 0.11.0
+
+## ✨ Breaking Changes
+
+* Dropped support for python 3.6
+* `Queryset.get_or_create` returns now a tuple with model and bool value indicating if the model was created (by @MojixCoder - thanks!) [#554](https://github.com/collerek/ormar/pull/554)
+* `Queryset.count()` now counts the number of distinct parent model rows by default, counting all rows is possible by setting `distinct=False` (by @erichaydel - thanks) [#588](https://github.com/collerek/ormar/pull/588)
+
+## ✨ Features
+
+* Added support for python 3.10
+
+## 🐛 Fixes
+
+* Fix inconsistent `JSON` fields behaviour in `save` and `bulk_create` [#584](https://github.com/collerek/ormar/issues/584)
+* Fix maximum recursion error [#580](https://github.com/collerek/ormar/pull/580)
+
+
 # 0.10.25
 
 ## ✨ Features

ormar/models/mixins/save_mixin.py

@@ -56,7 +56,6 @@ class SavePrepareMixin(RelationMixin, AliasMixin):
         new_kwargs = cls.substitute_models_with_pks(new_kwargs)
         new_kwargs = cls.populate_default_values(new_kwargs)
         new_kwargs = cls.reconvert_str_to_bytes(new_kwargs)
-        new_kwargs = cls.dump_all_json_fields_to_str(new_kwargs)
         new_kwargs = cls.translate_columns_to_aliases(new_kwargs)
         return new_kwargs
 

ormar/models/newbasemodel.py

@@ -23,8 +23,7 @@ import databases
 import pydantic
 import sqlalchemy
 
-from ormar.fields.parsers import encode_json
+
-from ormar.models.utils import Extra
 from pydantic import BaseModel
 
 
@@ -32,6 +31,7 @@ import ormar  # noqa I100
 from ormar.exceptions import ModelError, ModelPersistenceError
 from ormar.fields import BaseField
 from ormar.fields.foreign_key import ForeignKeyField
+from ormar.fields.parsers import encode_json
 from ormar.models.helpers import register_relation_in_alias_manager
 from ormar.models.helpers.relations import expand_reverse_relationship
 from ormar.models.helpers.sqlalchemy import (
@@ -40,6 +40,7 @@ from ormar.models.helpers.sqlalchemy import (
 )
 from ormar.models.metaclass import ModelMeta, ModelMetaclass
 from ormar.models.modelproxy import ModelTableProxy
+from ormar.models.utils import Extra
 from ormar.queryset.utils import translate_list_to_dict
 from ormar.relations.alias_manager import AliasManager
 from ormar.relations.relation_manager import RelationsManager

ormar/queryset/field_accessor.py

@@ -1,4 +1,4 @@
-from typing import Any, TYPE_CHECKING, Type
+from typing import Any, TYPE_CHECKING, Type, cast
 
 from ormar.queryset.actions import OrderAction
 from ormar.queryset.actions.filter_action import METHODS_TO_OPERATORS
@@ -45,11 +45,17 @@ class FieldAccessor:
         :return: FieldAccessor for field or nested model
         :rtype: ormar.queryset.field_accessor.FieldAccessor
         """
-        if self._field and item == self._field.name:
+        if (
+            object.__getattribute__(self, "_field")
+            and item == object.__getattribute__(self, "_field").name
+        ):
             return self._field
 
-        if self._model and item in self._model.Meta.model_fields:
+        if (
-            field = self._model.Meta.model_fields[item]
+            object.__getattribute__(self, "_model")
+            and item in object.__getattribute__(self, "_model").Meta.model_fields
+        ):
+            field = cast("Model", self._model).Meta.model_fields[item]
             if field.is_relation:
                 return FieldAccessor(
                     source_model=self._source_model,

ormar/queryset/queryset.py

@@ -682,9 +682,11 @@ class QuerySet(Generic[T]):
         """
         Returns number of rows matching the given criteria
         (applied with `filter` and `exclude` if set before).
-        If `distinct` is `True` (the default), this will return the number of primary rows selected. If `False`,
+        If `distinct` is `True` (the default), this will return
+        the number of primary rows selected. If `False`,
         the count will be the total number of rows returned
-        (including extra rows for `one-to-many` or `many-to-many` left `select_related` table joins).
+        (including extra rows for `one-to-many` or `many-to-many`
+        left `select_related` table joins).
         `False` is the legacy (buggy) behavior for workflows that depend on it.
 
         :param distinct: flag if the primary table rows should be distinct or not
@@ -695,7 +697,9 @@ class QuerySet(Generic[T]):
         expr = self.build_select_expression().alias("subquery_for_count")
         expr = sqlalchemy.func.count().select().select_from(expr)
         if distinct:
-            expr_distinct = expr.group_by(self.model_meta.pkname).alias("subquery_for_group")
+            expr_distinct = expr.group_by(self.model_meta.pkname).alias(
+                "subquery_for_group"
+            )
             expr = sqlalchemy.func.count().select().select_from(expr_distinct)
         return await self.database.fetch_val(expr)
 

ormar/relations/querysetproxy.py

@@ -198,9 +198,11 @@ class QuerysetProxy(Generic[T]):
         """
         Returns number of rows matching the given criteria
         (applied with `filter` and `exclude` if set before).
-        If `distinct` is `True` (the default), this will return the number of primary rows selected. If `False`,
+        If `distinct` is `True` (the default), this will return
+        the number of primary rows selected. If `False`,
         the count will be the total number of rows returned
-        (including extra rows for `one-to-many` or `many-to-many` left `select_related` table joins).
+        (including extra rows for `one-to-many` or `many-to-many`
+        left `select_related` table joins).
         `False` is the legacy (buggy) behavior for workflows that depend on it.
 
         Actual call delegated to QuerySet.

pyproject.toml

@@ -3,7 +3,7 @@ name = "ormar"
 
 [tool.poetry]
 name = "ormar"
-version = "0.10.25"
+version = "0.11.0"
 description = "A simple async ORM with fastapi in mind and pydantic validation."
 authors = ["Radosław Drążkiewicz <collerek@gmail.com>"]
 license = "MIT"
@@ -32,16 +32,16 @@ classifiers = [
     "Topic :: Internet :: WWW/HTTP",
     "Framework :: AsyncIO",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.6",
     "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3 :: Only",
 ]
 
 
 [tool.poetry.dependencies]
-python = "^3.6.2"
+python = "^3.7.0"
 databases = ">=0.3.2,!=0.5.0,!=0.5.1,!=0.5.2,!=0.5.3,<=0.5.5"
 pydantic = ">=1.6.1,!=1.7,!=1.7.1,!=1.7.2,!=1.7.3,!=1.8,!=1.8.1,<=1.9.1"
 SQLAlchemy = ">=1.3.18,<=1.4.31"
@@ -54,7 +54,6 @@ cryptography = { version = ">=35,<37", optional = true }
 [tool.poetry.dependencies.orjson]
 version = ">=3.6.4"
 optional = true
-python = ">=3.7"
 
 [tool.poetry.dependencies.typing-extensions]
 version = "^3.7"
@@ -106,13 +105,12 @@ types-aiofiles = "^0.8.3"
 types-pkg-resources = "^0.1.3"
 types-requests = "^2.26.1"
 types-toml = "^0.10.1"
-types-dataclasses = { version = "^0.6.1", markers = "python_version < '3.7'" }
 
 # Documantation
 mkdocs = "^1.2.3"
 mkdocs-material = ">=8.1.2,<8.3"
 mkdocs-material-extensions = "^1.0.3"
-pydoc-markdown = { version = "^4.5.0", markers = "python_version > '3.7'" }
+pydoc-markdown = "^4.5.0"
 dataclasses = { version = ">=0.6.0,<0.8 || >0.8,<1.0.0" }
 
 # Performance testing

tests/test_fastapi/test_recursion_error.py

@@ -0,0 +1,149 @@
+import json
+from datetime import datetime
+import uuid
+from typing import List
+
+import databases
+import pytest
+import sqlalchemy
+from fastapi import Depends, FastAPI
+from pydantic import BaseModel, Json
+from starlette.testclient import TestClient
+
+import ormar
+from tests.settings import DATABASE_URL
+
+router = FastAPI()
+metadata = sqlalchemy.MetaData()
+database = databases.Database(DATABASE_URL, force_rollback=True)
+router.state.database = database
+
+headers = {"content-type": "application/json"}
+
+
+@router.on_event("startup")
+async def startup() -> None:
+    database_ = router.state.database
+    if not database_.is_connected:
+        await database_.connect()
+
+
+@router.on_event("shutdown")
+async def shutdown() -> None:
+    database_ = router.state.database
+    if database_.is_connected:
+        await database_.disconnect()
+
+
+class User(ormar.Model):
+    """
+    The user model
+    """
+
+    id: uuid.UUID = ormar.UUID(primary_key=True, default=uuid.uuid4)
+    email: str = ormar.String(unique=True, max_length=100)
+    username: str = ormar.String(unique=True, max_length=100)
+    password: str = ormar.String(unique=True, max_length=100)
+    verified: bool = ormar.Boolean(default=False)
+    verify_key: str = ormar.String(unique=True, max_length=100, nullable=True)
+    created_at: datetime = ormar.DateTime(default=datetime.now())
+
+    class Meta:
+        tablename = "users"
+        metadata = metadata
+        database = database
+
+
+class UserSession(ormar.Model):
+    """
+    The user session model
+    """
+
+    id: uuid.UUID = ormar.UUID(primary_key=True, default=uuid.uuid4)
+    user: User = ormar.ForeignKey(User)
+    session_key: str = ormar.String(unique=True, max_length=64)
+    created_at: datetime = ormar.DateTime(default=datetime.now())
+
+    class Meta:
+        tablename = "user_sessions"
+        metadata = metadata
+        database = database
+
+
+class QuizAnswer(BaseModel):
+    right: bool
+    answer: str
+
+
+class QuizQuestion(BaseModel):
+    question: str
+    answers: List[QuizAnswer]
+
+
+class QuizInput(BaseModel):
+    title: str
+    description: str
+    questions: List[QuizQuestion]
+
+
+class Quiz(ormar.Model):
+    id: uuid.UUID = ormar.UUID(primary_key=True, default=uuid.uuid4)
+    title: str = ormar.String(max_length=100)
+    description: str = ormar.String(max_length=300, nullable=True)
+    created_at: datetime = ormar.DateTime(default=datetime.now())
+    updated_at: datetime = ormar.DateTime(default=datetime.now())
+    user_id: uuid.UUID = ormar.UUID(foreign_key=User.id)
+    questions: Json = ormar.JSON(nullable=False)
+
+    class Meta:
+        tablename = "quiz"
+        metadata = metadata
+        database = database
+
+
+@pytest.fixture(autouse=True, scope="module")
+def create_test_database():
+    engine = sqlalchemy.create_engine(DATABASE_URL)
+    metadata.create_all(engine)
+    yield
+    metadata.drop_all(engine)
+
+
+async def get_current_user():
+    return await User(email="mail@example.com", username="aa", password="pass").save()
+
+
+@router.post("/create", response_model=Quiz)
+async def create_quiz_lol(
+    quiz_input: QuizInput, user: User = Depends(get_current_user)
+):
+    quiz = Quiz(**quiz_input.dict(), user_id=user.id)
+    return await quiz.save()
+
+
+def test_quiz_creation():
+    client = TestClient(app=router)
+    with client as client:
+        payload = {
+            "title": "Some test question",
+            "description": "A description",
+            "questions": [
+                {
+                    "question": "Is ClassQuiz cool?",
+                    "answers": [
+                        {"right": True, "answer": "Yes"},
+                        {"right": False, "answer": "No"},
+                    ],
+                },
+                {
+                    "question": "Do you like open source?",
+                    "answers": [
+                        {"right": True, "answer": "Yes"},
+                        {"right": False, "answer": "No"},
+                        {"right": False, "answer": "Maybe"},
+                    ],
+                },
+            ],
+        }
+        response = client.post("/create", data=json.dumps(payload))
+        assert response.status_code == 200

tests/test_queries/test_aggr_functions.py

@@ -176,6 +176,7 @@ async def test_queryset_method():
             year=1930, title="Book 3"
         )
 
+
 @pytest.mark.asyncio
 async def test_count_method():
     async with database:

tests/test_queries/test_queryset_level_methods.py

@@ -4,6 +4,7 @@ import databases
 import pydantic
 import pytest
 import sqlalchemy
+from pydantic import Json
 
 import ormar
 from ormar import QuerySet
@@ -68,7 +69,7 @@ class Note(ormar.Model):
 
 
 class ItemConfig(ormar.Model):
-    class Meta:
+    class Meta(ormar.ModelMeta):
         metadata = metadata
         database = database
         tablename = "item_config"
@@ -98,6 +99,16 @@ class Customer(ormar.Model):
     name: str = ormar.String(max_length=32)
 
 
+class JsonTestModel(ormar.Model):
+    class Meta(ormar.ModelMeta):
+        metadata = metadata
+        database = database
+        tablename = "test_model"
+
+    id: int = ormar.Integer(primary_key=True)
+    json_field: Json = ormar.JSON()
+
+
 @pytest.fixture(autouse=True, scope="module")
 def create_test_database():
     engine = sqlalchemy.create_engine(DATABASE_URL)
@@ -268,6 +279,36 @@ async def test_bulk_create():
         assert len(completed) == 2
 
 
+@pytest.mark.asyncio
+async def test_bulk_create_json_field():
+    async with database:
+        json_value = {"a": 1}
+        test_model_1 = JsonTestModel(id=1, json_field=json_value)
+        test_model_2 = JsonTestModel(id=2, json_field=json_value)
+
+        # store one with .save() and the other with .bulk_create()
+        await test_model_1.save()
+        await JsonTestModel.objects.bulk_create([test_model_2])
+
+        # refresh from the database
+        await test_model_1.load()
+        await test_model_2.load()
+
+        assert test_model_1.json_field == test_model_2.json_field  # True
+
+        # try to query the json field
+        table = JsonTestModel.Meta.table
+        query = table.select().where(table.c.json_field["a"].as_integer() == 1)
+        res = [
+            JsonTestModel.from_row(record, source_model=JsonTestModel)
+            for record in await database.fetch_all(query)
+        ]
+
+        assert test_model_1 in res
+        assert test_model_2 in res
+        assert len(res) == 2
+
+
 @pytest.mark.asyncio
 async def test_bulk_create_with_relation():
     async with database:
@@ -408,6 +449,21 @@ async def test_bulk_operations_with_json():
         items = await ItemConfig.objects.all()
         assert all(x.pairs == ["1"] for x in items)
 
+        items = await ItemConfig.objects.filter(ItemConfig.id > 1).all()
+        for item in items:
+            item.pairs = {"b": 2}
+        await ItemConfig.objects.bulk_update(items)
+        items = await ItemConfig.objects.filter(ItemConfig.id > 1).all()
+        assert all(x.pairs == {"b": 2} for x in items)
+
+        table = ItemConfig.Meta.table
+        query = table.select().where(table.c.pairs["b"].as_integer() == 2)
+        res = [
+            ItemConfig.from_row(record, source_model=ItemConfig)
+            for record in await database.fetch_all(query)
+        ]
+        assert len(res) == 2
+
 
 @pytest.mark.asyncio
 async def test_custom_queryset_cls():