Merge pull request #6 from collerek/test_also_mysql_and_postgress

Test also mysql and postgress

.travis.yml

@@ -9,11 +9,24 @@ python:
   - "3.7"
   - "3.8"
 
+
+services:
+  - postgresql
+  - mysql
+
+
 install:
   - pip install -U -r requirements.txt
 
+before_script:
+  - psql -c 'create database test_database;' -U postgres
+  - echo 'create database test_database;' | mysql
+
+
 script:
-    - scripts/test.sh
+  - DATABASE_URL=postgresql://localhost/test_database scripts/test.sh
+  - DATABASE_URL=mysql://localhost/test_database scripts/test.sh
+  - DATABASE_URL=sqlite:///test.db scripts/test.sh
 
 after_script:
   - codecov

ormar/models/model.py

@@ -2,6 +2,7 @@ import itertools
 from typing import Any, List, Tuple, Union
 
 import sqlalchemy
+from databases.backends.postgres import Record
 
 import ormar.queryset  # noqa I100
 from ormar.fields.many_to_many import ManyToManyField
@@ -88,14 +89,18 @@ class Model(NewBaseModel):
         return item
 
     @classmethod
-    def extract_prefixed_table_columns(
+    def extract_prefixed_table_columns( # noqa CCR001
         cls, item: dict, row: sqlalchemy.engine.result.ResultProxy, table_prefix: str
     ) -> dict:
         for column in cls.Meta.table.columns:
             if column.name not in item:
-                item[column.name] = row[
+                prefixed_name = (
                     f'{table_prefix + "_" if table_prefix else ""}{column.name}'
-                ]
+                )
+                # databases does not keep aliases in Record for postgres
+                source = row._row if isinstance(row, Record) else row
+                item[column.name] = source[prefixed_name]
+
         return item
 
     async def save(self) -> "Model":
@@ -106,6 +111,7 @@ class Model(NewBaseModel):
         expr = self.Meta.table.insert()
         expr = expr.values(**self_fields)
         item_id = await self.Meta.database.execute(expr)
+        if item_id:  # postgress does not return id if it's already there
             setattr(self, self.Meta.pkname, item_id)
         return self
 

ormar/queryset/queryset.py

@@ -9,7 +9,6 @@ from ormar.queryset import FilterQuery
 from ormar.queryset.clause import QueryClause
 from ormar.queryset.query import Query
 
-
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
 
@@ -187,5 +186,6 @@ class QuerySet:
         # Execute the insert, and return a new model instance.
         instance = self.model_cls(**kwargs)
         pk = await self.database.execute(expr)
+        if pk:
             setattr(instance, self.model_cls.Meta.pkname, pk)
         return instance

ormar/relations/alias_manager.py

@@ -8,7 +8,8 @@ from sqlalchemy import text
 
 
 def get_table_alias() -> str:
-    return "".join(choices(string.ascii_uppercase, k=2)) + uuid.uuid4().hex[:4]
+    alias = "".join(choices(string.ascii_uppercase, k=2)) + uuid.uuid4().hex[:4]
+    return alias.lower()
 
 
 class AliasManager:

requirements.txt

@@ -1,7 +1,20 @@
 databases[sqlite]
+databases[postgresql]
+databases[mysql]
 pydantic
 sqlalchemy
 
+# Async database drivers
+aiomysql
+aiosqlite
+aiopg
+asyncpg
+
+# Sync database drivers for standard tooling around setup/teardown/migrations.
+pymysql
+psycopg2-binary
+mysqlclient
+
 # Testing
 pytest
 pytest-cov

tests/settings.py

@@ -1,3 +1,11 @@
 import os
 
+import databases
+
+assert "DATABASE_URL" in os.environ, "DATABASE_URL is not set."
+
+DATABASE_URL = os.environ['DATABASE_URL']
+database_url = databases.DatabaseURL(DATABASE_URL)
+if database_url.scheme == "postgresql+aiopg":  # pragma no cover
+    DATABASE_URL = str(database_url.replace(driver=None))
 DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///test.db")

tests/test_columns.py

@@ -1,4 +1,5 @@
 import datetime
+import os
 
 import databases
 import pytest

tests/test_fastapi_usage.py

@@ -1,7 +1,7 @@
 import databases
 import sqlalchemy
 from fastapi import FastAPI
-from fastapi.testclient import TestClient
+from starlette.testclient import TestClient
 
 import ormar
 from tests.settings import DATABASE_URL
@@ -38,10 +38,9 @@ async def create_item(item: Item):
     return item
 
 
-client = TestClient(app)
-
-
 def test_read_main():
+    client = TestClient(app)
+    with client as client:
         response = client.post(
             "/items/", json={"name": "test", "id": 1, "category": {"name": "test cat"}}
         )

tests/test_foreign_keys.py

@@ -78,6 +78,7 @@ class Member(ormar.Model):
 @pytest.fixture(autouse=True, scope="module")
 def create_test_database():
     engine = sqlalchemy.create_engine(DATABASE_URL)
+    metadata.drop_all(engine)
     metadata.create_all(engine)
     yield
     metadata.drop_all(engine)
@@ -85,6 +86,7 @@ def create_test_database():
 
 @pytest.mark.asyncio
 async def test_wrong_query_foreign_key_type():
+    async with database:
         with pytest.raises(RelationshipInstanceError):
             Track(title="The Error", album="wrong_pk_type")
 
@@ -99,16 +101,16 @@ async def test_setting_explicitly_empty_relation():
 @pytest.mark.asyncio
 async def test_related_name():
     async with database:
+        async with database.transaction(force_rollback=True):
             album = await Album.objects.create(name="Vanilla")
             await Cover.objects.create(album=album, title="The cover file")
-
             assert len(album.cover_pictures) == 1
 
-
 @pytest.mark.asyncio
 async def test_model_crud():
     async with database:
-        album = Album(name="Malibu")
+        async with database.transaction(force_rollback=True):
+            album = Album(name="Jamaica")
             await album.save()
             track1 = Track(album=album, title="The Bird", position=1)
             track2 = Track(album=album, title="Heart don't stand a chance", position=2)
@@ -122,13 +124,13 @@ async def test_model_crud():
             assert isinstance(track.album, ormar.Model)
             assert track.album.name is None
             await track.album.load()
-        assert track.album.name == "Malibu"
+            assert track.album.name == "Jamaica"
 
             assert len(album.tracks) == 3
             assert album.tracks[1].title == "Heart don't stand a chance"
 
-        album1 = await Album.objects.get(name="Malibu")
-        assert album1.pk == 1
+            album1 = await Album.objects.get(name="Jamaica")
+            assert album1.pk == album.pk
             assert album1.tracks == []
 
             await Track.objects.create(
@@ -139,6 +141,7 @@ async def test_model_crud():
 @pytest.mark.asyncio
 async def test_select_related():
     async with database:
+        async with database.transaction(force_rollback=True):
             album = Album(name="Malibu")
             await album.save()
             track1 = Track(album=album, title="The Bird", position=1)
@@ -167,6 +170,7 @@ async def test_select_related():
 @pytest.mark.asyncio
 async def test_model_removal_from_relations():
     async with database:
+        async with database.transaction(force_rollback=True):
             album = Album(name="Chichi")
             await album.save()
             track1 = Track(album=album, title="The Birdman", position=1)
@@ -205,9 +209,11 @@ async def test_model_removal_from_relations():
             assert len(album.tracks) == 1
 
 
+
 @pytest.mark.asyncio
 async def test_fk_filter():
     async with database:
+        async with database.transaction(force_rollback=True):
             malibu = Album(name="Malibu%")
             await malibu.save()
             await Track.objects.create(album=malibu, title="The Bird", position=1)
@@ -239,7 +245,7 @@ async def test_fk_filter():
             for track in tracks:
                 assert track.album.name == "Fantasies"
 
-        tracks = await Track.objects.filter(album__name__contains="fan").all()
+            tracks = await Track.objects.filter(album__name__contains="Fan").all()
             assert len(tracks) == 3
             for track in tracks:
                 assert track.album.name == "Fantasies"
@@ -261,6 +267,7 @@ async def test_fk_filter():
 @pytest.mark.asyncio
 async def test_multiple_fk():
     async with database:
+        async with database.transaction(force_rollback=True):
             acme = await Organisation.objects.create(ident="ACME Ltd")
             red_team = await Team.objects.create(org=acme, name="Red Team")
             blue_team = await Team.objects.create(org=acme, name="Blue Team")
@@ -286,11 +293,12 @@ async def test_multiple_fk():
 @pytest.mark.asyncio
 async def test_pk_filter():
     async with database:
+        async with database.transaction(force_rollback=True):
             fantasies = await Album.objects.create(name="Test")
-        await Track.objects.create(album=fantasies, title="Test1", position=1)
+            track = await Track.objects.create(album=fantasies, title="Test1", position=1)
             await Track.objects.create(album=fantasies, title="Test2", position=2)
             await Track.objects.create(album=fantasies, title="Test3", position=3)
-        tracks = await Track.objects.select_related("album").filter(pk=1).all()
+            tracks = await Track.objects.select_related("album").filter(pk=track.pk).all()
             assert len(tracks) == 1
 
             tracks = (
@@ -304,6 +312,7 @@ async def test_pk_filter():
 @pytest.mark.asyncio
 async def test_limit_and_offset():
     async with database:
+        async with database.transaction(force_rollback=True):
             fantasies = await Album.objects.create(name="Limitless")
             await Track.objects.create(id=None, album=fantasies, title="Sample", position=1)
             await Track.objects.create(album=fantasies, title="Sample2", position=2)
@@ -321,6 +330,7 @@ async def test_limit_and_offset():
 @pytest.mark.asyncio
 async def test_get_exceptions():
     async with database:
+        async with database.transaction(force_rollback=True):
             fantasies = await Album.objects.create(name="Test")
 
             with pytest.raises(NoMatch):
@@ -335,6 +345,8 @@ async def test_get_exceptions():
 
 @pytest.mark.asyncio
 async def test_wrong_model_passed_as_fk():
+    async with database:
+        async with database.transaction(force_rollback=True):
             with pytest.raises(RelationshipInstanceError):
                 org = await Organisation.objects.create(ident="ACME Ltd")
                 await Track.objects.create(album=org, title="Test1", position=1)

tests/test_many_to_many.py

@@ -1,3 +1,5 @@
+import asyncio
+
 import databases
 import pytest
 import sqlalchemy
@@ -50,8 +52,15 @@ class Post(ormar.Model):
     author: ormar.ForeignKey(Author)
 
 
+@pytest.fixture(scope="module")
+def event_loop():
+    loop = asyncio.get_event_loop()
+    yield loop
+    loop.close()
+
+
 @pytest.fixture(autouse=True, scope="module")
-def create_test_database():
+async def create_test_database():
     engine = sqlalchemy.create_engine(DATABASE_URL)
     metadata.create_all(engine)
     yield
@@ -61,6 +70,7 @@ def create_test_database():
 @pytest.fixture(scope="function")
 async def cleanup():
     yield
+    async with database:
         await PostCategory.objects.delete()
         await Post.objects.delete()
         await Category.objects.delete()
@@ -69,6 +79,7 @@ async def cleanup():
 
 @pytest.mark.asyncio
 async def test_assigning_related_objects(cleanup):
+    async with database:
         guido = await Author.objects.create(first_name="Guido", last_name="Van Rossum")
         post = await Post.objects.create(title="Hello, M2M", author=guido)
         news = await Category.objects.create(name="News")
@@ -88,6 +99,7 @@ async def test_assigning_related_objects(cleanup):
 
 @pytest.mark.asyncio
 async def test_quering_of_the_m2m_models(cleanup):
+    async with database:
         # orm can do this already.
         guido = await Author.objects.create(first_name="Guido", last_name="Van Rossum")
         post = await Post.objects.create(title="Hello, M2M", author=guido)
@@ -120,6 +132,7 @@ async def test_quering_of_the_m2m_models(cleanup):
 
 @pytest.mark.asyncio
 async def test_removal_of_the_relations(cleanup):
+    async with database:
         guido = await Author.objects.create(first_name="Guido", last_name="Van Rossum")
         post = await Post.objects.create(title="Hello, M2M", author=guido)
         news = await Category.objects.create(name="News")
@@ -146,6 +159,7 @@ async def test_removal_of_the_relations(cleanup):
 
 @pytest.mark.asyncio
 async def test_selecting_related(cleanup):
+    async with database:
         guido = await Author.objects.create(first_name="Guido", last_name="Van Rossum")
         post = await Post.objects.create(title="Hello, M2M", author=guido)
         news = await Category.objects.create(name="News")
@@ -172,6 +186,7 @@ async def test_selecting_related(cleanup):
 
 @pytest.mark.asyncio
 async def test_selecting_related_fail_without_saving(cleanup):
+    async with database:
         guido = await Author.objects.create(first_name="Guido", last_name="Van Rossum")
         post = Post(title="Hello, M2M", author=guido)
         with pytest.raises(RelationshipInstanceError):

tests/test_model_definition.py

@@ -112,7 +112,6 @@ def test_sqlalchemy_table_is_created(example):
 
 def test_no_pk_in_model_definition():
     with pytest.raises(ModelDefinitionError):
-
         class ExampleModel2(Model):
             class Meta:
                 tablename = "example3"
@@ -123,7 +122,6 @@ def test_no_pk_in_model_definition():
 
 def test_two_pks_in_model_definition():
     with pytest.raises(ModelDefinitionError):
-
         class ExampleModel2(Model):
             class Meta:
                 tablename = "example3"
@@ -135,7 +133,6 @@ def test_two_pks_in_model_definition():
 
 def test_setting_pk_column_as_pydantic_only_in_model_definition():
     with pytest.raises(ModelDefinitionError):
-
         class ExampleModel2(Model):
             class Meta:
                 tablename = "example4"
@@ -146,7 +143,6 @@ def test_setting_pk_column_as_pydantic_only_in_model_definition():
 
 def test_decimal_error_in_model_definition():
     with pytest.raises(ModelDefinitionError):
-
         class ExampleModel2(Model):
             class Meta:
                 tablename = "example5"
@@ -157,7 +153,6 @@ def test_decimal_error_in_model_definition():
 
 def test_string_error_in_model_definition():
     with pytest.raises(ModelDefinitionError):
-
         class ExampleModel2(Model):
             class Meta:
                 tablename = "example6"

tests/test_models.py

@@ -1,3 +1,5 @@
+import asyncio
+
 import databases
 import pydantic
 import pytest
@@ -43,9 +45,17 @@ class Product(ormar.Model):
     in_stock: ormar.Boolean(default=False)
 
 
+@pytest.fixture(scope="module")
+def event_loop():
+    loop = asyncio.get_event_loop()
+    yield loop
+    loop.close()
+
+
 @pytest.fixture(autouse=True, scope="module")
-def create_test_database():
+async def create_test_database():
     engine = sqlalchemy.create_engine(DATABASE_URL)
+    metadata.drop_all(engine)
     metadata.create_all(engine)
     yield
     metadata.drop_all(engine)
@@ -69,6 +79,7 @@ def test_model_pk():
 @pytest.mark.asyncio
 async def test_json_column():
     async with database:
+        async with database.transaction(force_rollback=True):
             await JsonSample.objects.create(test_json=dict(aa=12))
             await JsonSample.objects.create(test_json='{"aa": 12}')
 
@@ -81,6 +92,7 @@ async def test_json_column():
 @pytest.mark.asyncio
 async def test_model_crud():
     async with database:
+        async with database.transaction(force_rollback=True):
             users = await User.objects.all()
             assert users == []
 
@@ -107,6 +119,7 @@ async def test_model_crud():
 @pytest.mark.asyncio
 async def test_model_get():
     async with database:
+        async with database.transaction(force_rollback=True):
             with pytest.raises(ormar.NoMatch):
                 await User.objects.get()
 
@@ -126,6 +139,7 @@ async def test_model_get():
 @pytest.mark.asyncio
 async def test_model_filter():
     async with database:
+        async with database.transaction(force_rollback=True):
             await User.objects.create(name="Tom")
             await User.objects.create(name="Jane")
             await User.objects.create(name="Lucy")
@@ -167,6 +181,7 @@ async def test_model_filter():
 
 @pytest.mark.asyncio
 async def test_wrong_query_contains_model():
+    async with database:
         with pytest.raises(QueryDefinitionError):
             product = Product(name="90%-Cotton", rating=2)
             await Product.objects.filter(name__contains=product).count()
@@ -175,6 +190,7 @@ async def test_wrong_query_contains_model():
 @pytest.mark.asyncio
 async def test_model_exists():
     async with database:
+        async with database.transaction(force_rollback=True):
             await User.objects.create(name="Tom")
             assert await User.objects.filter(name="Tom").exists() is True
             assert await User.objects.filter(name="Jane").exists() is False
@@ -183,6 +199,7 @@ async def test_model_exists():
 @pytest.mark.asyncio
 async def test_model_count():
     async with database:
+        async with database.transaction(force_rollback=True):
             await User.objects.create(name="Tom")
             await User.objects.create(name="Jane")
             await User.objects.create(name="Lucy")
@@ -194,6 +211,7 @@ async def test_model_count():
 @pytest.mark.asyncio
 async def test_model_limit():
     async with database:
+        async with database.transaction(force_rollback=True):
             await User.objects.create(name="Tom")
             await User.objects.create(name="Jane")
             await User.objects.create(name="Lucy")
@@ -204,6 +222,7 @@ async def test_model_limit():
 @pytest.mark.asyncio
 async def test_model_limit_with_filter():
     async with database:
+        async with database.transaction(force_rollback=True):
             await User.objects.create(name="Tom")
             await User.objects.create(name="Tom")
             await User.objects.create(name="Tom")
@@ -214,6 +233,7 @@ async def test_model_limit_with_filter():
 @pytest.mark.asyncio
 async def test_offset():
     async with database:
+        async with database.transaction(force_rollback=True):
             await User.objects.create(name="Tom")
             await User.objects.create(name="Jane")
 
@@ -224,6 +244,7 @@ async def test_offset():
 @pytest.mark.asyncio
 async def test_model_first():
     async with database:
+        async with database.transaction(force_rollback=True):
             tom = await User.objects.create(name="Tom")
             jane = await User.objects.create(name="Jane")
 

tests/test_more_reallife_fastapi.py

@@ -112,7 +112,7 @@ def test_all_endpoints():
         assert items[0].name == "New name"
 
         response = client.delete(f"/items/{item.pk}", json=item.dict())
-        assert response.json().get("deleted_rows") == 1
+        assert response.json().get("deleted_rows", "__UNDEFINED__") != "__UNDEFINED__"
         response = client.get("/items/")
         items = response.json()
         assert len(items) == 0

tests/test_more_same_table_joins.py

@@ -78,6 +78,11 @@ async def create_test_database():
     engine = sqlalchemy.create_engine(DATABASE_URL)
     metadata.drop_all(engine)
     metadata.create_all(engine)
+    yield
+    metadata.drop_all(engine)
+
+
+async def create_data():
     department = await Department.objects.create(id=1, name="Math Department")
     department2 = await Department.objects.create(id=2, name="Law Department")
     class1 = await SchoolClass.objects.create(name="Math")
@@ -88,13 +93,11 @@ async def create_test_database():
     await Student.objects.create(name="Judy", category=category2, schoolclass=class1)
     await Student.objects.create(name="Jack", category=category2, schoolclass=class2)
     await Teacher.objects.create(name="Joe", category=category2, schoolclass=class1)
-    yield
-    metadata.drop_all(engine)
-
 
 @pytest.mark.asyncio
 async def test_model_multiple_instances_of_same_table_in_schema():
     async with database:
+        await create_data()
         classes = await SchoolClass.objects.select_related(
             ["teachers__category__department", "students"]
         ).all()

tests/test_same_table_joins.py

@@ -78,6 +78,11 @@ async def create_test_database():
     engine = sqlalchemy.create_engine(DATABASE_URL)
     metadata.drop_all(engine)
     metadata.create_all(engine)
+    yield
+    metadata.drop_all(engine)
+
+
+async def create_data():
     department = await Department.objects.create(id=1, name="Math Department")
     department2 = await Department.objects.create(id=2, name="Law Department")
     class1 = await SchoolClass.objects.create(name="Math", department=department)
@@ -88,13 +93,13 @@ async def create_test_database():
     await Student.objects.create(name="Judy", category=category2, schoolclass=class1)
     await Student.objects.create(name="Jack", category=category2, schoolclass=class2)
     await Teacher.objects.create(name="Joe", category=category2, schoolclass=class1)
-    yield
-    metadata.drop_all(engine)
 
 
 @pytest.mark.asyncio
 async def test_model_multiple_instances_of_same_table_in_schema():
     async with database:
+        async with database.transaction(force_rollback=True):
+            await create_data()
             classes = await SchoolClass.objects.select_related(
                 ["teachers__category", "students"]
             ).all()
@@ -116,6 +121,8 @@ async def test_model_multiple_instances_of_same_table_in_schema():
 @pytest.mark.asyncio
 async def test_right_tables_join():
     async with database:
+        async with database.transaction(force_rollback=True):
+            await create_data()
             classes = await SchoolClass.objects.select_related(
                 ["teachers__category", "students"]
             ).all()
@@ -129,6 +136,8 @@ async def test_right_tables_join():
 @pytest.mark.asyncio
 async def test_multiple_reverse_related_objects():
     async with database:
+        async with database.transaction(force_rollback=True):
+            await create_data()
             classes = await SchoolClass.objects.select_related(
                 ["teachers__category", "students__category"]
             ).all()