diff --git a/.coverage b/.coverage index 6da7884..7f7ce46 100644 Binary files a/.coverage and b/.coverage differ diff --git a/README.md b/README.md index b8f3f26..cb31386 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Ormar is built with: Because ormar is built on SQLAlchemy core, you can use [`alembic`][alembic] to provide database migrations. -The goal was to create a simple ORM that can be used directly with [`fastapi`][fastapi] that bases it's data validation on pydantic. +The goal was to create a simple ORM that can be used directly (as request and response models) with [`fastapi`][fastapi] that bases it's data validation on pydantic. Initial work was inspired by [`encode/orm`][encode/orm], later I found `ormantic` and used it as a further inspiration. The encode package was too simple (i.e. no ability to join two times to the same table) and used typesystem for data checks. @@ -241,19 +241,19 @@ await post.categories.add(news) # or from the other end: await news.posts.add(post) -# Creating related object from instance: +# Creating columns object from instance: await post.categories.create(name="Tips") assert len(await post.categories.all()) == 2 -# Many to many relation exposes a list of related models +# Many to many relation exposes a list of columns models # and an API of the Queryset: assert news == await post.categories.get(name="News") -# with all Queryset methods - filtering, selecting related, counting etc. +# with all Queryset methods - filtering, selecting columns, counting etc. await news.posts.filter(title__contains="M2M").all() await Category.objects.filter(posts__author=guido).get() -# related models of many to many relation can be prefetched +# columns models of many to many relation can be prefetched news_posts = await news.posts.select_related("author").all() assert news_posts[0].author == guido @@ -408,6 +408,83 @@ await Product.objects.create(name="Mars", company="Mars") ``` +Since version >=0.3.6 Ormar supports selecting subset of model columns to limit the data load. +Warning - mandatory fields cannot be excluded as it will raise validation error, to exclude a field it has to be nullable. +Pk column cannot be excluded - it's always auto added even if not explicitly included. +```python +import databases +import pydantic +import pytest +import sqlalchemy + +import ormar +from tests.settings import DATABASE_URL + +database = databases.Database(DATABASE_URL, force_rollback=True) +metadata = sqlalchemy.MetaData() + + +class Company(ormar.Model): + class Meta: + tablename = "companies" + metadata = metadata + database = database + + id: ormar.Integer(primary_key=True) + name: ormar.String(max_length=100) + founded: ormar.Integer(nullable=True) + + +class Car(ormar.Model): + class Meta: + tablename = "cars" + metadata = metadata + database = database + + id: ormar.Integer(primary_key=True) + manufacturer: ormar.ForeignKey(Company) + name: ormar.String(max_length=100) + year: ormar.Integer(nullable=True) + gearbox_type: ormar.String(max_length=20, nullable=True) + gears: ormar.Integer(nullable=True) + aircon_type: ormar.String(max_length=20, nullable=True) + + + +# build some sample data +toyota = await Company.objects.create(name="Toyota", founded=1937) +await Car.objects.create(manufacturer=toyota, name="Corolla", year=2020, gearbox_type='Manual', gears=5, + aircon_type='Manual') +await Car.objects.create(manufacturer=toyota, name="Yaris", year=2019, gearbox_type='Manual', gears=5, + aircon_type='Manual') +await Car.objects.create(manufacturer=toyota, name="Supreme", year=2020, gearbox_type='Auto', gears=6, + aircon_type='Auto') + +# select manufacturer but only name - to include related models use notation {model_name}__{column} +all_cars = await Car.objects.select_related('manufacturer').fields(['id', 'name', 'company__name']).all() +for car in all_cars: + # excluded columns will yield None + assert all(getattr(car, x) is None for x in ['year', 'gearbox_type', 'gears', 'aircon_type']) + # included column on related models will be available, pk column is always included + # even if you do not include it in fields list + assert car.manufacturer.name == 'Toyota' + # also in the nested related models - you cannot exclude pk - it's always auto added + assert car.manufacturer.founded is None + +# fields() can be called several times, building up the columns to select +# models selected in select_related but with no columns in fields list implies all fields +all_cars = await Car.objects.select_related('manufacturer').fields('id').fields( + ['name']).all() +# all fiels from company model are selected +assert all_cars[0].manufacturer.name == 'Toyota' +assert all_cars[0].manufacturer.founded == 1937 + +# cannot exclude mandatory model columns - company__name in this example +await Car.objects.select_related('manufacturer').fields(['id', 'name', 'company__founded']).all() +# will raise pydantic ValidationError as company.name is required + +``` + ## Data types @@ -441,6 +518,7 @@ Available Model Fields (with required args - optional ones in docs): * `JSON()` * `BigInteger()` * `Decimal(scale, precision)` +* `UUID()` * `ForeignKey(to)` * `Many2Many(to, through)` diff --git a/docs/queries.md b/docs/queries.md index 474a98e..00e3eb7 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -76,7 +76,7 @@ track = Track.objects.filter(name="The Bird").get() # will return a track with name equal to 'The Bird' tracks = Track.objects.filter(album__name="Fantasies").all() -# will return all tracks where the related album name = 'Fantasies' +# will return all tracks where the columns album name = 'Fantasies' ``` You can use special filter suffix to change the filter operands: @@ -106,7 +106,7 @@ To chain related `Models` relation use double underscore. ```python album = await Album.objects.select_related("tracks").all() -# will return album will all related tracks +# will return album will all columns tracks ``` You can provide a string or a list of strings diff --git a/docs_src/fields/docs003.py b/docs_src/fields/docs003.py index 32ff68c..998029d 100644 --- a/docs_src/fields/docs003.py +++ b/docs_src/fields/docs003.py @@ -29,7 +29,7 @@ class Track(ormar.Model): album = await Album.objects.create(name="Brooklyn") await Track.objects.create(album=album, title="The Bird", position=1) -# explicit preload of related Album Model +# explicit preload of columns Album Model track = await Track.objects.select_related("album").get(title="The Bird") assert track.album.name == 'Brooklyn' # Will produce: True diff --git a/docs_src/models/docs006.py b/docs_src/models/docs006.py index 368e771..d3151db 100644 --- a/docs_src/models/docs006.py +++ b/docs_src/models/docs006.py @@ -33,7 +33,7 @@ print('name' in course.__dict__) print(course.name) # Math <- value returned from underlying pydantic model print('department' in course.__dict__) -# False <- related model is not stored on Course instance +# False <- columns model is not stored on Course instance print(course.department) # Department(id=None, name='Science') <- Department model # returned from AliasManager diff --git a/ormar/__init__.py b/ormar/__init__.py index 1616879..1765386 100644 --- a/ormar/__init__.py +++ b/ormar/__init__.py @@ -28,7 +28,7 @@ class UndefinedType: # pragma no cover Undefined = UndefinedType() -__version__ = "0.3.5" +__version__ = "0.3.6" __all__ = [ "Integer", "BigInteger", diff --git a/ormar/models/model.py b/ormar/models/model.py index 6ce3b0b..fd71efe 100644 --- a/ormar/models/model.py +++ b/ormar/models/model.py @@ -34,6 +34,7 @@ class Model(NewBaseModel): select_related: List = None, related_models: Any = None, previous_table: str = None, + fields: List = None, ) -> Optional["Model"]: item: Dict[str, Any] = {} @@ -61,9 +62,11 @@ class Model(NewBaseModel): previous_table = cls.Meta.table.name item = cls.populate_nested_models_from_row( - item, row, related_models, previous_table + item, row, related_models, previous_table, fields + ) + item = cls.extract_prefixed_table_columns( + item, row, table_prefix, fields, nested=table_prefix != "" ) - item = cls.extract_prefixed_table_columns(item, row, table_prefix) instance = cls(**item) if item.get(cls.Meta.pkname, None) is not None else None return instance @@ -75,33 +78,47 @@ class Model(NewBaseModel): row: sqlalchemy.engine.ResultProxy, related_models: Any, previous_table: sqlalchemy.Table, + fields: List = None, ) -> dict: for related in related_models: if isinstance(related_models, dict) and related_models[related]: first_part, remainder = related, related_models[related] model_cls = cls.Meta.model_fields[first_part].to child = model_cls.from_row( - row, related_models=remainder, previous_table=previous_table + row, + related_models=remainder, + previous_table=previous_table, + fields=fields, ) item[first_part] = child else: model_cls = cls.Meta.model_fields[related].to - child = model_cls.from_row(row, previous_table=previous_table) + child = model_cls.from_row( + row, previous_table=previous_table, fields=fields + ) item[related] = child return item @classmethod def extract_prefixed_table_columns( # noqa CCR001 - cls, item: dict, row: sqlalchemy.engine.result.ResultProxy, table_prefix: str + cls, + item: dict, + row: sqlalchemy.engine.result.ResultProxy, + table_prefix: str, + fields: List = None, + nested: bool = False, ) -> dict: + + # databases does not keep aliases in Record for postgres, change to raw row + source = row._row if isinstance(row, Record) else row + + selected_columns = cls.own_table_columns(cls, fields or [], nested=nested) for column in cls.Meta.table.columns: - if column.name not in item: + if column.name not in item and column.name in selected_columns: prefixed_name = ( f'{table_prefix + "_" if table_prefix else ""}{column.name}' ) - # databases does not keep aliases in Record for postgres - source = row._row if isinstance(row, Record) else row item[column.name] = source[prefixed_name] return item diff --git a/ormar/models/modelproxy.py b/ormar/models/modelproxy.py index ab70385..0dadbb2 100644 --- a/ormar/models/modelproxy.py +++ b/ormar/models/modelproxy.py @@ -149,3 +149,32 @@ class ModelTableProxy: cls.merge_two_instances(current_field, getattr(other, field)), ) return other + + @staticmethod + def own_table_columns( + model: Type["Model"], fields: List, nested: bool = False + ) -> List[str]: + column_names = [col.name for col in model.Meta.table.columns] + if not fields: + return column_names + + if not nested: + columns = [ + name for name in fields if "__" not in name and name in column_names + ] + else: + model_name = f"{model.get_name()}__" + columns = [ + name[(name.find(model_name) + len(model_name)) :] # noqa: E203 + for name in fields + if f"{model.get_name()}__" in name + ] + + # if the model is in select and no columns in fields, all implied + if not columns: + columns = column_names + + # always has to return pk column + if model.Meta.pkname not in columns: + columns.append(model.Meta.pkname) + return columns diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index 64f7df3..140d118 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -93,7 +93,7 @@ class NewBaseModel(pydantic.BaseModel, ModelTableProxy, metaclass=ModelMetaclass object.__setattr__(self, "__dict__", values) object.__setattr__(self, "__fields_set__", fields_set) - # register the related models after initialization + # register the columns models after initialization for related in self.extract_related_names(): self.Meta.model_fields[related].expand_relationship( kwargs.get(related), self, to_register=True diff --git a/ormar/queryset/join.py b/ormar/queryset/join.py index 4b70636..f79f353 100644 --- a/ormar/queryset/join.py +++ b/ormar/queryset/join.py @@ -24,11 +24,13 @@ class SqlJoin: select_from: sqlalchemy.sql.select, order_bys: List[sqlalchemy.sql.elements.TextClause], columns: List[sqlalchemy.Column], + fields: List, ) -> None: self.used_aliases = used_aliases self.select_from = select_from self.order_bys = order_bys self.columns = columns + self.fields = fields @staticmethod def relation_manager(model_cls: Type["Model"]) -> AliasManager: @@ -105,9 +107,12 @@ class SqlJoin: self.select_from, target_table, on_clause ) self.order_bys.append(text(f"{alias}_{to_table}.{model_cls.Meta.pkname}")) + self_related_fields = model_cls.own_table_columns( + model_cls, self.fields, nested=True + ) self.columns.extend( self.relation_manager(model_cls).prefixed_columns( - alias, model_cls.Meta.table + alias, model_cls.Meta.table, self_related_fields ) ) self.used_aliases.append(alias) diff --git a/ormar/queryset/query.py b/ormar/queryset/query.py index d352e51..ada3437 100644 --- a/ormar/queryset/query.py +++ b/ormar/queryset/query.py @@ -20,12 +20,14 @@ class Query: select_related: List, limit_count: Optional[int], offset: Optional[int], + fields: Optional[List], ) -> None: self.query_offset = offset self.limit_count = limit_count self._select_related = select_related[:] self.filter_clauses = filter_clauses[:] self.exclude_clauses = exclude_clauses[:] + self.fields = fields[:] if fields else [] self.model_cls = model_cls self.table = self.model_cls.Meta.table @@ -41,7 +43,12 @@ class Query: return f"{self.table.name}.{self.model_cls.Meta.pkname}" def build_select_expression(self) -> Tuple[sqlalchemy.sql.select, List[str]]: - self.columns = list(self.table.columns) + self_related_fields = self.model_cls.own_table_columns( + self.model_cls, self.fields + ) + self.columns = self.model_cls.Meta.alias_manager.prefixed_columns( + "", self.table, self_related_fields + ) self.order_bys = [text(self.prefixed_pk_name)] self.select_from = self.table @@ -57,6 +64,7 @@ class Query: select_from=self.select_from, columns=self.columns, order_bys=self.order_bys, + fields=self.fields, ) ( @@ -93,3 +101,4 @@ class Query: self.columns = [] self.order_bys = [] self.used_aliases = [] + self.fields = [] diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index 427e95a..adece00 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -26,6 +26,7 @@ class QuerySet: select_related: List = None, limit_count: int = None, offset: int = None, + columns: List = None, ) -> None: self.model_cls = model_cls self.filter_clauses = [] if filter_clauses is None else filter_clauses @@ -33,6 +34,7 @@ class QuerySet: self._select_related = [] if select_related is None else select_related self.limit_count = limit_count self.query_offset = offset + self._columns = columns or [] self.order_bys = None def __get__( @@ -59,7 +61,9 @@ class QuerySet: def _process_query_result_rows(self, rows: List) -> List[Optional["Model"]]: result_rows = [ - self.model.from_row(row, select_related=self._select_related) + self.model.from_row( + row, select_related=self._select_related, fields=self._columns + ) for row in rows ] if result_rows: @@ -104,6 +108,7 @@ class QuerySet: exclude_clauses=self.exclude_clauses, offset=self.query_offset, limit_count=self.limit_count, + fields=self._columns, ) exp = qry.build_select_expression() # print(exp.compile(compile_kwargs={"literal_binds": True})) @@ -130,6 +135,7 @@ class QuerySet: select_related=select_related, limit_count=self.limit_count, offset=self.query_offset, + columns=self._columns, ) def exclude(self, **kwargs: Any) -> "QuerySet": # noqa: A003 @@ -147,6 +153,22 @@ class QuerySet: select_related=related, limit_count=self.limit_count, offset=self.query_offset, + columns=self._columns, + ) + + def fields(self, columns: Union[List, str]) -> "QuerySet": + if not isinstance(columns, list): + columns = [columns] + + columns = list(set(list(self._columns) + columns)) + return self.__class__( + model_cls=self.model, + filter_clauses=self.filter_clauses, + exclude_clauses=self.exclude_clauses, + select_related=self._select_related, + limit_count=self.limit_count, + offset=self.query_offset, + columns=columns, ) async def exists(self) -> bool: @@ -193,6 +215,7 @@ class QuerySet: select_related=self._select_related, limit_count=limit_count, offset=self.query_offset, + columns=self._columns, ) def offset(self, offset: int) -> "QuerySet": @@ -203,6 +226,7 @@ class QuerySet: select_related=self._select_related, limit_count=self.limit_count, offset=offset, + columns=self._columns, ) async def first(self, **kwargs: Any) -> "Model": diff --git a/ormar/relations/alias_manager.py b/ormar/relations/alias_manager.py index ff4842f..2eead1e 100644 --- a/ormar/relations/alias_manager.py +++ b/ormar/relations/alias_manager.py @@ -17,10 +17,18 @@ class AliasManager: self._aliases: Dict[str, str] = dict() @staticmethod - def prefixed_columns(alias: str, table: sqlalchemy.Table) -> List[text]: + def prefixed_columns( + alias: str, table: sqlalchemy.Table, fields: List = None + ) -> List[text]: + alias = f"{alias}_" if alias else "" + all_columns = ( + table.columns + if not fields + else [col for col in table.columns if col.name in fields] + ) return [ - text(f"{alias}_{table.name}.{column.name} as {alias}_{column.name}") - for column in table.columns + text(f"{alias}{table.name}.{column.name} as {alias}{column.name}") + for column in all_columns ] @staticmethod diff --git a/setup.py b/setup.py index 253810c..867d8a1 100644 --- a/setup.py +++ b/setup.py @@ -42,10 +42,10 @@ setup( version=get_version(PACKAGE), url=URL, license="MIT", - description="An simple async ORM with Fastapi in mind.", + description="An simple async ORM with fastapi in mind and pydantic validation.", long_description=get_long_description(), long_description_content_type="text/markdown", - keywords=['ORM', 'sqlalchemy', 'fastapi', 'pydantic', 'databases'], + keywords=['orm', 'sqlalchemy', 'fastapi', 'pydantic', 'databases', 'async', 'alembic'], author="collerek", author_email="collerek@gmail.com", packages=get_packages(PACKAGE), diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index b2efbc6..b85493d 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -1,9 +1,6 @@ -import asyncio - import databases import pytest import sqlalchemy -from pydantic import root_validator, validator import ormar from ormar.exceptions import NoMatch, MultipleMatches, RelationshipInstanceError diff --git a/tests/test_many_to_many.py b/tests/test_many_to_many.py index 2859b71..fc97b0d 100644 --- a/tests/test_many_to_many.py +++ b/tests/test_many_to_many.py @@ -89,7 +89,7 @@ async def test_assigning_related_objects(cleanup): # or from the other end: await news.posts.add(post) - # Creating related object from instance: + # Creating columns object from instance: await post.categories.create(name="Tips") assert len(post.categories) == 2 @@ -148,7 +148,7 @@ async def test_removal_of_the_relations(cleanup): await news.posts.remove(post) assert len(await news.posts.all()) == 0 - # Remove all related objects: + # Remove all columns objects: await post.categories.add(news) await post.categories.clear() assert len(await post.categories.all()) == 0 diff --git a/tests/test_selecting_subset_of_columns.py b/tests/test_selecting_subset_of_columns.py new file mode 100644 index 0000000..9397235 --- /dev/null +++ b/tests/test_selecting_subset_of_columns.py @@ -0,0 +1,118 @@ +import databases +import pydantic +import pytest +import sqlalchemy + +import ormar +from tests.settings import DATABASE_URL + +database = databases.Database(DATABASE_URL, force_rollback=True) +metadata = sqlalchemy.MetaData() + + +class Company(ormar.Model): + class Meta: + tablename = "companies" + metadata = metadata + database = database + + id: ormar.Integer(primary_key=True) + name: ormar.String(max_length=100) + founded: ormar.Integer(nullable=True) + + +class Car(ormar.Model): + class Meta: + tablename = "cars" + metadata = metadata + database = database + + id: ormar.Integer(primary_key=True) + manufacturer: ormar.ForeignKey(Company) + name: ormar.String(max_length=100) + year: ormar.Integer(nullable=True) + gearbox_type: ormar.String(max_length=20, nullable=True) + gears: ormar.Integer(nullable=True) + aircon_type: ormar.String(max_length=20, nullable=True) + + +@pytest.fixture(autouse=True, scope="module") +def create_test_database(): + engine = sqlalchemy.create_engine(DATABASE_URL) + metadata.drop_all(engine) + metadata.create_all(engine) + yield + metadata.drop_all(engine) + + +@pytest.mark.asyncio +async def test_selecting_subset(): + async with database: + async with database.transaction(force_rollback=True): + toyota = await Company.objects.create(name="Toyota", founded=1937) + await Car.objects.create( + manufacturer=toyota, + name="Corolla", + year=2020, + gearbox_type="Manual", + gears=5, + aircon_type="Manual", + ) + await Car.objects.create( + manufacturer=toyota, + name="Yaris", + year=2019, + gearbox_type="Manual", + gears=5, + aircon_type="Manual", + ) + await Car.objects.create( + manufacturer=toyota, + name="Supreme", + year=2020, + gearbox_type="Auto", + gears=6, + aircon_type="Auto", + ) + + all_cars = ( + await Car.objects.select_related("manufacturer") + .fields(["id", "name", "company__name"]) + .all() + ) + for car in all_cars: + assert all( + getattr(car, x) is None + for x in ["year", "gearbox_type", "gears", "aircon_type"] + ) + assert car.manufacturer.name == "Toyota" + assert car.manufacturer.founded is None + + all_cars = ( + await Car.objects.select_related("manufacturer") + .fields("id") + .fields(["name"]) + .all() + ) + for car in all_cars: + assert all( + getattr(car, x) is None + for x in ["year", "gearbox_type", "gears", "aircon_type"] + ) + assert car.manufacturer.name == "Toyota" + assert car.manufacturer.founded == 1937 + + all_cars_check = await Car.objects.select_related("manufacturer").all() + for car in all_cars_check: + assert all( + getattr(car, x) is not None + for x in ["year", "gearbox_type", "gears", "aircon_type"] + ) + assert car.manufacturer.name == "Toyota" + assert car.manufacturer.founded == 1937 + + with pytest.raises(pydantic.error_wrappers.ValidationError): + # cannot exclude mandatory model columns - company__name in this example + await Car.objects.select_related("manufacturer").fields( + ["id", "name", "company__founded"] + ).all()