Prodream/ormar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip with m2m fields

Browse Source
This commit is contained in:
collerek
2021-02-28 08:19:02 +01:00
parent ad9d065c6d
commit fd38ae2a40
7 changed files with 231 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
ormar/models/excludable.py
View File

@ -12,6 +12,20 @@ class Excludable:
include: Set = field(default_factory=set) include: Set = field(default_factory=set)
exclude: Set = field(default_factory=set) exclude: Set = field(default_factory=set)
@property
def include_all(self):
return ... in self.include
@property
def exclude_all(self):
return ... in self.exclude
def get_copy(self) -> "Excludable":
_copy = self.__class__()
_copy.include = {x for x in self.include}
_copy.exclude = {x for x in self.exclude}
return _copy
def set_values(self, value: Set, is_exclude: bool) -> None: def set_values(self, value: Set, is_exclude: bool) -> None:
prop = "exclude" if is_exclude else "include" prop = "exclude" if is_exclude else "include"
if ... in getattr(self, prop) or ... in value: if ... in getattr(self, prop) or ... in value:
@ -38,6 +52,13 @@ class ExcludableItems:
def __init__(self) -> None: def __init__(self) -> None:
self.items: Dict[str, Excludable] = dict() self.items: Dict[str, Excludable] = dict()
@classmethod
def from_excludable(cls, other: "ExcludableItems") -> "ExcludableItems":
new_excludable = cls()
for key, value in other.items.items():
new_excludable.items[key] = value.get_copy()
return new_excludable
def get(self, model_cls: Type["Model"], alias: str = "") -> Excludable: def get(self, model_cls: Type["Model"], alias: str = "") -> Excludable:
key = f"{alias + '_' if alias else ''}{model_cls.get_name(lower=True)}" key = f"{alias + '_' if alias else ''}{model_cls.get_name(lower=True)}"
return self.items.get(key, Excludable()) return self.items.get(key, Excludable())

51
ormar/models/mixins/excludable_mixin.py
View File

@ -9,9 +9,10 @@ from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Type, Type,
TypeVar, TypeVar,
Union, Union, cast,
) )
from ormar.models.excludable import ExcludableItems
from ormar.models.mixins.relation_mixin import RelationMixin from ormar.models.mixins.relation_mixin import RelationMixin
from ormar.queryset.utils import translate_list_to_dict, update from ormar.queryset.utils import translate_list_to_dict, update
@ -161,10 +162,11 @@ class ExcludableMixin(RelationMixin):
def own_table_columns( def own_table_columns(
cls, cls,
model: Union[Type["Model"], Type["ModelRow"]], model: Union[Type["Model"], Type["ModelRow"]],
fields: Optional[Union[Set, Dict]], excludable: ExcludableItems,
exclude_fields: Optional[Union[Set, Dict]], alias: str = '',
use_alias: bool = False, use_alias: bool = False,
) -> List[str]: ) -> List[str]:
# TODO update docstring
""" """
Returns list of aliases or field names for given model. Returns list of aliases or field names for given model.
Aliases/names switch is use_alias flag. Aliases/names switch is use_alias flag.
@ -176,15 +178,12 @@ class ExcludableMixin(RelationMixin):
:param model: model on columns are selected :param model: model on columns are selected
:type model: Type["Model"] :type model: Type["Model"]
:param fields: set/dict of fields to include
:type fields: Optional[Union[Set, Dict]]
:param exclude_fields: set/dict of fields to exclude
:type exclude_fields: Optional[Union[Set, Dict]]
:param use_alias: flag if aliases or field names should be used :param use_alias: flag if aliases or field names should be used
:type use_alias: bool :type use_alias: bool
:return: list of column field names or aliases :return: list of column field names or aliases
:rtype: List[str] :rtype: List[str]
""" """
model_excludable = excludable.get(model_cls=model, alias=alias)
columns = [ columns = [
model.get_column_name_from_alias(col.name) if not use_alias else col.name model.get_column_name_from_alias(col.name) if not use_alias else col.name
for col in model.Meta.table.columns for col in model.Meta.table.columns
@ -193,17 +192,17 @@ class ExcludableMixin(RelationMixin):
model.get_column_name_from_alias(col.name) model.get_column_name_from_alias(col.name)
for col in model.Meta.table.columns for col in model.Meta.table.columns
] ]
if fields: if model_excludable.include:
columns = [ columns = [
col col
for col, name in zip(columns, field_names) for col, name in zip(columns, field_names)
if model.is_included(fields, name) if model_excludable.is_included(name)
] ]
if exclude_fields: if model_excludable.exclude:
columns = [ columns = [
col col
for col, name in zip(columns, field_names) for col, name in zip(columns, field_names)
if not model.is_excluded(exclude_fields, name) if not model_excludable.is_excluded(name)
] ]
# always has to return pk column for ormar to work # always has to return pk column for ormar to work
@ -246,8 +245,8 @@ class ExcludableMixin(RelationMixin):
@classmethod @classmethod
def get_names_to_exclude( def get_names_to_exclude(
cls, cls,
fields: Optional[Union[Dict, Set]] = None, excludable: ExcludableItems,
exclude_fields: Optional[Union[Dict, Set]] = None, alias: str
) -> Set: ) -> Set:
""" """
Returns a set of models field names that should be explicitly excluded Returns a set of models field names that should be explicitly excluded
@ -259,33 +258,27 @@ class ExcludableMixin(RelationMixin):
Used in parsing data from database rows that construct Models by initializing Used in parsing data from database rows that construct Models by initializing
them with dicts constructed from those db rows. them with dicts constructed from those db rows.
:param fields: set/dict of fields to include :param alias: alias of current relation
:type fields: Optional[Union[Set, Dict]] :type alias: str
:param exclude_fields: set/dict of fields to exclude :param excludable: structure of fields to include and exclude
:type exclude_fields: Optional[Union[Set, Dict]] :type excludable: ExcludableItems
:return: set of field names that should be excluded :return: set of field names that should be excluded
:rtype: Set :rtype: Set
""" """
model = cast(Type["Model"], cls)
model_excludable = excludable.get(model_cls=model, alias=alias)
fields_names = cls.extract_db_own_fields() fields_names = cls.extract_db_own_fields()
if fields and fields is not Ellipsis: if model_excludable.include and model_excludable.include_all:
fields_to_keep = {name for name in fields if name in fields_names} fields_to_keep = model_excludable.include.intersection(fields_names)
else: else:
fields_to_keep = fields_names fields_to_keep = fields_names
fields_to_exclude = fields_names - fields_to_keep fields_to_exclude = fields_names - fields_to_keep
if isinstance(exclude_fields, Set): if model_excludable.exclude:
fields_to_exclude = fields_to_exclude.union( fields_to_exclude = fields_to_exclude.union(
{name for name in exclude_fields if name in fields_names} model_excludable.exclude.intersection(fields_names)
) )
elif isinstance(exclude_fields, Dict):
new_to_exclude = {
name
for name in exclude_fields
if name in fields_names and exclude_fields[name] is Ellipsis
}
fields_to_exclude = fields_to_exclude.union(new_to_exclude)
fields_to_exclude = fields_to_exclude - {cls.Meta.pkname} fields_to_exclude = fields_to_exclude - {cls.Meta.pkname}
return fields_to_exclude return fields_to_exclude

76
ormar/models/model_row.py
View File

@ -14,6 +14,7 @@ from typing import (
import sqlalchemy import sqlalchemy
from ormar.models import NewBaseModel # noqa: I202 from ormar.models import NewBaseModel # noqa: I202
from ormar.models.excludable import ExcludableItems
from ormar.models.helpers.models import group_related_list from ormar.models.helpers.models import group_related_list
@ -33,8 +34,7 @@ class ModelRow(NewBaseModel):
select_related: List = None, select_related: List = None,
related_models: Any = None, related_models: Any = None,
related_field: Type["ForeignKeyField"] = None, related_field: Type["ForeignKeyField"] = None,
fields: Optional[Union[Dict, Set]] = None, excludable: ExcludableItems = None,
exclude_fields: Optional[Union[Dict, Set]] = None,
current_relation_str: str = "", current_relation_str: str = "",
) -> Optional[T]: ) -> Optional[T]:
""" """
@ -50,6 +50,8 @@ class ModelRow(NewBaseModel):
where rows are populated in a different way as they do not have where rows are populated in a different way as they do not have
nested models in result. nested models in result.
:param excludable: structure of fields to include and exclude
:type excludable: ExcludableItems
:param current_relation_str: name of the relation field :param current_relation_str: name of the relation field
:type current_relation_str: str :type current_relation_str: str
:param source_model: model on which relation was defined :param source_model: model on which relation was defined
@ -62,12 +64,6 @@ class ModelRow(NewBaseModel):
:type related_models: Union[List, Dict] :type related_models: Union[List, Dict]
:param related_field: field with relation declaration :param related_field: field with relation declaration
:type related_field: Type[ForeignKeyField] :type related_field: Type[ForeignKeyField]
:param fields: fields and related model fields to include
if provided only those are included
:type fields: Optional[Union[Dict, Set]]
:param exclude_fields: fields and related model fields to exclude
excludes the fields even if they are provided in fields
:type exclude_fields: Optional[Union[Dict, Set]]
:return: returns model if model is populated from database :return: returns model if model is populated from database
:rtype: Optional[Model] :rtype: Optional[Model]
""" """
@ -75,6 +71,7 @@ class ModelRow(NewBaseModel):
select_related = select_related or [] select_related = select_related or []
related_models = related_models or [] related_models = related_models or []
table_prefix = "" table_prefix = ""
excludable = excludable or ExcludableItems()
if select_related: if select_related:
source_model = cast(Type[T], cls) source_model = cast(Type[T], cls)
@ -87,12 +84,11 @@ class ModelRow(NewBaseModel):
relation_field=related_field, relation_field=related_field,
) )
item = cls.populate_nested_models_from_row( item = cls._populate_nested_models_from_row(
item=item, item=item,
row=row, row=row,
related_models=related_models, related_models=related_models,
fields=fields, excludable=excludable,
exclude_fields=exclude_fields,
current_relation_str=current_relation_str, current_relation_str=current_relation_str,
source_model=source_model, source_model=source_model,
) )
@ -100,28 +96,26 @@ class ModelRow(NewBaseModel):
item=item, item=item,
row=row, row=row,
table_prefix=table_prefix, table_prefix=table_prefix,
fields=fields, excludable=excludable
exclude_fields=exclude_fields,
) )
instance: Optional[T] = None instance: Optional[T] = None
if item.get(cls.Meta.pkname, None) is not None: if item.get(cls.Meta.pkname, None) is not None:
item["__excluded__"] = cls.get_names_to_exclude( item["__excluded__"] = cls.get_names_to_exclude(
fields=fields, exclude_fields=exclude_fields excludable=excludable, alias=table_prefix
) )
instance = cast(T, cls(**item)) instance = cast(T, cls(**item))
instance.set_save_status(True) instance.set_save_status(True)
return instance return instance
@classmethod @classmethod
def populate_nested_models_from_row( # noqa: CFQ002 def _populate_nested_models_from_row( # noqa: CFQ002
cls, cls,
item: dict, item: dict,
row: sqlalchemy.engine.ResultProxy, row: sqlalchemy.engine.ResultProxy,
source_model: Type[T], source_model: Type[T],
related_models: Any, related_models: Any,
fields: Optional[Union[Dict, Set]] = None, excludable: ExcludableItems,
exclude_fields: Optional[Union[Dict, Set]] = None,
current_relation_str: str = None, current_relation_str: str = None,
) -> dict: ) -> dict:
""" """
@ -134,6 +128,8 @@ class ModelRow(NewBaseModel):
Recurrently calls from_row method on nested instances and create nested Recurrently calls from_row method on nested instances and create nested
instances. In the end those instances are added to the final model dictionary. instances. In the end those instances are added to the final model dictionary.
:param excludable: structure of fields to include and exclude
:type excludable: ExcludableItems
:param source_model: source model from which relation started :param source_model: source model from which relation started
:type source_model: Type[Model] :type source_model: Type[Model]
:param current_relation_str: joined related parts into one string :param current_relation_str: joined related parts into one string
@ -144,12 +140,6 @@ class ModelRow(NewBaseModel):
:type row: sqlalchemy.engine.result.ResultProxy :type row: sqlalchemy.engine.result.ResultProxy
:param related_models: list or dict of related models :param related_models: list or dict of related models
:type related_models: Union[Dict, List] :type related_models: Union[Dict, List]
:param fields: fields and related model fields to include -
if provided only those are included
:type fields: Optional[Union[Dict, Set]]
:param exclude_fields: fields and related model fields to exclude
excludes the fields even if they are provided in fields
:type exclude_fields: Optional[Union[Dict, Set]]
:return: dictionary with keys corresponding to model fields names :return: dictionary with keys corresponding to model fields names
and values are database values and values are database values
:rtype: Dict :rtype: Dict
@ -163,8 +153,6 @@ class ModelRow(NewBaseModel):
) )
field = cls.Meta.model_fields[related] field = cls.Meta.model_fields[related]
field = cast(Type["ForeignKeyField"], field) field = cast(Type["ForeignKeyField"], field)
fields = cls.get_included(fields, related)
exclude_fields = cls.get_excluded(exclude_fields, related)
model_cls = field.to model_cls = field.to
remainder = None remainder = None
@ -174,8 +162,7 @@ class ModelRow(NewBaseModel):
row, row,
related_models=remainder, related_models=remainder,
related_field=field, related_field=field,
fields=fields, excludable=excludable,
exclude_fields=exclude_fields,
current_relation_str=relation_str, current_relation_str=relation_str,
source_model=source_model, source_model=source_model,
) )
@ -188,8 +175,7 @@ class ModelRow(NewBaseModel):
row=row, row=row,
related=related, related=related,
through_name=through_name, through_name=through_name,
fields=fields, excludable=excludable
exclude_fields=exclude_fields,
) )
item[through_name] = through_child item[through_name] = through_child
setattr(child, through_name, through_child) setattr(child, through_name, through_child)
@ -203,35 +189,29 @@ class ModelRow(NewBaseModel):
row: sqlalchemy.engine.ResultProxy, row: sqlalchemy.engine.ResultProxy,
through_name: str, through_name: str,
related: str, related: str,
fields: Optional[Union[Dict, Set]] = None, excludable: ExcludableItems
exclude_fields: Optional[Union[Dict, Set]] = None,
) -> Dict: ) -> Dict:
# TODO: fix excludes and includes # TODO: fix excludes and includes and docstring
fields = cls.get_included(fields, through_name)
# exclude_fields = cls.get_excluded(exclude_fields, through_name)
model_cls = cls.Meta.model_fields[through_name].to model_cls = cls.Meta.model_fields[through_name].to
exclude_fields = model_cls.extract_related_names()
table_prefix = cls.Meta.alias_manager.resolve_relation_alias( table_prefix = cls.Meta.alias_manager.resolve_relation_alias(
from_model=cls, relation_name=related from_model=cls, relation_name=related
) )
child = model_cls.extract_prefixed_table_columns( child = model_cls.extract_prefixed_table_columns(
item={}, item={},
row=row, row=row,
table_prefix=table_prefix, excludable=excludable,
fields=fields, table_prefix=table_prefix
exclude_fields=exclude_fields,
) )
return child return child
@classmethod @classmethod
def extract_prefixed_table_columns( # noqa CCR001 def extract_prefixed_table_columns(
cls, cls,
item: dict, item: dict,
row: sqlalchemy.engine.result.ResultProxy, row: sqlalchemy.engine.result.ResultProxy,
table_prefix: str, table_prefix: str,
fields: Optional[Union[Dict, Set]] = None, excludable: ExcludableItems
exclude_fields: Optional[Union[Dict, Set]] = None, ) -> Dict:
) -> dict:
""" """
Extracts own fields from raw sql result, using a given prefix. Extracts own fields from raw sql result, using a given prefix.
Prefix changes depending on the table's position in a join. Prefix changes depending on the table's position in a join.
@ -244,6 +224,8 @@ class ModelRow(NewBaseModel):
Used in Model.from_row and PrefetchQuery._populate_rows methods. Used in Model.from_row and PrefetchQuery._populate_rows methods.
:param excludable: structure of fields to include and exclude
:type excludable: ExcludableItems
:param item: dictionary of already populated nested models, otherwise empty dict :param item: dictionary of already populated nested models, otherwise empty dict
:type item: Dict :type item: Dict
:param row: raw result row from the database :param row: raw result row from the database
@ -252,12 +234,6 @@ class ModelRow(NewBaseModel):
each pair of tables have own prefix (two of them depending on direction) - each pair of tables have own prefix (two of them depending on direction) -
used in joins to allow multiple joins to the same table. used in joins to allow multiple joins to the same table.
:type table_prefix: str :type table_prefix: str
:param fields: fields and related model fields to include -
if provided only those are included
:type fields: Optional[Union[Dict, Set]]
:param exclude_fields: fields and related model fields to exclude
excludes the fields even if they are provided in fields
:type exclude_fields: Optional[Union[Dict, Set]]
:return: dictionary with keys corresponding to model fields names :return: dictionary with keys corresponding to model fields names
and values are database values and values are database values
:rtype: Dict :rtype: Dict
@ -267,8 +243,8 @@ class ModelRow(NewBaseModel):
selected_columns = cls.own_table_columns( selected_columns = cls.own_table_columns(
model=cls, model=cls,
fields=fields or {}, excludable=excludable,
exclude_fields=exclude_fields or {}, alias=table_prefix,
use_alias=False, use_alias=False,
) )

16
ormar/queryset/join.py
View File

@ -16,6 +16,7 @@ from sqlalchemy import text
import ormar # noqa I100 import ormar # noqa I100
from ormar.exceptions import RelationshipInstanceError from ormar.exceptions import RelationshipInstanceError
from ormar.models.excludable import ExcludableItems
from ormar.relations import AliasManager from ormar.relations import AliasManager
if TYPE_CHECKING: # pragma no cover if TYPE_CHECKING: # pragma no cover
@ -29,8 +30,7 @@ class SqlJoin:
used_aliases: List, used_aliases: List,
select_from: sqlalchemy.sql.select, select_from: sqlalchemy.sql.select,
columns: List[sqlalchemy.Column], columns: List[sqlalchemy.Column],
fields: Optional[Union[Set, Dict]], excludable: ExcludableItems,
exclude_fields: Optional[Union[Set, Dict]],
order_columns: Optional[List["OrderAction"]], order_columns: Optional[List["OrderAction"]],
sorted_orders: OrderedDict, sorted_orders: OrderedDict,
main_model: Type["Model"], main_model: Type["Model"],
@ -44,8 +44,7 @@ class SqlJoin:
self.related_models = related_models or [] self.related_models = related_models or []
self.select_from = select_from self.select_from = select_from
self.columns = columns self.columns = columns
self.fields = fields self.excludable=excludable
self.exclude_fields = exclude_fields
self.order_columns = order_columns self.order_columns = order_columns
self.sorted_orders = sorted_orders self.sorted_orders = sorted_orders
self.main_model = main_model self.main_model = main_model
@ -200,10 +199,7 @@ class SqlJoin:
used_aliases=self.used_aliases, used_aliases=self.used_aliases,
select_from=self.select_from, select_from=self.select_from,
columns=self.columns, columns=self.columns,
fields=self.main_model.get_excluded(self.fields, related_name), excludable=self.excludable,
exclude_fields=self.main_model.get_excluded(
self.exclude_fields, related_name
),
order_columns=self.order_columns, order_columns=self.order_columns,
sorted_orders=self.sorted_orders, sorted_orders=self.sorted_orders,
main_model=self.next_model, main_model=self.next_model,
@ -303,8 +299,8 @@ class SqlJoin:
# TODO: fix fields and exclusions for through model? # TODO: fix fields and exclusions for through model?
self_related_fields = self.next_model.own_table_columns( self_related_fields = self.next_model.own_table_columns(
model=self.next_model, model=self.next_model,
fields=self.fields, excludable=self.excludable,
exclude_fields=self.exclude_fields, alias=self.next_alias,
use_alias=True, use_alias=True,
) )
self.columns.extend( self.columns.extend(

61
ormar/queryset/prefetch_query.py
View File

@ -13,6 +13,7 @@ from typing import (
) )
import ormar import ormar
from ormar.models.excludable import ExcludableItems
from ormar.queryset.clause import QueryClause from ormar.queryset.clause import QueryClause
from ormar.queryset.query import Query from ormar.queryset.query import Query
from ormar.queryset.utils import extract_models_to_dict_of_lists, translate_list_to_dict from ormar.queryset.utils import extract_models_to_dict_of_lists, translate_list_to_dict
@ -125,8 +126,7 @@ class PrefetchQuery:
def __init__( # noqa: CFQ002 def __init__( # noqa: CFQ002
self, self,
model_cls: Type["Model"], model_cls: Type["Model"],
fields: Optional[Union[Dict, Set]], excludable: ExcludableItems,
exclude_fields: Optional[Union[Dict, Set]],
prefetch_related: List, prefetch_related: List,
select_related: List, select_related: List,
orders_by: List["OrderAction"], orders_by: List["OrderAction"],
@ -136,8 +136,7 @@ class PrefetchQuery:
self.database = self.model.Meta.database self.database = self.model.Meta.database
self._prefetch_related = prefetch_related self._prefetch_related = prefetch_related
self._select_related = select_related self._select_related = select_related
self._exclude_columns = exclude_fields self.excludable = excludable
self._columns = fields
self.already_extracted: Dict = dict() self.already_extracted: Dict = dict()
self.models: Dict = {} self.models: Dict = {}
self.select_dict = translate_list_to_dict(self._select_related) self.select_dict = translate_list_to_dict(self._select_related)
@ -366,8 +365,6 @@ class PrefetchQuery:
select_dict = translate_list_to_dict(self._select_related) select_dict = translate_list_to_dict(self._select_related)
prefetch_dict = translate_list_to_dict(self._prefetch_related) prefetch_dict = translate_list_to_dict(self._prefetch_related)
target_model = self.model target_model = self.model
fields = self._columns
exclude_fields = self._exclude_columns
orders_by = self.order_dict orders_by = self.order_dict
for related in prefetch_dict.keys(): for related in prefetch_dict.keys():
await self._extract_related_models( await self._extract_related_models(
@ -375,8 +372,7 @@ class PrefetchQuery:
target_model=target_model, target_model=target_model,
prefetch_dict=prefetch_dict.get(related, {}), prefetch_dict=prefetch_dict.get(related, {}),
select_dict=select_dict.get(related, {}), select_dict=select_dict.get(related, {}),
fields=fields, excludable=self.excludable,
exclude_fields=exclude_fields,
orders_by=orders_by.get(related, {}), orders_by=orders_by.get(related, {}),
) )
final_models = [] final_models = []
@ -394,8 +390,7 @@ class PrefetchQuery:
target_model: Type["Model"], target_model: Type["Model"],
prefetch_dict: Dict, prefetch_dict: Dict,
select_dict: Dict, select_dict: Dict,
fields: Union[Set[Any], Dict[Any, Any], None], excludable: ExcludableItems,
exclude_fields: Union[Set[Any], Dict[Any, Any], None],
orders_by: Dict, orders_by: Dict,
) -> None: ) -> None:
""" """
@ -424,8 +419,6 @@ class PrefetchQuery:
:return: None :return: None
:rtype: None :rtype: None
""" """
fields = target_model.get_included(fields, related)
exclude_fields = target_model.get_excluded(exclude_fields, related)
target_field = target_model.Meta.model_fields[related] target_field = target_model.Meta.model_fields[related]
target_field = cast(Type["ForeignKeyField"], target_field) target_field = cast(Type["ForeignKeyField"], target_field)
reverse = False reverse = False
@ -450,14 +443,11 @@ class PrefetchQuery:
related_field_name = parent_model.get_related_field_name( related_field_name = parent_model.get_related_field_name(
target_field=target_field target_field=target_field
) )
fields = add_relation_field_to_fields(
fields=fields, related_field_name=related_field_name
)
table_prefix, rows = await self._run_prefetch_query( table_prefix, rows = await self._run_prefetch_query(
target_field=target_field, target_field=target_field,
fields=fields, excludable=excludable,
exclude_fields=exclude_fields,
filter_clauses=filter_clauses, filter_clauses=filter_clauses,
related_field_name=related_field_name
) )
else: else:
rows = [] rows = []
@ -472,8 +462,7 @@ class PrefetchQuery:
select_dict=self._get_select_related_if_apply( select_dict=self._get_select_related_if_apply(
subrelated, select_dict subrelated, select_dict
), ),
fields=fields, excludable=excludable,
exclude_fields=exclude_fields,
orders_by=self._get_select_related_if_apply(subrelated, orders_by), orders_by=self._get_select_related_if_apply(subrelated, orders_by),
) )
@ -483,8 +472,7 @@ class PrefetchQuery:
parent_model=parent_model, parent_model=parent_model,
target_field=target_field, target_field=target_field,
table_prefix=table_prefix, table_prefix=table_prefix,
fields=fields, excludable=excludable,
exclude_fields=exclude_fields,
prefetch_dict=prefetch_dict, prefetch_dict=prefetch_dict,
orders_by=orders_by, orders_by=orders_by,
) )
@ -498,9 +486,9 @@ class PrefetchQuery:
async def _run_prefetch_query( async def _run_prefetch_query(
self, self,
target_field: Type["BaseField"], target_field: Type["BaseField"],
fields: Union[Set[Any], Dict[Any, Any], None], excludable: ExcludableItems,
exclude_fields: Union[Set[Any], Dict[Any, Any], None],
filter_clauses: List, filter_clauses: List,
related_field_name: str
) -> Tuple[str, List]: ) -> Tuple[str, List]:
""" """
Actually runs the queries against the database and populates the raw response Actually runs the queries against the database and populates the raw response
@ -511,10 +499,6 @@ class PrefetchQuery:
:param target_field: ormar field with relation definition :param target_field: ormar field with relation definition
:type target_field: Type["BaseField"] :type target_field: Type["BaseField"]
:param fields: fields to include
:type fields: Union[Set[Any], Dict[Any, Any], None]
:param exclude_fields: fields to exclude
:type exclude_fields: Union[Set[Any], Dict[Any, Any], None]
:param filter_clauses: list of clauses, actually one clause with ids of relation :param filter_clauses: list of clauses, actually one clause with ids of relation
:type filter_clauses: List[sqlalchemy.sql.elements.TextClause] :type filter_clauses: List[sqlalchemy.sql.elements.TextClause]
:return: table prefix and raw rows from sql response :return: table prefix and raw rows from sql response
@ -533,6 +517,11 @@ class PrefetchQuery:
) )
self.already_extracted.setdefault(target_name, {})["prefix"] = table_prefix self.already_extracted.setdefault(target_name, {})["prefix"] = table_prefix
model_excludable = excludable.get(model_cls=target_model, alias=table_prefix)
if model_excludable.include and not model_excludable.is_included(
related_field_name):
model_excludable.set_values({related_field_name}, is_exclude=False)
qry = Query( qry = Query(
model_cls=query_target, model_cls=query_target,
select_related=select_related, select_related=select_related,
@ -540,8 +529,7 @@ class PrefetchQuery:
exclude_clauses=[], exclude_clauses=[],
offset=None, offset=None,
limit_count=None, limit_count=None,
fields=fields, excludable=excludable,
exclude_fields=exclude_fields,
order_bys=None, order_bys=None,
limit_raw_sql=False, limit_raw_sql=False,
) )
@ -595,8 +583,7 @@ class PrefetchQuery:
target_field: Type["ForeignKeyField"], target_field: Type["ForeignKeyField"],
parent_model: Type["Model"], parent_model: Type["Model"],
table_prefix: str, table_prefix: str,
fields: Union[Set[Any], Dict[Any, Any], None], excludable: ExcludableItems,
exclude_fields: Union[Set[Any], Dict[Any, Any], None],
prefetch_dict: Dict, prefetch_dict: Dict,
orders_by: Dict, orders_by: Dict,
) -> None: ) -> None:
@ -610,6 +597,8 @@ class PrefetchQuery:
already_extracted dictionary. Later those instances will be fetched by ids already_extracted dictionary. Later those instances will be fetched by ids
and set on the parent model after sorting if needed. and set on the parent model after sorting if needed.
:param excludable: structure of fields to include and exclude
:type excludable: ExcludableItems
:param rows: raw sql response from the prefetch query :param rows: raw sql response from the prefetch query
:type rows: List[sqlalchemy.engine.result.RowProxy] :type rows: List[sqlalchemy.engine.result.RowProxy]
:param target_field: field with relation definition from parent model :param target_field: field with relation definition from parent model
@ -618,10 +607,6 @@ class PrefetchQuery:
:type parent_model: Type[Model] :type parent_model: Type[Model]
:param table_prefix: prefix of the target table from current relation :param table_prefix: prefix of the target table from current relation
:type table_prefix: str :type table_prefix: str
:param fields: fields to include
:type fields: Union[Set[Any], Dict[Any, Any], None]
:param exclude_fields: fields to exclude
:type exclude_fields: Union[Set[Any], Dict[Any, Any], None]
:param prefetch_dict: dictionaries of related models to prefetch :param prefetch_dict: dictionaries of related models to prefetch
:type prefetch_dict: Dict :type prefetch_dict: Dict
:param orders_by: dictionary of order by clauses by model :param orders_by: dictionary of order by clauses by model
@ -629,16 +614,16 @@ class PrefetchQuery:
""" """
target_model = target_field.to target_model = target_field.to
for row in rows: for row in rows:
# TODO Fix fields
field_name = parent_model.get_related_field_name(target_field=target_field) field_name = parent_model.get_related_field_name(target_field=target_field)
item = target_model.extract_prefixed_table_columns( item = target_model.extract_prefixed_table_columns(
item={}, item={},
row=row, row=row,
table_prefix=table_prefix, table_prefix=table_prefix,
fields=fields, excludable=excludable,
exclude_fields=exclude_fields,
) )
item["__excluded__"] = target_model.get_names_to_exclude( item["__excluded__"] = target_model.get_names_to_exclude(
fields=fields, exclude_fields=exclude_fields excludable=excludable, alias=table_prefix
) )
instance = target_model(**item) instance = target_model(**item)
instance = self._populate_nested_related( instance = self._populate_nested_related(

17
ormar/queryset/query.py
View File

@ -6,6 +6,7 @@ import sqlalchemy
from sqlalchemy import text from sqlalchemy import text
import ormar # noqa I100 import ormar # noqa I100
from ormar.models.excludable import ExcludableItems
from ormar.models.helpers.models import group_related_list from ormar.models.helpers.models import group_related_list
from ormar.queryset import FilterQuery, LimitQuery, OffsetQuery, OrderQuery from ormar.queryset import FilterQuery, LimitQuery, OffsetQuery, OrderQuery
from ormar.queryset.actions.filter_action import FilterAction from ormar.queryset.actions.filter_action import FilterAction
@ -25,8 +26,7 @@ class Query:
select_related: List, select_related: List,
limit_count: Optional[int], limit_count: Optional[int],
offset: Optional[int], offset: Optional[int],
fields: Optional[Union[Dict, Set]], excludable: ExcludableItems,
exclude_fields: Optional[Union[Dict, Set]],
order_bys: Optional[List["OrderAction"]], order_bys: Optional[List["OrderAction"]],
limit_raw_sql: bool, limit_raw_sql: bool,
) -> None: ) -> None:
@ -35,8 +35,7 @@ class Query:
self._select_related = select_related[:] self._select_related = select_related[:]
self.filter_clauses = filter_clauses[:] self.filter_clauses = filter_clauses[:]
self.exclude_clauses = exclude_clauses[:] self.exclude_clauses = exclude_clauses[:]
self.fields = copy.deepcopy(fields) if fields else {} self.excludable = excludable
self.exclude_fields = copy.deepcopy(exclude_fields) if exclude_fields else {}
self.model_cls = model_cls self.model_cls = model_cls
self.table = self.model_cls.Meta.table self.table = self.model_cls.Meta.table
@ -105,8 +104,7 @@ class Query:
""" """
self_related_fields = self.model_cls.own_table_columns( self_related_fields = self.model_cls.own_table_columns(
model=self.model_cls, model=self.model_cls,
fields=self.fields, excludable=self.excludable,
exclude_fields=self.exclude_fields,
use_alias=True, use_alias=True,
) )
self.columns = self.model_cls.Meta.alias_manager.prefixed_columns( self.columns = self.model_cls.Meta.alias_manager.prefixed_columns(
@ -121,8 +119,6 @@ class Query:
related_models = group_related_list(self._select_related) related_models = group_related_list(self._select_related)
for related in related_models: for related in related_models:
fields = self.model_cls.get_included(self.fields, related)
exclude_fields = self.model_cls.get_excluded(self.exclude_fields, related)
remainder = None remainder = None
if isinstance(related_models, dict) and related_models[related]: if isinstance(related_models, dict) and related_models[related]:
remainder = related_models[related] remainder = related_models[related]
@ -130,8 +126,7 @@ class Query:
used_aliases=self.used_aliases, used_aliases=self.used_aliases,
select_from=self.select_from, select_from=self.select_from,
columns=self.columns, columns=self.columns,
fields=fields, excludable=self.excludable,
exclude_fields=exclude_fields,
order_columns=self.order_columns, order_columns=self.order_columns,
sorted_orders=self.sorted_orders, sorted_orders=self.sorted_orders,
main_model=self.model_cls, main_model=self.model_cls,
@ -231,5 +226,3 @@ class Query:
self.select_from = [] self.select_from = []
self.columns = [] self.columns = []
self.used_aliases = [] self.used_aliases = []
self.fields = {}
self.exclude_fields = {}

79
ormar/queryset/queryset.py
View File

@ -20,6 +20,7 @@ from sqlalchemy import bindparam
import ormar # noqa I100 import ormar # noqa I100
from ormar import MultipleMatches, NoMatch from ormar import MultipleMatches, NoMatch
from ormar.exceptions import ModelError, ModelPersistenceError, QueryDefinitionError from ormar.exceptions import ModelError, ModelPersistenceError, QueryDefinitionError
from ormar.models.excludable import ExcludableItems
from ormar.queryset import FilterQuery from ormar.queryset import FilterQuery
from ormar.queryset.actions.order_action import OrderAction from ormar.queryset.actions.order_action import OrderAction
from ormar.queryset.clause import QueryClause from ormar.queryset.clause import QueryClause
@ -48,8 +49,7 @@ class QuerySet(Generic[T]):
select_related: List = None, select_related: List = None,
limit_count: int = None, limit_count: int = None,
offset: int = None, offset: int = None,
columns: Dict = None, excludable: ExcludableItems = None,
exclude_columns: Dict = None,
order_bys: List = None, order_bys: List = None,
prefetch_related: List = None, prefetch_related: List = None,
limit_raw_sql: bool = False, limit_raw_sql: bool = False,
@ -61,8 +61,7 @@ class QuerySet(Generic[T]):
self._prefetch_related = [] if prefetch_related is None else prefetch_related self._prefetch_related = [] if prefetch_related is None else prefetch_related
self.limit_count = limit_count self.limit_count = limit_count
self.query_offset = offset self.query_offset = offset
self._columns = columns or {} self._excludable = excludable or ExcludableItems()
self._exclude_columns = exclude_columns or {}
self.order_bys = order_bys or [] self.order_bys = order_bys or []
self.limit_sql_raw = limit_raw_sql self.limit_sql_raw = limit_raw_sql
@ -121,8 +120,7 @@ class QuerySet(Generic[T]):
""" """
query = PrefetchQuery( query = PrefetchQuery(
model_cls=self.model, model_cls=self.model,
fields=self._columns, excludable=self._excludable,
exclude_fields=self._exclude_columns,
prefetch_related=self._prefetch_related, prefetch_related=self._prefetch_related,
select_related=self._select_related, select_related=self._select_related,
orders_by=self.order_bys, orders_by=self.order_bys,
@ -142,8 +140,7 @@ class QuerySet(Generic[T]):
self.model.from_row( self.model.from_row(
row=row, row=row,
select_related=self._select_related, select_related=self._select_related,
fields=self._columns, excludable=self._excludable,
exclude_fields=self._exclude_columns,
source_model=self.model, source_model=self.model,
) )
for row in rows for row in rows
@ -208,8 +205,7 @@ class QuerySet(Generic[T]):
exclude_clauses=self.exclude_clauses, exclude_clauses=self.exclude_clauses,
offset=offset or self.query_offset, offset=offset or self.query_offset,
limit_count=limit or self.limit_count, limit_count=limit or self.limit_count,
fields=self._columns, excludable=self._excludable,
exclude_fields=self._exclude_columns,
order_bys=order_bys or self.order_bys, order_bys=order_bys or self.order_bys,
limit_raw_sql=self.limit_sql_raw, limit_raw_sql=self.limit_sql_raw,
) )
@ -265,8 +261,7 @@ class QuerySet(Generic[T]):
select_related=select_related, select_related=select_related,
limit_count=self.limit_count, limit_count=self.limit_count,
offset=self.query_offset, offset=self.query_offset,
columns=self._columns, excludable=self._excludable,
exclude_columns=self._exclude_columns,
order_bys=self.order_bys, order_bys=self.order_bys,
prefetch_related=self._prefetch_related, prefetch_related=self._prefetch_related,
limit_raw_sql=self.limit_sql_raw, limit_raw_sql=self.limit_sql_raw,
@ -321,8 +316,7 @@ class QuerySet(Generic[T]):
select_related=related, select_related=related,
limit_count=self.limit_count, limit_count=self.limit_count,
offset=self.query_offset, offset=self.query_offset,
columns=self._columns, excludable=self._excludable,
exclude_columns=self._exclude_columns,
order_bys=self.order_bys, order_bys=self.order_bys,
prefetch_related=self._prefetch_related, prefetch_related=self._prefetch_related,
limit_raw_sql=self.limit_sql_raw, limit_raw_sql=self.limit_sql_raw,
@ -357,14 +351,14 @@ class QuerySet(Generic[T]):
select_related=self._select_related, select_related=self._select_related,
limit_count=self.limit_count, limit_count=self.limit_count,
offset=self.query_offset, offset=self.query_offset,
columns=self._columns, excludable=self._excludable,
exclude_columns=self._exclude_columns,
order_bys=self.order_bys, order_bys=self.order_bys,
prefetch_related=related, prefetch_related=related,
limit_raw_sql=self.limit_sql_raw, limit_raw_sql=self.limit_sql_raw,
) )
def fields(self, columns: Union[List, str, Set, Dict]) -> "QuerySet": def fields(self, columns: Union[List, str, Set, Dict],
_is_exclude: bool = False) -> "QuerySet":
""" """
With `fields()` you can select subset of model columns to limit the data load. With `fields()` you can select subset of model columns to limit the data load.
@ -407,15 +401,10 @@ class QuerySet(Generic[T]):
:return: QuerySet :return: QuerySet
:rtype: QuerySet :rtype: QuerySet
""" """
if isinstance(columns, str): excludable = ExcludableItems.from_excludable(self._excludable)
columns = [columns] excludable.build(items=columns,
model_cls=self.model_cls,
# TODO: Flatten all excludes into one dict-like structure with alias + model key is_exclude=_is_exclude)
current_included = self._columns
if not isinstance(columns, dict):
current_included = update_dict_from_list(current_included, columns)
else:
current_included = update(current_included, columns)
return self.__class__( return self.__class__(
model_cls=self.model, model_cls=self.model,
@ -424,8 +413,7 @@ class QuerySet(Generic[T]):
select_related=self._select_related, select_related=self._select_related,
limit_count=self.limit_count, limit_count=self.limit_count,
offset=self.query_offset, offset=self.query_offset,
columns=current_included, excludable=excludable,
exclude_columns=self._exclude_columns,
order_bys=self.order_bys, order_bys=self.order_bys,
prefetch_related=self._prefetch_related, prefetch_related=self._prefetch_related,
limit_raw_sql=self.limit_sql_raw, limit_raw_sql=self.limit_sql_raw,
@ -458,28 +446,7 @@ class QuerySet(Generic[T]):
:return: QuerySet :return: QuerySet
:rtype: QuerySet :rtype: QuerySet
""" """
if isinstance(columns, str): return self.fields(columns=columns, _is_exclude=True)
columns = [columns]
current_excluded = self._exclude_columns
if not isinstance(columns, dict):
current_excluded = update_dict_from_list(current_excluded, columns)
else:
current_excluded = update(current_excluded, columns)
return self.__class__(
model_cls=self.model,
filter_clauses=self.filter_clauses,
exclude_clauses=self.exclude_clauses,
select_related=self._select_related,
limit_count=self.limit_count,
offset=self.query_offset,
columns=self._columns,
exclude_columns=current_excluded,
order_bys=self.order_bys,
prefetch_related=self._prefetch_related,
limit_raw_sql=self.limit_sql_raw,
)
def order_by(self, columns: Union[List, str]) -> "QuerySet": def order_by(self, columns: Union[List, str]) -> "QuerySet":
""" """
@ -529,8 +496,7 @@ class QuerySet(Generic[T]):
select_related=self._select_related, select_related=self._select_related,
limit_count=self.limit_count, limit_count=self.limit_count,
offset=self.query_offset, offset=self.query_offset,
columns=self._columns, excludable=self._excludable,
exclude_columns=self._exclude_columns,
order_bys=order_bys, order_bys=order_bys,
prefetch_related=self._prefetch_related, prefetch_related=self._prefetch_related,
limit_raw_sql=self.limit_sql_raw, limit_raw_sql=self.limit_sql_raw,
@ -642,8 +608,7 @@ class QuerySet(Generic[T]):
select_related=self._select_related, select_related=self._select_related,
limit_count=limit_count, limit_count=limit_count,
offset=query_offset, offset=query_offset,
columns=self._columns, excludable=self._excludable,
exclude_columns=self._exclude_columns,
order_bys=self.order_bys, order_bys=self.order_bys,
prefetch_related=self._prefetch_related, prefetch_related=self._prefetch_related,
limit_raw_sql=self.limit_sql_raw, limit_raw_sql=self.limit_sql_raw,
@ -671,8 +636,7 @@ class QuerySet(Generic[T]):
select_related=self._select_related, select_related=self._select_related,
limit_count=limit_count, limit_count=limit_count,
offset=self.query_offset, offset=self.query_offset,
columns=self._columns, excludable=self._excludable,
exclude_columns=self._exclude_columns,
order_bys=self.order_bys, order_bys=self.order_bys,
prefetch_related=self._prefetch_related, prefetch_related=self._prefetch_related,
limit_raw_sql=limit_raw_sql, limit_raw_sql=limit_raw_sql,
@ -700,8 +664,7 @@ class QuerySet(Generic[T]):
select_related=self._select_related, select_related=self._select_related,
limit_count=self.limit_count, limit_count=self.limit_count,
offset=offset, offset=offset,
columns=self._columns, excludable=self._excludable,
exclude_columns=self._exclude_columns,
order_bys=self.order_bys, order_bys=self.order_bys,
prefetch_related=self._prefetch_related, prefetch_related=self._prefetch_related,
limit_raw_sql=limit_raw_sql, limit_raw_sql=limit_raw_sql,