From a7d1b657122ad7ff76497cf24aff311719c4e715 Mon Sep 17 00:00:00 2001 From: collerek Date: Thu, 17 Sep 2020 15:49:39 +0200 Subject: [PATCH 1/2] extracted join --- .coverage | Bin 53248 -> 53248 bytes ormar/fields/__init__.py | 3 +- ormar/queryset/join.py | 131 +++++++++++++++++++++++++++++++++++++++ ormar/queryset/query.py | 116 +++++----------------------------- 4 files changed, 148 insertions(+), 102 deletions(-) create mode 100644 ormar/queryset/join.py diff --git a/.coverage b/.coverage index 5c9ea5d2bcdc4ff651dc133eae0146c82fbc4b8c..6a3129e1624e9f90986c5987781b7bbdb3eb3e26 100644 GIT binary patch delta 1228 zcmZ9JTTC2f6vt=yhMk?6oo{x_-q~ID%Cy^Lp%ltRp@u}eTj)d4hi#U$QAAu**vcik z8E|PYrj#aBiovpK@P5DtASoEKge-27ksXZ)Ld4Nq}DbDwfO zTs_C=f6!mnpVoWXT-Ic~z!ufYWFs0RO;IylPA)_bkin>#v_->Y8k``RjYjAIBPDbDgk|$v9GhgKiSjO7de-Xco5YArj;05b>{BW z>z!x&+BZMYJnY_Dve_hU$1RKtQOTV&dCTP?Lkee+g+6s~C;tC6* zI=9Cq!&|9j4-8bA5!I!cS!3%H)F!iwl$4fQBs=~GXYn9@5$_YPb8|c`&WpW<{bIdv zN2Cqk8ODUXAt8(kukhCe51-^0_#@IU+%@SMm*N_@V*QW$kMw8sK6VkfTx5wOUPsGh zEFK0RCFErMAOI<$L*%D;gbCK60Wueti5$aBpc-{U5dbcj@oyD=;3CmJGI6Sg@g778 z=)0v|T*@j?2Y5E^q9uEi5kXQi8{B3C1IP0|FXv?!WmzheAPcTZ81K5uYvqvMxZnM{GJj z$R%Amq!4{aX!R(d8^r+%NA9sgpou(aN_ ztSZl%Y;LXL&WFgFy0NjGQPhsi;*_e;f(sgUSAOAWVZ|+#Lg6jV8w<)&W##WLGbSe( zL1*{_<)*4m4$UhoNOXYZa-5oeu%=|r+!~VX+KfH_O|~$eQE#aU-Uf!%cK`CTKWbSk zXrtA$VYPsWSne%k3y)UTew|wW_i9R6SFydd1*FveZb}+b4>oMrh`6F_IA_lIGc&{{f)EvY|7qiCfAT8}(lf$>P=fywC6dyoYCc zkxnnt^F57K&8?kHEtwY@myI;Y+Q9H!$d?mN2`>sY{5Ae#ewc6O33rwIgnON9V;``W z*je@^)|RO@erKp^4^rI(d>Tuw4~$0J7&q0$kvom!Lnls*yp_QcRw3a=U+OTL@J z(>iTbr|nXV)DSDo#rC3Np+CkjmPf*HGFDB@h+C_gOjMVs#4^UdyR<*Z+I31wi77Vu zZ#g4R%FoL+(wF>We2bKq#>73+QxXw>5a)!F7#A)I@AE$k2l&%MEuT=Xaz80&xkc^` z?n(9``y)HYzRcRO>F*8j*DD<`aIgXUttA0a4~F1_NG;B4@;#M{I4POVH z*frtbNDc7R!{jrOOhhM%N;WE(>+&+v-Yx zc9)AT#eti8N6A>$y0zlXZ_?FvH2n6^N7}hUVQDI_6{%!H%W1n1`(sT@59X#6Yb9eX zeVZwtNf&a3IB!A2V!1xQ@mssWjM`{+Z#qpVI!(Vec5N(AymM|m8GAUDA1JTizLncp rBbm=W3==}&%IrQGl<$FIZa>!gg`By!AgVN|^nl@F#knm*&*kiYR9ItC diff --git a/ormar/fields/__init__.py b/ormar/fields/__init__.py index 22c2665..493ebab 100644 --- a/ormar/fields/__init__.py +++ b/ormar/fields/__init__.py @@ -1,6 +1,6 @@ from ormar.fields.base import BaseField from ormar.fields.foreign_key import ForeignKey -from ormar.fields.many_to_many import ManyToMany +from ormar.fields.many_to_many import ManyToMany, ManyToManyField from ormar.fields.model_fields import ( BigInteger, Boolean, @@ -29,5 +29,6 @@ __all__ = [ "Time", "ForeignKey", "ManyToMany", + "ManyToManyField", "BaseField", ] diff --git a/ormar/queryset/join.py b/ormar/queryset/join.py new file mode 100644 index 0000000..2d59f58 --- /dev/null +++ b/ormar/queryset/join.py @@ -0,0 +1,131 @@ +from typing import List, NamedTuple, TYPE_CHECKING, Tuple, Type + +import sqlalchemy +from sqlalchemy import text + +from ormar.fields import ManyToManyField # noqa I100 +from ormar.relations import AliasManager + +if TYPE_CHECKING: # pragma no cover + from ormar import Model + + +class JoinParameters(NamedTuple): + prev_model: Type["Model"] + previous_alias: str + from_table: str + model_cls: Type["Model"] + + +class SqlJoin: + def __init__( + self, + used_aliases: List, + select_from: sqlalchemy.sql.select, + order_bys: List, + columns: List, + ) -> None: + self.used_aliases = used_aliases + self.select_from = select_from + self.order_bys = order_bys + self.columns = columns + + @staticmethod + def relation_manager(model_cls: Type["Model"]) -> AliasManager: + return model_cls.Meta.alias_manager + + @staticmethod + def on_clause( + previous_alias: str, alias: str, from_clause: str, to_clause: str, + ) -> text: + left_part = f"{alias}_{to_clause}" + right_part = f"{previous_alias + '_' if previous_alias else ''}{from_clause}" + return text(f"{left_part}={right_part}") + + def build_join( + self, item: str, join_parameters: JoinParameters + ) -> Tuple[List, sqlalchemy.sql.select, List, List]: + for part in item.split("__"): + if issubclass( + join_parameters.model_cls.Meta.model_fields[part], ManyToManyField + ): + _fields = join_parameters.model_cls.Meta.model_fields + new_part = _fields[part].to.get_name() + join_parameters = self._build_join_parameters( + part, join_parameters, is_multi=True + ) + part = new_part + join_parameters = self._build_join_parameters(part, join_parameters) + + return self.used_aliases, self.select_from, self.columns, self.order_bys + + def _build_join_parameters( + self, part: str, join_params: JoinParameters, is_multi: bool = False + ) -> JoinParameters: + if is_multi: + model_cls = join_params.model_cls.Meta.model_fields[part].through + else: + model_cls = join_params.model_cls.Meta.model_fields[part].to + to_table = model_cls.Meta.table.name + + alias = model_cls.Meta.alias_manager.resolve_relation_join( + join_params.from_table, to_table + ) + if alias not in self.used_aliases: + self._process_join(join_params, is_multi, model_cls, part, alias) + + previous_alias = alias + from_table = to_table + prev_model = model_cls + return JoinParameters(prev_model, previous_alias, from_table, model_cls) + + def _process_join( + self, + join_params: JoinParameters, + is_multi: bool, + model_cls: Type["Model"], + part: str, + alias: str, + ) -> None: + to_table = model_cls.Meta.table.name + to_key, from_key = self.get_to_and_from_keys( + join_params, is_multi, model_cls, part + ) + + on_clause = self.on_clause( + previous_alias=join_params.previous_alias, + alias=alias, + from_clause=f"{join_params.from_table}.{from_key}", + to_clause=f"{to_table}.{to_key}", + ) + target_table = self.relation_manager(model_cls).prefixed_table_name( + alias, to_table + ) + self.select_from = sqlalchemy.sql.outerjoin( + self.select_from, target_table, on_clause + ) + self.order_bys.append(text(f"{alias}_{to_table}.{model_cls.Meta.pkname}")) + self.columns.extend( + self.relation_manager(model_cls).prefixed_columns( + alias, model_cls.Meta.table + ) + ) + self.used_aliases.append(alias) + + @staticmethod + def get_to_and_from_keys( + join_params: JoinParameters, + is_multi: bool, + model_cls: Type["Model"], + part: str, + ) -> Tuple[str, str]: + if join_params.prev_model.Meta.model_fields[part].virtual or is_multi: + to_field = model_cls.resolve_relation_field( + model_cls, join_params.prev_model + ) + to_key = to_field.name + from_key = model_cls.Meta.pkname + else: + to_key = model_cls.Meta.pkname + from_key = part + return to_key, from_key diff --git a/ormar/queryset/query.py b/ormar/queryset/query.py index 22578a7..6fdbdd4 100644 --- a/ormar/queryset/query.py +++ b/ormar/queryset/query.py @@ -1,24 +1,16 @@ -from typing import List, NamedTuple, TYPE_CHECKING, Tuple, Type +from typing import List, TYPE_CHECKING, Tuple, Type import sqlalchemy from sqlalchemy import text import ormar # noqa I100 -from ormar.fields.many_to_many import ManyToManyField from ormar.queryset import FilterQuery, LimitQuery, OffsetQuery, OrderQuery -from ormar.relations.alias_manager import AliasManager +from ormar.queryset.join import JoinParameters, SqlJoin if TYPE_CHECKING: # pragma no cover from ormar import Model -class JoinParameters(NamedTuple): - prev_model: Type["Model"] - previous_alias: str - from_table: str - model_cls: Type["Model"] - - class Query: def __init__( self, @@ -28,7 +20,6 @@ class Query: limit_count: int, offset: int, ) -> None: - self.query_offset = offset self.limit_count = limit_count self._select_related = select_related @@ -43,10 +34,6 @@ class Query: self.columns = None self.order_bys = None - @property - def relation_manager(self) -> AliasManager: - return self.model_cls.Meta.alias_manager - @property def prefixed_pk_name(self) -> str: return f"{self.table.name}.{self.model_cls.Meta.pkname}" @@ -63,17 +50,19 @@ class Query: self.model_cls, "", self.table.name, self.model_cls ) - for part in item.split("__"): - if issubclass( - join_parameters.model_cls.Meta.model_fields[part], ManyToManyField - ): - _fields = join_parameters.model_cls.Meta.model_fields - new_part = _fields[part].to.get_name() - join_parameters = self._build_join_parameters( - part, join_parameters, is_multi=True - ) - part = new_part - join_parameters = self._build_join_parameters(part, join_parameters) + sql_join = SqlJoin( + used_aliases=self.used_aliases, + select_from=self.select_from, + columns=self.columns, + order_bys=self.order_bys, + ) + + ( + self.used_aliases, + self.select_from, + self.columns, + self.order_bys, + ) = sql_join.build_join(item, join_parameters) expr = sqlalchemy.sql.select(self.columns) expr = expr.select_from(self.select_from) @@ -85,81 +74,6 @@ class Query: return expr - @staticmethod - def on_clause( - previous_alias: str, alias: str, from_clause: str, to_clause: str, - ) -> text: - left_part = f"{alias}_{to_clause}" - right_part = f"{previous_alias + '_' if previous_alias else ''}{from_clause}" - return text(f"{left_part}={right_part}") - - def _build_join_parameters( - self, part: str, join_params: JoinParameters, is_multi: bool = False - ) -> JoinParameters: - if is_multi: - model_cls = join_params.model_cls.Meta.model_fields[part].through - else: - model_cls = join_params.model_cls.Meta.model_fields[part].to - to_table = model_cls.Meta.table.name - - alias = model_cls.Meta.alias_manager.resolve_relation_join( - join_params.from_table, to_table - ) - if alias not in self.used_aliases: - self._process_join(join_params, is_multi, model_cls, part, alias) - - previous_alias = alias - from_table = to_table - prev_model = model_cls - return JoinParameters(prev_model, previous_alias, from_table, model_cls) - - def _process_join( - self, - join_params: JoinParameters, - is_multi: bool, - model_cls: Type["Model"], - part: str, - alias: str, - ) -> None: - to_table = model_cls.Meta.table.name - to_key, from_key = self._get_to_and_from_keys( - join_params, is_multi, model_cls, part - ) - - on_clause = self.on_clause( - previous_alias=join_params.previous_alias, - alias=alias, - from_clause=f"{join_params.from_table}.{from_key}", - to_clause=f"{to_table}.{to_key}", - ) - target_table = self.relation_manager.prefixed_table_name(alias, to_table) - self.select_from = sqlalchemy.sql.outerjoin( - self.select_from, target_table, on_clause - ) - self.order_bys.append(text(f"{alias}_{to_table}.{model_cls.Meta.pkname}")) - self.columns.extend( - self.relation_manager.prefixed_columns(alias, model_cls.Meta.table) - ) - self.used_aliases.append(alias) - - def _get_to_and_from_keys( - self, - join_params: JoinParameters, - is_multi: bool, - model_cls: Type["Model"], - part: str, - ) -> Tuple[str, str]: - if join_params.prev_model.Meta.model_fields[part].virtual or is_multi: - to_field = model_cls.resolve_relation_field( - model_cls, join_params.prev_model - ) - to_key = to_field.name - from_key = model_cls.Meta.pkname - else: - to_key = model_cls.Meta.pkname - from_key = part - return to_key, from_key - def _apply_expression_modifiers( self, expr: sqlalchemy.sql.select ) -> sqlalchemy.sql.select: From 0c4f6a0745d619cce304be852d1737e2996980e5 Mon Sep 17 00:00:00 2001 From: collerek Date: Thu, 17 Sep 2020 16:18:33 +0200 Subject: [PATCH 2/2] some refactors --- .coverage | Bin 53248 -> 53248 bytes ormar/queryset/queryset.py | 2 -- 2 files changed, 2 deletions(-) diff --git a/.coverage b/.coverage index 6a3129e1624e9f90986c5987781b7bbdb3eb3e26..49a54560217fb1081c660ae9a6013a303ccd779c 100644 GIT binary patch delta 39 ucmZozz}&Ead4q#KyP1`tIS_4j*Pjs2{rTSY`S#`I@7LwoZGO}z=Li5I`w&S0 delta 39 vcmZozz}&Ead4q#KyQ!6lrInG{W_SGw@!Z9+@%8`i+}*o-*U!z5`s5q|A_x&6 diff --git a/ormar/queryset/queryset.py b/ormar/queryset/queryset.py index a6c82f8..ef3d636 100644 --- a/ormar/queryset/queryset.py +++ b/ormar/queryset/queryset.py @@ -108,13 +108,11 @@ class QuerySet: async def exists(self) -> bool: expr = self.build_select_expression() expr = sqlalchemy.exists(expr).select() - # print(expr.compile(compile_kwargs={"literal_binds": True})) return await self.database.fetch_val(expr) async def count(self) -> int: expr = self.build_select_expression().alias("subquery_for_count") expr = sqlalchemy.func.count().select().select_from(expr) - # print(expr.compile(compile_kwargs={"literal_binds": True})) return await self.database.fetch_val(expr) async def delete(self, **kwargs: Any) -> int: