From 3e615a80571574097f5f3b96a5c0fe33541b2bf0 Mon Sep 17 00:00:00 2001 From: collerek Date: Wed, 2 Dec 2020 19:15:55 +0100 Subject: [PATCH 1/7] work in progres pydantic_only and properties --- ormar/fields/model_fields.py | 24 +++- ormar/models/metaclass.py | 136 +++++++++++++++------- ormar/models/modelproxy.py | 70 +++++------ ormar/models/newbasemodel.py | 94 +++++++-------- tests/test_excluding_fields_in_fastapi.py | 38 ++++++ tests/test_pydantic_only_fields.py | 75 ++++++++++++ 6 files changed, 299 insertions(+), 138 deletions(-) create mode 100644 tests/test_pydantic_only_fields.py diff --git a/ormar/fields/model_fields.py b/ormar/fields/model_fields.py index b92da65..f90ed87 100644 --- a/ormar/fields/model_fields.py +++ b/ormar/fields/model_fields.py @@ -12,13 +12,20 @@ from ormar.fields.base import BaseField # noqa I101 def is_field_nullable( - nullable: Optional[bool], default: Any, server_default: Any + nullable: Optional[bool], + default: Any, + server_default: Any, + pydantic_only: Optional[bool], ) -> bool: if nullable is None: - return default is not None or server_default is not None + return default is not None or server_default is not None or pydantic_only return nullable +def is_auto_primary_key(primary_key: bool, autoincrement: bool): + return primary_key and autoincrement + + class ModelFieldFactory: _bases: Any = (BaseField,) _type: Any = None @@ -29,19 +36,24 @@ class ModelFieldFactory: default = kwargs.pop("default", None) server_default = kwargs.pop("server_default", None) nullable = kwargs.pop("nullable", None) + pydantic_only = kwargs.pop("pydantic_only", False) + + primary_key = kwargs.pop("primary_key", False) + autoincrement = kwargs.pop("autoincrement", False) namespace = dict( __type__=cls._type, alias=kwargs.pop("name", None), name=None, - primary_key=kwargs.pop("primary_key", False), + primary_key=primary_key, default=default, server_default=server_default, - nullable=is_field_nullable(nullable, default, server_default), + nullable=is_field_nullable(nullable, default, server_default, pydantic_only) + or is_auto_primary_key(primary_key, autoincrement), index=kwargs.pop("index", False), unique=kwargs.pop("unique", False), - pydantic_only=kwargs.pop("pydantic_only", False), - autoincrement=kwargs.pop("autoincrement", False), + pydantic_only=pydantic_only, + autoincrement=autoincrement, column_type=cls.get_column_type(**kwargs), choices=set(kwargs.pop("choices", [])), **kwargs diff --git a/ormar/models/metaclass.py b/ormar/models/metaclass.py index 80265bd..2280c11 100644 --- a/ormar/models/metaclass.py +++ b/ormar/models/metaclass.py @@ -1,6 +1,7 @@ +import inspect import logging import warnings -from typing import Any, Dict, List, Optional, TYPE_CHECKING, Tuple, Type, Union +from typing import Any, Dict, List, Optional, Set, TYPE_CHECKING, Tuple, Type, Union import databases import pydantic @@ -15,6 +16,7 @@ from ormar import ForeignKey, ModelDefinitionError, Integer # noqa I100 from ormar.fields import BaseField from ormar.fields.foreign_key import ForeignKeyField from ormar.fields.many_to_many import ManyToMany, ManyToManyField +from ormar.fields.model_fields import ModelFieldFactory from ormar.queryset import QuerySet from ormar.relations.alias_manager import AliasManager @@ -36,6 +38,8 @@ class ModelMeta: str, Union[Type[BaseField], Type[ForeignKeyField], Type[ManyToManyField]] ] alias_manager: AliasManager + include_props_in_dict: bool + include_props_in_fields: bool def register_relation_on_build(table_name: str, field: Type[ForeignKeyField]) -> None: @@ -43,7 +47,7 @@ def register_relation_on_build(table_name: str, field: Type[ForeignKeyField]) -> def register_many_to_many_relation_on_build( - table_name: str, field: Type[ManyToManyField] + table_name: str, field: Type[ManyToManyField] ) -> None: alias_manager.add_relation_type(field.through.Meta.tablename, table_name) alias_manager.add_relation_type( @@ -52,11 +56,11 @@ def register_many_to_many_relation_on_build( def reverse_field_not_already_registered( - child: Type["Model"], child_model_name: str, parent_model: Type["Model"] + child: Type["Model"], child_model_name: str, parent_model: Type["Model"] ) -> bool: return ( - child_model_name not in parent_model.__fields__ - and child.get_name() not in parent_model.__fields__ + child_model_name not in parent_model.__fields__ + and child.get_name() not in parent_model.__fields__ ) @@ -67,7 +71,7 @@ def expand_reverse_relationships(model: Type["Model"]) -> None: parent_model = model_field.to child = model if reverse_field_not_already_registered( - child, child_model_name, parent_model + child, child_model_name, parent_model ): register_reverse_model_fields( parent_model, child, child_model_name, model_field @@ -75,10 +79,10 @@ def expand_reverse_relationships(model: Type["Model"]) -> None: def register_reverse_model_fields( - model: Type["Model"], - child: Type["Model"], - child_model_name: str, - model_field: Type["ForeignKeyField"], + model: Type["Model"], + child: Type["Model"], + child_model_name: str, + model_field: Type["ForeignKeyField"], ) -> None: if issubclass(model_field, ManyToManyField): model.Meta.model_fields[child_model_name] = ManyToMany( @@ -93,7 +97,7 @@ def register_reverse_model_fields( def adjust_through_many_to_many_model( - model: Type["Model"], child: Type["Model"], model_field: Type[ManyToManyField] + model: Type["Model"], child: Type["Model"], model_field: Type[ManyToManyField] ) -> None: model_field.through.Meta.model_fields[model.get_name()] = ForeignKey( model, real_name=model.get_name(), ondelete="CASCADE" @@ -110,7 +114,7 @@ def adjust_through_many_to_many_model( def create_pydantic_field( - field_name: str, model: Type["Model"], model_field: Type[ManyToManyField] + field_name: str, model: Type["Model"], model_field: Type[ManyToManyField] ) -> None: model_field.through.__fields__[field_name] = ModelField( name=field_name, @@ -121,8 +125,18 @@ def create_pydantic_field( ) +def get_pydantic_field(field_name: str, model: Type["Model"]) -> "ModelField": + return ModelField( + name=field_name, + type_=model.Meta.model_fields[field_name].__type__, + model_config=model.__config__, + required=not model.Meta.model_fields[field_name].nullable, + class_validators={}, + ) + + def create_and_append_m2m_fk( - model: Type["Model"], model_field: Type[ManyToManyField] + model: Type["Model"], model_field: Type[ManyToManyField] ) -> None: column = sqlalchemy.Column( model.get_name(), @@ -138,7 +152,7 @@ def create_and_append_m2m_fk( def check_pk_column_validity( - field_name: str, field: BaseField, pkname: Optional[str] + field_name: str, field: BaseField, pkname: Optional[str] ) -> Optional[str]: if pkname is not None: raise ModelDefinitionError("Only one primary key column is allowed.") @@ -148,7 +162,7 @@ def check_pk_column_validity( def sqlalchemy_columns_from_model_fields( - model_fields: Dict, table_name: str + model_fields: Dict, table_name: str ) -> Tuple[Optional[str], List[sqlalchemy.Column]]: columns = [] pkname = None @@ -162,9 +176,9 @@ def sqlalchemy_columns_from_model_fields( if field.primary_key: pkname = check_pk_column_validity(field_name, field, pkname) if ( - not field.pydantic_only - and not field.virtual - and not issubclass(field, ManyToManyField) + not field.pydantic_only + and not field.virtual + and not issubclass(field, ManyToManyField) ): columns.append(field.get_column(field.get_alias())) register_relation_in_alias_manager(table_name, field) @@ -172,7 +186,7 @@ def sqlalchemy_columns_from_model_fields( def register_relation_in_alias_manager( - table_name: str, field: Type[ForeignKeyField] + table_name: str, field: Type[ForeignKeyField] ) -> None: if issubclass(field, ManyToManyField): register_many_to_many_relation_on_build(table_name, field) @@ -181,7 +195,7 @@ def register_relation_in_alias_manager( def populate_default_pydantic_field_value( - ormar_field: Type[BaseField], field_name: str, attrs: dict + ormar_field: Type[BaseField], field_name: str, attrs: dict ) -> dict: curr_def_value = attrs.get(field_name, ormar.Undefined) if lenient_issubclass(curr_def_value, ormar.fields.BaseField): @@ -228,7 +242,7 @@ def extract_annotations_and_default_vals(attrs: dict) -> Tuple[Dict, Dict]: def populate_meta_tablename_columns_and_pk( - name: str, new_model: Type["Model"] + name: str, new_model: Type["Model"] ) -> Type["Model"]: tablename = name.lower() + "s" new_model.Meta.tablename = ( @@ -254,7 +268,7 @@ def populate_meta_tablename_columns_and_pk( def populate_meta_sqlalchemy_table_if_required( - new_model: Type["Model"], + new_model: Type["Model"], ) -> Type["Model"]: if not hasattr(new_model.Meta, "table"): new_model.Meta.table = sqlalchemy.Table( @@ -295,21 +309,66 @@ def choices_validator(cls: Type["Model"], values: Dict[str, Any]) -> Dict[str, A return values -def populate_choices_validators( # noqa CCR001 - model: Type["Model"], attrs: Dict -) -> None: +def populate_choices_validators(model: Type["Model"]) -> None: # noqa CCR001 if model_initialized_and_has_model_fields(model): for _, field in model.Meta.model_fields.items(): if check_if_field_has_choices(field): - validators = attrs.get("__pre_root_validators__", []) + validators = getattr(model, "__pre_root_validators__", []) if choices_validator not in validators: validators.append(choices_validator) - attrs["__pre_root_validators__"] = validators + setattr(model, "__pre_root_validators__", validators) + + +def populate_default_options_values(new_model: Type["Model"], model_fields: Dict): + if not hasattr(new_model.Meta, "constraints"): + new_model.Meta.constraints = [] + if not hasattr(new_model.Meta, "model_fields"): + new_model.Meta.model_fields = model_fields + + if not hasattr(new_model.Meta, "include_props_in_dict"): + new_model.Meta.include_props_in_dict = True + if not hasattr(new_model.Meta, "include_props_in_fields"): + new_model.Meta.include_props_in_fields = False + + +def add_cached_properties(new_model): + new_model._props = { + prop + for prop in vars(new_model) + if isinstance(getattr(new_model, prop), property) + and prop + not in ("__values__", "__fields__", "fields", "pk_column", "saved") + } + new_model._quick_access_fields = { + "_orm_id", + "_orm_saved", + "_orm", + "__fields__", + "_related_names", + "_props", + "__class__", + "extract_related_names", + } + new_model._related_names = None + + +def add_property_fields(new_model): + if new_model.Meta.include_props_in_fields: + for prop in new_model._props: + field_type = getattr(new_model, prop).fget.__annotations__.get('return') + new_model.Meta.model_fields[prop] = ModelFieldFactory(nullable=True, pydantic_only=True) + new_model.__fields__[prop] = ModelField( + name=prop, + type_=Optional[field_type] if field_type else Any, + model_config=new_model.__config__, + required=False, + class_validators={}, + ) class ModelMetaclass(pydantic.main.ModelMetaclass): def __new__( # type: ignore - mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict + mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict ) -> "ModelMetaclass": attrs["Config"] = get_pydantic_base_orm_config() attrs["__name__"] = name @@ -319,28 +378,21 @@ class ModelMetaclass(pydantic.main.ModelMetaclass): ) if hasattr(new_model, "Meta"): - if not hasattr(new_model.Meta, "constraints"): - new_model.Meta.constraints = [] - if not hasattr(new_model.Meta, "model_fields"): - new_model.Meta.model_fields = model_fields + populate_default_options_values(new_model, model_fields) new_model = populate_meta_tablename_columns_and_pk(name, new_model) new_model = populate_meta_sqlalchemy_table_if_required(new_model) expand_reverse_relationships(new_model) - populate_choices_validators(new_model, attrs) - + populate_choices_validators(new_model) if new_model.Meta.pkname not in attrs["__annotations__"]: field_name = new_model.Meta.pkname - field = Integer(name=field_name, primary_key=True) attrs["__annotations__"][field_name] = Optional[int] # type: ignore - populate_default_pydantic_field_value( - field, field_name, attrs # type: ignore + attrs[field_name] = None + new_model.__fields__[field_name] = get_pydantic_field( + field_name=field_name, model=new_model ) - - new_model = super().__new__( # type: ignore - mcs, name, bases, attrs - ) - new_model.Meta.alias_manager = alias_manager new_model.objects = QuerySet(new_model) + add_cached_properties(new_model) + add_property_fields(new_model) return new_model diff --git a/ormar/models/modelproxy.py b/ormar/models/modelproxy.py index ac3c5ed..1f637cc 100644 --- a/ormar/models/modelproxy.py +++ b/ormar/models/modelproxy.py @@ -49,6 +49,7 @@ class ModelTableProxy: _related_names_hash: Union[str, bytes] pk: Any get_name: Callable + _props: Set def dict(self): # noqa A003 raise NotImplementedError # pragma no cover @@ -68,7 +69,7 @@ class ModelTableProxy: @staticmethod def get_clause_target_and_filter_column_name( - parent_model: Type["Model"], target_model: Type["Model"], reverse: bool + parent_model: Type["Model"], target_model: Type["Model"], reverse: bool ) -> Tuple[Type["Model"], str]: if reverse: field = target_model.resolve_relation_field(target_model, parent_model) @@ -83,10 +84,10 @@ class ModelTableProxy: @staticmethod def get_column_name_for_id_extraction( - parent_model: Type["Model"], - target_model: Type["Model"], - reverse: bool, - use_raw: bool, + parent_model: Type["Model"], + target_model: Type["Model"], + reverse: bool, + use_raw: bool, ) -> str: if reverse: column_name = parent_model.Meta.pkname @@ -109,7 +110,7 @@ class ModelTableProxy: def get_relation_model_id(self, target_field: Type["BaseField"]) -> Optional[int]: if target_field.virtual or issubclass( - target_field, ormar.fields.ManyToManyField + target_field, ormar.fields.ManyToManyField ): return self.pk related_name = self.resolve_relation_name(self, target_field.to) @@ -126,9 +127,9 @@ class ModelTableProxy: @classmethod def get_names_to_exclude( - cls, - fields: Optional[Union[Dict, Set]] = None, - exclude_fields: Optional[Union[Dict, Set]] = None, + cls, + fields: Optional[Union[Dict, Set]] = None, + exclude_fields: Optional[Union[Dict, Set]] = None, ) -> Set: fields_names = cls.extract_db_own_fields() if fields and fields is not Ellipsis: @@ -207,14 +208,13 @@ class ModelTableProxy: @classmethod def extract_related_names(cls) -> Set: - if isinstance(cls._related_names_hash, (str, bytes)): + if isinstance(cls._related_names, Set): return cls._related_names related_names = set() for name, field in cls.Meta.model_fields.items(): if inspect.isclass(field) and issubclass(field, ForeignKeyField): related_names.add(name) - cls._related_names_hash = json.dumps(list(cls.Meta.model_fields.keys())) cls._related_names = related_names return related_names @@ -241,9 +241,9 @@ class ModelTableProxy: @classmethod def _update_excluded_with_related_not_required( - cls, - exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None], - nested: bool = False, + cls, + exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None], + nested: bool = False, ) -> Union[Set, Dict]: exclude = exclude or {} related_set = cls._exclude_related_names_not_required(nested=nested) @@ -269,18 +269,18 @@ class ModelTableProxy: @staticmethod def resolve_relation_name( # noqa CCR001 - item: Union[ - "NewBaseModel", - Type["NewBaseModel"], - "ModelTableProxy", - Type["ModelTableProxy"], - ], - related: Union[ - "NewBaseModel", - Type["NewBaseModel"], - "ModelTableProxy", - Type["ModelTableProxy"], - ], + item: Union[ + "NewBaseModel", + Type["NewBaseModel"], + "ModelTableProxy", + Type["ModelTableProxy"], + ], + related: Union[ + "NewBaseModel", + Type["NewBaseModel"], + "ModelTableProxy", + Type["ModelTableProxy"], + ], ) -> str: for name, field in item.Meta.model_fields.items(): if issubclass(field, ForeignKeyField): @@ -296,7 +296,7 @@ class ModelTableProxy: @staticmethod def resolve_relation_field( - item: Union["Model", Type["Model"]], related: Union["Model", Type["Model"]] + item: Union["Model", Type["Model"]], related: Union["Model", Type["Model"]] ) -> Type[BaseField]: name = ModelTableProxy.resolve_relation_name(item, related) to_field = item.Meta.model_fields.get(name) @@ -343,12 +343,12 @@ class ModelTableProxy: for field in one.Meta.model_fields.keys(): current_field = getattr(one, field) if isinstance(current_field, list) and not isinstance( - current_field, ormar.Model + current_field, ormar.Model ): setattr(other, field, current_field + getattr(other, field)) elif ( - isinstance(current_field, ormar.Model) - and current_field.pk == getattr(other, field).pk + isinstance(current_field, ormar.Model) + and current_field.pk == getattr(other, field).pk ): setattr( other, @@ -360,7 +360,7 @@ class ModelTableProxy: @staticmethod def _populate_pk_column( - model: Type["Model"], columns: List[str], use_alias: bool = False, + model: Type["Model"], columns: List[str], use_alias: bool = False, ) -> List[str]: pk_alias = ( model.get_column_alias(model.Meta.pkname) @@ -373,10 +373,10 @@ class ModelTableProxy: @staticmethod def own_table_columns( - model: Type["Model"], - fields: Optional[Union[Set, Dict]], - exclude_fields: Optional[Union[Set, Dict]], - use_alias: bool = False, + model: Type["Model"], + fields: Optional[Union[Set, Dict]], + exclude_fields: Optional[Union[Set, Dict]], + use_alias: bool = False, ) -> List[str]: columns = [ model.get_column_name_from_alias(col.name) if not use_alias else col.name diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index 837c10f..77879c4 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -51,9 +51,6 @@ class NewBaseModel( "_orm_id", "_orm_saved", "_orm", - "_related_names", - "_related_names_hash", - "_props", ) if TYPE_CHECKING: # pragma no cover @@ -70,7 +67,7 @@ class NewBaseModel( _orm_saved: bool _related_names: Set _related_names_hash: str - _props: List[str] + _props: Set Meta: ModelMeta # noinspection PyMissingConstructor @@ -158,27 +155,22 @@ class NewBaseModel( self.set_save_status(False) def __getattribute__(self, item: str) -> Any: - if item in ( - "_orm_id", - "_orm_saved", - "_orm", - "__fields__", - "_related_names", - "_props", - ): + if item in object.__getattribute__(self, '_quick_access_fields'): return object.__getattribute__(self, item) if item == "pk": return self.__dict__.get(self.Meta.pkname, None) - if item != "extract_related_names" and item in self.extract_related_names(): + if item in self.extract_related_names(): return self._extract_related_model_instead_of_field(item) - if item != "__fields__" and item in self.__fields__: + if item in self._props: + return object.__getattribute__(self, item) + if item in self.__fields__: value = self.__dict__.get(item, None) value = self._convert_json(item, value, "loads") return value return super().__getattribute__(item) def _extract_related_model_instead_of_field( - self, item: str + self, item: str ) -> Optional[Union["T", Sequence["T"]]]: # alias = self.get_column_alias(item) if item in self._orm: @@ -192,9 +184,9 @@ class NewBaseModel( def __same__(self, other: "NewBaseModel") -> bool: return ( - self._orm_id == other._orm_id - or self.dict() == other.dict() - or (self.pk == other.pk and self.pk is not None) + self._orm_id == other._orm_id + or self.dict() == other.dict() + or (self.pk == other.pk and self.pk is not None) ) @classmethod @@ -228,19 +220,10 @@ class NewBaseModel( @classmethod def get_properties( - cls, include: Union[Set, Dict, None], exclude: Union[Set, Dict, None] + cls, include: Union[Set, Dict, None], exclude: Union[Set, Dict, None] ) -> List[str]: - if isinstance(cls._props, list): - props = cls._props - else: - props = [ - prop - for prop in dir(cls) - if isinstance(getattr(cls, prop), property) - and prop - not in ("__values__", "__fields__", "fields", "pk_column", "saved") - ] - cls._props = props + + props = cls._props if include: props = [prop for prop in props if prop in include] if exclude: @@ -248,7 +231,7 @@ class NewBaseModel( return props def _get_related_not_excluded_fields( - self, include: Optional[Dict], exclude: Optional[Dict], + self, include: Optional[Dict], exclude: Optional[Dict], ) -> List: fields = [field for field in self.extract_related_names()] if include: @@ -263,15 +246,15 @@ class NewBaseModel( @staticmethod def _extract_nested_models_from_list( - models: MutableSequence, - include: Union[Set, Dict, None], - exclude: Union[Set, Dict, None], + models: MutableSequence, + include: Union[Set, Dict, None], + exclude: Union[Set, Dict, None], ) -> List: result = [] for model in models: try: result.append( - model.dict(nested=True, include=include, exclude=exclude,) + model.dict(nested=True, include=include, exclude=exclude, ) ) except ReferenceError: # pragma no cover continue @@ -279,17 +262,17 @@ class NewBaseModel( @staticmethod def _skip_ellipsis( - items: Union[Set, Dict, None], key: str + items: Union[Set, Dict, None], key: str ) -> Union[Set, Dict, None]: result = Excludable.get_child(items, key) return result if result is not Ellipsis else None def _extract_nested_models( # noqa: CCR001 - self, - nested: bool, - dict_instance: Dict, - include: Optional[Dict], - exclude: Optional[Dict], + self, + nested: bool, + dict_instance: Dict, + include: Optional[Dict], + exclude: Optional[Dict], ) -> Dict: fields = self._get_related_not_excluded_fields(include=include, exclude=exclude) @@ -315,16 +298,16 @@ class NewBaseModel( return dict_instance def dict( # type: ignore # noqa A003 - self, - *, - include: Union[Set, Dict] = None, - exclude: Union[Set, Dict] = None, - by_alias: bool = False, - skip_defaults: bool = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - nested: bool = False, + self, + *, + include: Union[Set, Dict] = None, + exclude: Union[Set, Dict] = None, + by_alias: bool = False, + skip_defaults: bool = None, + exclude_unset: bool = False, + exclude_defaults: bool = False, + exclude_none: bool = False, + nested: bool = False, ) -> "DictStrAny": # noqa: A003' dict_instance = super().dict( include=include, @@ -349,9 +332,10 @@ class NewBaseModel( ) # include model properties as fields - props = self.get_properties(include=include, exclude=exclude) - if props: - dict_instance.update({prop: getattr(self, prop) for prop in props}) + if self.Meta.include_props_in_dict: + props = self.get_properties(include=include, exclude=exclude) + if props: + dict_instance.update({prop: getattr(self, prop) for prop in props}) return dict_instance @@ -379,4 +363,4 @@ class NewBaseModel( return value def _is_conversion_to_json_needed(self, column_name: str) -> bool: - return self.Meta.model_fields[column_name].__type__ == pydantic.Json + return column_name in self.Meta.model_fields and self.Meta.model_fields[column_name].__type__ == pydantic.Json diff --git a/tests/test_excluding_fields_in_fastapi.py b/tests/test_excluding_fields_in_fastapi.py index c9ebc46..c04ac94 100644 --- a/tests/test_excluding_fields_in_fastapi.py +++ b/tests/test_excluding_fields_in_fastapi.py @@ -64,6 +64,8 @@ class RandomModel(ormar.Model): metadata = metadata database = database + include_props_in_fields = True + id: int = ormar.Integer(primary_key=True) password: str = ormar.String(max_length=255, default=gen_pass) first_name: str = ormar.String(max_length=255, default="John") @@ -72,6 +74,10 @@ class RandomModel(ormar.Model): server_default=sqlalchemy.func.now() ) + @property + def full_name(self): + return ' '.join([self.first_name, self.last_name]) + class User(ormar.Model): class Meta: @@ -136,6 +142,12 @@ async def create_user5(user: RandomModel): return await user.save() +@app.post("/random2/", response_model=RandomModel) +async def create_user6(user: RandomModel): + user = await user.save() + return user.dict() + + def test_all_endpoints(): client = TestClient(app) with client as client: @@ -187,4 +199,30 @@ def test_all_endpoints(): "first_name", "last_name", "created_date", + "full_name" + ] + assert response.json().get("full_name") == "John Test" + + RandomModel.Meta.include_props_in_fields = False + user3 = {"last_name": "Test"} + response = client.post("/random/", json=user3) + assert list(response.json().keys()) == [ + "id", + "password", + "first_name", + "last_name", + "created_date", + "full_name" + ] + + RandomModel.Meta.include_props_in_dict = True + user3 = {"last_name": "Test"} + response = client.post("/random2/", json=user3) + assert list(response.json().keys()) == [ + "id", + "password", + "first_name", + "last_name", + "created_date", + "full_name" ] diff --git a/tests/test_pydantic_only_fields.py b/tests/test_pydantic_only_fields.py new file mode 100644 index 0000000..19ac37f --- /dev/null +++ b/tests/test_pydantic_only_fields.py @@ -0,0 +1,75 @@ +import datetime + +import databases +import pytest +import sqlalchemy +from pydantic import validator + +import ormar +from tests.settings import DATABASE_URL + +database = databases.Database(DATABASE_URL, force_rollback=True) +metadata = sqlalchemy.MetaData() + + +class Album(ormar.Model): + class Meta: + tablename = "albums" + metadata = metadata + database = database + include_props_in_dict = True + include_props_in_fields = True + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + timestamp: datetime.datetime = ormar.DateTime(pydantic_only=True) + + @property + def name10(self) -> str: + return self.name + '_10' + + @validator('name') + def test(cls, v): + return v + + +@pytest.fixture(autouse=True, scope="module") +def create_test_database(): + engine = sqlalchemy.create_engine(DATABASE_URL) + metadata.drop_all(engine) + metadata.create_all(engine) + yield + metadata.drop_all(engine) + + +@pytest.mark.asyncio +async def test_pydantic_only_fields(): + async with database: + async with database.transaction(force_rollback=True): + album = await Album.objects.create(name='Hitchcock') + assert album.pk is not None + assert album.saved + assert album.timestamp is None + + album = await Album.objects.exclude_fields('timestamp').get() + assert album.timestamp is None + + album = await Album.objects.fields({'name', 'timestamp'}).get() + assert album.timestamp is None + + test_dict = album.dict() + assert 'timestamp' in test_dict + assert test_dict['timestamp'] is None + + album.timestamp = datetime.datetime.now() + test_dict = album.dict() + assert 'timestamp' in test_dict + assert test_dict['timestamp'] is not None + assert test_dict.get('name10') == 'Hitchcock_10' + + Album.Meta.include_props_in_dict = False + test_dict = album.dict() + assert 'timestamp' in test_dict + assert test_dict['timestamp'] is not None + # key is still there as now it's a field + assert test_dict['name10'] is None From 4e10ff03e210c6e00407cd760b30f9344fbdd87e Mon Sep 17 00:00:00 2001 From: collerek Date: Wed, 2 Dec 2020 20:39:30 +0100 Subject: [PATCH 2/7] more optimizations --- ormar/models/metaclass.py | 102 +++++++++++++++++++++-------------- ormar/models/modelproxy.py | 66 +++++++++++------------ ormar/models/newbasemodel.py | 79 ++++++++++++++------------- 3 files changed, 138 insertions(+), 109 deletions(-) diff --git a/ormar/models/metaclass.py b/ormar/models/metaclass.py index 2280c11..5c569dd 100644 --- a/ormar/models/metaclass.py +++ b/ormar/models/metaclass.py @@ -47,7 +47,7 @@ def register_relation_on_build(table_name: str, field: Type[ForeignKeyField]) -> def register_many_to_many_relation_on_build( - table_name: str, field: Type[ManyToManyField] + table_name: str, field: Type[ManyToManyField] ) -> None: alias_manager.add_relation_type(field.through.Meta.tablename, table_name) alias_manager.add_relation_type( @@ -56,11 +56,11 @@ def register_many_to_many_relation_on_build( def reverse_field_not_already_registered( - child: Type["Model"], child_model_name: str, parent_model: Type["Model"] + child: Type["Model"], child_model_name: str, parent_model: Type["Model"] ) -> bool: return ( - child_model_name not in parent_model.__fields__ - and child.get_name() not in parent_model.__fields__ + child_model_name not in parent_model.__fields__ + and child.get_name() not in parent_model.__fields__ ) @@ -71,7 +71,7 @@ def expand_reverse_relationships(model: Type["Model"]) -> None: parent_model = model_field.to child = model if reverse_field_not_already_registered( - child, child_model_name, parent_model + child, child_model_name, parent_model ): register_reverse_model_fields( parent_model, child, child_model_name, model_field @@ -79,10 +79,10 @@ def expand_reverse_relationships(model: Type["Model"]) -> None: def register_reverse_model_fields( - model: Type["Model"], - child: Type["Model"], - child_model_name: str, - model_field: Type["ForeignKeyField"], + model: Type["Model"], + child: Type["Model"], + child_model_name: str, + model_field: Type["ForeignKeyField"], ) -> None: if issubclass(model_field, ManyToManyField): model.Meta.model_fields[child_model_name] = ManyToMany( @@ -97,7 +97,7 @@ def register_reverse_model_fields( def adjust_through_many_to_many_model( - model: Type["Model"], child: Type["Model"], model_field: Type[ManyToManyField] + model: Type["Model"], child: Type["Model"], model_field: Type[ManyToManyField] ) -> None: model_field.through.Meta.model_fields[model.get_name()] = ForeignKey( model, real_name=model.get_name(), ondelete="CASCADE" @@ -114,7 +114,7 @@ def adjust_through_many_to_many_model( def create_pydantic_field( - field_name: str, model: Type["Model"], model_field: Type[ManyToManyField] + field_name: str, model: Type["Model"], model_field: Type[ManyToManyField] ) -> None: model_field.through.__fields__[field_name] = ModelField( name=field_name, @@ -136,7 +136,7 @@ def get_pydantic_field(field_name: str, model: Type["Model"]) -> "ModelField": def create_and_append_m2m_fk( - model: Type["Model"], model_field: Type[ManyToManyField] + model: Type["Model"], model_field: Type[ManyToManyField] ) -> None: column = sqlalchemy.Column( model.get_name(), @@ -152,7 +152,7 @@ def create_and_append_m2m_fk( def check_pk_column_validity( - field_name: str, field: BaseField, pkname: Optional[str] + field_name: str, field: BaseField, pkname: Optional[str] ) -> Optional[str]: if pkname is not None: raise ModelDefinitionError("Only one primary key column is allowed.") @@ -162,7 +162,7 @@ def check_pk_column_validity( def sqlalchemy_columns_from_model_fields( - model_fields: Dict, table_name: str + model_fields: Dict, table_name: str ) -> Tuple[Optional[str], List[sqlalchemy.Column]]: columns = [] pkname = None @@ -176,9 +176,9 @@ def sqlalchemy_columns_from_model_fields( if field.primary_key: pkname = check_pk_column_validity(field_name, field, pkname) if ( - not field.pydantic_only - and not field.virtual - and not issubclass(field, ManyToManyField) + not field.pydantic_only + and not field.virtual + and not issubclass(field, ManyToManyField) ): columns.append(field.get_column(field.get_alias())) register_relation_in_alias_manager(table_name, field) @@ -186,7 +186,7 @@ def sqlalchemy_columns_from_model_fields( def register_relation_in_alias_manager( - table_name: str, field: Type[ForeignKeyField] + table_name: str, field: Type[ForeignKeyField] ) -> None: if issubclass(field, ManyToManyField): register_many_to_many_relation_on_build(table_name, field) @@ -195,7 +195,7 @@ def register_relation_in_alias_manager( def populate_default_pydantic_field_value( - ormar_field: Type[BaseField], field_name: str, attrs: dict + ormar_field: Type[BaseField], field_name: str, attrs: dict ) -> dict: curr_def_value = attrs.get(field_name, ormar.Undefined) if lenient_issubclass(curr_def_value, ormar.fields.BaseField): @@ -242,7 +242,7 @@ def extract_annotations_and_default_vals(attrs: dict) -> Tuple[Dict, Dict]: def populate_meta_tablename_columns_and_pk( - name: str, new_model: Type["Model"] + name: str, new_model: Type["Model"] ) -> Type["Model"]: tablename = name.lower() + "s" new_model.Meta.tablename = ( @@ -268,7 +268,7 @@ def populate_meta_tablename_columns_and_pk( def populate_meta_sqlalchemy_table_if_required( - new_model: Type["Model"], + new_model: Type["Model"], ) -> Type["Model"]: if not hasattr(new_model.Meta, "table"): new_model.Meta.table = sqlalchemy.Table( @@ -333,30 +333,54 @@ def populate_default_options_values(new_model: Type["Model"], model_fields: Dict def add_cached_properties(new_model): new_model._props = { - prop - for prop in vars(new_model) - if isinstance(getattr(new_model, prop), property) - and prop - not in ("__values__", "__fields__", "fields", "pk_column", "saved") - } + prop + for prop in vars(new_model) + if isinstance(getattr(new_model, prop), property) + and prop not in ("__values__", "__fields__", "fields", "pk_column", "saved") + } new_model._quick_access_fields = { - "_orm_id", - "_orm_saved", - "_orm", - "__fields__", - "_related_names", - "_props", - "__class__", - "extract_related_names", - } + "_orm_id", + "_orm_saved", + "_orm", + "_convert_json", + "__fields__", + "_related_names", + "_props", + "__class__", + "__dict__", + "__config__", + "_iter", + "_get_value", + "_is_conversion_to_json_needed", + "__fields_set__", + "_skip_ellipsis", + "_calculate_keys", + "dict", + "_update_excluded_with_related_not_required", + "_extract_nested_models", + "_get_related_not_excluded_fields", + "get_properties", + "resolve_relation_name", + "resolve_relation_field", + "set_save_status", + "__pre_root_validators__", + "__post_root_validators__", + "_extract_nested_models_from_list", + "get_name", + "extract_related_names", + "Meta", + } new_model._related_names = None + new_model._pydantic_fields = {name for name in new_model.__fields__} def add_property_fields(new_model): if new_model.Meta.include_props_in_fields: for prop in new_model._props: - field_type = getattr(new_model, prop).fget.__annotations__.get('return') - new_model.Meta.model_fields[prop] = ModelFieldFactory(nullable=True, pydantic_only=True) + field_type = getattr(new_model, prop).fget.__annotations__.get("return") + new_model.Meta.model_fields[prop] = ModelFieldFactory( + nullable=True, pydantic_only=True + ) new_model.__fields__[prop] = ModelField( name=prop, type_=Optional[field_type] if field_type else Any, @@ -368,7 +392,7 @@ def add_property_fields(new_model): class ModelMetaclass(pydantic.main.ModelMetaclass): def __new__( # type: ignore - mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict + mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict ) -> "ModelMetaclass": attrs["Config"] = get_pydantic_base_orm_config() attrs["__name__"] = name diff --git a/ormar/models/modelproxy.py b/ormar/models/modelproxy.py index 1f637cc..ca12672 100644 --- a/ormar/models/modelproxy.py +++ b/ormar/models/modelproxy.py @@ -69,7 +69,7 @@ class ModelTableProxy: @staticmethod def get_clause_target_and_filter_column_name( - parent_model: Type["Model"], target_model: Type["Model"], reverse: bool + parent_model: Type["Model"], target_model: Type["Model"], reverse: bool ) -> Tuple[Type["Model"], str]: if reverse: field = target_model.resolve_relation_field(target_model, parent_model) @@ -84,10 +84,10 @@ class ModelTableProxy: @staticmethod def get_column_name_for_id_extraction( - parent_model: Type["Model"], - target_model: Type["Model"], - reverse: bool, - use_raw: bool, + parent_model: Type["Model"], + target_model: Type["Model"], + reverse: bool, + use_raw: bool, ) -> str: if reverse: column_name = parent_model.Meta.pkname @@ -110,7 +110,7 @@ class ModelTableProxy: def get_relation_model_id(self, target_field: Type["BaseField"]) -> Optional[int]: if target_field.virtual or issubclass( - target_field, ormar.fields.ManyToManyField + target_field, ormar.fields.ManyToManyField ): return self.pk related_name = self.resolve_relation_name(self, target_field.to) @@ -127,9 +127,9 @@ class ModelTableProxy: @classmethod def get_names_to_exclude( - cls, - fields: Optional[Union[Dict, Set]] = None, - exclude_fields: Optional[Union[Dict, Set]] = None, + cls, + fields: Optional[Union[Dict, Set]] = None, + exclude_fields: Optional[Union[Dict, Set]] = None, ) -> Set: fields_names = cls.extract_db_own_fields() if fields and fields is not Ellipsis: @@ -241,9 +241,9 @@ class ModelTableProxy: @classmethod def _update_excluded_with_related_not_required( - cls, - exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None], - nested: bool = False, + cls, + exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None], + nested: bool = False, ) -> Union[Set, Dict]: exclude = exclude or {} related_set = cls._exclude_related_names_not_required(nested=nested) @@ -269,18 +269,18 @@ class ModelTableProxy: @staticmethod def resolve_relation_name( # noqa CCR001 - item: Union[ - "NewBaseModel", - Type["NewBaseModel"], - "ModelTableProxy", - Type["ModelTableProxy"], - ], - related: Union[ - "NewBaseModel", - Type["NewBaseModel"], - "ModelTableProxy", - Type["ModelTableProxy"], - ], + item: Union[ + "NewBaseModel", + Type["NewBaseModel"], + "ModelTableProxy", + Type["ModelTableProxy"], + ], + related: Union[ + "NewBaseModel", + Type["NewBaseModel"], + "ModelTableProxy", + Type["ModelTableProxy"], + ], ) -> str: for name, field in item.Meta.model_fields.items(): if issubclass(field, ForeignKeyField): @@ -296,7 +296,7 @@ class ModelTableProxy: @staticmethod def resolve_relation_field( - item: Union["Model", Type["Model"]], related: Union["Model", Type["Model"]] + item: Union["Model", Type["Model"]], related: Union["Model", Type["Model"]] ) -> Type[BaseField]: name = ModelTableProxy.resolve_relation_name(item, related) to_field = item.Meta.model_fields.get(name) @@ -343,12 +343,12 @@ class ModelTableProxy: for field in one.Meta.model_fields.keys(): current_field = getattr(one, field) if isinstance(current_field, list) and not isinstance( - current_field, ormar.Model + current_field, ormar.Model ): setattr(other, field, current_field + getattr(other, field)) elif ( - isinstance(current_field, ormar.Model) - and current_field.pk == getattr(other, field).pk + isinstance(current_field, ormar.Model) + and current_field.pk == getattr(other, field).pk ): setattr( other, @@ -360,7 +360,7 @@ class ModelTableProxy: @staticmethod def _populate_pk_column( - model: Type["Model"], columns: List[str], use_alias: bool = False, + model: Type["Model"], columns: List[str], use_alias: bool = False, ) -> List[str]: pk_alias = ( model.get_column_alias(model.Meta.pkname) @@ -373,10 +373,10 @@ class ModelTableProxy: @staticmethod def own_table_columns( - model: Type["Model"], - fields: Optional[Union[Set, Dict]], - exclude_fields: Optional[Union[Set, Dict]], - use_alias: bool = False, + model: Type["Model"], + fields: Optional[Union[Set, Dict]], + exclude_fields: Optional[Union[Set, Dict]], + use_alias: bool = False, ) -> List[str]: columns = [ model.get_column_name_from_alias(col.name) if not use_alias else col.name diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index 77879c4..64f703f 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -1,5 +1,6 @@ import json import uuid +from collections import Counter from typing import ( AbstractSet, Any, @@ -155,24 +156,25 @@ class NewBaseModel( self.set_save_status(False) def __getattribute__(self, item: str) -> Any: - if item in object.__getattribute__(self, '_quick_access_fields'): + if item in object.__getattribute__(self, "_quick_access_fields"): return object.__getattribute__(self, item) if item == "pk": - return self.__dict__.get(self.Meta.pkname, None) - if item in self.extract_related_names(): - return self._extract_related_model_instead_of_field(item) - if item in self._props: + return object.__getattribute__(self, "__dict__").get(self.Meta.pkname, None) + if item in object.__getattribute__(self, "extract_related_names")(): + return object.__getattribute__( + self, "_extract_related_model_instead_of_field" + )(item) + if item in object.__getattribute__(self, "_props"): return object.__getattribute__(self, item) - if item in self.__fields__: - value = self.__dict__.get(item, None) - value = self._convert_json(item, value, "loads") + if item in object.__getattribute__(self, "_pydantic_fields"): + value = object.__getattribute__(self, "__dict__").get(item, None) + value = object.__getattribute__(self, "_convert_json")(item, value, "loads") return value - return super().__getattribute__(item) + return object.__getattribute__(self, item) def _extract_related_model_instead_of_field( - self, item: str + self, item: str ) -> Optional[Union["T", Sequence["T"]]]: - # alias = self.get_column_alias(item) if item in self._orm: return self._orm.get(item) return None # pragma no cover @@ -184,9 +186,9 @@ class NewBaseModel( def __same__(self, other: "NewBaseModel") -> bool: return ( - self._orm_id == other._orm_id - or self.dict() == other.dict() - or (self.pk == other.pk and self.pk is not None) + self._orm_id == other._orm_id + or (self.pk == other.pk and self.pk is not None) + or self.dict() == other.dict() ) @classmethod @@ -220,7 +222,7 @@ class NewBaseModel( @classmethod def get_properties( - cls, include: Union[Set, Dict, None], exclude: Union[Set, Dict, None] + cls, include: Union[Set, Dict, None], exclude: Union[Set, Dict, None] ) -> List[str]: props = cls._props @@ -231,7 +233,7 @@ class NewBaseModel( return props def _get_related_not_excluded_fields( - self, include: Optional[Dict], exclude: Optional[Dict], + self, include: Optional[Dict], exclude: Optional[Dict], ) -> List: fields = [field for field in self.extract_related_names()] if include: @@ -246,15 +248,15 @@ class NewBaseModel( @staticmethod def _extract_nested_models_from_list( - models: MutableSequence, - include: Union[Set, Dict, None], - exclude: Union[Set, Dict, None], + models: MutableSequence, + include: Union[Set, Dict, None], + exclude: Union[Set, Dict, None], ) -> List: result = [] for model in models: try: result.append( - model.dict(nested=True, include=include, exclude=exclude, ) + model.dict(nested=True, include=include, exclude=exclude,) ) except ReferenceError: # pragma no cover continue @@ -262,17 +264,17 @@ class NewBaseModel( @staticmethod def _skip_ellipsis( - items: Union[Set, Dict, None], key: str + items: Union[Set, Dict, None], key: str ) -> Union[Set, Dict, None]: result = Excludable.get_child(items, key) return result if result is not Ellipsis else None def _extract_nested_models( # noqa: CCR001 - self, - nested: bool, - dict_instance: Dict, - include: Optional[Dict], - exclude: Optional[Dict], + self, + nested: bool, + dict_instance: Dict, + include: Optional[Dict], + exclude: Optional[Dict], ) -> Dict: fields = self._get_related_not_excluded_fields(include=include, exclude=exclude) @@ -298,16 +300,16 @@ class NewBaseModel( return dict_instance def dict( # type: ignore # noqa A003 - self, - *, - include: Union[Set, Dict] = None, - exclude: Union[Set, Dict] = None, - by_alias: bool = False, - skip_defaults: bool = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - nested: bool = False, + self, + *, + include: Union[Set, Dict] = None, + exclude: Union[Set, Dict] = None, + by_alias: bool = False, + skip_defaults: bool = None, + exclude_unset: bool = False, + exclude_defaults: bool = False, + exclude_none: bool = False, + nested: bool = False, ) -> "DictStrAny": # noqa: A003' dict_instance = super().dict( include=include, @@ -363,4 +365,7 @@ class NewBaseModel( return value def _is_conversion_to_json_needed(self, column_name: str) -> bool: - return column_name in self.Meta.model_fields and self.Meta.model_fields[column_name].__type__ == pydantic.Json + return ( + column_name in self.Meta.model_fields + and self.Meta.model_fields[column_name].__type__ == pydantic.Json + ) From b838fa1edf74e586be71d123194144a1da99a207 Mon Sep 17 00:00:00 2001 From: collerek Date: Thu, 3 Dec 2020 09:15:19 +0100 Subject: [PATCH 3/7] some cleanup and optimization --- ormar/fields/model_fields.py | 8 ++- ormar/models/metaclass.py | 55 +++++--------------- ormar/models/modelproxy.py | 6 +-- ormar/models/newbasemodel.py | 13 ++--- ormar/models/quick_access_views.py | 62 +++++++++++++++++++++++ tests/test_excluding_fields_in_fastapi.py | 8 +-- tests/test_pydantic_only_fields.py | 26 +++++----- 7 files changed, 106 insertions(+), 72 deletions(-) create mode 100644 ormar/models/quick_access_views.py diff --git a/ormar/fields/model_fields.py b/ormar/fields/model_fields.py index f90ed87..8cbbd5e 100644 --- a/ormar/fields/model_fields.py +++ b/ormar/fields/model_fields.py @@ -18,11 +18,15 @@ def is_field_nullable( pydantic_only: Optional[bool], ) -> bool: if nullable is None: - return default is not None or server_default is not None or pydantic_only + return ( + default is not None + or server_default is not None + or (pydantic_only is not None and pydantic_only) + ) return nullable -def is_auto_primary_key(primary_key: bool, autoincrement: bool): +def is_auto_primary_key(primary_key: bool, autoincrement: bool) -> bool: return primary_key and autoincrement diff --git a/ormar/models/metaclass.py b/ormar/models/metaclass.py index 5c569dd..4706029 100644 --- a/ormar/models/metaclass.py +++ b/ormar/models/metaclass.py @@ -1,7 +1,6 @@ -import inspect import logging import warnings -from typing import Any, Dict, List, Optional, Set, TYPE_CHECKING, Tuple, Type, Union +from typing import Any, Dict, List, Optional, TYPE_CHECKING, Tuple, Type, Union import databases import pydantic @@ -12,11 +11,12 @@ from pydantic.utils import lenient_issubclass from sqlalchemy.sql.schema import ColumnCollectionConstraint import ormar # noqa I100 -from ormar import ForeignKey, ModelDefinitionError, Integer # noqa I100 +from ormar import ForeignKey, Integer, ModelDefinitionError # noqa I100 from ormar.fields import BaseField from ormar.fields.foreign_key import ForeignKeyField from ormar.fields.many_to_many import ManyToMany, ManyToManyField from ormar.fields.model_fields import ModelFieldFactory +from ormar.models.quick_access_views import quick_access_set from ormar.queryset import QuerySet from ormar.relations.alias_manager import AliasManager @@ -128,7 +128,7 @@ def create_pydantic_field( def get_pydantic_field(field_name: str, model: Type["Model"]) -> "ModelField": return ModelField( name=field_name, - type_=model.Meta.model_fields[field_name].__type__, + type_=model.Meta.model_fields[field_name].__type__, # type: ignore model_config=model.__config__, required=not model.Meta.model_fields[field_name].nullable, class_validators={}, @@ -316,10 +316,12 @@ def populate_choices_validators(model: Type["Model"]) -> None: # noqa CCR001 validators = getattr(model, "__pre_root_validators__", []) if choices_validator not in validators: validators.append(choices_validator) - setattr(model, "__pre_root_validators__", validators) + model.__pre_root_validators__ = validators -def populate_default_options_values(new_model: Type["Model"], model_fields: Dict): +def populate_default_options_values( + new_model: Type["Model"], model_fields: Dict +) -> None: if not hasattr(new_model.Meta, "constraints"): new_model.Meta.constraints = [] if not hasattr(new_model.Meta, "model_fields"): @@ -331,59 +333,28 @@ def populate_default_options_values(new_model: Type["Model"], model_fields: Dict new_model.Meta.include_props_in_fields = False -def add_cached_properties(new_model): +def add_cached_properties(new_model: Type["Model"]) -> None: new_model._props = { prop for prop in vars(new_model) if isinstance(getattr(new_model, prop), property) and prop not in ("__values__", "__fields__", "fields", "pk_column", "saved") } - new_model._quick_access_fields = { - "_orm_id", - "_orm_saved", - "_orm", - "_convert_json", - "__fields__", - "_related_names", - "_props", - "__class__", - "__dict__", - "__config__", - "_iter", - "_get_value", - "_is_conversion_to_json_needed", - "__fields_set__", - "_skip_ellipsis", - "_calculate_keys", - "dict", - "_update_excluded_with_related_not_required", - "_extract_nested_models", - "_get_related_not_excluded_fields", - "get_properties", - "resolve_relation_name", - "resolve_relation_field", - "set_save_status", - "__pre_root_validators__", - "__post_root_validators__", - "_extract_nested_models_from_list", - "get_name", - "extract_related_names", - "Meta", - } + new_model._quick_access_fields = quick_access_set new_model._related_names = None new_model._pydantic_fields = {name for name in new_model.__fields__} -def add_property_fields(new_model): +def add_property_fields(new_model: Type["Model"]) -> None: if new_model.Meta.include_props_in_fields: for prop in new_model._props: field_type = getattr(new_model, prop).fget.__annotations__.get("return") - new_model.Meta.model_fields[prop] = ModelFieldFactory( + new_model.Meta.model_fields[prop] = ModelFieldFactory( # type: ignore nullable=True, pydantic_only=True ) new_model.__fields__[prop] = ModelField( name=prop, - type_=Optional[field_type] if field_type else Any, + type_=Optional[field_type] if field_type is not None else Any, # type: ignore model_config=new_model.__config__, required=False, class_validators={}, diff --git a/ormar/models/modelproxy.py b/ormar/models/modelproxy.py index ca12672..0523752 100644 --- a/ormar/models/modelproxy.py +++ b/ormar/models/modelproxy.py @@ -20,10 +20,6 @@ from typing import ( from ormar.exceptions import ModelPersistenceError, RelationshipInstanceError from ormar.queryset.utils import translate_list_to_dict, update -try: - import orjson as json -except ImportError: # pragma: nocover - import json # type: ignore import ormar # noqa: I100 from ormar.fields import BaseField @@ -45,7 +41,7 @@ Field = TypeVar("Field", bound=BaseField) class ModelTableProxy: if TYPE_CHECKING: # pragma no cover Meta: ModelMeta - _related_names: Set + _related_names: Optional[Set] _related_names_hash: Union[str, bytes] pk: Any get_name: Callable diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index 64f703f..82adee9 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -1,6 +1,5 @@ import json import uuid -from collections import Counter from typing import ( AbstractSet, Any, @@ -66,9 +65,11 @@ class NewBaseModel( _orm_relationship_manager: AliasManager _orm: RelationsManager _orm_saved: bool - _related_names: Set + _related_names: Optional[Set] _related_names_hash: str _props: Set + _pydantic_fields: Set + _quick_access_fields: Set Meta: ModelMeta # noinspection PyMissingConstructor @@ -170,7 +171,7 @@ class NewBaseModel( value = object.__getattribute__(self, "__dict__").get(item, None) value = object.__getattribute__(self, "_convert_json")(item, value, "loads") return value - return object.__getattribute__(self, item) + return object.__getattribute__(self, item) # pragma: no cover def _extract_related_model_instead_of_field( self, item: str @@ -223,13 +224,13 @@ class NewBaseModel( @classmethod def get_properties( cls, include: Union[Set, Dict, None], exclude: Union[Set, Dict, None] - ) -> List[str]: + ) -> Set[str]: props = cls._props if include: - props = [prop for prop in props if prop in include] + props = {prop for prop in props if prop in include} if exclude: - props = [prop for prop in props if prop not in exclude] + props = {prop for prop in props if prop not in exclude} return props def _get_related_not_excluded_fields( diff --git a/ormar/models/quick_access_views.py b/ormar/models/quick_access_views.py new file mode 100644 index 0000000..bbab89e --- /dev/null +++ b/ormar/models/quick_access_views.py @@ -0,0 +1,62 @@ +quick_access_set = { + "Config", + "Meta", + "__class__", + "__config__", + "__custom_root_type__", + "__dict__", + "__fields__", + "__fields_set__", + "__json_encoder__", + "__post_root_validators__", + "__pre_root_validators__", + "__same__", + "_calculate_keys", + "_convert_json", + "_extract_db_related_names", + "_extract_model_db_fields", + "_extract_nested_models", + "_extract_nested_models_from_list", + "_extract_own_model_fields", + "_get_related_not_excluded_fields", + "_get_value", + "_is_conversion_to_json_needed", + "_iter", + "_orm", + "_orm_id", + "_orm_saved", + "_props", + "_related_names", + "_skip_ellipsis", + "_update_and_follow", + "_update_excluded_with_related_not_required", + "copy", + "delete", + "dict", + "extract_related_names", + "from_dict", + "get_column_alias", + "get_column_name_from_alias", + "get_filtered_names_to_extract", + "get_name", + "get_properties", + "get_related_field_name", + "get_relation_model_id", + "json", + "keys", + "load", + "pk_column", + "pk_type", + "populate_default_values", + "remove", + "resolve_relation_field", + "resolve_relation_name", + "save", + "save_related", + "saved", + "set_save_status", + "translate_aliases_to_columns", + "translate_columns_to_aliases", + "update", + "upsert", +} diff --git a/tests/test_excluding_fields_in_fastapi.py b/tests/test_excluding_fields_in_fastapi.py index c04ac94..819f3a7 100644 --- a/tests/test_excluding_fields_in_fastapi.py +++ b/tests/test_excluding_fields_in_fastapi.py @@ -76,7 +76,7 @@ class RandomModel(ormar.Model): @property def full_name(self): - return ' '.join([self.first_name, self.last_name]) + return " ".join([self.first_name, self.last_name]) class User(ormar.Model): @@ -199,7 +199,7 @@ def test_all_endpoints(): "first_name", "last_name", "created_date", - "full_name" + "full_name", ] assert response.json().get("full_name") == "John Test" @@ -212,7 +212,7 @@ def test_all_endpoints(): "first_name", "last_name", "created_date", - "full_name" + "full_name", ] RandomModel.Meta.include_props_in_dict = True @@ -224,5 +224,5 @@ def test_all_endpoints(): "first_name", "last_name", "created_date", - "full_name" + "full_name", ] diff --git a/tests/test_pydantic_only_fields.py b/tests/test_pydantic_only_fields.py index 19ac37f..74d6e38 100644 --- a/tests/test_pydantic_only_fields.py +++ b/tests/test_pydantic_only_fields.py @@ -26,9 +26,9 @@ class Album(ormar.Model): @property def name10(self) -> str: - return self.name + '_10' + return self.name + "_10" - @validator('name') + @validator("name") def test(cls, v): return v @@ -46,30 +46,30 @@ def create_test_database(): async def test_pydantic_only_fields(): async with database: async with database.transaction(force_rollback=True): - album = await Album.objects.create(name='Hitchcock') + album = await Album.objects.create(name="Hitchcock") assert album.pk is not None assert album.saved assert album.timestamp is None - album = await Album.objects.exclude_fields('timestamp').get() + album = await Album.objects.exclude_fields("timestamp").get() assert album.timestamp is None - album = await Album.objects.fields({'name', 'timestamp'}).get() + album = await Album.objects.fields({"name", "timestamp"}).get() assert album.timestamp is None test_dict = album.dict() - assert 'timestamp' in test_dict - assert test_dict['timestamp'] is None + assert "timestamp" in test_dict + assert test_dict["timestamp"] is None album.timestamp = datetime.datetime.now() test_dict = album.dict() - assert 'timestamp' in test_dict - assert test_dict['timestamp'] is not None - assert test_dict.get('name10') == 'Hitchcock_10' + assert "timestamp" in test_dict + assert test_dict["timestamp"] is not None + assert test_dict.get("name10") == "Hitchcock_10" Album.Meta.include_props_in_dict = False test_dict = album.dict() - assert 'timestamp' in test_dict - assert test_dict['timestamp'] is not None + assert "timestamp" in test_dict + assert test_dict["timestamp"] is not None # key is still there as now it's a field - assert test_dict['name10'] is None + assert test_dict["name10"] is None From 3b164c76ded88a02d322d404768bde412bebbb52 Mon Sep 17 00:00:00 2001 From: collerek Date: Thu, 3 Dec 2020 16:39:14 +0100 Subject: [PATCH 4/7] revert adding props to fields --- ormar/models/metaclass.py | 85 ++++++++++++----------- ormar/models/modelproxy.py | 7 +- ormar/models/newbasemodel.py | 63 +++++++++-------- tests/test_excluding_fields_in_fastapi.py | 65 ++++++++++------- tests/test_pydantic_only_fields.py | 7 +- 5 files changed, 121 insertions(+), 106 deletions(-) diff --git a/ormar/models/metaclass.py b/ormar/models/metaclass.py index 4706029..55b0bbb 100644 --- a/ormar/models/metaclass.py +++ b/ormar/models/metaclass.py @@ -1,3 +1,4 @@ +import inspect import logging import warnings from typing import Any, Dict, List, Optional, TYPE_CHECKING, Tuple, Type, Union @@ -47,7 +48,7 @@ def register_relation_on_build(table_name: str, field: Type[ForeignKeyField]) -> def register_many_to_many_relation_on_build( - table_name: str, field: Type[ManyToManyField] + table_name: str, field: Type[ManyToManyField] ) -> None: alias_manager.add_relation_type(field.through.Meta.tablename, table_name) alias_manager.add_relation_type( @@ -56,11 +57,11 @@ def register_many_to_many_relation_on_build( def reverse_field_not_already_registered( - child: Type["Model"], child_model_name: str, parent_model: Type["Model"] + child: Type["Model"], child_model_name: str, parent_model: Type["Model"] ) -> bool: return ( - child_model_name not in parent_model.__fields__ - and child.get_name() not in parent_model.__fields__ + child_model_name not in parent_model.__fields__ + and child.get_name() not in parent_model.__fields__ ) @@ -71,7 +72,7 @@ def expand_reverse_relationships(model: Type["Model"]) -> None: parent_model = model_field.to child = model if reverse_field_not_already_registered( - child, child_model_name, parent_model + child, child_model_name, parent_model ): register_reverse_model_fields( parent_model, child, child_model_name, model_field @@ -79,10 +80,10 @@ def expand_reverse_relationships(model: Type["Model"]) -> None: def register_reverse_model_fields( - model: Type["Model"], - child: Type["Model"], - child_model_name: str, - model_field: Type["ForeignKeyField"], + model: Type["Model"], + child: Type["Model"], + child_model_name: str, + model_field: Type["ForeignKeyField"], ) -> None: if issubclass(model_field, ManyToManyField): model.Meta.model_fields[child_model_name] = ManyToMany( @@ -97,7 +98,7 @@ def register_reverse_model_fields( def adjust_through_many_to_many_model( - model: Type["Model"], child: Type["Model"], model_field: Type[ManyToManyField] + model: Type["Model"], child: Type["Model"], model_field: Type[ManyToManyField] ) -> None: model_field.through.Meta.model_fields[model.get_name()] = ForeignKey( model, real_name=model.get_name(), ondelete="CASCADE" @@ -114,7 +115,7 @@ def adjust_through_many_to_many_model( def create_pydantic_field( - field_name: str, model: Type["Model"], model_field: Type[ManyToManyField] + field_name: str, model: Type["Model"], model_field: Type[ManyToManyField] ) -> None: model_field.through.__fields__[field_name] = ModelField( name=field_name, @@ -136,7 +137,7 @@ def get_pydantic_field(field_name: str, model: Type["Model"]) -> "ModelField": def create_and_append_m2m_fk( - model: Type["Model"], model_field: Type[ManyToManyField] + model: Type["Model"], model_field: Type[ManyToManyField] ) -> None: column = sqlalchemy.Column( model.get_name(), @@ -152,7 +153,7 @@ def create_and_append_m2m_fk( def check_pk_column_validity( - field_name: str, field: BaseField, pkname: Optional[str] + field_name: str, field: BaseField, pkname: Optional[str] ) -> Optional[str]: if pkname is not None: raise ModelDefinitionError("Only one primary key column is allowed.") @@ -162,7 +163,7 @@ def check_pk_column_validity( def sqlalchemy_columns_from_model_fields( - model_fields: Dict, table_name: str + model_fields: Dict, table_name: str ) -> Tuple[Optional[str], List[sqlalchemy.Column]]: columns = [] pkname = None @@ -176,9 +177,9 @@ def sqlalchemy_columns_from_model_fields( if field.primary_key: pkname = check_pk_column_validity(field_name, field, pkname) if ( - not field.pydantic_only - and not field.virtual - and not issubclass(field, ManyToManyField) + not field.pydantic_only + and not field.virtual + and not issubclass(field, ManyToManyField) ): columns.append(field.get_column(field.get_alias())) register_relation_in_alias_manager(table_name, field) @@ -186,7 +187,7 @@ def sqlalchemy_columns_from_model_fields( def register_relation_in_alias_manager( - table_name: str, field: Type[ForeignKeyField] + table_name: str, field: Type[ForeignKeyField] ) -> None: if issubclass(field, ManyToManyField): register_many_to_many_relation_on_build(table_name, field) @@ -195,7 +196,7 @@ def register_relation_in_alias_manager( def populate_default_pydantic_field_value( - ormar_field: Type[BaseField], field_name: str, attrs: dict + ormar_field: Type[BaseField], field_name: str, attrs: dict ) -> dict: curr_def_value = attrs.get(field_name, ormar.Undefined) if lenient_issubclass(curr_def_value, ormar.fields.BaseField): @@ -242,7 +243,7 @@ def extract_annotations_and_default_vals(attrs: dict) -> Tuple[Dict, Dict]: def populate_meta_tablename_columns_and_pk( - name: str, new_model: Type["Model"] + name: str, new_model: Type["Model"] ) -> Type["Model"]: tablename = name.lower() + "s" new_model.Meta.tablename = ( @@ -268,7 +269,7 @@ def populate_meta_tablename_columns_and_pk( def populate_meta_sqlalchemy_table_if_required( - new_model: Type["Model"], + new_model: Type["Model"], ) -> Type["Model"]: if not hasattr(new_model.Meta, "table"): new_model.Meta.table = sqlalchemy.Table( @@ -320,7 +321,7 @@ def populate_choices_validators(model: Type["Model"]) -> None: # noqa CCR001 def populate_default_options_values( - new_model: Type["Model"], model_fields: Dict + new_model: Type["Model"], model_fields: Dict ) -> None: if not hasattr(new_model.Meta, "constraints"): new_model.Meta.constraints = [] @@ -335,10 +336,9 @@ def populate_default_options_values( def add_cached_properties(new_model: Type["Model"]) -> None: new_model._props = { - prop - for prop in vars(new_model) - if isinstance(getattr(new_model, prop), property) - and prop not in ("__values__", "__fields__", "fields", "pk_column", "saved") + prop[0] + for prop in inspect.getmembers(new_model, lambda o: isinstance(o, property)) + if prop[0] not in ("__values__", "__fields__", "fields", "pk_column", "saved") } new_model._quick_access_fields = quick_access_set new_model._related_names = None @@ -346,24 +346,27 @@ def add_cached_properties(new_model: Type["Model"]) -> None: def add_property_fields(new_model: Type["Model"]) -> None: - if new_model.Meta.include_props_in_fields: - for prop in new_model._props: - field_type = getattr(new_model, prop).fget.__annotations__.get("return") - new_model.Meta.model_fields[prop] = ModelFieldFactory( # type: ignore - nullable=True, pydantic_only=True - ) - new_model.__fields__[prop] = ModelField( - name=prop, - type_=Optional[field_type] if field_type is not None else Any, # type: ignore - model_config=new_model.__config__, - required=False, - class_validators={}, - ) + pass + + +# if new_model.Meta.include_props_in_fields: +# for prop in new_model._props: +# field_type = getattr(new_model, prop).fget.__annotations__.get("return") +# new_model.Meta.model_fields[prop] = ModelFieldFactory( # type: ignore +# nullable=True, pydantic_only=True +# ) +# new_model.__fields__[prop] = ModelField( +# name=prop, +# type_=Optional[field_type] if field_type is not None else Any, # type: ignore +# model_config=new_model.__config__, +# required=False, +# class_validators={}, +# ) class ModelMetaclass(pydantic.main.ModelMetaclass): def __new__( # type: ignore - mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict + mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict ) -> "ModelMetaclass": attrs["Config"] = get_pydantic_base_orm_config() attrs["__name__"] = name @@ -371,6 +374,7 @@ class ModelMetaclass(pydantic.main.ModelMetaclass): new_model = super().__new__( # type: ignore mcs, name, bases, attrs ) + add_cached_properties(new_model) if hasattr(new_model, "Meta"): populate_default_options_values(new_model, model_fields) @@ -387,7 +391,6 @@ class ModelMetaclass(pydantic.main.ModelMetaclass): ) new_model.Meta.alias_manager = alias_manager new_model.objects = QuerySet(new_model) - add_cached_properties(new_model) add_property_fields(new_model) return new_model diff --git a/ormar/models/modelproxy.py b/ormar/models/modelproxy.py index 0523752..eee9ae9 100644 --- a/ormar/models/modelproxy.py +++ b/ormar/models/modelproxy.py @@ -20,7 +20,6 @@ from typing import ( from ormar.exceptions import ModelPersistenceError, RelationshipInstanceError from ormar.queryset.utils import translate_list_to_dict, update - import ormar # noqa: I100 from ormar.fields import BaseField from ormar.fields.foreign_key import ForeignKeyField @@ -46,13 +45,11 @@ class ModelTableProxy: pk: Any get_name: Callable _props: Set - - def dict(self): # noqa A003 - raise NotImplementedError # pragma no cover + dict: Callable def _extract_own_model_fields(self) -> Dict: related_names = self.extract_related_names() - self_fields = {k: v for k, v in self.dict().items() if k not in related_names} + self_fields = self.dict(exclude=related_names) return self_fields @classmethod diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index 82adee9..7a7ee59 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -1,3 +1,4 @@ +import inspect import json import uuid from typing import ( @@ -74,7 +75,7 @@ class NewBaseModel( # noinspection PyMissingConstructor def __init__(self, *args: Any, **kwargs: Any) -> None: # type: ignore - + caller_name = inspect.currentframe().f_back.f_code.co_name object.__setattr__(self, "_orm_id", uuid.uuid4().hex) object.__setattr__(self, "_orm_saved", False) object.__setattr__( @@ -96,6 +97,8 @@ class NewBaseModel( if "pk" in kwargs: kwargs[self.Meta.pkname] = kwargs.pop("pk") # build the models to set them and validate but don't register + if self.Meta.include_props_in_dict: + kwargs = {k: v for k, v in kwargs.items() if k not in object.__getattribute__(self, '_props')} try: new_kwargs: Dict[str, Any] = { k: self._convert_json( @@ -174,7 +177,7 @@ class NewBaseModel( return object.__getattribute__(self, item) # pragma: no cover def _extract_related_model_instead_of_field( - self, item: str + self, item: str ) -> Optional[Union["T", Sequence["T"]]]: if item in self._orm: return self._orm.get(item) @@ -187,9 +190,9 @@ class NewBaseModel( def __same__(self, other: "NewBaseModel") -> bool: return ( - self._orm_id == other._orm_id - or (self.pk == other.pk and self.pk is not None) - or self.dict() == other.dict() + self._orm_id == other._orm_id + or (self.pk == other.pk and self.pk is not None) + or self.dict() == other.dict() ) @classmethod @@ -223,7 +226,7 @@ class NewBaseModel( @classmethod def get_properties( - cls, include: Union[Set, Dict, None], exclude: Union[Set, Dict, None] + cls, include: Union[Set, Dict, None], exclude: Union[Set, Dict, None] ) -> Set[str]: props = cls._props @@ -234,7 +237,7 @@ class NewBaseModel( return props def _get_related_not_excluded_fields( - self, include: Optional[Dict], exclude: Optional[Dict], + self, include: Optional[Dict], exclude: Optional[Dict], ) -> List: fields = [field for field in self.extract_related_names()] if include: @@ -249,15 +252,15 @@ class NewBaseModel( @staticmethod def _extract_nested_models_from_list( - models: MutableSequence, - include: Union[Set, Dict, None], - exclude: Union[Set, Dict, None], + models: MutableSequence, + include: Union[Set, Dict, None], + exclude: Union[Set, Dict, None], ) -> List: result = [] for model in models: try: result.append( - model.dict(nested=True, include=include, exclude=exclude,) + model.dict(nested=True, include=include, exclude=exclude, ) ) except ReferenceError: # pragma no cover continue @@ -265,17 +268,17 @@ class NewBaseModel( @staticmethod def _skip_ellipsis( - items: Union[Set, Dict, None], key: str + items: Union[Set, Dict, None], key: str ) -> Union[Set, Dict, None]: result = Excludable.get_child(items, key) return result if result is not Ellipsis else None def _extract_nested_models( # noqa: CCR001 - self, - nested: bool, - dict_instance: Dict, - include: Optional[Dict], - exclude: Optional[Dict], + self, + nested: bool, + dict_instance: Dict, + include: Optional[Dict], + exclude: Optional[Dict], ) -> Dict: fields = self._get_related_not_excluded_fields(include=include, exclude=exclude) @@ -301,17 +304,19 @@ class NewBaseModel( return dict_instance def dict( # type: ignore # noqa A003 - self, - *, - include: Union[Set, Dict] = None, - exclude: Union[Set, Dict] = None, - by_alias: bool = False, - skip_defaults: bool = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - nested: bool = False, + self, + *, + include: Union[Set, Dict] = None, + exclude: Union[Set, Dict] = None, + by_alias: bool = False, + skip_defaults: bool = None, + exclude_unset: bool = False, + exclude_defaults: bool = False, + exclude_none: bool = False, + nested: bool = False, ) -> "DictStrAny": # noqa: A003' + # callable_name = inspect.currentframe().f_back.f_code.co_name + # print('dict', callable_name) dict_instance = super().dict( include=include, exclude=self._update_excluded_with_related_not_required(exclude, nested), @@ -367,6 +372,6 @@ class NewBaseModel( def _is_conversion_to_json_needed(self, column_name: str) -> bool: return ( - column_name in self.Meta.model_fields - and self.Meta.model_fields[column_name].__type__ == pydantic.Json + column_name in self.Meta.model_fields + and self.Meta.model_fields[column_name].__type__ == pydantic.Json ) diff --git a/tests/test_excluding_fields_in_fastapi.py b/tests/test_excluding_fields_in_fastapi.py index 819f3a7..cb801ce 100644 --- a/tests/test_excluding_fields_in_fastapi.py +++ b/tests/test_excluding_fields_in_fastapi.py @@ -64,7 +64,7 @@ class RandomModel(ormar.Model): metadata = metadata database = database - include_props_in_fields = True + include_props_in_dict = True id: int = ormar.Integer(primary_key=True) password: str = ormar.String(max_length=255, default=gen_pass) @@ -139,7 +139,9 @@ async def create_user4(user: User2): @app.post("/random/", response_model=RandomModel) async def create_user5(user: RandomModel): - return await user.save() + user = await user.save() + print('returning') + return user @app.post("/random2/", response_model=RandomModel) @@ -148,7 +150,7 @@ async def create_user6(user: RandomModel): return user.dict() -def test_all_endpoints(): +def test_excluding_fields_in_endpoints(): client = TestClient(app) with client as client: user = { @@ -168,20 +170,24 @@ def test_all_endpoints(): "last_name": "Doe", } + print('before call') response = client.post("/users/", json=user2) created_user = User(**response.json()) assert created_user.pk is not None assert created_user.password is None + print('before call') response = client.post("/users2/", json=user) created_user2 = User(**response.json()) assert created_user2.pk is not None assert created_user2.password is None # response has only 3 fields from UserBase + print('before call') response = client.post("/users3/", json=user) assert list(response.json().keys()) == ["email", "first_name", "last_name"] + print('before call') response = client.post("/users4/", json=user) assert list(response.json().keys()) == [ "id", @@ -191,32 +197,40 @@ def test_all_endpoints(): "category", ] - user3 = {"last_name": "Test"} - response = client.post("/random/", json=user3) - assert list(response.json().keys()) == [ - "id", - "password", - "first_name", - "last_name", - "created_date", - "full_name", - ] - assert response.json().get("full_name") == "John Test" + # user3 = {"last_name": "Test"} + # print('before call') + # response = client.post("/random/", json=user3) + # assert list(response.json().keys()) == [ + # "id", + # "password", + # "first_name", + # "last_name", + # "created_date", + # "full_name", + # ] + # assert response.json().get("full_name") == "John Test" + # + # RandomModel.Meta.include_props_in_fields = False + # user3 = {"last_name": "Test"} + # print('before call') + # response = client.post("/random/", json=user3) + # assert list(response.json().keys()) == [ + # "id", + # "password", + # "first_name", + # "last_name", + # "created_date", + # "full_name", + # ] + # assert response.json().get("full_name") == "John Test" - RandomModel.Meta.include_props_in_fields = False - user3 = {"last_name": "Test"} - response = client.post("/random/", json=user3) - assert list(response.json().keys()) == [ - "id", - "password", - "first_name", - "last_name", - "created_date", - "full_name", - ] +def test_adding_fields_in_endpoints(): + client = TestClient(app) + with client as client: RandomModel.Meta.include_props_in_dict = True user3 = {"last_name": "Test"} + print('before call') response = client.post("/random2/", json=user3) assert list(response.json().keys()) == [ "id", @@ -226,3 +240,4 @@ def test_all_endpoints(): "created_date", "full_name", ] + assert response.json().get("full_name") == "John Test" diff --git a/tests/test_pydantic_only_fields.py b/tests/test_pydantic_only_fields.py index 74d6e38..6c6b14e 100644 --- a/tests/test_pydantic_only_fields.py +++ b/tests/test_pydantic_only_fields.py @@ -28,10 +28,6 @@ class Album(ormar.Model): def name10(self) -> str: return self.name + "_10" - @validator("name") - def test(cls, v): - return v - @pytest.fixture(autouse=True, scope="module") def create_test_database(): @@ -71,5 +67,4 @@ async def test_pydantic_only_fields(): test_dict = album.dict() assert "timestamp" in test_dict assert test_dict["timestamp"] is not None - # key is still there as now it's a field - assert test_dict["name10"] is None + assert test_dict.get("name10", 'aa') == 'aa' From 00ab8a6d1df8d86256dd7ff39a743e088d87e274 Mon Sep 17 00:00:00 2001 From: collerek Date: Fri, 4 Dec 2020 09:51:00 +0100 Subject: [PATCH 5/7] switch to decorator used to register property_fields and save it on Meta inner class to expose to cloned fastapi models --- ormar/__init__.py | 2 + ormar/decorators/__init__.py | 5 ++ ormar/decorators/property_field.py | 19 +++++ ormar/models/metaclass.py | 95 ++++++++++------------- ormar/models/modelproxy.py | 2 +- ormar/models/newbasemodel.py | 81 +++++++++---------- ormar/models/quick_access_views.py | 1 - tests/test_excluding_fields_in_fastapi.py | 67 ++++++++-------- tests/test_properties.py | 17 +++- tests/test_pydantic_only_fields.py | 29 ++++--- 10 files changed, 175 insertions(+), 143 deletions(-) create mode 100644 ormar/decorators/__init__.py create mode 100644 ormar/decorators/property_field.py diff --git a/ormar/__init__.py b/ormar/__init__.py index 41913b8..c970f7d 100644 --- a/ormar/__init__.py +++ b/ormar/__init__.py @@ -1,3 +1,4 @@ +from ormar.decorators import property_field from ormar.exceptions import ModelDefinitionError, ModelNotSet, MultipleMatches, NoMatch from ormar.protocols import QuerySetProtocol, RelationProtocol # noqa: I100 from ormar.fields import ( # noqa: I100 @@ -58,4 +59,5 @@ __all__ = [ "QuerySetProtocol", "RelationProtocol", "ModelMeta", + "property_field", ] diff --git a/ormar/decorators/__init__.py b/ormar/decorators/__init__.py new file mode 100644 index 0000000..7dfbe5e --- /dev/null +++ b/ormar/decorators/__init__.py @@ -0,0 +1,5 @@ +from ormar.decorators.property_field import property_field + +__all__ = [ + "property_field", +] diff --git a/ormar/decorators/property_field.py b/ormar/decorators/property_field.py new file mode 100644 index 0000000..8d6a2e2 --- /dev/null +++ b/ormar/decorators/property_field.py @@ -0,0 +1,19 @@ +import inspect +from collections.abc import Callable +from typing import Union + +from ormar.exceptions import ModelDefinitionError + + +def property_field(func: Callable) -> Union[property, Callable]: + if isinstance(func, property): # pragma: no cover + func.fget.__property_field__ = True + else: + arguments = list(inspect.signature(func).parameters.keys()) + if len(arguments) > 1 or arguments[0] != "self": + raise ModelDefinitionError( + "property_field decorator can be used " + "only on class methods with no arguments" + ) + func.__dict__["__property_field__"] = True + return func diff --git a/ormar/models/metaclass.py b/ormar/models/metaclass.py index 55b0bbb..8460e13 100644 --- a/ormar/models/metaclass.py +++ b/ormar/models/metaclass.py @@ -1,7 +1,6 @@ -import inspect import logging import warnings -from typing import Any, Dict, List, Optional, TYPE_CHECKING, Tuple, Type, Union +from typing import Any, Dict, List, Optional, Set, TYPE_CHECKING, Tuple, Type, Union import databases import pydantic @@ -16,7 +15,6 @@ from ormar import ForeignKey, Integer, ModelDefinitionError # noqa I100 from ormar.fields import BaseField from ormar.fields.foreign_key import ForeignKeyField from ormar.fields.many_to_many import ManyToMany, ManyToManyField -from ormar.fields.model_fields import ModelFieldFactory from ormar.models.quick_access_views import quick_access_set from ormar.queryset import QuerySet from ormar.relations.alias_manager import AliasManager @@ -39,8 +37,7 @@ class ModelMeta: str, Union[Type[BaseField], Type[ForeignKeyField], Type[ManyToManyField]] ] alias_manager: AliasManager - include_props_in_dict: bool - include_props_in_fields: bool + property_fields: Set def register_relation_on_build(table_name: str, field: Type[ForeignKeyField]) -> None: @@ -48,7 +45,7 @@ def register_relation_on_build(table_name: str, field: Type[ForeignKeyField]) -> def register_many_to_many_relation_on_build( - table_name: str, field: Type[ManyToManyField] + table_name: str, field: Type[ManyToManyField] ) -> None: alias_manager.add_relation_type(field.through.Meta.tablename, table_name) alias_manager.add_relation_type( @@ -57,11 +54,11 @@ def register_many_to_many_relation_on_build( def reverse_field_not_already_registered( - child: Type["Model"], child_model_name: str, parent_model: Type["Model"] + child: Type["Model"], child_model_name: str, parent_model: Type["Model"] ) -> bool: return ( - child_model_name not in parent_model.__fields__ - and child.get_name() not in parent_model.__fields__ + child_model_name not in parent_model.__fields__ + and child.get_name() not in parent_model.__fields__ ) @@ -72,7 +69,7 @@ def expand_reverse_relationships(model: Type["Model"]) -> None: parent_model = model_field.to child = model if reverse_field_not_already_registered( - child, child_model_name, parent_model + child, child_model_name, parent_model ): register_reverse_model_fields( parent_model, child, child_model_name, model_field @@ -80,10 +77,10 @@ def expand_reverse_relationships(model: Type["Model"]) -> None: def register_reverse_model_fields( - model: Type["Model"], - child: Type["Model"], - child_model_name: str, - model_field: Type["ForeignKeyField"], + model: Type["Model"], + child: Type["Model"], + child_model_name: str, + model_field: Type["ForeignKeyField"], ) -> None: if issubclass(model_field, ManyToManyField): model.Meta.model_fields[child_model_name] = ManyToMany( @@ -98,7 +95,7 @@ def register_reverse_model_fields( def adjust_through_many_to_many_model( - model: Type["Model"], child: Type["Model"], model_field: Type[ManyToManyField] + model: Type["Model"], child: Type["Model"], model_field: Type[ManyToManyField] ) -> None: model_field.through.Meta.model_fields[model.get_name()] = ForeignKey( model, real_name=model.get_name(), ondelete="CASCADE" @@ -115,7 +112,7 @@ def adjust_through_many_to_many_model( def create_pydantic_field( - field_name: str, model: Type["Model"], model_field: Type[ManyToManyField] + field_name: str, model: Type["Model"], model_field: Type[ManyToManyField] ) -> None: model_field.through.__fields__[field_name] = ModelField( name=field_name, @@ -137,7 +134,7 @@ def get_pydantic_field(field_name: str, model: Type["Model"]) -> "ModelField": def create_and_append_m2m_fk( - model: Type["Model"], model_field: Type[ManyToManyField] + model: Type["Model"], model_field: Type[ManyToManyField] ) -> None: column = sqlalchemy.Column( model.get_name(), @@ -153,7 +150,7 @@ def create_and_append_m2m_fk( def check_pk_column_validity( - field_name: str, field: BaseField, pkname: Optional[str] + field_name: str, field: BaseField, pkname: Optional[str] ) -> Optional[str]: if pkname is not None: raise ModelDefinitionError("Only one primary key column is allowed.") @@ -163,7 +160,7 @@ def check_pk_column_validity( def sqlalchemy_columns_from_model_fields( - model_fields: Dict, table_name: str + model_fields: Dict, table_name: str ) -> Tuple[Optional[str], List[sqlalchemy.Column]]: columns = [] pkname = None @@ -177,9 +174,9 @@ def sqlalchemy_columns_from_model_fields( if field.primary_key: pkname = check_pk_column_validity(field_name, field, pkname) if ( - not field.pydantic_only - and not field.virtual - and not issubclass(field, ManyToManyField) + not field.pydantic_only + and not field.virtual + and not issubclass(field, ManyToManyField) ): columns.append(field.get_column(field.get_alias())) register_relation_in_alias_manager(table_name, field) @@ -187,7 +184,7 @@ def sqlalchemy_columns_from_model_fields( def register_relation_in_alias_manager( - table_name: str, field: Type[ForeignKeyField] + table_name: str, field: Type[ForeignKeyField] ) -> None: if issubclass(field, ManyToManyField): register_many_to_many_relation_on_build(table_name, field) @@ -196,7 +193,7 @@ def register_relation_in_alias_manager( def populate_default_pydantic_field_value( - ormar_field: Type[BaseField], field_name: str, attrs: dict + ormar_field: Type[BaseField], field_name: str, attrs: dict ) -> dict: curr_def_value = attrs.get(field_name, ormar.Undefined) if lenient_issubclass(curr_def_value, ormar.fields.BaseField): @@ -243,7 +240,7 @@ def extract_annotations_and_default_vals(attrs: dict) -> Tuple[Dict, Dict]: def populate_meta_tablename_columns_and_pk( - name: str, new_model: Type["Model"] + name: str, new_model: Type["Model"] ) -> Type["Model"]: tablename = name.lower() + "s" new_model.Meta.tablename = ( @@ -269,7 +266,7 @@ def populate_meta_tablename_columns_and_pk( def populate_meta_sqlalchemy_table_if_required( - new_model: Type["Model"], + new_model: Type["Model"], ) -> Type["Model"]: if not hasattr(new_model.Meta, "table"): new_model.Meta.table = sqlalchemy.Table( @@ -321,52 +318,42 @@ def populate_choices_validators(model: Type["Model"]) -> None: # noqa CCR001 def populate_default_options_values( - new_model: Type["Model"], model_fields: Dict + new_model: Type["Model"], model_fields: Dict ) -> None: if not hasattr(new_model.Meta, "constraints"): new_model.Meta.constraints = [] if not hasattr(new_model.Meta, "model_fields"): new_model.Meta.model_fields = model_fields - if not hasattr(new_model.Meta, "include_props_in_dict"): - new_model.Meta.include_props_in_dict = True - if not hasattr(new_model.Meta, "include_props_in_fields"): - new_model.Meta.include_props_in_fields = False - def add_cached_properties(new_model: Type["Model"]) -> None: - new_model._props = { - prop[0] - for prop in inspect.getmembers(new_model, lambda o: isinstance(o, property)) - if prop[0] not in ("__values__", "__fields__", "fields", "pk_column", "saved") - } new_model._quick_access_fields = quick_access_set new_model._related_names = None new_model._pydantic_fields = {name for name in new_model.__fields__} -def add_property_fields(new_model: Type["Model"]) -> None: - pass +def property_fields_not_set(new_model: Type["Model"]) -> bool: + return ( + not hasattr(new_model.Meta, "property_fields") + or not new_model.Meta.property_fields + ) -# if new_model.Meta.include_props_in_fields: -# for prop in new_model._props: -# field_type = getattr(new_model, prop).fget.__annotations__.get("return") -# new_model.Meta.model_fields[prop] = ModelFieldFactory( # type: ignore -# nullable=True, pydantic_only=True -# ) -# new_model.__fields__[prop] = ModelField( -# name=prop, -# type_=Optional[field_type] if field_type is not None else Any, # type: ignore -# model_config=new_model.__config__, -# required=False, -# class_validators={}, -# ) +def add_property_fields(new_model: Type["Model"], attrs: Dict) -> None: # noqa: CCR001 + if property_fields_not_set(new_model): + props = set() + for var_name, value in attrs.items(): + if isinstance(value, property): + value = value.fget + field_config = getattr(value, "__property_field__", None) + if field_config: + props.add(var_name) + new_model.Meta.property_fields = props class ModelMetaclass(pydantic.main.ModelMetaclass): def __new__( # type: ignore - mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict + mcs: "ModelMetaclass", name: str, bases: Any, attrs: dict ) -> "ModelMetaclass": attrs["Config"] = get_pydantic_base_orm_config() attrs["__name__"] = name @@ -391,6 +378,6 @@ class ModelMetaclass(pydantic.main.ModelMetaclass): ) new_model.Meta.alias_manager = alias_manager new_model.objects = QuerySet(new_model) - add_property_fields(new_model) + add_property_fields(new_model, attrs) return new_model diff --git a/ormar/models/modelproxy.py b/ormar/models/modelproxy.py index eee9ae9..3a82230 100644 --- a/ormar/models/modelproxy.py +++ b/ormar/models/modelproxy.py @@ -45,7 +45,7 @@ class ModelTableProxy: pk: Any get_name: Callable _props: Set - dict: Callable + dict: Callable # noqa: A001, VNE003 def _extract_own_model_fields(self) -> Dict: related_names = self.extract_related_names() diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index 7a7ee59..35a5d57 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -1,4 +1,3 @@ -import inspect import json import uuid from typing import ( @@ -68,14 +67,12 @@ class NewBaseModel( _orm_saved: bool _related_names: Optional[Set] _related_names_hash: str - _props: Set _pydantic_fields: Set _quick_access_fields: Set Meta: ModelMeta # noinspection PyMissingConstructor def __init__(self, *args: Any, **kwargs: Any) -> None: # type: ignore - caller_name = inspect.currentframe().f_back.f_code.co_name object.__setattr__(self, "_orm_id", uuid.uuid4().hex) object.__setattr__(self, "_orm_saved", False) object.__setattr__( @@ -96,9 +93,14 @@ class NewBaseModel( if "pk" in kwargs: kwargs[self.Meta.pkname] = kwargs.pop("pk") + + # remove property fields values from validation + kwargs = { + k: v + for k, v in kwargs.items() + if k not in object.__getattribute__(self, "Meta").property_fields + } # build the models to set them and validate but don't register - if self.Meta.include_props_in_dict: - kwargs = {k: v for k, v in kwargs.items() if k not in object.__getattribute__(self, '_props')} try: new_kwargs: Dict[str, Any] = { k: self._convert_json( @@ -136,7 +138,7 @@ class NewBaseModel( ) def __setattr__(self, name: str, value: Any) -> None: # noqa CCR001 - if name in ("_orm_id", "_orm_saved", "_orm", "_related_names", "_props"): + if name in object.__getattribute__(self, "_quick_access_fields"): object.__setattr__(self, name, value) elif name == "pk": object.__setattr__(self, self.Meta.pkname, value) @@ -168,8 +170,9 @@ class NewBaseModel( return object.__getattribute__( self, "_extract_related_model_instead_of_field" )(item) - if item in object.__getattribute__(self, "_props"): - return object.__getattribute__(self, item) + if item in object.__getattribute__(self, "Meta").property_fields: + value = object.__getattribute__(self, item) + return value() if callable(value) else value if item in object.__getattribute__(self, "_pydantic_fields"): value = object.__getattribute__(self, "__dict__").get(item, None) value = object.__getattribute__(self, "_convert_json")(item, value, "loads") @@ -177,7 +180,7 @@ class NewBaseModel( return object.__getattribute__(self, item) # pragma: no cover def _extract_related_model_instead_of_field( - self, item: str + self, item: str ) -> Optional[Union["T", Sequence["T"]]]: if item in self._orm: return self._orm.get(item) @@ -190,9 +193,9 @@ class NewBaseModel( def __same__(self, other: "NewBaseModel") -> bool: return ( - self._orm_id == other._orm_id - or (self.pk == other.pk and self.pk is not None) - or self.dict() == other.dict() + self._orm_id == other._orm_id + or (self.pk == other.pk and self.pk is not None) + or self.dict() == other.dict() ) @classmethod @@ -226,10 +229,10 @@ class NewBaseModel( @classmethod def get_properties( - cls, include: Union[Set, Dict, None], exclude: Union[Set, Dict, None] + cls, include: Union[Set, Dict, None], exclude: Union[Set, Dict, None] ) -> Set[str]: - props = cls._props + props = cls.Meta.property_fields if include: props = {prop for prop in props if prop in include} if exclude: @@ -237,7 +240,7 @@ class NewBaseModel( return props def _get_related_not_excluded_fields( - self, include: Optional[Dict], exclude: Optional[Dict], + self, include: Optional[Dict], exclude: Optional[Dict], ) -> List: fields = [field for field in self.extract_related_names()] if include: @@ -252,15 +255,15 @@ class NewBaseModel( @staticmethod def _extract_nested_models_from_list( - models: MutableSequence, - include: Union[Set, Dict, None], - exclude: Union[Set, Dict, None], + models: MutableSequence, + include: Union[Set, Dict, None], + exclude: Union[Set, Dict, None], ) -> List: result = [] for model in models: try: result.append( - model.dict(nested=True, include=include, exclude=exclude, ) + model.dict(nested=True, include=include, exclude=exclude,) ) except ReferenceError: # pragma no cover continue @@ -268,17 +271,17 @@ class NewBaseModel( @staticmethod def _skip_ellipsis( - items: Union[Set, Dict, None], key: str + items: Union[Set, Dict, None], key: str ) -> Union[Set, Dict, None]: result = Excludable.get_child(items, key) return result if result is not Ellipsis else None def _extract_nested_models( # noqa: CCR001 - self, - nested: bool, - dict_instance: Dict, - include: Optional[Dict], - exclude: Optional[Dict], + self, + nested: bool, + dict_instance: Dict, + include: Optional[Dict], + exclude: Optional[Dict], ) -> Dict: fields = self._get_related_not_excluded_fields(include=include, exclude=exclude) @@ -304,16 +307,16 @@ class NewBaseModel( return dict_instance def dict( # type: ignore # noqa A003 - self, - *, - include: Union[Set, Dict] = None, - exclude: Union[Set, Dict] = None, - by_alias: bool = False, - skip_defaults: bool = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - nested: bool = False, + self, + *, + include: Union[Set, Dict] = None, + exclude: Union[Set, Dict] = None, + by_alias: bool = False, + skip_defaults: bool = None, + exclude_unset: bool = False, + exclude_defaults: bool = False, + exclude_none: bool = False, + nested: bool = False, ) -> "DictStrAny": # noqa: A003' # callable_name = inspect.currentframe().f_back.f_code.co_name # print('dict', callable_name) @@ -339,8 +342,8 @@ class NewBaseModel( exclude=exclude, # type: ignore ) - # include model properties as fields - if self.Meta.include_props_in_dict: + # include model properties as fields in dict + if object.__getattribute__(self, "Meta").property_fields: props = self.get_properties(include=include, exclude=exclude) if props: dict_instance.update({prop: getattr(self, prop) for prop in props}) @@ -372,6 +375,6 @@ class NewBaseModel( def _is_conversion_to_json_needed(self, column_name: str) -> bool: return ( - column_name in self.Meta.model_fields - and self.Meta.model_fields[column_name].__type__ == pydantic.Json + column_name in self.Meta.model_fields + and self.Meta.model_fields[column_name].__type__ == pydantic.Json ) diff --git a/ormar/models/quick_access_views.py b/ormar/models/quick_access_views.py index bbab89e..471c809 100644 --- a/ormar/models/quick_access_views.py +++ b/ormar/models/quick_access_views.py @@ -25,7 +25,6 @@ quick_access_set = { "_orm", "_orm_id", "_orm_saved", - "_props", "_related_names", "_skip_ellipsis", "_update_and_follow", diff --git a/tests/test_excluding_fields_in_fastapi.py b/tests/test_excluding_fields_in_fastapi.py index cb801ce..b9398ca 100644 --- a/tests/test_excluding_fields_in_fastapi.py +++ b/tests/test_excluding_fields_in_fastapi.py @@ -10,6 +10,7 @@ from fastapi import FastAPI from starlette.testclient import TestClient import ormar +from ormar import property_field from tests.settings import DATABASE_URL app = FastAPI() @@ -74,8 +75,8 @@ class RandomModel(ormar.Model): server_default=sqlalchemy.func.now() ) - @property - def full_name(self): + @property_field + def full_name(self) -> str: return " ".join([self.first_name, self.last_name]) @@ -140,7 +141,6 @@ async def create_user4(user: User2): @app.post("/random/", response_model=RandomModel) async def create_user5(user: RandomModel): user = await user.save() - print('returning') return user @@ -170,24 +170,20 @@ def test_excluding_fields_in_endpoints(): "last_name": "Doe", } - print('before call') response = client.post("/users/", json=user2) created_user = User(**response.json()) assert created_user.pk is not None assert created_user.password is None - print('before call') response = client.post("/users2/", json=user) created_user2 = User(**response.json()) assert created_user2.pk is not None assert created_user2.password is None # response has only 3 fields from UserBase - print('before call') response = client.post("/users3/", json=user) assert list(response.json().keys()) == ["email", "first_name", "last_name"] - print('before call') response = client.post("/users4/", json=user) assert list(response.json().keys()) == [ "id", @@ -197,40 +193,41 @@ def test_excluding_fields_in_endpoints(): "category", ] - # user3 = {"last_name": "Test"} - # print('before call') - # response = client.post("/random/", json=user3) - # assert list(response.json().keys()) == [ - # "id", - # "password", - # "first_name", - # "last_name", - # "created_date", - # "full_name", - # ] - # assert response.json().get("full_name") == "John Test" - # - # RandomModel.Meta.include_props_in_fields = False - # user3 = {"last_name": "Test"} - # print('before call') - # response = client.post("/random/", json=user3) - # assert list(response.json().keys()) == [ - # "id", - # "password", - # "first_name", - # "last_name", - # "created_date", - # "full_name", - # ] - # assert response.json().get("full_name") == "John Test" - def test_adding_fields_in_endpoints(): + client = TestClient(app) + with client as client: + user3 = {"last_name": "Test"} + response = client.post("/random/", json=user3) + assert list(response.json().keys()) == [ + "id", + "password", + "first_name", + "last_name", + "created_date", + "full_name", + ] + assert response.json().get("full_name") == "John Test" + + RandomModel.Meta.include_props_in_fields = False + user3 = {"last_name": "Test"} + response = client.post("/random/", json=user3) + assert list(response.json().keys()) == [ + "id", + "password", + "first_name", + "last_name", + "created_date", + "full_name", + ] + assert response.json().get("full_name") == "John Test" + + +def test_adding_fields_in_endpoints2(): client = TestClient(app) with client as client: RandomModel.Meta.include_props_in_dict = True user3 = {"last_name": "Test"} - print('before call') response = client.post("/random2/", json=user3) assert list(response.json().keys()) == [ "id", diff --git a/tests/test_properties.py b/tests/test_properties.py index b89db67..6456c6a 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -1,8 +1,10 @@ +# type: ignore import databases import pytest import sqlalchemy import ormar +from ormar import ModelDefinitionError, property_field from tests.settings import DATABASE_URL database = databases.Database(DATABASE_URL, force_rollback=True) @@ -19,15 +21,15 @@ class Song(ormar.Model): name: str = ormar.String(max_length=100) sort_order: int = ormar.Integer() - @property + @property_field def sorted_name(self): return f"{self.sort_order}: {self.name}" - @property + @property_field def sample(self): return "sample" - @property + @property_field def sample2(self): return "sample2" @@ -66,3 +68,12 @@ async def test_sort_order_on_main_model(): assert "sample" not in check_include assert "sample2" in check_include assert "sorted_name" in check_include + + +def test_wrong_definition(): + with pytest.raises(ModelDefinitionError): + + class WrongModel(ormar.Model): # pragma: no cover + @property_field + def test(self, aa=10, bb=30): + pass diff --git a/tests/test_pydantic_only_fields.py b/tests/test_pydantic_only_fields.py index 6c6b14e..ee58177 100644 --- a/tests/test_pydantic_only_fields.py +++ b/tests/test_pydantic_only_fields.py @@ -3,9 +3,9 @@ import datetime import databases import pytest import sqlalchemy -from pydantic import validator import ormar +from ormar import property_field from tests.settings import DATABASE_URL database = databases.Database(DATABASE_URL, force_rollback=True) @@ -17,17 +17,27 @@ class Album(ormar.Model): tablename = "albums" metadata = metadata database = database - include_props_in_dict = True - include_props_in_fields = True id: int = ormar.Integer(primary_key=True) name: str = ormar.String(max_length=100) timestamp: datetime.datetime = ormar.DateTime(pydantic_only=True) - @property + @property_field def name10(self) -> str: return self.name + "_10" + @property_field + def name20(self) -> str: + return self.name + "_20" + + @property + def name30(self) -> str: + return self.name + "_30" + + @property_field + def name40(self) -> str: + return self.name + "_40" + @pytest.fixture(autouse=True, scope="module") def create_test_database(): @@ -57,14 +67,13 @@ async def test_pydantic_only_fields(): assert "timestamp" in test_dict assert test_dict["timestamp"] is None + assert album.name30 == "Hitchcock_30" + album.timestamp = datetime.datetime.now() test_dict = album.dict() assert "timestamp" in test_dict assert test_dict["timestamp"] is not None assert test_dict.get("name10") == "Hitchcock_10" - - Album.Meta.include_props_in_dict = False - test_dict = album.dict() - assert "timestamp" in test_dict - assert test_dict["timestamp"] is not None - assert test_dict.get("name10", 'aa') == 'aa' + assert test_dict.get("name20") == "Hitchcock_20" + assert test_dict.get("name40") == "Hitchcock_40" + assert "name30" not in test_dict From f071d4538ee9093491839745933c6ea364819c9e Mon Sep 17 00:00:00 2001 From: collerek Date: Fri, 4 Dec 2020 15:10:00 +0100 Subject: [PATCH 6/7] bump version, update docs --- docs/models.md | 212 +++++++++++++++++++++- docs/releases.md | 8 + docs_src/models/docs014.py | 18 ++ docs_src/models/docs015.py | 23 +++ ormar/__init__.py | 2 +- ormar/models/modelproxy.py | 6 +- ormar/models/newbasemodel.py | 2 - tests/test_excluding_fields_in_fastapi.py | 68 ++++++- 8 files changed, 326 insertions(+), 13 deletions(-) create mode 100644 docs_src/models/docs014.py create mode 100644 docs_src/models/docs015.py diff --git a/docs/models.md b/docs/models.md index be91d12..a29fb70 100644 --- a/docs/models.md +++ b/docs/models.md @@ -17,6 +17,8 @@ To build an ormar model you simply need to inherit a `ormar.Model` class. Next assign one or more of the [Fields][fields] as a class level variables. +#### Basic Field Types + Each table **has to** have a primary key column, which you specify by setting `primary_key=True` on selected field. Only one primary key column is allowed. @@ -37,7 +39,212 @@ You can disable by passing `autoincremant=False`. id: int = ormar.Integer(primary_key=True, autoincrement=False) ``` -### Fields names vs Column names +#### Non Database Fields + +Note that if you need a normal pydantic field in your model (used to store value on model or pass around some value) you can define a +field with parameter `pydantic_only=True`. + +Fields created like this are added to the `pydantic` model fields -> so are subject to validation according to `Field` type, +also appear in `dict()` and `json()` result. + +The difference is that **those fields are not saved in the database**. So they won't be included in underlying sqlalchemy `columns`, +or `table` variables (check [Internals][Internals] section below to see how you can access those if you need). + +Subsequently `pydantic_only` fields won't be included in migrations or any database operation (like `save`, `update` etc.) + +Fields like those can be passed around into payload in `fastapi` request and will be returned in `fastapi` response +(of course only if you set their value somewhere in your code as the value is **not** fetched from the db. +If you pass a value in `fastapi` `request` and return the same instance that `fastapi` constructs for you in `request_model` +you should get back exactly same value in `response`.). + +!!!warning + `pydantic_only=True` fields are always **Optional** and it cannot be changed (otherwise db load validation would fail) + +!!!tip + `pydantic_only=True` fields are a good solution if you need to pass additional information from outside of your API + (i.e. frontend). They are not stored in db but you can access them in your `APIRoute` code and they also have `pydantic` validation. + +```Python hl_lines="18" +--8<-- "../docs_src/models/docs014.py" +``` + +If you combine `pydantic_only=True` field with `default` parameter and do not pass actual value in request you will always get default value. +Since it can be a function you can set `default=datetime.datetime.now` and get current timestamp each time you call an endpoint etc. + +!!!note + Note that both `pydantic_only` and `property_field` decorated field can be included/excluded in both `dict()` and `fastapi` + response with `include`/`exclude` and `response_model_include`/`response_model_exclude` accordingly. + +```python +# <==part of code removed for clarity==> +class User(ormar.Model): + class Meta: + tablename: str = "users2" + metadata = metadata + database = database + + id: int = ormar.Integer(primary_key=True) + email: str = ormar.String(max_length=255, nullable=False) + password: str = ormar.String(max_length=255) + first_name: str = ormar.String(max_length=255) + last_name: str = ormar.String(max_length=255) + category: str = ormar.String(max_length=255, nullable=True) + timestamp: datetime.datetime = ormar.DateTime( + pydantic_only=True, default=datetime.datetime.now + ) + +# <==part of code removed for clarity==> +app =FastAPI() + +@app.post("/users/") +async def create_user(user: User): + return await user.save() + +# <==part of code removed for clarity==> + +def test_excluding_fields_in_endpoints(): + client = TestClient(app) + with client as client: + timestamp = datetime.datetime.now() + + user = { + "email": "test@domain.com", + "password": "^*^%A*DA*IAAA", + "first_name": "John", + "last_name": "Doe", + "timestamp": str(timestamp), + } + response = client.post("/users/", json=user) + assert list(response.json().keys()) == [ + "id", + "email", + "first_name", + "last_name", + "category", + "timestamp", + ] + # returned is the same timestamp + assert response.json().get("timestamp") == str(timestamp).replace(" ", "T") + + +# <==part of code removed for clarity==> +``` + +#### Property fields + +Sometimes it's desirable to do some kind of calculation on the model instance. One of the most common examples can be concatenating +two or more fields. Imagine you have `first_name` and `last_name` fields on your model, but would like to have `full_name` in the result +of the `fastapi` query. + +You can create a new `pydantic` model with a `method` that accepts only `self` (so like default python `@property`) +and populate it in your code. + +But it's so common that `ormar` has you covered. You can "materialize" a `property_field` on you `Model`. + +!!!warning + `property_field` fields are always **Optional** and it cannot be changed (otherwise db load validation would fail) + +```Python hl_lines="20-22" +--8<-- "../docs_src/models/docs015.py" +``` + +!!!warning + The decorated function has to accept only one parameter, and that parameter have to be `self`. + + If you try to decorate a function with more parameters `ormar` will raise `ModelDefinitionError`. + + Sample: + + ```python + # will raise ModelDefinitionError + @property_field + def prefixed_name(self, prefix="prefix_"): + return 'custom_prefix__' + self.name + + # will raise ModelDefinitionError + # (calling first param something else than 'self' is a bad practice anyway) + @property_field + def prefixed_name(instance): + return 'custom_prefix__' + self.name + ``` + +Note that `property_field` decorated methods do not go through verification (but that might change in future) and are only available +in the response from `fastapi` and `dict()` and `json()` methods. You cannot pass a value for this field in the request +(or rather you can but it will be discarded by ormar so really no point but no Exception will be raised). + +!!!note + Note that both `pydantic_only` and `property_field` decorated field can be included/excluded in both `dict()` and `fastapi` + response with `include`/`exclude` and `response_model_include`/`response_model_exclude` accordingly. + +!!!tip + Note that `@property_field` decorator is designed to replace the python `@property` decorator, you do not have to combine them. + + In theory you can cause `ormar` have a failsafe mechanism, but note that i.e. `mypy` will complain about re-decorating a property. + + ```python + # valid and working but unnecessary and mypy will complain + @property_field + @property + def prefixed_name(self): + return 'custom_prefix__' + self.name + ``` + +```python +# <==part of code removed for clarity==> +def gen_pass(): # note: NOT production ready + choices = string.ascii_letters + string.digits + "!@#$%^&*()" + return "".join(random.choice(choices) for _ in range(20)) + +class RandomModel(ormar.Model): + class Meta: + tablename: str = "random_users" + metadata = metadata + database = database + + include_props_in_dict = True + + id: int = ormar.Integer(primary_key=True) + password: str = ormar.String(max_length=255, default=gen_pass) + first_name: str = ormar.String(max_length=255, default="John") + last_name: str = ormar.String(max_length=255) + created_date: datetime.datetime = ormar.DateTime( + server_default=sqlalchemy.func.now() + ) + + @property_field + def full_name(self) -> str: + return " ".join([self.first_name, self.last_name]) + +# <==part of code removed for clarity==> +app =FastAPI() + +# explicitly exclude property_field in this endpoint +@app.post("/random/", response_model=RandomModel, response_model_exclude={"full_name"}) +async def create_user(user: RandomModel): + return await user.save() + +# <==part of code removed for clarity==> + +def test_excluding_property_field_in_endpoints2(): + client = TestClient(app) + with client as client: + RandomModel.Meta.include_props_in_dict = True + user3 = {"last_name": "Test"} + response = client.post("/random3/", json=user3) + assert list(response.json().keys()) == [ + "id", + "password", + "first_name", + "last_name", + "created_date", + ] + # despite being decorated with property_field if you explictly exclude it it will be gone + assert response.json().get("full_name") is None + +# <==part of code removed for clarity==> +``` + +#### Fields names vs Column names By default names of the fields will be used for both the underlying `pydantic` model and `sqlalchemy` table. @@ -330,7 +537,7 @@ You can set this parameter by providing `Meta` class `constraints` argument. --8<-- "../docs_src/models/docs006.py" ``` -## Initialization +## Model Initialization There are two ways to create and persist the `Model` instance in the database. @@ -560,3 +767,4 @@ For example to list table model fields you can: [sqlalchemy table creation]: https://docs.sqlalchemy.org/en/13/core/metadata.html#creating-and-dropping-database-tables [alembic]: https://alembic.sqlalchemy.org/en/latest/tutorial.html [save status]: ../models/#model-save-status +[Internals]: #internals diff --git a/docs/releases.md b/docs/releases.md index e931b13..f940a36 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,3 +1,11 @@ +# 0.6.2 + +* Performance optimization +* Fix for bug with `pydantic_only` fields being required +* Add `property_field` decorator that registers a function as a property that will + be included in `Model.dict()` and in `fastapi` response +* Update docs + # 0.6.1 * Explicitly set None to excluded nullable fields to avoid pydantic setting a default value (fix [#60][#60]). diff --git a/docs_src/models/docs014.py b/docs_src/models/docs014.py new file mode 100644 index 0000000..fc02ddb --- /dev/null +++ b/docs_src/models/docs014.py @@ -0,0 +1,18 @@ +import databases +import sqlalchemy + +import ormar + +database = databases.Database("sqlite:///db.sqlite") +metadata = sqlalchemy.MetaData() + + +class Course(ormar.Model): + class Meta: + database = database + metadata = metadata + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + completed: bool = ormar.Boolean(default=False) + non_db_field: str = ormar.String(max_length=100, pydantic_only=True) diff --git a/docs_src/models/docs015.py b/docs_src/models/docs015.py new file mode 100644 index 0000000..b67bff5 --- /dev/null +++ b/docs_src/models/docs015.py @@ -0,0 +1,23 @@ +import databases +import sqlalchemy + +import ormar +from ormar import property_field + +database = databases.Database("sqlite:///db.sqlite") +metadata = sqlalchemy.MetaData() + + +class Course(ormar.Model): + class Meta: + database = database + metadata = metadata + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + completed: bool = ormar.Boolean(default=False) + + @property_field + def prefixed_name(self): + return 'custom_prefix__' + self.name + diff --git a/ormar/__init__.py b/ormar/__init__.py index c970f7d..6e196db 100644 --- a/ormar/__init__.py +++ b/ormar/__init__.py @@ -31,7 +31,7 @@ class UndefinedType: # pragma no cover Undefined = UndefinedType() -__version__ = "0.6.1" +__version__ = "0.6.2" __all__ = [ "Integer", "BigInteger", diff --git a/ormar/models/modelproxy.py b/ormar/models/modelproxy.py index 3a82230..b545ecf 100644 --- a/ormar/models/modelproxy.py +++ b/ormar/models/modelproxy.py @@ -177,7 +177,11 @@ class ModelTableProxy: @classmethod def populate_default_values(cls, new_kwargs: Dict) -> Dict: for field_name, field in cls.Meta.model_fields.items(): - if field_name not in new_kwargs and field.has_default(use_server=False): + if ( + field_name not in new_kwargs + and field.has_default(use_server=False) + and not field.pydantic_only + ): new_kwargs[field_name] = field.get_default() # clear fields with server_default set as None if field.server_default is not None and not new_kwargs.get(field_name): diff --git a/ormar/models/newbasemodel.py b/ormar/models/newbasemodel.py index 35a5d57..9105f7b 100644 --- a/ormar/models/newbasemodel.py +++ b/ormar/models/newbasemodel.py @@ -318,8 +318,6 @@ class NewBaseModel( exclude_none: bool = False, nested: bool = False, ) -> "DictStrAny": # noqa: A003' - # callable_name = inspect.currentframe().f_back.f_code.co_name - # print('dict', callable_name) dict_instance = super().dict( include=include, exclude=self._update_excluded_with_related_not_required(exclude, nested), diff --git a/tests/test_excluding_fields_in_fastapi.py b/tests/test_excluding_fields_in_fastapi.py index b9398ca..a75d0e1 100644 --- a/tests/test_excluding_fields_in_fastapi.py +++ b/tests/test_excluding_fields_in_fastapi.py @@ -87,10 +87,10 @@ class User(ormar.Model): database = database id: int = ormar.Integer(primary_key=True) - email: str = ormar.String(max_length=255, nullable=False) + email: str = ormar.String(max_length=255) password: str = ormar.String(max_length=255, nullable=True) - first_name: str = ormar.String(max_length=255, nullable=False) - last_name: str = ormar.String(max_length=255, nullable=False) + first_name: str = ormar.String(max_length=255) + last_name: str = ormar.String(max_length=255) category: str = ormar.String(max_length=255, nullable=True) @@ -102,10 +102,13 @@ class User2(ormar.Model): id: int = ormar.Integer(primary_key=True) email: str = ormar.String(max_length=255, nullable=False) - password: str = ormar.String(max_length=255, nullable=False) - first_name: str = ormar.String(max_length=255, nullable=False) - last_name: str = ormar.String(max_length=255, nullable=False) + password: str = ormar.String(max_length=255) + first_name: str = ormar.String(max_length=255) + last_name: str = ormar.String(max_length=255) category: str = ormar.String(max_length=255, nullable=True) + timestamp: datetime.datetime = ormar.DateTime( + pydantic_only=True, default=datetime.datetime.now + ) @pytest.fixture(autouse=True, scope="module") @@ -150,6 +153,12 @@ async def create_user6(user: RandomModel): return user.dict() +@app.post("/random3/", response_model=RandomModel, response_model_exclude={"full_name"}) +async def create_user7(user: RandomModel): + user = await user.save() + return user.dict() + + def test_excluding_fields_in_endpoints(): client = TestClient(app) with client as client: @@ -184,6 +193,32 @@ def test_excluding_fields_in_endpoints(): response = client.post("/users3/", json=user) assert list(response.json().keys()) == ["email", "first_name", "last_name"] + timestamp = datetime.datetime.now() + + user3 = { + "email": "test@domain.com", + "password": "^*^%A*DA*IAAA", + "first_name": "John", + "last_name": "Doe", + "timestamp": str(timestamp), + } + response = client.post("/users4/", json=user3) + assert list(response.json().keys()) == [ + "id", + "email", + "first_name", + "last_name", + "category", + "timestamp", + ] + assert response.json().get("timestamp") == str(timestamp).replace(" ", "T") + resp_dict = response.json() + resp_dict.update({"password": "random"}) + user_instance = User2(**resp_dict) + assert user_instance.timestamp is not None + assert isinstance(user_instance.timestamp, datetime.datetime) + assert user_instance.timestamp == timestamp + response = client.post("/users4/", json=user) assert list(response.json().keys()) == [ "id", @@ -191,13 +226,16 @@ def test_excluding_fields_in_endpoints(): "first_name", "last_name", "category", + "timestamp", ] + assert response.json().get("timestamp") != str(timestamp).replace(" ", "T") + assert response.json().get("timestamp") is not None def test_adding_fields_in_endpoints(): client = TestClient(app) with client as client: - user3 = {"last_name": "Test"} + user3 = {"last_name": "Test", "full_name": "deleted"} response = client.post("/random/", json=user3) assert list(response.json().keys()) == [ "id", @@ -238,3 +276,19 @@ def test_adding_fields_in_endpoints2(): "full_name", ] assert response.json().get("full_name") == "John Test" + + +def test_excluding_property_field_in_endpoints2(): + client = TestClient(app) + with client as client: + RandomModel.Meta.include_props_in_dict = True + user3 = {"last_name": "Test"} + response = client.post("/random3/", json=user3) + assert list(response.json().keys()) == [ + "id", + "password", + "first_name", + "last_name", + "created_date", + ] + assert response.json().get("full_name") is None From ad44653acd451f98fdec13fad83d84911f247bbc Mon Sep 17 00:00:00 2001 From: collerek Date: Fri, 4 Dec 2020 15:19:18 +0100 Subject: [PATCH 7/7] remove trailing line in docs --- docs_src/models/docs015.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs_src/models/docs015.py b/docs_src/models/docs015.py index b67bff5..828da0b 100644 --- a/docs_src/models/docs015.py +++ b/docs_src/models/docs015.py @@ -20,4 +20,3 @@ class Course(ormar.Model): @property_field def prefixed_name(self): return 'custom_prefix__' + self.name -