version with pydantic inheritance passing all the tests

This commit is contained in:
collerek
2020-08-19 18:40:57 +07:00
commit 0b156caf0a
58 changed files with 4853 additions and 0 deletions

4
ormar/models/__init__.py Normal file
View File

@ -0,0 +1,4 @@
from ormar.models.fakepydantic import FakePydantic
from ormar.models.model import Model
__all__ = ["FakePydantic", "Model"]

View File

@ -0,0 +1,290 @@
import inspect
import json
import uuid
from typing import (
Any,
Callable,
Dict,
List,
Optional,
Set,
TYPE_CHECKING,
Type,
TypeVar,
Union, AbstractSet, Mapping,
)
import databases
import pydantic
import sqlalchemy
from pydantic import BaseModel
import ormar # noqa I100
from ormar import ForeignKey
from ormar.fields import BaseField
from ormar.fields.foreign_key import ForeignKeyField
from ormar.models.metaclass import ModelMetaclass, ModelMeta
from ormar.relations import RelationshipManager
if TYPE_CHECKING: # pragma no cover
from ormar.models.model import Model
IntStr = Union[int, str]
DictStrAny = Dict[str, Any]
AbstractSetIntStr = AbstractSet[IntStr]
MappingIntStrAny = Mapping[IntStr, Any]
class FakePydantic(pydantic.BaseModel, metaclass=ModelMetaclass):
# FakePydantic inherits from list in order to be treated as
# request.Body parameter in fastapi routes,
# inheriting from pydantic.BaseModel causes metaclass conflicts
__slots__ = ('_orm_id', '_orm_saved')
if TYPE_CHECKING: # pragma no cover
__model_fields__: Dict[str, TypeVar[BaseField]]
__table__: sqlalchemy.Table
__fields__: Dict[str, pydantic.fields.ModelField]
__pydantic_model__: Type[BaseModel]
__pkname__: str
__tablename__: str
__metadata__: sqlalchemy.MetaData
__database__: databases.Database
_orm_relationship_manager: RelationshipManager
Meta: ModelMeta
# noinspection PyMissingConstructor
def __init__(self, *args: Any, **kwargs: Any) -> None:
object.__setattr__(self, "_orm_id", uuid.uuid4().hex)
object.__setattr__(self, "_orm_saved", False)
pk_only = kwargs.pop("__pk_only__", False)
if "pk" in kwargs:
kwargs[self.Meta.pkname] = kwargs.pop("pk")
kwargs = {
k: self.Meta.model_fields[k].expand_relationship(v, self)
for k, v in kwargs.items()
}
values, fields_set, validation_error = pydantic.validate_model(
self, kwargs
)
if validation_error and not pk_only:
raise validation_error
object.__setattr__(self, '__dict__', values)
object.__setattr__(self, '__fields_set__', fields_set)
# super().__init__(**kwargs)
# self.values = self.__pydantic_model__(**kwargs)
def __del__(self) -> None:
self.Meta._orm_relationship_manager.deregister(self)
def __setattr__(self, name, value):
relation_key = self.get_name(title=True) + "_" + name
if name in self.__slots__:
object.__setattr__(self, name, value)
elif name == 'pk':
object.__setattr__(self, self.Meta.pkname, value)
elif self.Meta._orm_relationship_manager.contains(relation_key, self):
self.Meta.model_fields[name].expand_relationship(value, self)
else:
super().__setattr__(name, value)
def __getattr__(self, item):
relation_key = self.get_name(title=True) + "_" + item
if self.Meta._orm_relationship_manager.contains(relation_key, self):
return self.Meta._orm_relationship_manager.get(relation_key, self)
# def __setattr__(self, key: str, value: Any) -> None:
# if key in ('_orm_id', '_orm_relationship_manager', '_orm_saved', 'objects', '__model_fields__'):
# return setattr(self, key, value)
# # elif key in self._extract_related_names():
# # value = self._convert_json(key, value, op="dumps")
# # value = self.Meta.model_fields[key].expand_relationship(value, self)
# # relation_key = self.get_name(title=True) + "_" + key
# # if not self.Meta._orm_relationship_manager.contains(relation_key, self):
# # setattr(self.values, key, value)
# else:
# super().__setattr__(key, value)
# def __getattribute__(self, key: str) -> Any:
# if key != 'Meta' and key in self.Meta.model_fields:
# relation_key = self.get_name(title=True) + "_" + key
# if self.Meta._orm_relationship_manager.contains(relation_key, self):
# return self.Meta._orm_relationship_manager.get(relation_key, self)
# item = getattr(self.__fields__, key, None)
# item = self._convert_json(key, item, op="loads")
# return item
# return super().__getattribute__(key)
def __same__(self, other: "Model") -> bool:
if self.__class__ != other.__class__: # pragma no cover
return False
return (self._orm_id == other._orm_id or
self.__dict__ == other.__dict__ or
(self.pk == other.pk and self.pk is not None
))
# def __repr__(self) -> str: # pragma no cover
# return self.values.__repr__()
# @classmethod
# def __get_validators__(cls) -> Callable: # pragma no cover
# yield cls.__pydantic_model__.validate
@classmethod
def get_name(cls, title: bool = False, lower: bool = True) -> str:
name = cls.__name__
if lower:
name = name.lower()
if title:
name = name.title()
return name
@property
def pk(self) -> Any:
return getattr(self, self.Meta.pkname)
@pk.setter
def pk(self, value: Any) -> None:
setattr(self, self.Meta.pkname, value)
@property
def pk_column(self) -> sqlalchemy.Column:
return self.Meta.table.primary_key.columns.values()[0]
@classmethod
def pk_type(cls) -> Any:
return cls.Meta.model_fields[cls.Meta.pkname].__type__
def dict(
self,
*,
include: Union['AbstractSetIntStr', 'MappingIntStrAny'] = None,
exclude: Union['AbstractSetIntStr', 'MappingIntStrAny'] = None,
by_alias: bool = False,
skip_defaults: bool = None,
exclude_unset: bool = False,
exclude_defaults: bool = False,
exclude_none: bool = False,
nested: bool = False
) -> 'DictStrAny': # noqa: A003
print('callin super', self.__class__)
print('to exclude', self._exclude_related_names_not_required(nested))
dict_instance = super().dict(include=include,
exclude=self._exclude_related_names_not_required(nested),
by_alias=by_alias,
skip_defaults=skip_defaults,
exclude_unset=exclude_unset,
exclude_defaults=exclude_defaults,
exclude_none=exclude_none)
print('after super')
for field in self._extract_related_names():
print(self.__class__, field, nested)
nested_model = getattr(self, field)
if self.Meta.model_fields[field].virtual and nested:
continue
if isinstance(nested_model, list) and not isinstance(
nested_model, ormar.Model
):
print('nested list')
dict_instance[field] = [x.dict(nested=True) for x in nested_model]
else:
print('instance')
if nested_model is not None:
dict_instance[field] = nested_model.dict(nested=True)
return dict_instance
def from_dict(self, value_dict: Dict) -> None:
for key, value in value_dict.items():
setattr(self, key, value)
def _convert_json(self, column_name: str, value: Any, op: str) -> Union[str, dict]:
if not self._is_conversion_to_json_needed(column_name):
return value
condition = (
isinstance(value, str) if op == "loads" else not isinstance(value, str)
)
operand = json.loads if op == "loads" else json.dumps
if condition:
try:
return operand(value)
except TypeError: # pragma no cover
pass
return value
def _is_conversion_to_json_needed(self, column_name: str) -> bool:
return self.Meta.model_fields.get(column_name).__type__ == pydantic.Json
def _extract_own_model_fields(self) -> Dict:
related_names = self._extract_related_names()
self_fields = {k: v for k, v in self.dict().items() if k not in related_names}
return self_fields
@classmethod
def _extract_related_names(cls) -> Set:
related_names = set()
for name, field in cls.Meta.model_fields.items():
if inspect.isclass(field) and issubclass(
field, ForeignKeyField
):
related_names.add(name)
return related_names
@classmethod
def _exclude_related_names_not_required(cls, nested:bool=False) -> Set:
if nested:
return cls._extract_related_names()
related_names = set()
for name, field in cls.Meta.model_fields.items():
if inspect.isclass(field) and issubclass(field, ForeignKeyField) and field.nullable:
related_names.add(name)
return related_names
def _extract_model_db_fields(self) -> Dict:
self_fields = self._extract_own_model_fields()
self_fields = {
k: v for k, v in self_fields.items() if k in self.Meta.table.columns
}
for field in self._extract_related_names():
if getattr(self, field) is not None:
self_fields[field] = getattr(
getattr(self, field), self.Meta.model_fields[field].to.Meta.pkname
)
return self_fields
@classmethod
def merge_instances_list(cls, result_rows: List["Model"]) -> List["Model"]:
merged_rows = []
for index, model in enumerate(result_rows):
if index > 0 and model.pk == result_rows[index - 1].pk:
result_rows[-1] = cls.merge_two_instances(model, merged_rows[-1])
else:
merged_rows.append(model)
return merged_rows
@classmethod
def merge_two_instances(cls, one: "Model", other: "Model") -> "Model":
for field in one.Meta.model_fields.keys():
if isinstance(getattr(one, field), list) and not isinstance(
getattr(one, field), ormar.Model
):
setattr(other, field, getattr(one, field) + getattr(other, field))
elif isinstance(getattr(one, field), ormar.Model):
if getattr(one, field).pk == getattr(other, field).pk:
setattr(
other,
field,
cls.merge_two_instances(
getattr(one, field), getattr(other, field)
),
)
return other

196
ormar/models/metaclass.py Normal file
View File

@ -0,0 +1,196 @@
from typing import Any, Dict, List, Optional, TYPE_CHECKING, Tuple, Type, Union
import databases
import pydantic
import sqlalchemy
from pydantic import BaseConfig, create_model, Extra
from pydantic.fields import ModelField, FieldInfo
from ormar import ForeignKey, ModelDefinitionError # noqa I100
from ormar.fields import BaseField
from ormar.fields.foreign_key import ForeignKeyField
from ormar.queryset import QuerySet
from ormar.relations import RelationshipManager
if TYPE_CHECKING: # pragma no cover
from ormar import Model
relationship_manager = RelationshipManager()
class ModelMeta:
tablename: str
table: sqlalchemy.Table
metadata: sqlalchemy.MetaData
database: databases.Database
columns: List[sqlalchemy.Column]
pkname: str
model_fields: Dict[str, Union[BaseField, ForeignKey]]
_orm_relationship_manager: RelationshipManager
def parse_pydantic_field_from_model_fields(object_dict: dict) -> Dict[str, Tuple]:
pydantic_fields = {
field_name: (
base_field.__type__,
... if base_field.is_required else base_field.default_value,
)
for field_name, base_field in object_dict.items()
if isinstance(base_field, BaseField)
}
return pydantic_fields
def register_relation_on_build(table_name: str, field: ForeignKey, name: str) -> None:
child_relation_name = (
field.to.get_name(title=True)
+ "_"
+ (field.related_name or (name.lower() + "s"))
)
reverse_name = child_relation_name
relation_name = name.lower().title() + "_" + field.to.get_name()
relationship_manager.add_relation_type(
relation_name, reverse_name, field, table_name
)
def expand_reverse_relationships(model: Type["Model"]) -> None:
for model_field in model.Meta.model_fields.values():
if issubclass(model_field, ForeignKeyField):
child_model_name = model_field.related_name or model.get_name() + "s"
parent_model = model_field.to
child = model
if (
child_model_name not in parent_model.__fields__
and child.get_name() not in parent_model.__fields__
):
register_reverse_model_fields(parent_model, child, child_model_name)
def register_reverse_model_fields(
model: Type["Model"], child: Type["Model"], child_model_name: str
) -> None:
# model.__fields__[child_model_name] = ModelField(
# name=child_model_name,
# type_=Optional[Union[List[child], child]],
# model_config=child.__config__,
# class_validators=child.__validators__,
# )
model.Meta.model_fields[child_model_name] = ForeignKey(
child, name=child_model_name, virtual=True
)
def sqlalchemy_columns_from_model_fields(
name: str, object_dict: Dict, table_name: str
) -> Tuple[Optional[str], List[sqlalchemy.Column], Dict[str, BaseField]]:
columns = []
pkname = None
model_fields = {
field_name: field
for field_name, field in object_dict['__annotations__'].items()
if issubclass(field, BaseField)
}
for field_name, field in model_fields.items():
if field.primary_key:
if pkname is not None:
raise ModelDefinitionError("Only one primary key column is allowed.")
if field.pydantic_only:
raise ModelDefinitionError('Primary key column cannot be pydantic only')
pkname = field_name
if not field.pydantic_only:
columns.append(field.get_column(field_name))
if issubclass(field, ForeignKeyField):
register_relation_on_build(table_name, field, name)
return pkname, columns, model_fields
def populate_pydantic_default_values(attrs: Dict) -> Dict:
for field, type_ in attrs["__annotations__"].items():
if issubclass(type_, BaseField):
if type_.name is None:
type_.name = field
def_value = type_.default_value()
curr_def_value = attrs.get(field, 'NONE')
print(field, curr_def_value, 'def val', type_.nullable)
if curr_def_value == 'NONE' and isinstance(def_value, FieldInfo):
attrs[field] = def_value
elif curr_def_value == 'NONE' and type_.nullable:
print(field, 'defsults tp none')
attrs[field] = FieldInfo(default=None)
return attrs
def get_pydantic_base_orm_config() -> Type[BaseConfig]:
class Config(BaseConfig):
orm_mode = True
arbitrary_types_allowed = True
# extra = Extra.allow
return Config
class ModelMetaclass(pydantic.main.ModelMetaclass):
def __new__(mcs: type, name: str, bases: Any, attrs: dict) -> type:
attrs['Config'] = get_pydantic_base_orm_config()
new_model = super().__new__( # type: ignore
mcs, name, bases, attrs
)
if hasattr(new_model, 'Meta'):
if attrs.get("__abstract__"):
return new_model
annotations = attrs.get("__annotations__") or new_model.__annotations__
attrs["__annotations__"]= annotations
attrs = populate_pydantic_default_values(attrs)
print(attrs)
tablename = name.lower() + "s"
new_model.Meta.tablename = new_model.Meta.tablename or tablename
# sqlalchemy table creation
pkname, columns, model_fields = sqlalchemy_columns_from_model_fields(
name, attrs, new_model.Meta.tablename
)
if hasattr(new_model.Meta, "model_fields") and not pkname:
model_fields = new_model.Meta.model_fields
for fieldname, field in new_model.Meta.model_fields.items():
if field.primary_key:
pkname=fieldname
columns = new_model.Meta.table.columns
if not hasattr(new_model.Meta, "table"):
new_model.Meta.table = sqlalchemy.Table(new_model.Meta.tablename, new_model.Meta.metadata, *columns)
new_model.Meta.columns = columns
new_model.Meta.pkname = pkname
if not pkname:
raise ModelDefinitionError("Table has to have a primary key.")
# pydantic model creation
new_model.Meta.pydantic_fields = parse_pydantic_field_from_model_fields(attrs)
new_model.Meta.pydantic_model = create_model(
name, __config__=get_pydantic_base_orm_config(), **new_model.Meta.pydantic_fields
)
new_model.Meta.model_fields = model_fields
print(attrs, 'before super')
print(new_model.Meta.__dict__)
new_model = super().__new__( # type: ignore
mcs, name, bases, attrs
)
expand_reverse_relationships(new_model)
new_model.Meta._orm_relationship_manager = relationship_manager
new_model.objects = QuerySet(new_model)
# breakpoint()
return new_model

85
ormar/models/model.py Normal file
View File

@ -0,0 +1,85 @@
from typing import Any, List
import sqlalchemy
import ormar.queryset # noqa I100
from ormar.models import FakePydantic # noqa I100
class Model(FakePydantic):
__abstract__ = False
# objects = ormar.queryset.QuerySet()
@classmethod
def from_row(
cls,
row: sqlalchemy.engine.ResultProxy,
select_related: List = None,
previous_table: str = None,
) -> "Model":
item = {}
select_related = select_related or []
table_prefix = cls.Meta._orm_relationship_manager.resolve_relation_join(
previous_table, cls.Meta.table.name
)
previous_table = cls.Meta.table.name
for related in select_related:
if "__" in related:
first_part, remainder = related.split("__", 1)
model_cls = cls.Meta.model_fields[first_part].to
child = model_cls.from_row(
row, select_related=[remainder], previous_table=previous_table
)
item[first_part] = child
else:
model_cls = cls.Meta.model_fields[related].to
child = model_cls.from_row(row, previous_table=previous_table)
item[related] = child
for column in cls.Meta.table.columns:
if column.name not in item:
item[column.name] = row[
f'{table_prefix + "_" if table_prefix else ""}{column.name}'
]
return cls(**item)
async def save(self) -> "Model":
self_fields = self._extract_model_db_fields()
if self.Meta.model_fields.get(self.Meta.pkname).autoincrement:
self_fields.pop(self.Meta.pkname, None)
expr = self.Meta.table.insert()
expr = expr.values(**self_fields)
item_id = await self.Meta.database.execute(expr)
setattr(self, self.Meta.pkname, item_id)
return self
async def update(self, **kwargs: Any) -> int:
if kwargs:
new_values = {**self.dict(), **kwargs}
self.from_dict(new_values)
self_fields = self._extract_model_db_fields()
self_fields.pop(self.Meta.pkname)
expr = (
self.Meta.table.update()
.values(**self_fields)
.where(self.pk_column == getattr(self, self.Meta.pkname))
)
result = await self.Meta.database.execute(expr)
return result
async def delete(self) -> int:
expr = self.Meta.table.delete()
expr = expr.where(self.pk_column == (getattr(self, self.Meta.pkname)))
result = await self.Meta.database.execute(expr)
return result
async def load(self) -> "Model":
expr = self.Meta.table.select().where(self.pk_column == self.pk)
row = await self.Meta.database.fetch_one(expr)
self.from_dict(dict(row))
return self