From f424e65a4075eda3bd86c880a990116c60830cc4 Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 9 Mar 2021 17:00:13 +0100 Subject: [PATCH 01/13] check if data binding not work only in sqlite --- ormar/__init__.py | 4 +- ormar/fields/__init__.py | 3 + ormar/fields/base.py | 74 ++++++--- ormar/fields/model_fields.py | 11 ++ ormar/fields/sqlalchemy_encrypted.py | 219 +++++++++++++++++++++++++++ ormar/models/helpers/sqlalchemy.py | 2 +- tests/test_encrypted_columns.py | 60 ++++++++ 7 files changed, 349 insertions(+), 24 deletions(-) create mode 100644 ormar/fields/sqlalchemy_encrypted.py create mode 100644 tests/test_encrypted_columns.py diff --git a/ormar/__init__.py b/ormar/__init__.py index 98f7e78..d68a23c 100644 --- a/ormar/__init__.py +++ b/ormar/__init__.py @@ -53,6 +53,7 @@ from ormar.fields import ( Time, UUID, UniqueColumns, + EncryptBackends ) # noqa: I100 from ormar.models import ExcludableItems, Model from ormar.models.metaclass import ModelMeta @@ -68,7 +69,7 @@ class UndefinedType: # pragma no cover Undefined = UndefinedType() -__version__ = "0.9.7" +__version__ = "0.9.8" __all__ = [ "Integer", "BigInteger", @@ -110,4 +111,5 @@ __all__ = [ "ExcludableItems", "and_", "or_", + "EncryptBackends" ] diff --git a/ormar/fields/__init__.py b/ormar/fields/__init__.py index c5f61d4..7a22c51 100644 --- a/ormar/fields/__init__.py +++ b/ormar/fields/__init__.py @@ -22,6 +22,7 @@ from ormar.fields.model_fields import ( UUID, ) from ormar.fields.through_field import Through, ThroughField +from ormar.fields.sqlalchemy_encrypted import EncryptBackend, EncryptBackends __all__ = [ "Decimal", @@ -44,4 +45,6 @@ __all__ = [ "ForeignKeyField", "ThroughField", "Through", + "EncryptBackends", + "EncryptBackend" ] diff --git a/ormar/fields/base.py b/ormar/fields/base.py index 1fada90..9fdadb7 100644 --- a/ormar/fields/base.py +++ b/ormar/fields/base.py @@ -5,6 +5,9 @@ from pydantic import Field, Json, typing from pydantic.fields import FieldInfo, Required, Undefined import ormar # noqa I101 +from ormar import ModelDefinitionError +from ormar.fields.sqlalchemy_encrypted import EncryptBackend, EncryptBackends, \ + EncryptedString if TYPE_CHECKING: # pragma no cover from ormar.models import Model @@ -49,6 +52,11 @@ class BaseField(FieldInfo): self_reference: bool = False self_reference_primary: Optional[str] = None + encrypt_secret: str + encrypt_backend: EncryptBackends = EncryptBackends.NONE + encrypt_custom_backend: Type[EncryptBackend] = None + encrypt_max_length: int = 5000 + default: Any server_default: Any @@ -93,10 +101,11 @@ class BaseField(FieldInfo): :rtype: bool """ return ( - field_name not in ["default", "default_factory", "alias", "allow_mutation"] - and not field_name.startswith("__") - and hasattr(cls, field_name) - and not callable(getattr(cls, field_name)) + field_name not in ["default", "default_factory", "alias", + "allow_mutation"] + and not field_name.startswith("__") + and hasattr(cls, field_name) + and not callable(getattr(cls, field_name)) ) @classmethod @@ -205,7 +214,7 @@ class BaseField(FieldInfo): :rtype: bool """ return cls.default is not None or ( - cls.server_default is not None and use_server + cls.server_default is not None and use_server ) @classmethod @@ -238,7 +247,7 @@ class BaseField(FieldInfo): ondelete=con.ondelete, onupdate=con.onupdate, name=f"fk_{cls.owner.Meta.tablename}_{cls.to.Meta.tablename}" - f"_{cls.to.get_column_alias(cls.to.Meta.pkname)}_{cls.name}", + f"_{cls.to.get_column_alias(cls.to.Meta.pkname)}_{cls.name}", ) for con in cls.constraints ] @@ -256,25 +265,46 @@ class BaseField(FieldInfo): :return: actual definition of the database column as sqlalchemy requires. :rtype: sqlalchemy.Column """ - column = sqlalchemy.Column( - cls.alias or name, - cls.column_type, - *cls.construct_constraints(), - primary_key=cls.primary_key, - nullable=cls.nullable and not cls.primary_key, - index=cls.index, - unique=cls.unique, - default=cls.default, - server_default=cls.server_default, - ) + if cls.encrypt_backend == EncryptBackends.NONE: + column = sqlalchemy.Column( + cls.alias or name, + cls.column_type, + *cls.construct_constraints(), + primary_key=cls.primary_key, + nullable=cls.nullable and not cls.primary_key, + index=cls.index, + unique=cls.unique, + default=cls.default, + server_default=cls.server_default, + ) + else: + if cls.primary_key or cls.is_relation: + raise ModelDefinitionError("Primary key field and relations fields" + "cannot be encrypted!") + column = sqlalchemy.Column( + cls.alias or name, + EncryptedString( + _field_type=cls, + encrypt_secret=cls.encrypt_secret, + encrypt_backend=cls.encrypt_backend, + encrypt_custom_backend=cls.encrypt_custom_backend, + encrypt_max_length=cls.encrypt_max_length + ), + nullable=cls.nullable, + index=cls.index, + unique=cls.unique, + default=cls.default, + server_default=cls.server_default, + ) + return column @classmethod def expand_relationship( - cls, - value: Any, - child: Union["Model", "NewBaseModel"], - to_register: bool = True, + cls, + value: Any, + child: Union["Model", "NewBaseModel"], + to_register: bool = True, ) -> Any: """ Function overwritten for relations, in basic field the value is returned as is. @@ -302,7 +332,7 @@ class BaseField(FieldInfo): :rtype: None """ if cls.owner is not None and ( - cls.owner == cls.to or cls.owner.Meta == cls.to.Meta + cls.owner == cls.to or cls.owner.Meta == cls.to.Meta ): cls.self_reference = True cls.self_reference_primary = cls.name diff --git a/ormar/fields/model_fields.py b/ormar/fields/model_fields.py index 2222b78..67ea3e6 100644 --- a/ormar/fields/model_fields.py +++ b/ormar/fields/model_fields.py @@ -9,6 +9,7 @@ import sqlalchemy from ormar import ModelDefinitionError # noqa I101 from ormar.fields import sqlalchemy_uuid from ormar.fields.base import BaseField # noqa I101 +from ormar.fields.sqlalchemy_encrypted import EncryptBackends def is_field_nullable( @@ -73,6 +74,12 @@ class ModelFieldFactory: primary_key = kwargs.pop("primary_key", False) autoincrement = kwargs.pop("autoincrement", False) + encrypt_secret = kwargs.pop("encrypt_secret", None) + encrypt_backend = kwargs.pop("encrypt_backend", EncryptBackends.NONE) + encrypt_custom_backend = kwargs.pop("encrypt_custom_backend", + None) + encrypt_max_length = kwargs.pop("encrypt_max_length", 5000) + namespace = dict( __type__=cls._type, alias=kwargs.pop("name", None), @@ -88,6 +95,10 @@ class ModelFieldFactory: autoincrement=autoincrement, column_type=cls.get_column_type(**kwargs), choices=set(kwargs.pop("choices", [])), + encrypt_secret=encrypt_secret, + encrypt_backend=encrypt_backend, + encrypt_custom_backend=encrypt_custom_backend, + encrypt_max_length=encrypt_max_length, **kwargs ) return type(cls.__name__, cls._bases, namespace) diff --git a/ormar/fields/sqlalchemy_encrypted.py b/ormar/fields/sqlalchemy_encrypted.py new file mode 100644 index 0000000..3b5b18c --- /dev/null +++ b/ormar/fields/sqlalchemy_encrypted.py @@ -0,0 +1,219 @@ +# inspired by sqlalchemy-utils (https://github.com/kvesteri/sqlalchemy-utils) +import abc +import base64 +import datetime +import json +from enum import Enum +from typing import Any, Callable, TYPE_CHECKING, Type, Union + +from sqlalchemy import String +from sqlalchemy.engine.default import DefaultDialect +from sqlalchemy.types import TypeDecorator + +from ormar import ModelDefinitionError + +try: + import cryptography + from cryptography.fernet import Fernet + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import hashes +except ImportError: + pass + +if TYPE_CHECKING: + from ormar import BaseField + + +class EncryptBackend(abc.ABC): + + def _update_key(self, key): + if isinstance(key, str): + key = key.encode() + digest = hashes.Hash(hashes.SHA256(), backend=default_backend()) + digest.update(key) + engine_key = digest.finalize() + + self._initialize_engine(engine_key) + + @abc.abstractmethod + def _initialize_engine(self, secret_key: bytes): + pass + + @abc.abstractmethod + def encrypt(self, value: Any) -> str: + pass + + @abc.abstractmethod + def decrypt(self, value: Any) -> str: + pass + + +class HashBackend(EncryptBackend): + """ + One-way hashing - in example for passwords, no way to decrypt the value! + """ + + def _initialize_engine(self, secret_key: bytes): + self.secret_key = base64.urlsafe_b64encode(secret_key) + + def encrypt(self, value: Any) -> str: + if not isinstance(value, str): + value = repr(value) + value = value.encode() + digest = hashes.Hash(hashes.SHA512(), backend=default_backend()) + digest.update(self.secret_key) + digest.update(value) + hashed_value = digest.finalize() + return hashed_value.hex() + + def decrypt(self, value: Any) -> str: + if not isinstance(value, str): + value = str(value) + return value + + +class FernetBackend(EncryptBackend): + """ + Two-way encryption, data stored in db are encrypted but decrypted during query. + """ + + def _initialize_engine(self, secret_key: bytes): + self.secret_key = base64.urlsafe_b64encode(secret_key) + self.fernet = Fernet(self.secret_key) + + def encrypt(self, value: Any) -> str: + if not isinstance(value, str): + value = repr(value) + value = value.encode() + encrypted = self.fernet.encrypt(value) + return encrypted.decode('utf-8') + + def decrypt(self, value: Any) -> str: + if not isinstance(value, str): + value = str(value) + decrypted = self.fernet.decrypt(value.encode()) + if not isinstance(decrypted, str): + decrypted = decrypted.decode('utf-8') + return decrypted + + +class EncryptBackends(Enum): + NONE = 0 + FERNET = 1 + HASH = 2 + CUSTOM = 3 + + +backends_map = { + EncryptBackends.FERNET: FernetBackend, + EncryptBackends.HASH: HashBackend, + EncryptBackends.CUSTOM: None +} + + +class EncryptedString(TypeDecorator): # pragma nocover + """ + Used to store encrypted values in a database + """ + + impl = String + + def __init__(self, + *args: Any, + encrypt_secret: Union[str, Callable], + _field_type: Type["BaseField"], + encrypt_max_length: int = 5000, + encrypt_backend: EncryptBackends = EncryptBackends.FERNET, + encrypt_custom_backend: Type[EncryptBackend] = None, + **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + if not cryptography: + raise ModelDefinitionError( + "In order to encrypt a column 'cryptography' is required!" + ) + backend = backends_map.get(encrypt_backend, encrypt_custom_backend) + if not backend or not issubclass(backend, EncryptBackend): + raise ModelDefinitionError("Wrong or no encrypt backend provided!") + self.backend = backend() + self._field_type = _field_type + self._underlying_type = _field_type.column_type + self._key = encrypt_secret + self.max_length = encrypt_max_length + + def __repr__(self) -> str: + return f"String({self.max_length})" + # + # def load_dialect_impl(self, dialect: DefaultDialect) -> Any: + # dialect.type_descriptor(VARCHAR(self.max_length)) + + @property + def key(self): + return self._key + + @key.setter + def key(self, value): + self._key = value + + def _update_key(self): + key = self._key() if callable(self._key) else self._key + self.backend._update_key(key) + + def process_bind_param(self, value, dialect): + """Encrypt a value on the way in.""" + if value is not None: + self._update_key() + + try: + value = self._underlying_type.process_bind_param( + value, dialect + ) + + except AttributeError: + # Doesn't have 'process_bind_param' + type_ = self._field_type.__type__ + if issubclass(type_, bool): + value = 'true' if value else 'false' + + elif issubclass(type_, (datetime.date, datetime.time)): + value = value.isoformat() + + # elif issubclass(type_, JSONType): + # value = json.dumps(value) + + return self.backend.encrypt(value) + + def process_result_value(self, value, dialect): + """Decrypt value on the way out.""" + if value is not None: + self._update_key() + decrypted_value = self.backend.decrypt(value) + + try: + return self.underlying_type.process_result_value( + decrypted_value, dialect + ) + + except AttributeError: + # Doesn't have 'process_result_value' + + # Handle 'boolean' and 'dates' + type_ = self._field_type.__type__ + # date_types = [datetime.datetime, datetime.time, datetime.date] + + if issubclass(type_, bool): + return decrypted_value == 'true' + + # elif type_ in date_types: + # return DatetimeHandler.process_value( + # decrypted_value, type_ + # ) + + # elif issubclass(type_, JSONType): + # return json.loads(decrypted_value) + + # Handle all others + return self.underlying_type.python_type(decrypted_value) + + def _coerce(self, value): + return self.underlying_type._coerce(value) + diff --git a/ormar/models/helpers/sqlalchemy.py b/ormar/models/helpers/sqlalchemy.py index e40239d..472bbad 100644 --- a/ormar/models/helpers/sqlalchemy.py +++ b/ormar/models/helpers/sqlalchemy.py @@ -289,7 +289,7 @@ def populate_meta_sqlalchemy_table_if_required(meta: "ModelMeta") -> None: f'{"_".join([str(col) for col in constraint._pending_colargs])}' ) table = sqlalchemy.Table( - meta.tablename, meta.metadata, *meta.columns, *meta.constraints, + meta.tablename, meta.metadata, *meta.columns, *meta.constraints ) meta.table = table diff --git a/tests/test_encrypted_columns.py b/tests/test_encrypted_columns.py new file mode 100644 index 0000000..4de0459 --- /dev/null +++ b/tests/test_encrypted_columns.py @@ -0,0 +1,60 @@ +import uuid +from typing import Optional + +import databases +import pytest +import sqlalchemy + +import ormar +from ormar.exceptions import QueryDefinitionError +from tests.settings import DATABASE_URL + +database = databases.Database(DATABASE_URL) +metadata = sqlalchemy.MetaData() + + +class BaseMeta(ormar.ModelMeta): + metadata = metadata + database = database + + +class Author(ormar.Model): + class Meta(BaseMeta): + tablename = "authors" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, + encrypt_secret='asd123', + encrypt_backend=ormar.EncryptBackends.FERNET) + uuid_test = ormar.UUID(default=uuid.uuid4, uuid_format='string') + password: str = ormar.String(max_length=100, + encrypt_secret='udxc32', + encrypt_backend=ormar.EncryptBackends.HASH) + birth_year: int = ormar.Integer(nullable=True, + encrypt_secret='secure89key%^&psdijfipew', + encrypt_max_length=200, + encrypt_backend=ormar.EncryptBackends.FERNET) + + +@pytest.fixture(autouse=True, scope="module") +def create_test_database(): + engine = sqlalchemy.create_engine(DATABASE_URL) + metadata.drop_all(engine) + metadata.create_all(engine) + yield + metadata.drop_all(engine) + + +def test_db_structure(): + assert Author.Meta.table.c.get('name').type.impl.__class__ == sqlalchemy.NVARCHAR + assert Author.Meta.table.c.get('birth_year').type.max_length == 200 + + +@pytest.mark.asyncio +async def test_wrong_query_foreign_key_type(): + async with database: + await Author(name='Test', birth_year=1988, password='test123').save() + author = await Author.objects.get() + + assert author.name == 'Test' + assert author.birth_year == 1988 From 5695bb8f57e3612811e5fb9883963e80d3bf3d8b Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 9 Mar 2021 17:06:18 +0100 Subject: [PATCH 02/13] specify crypto to none --- ormar/fields/sqlalchemy_encrypted.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ormar/fields/sqlalchemy_encrypted.py b/ormar/fields/sqlalchemy_encrypted.py index 3b5b18c..10a017b 100644 --- a/ormar/fields/sqlalchemy_encrypted.py +++ b/ormar/fields/sqlalchemy_encrypted.py @@ -12,6 +12,7 @@ from sqlalchemy.types import TypeDecorator from ormar import ModelDefinitionError +cryptography = None try: import cryptography from cryptography.fernet import Fernet From 2c31ad735d676b38a4dc4861adf1105b880f7f6d Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 9 Mar 2021 17:10:39 +0100 Subject: [PATCH 03/13] add crypto to reqs --- ormar/fields/sqlalchemy_encrypted.py | 10 ++++------ requirements.txt | 1 + 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ormar/fields/sqlalchemy_encrypted.py b/ormar/fields/sqlalchemy_encrypted.py index 10a017b..cf2b6af 100644 --- a/ormar/fields/sqlalchemy_encrypted.py +++ b/ormar/fields/sqlalchemy_encrypted.py @@ -6,9 +6,7 @@ import json from enum import Enum from typing import Any, Callable, TYPE_CHECKING, Type, Union -from sqlalchemy import String -from sqlalchemy.engine.default import DefaultDialect -from sqlalchemy.types import TypeDecorator +import sqlalchemy.types as types from ormar import ModelDefinitionError @@ -112,12 +110,12 @@ backends_map = { } -class EncryptedString(TypeDecorator): # pragma nocover +class EncryptedString(types.TypeDecorator): # pragma nocover """ Used to store encrypted values in a database """ - impl = String + impl = types.String def __init__(self, *args: Any, @@ -143,6 +141,7 @@ class EncryptedString(TypeDecorator): # pragma nocover def __repr__(self) -> str: return f"String({self.max_length})" + # # def load_dialect_impl(self, dialect: DefaultDialect) -> Any: # dialect.type_descriptor(VARCHAR(self.max_length)) @@ -217,4 +216,3 @@ class EncryptedString(TypeDecorator): # pragma nocover def _coerce(self, value): return self.underlying_type._coerce(value) - diff --git a/requirements.txt b/requirements.txt index f47a05d..4d7610c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ pydantic sqlalchemy typing_extensions orjson +cryptography # Async database drivers aiomysql From 8d96a3fb840f180f046223ee6686c447ae3507d5 Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 9 Mar 2021 17:29:29 +0100 Subject: [PATCH 04/13] add length in imp --- ormar/fields/sqlalchemy_encrypted.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ormar/fields/sqlalchemy_encrypted.py b/ormar/fields/sqlalchemy_encrypted.py index cf2b6af..a2e10b7 100644 --- a/ormar/fields/sqlalchemy_encrypted.py +++ b/ormar/fields/sqlalchemy_encrypted.py @@ -7,6 +7,7 @@ from enum import Enum from typing import Any, Callable, TYPE_CHECKING, Type, Union import sqlalchemy.types as types +from sqlalchemy.engine.default import DefaultDialect from ormar import ModelDefinitionError @@ -115,7 +116,7 @@ class EncryptedString(types.TypeDecorator): # pragma nocover Used to store encrypted values in a database """ - impl = types.String + impl = types.TypeEngine def __init__(self, *args: Any, @@ -140,11 +141,10 @@ class EncryptedString(types.TypeDecorator): # pragma nocover self.max_length = encrypt_max_length def __repr__(self) -> str: - return f"String({self.max_length})" + return f"VARCHAR({self.max_length})" - # - # def load_dialect_impl(self, dialect: DefaultDialect) -> Any: - # dialect.type_descriptor(VARCHAR(self.max_length)) + def load_dialect_impl(self, dialect: DefaultDialect) -> Any: + return dialect.type_descriptor(types.VARCHAR(self.max_length)) @property def key(self): From e29bea6f85dacb95a276ae3a10ef25b58d67341e Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 9 Mar 2021 20:29:27 +0100 Subject: [PATCH 05/13] revert to use tables and columns with labels and aliases instead of text clauses, add encryption, mostly working encryption column type with configurable backends --- ormar/__init__.py | 8 +- ormar/fields/__init__.py | 7 +- ormar/fields/base.py | 81 +++++++----- ormar/fields/foreign_key.py | 21 +++- ormar/fields/many_to_many.py | 18 ++- ormar/fields/model_fields.py | 3 +- ormar/fields/parsers.py | 44 +++++++ ormar/fields/sqlalchemy_encrypted.py | 182 +++++++++++---------------- ormar/fields/sqlalchemy_uuid.py | 32 +---- ormar/models/helpers/validation.py | 1 + ormar/queryset/join.py | 8 +- ormar/relations/alias_manager.py | 13 +- tests/test_encrypted_columns.py | 164 +++++++++++++++++++++--- tests/test_or_filters.py | 86 ++++++------- 14 files changed, 415 insertions(+), 253 deletions(-) create mode 100644 ormar/fields/parsers.py diff --git a/ormar/__init__.py b/ormar/__init__.py index d68a23c..9193543 100644 --- a/ormar/__init__.py +++ b/ormar/__init__.py @@ -38,9 +38,12 @@ from ormar.fields import ( BaseField, BigInteger, Boolean, + DECODERS_MAP, Date, DateTime, Decimal, + ENCODERS_MAP, + EncryptBackends, Float, ForeignKey, ForeignKeyField, @@ -53,7 +56,6 @@ from ormar.fields import ( Time, UUID, UniqueColumns, - EncryptBackends ) # noqa: I100 from ormar.models import ExcludableItems, Model from ormar.models.metaclass import ModelMeta @@ -111,5 +113,7 @@ __all__ = [ "ExcludableItems", "and_", "or_", - "EncryptBackends" + "EncryptBackends", + "ENCODERS_MAP", + "DECODERS_MAP", ] diff --git a/ormar/fields/__init__.py b/ormar/fields/__init__.py index 7a22c51..e0cb3b0 100644 --- a/ormar/fields/__init__.py +++ b/ormar/fields/__init__.py @@ -21,8 +21,9 @@ from ormar.fields.model_fields import ( Time, UUID, ) -from ormar.fields.through_field import Through, ThroughField +from ormar.fields.parsers import DECODERS_MAP, ENCODERS_MAP from ormar.fields.sqlalchemy_encrypted import EncryptBackend, EncryptBackends +from ormar.fields.through_field import Through, ThroughField __all__ = [ "Decimal", @@ -46,5 +47,7 @@ __all__ = [ "ThroughField", "Through", "EncryptBackends", - "EncryptBackend" + "EncryptBackend", + "DECODERS_MAP", + "ENCODERS_MAP", ] diff --git a/ormar/fields/base.py b/ormar/fields/base.py index 9fdadb7..041d934 100644 --- a/ormar/fields/base.py +++ b/ormar/fields/base.py @@ -6,8 +6,11 @@ from pydantic.fields import FieldInfo, Required, Undefined import ormar # noqa I101 from ormar import ModelDefinitionError -from ormar.fields.sqlalchemy_encrypted import EncryptBackend, EncryptBackends, \ - EncryptedString +from ormar.fields.sqlalchemy_encrypted import ( + EncryptBackend, + EncryptBackends, + EncryptedString, +) if TYPE_CHECKING: # pragma no cover from ormar.models import Model @@ -54,7 +57,7 @@ class BaseField(FieldInfo): encrypt_secret: str encrypt_backend: EncryptBackends = EncryptBackends.NONE - encrypt_custom_backend: Type[EncryptBackend] = None + encrypt_custom_backend: Optional[Type[EncryptBackend]] = None encrypt_max_length: int = 5000 default: Any @@ -101,11 +104,10 @@ class BaseField(FieldInfo): :rtype: bool """ return ( - field_name not in ["default", "default_factory", "alias", - "allow_mutation"] - and not field_name.startswith("__") - and hasattr(cls, field_name) - and not callable(getattr(cls, field_name)) + field_name not in ["default", "default_factory", "alias", "allow_mutation"] + and not field_name.startswith("__") + and hasattr(cls, field_name) + and not callable(getattr(cls, field_name)) ) @classmethod @@ -214,7 +216,7 @@ class BaseField(FieldInfo): :rtype: bool """ return cls.default is not None or ( - cls.server_default is not None and use_server + cls.server_default is not None and use_server ) @classmethod @@ -247,7 +249,7 @@ class BaseField(FieldInfo): ondelete=con.ondelete, onupdate=con.onupdate, name=f"fk_{cls.owner.Meta.tablename}_{cls.to.Meta.tablename}" - f"_{cls.to.get_column_alias(cls.to.Meta.pkname)}_{cls.name}", + f"_{cls.to.get_column_alias(cls.to.Meta.pkname)}_{cls.name}", ) for con in cls.constraints ] @@ -278,33 +280,46 @@ class BaseField(FieldInfo): server_default=cls.server_default, ) else: - if cls.primary_key or cls.is_relation: - raise ModelDefinitionError("Primary key field and relations fields" - "cannot be encrypted!") - column = sqlalchemy.Column( - cls.alias or name, - EncryptedString( - _field_type=cls, - encrypt_secret=cls.encrypt_secret, - encrypt_backend=cls.encrypt_backend, - encrypt_custom_backend=cls.encrypt_custom_backend, - encrypt_max_length=cls.encrypt_max_length - ), - nullable=cls.nullable, - index=cls.index, - unique=cls.unique, - default=cls.default, - server_default=cls.server_default, - ) + column = cls._get_encrypted_column(name=name) + return column + @classmethod + def _get_encrypted_column(cls, name: str) -> sqlalchemy.Column: + """ + Returns EncryptedString column type instead of actual column. + + :param name: column name + :type name: str + :return: newly defined column + :rtype: sqlalchemy.Column + """ + if cls.primary_key or cls.is_relation: + raise ModelDefinitionError( + "Primary key field and relations fields" "cannot be encrypted!" + ) + column = sqlalchemy.Column( + cls.alias or name, + EncryptedString( + _field_type=cls, + encrypt_secret=cls.encrypt_secret, + encrypt_backend=cls.encrypt_backend, + encrypt_custom_backend=cls.encrypt_custom_backend, + encrypt_max_length=cls.encrypt_max_length, + ), + nullable=cls.nullable, + index=cls.index, + unique=cls.unique, + default=cls.default, + server_default=cls.server_default, + ) return column @classmethod def expand_relationship( - cls, - value: Any, - child: Union["Model", "NewBaseModel"], - to_register: bool = True, + cls, + value: Any, + child: Union["Model", "NewBaseModel"], + to_register: bool = True, ) -> Any: """ Function overwritten for relations, in basic field the value is returned as is. @@ -332,7 +347,7 @@ class BaseField(FieldInfo): :rtype: None """ if cls.owner is not None and ( - cls.owner == cls.to or cls.owner.Meta == cls.to.Meta + cls.owner == cls.to or cls.owner.Meta == cls.to.Meta ): cls.self_reference = True cls.self_reference_primary = cls.name diff --git a/ormar/fields/foreign_key.py b/ormar/fields/foreign_key.py index 7f1a500..0aafd5e 100644 --- a/ormar/fields/foreign_key.py +++ b/ormar/fields/foreign_key.py @@ -184,10 +184,25 @@ def ForeignKey( # noqa CFQ002 owner = kwargs.pop("owner", None) self_reference = kwargs.pop("self_reference", False) + default = kwargs.pop("default", None) - if default is not None: + encrypt_secret = kwargs.pop("encrypt_secret", None) + encrypt_backend = kwargs.pop("encrypt_backend", None) + encrypt_custom_backend = kwargs.pop("encrypt_custom_backend", None) + encrypt_max_length = kwargs.pop("encrypt_max_length", None) + + not_supported = [ + default, + encrypt_secret, + encrypt_backend, + encrypt_custom_backend, + encrypt_max_length, + ] + if any(x is not None for x in not_supported): raise ModelDefinitionError( - "Argument 'default' is not supported " "on relation fields!" + f"Argument {next((x for x in not_supported if x is not None))} " + f"is not supported " + "on relation fields!" ) if to.__class__ == ForwardRef: @@ -386,8 +401,6 @@ class ForeignKeyField(BaseField): :return: (if needed) registered Model :rtype: Model """ - if cls.to.pk_type() == uuid.UUID and isinstance(value, str): - value = uuid.UUID(value) if not isinstance(value, cls.to.pk_type()): raise RelationshipInstanceError( f"Relationship error - ForeignKey {cls.to.__name__} " diff --git a/ormar/fields/many_to_many.py b/ormar/fields/many_to_many.py index db763e3..ca26364 100644 --- a/ormar/fields/many_to_many.py +++ b/ormar/fields/many_to_many.py @@ -97,9 +97,23 @@ def ManyToMany( forbid_through_relations(cast(Type["Model"], through)) default = kwargs.pop("default", None) - if default is not None: + encrypt_secret = kwargs.pop("encrypt_secret", None) + encrypt_backend = kwargs.pop("encrypt_backend", None) + encrypt_custom_backend = kwargs.pop("encrypt_custom_backend", None) + encrypt_max_length = kwargs.pop("encrypt_max_length", None) + + not_supported = [ + default, + encrypt_secret, + encrypt_backend, + encrypt_custom_backend, + encrypt_max_length, + ] + if any(x is not None for x in not_supported): raise ModelDefinitionError( - "Argument 'default' is not supported " "on relation fields!" + f"Argument {next((x for x in not_supported if x is not None))} " + f"is not supported " + "on relation fields!" ) if to.__class__ == ForwardRef: diff --git a/ormar/fields/model_fields.py b/ormar/fields/model_fields.py index 67ea3e6..9cfd4f4 100644 --- a/ormar/fields/model_fields.py +++ b/ormar/fields/model_fields.py @@ -76,8 +76,7 @@ class ModelFieldFactory: encrypt_secret = kwargs.pop("encrypt_secret", None) encrypt_backend = kwargs.pop("encrypt_backend", EncryptBackends.NONE) - encrypt_custom_backend = kwargs.pop("encrypt_custom_backend", - None) + encrypt_custom_backend = kwargs.pop("encrypt_custom_backend", None) encrypt_max_length = kwargs.pop("encrypt_max_length", 5000) namespace = dict( diff --git a/ormar/fields/parsers.py b/ormar/fields/parsers.py new file mode 100644 index 0000000..e0f1a53 --- /dev/null +++ b/ormar/fields/parsers.py @@ -0,0 +1,44 @@ +import datetime +import decimal +from typing import Any + +import pydantic +from pydantic.datetime_parse import parse_date, parse_datetime, parse_time + +try: + import orjson as json +except ImportError: # pragma: no cover + import json # type: ignore + + +def parse_bool(value: str) -> bool: + return value == "true" + + +def encode_bool(value: bool) -> str: + return "true" if value else "false" + + +def encode_json(value: Any) -> str: + value = json.dumps(value) if not isinstance(value, str) else value + value = value.decode("utf-8") if isinstance(value, bytes) else value + return value + + +ENCODERS_MAP = { + bool: encode_bool, + datetime.datetime: lambda x: x.isoformat(), + datetime.date: lambda x: x.isoformat(), + datetime.time: lambda x: x.isoformat(), + pydantic.Json: encode_json, + decimal.Decimal: float, +} + +DECODERS_MAP = { + bool: parse_bool, + datetime.datetime: parse_datetime, + datetime.date: parse_date, + datetime.time: parse_time, + pydantic.Json: json.loads, + decimal.Decimal: decimal.Decimal, +} diff --git a/ormar/fields/sqlalchemy_encrypted.py b/ormar/fields/sqlalchemy_encrypted.py index a2e10b7..bdb1832 100644 --- a/ormar/fields/sqlalchemy_encrypted.py +++ b/ormar/fields/sqlalchemy_encrypted.py @@ -1,50 +1,49 @@ # inspired by sqlalchemy-utils (https://github.com/kvesteri/sqlalchemy-utils) import abc import base64 -import datetime -import json from enum import Enum -from typing import Any, Callable, TYPE_CHECKING, Type, Union +from typing import Any, Callable, Optional, TYPE_CHECKING, Type, Union import sqlalchemy.types as types +from pydantic.utils import lenient_issubclass from sqlalchemy.engine.default import DefaultDialect -from ormar import ModelDefinitionError +import ormar # noqa: I100, I202 +from ormar import ModelDefinitionError # noqa: I202, I100 cryptography = None -try: - import cryptography +try: # pragma: nocover + import cryptography # type: ignore from cryptography.fernet import Fernet from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes -except ImportError: +except ImportError: # pragma: nocover pass -if TYPE_CHECKING: +if TYPE_CHECKING: # pragma: nocover from ormar import BaseField class EncryptBackend(abc.ABC): - - def _update_key(self, key): + def _refresh(self, key: Union[str, bytes]) -> None: if isinstance(key, str): key = key.encode() digest = hashes.Hash(hashes.SHA256(), backend=default_backend()) digest.update(key) engine_key = digest.finalize() - self._initialize_engine(engine_key) + self._initialize_backend(engine_key) @abc.abstractmethod - def _initialize_engine(self, secret_key: bytes): + def _initialize_backend(self, secret_key: bytes) -> None: # pragma: nocover pass @abc.abstractmethod - def encrypt(self, value: Any) -> str: + def encrypt(self, value: Any) -> str: # pragma: nocover pass @abc.abstractmethod - def decrypt(self, value: Any) -> str: + def decrypt(self, value: Any) -> str: # pragma: nocover pass @@ -53,11 +52,11 @@ class HashBackend(EncryptBackend): One-way hashing - in example for passwords, no way to decrypt the value! """ - def _initialize_engine(self, secret_key: bytes): + def _initialize_backend(self, secret_key: bytes) -> None: self.secret_key = base64.urlsafe_b64encode(secret_key) def encrypt(self, value: Any) -> str: - if not isinstance(value, str): + if not isinstance(value, str): # pragma: nocover value = repr(value) value = value.encode() digest = hashes.Hash(hashes.SHA512(), backend=default_backend()) @@ -67,7 +66,7 @@ class HashBackend(EncryptBackend): return hashed_value.hex() def decrypt(self, value: Any) -> str: - if not isinstance(value, str): + if not isinstance(value, str): # pragma: nocover value = str(value) return value @@ -77,7 +76,7 @@ class FernetBackend(EncryptBackend): Two-way encryption, data stored in db are encrypted but decrypted during query. """ - def _initialize_engine(self, secret_key: bytes): + def _initialize_backend(self, secret_key: bytes) -> None: self.secret_key = base64.urlsafe_b64encode(secret_key) self.fernet = Fernet(self.secret_key) @@ -86,14 +85,14 @@ class FernetBackend(EncryptBackend): value = repr(value) value = value.encode() encrypted = self.fernet.encrypt(value) - return encrypted.decode('utf-8') + return encrypted.decode("utf-8") def decrypt(self, value: Any) -> str: - if not isinstance(value, str): + if not isinstance(value, str): # pragma: nocover value = str(value) - decrypted = self.fernet.decrypt(value.encode()) + decrypted: Union[str, bytes] = self.fernet.decrypt(value.encode()) if not isinstance(decrypted, str): - decrypted = decrypted.decode('utf-8') + decrypted = decrypted.decode("utf-8") return decrypted @@ -104,115 +103,82 @@ class EncryptBackends(Enum): CUSTOM = 3 -backends_map = { +BACKENDS_MAP = { EncryptBackends.FERNET: FernetBackend, EncryptBackends.HASH: HashBackend, - EncryptBackends.CUSTOM: None } -class EncryptedString(types.TypeDecorator): # pragma nocover +class EncryptedString(types.TypeDecorator): """ Used to store encrypted values in a database """ impl = types.TypeEngine - def __init__(self, - *args: Any, - encrypt_secret: Union[str, Callable], - _field_type: Type["BaseField"], - encrypt_max_length: int = 5000, - encrypt_backend: EncryptBackends = EncryptBackends.FERNET, - encrypt_custom_backend: Type[EncryptBackend] = None, - **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - if not cryptography: + def __init__( + self, + encrypt_secret: Union[str, Callable], + encrypt_backend: EncryptBackends = EncryptBackends.FERNET, + encrypt_custom_backend: Type[EncryptBackend] = None, + **kwargs: Any, + ) -> None: + _field_type = kwargs.pop("_field_type") + encrypt_max_length = kwargs.pop("encrypt_max_length", 5000) + super().__init__() + if not cryptography: # pragma: nocover raise ModelDefinitionError( "In order to encrypt a column 'cryptography' is required!" ) - backend = backends_map.get(encrypt_backend, encrypt_custom_backend) - if not backend or not issubclass(backend, EncryptBackend): + backend = BACKENDS_MAP.get(encrypt_backend, encrypt_custom_backend) + if not backend or not lenient_issubclass(backend, EncryptBackend): raise ModelDefinitionError("Wrong or no encrypt backend provided!") - self.backend = backend() - self._field_type = _field_type - self._underlying_type = _field_type.column_type - self._key = encrypt_secret - self.max_length = encrypt_max_length - def __repr__(self) -> str: + self.backend: EncryptBackend = backend() + self._field_type: Type["BaseField"] = _field_type + self._underlying_type: Any = _field_type.column_type + self._key: Union[str, Callable] = encrypt_secret + self.max_length: int = encrypt_max_length + type_ = self._field_type.__type__ + if type_ is None: # pragma: nocover + raise ModelDefinitionError( + f"Improperly configured field " f"{self._field_type.name}" + ) + self.type_: Any = type_ + + def __repr__(self) -> str: # pragma: nocover return f"VARCHAR({self.max_length})" def load_dialect_impl(self, dialect: DefaultDialect) -> Any: return dialect.type_descriptor(types.VARCHAR(self.max_length)) - @property - def key(self): - return self._key - - @key.setter - def key(self, value): - self._key = value - - def _update_key(self): + def _refresh(self) -> None: key = self._key() if callable(self._key) else self._key - self.backend._update_key(key) + self.backend._refresh(key) - def process_bind_param(self, value, dialect): - """Encrypt a value on the way in.""" - if value is not None: - self._update_key() + def process_bind_param(self, value: Any, dialect: DefaultDialect) -> Optional[str]: + if value is None: + return value + self._refresh() + try: + value = self._underlying_type.process_bind_param(value, dialect) + except AttributeError: + encoder = ormar.ENCODERS_MAP.get(self.type_, None) + if encoder: + value = encoder(value) # type: ignore - try: - value = self._underlying_type.process_bind_param( - value, dialect - ) + return self.backend.encrypt(value) - except AttributeError: - # Doesn't have 'process_bind_param' - type_ = self._field_type.__type__ - if issubclass(type_, bool): - value = 'true' if value else 'false' + def process_result_value(self, value: Any, dialect: DefaultDialect) -> Any: + if value is None: + return value + self._refresh() + decrypted_value = self.backend.decrypt(value) + try: + return self._underlying_type.process_result_value(decrypted_value, dialect) + except AttributeError: + decoder = ormar.DECODERS_MAP.get(self.type_, None) + if decoder: + return decoder(decrypted_value) # type: ignore - elif issubclass(type_, (datetime.date, datetime.time)): - value = value.isoformat() - - # elif issubclass(type_, JSONType): - # value = json.dumps(value) - - return self.backend.encrypt(value) - - def process_result_value(self, value, dialect): - """Decrypt value on the way out.""" - if value is not None: - self._update_key() - decrypted_value = self.backend.decrypt(value) - - try: - return self.underlying_type.process_result_value( - decrypted_value, dialect - ) - - except AttributeError: - # Doesn't have 'process_result_value' - - # Handle 'boolean' and 'dates' - type_ = self._field_type.__type__ - # date_types = [datetime.datetime, datetime.time, datetime.date] - - if issubclass(type_, bool): - return decrypted_value == 'true' - - # elif type_ in date_types: - # return DatetimeHandler.process_value( - # decrypted_value, type_ - # ) - - # elif issubclass(type_, JSONType): - # return json.loads(decrypted_value) - - # Handle all others - return self.underlying_type.python_type(decrypted_value) - - def _coerce(self, value): - return self.underlying_type._coerce(value) + return self._field_type.__type__(decrypted_value) # type: ignore diff --git a/ormar/fields/sqlalchemy_uuid.py b/ormar/fields/sqlalchemy_uuid.py index 2a6bfd7..826c0c8 100644 --- a/ormar/fields/sqlalchemy_uuid.py +++ b/ormar/fields/sqlalchemy_uuid.py @@ -1,12 +1,12 @@ import uuid -from typing import Any, Optional, Union +from typing import Any, Optional from sqlalchemy import CHAR from sqlalchemy.engine.default import DefaultDialect from sqlalchemy.types import TypeDecorator -class UUID(TypeDecorator): # pragma nocover +class UUID(TypeDecorator): """ Platform-independent GUID type. Uses CHAR(36) if in a string mode, otherwise uses CHAR(32), to store UUID. @@ -20,31 +20,11 @@ class UUID(TypeDecorator): # pragma nocover super().__init__(*args, **kwargs) self.uuid_format = uuid_format - def __repr__(self) -> str: + def __repr__(self) -> str: # pragma: nocover if self.uuid_format == "string": return "CHAR(36)" return "CHAR(32)" - def _cast_to_uuid(self, value: Union[str, int, bytes]) -> uuid.UUID: - """ - Parses given value into uuid.UUID field. - - :param value: value to be parsed - :type value: Union[str, int, bytes] - :return: initialized uuid - :rtype: uuid.UUID - """ - if not isinstance(value, uuid.UUID): - if isinstance(value, bytes): - ret_value = uuid.UUID(bytes=value) - elif isinstance(value, int): - ret_value = uuid.UUID(int=value) - elif isinstance(value, str): - ret_value = uuid.UUID(value) - else: - ret_value = value - return ret_value - def load_dialect_impl(self, dialect: DefaultDialect) -> Any: return ( dialect.type_descriptor(CHAR(36)) @@ -53,12 +33,10 @@ class UUID(TypeDecorator): # pragma nocover ) def process_bind_param( - self, value: Union[str, int, bytes, uuid.UUID, None], dialect: DefaultDialect + self, value: uuid.UUID, dialect: DefaultDialect ) -> Optional[str]: if value is None: return value - if not isinstance(value, uuid.UUID): - value = self._cast_to_uuid(value) return str(value) if self.uuid_format == "string" else "%.32x" % value.int def process_result_value( @@ -68,4 +46,4 @@ class UUID(TypeDecorator): # pragma nocover return value if not isinstance(value, uuid.UUID): return uuid.UUID(value) - return value + return value # pragma: nocover diff --git a/ormar/models/helpers/validation.py b/ormar/models/helpers/validation.py index 582c3fa..bc87235 100644 --- a/ormar/models/helpers/validation.py +++ b/ormar/models/helpers/validation.py @@ -53,6 +53,7 @@ def convert_choices_if_needed( # noqa: CCR001 :return: value, choices list :rtype: Tuple[Any, List] """ + # TODO use same maps as with EncryptedString choices = [o.value if isinstance(o, Enum) else o for o in field.choices] if field.__type__ in [datetime.datetime, datetime.date, datetime.time]: diff --git a/ormar/queryset/join.py b/ormar/queryset/join.py index b9e71df..e710aef 100644 --- a/ormar/queryset/join.py +++ b/ormar/queryset/join.py @@ -88,13 +88,13 @@ class SqlJoin: return self.main_model.Meta.alias_manager @property - def to_table(self) -> str: + def to_table(self) -> sqlalchemy.Table: """ Shortcut to table name of the next model :return: name of the target table :rtype: str """ - return self.next_model.Meta.table.name + return self.next_model.Meta.table def _on_clause( self, previous_alias: str, from_clause: str, to_clause: str, @@ -282,7 +282,7 @@ class SqlJoin: on_clause = self._on_clause( previous_alias=self.own_alias, from_clause=f"{self.target_field.owner.Meta.tablename}.{from_key}", - to_clause=f"{self.to_table}.{to_key}", + to_clause=f"{self.to_table.name}.{to_key}", ) target_table = self.alias_manager.prefixed_table_name( self.next_alias, self.to_table @@ -301,7 +301,7 @@ class SqlJoin: ) self.columns.extend( self.alias_manager.prefixed_columns( - self.next_alias, self.next_model.Meta.table, self_related_fields + self.next_alias, target_table, self_related_fields ) ) self.used_aliases.append(self.next_alias) diff --git a/ormar/relations/alias_manager.py b/ormar/relations/alias_manager.py index 815a4dc..2ec6159 100644 --- a/ormar/relations/alias_manager.py +++ b/ormar/relations/alias_manager.py @@ -67,24 +67,21 @@ class AliasManager: if not fields else [col for col in table.columns if col.name in fields] ) - return [ - text(f"{alias}{table.name}.{column.name} as {alias}{column.name}") - for column in all_columns - ] + return [column.label(f"{alias}{column.name}") for column in all_columns] @staticmethod - def prefixed_table_name(alias: str, name: str) -> text: + def prefixed_table_name(alias: str, table: sqlalchemy.Table) -> text: """ Creates text clause with table name with aliased name. :param alias: alias of given table :type alias: str - :param name: table name - :type name: str + :param table: table + :type table: sqlalchemy.Table :return: sqlalchemy text clause as "table_name aliased_name" :rtype: sqlalchemy text clause """ - return text(f"{name} {alias}_{name}") + return table.alias(f"{alias}_{table.name}") def add_relation_type( self, source_model: Type["Model"], relation_name: str, reverse_name: str = None, diff --git a/tests/test_encrypted_columns.py b/tests/test_encrypted_columns.py index 4de0459..5eff02f 100644 --- a/tests/test_encrypted_columns.py +++ b/tests/test_encrypted_columns.py @@ -1,12 +1,16 @@ +# type: ignore +import decimal import uuid -from typing import Optional +import datetime +from typing import Any import databases import pytest import sqlalchemy import ormar -from ormar.exceptions import QueryDefinitionError +from ormar import ModelDefinitionError +from ormar.fields.sqlalchemy_encrypted import EncryptedString from tests.settings import DATABASE_URL database = databases.Database(DATABASE_URL) @@ -18,22 +22,58 @@ class BaseMeta(ormar.ModelMeta): database = database +default_fernet = dict( + encrypt_secret="asd123", encrypt_backend=ormar.EncryptBackends.FERNET, +) + + +class DummyBackend(ormar.fields.EncryptBackend): + def _initialize_backend(self, secret_key: bytes) -> None: + pass + + def encrypt(self, value: Any) -> str: + return value + + def decrypt(self, value: Any) -> str: + return value + + class Author(ormar.Model): class Meta(BaseMeta): tablename = "authors" id: int = ormar.Integer(primary_key=True) - name: str = ormar.String(max_length=100, - encrypt_secret='asd123', - encrypt_backend=ormar.EncryptBackends.FERNET) - uuid_test = ormar.UUID(default=uuid.uuid4, uuid_format='string') - password: str = ormar.String(max_length=100, - encrypt_secret='udxc32', - encrypt_backend=ormar.EncryptBackends.HASH) - birth_year: int = ormar.Integer(nullable=True, - encrypt_secret='secure89key%^&psdijfipew', - encrypt_max_length=200, - encrypt_backend=ormar.EncryptBackends.FERNET) + name: str = ormar.String(max_length=100, **default_fernet) + uuid_test = ormar.UUID(default=uuid.uuid4, uuid_format="string") + uuid_test2 = ormar.UUID(nullable=True, uuid_format="string") + password: str = ormar.String( + max_length=128, + encrypt_secret="udxc32", + encrypt_backend=ormar.EncryptBackends.HASH, + ) + birth_year: int = ormar.Integer( + nullable=True, + encrypt_secret="secure89key%^&psdijfipew", + encrypt_max_length=200, + encrypt_backend=ormar.EncryptBackends.FERNET, + ) + test_text: str = ormar.Text(default="", **default_fernet) + test_bool: bool = ormar.Boolean(nullable=False, **default_fernet) + test_float: float = ormar.Float(**default_fernet) + test_float2: float = ormar.Float(nullable=True, **default_fernet) + test_datetime = ormar.DateTime(default=datetime.datetime.now, **default_fernet) + test_date = ormar.Date(default=datetime.date.today, **default_fernet) + test_time = ormar.Time(default=datetime.time, **default_fernet) + test_json = ormar.JSON(default={}, **default_fernet) + test_bigint: int = ormar.BigInteger(default=0, **default_fernet) + test_decimal = ormar.Decimal(scale=2, precision=10, **default_fernet) + test_decimal2 = ormar.Decimal(max_digits=10, decimal_places=2, **default_fernet) + custom_backend: str = ormar.String( + max_length=200, + encrypt_secret="asda8", + encrypt_backend=ormar.EncryptBackends.CUSTOM, + encrypt_custom_backend=DummyBackend, + ) @pytest.fixture(autouse=True, scope="module") @@ -45,16 +85,104 @@ def create_test_database(): metadata.drop_all(engine) +def test_error_on_encrypted_pk(): + with pytest.raises(ModelDefinitionError): + + class Wrong(ormar.Model): + class Meta(BaseMeta): + tablename = "wrongs" + + id: int = ormar.Integer( + primary_key=True, + encrypt_secret="asd123", + encrypt_backend=ormar.EncryptBackends.FERNET, + ) + + +def test_error_on_encrypted_relation(): + with pytest.raises(ModelDefinitionError): + + class Wrong2(ormar.Model): + class Meta(BaseMeta): + tablename = "wrongs2" + + id: int = ormar.Integer(primary_key=True) + author = ormar.ForeignKey( + Author, + encrypt_secret="asd123", + encrypt_backend=ormar.EncryptBackends.FERNET, + ) + + +def test_error_on_encrypted_m2m_relation(): + with pytest.raises(ModelDefinitionError): + + class Wrong3(ormar.Model): + class Meta(BaseMeta): + tablename = "wrongs3" + + id: int = ormar.Integer(primary_key=True) + author = ormar.ManyToMany( + Author, + encrypt_secret="asd123", + encrypt_backend=ormar.EncryptBackends.FERNET, + ) + + +def test_wrong_backend(): + with pytest.raises(ModelDefinitionError): + + class Wrong3(ormar.Model): + class Meta(BaseMeta): + tablename = "wrongs3" + + id: int = ormar.Integer(primary_key=True) + author = ormar.Integer( + encrypt_secret="asd123", + encrypt_backend=ormar.EncryptBackends.CUSTOM, + encrypt_custom_backend="aa", + ) + + def test_db_structure(): - assert Author.Meta.table.c.get('name').type.impl.__class__ == sqlalchemy.NVARCHAR - assert Author.Meta.table.c.get('birth_year').type.max_length == 200 + assert Author.Meta.table.c.get("name").type.__class__ == EncryptedString + assert Author.Meta.table.c.get("birth_year").type.max_length == 200 @pytest.mark.asyncio -async def test_wrong_query_foreign_key_type(): +async def test_save_and_retrieve(): async with database: - await Author(name='Test', birth_year=1988, password='test123').save() + test_uuid = uuid.uuid4() + await Author( + name="Test", + birth_year=1988, + password="test123", + uuid_test=test_uuid, + test_float=1.2, + test_bool=True, + test_decimal=decimal.Decimal(3.5), + test_decimal2=decimal.Decimal(5.5), + test_json=dict(aa=12), + custom_backend="test12", + ).save() author = await Author.objects.get() - assert author.name == 'Test' + assert author.name == "Test" assert author.birth_year == 1988 + password = ( + "03e4a4d513e99cb3fe4ee3db282c053daa3f3572b849c3868939a306944ad5c08" + "22b50d4886e10f4cd418c3f2df3ceb02e2e7ac6e920ae0c90f2dedfc8fa16e2" + ) + assert author.password == password + assert author.uuid_test == test_uuid + assert author.uuid_test2 is None + assert author.test_datetime.date() == datetime.date.today() + assert author.test_date == datetime.date.today() + assert author.test_text == "" + assert author.test_float == 1.2 + assert author.test_float2 is None + assert author.test_bigint == 0 + assert author.test_json == {"aa": 12} + assert author.test_decimal == 3.5 + assert author.test_decimal2 == 5.5 + assert author.custom_backend == "test12" diff --git a/tests/test_or_filters.py b/tests/test_or_filters.py index 6d6f8ab..81a412c 100644 --- a/tests/test_or_filters.py +++ b/tests/test_or_filters.py @@ -57,24 +57,24 @@ async def test_or_filters(): books = ( await Book.objects.select_related("author") - .filter(ormar.or_(author__name="J.R.R. Tolkien", year__gt=1970)) - .all() + .filter(ormar.or_(author__name="J.R.R. Tolkien", year__gt=1970)) + .all() ) assert len(books) == 5 books = ( await Book.objects.select_related("author") - .filter(ormar.or_(author__name="J.R.R. Tolkien", year__lt=1995)) - .all() + .filter(ormar.or_(author__name="J.R.R. Tolkien", year__lt=1995)) + .all() ) assert len(books) == 4 assert not any([x.title == "The Tower of Fools" for x in books]) books = ( await Book.objects.select_related("author") - .filter(ormar.or_(year__gt=1960, year__lt=1940)) - .filter(author__name="J.R.R. Tolkien") - .all() + .filter(ormar.or_(year__gt=1960, year__lt=1940)) + .filter(author__name="J.R.R. Tolkien") + .all() ) assert len(books) == 2 assert books[0].title == "The Hobbit" @@ -82,13 +82,13 @@ async def test_or_filters(): books = ( await Book.objects.select_related("author") - .filter( + .filter( ormar.and_( ormar.or_(year__gt=1960, year__lt=1940), author__name="J.R.R. Tolkien", ) ) - .all() + .all() ) assert len(books) == 2 @@ -97,14 +97,14 @@ async def test_or_filters(): books = ( await Book.objects.select_related("author") - .filter( + .filter( ormar.or_( ormar.and_(year__gt=1960, author__name="J.R.R. Tolkien"), ormar.and_(year__lt=2000, author__name="Andrzej Sapkowski"), ) ) - .filter(title__startswith="The") - .all() + .filter(title__startswith="The") + .all() ) assert len(books) == 2 assert books[0].title == "The Silmarillion" @@ -112,7 +112,7 @@ async def test_or_filters(): books = ( await Book.objects.select_related("author") - .filter( + .filter( ormar.or_( ormar.and_( ormar.or_(year__gt=1960, year__lt=1940), @@ -121,7 +121,7 @@ async def test_or_filters(): ormar.and_(year__lt=2000, author__name="Andrzej Sapkowski"), ) ) - .all() + .all() ) assert len(books) == 3 assert books[0].title == "The Hobbit" @@ -130,29 +130,29 @@ async def test_or_filters(): books = ( await Book.objects.select_related("author") - .exclude( + .exclude( ormar.or_( ormar.and_(year__gt=1960, author__name="J.R.R. Tolkien"), ormar.and_(year__lt=2000, author__name="Andrzej Sapkowski"), ) ) - .filter(title__startswith="The") - .all() + .filter(title__startswith="The") + .all() ) assert len(books) == 3 assert not any([x.title in ["The Silmarillion", "The Witcher"] for x in books]) books = ( await Book.objects.select_related("author") - .filter( + .filter( ormar.or_( ormar.and_(year__gt=1960, author__name="J.R.R. Tolkien"), ormar.and_(year__lt=2000, author__name="Andrzej Sapkowski"), title__icontains="hobbit", ) ) - .filter(title__startswith="The") - .all() + .filter(title__startswith="The") + .all() ) assert len(books) == 3 assert not any( @@ -161,43 +161,43 @@ async def test_or_filters(): books = ( await Book.objects.select_related("author") - .filter(ormar.or_(year__gt=1980, year__lt=1910)) - .filter(title__startswith="The") - .limit(1) - .all() + .filter(ormar.or_(year__gt=1980, year__lt=1910)) + .filter(title__startswith="The") + .limit(1) + .all() ) assert len(books) == 1 assert books[0].title == "The Witcher" books = ( await Book.objects.select_related("author") - .filter(ormar.or_(year__gt=1980, author__name="Andrzej Sapkowski")) - .filter(title__startswith="The") - .limit(1) - .all() + .filter(ormar.or_(year__gt=1980, author__name="Andrzej Sapkowski")) + .filter(title__startswith="The") + .limit(1) + .all() ) assert len(books) == 1 assert books[0].title == "The Witcher" books = ( await Book.objects.select_related("author") - .filter(ormar.or_(year__gt=1980, author__name="Andrzej Sapkowski")) - .filter(title__startswith="The") - .limit(1) - .offset(1) - .all() + .filter(ormar.or_(year__gt=1980, author__name="Andrzej Sapkowski")) + .filter(title__startswith="The") + .limit(1) + .offset(1) + .all() ) assert len(books) == 1 assert books[0].title == "The Tower of Fools" books = ( await Book.objects.select_related("author") - .filter(ormar.or_(year__gt=1980, author__name="Andrzej Sapkowski")) - .filter(title__startswith="The") - .limit(1) - .offset(1) - .order_by("-id") - .all() + .filter(ormar.or_(year__gt=1980, author__name="Andrzej Sapkowski")) + .filter(title__startswith="The") + .limit(1) + .offset(1) + .order_by("-id") + .all() ) assert len(books) == 1 assert books[0].title == "The Witcher" @@ -220,19 +220,19 @@ async def test_or_filters(): books = ( await Book.objects.select_related("author") - .filter(ormar.or_(author__name="J.R.R. Tolkien")) - .all() + .filter(ormar.or_(author__name="J.R.R. Tolkien")) + .all() ) assert len(books) == 3 books = ( await Book.objects.select_related("author") - .filter( + .filter( ormar.or_( ormar.and_(author__name__icontains="tolkien"), ormar.and_(author__name__icontains="sapkowski"), ) ) - .all() + .all() ) assert len(books) == 5 From fbde6d4624cab4d3c4dbbbbf1ee49571de438228 Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 9 Mar 2021 20:35:07 +0100 Subject: [PATCH 06/13] use TEXT col for mysql --- ormar/fields/sqlalchemy_encrypted.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ormar/fields/sqlalchemy_encrypted.py b/ormar/fields/sqlalchemy_encrypted.py index bdb1832..f08c43c 100644 --- a/ormar/fields/sqlalchemy_encrypted.py +++ b/ormar/fields/sqlalchemy_encrypted.py @@ -150,6 +150,8 @@ class EncryptedString(types.TypeDecorator): return f"VARCHAR({self.max_length})" def load_dialect_impl(self, dialect: DefaultDialect) -> Any: + if dialect.name == 'mysql': # pragma: nocover + return dialect.type_descriptor(types.TEXT()) return dialect.type_descriptor(types.VARCHAR(self.max_length)) def _refresh(self) -> None: From 082405d8ef63d60a1c594a7a640beaf182327569 Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 9 Mar 2021 20:41:17 +0100 Subject: [PATCH 07/13] use TEXT for all backends --- ormar/fields/base.py | 2 -- ormar/fields/foreign_key.py | 2 -- ormar/fields/many_to_many.py | 2 -- ormar/fields/sqlalchemy_encrypted.py | 8 ++------ tests/test_encrypted_columns.py | 2 -- 5 files changed, 2 insertions(+), 14 deletions(-) diff --git a/ormar/fields/base.py b/ormar/fields/base.py index 041d934..c58348c 100644 --- a/ormar/fields/base.py +++ b/ormar/fields/base.py @@ -58,7 +58,6 @@ class BaseField(FieldInfo): encrypt_secret: str encrypt_backend: EncryptBackends = EncryptBackends.NONE encrypt_custom_backend: Optional[Type[EncryptBackend]] = None - encrypt_max_length: int = 5000 default: Any server_default: Any @@ -304,7 +303,6 @@ class BaseField(FieldInfo): encrypt_secret=cls.encrypt_secret, encrypt_backend=cls.encrypt_backend, encrypt_custom_backend=cls.encrypt_custom_backend, - encrypt_max_length=cls.encrypt_max_length, ), nullable=cls.nullable, index=cls.index, diff --git a/ormar/fields/foreign_key.py b/ormar/fields/foreign_key.py index 0aafd5e..a0122d4 100644 --- a/ormar/fields/foreign_key.py +++ b/ormar/fields/foreign_key.py @@ -189,14 +189,12 @@ def ForeignKey( # noqa CFQ002 encrypt_secret = kwargs.pop("encrypt_secret", None) encrypt_backend = kwargs.pop("encrypt_backend", None) encrypt_custom_backend = kwargs.pop("encrypt_custom_backend", None) - encrypt_max_length = kwargs.pop("encrypt_max_length", None) not_supported = [ default, encrypt_secret, encrypt_backend, encrypt_custom_backend, - encrypt_max_length, ] if any(x is not None for x in not_supported): raise ModelDefinitionError( diff --git a/ormar/fields/many_to_many.py b/ormar/fields/many_to_many.py index ca26364..2382fa5 100644 --- a/ormar/fields/many_to_many.py +++ b/ormar/fields/many_to_many.py @@ -100,14 +100,12 @@ def ManyToMany( encrypt_secret = kwargs.pop("encrypt_secret", None) encrypt_backend = kwargs.pop("encrypt_backend", None) encrypt_custom_backend = kwargs.pop("encrypt_custom_backend", None) - encrypt_max_length = kwargs.pop("encrypt_max_length", None) not_supported = [ default, encrypt_secret, encrypt_backend, encrypt_custom_backend, - encrypt_max_length, ] if any(x is not None for x in not_supported): raise ModelDefinitionError( diff --git a/ormar/fields/sqlalchemy_encrypted.py b/ormar/fields/sqlalchemy_encrypted.py index f08c43c..a198bd6 100644 --- a/ormar/fields/sqlalchemy_encrypted.py +++ b/ormar/fields/sqlalchemy_encrypted.py @@ -124,7 +124,6 @@ class EncryptedString(types.TypeDecorator): **kwargs: Any, ) -> None: _field_type = kwargs.pop("_field_type") - encrypt_max_length = kwargs.pop("encrypt_max_length", 5000) super().__init__() if not cryptography: # pragma: nocover raise ModelDefinitionError( @@ -138,7 +137,6 @@ class EncryptedString(types.TypeDecorator): self._field_type: Type["BaseField"] = _field_type self._underlying_type: Any = _field_type.column_type self._key: Union[str, Callable] = encrypt_secret - self.max_length: int = encrypt_max_length type_ = self._field_type.__type__ if type_ is None: # pragma: nocover raise ModelDefinitionError( @@ -147,12 +145,10 @@ class EncryptedString(types.TypeDecorator): self.type_: Any = type_ def __repr__(self) -> str: # pragma: nocover - return f"VARCHAR({self.max_length})" + return f"TEXT()" def load_dialect_impl(self, dialect: DefaultDialect) -> Any: - if dialect.name == 'mysql': # pragma: nocover - return dialect.type_descriptor(types.TEXT()) - return dialect.type_descriptor(types.VARCHAR(self.max_length)) + return dialect.type_descriptor(types.TEXT()) def _refresh(self) -> None: key = self._key() if callable(self._key) else self._key diff --git a/tests/test_encrypted_columns.py b/tests/test_encrypted_columns.py index 5eff02f..323563f 100644 --- a/tests/test_encrypted_columns.py +++ b/tests/test_encrypted_columns.py @@ -54,7 +54,6 @@ class Author(ormar.Model): birth_year: int = ormar.Integer( nullable=True, encrypt_secret="secure89key%^&psdijfipew", - encrypt_max_length=200, encrypt_backend=ormar.EncryptBackends.FERNET, ) test_text: str = ormar.Text(default="", **default_fernet) @@ -146,7 +145,6 @@ def test_wrong_backend(): def test_db_structure(): assert Author.Meta.table.c.get("name").type.__class__ == EncryptedString - assert Author.Meta.table.c.get("birth_year").type.max_length == 200 @pytest.mark.asyncio From 869f4d9d9755b0f7aae229140dcdcb9ff418defc Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 9 Mar 2021 20:49:20 +0100 Subject: [PATCH 08/13] restore uuid check for fk --- ormar/fields/foreign_key.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ormar/fields/foreign_key.py b/ormar/fields/foreign_key.py index a0122d4..dfc3f4b 100644 --- a/ormar/fields/foreign_key.py +++ b/ormar/fields/foreign_key.py @@ -399,6 +399,8 @@ class ForeignKeyField(BaseField): :return: (if needed) registered Model :rtype: Model """ + if cls.to.pk_type() == uuid.UUID and isinstance(value, str): + value = uuid.UUID(value) if not isinstance(value, cls.to.pk_type()): raise RelationshipInstanceError( f"Relationship error - ForeignKey {cls.to.__name__} " From b8a85436f7b7ca9f59012c05b4b2952bfe1ed0e9 Mon Sep 17 00:00:00 2001 From: collerek Date: Tue, 9 Mar 2021 20:52:58 +0100 Subject: [PATCH 09/13] restore uuid check for fk --- ormar/fields/foreign_key.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ormar/fields/foreign_key.py b/ormar/fields/foreign_key.py index dfc3f4b..e981d9e 100644 --- a/ormar/fields/foreign_key.py +++ b/ormar/fields/foreign_key.py @@ -399,7 +399,7 @@ class ForeignKeyField(BaseField): :return: (if needed) registered Model :rtype: Model """ - if cls.to.pk_type() == uuid.UUID and isinstance(value, str): + if cls.to.pk_type() == uuid.UUID and isinstance(value, str): # pragma: nocover value = uuid.UUID(value) if not isinstance(value, cls.to.pk_type()): raise RelationshipInstanceError( From f6c845c31876aea596caba1f8b8c39687ca14d0a Mon Sep 17 00:00:00 2001 From: collerek Date: Wed, 10 Mar 2021 09:06:23 +0100 Subject: [PATCH 10/13] add makefile and local test for pb mysql to avoid uploading garbage --- .gitignore | 3 ++- Makefile | 19 +++++++++++++++++++ scripts/docker-compose.yml | 20 ++++++++++++++++++++ tests/settings.py | 1 + 4 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 Makefile create mode 100644 scripts/docker-compose.yml diff --git a/.gitignore b/.gitignore index 6c5114b..2b9985d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ alembic.ini .idea .pytest_cache .mypy_cache -.coverage +*.coverage *.pyc *.log test.db @@ -14,3 +14,4 @@ site profile.py *.db *.db-journal +*coverage.xml \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0995692 --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +PIPENV_RUN := pipenv run +PG_DOCKERFILE_NAME := fastapi-users-test-mongo + +test_all: test_pg test_mysql test_sqlite + +test_pg: export DATABASE_URL=postgresql://username:password@localhost:5432/testsuite +test_pg: + docker-compose -f scripts/docker-compose.yml up -d postgres + bash scripts/test.sh -svv + docker-compose stop postgres + +test_mysql: export DATABASE_URL=mysql://username:password@127.0.0.1:3306/testsuite +test_mysql: + docker-compose -f "scripts/docker-compose.yml" up -d mysql + bash scripts/test.sh -svv + docker-compose stop mysql + +test_sqlite: + bash scripts/test.sh -svv \ No newline at end of file diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml new file mode 100644 index 0000000..c6ad36e --- /dev/null +++ b/scripts/docker-compose.yml @@ -0,0 +1,20 @@ +version: '2.1' +services: + postgres: + image: postgres:10.8 + environment: + POSTGRES_USER: username + POSTGRES_PASSWORD: password + POSTGRES_DB: testsuite + ports: + - 5432:5432 + + mysql: + image: mysql:5.7 + environment: + MYSQL_USER: username + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: password + MYSQL_DATABASE: testsuite + ports: + - 3306:3306 \ No newline at end of file diff --git a/tests/settings.py b/tests/settings.py index 6d89f4e..fededf3 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -6,3 +6,4 @@ DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///test.db") database_url = databases.DatabaseURL(DATABASE_URL) if database_url.scheme == "postgresql+aiopg": # pragma no cover DATABASE_URL = str(database_url.replace(driver=None)) +print('USED DB:', DATABASE_URL) From 8f2704146df7b436127390cf28561ca4961ce8ad Mon Sep 17 00:00:00 2001 From: collerek Date: Wed, 10 Mar 2021 09:12:31 +0100 Subject: [PATCH 11/13] remove switch to _row for pg backend --- Makefile | 4 ++-- ormar/models/model_row.py | 5 +---- tests/settings.py | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 0995692..721821b 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,13 @@ test_pg: export DATABASE_URL=postgresql://username:password@localhost:5432/tests test_pg: docker-compose -f scripts/docker-compose.yml up -d postgres bash scripts/test.sh -svv - docker-compose stop postgres + docker-compose -f scripts/docker-compose.yml stop postgres test_mysql: export DATABASE_URL=mysql://username:password@127.0.0.1:3306/testsuite test_mysql: docker-compose -f "scripts/docker-compose.yml" up -d mysql bash scripts/test.sh -svv - docker-compose stop mysql + docker-compose -f scripts/docker-compose.yml stop mysql test_sqlite: bash scripts/test.sh -svv \ No newline at end of file diff --git a/ormar/models/model_row.py b/ormar/models/model_row.py index 2cd488a..d06d0d9 100644 --- a/ormar/models/model_row.py +++ b/ormar/models/model_row.py @@ -369,9 +369,6 @@ class ModelRow(NewBaseModel): and values are database values :rtype: Dict """ - # databases does not keep aliases in Record for postgres, change to raw row - source = row._row if cls.db_backend_name() == "postgresql" else row - selected_columns = cls.own_table_columns( model=cls, excludable=excludable, alias=table_prefix, use_alias=False, ) @@ -381,6 +378,6 @@ class ModelRow(NewBaseModel): alias = cls.get_column_name_from_alias(column.name) if alias not in item and alias in selected_columns: prefixed_name = f"{column_prefix}{column.name}" - item[alias] = source[prefixed_name] + item[alias] = row[prefixed_name] return item diff --git a/tests/settings.py b/tests/settings.py index fededf3..be1bed2 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -6,4 +6,4 @@ DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///test.db") database_url = databases.DatabaseURL(DATABASE_URL) if database_url.scheme == "postgresql+aiopg": # pragma no cover DATABASE_URL = str(database_url.replace(driver=None)) -print('USED DB:', DATABASE_URL) +print("USED DB:", DATABASE_URL) From 01904580e5dd92527ece34d3c2533593b9fa4186 Mon Sep 17 00:00:00 2001 From: collerek Date: Wed, 10 Mar 2021 11:39:51 +0100 Subject: [PATCH 12/13] add release docs, add docs, finish tests --- docs/fields/encryption.md | 163 +++++++++++++++++++++++++++ docs/releases.md | 18 +++ mkdocs.yml | 1 + ormar/fields/sqlalchemy_encrypted.py | 4 +- setup.py | 3 +- tests/test_encrypted_columns.py | 54 ++++++++- 6 files changed, 239 insertions(+), 4 deletions(-) create mode 100644 docs/fields/encryption.md diff --git a/docs/fields/encryption.md b/docs/fields/encryption.md new file mode 100644 index 0000000..e4a11c0 --- /dev/null +++ b/docs/fields/encryption.md @@ -0,0 +1,163 @@ +# Encryption + +`ormar` provides you with a way to encrypt a field in the database only. +Provided encryption backends allow for both one-way encryption (`HASH` backend) as +well as both-way encryption/decryption (`FERNET` backend). + +!!!warning + Note that in order for encryption to work you need to install optional `cryptography` package. + + You can do it manually `pip install cryptography` or with ormar by `pip install ormar[crypto]` + +!!!warning + Note that adding `encrypt_backend` changes the database column type to `TEXT`, + which needs to be reflected in db either by migration (`alembic`) or manual change + +## Defining a field encryption + +To encrypt a field you need to pass at minimum `encrypt_secret` and `encrypt_backend` parameters. + +```python hl_lines="7-8" +class Filter(ormar.Model): + class Meta(BaseMeta): + tablename = "filters" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, + encrypt_secret="secret123", + encrypt_backend=ormar.EncryptBackends.FERNET) +``` + +!!!warning + You can encrypt all `Field` types apart from `primary_key` column and relation + columns (`ForeignKey` and `ManyToMany`). Check backends details for more information. + +## Available backends + +### HASH + +HASH is a one-way hash (like for password), never decrypted on retrieval + +To set it up pass appropriate backend value. + +```python +... # rest of model definition +password: str = ormar.String(max_length=128, + encrypt_secret="secret123", + encrypt_backend=ormar.EncryptBackends.HASH) +``` + +Note that since this backend never decrypt the stored value it's only applicable for +`String` fields. Used hash is a `sha512` hash, so the field length has to be >=128. + +!!!warning + Note that in `HASH` backend you can filter by full value but filters like `contain` will not work as comparison is make on encrypted values + +!!!note + Note that provided `encrypt_secret` is first hashed itself and used as salt, so in order to + compare to stored string you need to recreate this steps. + +```python +class Hash(ormar.Model): + class Meta(BaseMeta): + tablename = "hashes" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=128, + encrypt_secret="udxc32", + encrypt_backend=ormar.EncryptBackends.HASH) + + +await Hash(name='test1').save() + +# note the steps to recreate the stored value +# you can use also cryptography package instead of hashlib +secret = hashlib.sha256("udxc32".encode()).digest() +secret = base64.urlsafe_b64encode(secret) +hashed_test1 = hashlib.sha512(secret + 'test1'.encode()).hexdigest() + +# full value comparison works +hash1 = await Hash.objects.get(name='test1') +assert hash1.name == hashed_test1 + +# but partial comparison does not (hashed strings are compared) +with pytest.raises(NoMatch): + await Filter.objects.get(name__icontains='test') +``` + +### FERNET + +FERNET is a two-way encrypt/decrypt backend + +To set it up pass appropriate backend value. + +```python +... # rest of model definition +year: int = ormar.Integer(encrypt_secret="secret123", + encrypt_backend=ormar.EncryptBackends.FERNET) +``` + +Value is encrypted on way to database end decrypted on way out. Can be used on all types, +as the returned value is parsed to corresponding python type. + +!!!warning + Note that in `FERNET` backend you loose `filtering` possibility altogether as part of the encrypted value is a timestamp + +```python +class Filter(ormar.Model): + class Meta(BaseMeta): + tablename = "filters" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, + encrypt_secret="asd123", + encrypt_backend=ormar.EncryptBackends.FERNET) + +await Filter(name='test1').save() +await Filter(name='test1').save() + +# values are properly encrypted and later decrypted +filters = await Filter.objects.all() +assert filters[0].name == filters[1].name == 'test1' + +# but you cannot filter at all since part of the fernet hash is a timestamp +# which means that even if you encrypt the same string 2 times it will be different +with pytest.raises(NoMatch): + await Filter.objects.get(name='test1') +``` + +## Custom Backends + +If you wish to support other type of encryption (i.e. AES) you can provide your own `EncryptionBackend`. + +To setup a backend all you need to do is subclass `ormar.fields.EncryptBackend` class and provide required backend. + +Sample dummy backend (that does nothing) can look like following: + +```python +class DummyBackend(ormar.fields.EncryptBackend): + def _initialize_backend(self, secret_key: bytes) -> None: + pass + + def encrypt(self, value: Any) -> str: + return value + + def decrypt(self, value: Any) -> str: + return value +``` + +To use this backend set `encrypt_backend` to `CUSTOM` and provide your backend as +argument by `encrypt_custom_backend`. + +```python +class Filter(ormar.Model): + class Meta(BaseMeta): + tablename = "filters" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, + encrypt_secret="secret123", + encrypt_backend=ormar.EncryptBackends.CUSTOM, + encrypt_custom_backend=DummyBackend + ) +``` \ No newline at end of file diff --git a/docs/releases.md b/docs/releases.md index 2e85e43..e1795c5 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,3 +1,21 @@ +# 0.9.8 + +## Features +* Add possibility to encrypt the selected field(s) in the database + * As minimum you need to provide `encrypt_secret` and `encrypt_backend` + * `encrypt_backend` can be one of the `ormar.EncryptBackends` enum (`NONE, FERNET, HASH, CUSTOM`) - default: `NONE` + * When custom backend is selected you need to provide your backend class that subclasses `ormar.fields.EncryptBackend` + * You cannot encrypt `primary_key` column and relation columns (FK and M2M). + * Provided are 2 backends: HASH and FERNET + * HASH is a one-way hash (like for password), never decrypted on retrieval + * FERNET is a two-way encrypt/decrypt backend + * Note that in FERNET backend you loose `filtering` possibility altogether as part of the encrypted value is a timestamp. + * Note that in HASH backend you can filter by full value but filters like `contain` will not work as comparison is make on encrypted values + * Note that adding `encrypt_backend` changes the database column type to `TEXT`, which needs to be reflected in db either by migration or manual change + +## Fixes +* (Advanced/ Internal) Restore custom sqlalchemy types (by `types.TypeDecorator` subclass) functionality that ceased to working so `process_result_value` was never called + # 0.9.7 ## Features diff --git a/mkdocs.yml b/mkdocs.yml index 8aba336..735c732 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,6 +12,7 @@ nav: - Fields: - Common parameters: fields/common-parameters.md - Fields types: fields/field-types.md + - Fields encryption: fields/encryption.md - Relations: - relations/index.md - relations/postponed-annotations.md diff --git a/ormar/fields/sqlalchemy_encrypted.py b/ormar/fields/sqlalchemy_encrypted.py index a198bd6..06483c6 100644 --- a/ormar/fields/sqlalchemy_encrypted.py +++ b/ormar/fields/sqlalchemy_encrypted.py @@ -31,7 +31,6 @@ class EncryptBackend(abc.ABC): digest = hashes.Hash(hashes.SHA256(), backend=default_backend()) digest.update(key) engine_key = digest.finalize() - self._initialize_backend(engine_key) @abc.abstractmethod @@ -165,7 +164,8 @@ class EncryptedString(types.TypeDecorator): if encoder: value = encoder(value) # type: ignore - return self.backend.encrypt(value) + encrypted_value = self.backend.encrypt(value) + return encrypted_value def process_result_value(self, value: Any, dialect: DefaultDialect) -> Any: if value is None: diff --git a/setup.py b/setup.py index a37f0b8..d30a8ac 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,8 @@ setup( "postgresql": ["asyncpg", "psycopg2"], "mysql": ["aiomysql", "pymysql"], "sqlite": ["aiosqlite"], - "orjson": ["orjson"] + "orjson": ["orjson"], + "crypto": ["cryptography"] }, classifiers=[ "Development Status :: 4 - Beta", diff --git a/tests/test_encrypted_columns.py b/tests/test_encrypted_columns.py index 323563f..3cfc345 100644 --- a/tests/test_encrypted_columns.py +++ b/tests/test_encrypted_columns.py @@ -1,5 +1,7 @@ # type: ignore +import base64 import decimal +import hashlib import uuid import datetime from typing import Any @@ -9,7 +11,7 @@ import pytest import sqlalchemy import ormar -from ormar import ModelDefinitionError +from ormar import ModelDefinitionError, NoMatch from ormar.fields.sqlalchemy_encrypted import EncryptedString from tests.settings import DATABASE_URL @@ -75,6 +77,26 @@ class Author(ormar.Model): ) +class Filter(ormar.Model): + class Meta(BaseMeta): + tablename = "filters" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, **default_fernet) + + +class Hash(ormar.Model): + class Meta(BaseMeta): + tablename = "hashes" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String( + max_length=128, + encrypt_secret="udxc32", + encrypt_backend=ormar.EncryptBackends.HASH, + ) + + @pytest.fixture(autouse=True, scope="module") def create_test_database(): engine = sqlalchemy.create_engine(DATABASE_URL) @@ -184,3 +206,33 @@ async def test_save_and_retrieve(): assert author.test_decimal == 3.5 assert author.test_decimal2 == 5.5 assert author.custom_backend == "test12" + + +@pytest.mark.asyncio +async def test_fernet_filters_nomatch(): + async with database: + await Filter(name="test1").save() + await Filter(name="test1").save() + + filters = await Filter.objects.all() + assert filters[0].name == filters[1].name == "test1" + + with pytest.raises(NoMatch): + await Filter.objects.get(name="test1") + + +@pytest.mark.asyncio +async def test_hash_filters_works(): + async with database: + await Hash(name="test1").save() + await Hash(name="test2").save() + + secret = hashlib.sha256("udxc32".encode()).digest() + secret = base64.urlsafe_b64encode(secret) + hashed_test1 = hashlib.sha512(secret + "test1".encode()).hexdigest() + + hash1 = await Hash.objects.get(name="test1") + assert hash1.name == hashed_test1 + + with pytest.raises(NoMatch): + await Filter.objects.get(name__icontains="test") From 50ddd1c2bb546c947a00d4a3c301554e27bfee38 Mon Sep 17 00:00:00 2001 From: collerek Date: Wed, 10 Mar 2021 14:01:16 +0100 Subject: [PATCH 13/13] add related model load tests --- README.md | 1 + docs/fields/encryption.md | 5 +-- docs/index.md | 1 + ormar/fields/sqlalchemy_encrypted.py | 2 +- tests/test_encrypted_columns.py | 47 +++++++++++++++++++++++----- 5 files changed, 45 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index fd526bd..22db942 100644 --- a/README.md +++ b/README.md @@ -472,6 +472,7 @@ Available Model Fields (with required args - optional ones in docs): * `Decimal(scale, precision)` * `UUID()` * `EnumField` - by passing `choices` to any other Field type +* `EncryptedString` - by passing `encrypt_secret` and `encrypt_backend` * `ForeignKey(to)` * `ManyToMany(to, through)` diff --git a/docs/fields/encryption.md b/docs/fields/encryption.md index e4a11c0..9cc06b0 100644 --- a/docs/fields/encryption.md +++ b/docs/fields/encryption.md @@ -55,7 +55,7 @@ Note that since this backend never decrypt the stored value it's only applicable !!!note Note that provided `encrypt_secret` is first hashed itself and used as salt, so in order to - compare to stored string you need to recreate this steps. + compare to stored string you need to recreate this steps. The `order_by` will not work as encrypted strings are compared so you cannot reliably order by. ```python class Hash(ormar.Model): @@ -101,7 +101,8 @@ Value is encrypted on way to database end decrypted on way out. Can be used on a as the returned value is parsed to corresponding python type. !!!warning - Note that in `FERNET` backend you loose `filtering` possibility altogether as part of the encrypted value is a timestamp + Note that in `FERNET` backend you loose `filter`ing possibility altogether as part of the encrypted value is a timestamp. + The same goes for `order_by` as encrypted strings are compared so you cannot reliably order by. ```python class Filter(ormar.Model): diff --git a/docs/index.md b/docs/index.md index fd526bd..22db942 100644 --- a/docs/index.md +++ b/docs/index.md @@ -472,6 +472,7 @@ Available Model Fields (with required args - optional ones in docs): * `Decimal(scale, precision)` * `UUID()` * `EnumField` - by passing `choices` to any other Field type +* `EncryptedString` - by passing `encrypt_secret` and `encrypt_backend` * `ForeignKey(to)` * `ManyToMany(to, through)` diff --git a/ormar/fields/sqlalchemy_encrypted.py b/ormar/fields/sqlalchemy_encrypted.py index 06483c6..1e6eda1 100644 --- a/ormar/fields/sqlalchemy_encrypted.py +++ b/ormar/fields/sqlalchemy_encrypted.py @@ -144,7 +144,7 @@ class EncryptedString(types.TypeDecorator): self.type_: Any = type_ def __repr__(self) -> str: # pragma: nocover - return f"TEXT()" + return "TEXT()" def load_dialect_impl(self, dialect: DefaultDialect) -> Any: return dialect.type_descriptor(types.TEXT()) diff --git a/tests/test_encrypted_columns.py b/tests/test_encrypted_columns.py index 3cfc345..1f0d7d7 100644 --- a/tests/test_encrypted_columns.py +++ b/tests/test_encrypted_columns.py @@ -77,14 +77,6 @@ class Author(ormar.Model): ) -class Filter(ormar.Model): - class Meta(BaseMeta): - tablename = "filters" - - id: int = ormar.Integer(primary_key=True) - name: str = ormar.String(max_length=100, **default_fernet) - - class Hash(ormar.Model): class Meta(BaseMeta): tablename = "hashes" @@ -97,6 +89,24 @@ class Hash(ormar.Model): ) +class Filter(ormar.Model): + class Meta(BaseMeta): + tablename = "filters" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, **default_fernet) + hash = ormar.ForeignKey(Hash) + + +class Report(ormar.Model): + class Meta(BaseMeta): + tablename = "reports" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + filters = ormar.ManyToMany(Filter) + + @pytest.fixture(autouse=True, scope="module") def create_test_database(): engine = sqlalchemy.create_engine(DATABASE_URL) @@ -236,3 +246,24 @@ async def test_hash_filters_works(): with pytest.raises(NoMatch): await Filter.objects.get(name__icontains="test") + + +@pytest.mark.asyncio +async def test_related_model_fields_properly_decrypted(): + async with database: + hash1 = await Hash(name="test1").save() + report = await Report.objects.create(name="Report1") + await report.filters.create(name="test1", hash=hash1) + await report.filters.create(name="test2") + + report2 = await Report.objects.select_related("filters").get() + assert report2.filters[0].name == "test1" + assert report2.filters[1].name == "test2" + + secret = hashlib.sha256("udxc32".encode()).digest() + secret = base64.urlsafe_b64encode(secret) + hashed_test1 = hashlib.sha512(secret + "test1".encode()).hexdigest() + + report2 = await Report.objects.select_related("filters__hash").get() + assert report2.filters[0].name == "test1" + assert report2.filters[0].hash.name == hashed_test1