diff --git a/.gitignore b/.gitignore index 6c5114b..2b9985d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ alembic.ini .idea .pytest_cache .mypy_cache -.coverage +*.coverage *.pyc *.log test.db @@ -14,3 +14,4 @@ site profile.py *.db *.db-journal +*coverage.xml \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..721821b --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +PIPENV_RUN := pipenv run +PG_DOCKERFILE_NAME := fastapi-users-test-mongo + +test_all: test_pg test_mysql test_sqlite + +test_pg: export DATABASE_URL=postgresql://username:password@localhost:5432/testsuite +test_pg: + docker-compose -f scripts/docker-compose.yml up -d postgres + bash scripts/test.sh -svv + docker-compose -f scripts/docker-compose.yml stop postgres + +test_mysql: export DATABASE_URL=mysql://username:password@127.0.0.1:3306/testsuite +test_mysql: + docker-compose -f "scripts/docker-compose.yml" up -d mysql + bash scripts/test.sh -svv + docker-compose -f scripts/docker-compose.yml stop mysql + +test_sqlite: + bash scripts/test.sh -svv \ No newline at end of file diff --git a/README.md b/README.md index fd526bd..22db942 100644 --- a/README.md +++ b/README.md @@ -472,6 +472,7 @@ Available Model Fields (with required args - optional ones in docs): * `Decimal(scale, precision)` * `UUID()` * `EnumField` - by passing `choices` to any other Field type +* `EncryptedString` - by passing `encrypt_secret` and `encrypt_backend` * `ForeignKey(to)` * `ManyToMany(to, through)` diff --git a/docs/fields/encryption.md b/docs/fields/encryption.md new file mode 100644 index 0000000..9cc06b0 --- /dev/null +++ b/docs/fields/encryption.md @@ -0,0 +1,164 @@ +# Encryption + +`ormar` provides you with a way to encrypt a field in the database only. +Provided encryption backends allow for both one-way encryption (`HASH` backend) as +well as both-way encryption/decryption (`FERNET` backend). + +!!!warning + Note that in order for encryption to work you need to install optional `cryptography` package. + + You can do it manually `pip install cryptography` or with ormar by `pip install ormar[crypto]` + +!!!warning + Note that adding `encrypt_backend` changes the database column type to `TEXT`, + which needs to be reflected in db either by migration (`alembic`) or manual change + +## Defining a field encryption + +To encrypt a field you need to pass at minimum `encrypt_secret` and `encrypt_backend` parameters. + +```python hl_lines="7-8" +class Filter(ormar.Model): + class Meta(BaseMeta): + tablename = "filters" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, + encrypt_secret="secret123", + encrypt_backend=ormar.EncryptBackends.FERNET) +``` + +!!!warning + You can encrypt all `Field` types apart from `primary_key` column and relation + columns (`ForeignKey` and `ManyToMany`). Check backends details for more information. + +## Available backends + +### HASH + +HASH is a one-way hash (like for password), never decrypted on retrieval + +To set it up pass appropriate backend value. + +```python +... # rest of model definition +password: str = ormar.String(max_length=128, + encrypt_secret="secret123", + encrypt_backend=ormar.EncryptBackends.HASH) +``` + +Note that since this backend never decrypt the stored value it's only applicable for +`String` fields. Used hash is a `sha512` hash, so the field length has to be >=128. + +!!!warning + Note that in `HASH` backend you can filter by full value but filters like `contain` will not work as comparison is make on encrypted values + +!!!note + Note that provided `encrypt_secret` is first hashed itself and used as salt, so in order to + compare to stored string you need to recreate this steps. The `order_by` will not work as encrypted strings are compared so you cannot reliably order by. + +```python +class Hash(ormar.Model): + class Meta(BaseMeta): + tablename = "hashes" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=128, + encrypt_secret="udxc32", + encrypt_backend=ormar.EncryptBackends.HASH) + + +await Hash(name='test1').save() + +# note the steps to recreate the stored value +# you can use also cryptography package instead of hashlib +secret = hashlib.sha256("udxc32".encode()).digest() +secret = base64.urlsafe_b64encode(secret) +hashed_test1 = hashlib.sha512(secret + 'test1'.encode()).hexdigest() + +# full value comparison works +hash1 = await Hash.objects.get(name='test1') +assert hash1.name == hashed_test1 + +# but partial comparison does not (hashed strings are compared) +with pytest.raises(NoMatch): + await Filter.objects.get(name__icontains='test') +``` + +### FERNET + +FERNET is a two-way encrypt/decrypt backend + +To set it up pass appropriate backend value. + +```python +... # rest of model definition +year: int = ormar.Integer(encrypt_secret="secret123", + encrypt_backend=ormar.EncryptBackends.FERNET) +``` + +Value is encrypted on way to database end decrypted on way out. Can be used on all types, +as the returned value is parsed to corresponding python type. + +!!!warning + Note that in `FERNET` backend you loose `filter`ing possibility altogether as part of the encrypted value is a timestamp. + The same goes for `order_by` as encrypted strings are compared so you cannot reliably order by. + +```python +class Filter(ormar.Model): + class Meta(BaseMeta): + tablename = "filters" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, + encrypt_secret="asd123", + encrypt_backend=ormar.EncryptBackends.FERNET) + +await Filter(name='test1').save() +await Filter(name='test1').save() + +# values are properly encrypted and later decrypted +filters = await Filter.objects.all() +assert filters[0].name == filters[1].name == 'test1' + +# but you cannot filter at all since part of the fernet hash is a timestamp +# which means that even if you encrypt the same string 2 times it will be different +with pytest.raises(NoMatch): + await Filter.objects.get(name='test1') +``` + +## Custom Backends + +If you wish to support other type of encryption (i.e. AES) you can provide your own `EncryptionBackend`. + +To setup a backend all you need to do is subclass `ormar.fields.EncryptBackend` class and provide required backend. + +Sample dummy backend (that does nothing) can look like following: + +```python +class DummyBackend(ormar.fields.EncryptBackend): + def _initialize_backend(self, secret_key: bytes) -> None: + pass + + def encrypt(self, value: Any) -> str: + return value + + def decrypt(self, value: Any) -> str: + return value +``` + +To use this backend set `encrypt_backend` to `CUSTOM` and provide your backend as +argument by `encrypt_custom_backend`. + +```python +class Filter(ormar.Model): + class Meta(BaseMeta): + tablename = "filters" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, + encrypt_secret="secret123", + encrypt_backend=ormar.EncryptBackends.CUSTOM, + encrypt_custom_backend=DummyBackend + ) +``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index fd526bd..22db942 100644 --- a/docs/index.md +++ b/docs/index.md @@ -472,6 +472,7 @@ Available Model Fields (with required args - optional ones in docs): * `Decimal(scale, precision)` * `UUID()` * `EnumField` - by passing `choices` to any other Field type +* `EncryptedString` - by passing `encrypt_secret` and `encrypt_backend` * `ForeignKey(to)` * `ManyToMany(to, through)` diff --git a/docs/releases.md b/docs/releases.md index 2e85e43..e1795c5 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,3 +1,21 @@ +# 0.9.8 + +## Features +* Add possibility to encrypt the selected field(s) in the database + * As minimum you need to provide `encrypt_secret` and `encrypt_backend` + * `encrypt_backend` can be one of the `ormar.EncryptBackends` enum (`NONE, FERNET, HASH, CUSTOM`) - default: `NONE` + * When custom backend is selected you need to provide your backend class that subclasses `ormar.fields.EncryptBackend` + * You cannot encrypt `primary_key` column and relation columns (FK and M2M). + * Provided are 2 backends: HASH and FERNET + * HASH is a one-way hash (like for password), never decrypted on retrieval + * FERNET is a two-way encrypt/decrypt backend + * Note that in FERNET backend you loose `filtering` possibility altogether as part of the encrypted value is a timestamp. + * Note that in HASH backend you can filter by full value but filters like `contain` will not work as comparison is make on encrypted values + * Note that adding `encrypt_backend` changes the database column type to `TEXT`, which needs to be reflected in db either by migration or manual change + +## Fixes +* (Advanced/ Internal) Restore custom sqlalchemy types (by `types.TypeDecorator` subclass) functionality that ceased to working so `process_result_value` was never called + # 0.9.7 ## Features diff --git a/mkdocs.yml b/mkdocs.yml index 8aba336..735c732 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,6 +12,7 @@ nav: - Fields: - Common parameters: fields/common-parameters.md - Fields types: fields/field-types.md + - Fields encryption: fields/encryption.md - Relations: - relations/index.md - relations/postponed-annotations.md diff --git a/ormar/__init__.py b/ormar/__init__.py index 98f7e78..9193543 100644 --- a/ormar/__init__.py +++ b/ormar/__init__.py @@ -38,9 +38,12 @@ from ormar.fields import ( BaseField, BigInteger, Boolean, + DECODERS_MAP, Date, DateTime, Decimal, + ENCODERS_MAP, + EncryptBackends, Float, ForeignKey, ForeignKeyField, @@ -68,7 +71,7 @@ class UndefinedType: # pragma no cover Undefined = UndefinedType() -__version__ = "0.9.7" +__version__ = "0.9.8" __all__ = [ "Integer", "BigInteger", @@ -110,4 +113,7 @@ __all__ = [ "ExcludableItems", "and_", "or_", + "EncryptBackends", + "ENCODERS_MAP", + "DECODERS_MAP", ] diff --git a/ormar/fields/__init__.py b/ormar/fields/__init__.py index c5f61d4..e0cb3b0 100644 --- a/ormar/fields/__init__.py +++ b/ormar/fields/__init__.py @@ -21,6 +21,8 @@ from ormar.fields.model_fields import ( Time, UUID, ) +from ormar.fields.parsers import DECODERS_MAP, ENCODERS_MAP +from ormar.fields.sqlalchemy_encrypted import EncryptBackend, EncryptBackends from ormar.fields.through_field import Through, ThroughField __all__ = [ @@ -44,4 +46,8 @@ __all__ = [ "ForeignKeyField", "ThroughField", "Through", + "EncryptBackends", + "EncryptBackend", + "DECODERS_MAP", + "ENCODERS_MAP", ] diff --git a/ormar/fields/base.py b/ormar/fields/base.py index 1fada90..c58348c 100644 --- a/ormar/fields/base.py +++ b/ormar/fields/base.py @@ -5,6 +5,12 @@ from pydantic import Field, Json, typing from pydantic.fields import FieldInfo, Required, Undefined import ormar # noqa I101 +from ormar import ModelDefinitionError +from ormar.fields.sqlalchemy_encrypted import ( + EncryptBackend, + EncryptBackends, + EncryptedString, +) if TYPE_CHECKING: # pragma no cover from ormar.models import Model @@ -49,6 +55,10 @@ class BaseField(FieldInfo): self_reference: bool = False self_reference_primary: Optional[str] = None + encrypt_secret: str + encrypt_backend: EncryptBackends = EncryptBackends.NONE + encrypt_custom_backend: Optional[Type[EncryptBackend]] = None + default: Any server_default: Any @@ -256,12 +266,45 @@ class BaseField(FieldInfo): :return: actual definition of the database column as sqlalchemy requires. :rtype: sqlalchemy.Column """ + if cls.encrypt_backend == EncryptBackends.NONE: + column = sqlalchemy.Column( + cls.alias or name, + cls.column_type, + *cls.construct_constraints(), + primary_key=cls.primary_key, + nullable=cls.nullable and not cls.primary_key, + index=cls.index, + unique=cls.unique, + default=cls.default, + server_default=cls.server_default, + ) + else: + column = cls._get_encrypted_column(name=name) + return column + + @classmethod + def _get_encrypted_column(cls, name: str) -> sqlalchemy.Column: + """ + Returns EncryptedString column type instead of actual column. + + :param name: column name + :type name: str + :return: newly defined column + :rtype: sqlalchemy.Column + """ + if cls.primary_key or cls.is_relation: + raise ModelDefinitionError( + "Primary key field and relations fields" "cannot be encrypted!" + ) column = sqlalchemy.Column( cls.alias or name, - cls.column_type, - *cls.construct_constraints(), - primary_key=cls.primary_key, - nullable=cls.nullable and not cls.primary_key, + EncryptedString( + _field_type=cls, + encrypt_secret=cls.encrypt_secret, + encrypt_backend=cls.encrypt_backend, + encrypt_custom_backend=cls.encrypt_custom_backend, + ), + nullable=cls.nullable, index=cls.index, unique=cls.unique, default=cls.default, diff --git a/ormar/fields/foreign_key.py b/ormar/fields/foreign_key.py index 7f1a500..e981d9e 100644 --- a/ormar/fields/foreign_key.py +++ b/ormar/fields/foreign_key.py @@ -184,10 +184,23 @@ def ForeignKey( # noqa CFQ002 owner = kwargs.pop("owner", None) self_reference = kwargs.pop("self_reference", False) + default = kwargs.pop("default", None) - if default is not None: + encrypt_secret = kwargs.pop("encrypt_secret", None) + encrypt_backend = kwargs.pop("encrypt_backend", None) + encrypt_custom_backend = kwargs.pop("encrypt_custom_backend", None) + + not_supported = [ + default, + encrypt_secret, + encrypt_backend, + encrypt_custom_backend, + ] + if any(x is not None for x in not_supported): raise ModelDefinitionError( - "Argument 'default' is not supported " "on relation fields!" + f"Argument {next((x for x in not_supported if x is not None))} " + f"is not supported " + "on relation fields!" ) if to.__class__ == ForwardRef: @@ -386,7 +399,7 @@ class ForeignKeyField(BaseField): :return: (if needed) registered Model :rtype: Model """ - if cls.to.pk_type() == uuid.UUID and isinstance(value, str): + if cls.to.pk_type() == uuid.UUID and isinstance(value, str): # pragma: nocover value = uuid.UUID(value) if not isinstance(value, cls.to.pk_type()): raise RelationshipInstanceError( diff --git a/ormar/fields/many_to_many.py b/ormar/fields/many_to_many.py index db763e3..2382fa5 100644 --- a/ormar/fields/many_to_many.py +++ b/ormar/fields/many_to_many.py @@ -97,9 +97,21 @@ def ManyToMany( forbid_through_relations(cast(Type["Model"], through)) default = kwargs.pop("default", None) - if default is not None: + encrypt_secret = kwargs.pop("encrypt_secret", None) + encrypt_backend = kwargs.pop("encrypt_backend", None) + encrypt_custom_backend = kwargs.pop("encrypt_custom_backend", None) + + not_supported = [ + default, + encrypt_secret, + encrypt_backend, + encrypt_custom_backend, + ] + if any(x is not None for x in not_supported): raise ModelDefinitionError( - "Argument 'default' is not supported " "on relation fields!" + f"Argument {next((x for x in not_supported if x is not None))} " + f"is not supported " + "on relation fields!" ) if to.__class__ == ForwardRef: diff --git a/ormar/fields/model_fields.py b/ormar/fields/model_fields.py index 2222b78..9cfd4f4 100644 --- a/ormar/fields/model_fields.py +++ b/ormar/fields/model_fields.py @@ -9,6 +9,7 @@ import sqlalchemy from ormar import ModelDefinitionError # noqa I101 from ormar.fields import sqlalchemy_uuid from ormar.fields.base import BaseField # noqa I101 +from ormar.fields.sqlalchemy_encrypted import EncryptBackends def is_field_nullable( @@ -73,6 +74,11 @@ class ModelFieldFactory: primary_key = kwargs.pop("primary_key", False) autoincrement = kwargs.pop("autoincrement", False) + encrypt_secret = kwargs.pop("encrypt_secret", None) + encrypt_backend = kwargs.pop("encrypt_backend", EncryptBackends.NONE) + encrypt_custom_backend = kwargs.pop("encrypt_custom_backend", None) + encrypt_max_length = kwargs.pop("encrypt_max_length", 5000) + namespace = dict( __type__=cls._type, alias=kwargs.pop("name", None), @@ -88,6 +94,10 @@ class ModelFieldFactory: autoincrement=autoincrement, column_type=cls.get_column_type(**kwargs), choices=set(kwargs.pop("choices", [])), + encrypt_secret=encrypt_secret, + encrypt_backend=encrypt_backend, + encrypt_custom_backend=encrypt_custom_backend, + encrypt_max_length=encrypt_max_length, **kwargs ) return type(cls.__name__, cls._bases, namespace) diff --git a/ormar/fields/parsers.py b/ormar/fields/parsers.py new file mode 100644 index 0000000..e0f1a53 --- /dev/null +++ b/ormar/fields/parsers.py @@ -0,0 +1,44 @@ +import datetime +import decimal +from typing import Any + +import pydantic +from pydantic.datetime_parse import parse_date, parse_datetime, parse_time + +try: + import orjson as json +except ImportError: # pragma: no cover + import json # type: ignore + + +def parse_bool(value: str) -> bool: + return value == "true" + + +def encode_bool(value: bool) -> str: + return "true" if value else "false" + + +def encode_json(value: Any) -> str: + value = json.dumps(value) if not isinstance(value, str) else value + value = value.decode("utf-8") if isinstance(value, bytes) else value + return value + + +ENCODERS_MAP = { + bool: encode_bool, + datetime.datetime: lambda x: x.isoformat(), + datetime.date: lambda x: x.isoformat(), + datetime.time: lambda x: x.isoformat(), + pydantic.Json: encode_json, + decimal.Decimal: float, +} + +DECODERS_MAP = { + bool: parse_bool, + datetime.datetime: parse_datetime, + datetime.date: parse_date, + datetime.time: parse_time, + pydantic.Json: json.loads, + decimal.Decimal: decimal.Decimal, +} diff --git a/ormar/fields/sqlalchemy_encrypted.py b/ormar/fields/sqlalchemy_encrypted.py new file mode 100644 index 0000000..1e6eda1 --- /dev/null +++ b/ormar/fields/sqlalchemy_encrypted.py @@ -0,0 +1,182 @@ +# inspired by sqlalchemy-utils (https://github.com/kvesteri/sqlalchemy-utils) +import abc +import base64 +from enum import Enum +from typing import Any, Callable, Optional, TYPE_CHECKING, Type, Union + +import sqlalchemy.types as types +from pydantic.utils import lenient_issubclass +from sqlalchemy.engine.default import DefaultDialect + +import ormar # noqa: I100, I202 +from ormar import ModelDefinitionError # noqa: I202, I100 + +cryptography = None +try: # pragma: nocover + import cryptography # type: ignore + from cryptography.fernet import Fernet + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import hashes +except ImportError: # pragma: nocover + pass + +if TYPE_CHECKING: # pragma: nocover + from ormar import BaseField + + +class EncryptBackend(abc.ABC): + def _refresh(self, key: Union[str, bytes]) -> None: + if isinstance(key, str): + key = key.encode() + digest = hashes.Hash(hashes.SHA256(), backend=default_backend()) + digest.update(key) + engine_key = digest.finalize() + self._initialize_backend(engine_key) + + @abc.abstractmethod + def _initialize_backend(self, secret_key: bytes) -> None: # pragma: nocover + pass + + @abc.abstractmethod + def encrypt(self, value: Any) -> str: # pragma: nocover + pass + + @abc.abstractmethod + def decrypt(self, value: Any) -> str: # pragma: nocover + pass + + +class HashBackend(EncryptBackend): + """ + One-way hashing - in example for passwords, no way to decrypt the value! + """ + + def _initialize_backend(self, secret_key: bytes) -> None: + self.secret_key = base64.urlsafe_b64encode(secret_key) + + def encrypt(self, value: Any) -> str: + if not isinstance(value, str): # pragma: nocover + value = repr(value) + value = value.encode() + digest = hashes.Hash(hashes.SHA512(), backend=default_backend()) + digest.update(self.secret_key) + digest.update(value) + hashed_value = digest.finalize() + return hashed_value.hex() + + def decrypt(self, value: Any) -> str: + if not isinstance(value, str): # pragma: nocover + value = str(value) + return value + + +class FernetBackend(EncryptBackend): + """ + Two-way encryption, data stored in db are encrypted but decrypted during query. + """ + + def _initialize_backend(self, secret_key: bytes) -> None: + self.secret_key = base64.urlsafe_b64encode(secret_key) + self.fernet = Fernet(self.secret_key) + + def encrypt(self, value: Any) -> str: + if not isinstance(value, str): + value = repr(value) + value = value.encode() + encrypted = self.fernet.encrypt(value) + return encrypted.decode("utf-8") + + def decrypt(self, value: Any) -> str: + if not isinstance(value, str): # pragma: nocover + value = str(value) + decrypted: Union[str, bytes] = self.fernet.decrypt(value.encode()) + if not isinstance(decrypted, str): + decrypted = decrypted.decode("utf-8") + return decrypted + + +class EncryptBackends(Enum): + NONE = 0 + FERNET = 1 + HASH = 2 + CUSTOM = 3 + + +BACKENDS_MAP = { + EncryptBackends.FERNET: FernetBackend, + EncryptBackends.HASH: HashBackend, +} + + +class EncryptedString(types.TypeDecorator): + """ + Used to store encrypted values in a database + """ + + impl = types.TypeEngine + + def __init__( + self, + encrypt_secret: Union[str, Callable], + encrypt_backend: EncryptBackends = EncryptBackends.FERNET, + encrypt_custom_backend: Type[EncryptBackend] = None, + **kwargs: Any, + ) -> None: + _field_type = kwargs.pop("_field_type") + super().__init__() + if not cryptography: # pragma: nocover + raise ModelDefinitionError( + "In order to encrypt a column 'cryptography' is required!" + ) + backend = BACKENDS_MAP.get(encrypt_backend, encrypt_custom_backend) + if not backend or not lenient_issubclass(backend, EncryptBackend): + raise ModelDefinitionError("Wrong or no encrypt backend provided!") + + self.backend: EncryptBackend = backend() + self._field_type: Type["BaseField"] = _field_type + self._underlying_type: Any = _field_type.column_type + self._key: Union[str, Callable] = encrypt_secret + type_ = self._field_type.__type__ + if type_ is None: # pragma: nocover + raise ModelDefinitionError( + f"Improperly configured field " f"{self._field_type.name}" + ) + self.type_: Any = type_ + + def __repr__(self) -> str: # pragma: nocover + return "TEXT()" + + def load_dialect_impl(self, dialect: DefaultDialect) -> Any: + return dialect.type_descriptor(types.TEXT()) + + def _refresh(self) -> None: + key = self._key() if callable(self._key) else self._key + self.backend._refresh(key) + + def process_bind_param(self, value: Any, dialect: DefaultDialect) -> Optional[str]: + if value is None: + return value + self._refresh() + try: + value = self._underlying_type.process_bind_param(value, dialect) + except AttributeError: + encoder = ormar.ENCODERS_MAP.get(self.type_, None) + if encoder: + value = encoder(value) # type: ignore + + encrypted_value = self.backend.encrypt(value) + return encrypted_value + + def process_result_value(self, value: Any, dialect: DefaultDialect) -> Any: + if value is None: + return value + self._refresh() + decrypted_value = self.backend.decrypt(value) + try: + return self._underlying_type.process_result_value(decrypted_value, dialect) + except AttributeError: + decoder = ormar.DECODERS_MAP.get(self.type_, None) + if decoder: + return decoder(decrypted_value) # type: ignore + + return self._field_type.__type__(decrypted_value) # type: ignore diff --git a/ormar/fields/sqlalchemy_uuid.py b/ormar/fields/sqlalchemy_uuid.py index 2a6bfd7..826c0c8 100644 --- a/ormar/fields/sqlalchemy_uuid.py +++ b/ormar/fields/sqlalchemy_uuid.py @@ -1,12 +1,12 @@ import uuid -from typing import Any, Optional, Union +from typing import Any, Optional from sqlalchemy import CHAR from sqlalchemy.engine.default import DefaultDialect from sqlalchemy.types import TypeDecorator -class UUID(TypeDecorator): # pragma nocover +class UUID(TypeDecorator): """ Platform-independent GUID type. Uses CHAR(36) if in a string mode, otherwise uses CHAR(32), to store UUID. @@ -20,31 +20,11 @@ class UUID(TypeDecorator): # pragma nocover super().__init__(*args, **kwargs) self.uuid_format = uuid_format - def __repr__(self) -> str: + def __repr__(self) -> str: # pragma: nocover if self.uuid_format == "string": return "CHAR(36)" return "CHAR(32)" - def _cast_to_uuid(self, value: Union[str, int, bytes]) -> uuid.UUID: - """ - Parses given value into uuid.UUID field. - - :param value: value to be parsed - :type value: Union[str, int, bytes] - :return: initialized uuid - :rtype: uuid.UUID - """ - if not isinstance(value, uuid.UUID): - if isinstance(value, bytes): - ret_value = uuid.UUID(bytes=value) - elif isinstance(value, int): - ret_value = uuid.UUID(int=value) - elif isinstance(value, str): - ret_value = uuid.UUID(value) - else: - ret_value = value - return ret_value - def load_dialect_impl(self, dialect: DefaultDialect) -> Any: return ( dialect.type_descriptor(CHAR(36)) @@ -53,12 +33,10 @@ class UUID(TypeDecorator): # pragma nocover ) def process_bind_param( - self, value: Union[str, int, bytes, uuid.UUID, None], dialect: DefaultDialect + self, value: uuid.UUID, dialect: DefaultDialect ) -> Optional[str]: if value is None: return value - if not isinstance(value, uuid.UUID): - value = self._cast_to_uuid(value) return str(value) if self.uuid_format == "string" else "%.32x" % value.int def process_result_value( @@ -68,4 +46,4 @@ class UUID(TypeDecorator): # pragma nocover return value if not isinstance(value, uuid.UUID): return uuid.UUID(value) - return value + return value # pragma: nocover diff --git a/ormar/models/helpers/sqlalchemy.py b/ormar/models/helpers/sqlalchemy.py index e40239d..472bbad 100644 --- a/ormar/models/helpers/sqlalchemy.py +++ b/ormar/models/helpers/sqlalchemy.py @@ -289,7 +289,7 @@ def populate_meta_sqlalchemy_table_if_required(meta: "ModelMeta") -> None: f'{"_".join([str(col) for col in constraint._pending_colargs])}' ) table = sqlalchemy.Table( - meta.tablename, meta.metadata, *meta.columns, *meta.constraints, + meta.tablename, meta.metadata, *meta.columns, *meta.constraints ) meta.table = table diff --git a/ormar/models/helpers/validation.py b/ormar/models/helpers/validation.py index 582c3fa..bc87235 100644 --- a/ormar/models/helpers/validation.py +++ b/ormar/models/helpers/validation.py @@ -53,6 +53,7 @@ def convert_choices_if_needed( # noqa: CCR001 :return: value, choices list :rtype: Tuple[Any, List] """ + # TODO use same maps as with EncryptedString choices = [o.value if isinstance(o, Enum) else o for o in field.choices] if field.__type__ in [datetime.datetime, datetime.date, datetime.time]: diff --git a/ormar/models/model_row.py b/ormar/models/model_row.py index 2cd488a..d06d0d9 100644 --- a/ormar/models/model_row.py +++ b/ormar/models/model_row.py @@ -369,9 +369,6 @@ class ModelRow(NewBaseModel): and values are database values :rtype: Dict """ - # databases does not keep aliases in Record for postgres, change to raw row - source = row._row if cls.db_backend_name() == "postgresql" else row - selected_columns = cls.own_table_columns( model=cls, excludable=excludable, alias=table_prefix, use_alias=False, ) @@ -381,6 +378,6 @@ class ModelRow(NewBaseModel): alias = cls.get_column_name_from_alias(column.name) if alias not in item and alias in selected_columns: prefixed_name = f"{column_prefix}{column.name}" - item[alias] = source[prefixed_name] + item[alias] = row[prefixed_name] return item diff --git a/ormar/queryset/join.py b/ormar/queryset/join.py index b9e71df..e710aef 100644 --- a/ormar/queryset/join.py +++ b/ormar/queryset/join.py @@ -88,13 +88,13 @@ class SqlJoin: return self.main_model.Meta.alias_manager @property - def to_table(self) -> str: + def to_table(self) -> sqlalchemy.Table: """ Shortcut to table name of the next model :return: name of the target table :rtype: str """ - return self.next_model.Meta.table.name + return self.next_model.Meta.table def _on_clause( self, previous_alias: str, from_clause: str, to_clause: str, @@ -282,7 +282,7 @@ class SqlJoin: on_clause = self._on_clause( previous_alias=self.own_alias, from_clause=f"{self.target_field.owner.Meta.tablename}.{from_key}", - to_clause=f"{self.to_table}.{to_key}", + to_clause=f"{self.to_table.name}.{to_key}", ) target_table = self.alias_manager.prefixed_table_name( self.next_alias, self.to_table @@ -301,7 +301,7 @@ class SqlJoin: ) self.columns.extend( self.alias_manager.prefixed_columns( - self.next_alias, self.next_model.Meta.table, self_related_fields + self.next_alias, target_table, self_related_fields ) ) self.used_aliases.append(self.next_alias) diff --git a/ormar/relations/alias_manager.py b/ormar/relations/alias_manager.py index 815a4dc..2ec6159 100644 --- a/ormar/relations/alias_manager.py +++ b/ormar/relations/alias_manager.py @@ -67,24 +67,21 @@ class AliasManager: if not fields else [col for col in table.columns if col.name in fields] ) - return [ - text(f"{alias}{table.name}.{column.name} as {alias}{column.name}") - for column in all_columns - ] + return [column.label(f"{alias}{column.name}") for column in all_columns] @staticmethod - def prefixed_table_name(alias: str, name: str) -> text: + def prefixed_table_name(alias: str, table: sqlalchemy.Table) -> text: """ Creates text clause with table name with aliased name. :param alias: alias of given table :type alias: str - :param name: table name - :type name: str + :param table: table + :type table: sqlalchemy.Table :return: sqlalchemy text clause as "table_name aliased_name" :rtype: sqlalchemy text clause """ - return text(f"{name} {alias}_{name}") + return table.alias(f"{alias}_{table.name}") def add_relation_type( self, source_model: Type["Model"], relation_name: str, reverse_name: str = None, diff --git a/requirements.txt b/requirements.txt index f47a05d..4d7610c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ pydantic sqlalchemy typing_extensions orjson +cryptography # Async database drivers aiomysql diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml new file mode 100644 index 0000000..c6ad36e --- /dev/null +++ b/scripts/docker-compose.yml @@ -0,0 +1,20 @@ +version: '2.1' +services: + postgres: + image: postgres:10.8 + environment: + POSTGRES_USER: username + POSTGRES_PASSWORD: password + POSTGRES_DB: testsuite + ports: + - 5432:5432 + + mysql: + image: mysql:5.7 + environment: + MYSQL_USER: username + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: password + MYSQL_DATABASE: testsuite + ports: + - 3306:3306 \ No newline at end of file diff --git a/setup.py b/setup.py index a37f0b8..d30a8ac 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,8 @@ setup( "postgresql": ["asyncpg", "psycopg2"], "mysql": ["aiomysql", "pymysql"], "sqlite": ["aiosqlite"], - "orjson": ["orjson"] + "orjson": ["orjson"], + "crypto": ["cryptography"] }, classifiers=[ "Development Status :: 4 - Beta", diff --git a/tests/settings.py b/tests/settings.py index 6d89f4e..be1bed2 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -6,3 +6,4 @@ DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///test.db") database_url = databases.DatabaseURL(DATABASE_URL) if database_url.scheme == "postgresql+aiopg": # pragma no cover DATABASE_URL = str(database_url.replace(driver=None)) +print("USED DB:", DATABASE_URL) diff --git a/tests/test_encrypted_columns.py b/tests/test_encrypted_columns.py new file mode 100644 index 0000000..1f0d7d7 --- /dev/null +++ b/tests/test_encrypted_columns.py @@ -0,0 +1,269 @@ +# type: ignore +import base64 +import decimal +import hashlib +import uuid +import datetime +from typing import Any + +import databases +import pytest +import sqlalchemy + +import ormar +from ormar import ModelDefinitionError, NoMatch +from ormar.fields.sqlalchemy_encrypted import EncryptedString +from tests.settings import DATABASE_URL + +database = databases.Database(DATABASE_URL) +metadata = sqlalchemy.MetaData() + + +class BaseMeta(ormar.ModelMeta): + metadata = metadata + database = database + + +default_fernet = dict( + encrypt_secret="asd123", encrypt_backend=ormar.EncryptBackends.FERNET, +) + + +class DummyBackend(ormar.fields.EncryptBackend): + def _initialize_backend(self, secret_key: bytes) -> None: + pass + + def encrypt(self, value: Any) -> str: + return value + + def decrypt(self, value: Any) -> str: + return value + + +class Author(ormar.Model): + class Meta(BaseMeta): + tablename = "authors" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, **default_fernet) + uuid_test = ormar.UUID(default=uuid.uuid4, uuid_format="string") + uuid_test2 = ormar.UUID(nullable=True, uuid_format="string") + password: str = ormar.String( + max_length=128, + encrypt_secret="udxc32", + encrypt_backend=ormar.EncryptBackends.HASH, + ) + birth_year: int = ormar.Integer( + nullable=True, + encrypt_secret="secure89key%^&psdijfipew", + encrypt_backend=ormar.EncryptBackends.FERNET, + ) + test_text: str = ormar.Text(default="", **default_fernet) + test_bool: bool = ormar.Boolean(nullable=False, **default_fernet) + test_float: float = ormar.Float(**default_fernet) + test_float2: float = ormar.Float(nullable=True, **default_fernet) + test_datetime = ormar.DateTime(default=datetime.datetime.now, **default_fernet) + test_date = ormar.Date(default=datetime.date.today, **default_fernet) + test_time = ormar.Time(default=datetime.time, **default_fernet) + test_json = ormar.JSON(default={}, **default_fernet) + test_bigint: int = ormar.BigInteger(default=0, **default_fernet) + test_decimal = ormar.Decimal(scale=2, precision=10, **default_fernet) + test_decimal2 = ormar.Decimal(max_digits=10, decimal_places=2, **default_fernet) + custom_backend: str = ormar.String( + max_length=200, + encrypt_secret="asda8", + encrypt_backend=ormar.EncryptBackends.CUSTOM, + encrypt_custom_backend=DummyBackend, + ) + + +class Hash(ormar.Model): + class Meta(BaseMeta): + tablename = "hashes" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String( + max_length=128, + encrypt_secret="udxc32", + encrypt_backend=ormar.EncryptBackends.HASH, + ) + + +class Filter(ormar.Model): + class Meta(BaseMeta): + tablename = "filters" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100, **default_fernet) + hash = ormar.ForeignKey(Hash) + + +class Report(ormar.Model): + class Meta(BaseMeta): + tablename = "reports" + + id: int = ormar.Integer(primary_key=True) + name: str = ormar.String(max_length=100) + filters = ormar.ManyToMany(Filter) + + +@pytest.fixture(autouse=True, scope="module") +def create_test_database(): + engine = sqlalchemy.create_engine(DATABASE_URL) + metadata.drop_all(engine) + metadata.create_all(engine) + yield + metadata.drop_all(engine) + + +def test_error_on_encrypted_pk(): + with pytest.raises(ModelDefinitionError): + + class Wrong(ormar.Model): + class Meta(BaseMeta): + tablename = "wrongs" + + id: int = ormar.Integer( + primary_key=True, + encrypt_secret="asd123", + encrypt_backend=ormar.EncryptBackends.FERNET, + ) + + +def test_error_on_encrypted_relation(): + with pytest.raises(ModelDefinitionError): + + class Wrong2(ormar.Model): + class Meta(BaseMeta): + tablename = "wrongs2" + + id: int = ormar.Integer(primary_key=True) + author = ormar.ForeignKey( + Author, + encrypt_secret="asd123", + encrypt_backend=ormar.EncryptBackends.FERNET, + ) + + +def test_error_on_encrypted_m2m_relation(): + with pytest.raises(ModelDefinitionError): + + class Wrong3(ormar.Model): + class Meta(BaseMeta): + tablename = "wrongs3" + + id: int = ormar.Integer(primary_key=True) + author = ormar.ManyToMany( + Author, + encrypt_secret="asd123", + encrypt_backend=ormar.EncryptBackends.FERNET, + ) + + +def test_wrong_backend(): + with pytest.raises(ModelDefinitionError): + + class Wrong3(ormar.Model): + class Meta(BaseMeta): + tablename = "wrongs3" + + id: int = ormar.Integer(primary_key=True) + author = ormar.Integer( + encrypt_secret="asd123", + encrypt_backend=ormar.EncryptBackends.CUSTOM, + encrypt_custom_backend="aa", + ) + + +def test_db_structure(): + assert Author.Meta.table.c.get("name").type.__class__ == EncryptedString + + +@pytest.mark.asyncio +async def test_save_and_retrieve(): + async with database: + test_uuid = uuid.uuid4() + await Author( + name="Test", + birth_year=1988, + password="test123", + uuid_test=test_uuid, + test_float=1.2, + test_bool=True, + test_decimal=decimal.Decimal(3.5), + test_decimal2=decimal.Decimal(5.5), + test_json=dict(aa=12), + custom_backend="test12", + ).save() + author = await Author.objects.get() + + assert author.name == "Test" + assert author.birth_year == 1988 + password = ( + "03e4a4d513e99cb3fe4ee3db282c053daa3f3572b849c3868939a306944ad5c08" + "22b50d4886e10f4cd418c3f2df3ceb02e2e7ac6e920ae0c90f2dedfc8fa16e2" + ) + assert author.password == password + assert author.uuid_test == test_uuid + assert author.uuid_test2 is None + assert author.test_datetime.date() == datetime.date.today() + assert author.test_date == datetime.date.today() + assert author.test_text == "" + assert author.test_float == 1.2 + assert author.test_float2 is None + assert author.test_bigint == 0 + assert author.test_json == {"aa": 12} + assert author.test_decimal == 3.5 + assert author.test_decimal2 == 5.5 + assert author.custom_backend == "test12" + + +@pytest.mark.asyncio +async def test_fernet_filters_nomatch(): + async with database: + await Filter(name="test1").save() + await Filter(name="test1").save() + + filters = await Filter.objects.all() + assert filters[0].name == filters[1].name == "test1" + + with pytest.raises(NoMatch): + await Filter.objects.get(name="test1") + + +@pytest.mark.asyncio +async def test_hash_filters_works(): + async with database: + await Hash(name="test1").save() + await Hash(name="test2").save() + + secret = hashlib.sha256("udxc32".encode()).digest() + secret = base64.urlsafe_b64encode(secret) + hashed_test1 = hashlib.sha512(secret + "test1".encode()).hexdigest() + + hash1 = await Hash.objects.get(name="test1") + assert hash1.name == hashed_test1 + + with pytest.raises(NoMatch): + await Filter.objects.get(name__icontains="test") + + +@pytest.mark.asyncio +async def test_related_model_fields_properly_decrypted(): + async with database: + hash1 = await Hash(name="test1").save() + report = await Report.objects.create(name="Report1") + await report.filters.create(name="test1", hash=hash1) + await report.filters.create(name="test2") + + report2 = await Report.objects.select_related("filters").get() + assert report2.filters[0].name == "test1" + assert report2.filters[1].name == "test2" + + secret = hashlib.sha256("udxc32".encode()).digest() + secret = base64.urlsafe_b64encode(secret) + hashed_test1 = hashlib.sha512(secret + "test1".encode()).hexdigest() + + report2 = await Report.objects.select_related("filters__hash").get() + assert report2.filters[0].name == "test1" + assert report2.filters[0].hash.name == hashed_test1 diff --git a/tests/test_or_filters.py b/tests/test_or_filters.py index 6d6f8ab..81a412c 100644 --- a/tests/test_or_filters.py +++ b/tests/test_or_filters.py @@ -57,24 +57,24 @@ async def test_or_filters(): books = ( await Book.objects.select_related("author") - .filter(ormar.or_(author__name="J.R.R. Tolkien", year__gt=1970)) - .all() + .filter(ormar.or_(author__name="J.R.R. Tolkien", year__gt=1970)) + .all() ) assert len(books) == 5 books = ( await Book.objects.select_related("author") - .filter(ormar.or_(author__name="J.R.R. Tolkien", year__lt=1995)) - .all() + .filter(ormar.or_(author__name="J.R.R. Tolkien", year__lt=1995)) + .all() ) assert len(books) == 4 assert not any([x.title == "The Tower of Fools" for x in books]) books = ( await Book.objects.select_related("author") - .filter(ormar.or_(year__gt=1960, year__lt=1940)) - .filter(author__name="J.R.R. Tolkien") - .all() + .filter(ormar.or_(year__gt=1960, year__lt=1940)) + .filter(author__name="J.R.R. Tolkien") + .all() ) assert len(books) == 2 assert books[0].title == "The Hobbit" @@ -82,13 +82,13 @@ async def test_or_filters(): books = ( await Book.objects.select_related("author") - .filter( + .filter( ormar.and_( ormar.or_(year__gt=1960, year__lt=1940), author__name="J.R.R. Tolkien", ) ) - .all() + .all() ) assert len(books) == 2 @@ -97,14 +97,14 @@ async def test_or_filters(): books = ( await Book.objects.select_related("author") - .filter( + .filter( ormar.or_( ormar.and_(year__gt=1960, author__name="J.R.R. Tolkien"), ormar.and_(year__lt=2000, author__name="Andrzej Sapkowski"), ) ) - .filter(title__startswith="The") - .all() + .filter(title__startswith="The") + .all() ) assert len(books) == 2 assert books[0].title == "The Silmarillion" @@ -112,7 +112,7 @@ async def test_or_filters(): books = ( await Book.objects.select_related("author") - .filter( + .filter( ormar.or_( ormar.and_( ormar.or_(year__gt=1960, year__lt=1940), @@ -121,7 +121,7 @@ async def test_or_filters(): ormar.and_(year__lt=2000, author__name="Andrzej Sapkowski"), ) ) - .all() + .all() ) assert len(books) == 3 assert books[0].title == "The Hobbit" @@ -130,29 +130,29 @@ async def test_or_filters(): books = ( await Book.objects.select_related("author") - .exclude( + .exclude( ormar.or_( ormar.and_(year__gt=1960, author__name="J.R.R. Tolkien"), ormar.and_(year__lt=2000, author__name="Andrzej Sapkowski"), ) ) - .filter(title__startswith="The") - .all() + .filter(title__startswith="The") + .all() ) assert len(books) == 3 assert not any([x.title in ["The Silmarillion", "The Witcher"] for x in books]) books = ( await Book.objects.select_related("author") - .filter( + .filter( ormar.or_( ormar.and_(year__gt=1960, author__name="J.R.R. Tolkien"), ormar.and_(year__lt=2000, author__name="Andrzej Sapkowski"), title__icontains="hobbit", ) ) - .filter(title__startswith="The") - .all() + .filter(title__startswith="The") + .all() ) assert len(books) == 3 assert not any( @@ -161,43 +161,43 @@ async def test_or_filters(): books = ( await Book.objects.select_related("author") - .filter(ormar.or_(year__gt=1980, year__lt=1910)) - .filter(title__startswith="The") - .limit(1) - .all() + .filter(ormar.or_(year__gt=1980, year__lt=1910)) + .filter(title__startswith="The") + .limit(1) + .all() ) assert len(books) == 1 assert books[0].title == "The Witcher" books = ( await Book.objects.select_related("author") - .filter(ormar.or_(year__gt=1980, author__name="Andrzej Sapkowski")) - .filter(title__startswith="The") - .limit(1) - .all() + .filter(ormar.or_(year__gt=1980, author__name="Andrzej Sapkowski")) + .filter(title__startswith="The") + .limit(1) + .all() ) assert len(books) == 1 assert books[0].title == "The Witcher" books = ( await Book.objects.select_related("author") - .filter(ormar.or_(year__gt=1980, author__name="Andrzej Sapkowski")) - .filter(title__startswith="The") - .limit(1) - .offset(1) - .all() + .filter(ormar.or_(year__gt=1980, author__name="Andrzej Sapkowski")) + .filter(title__startswith="The") + .limit(1) + .offset(1) + .all() ) assert len(books) == 1 assert books[0].title == "The Tower of Fools" books = ( await Book.objects.select_related("author") - .filter(ormar.or_(year__gt=1980, author__name="Andrzej Sapkowski")) - .filter(title__startswith="The") - .limit(1) - .offset(1) - .order_by("-id") - .all() + .filter(ormar.or_(year__gt=1980, author__name="Andrzej Sapkowski")) + .filter(title__startswith="The") + .limit(1) + .offset(1) + .order_by("-id") + .all() ) assert len(books) == 1 assert books[0].title == "The Witcher" @@ -220,19 +220,19 @@ async def test_or_filters(): books = ( await Book.objects.select_related("author") - .filter(ormar.or_(author__name="J.R.R. Tolkien")) - .all() + .filter(ormar.or_(author__name="J.R.R. Tolkien")) + .all() ) assert len(books) == 3 books = ( await Book.objects.select_related("author") - .filter( + .filter( ormar.or_( ormar.and_(author__name__icontains="tolkien"), ormar.and_(author__name__icontains="sapkowski"), ) ) - .all() + .all() ) assert len(books) == 5