Merge pull request #128 from collerek/encrypt

Fields encryption

.gitignore

@@ -4,7 +4,7 @@ alembic.ini
 .idea
 .pytest_cache
 .mypy_cache
-.coverage
+*.coverage
 *.pyc
 *.log
 test.db
@@ -14,3 +14,4 @@ site
 profile.py
 *.db
 *.db-journal
+*coverage.xml

Makefile

@@ -0,0 +1,19 @@
+PIPENV_RUN := pipenv run
+PG_DOCKERFILE_NAME := fastapi-users-test-mongo
+
+test_all: test_pg test_mysql test_sqlite
+
+test_pg: export DATABASE_URL=postgresql://username:password@localhost:5432/testsuite
+test_pg:
+	docker-compose -f scripts/docker-compose.yml up -d postgres
+	bash scripts/test.sh -svv
+	docker-compose -f scripts/docker-compose.yml stop postgres
+
+test_mysql: export DATABASE_URL=mysql://username:password@127.0.0.1:3306/testsuite
+test_mysql:
+	docker-compose -f "scripts/docker-compose.yml" up -d mysql
+	bash scripts/test.sh -svv
+	docker-compose -f scripts/docker-compose.yml stop mysql
+
+test_sqlite:
+	bash scripts/test.sh -svv

README.md

@@ -472,6 +472,7 @@ Available Model Fields (with required args - optional ones in docs):
 * `Decimal(scale, precision)`
 * `UUID()`
 * `EnumField` - by passing `choices` to any other Field type
+* `EncryptedString` - by passing `encrypt_secret` and `encrypt_backend`
 * `ForeignKey(to)`
 * `ManyToMany(to, through)`
 

docs/fields/encryption.md

@@ -0,0 +1,164 @@
+# Encryption
+
+`ormar` provides you with a way to encrypt a field in the database only.
+Provided encryption backends allow for both one-way encryption (`HASH` backend) as
+well as both-way encryption/decryption (`FERNET` backend).
+
+!!!warning
+    Note that in order for encryption to work you need to install optional `cryptography` package.
+
+    You can do it manually `pip install cryptography` or with ormar by `pip install ormar[crypto]`
+
+!!!warning
+    Note that adding `encrypt_backend` changes the database column type to `TEXT`, 
+    which needs to be reflected in db either by migration (`alembic`) or manual change
+
+## Defining a field encryption
+
+To encrypt a field you need to pass at minimum `encrypt_secret` and `encrypt_backend` parameters.
+
+```python hl_lines="7-8"
+class Filter(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "filters"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100, 
+                             encrypt_secret="secret123", 
+                             encrypt_backend=ormar.EncryptBackends.FERNET)
+```
+
+!!!warning
+    You can encrypt all `Field` types apart from `primary_key` column and relation 
+    columns (`ForeignKey` and `ManyToMany`). Check backends details for more information.
+
+## Available backends
+
+### HASH
+
+HASH is a one-way hash (like for password), never decrypted on retrieval
+
+To set it up pass appropriate backend value.
+
+```python
+... # rest of model definition
+password: str = ormar.String(max_length=128,
+                         encrypt_secret="secret123", 
+                         encrypt_backend=ormar.EncryptBackends.HASH)
+```
+
+Note that since this backend never decrypt the stored value it's only applicable for
+`String` fields. Used hash is a `sha512` hash, so the field length has to be >=128.
+
+!!!warning
+    Note that in `HASH` backend you can filter by full value but filters like `contain` will not work as comparison is make on encrypted values
+
+!!!note 
+    Note that provided `encrypt_secret` is first hashed itself and used as salt, so in order to
+    compare to stored string you need to recreate this steps. The `order_by` will not work as encrypted strings are compared so you cannot reliably order by.
+
+```python
+class Hash(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "hashes"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=128,
+                             encrypt_secret="udxc32",
+                             encrypt_backend=ormar.EncryptBackends.HASH)
+
+
+await Hash(name='test1').save()
+
+# note the steps to recreate the stored value
+# you can use also cryptography package instead of hashlib
+secret = hashlib.sha256("udxc32".encode()).digest()
+secret = base64.urlsafe_b64encode(secret)
+hashed_test1 = hashlib.sha512(secret + 'test1'.encode()).hexdigest()
+
+# full value comparison works
+hash1 = await Hash.objects.get(name='test1')
+assert hash1.name == hashed_test1
+
+# but partial comparison does not (hashed strings are compared)
+with pytest.raises(NoMatch):
+    await Filter.objects.get(name__icontains='test')
+```
+
+### FERNET
+
+FERNET is a two-way encrypt/decrypt backend
+
+To set it up pass appropriate backend value.
+
+```python
+... # rest of model definition
+year: int = ormar.Integer(encrypt_secret="secret123", 
+                          encrypt_backend=ormar.EncryptBackends.FERNET)
+```
+
+Value is encrypted on way to database end decrypted on way out. Can be used on all types,
+as the returned value is parsed to corresponding python type.
+
+!!!warning
+    Note that in `FERNET` backend you loose `filter`ing possibility altogether as part of the encrypted value is a timestamp.
+    The same goes for `order_by` as encrypted strings are compared so you cannot reliably order by. 
+
+```python
+class Filter(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "filters"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100, 
+                             encrypt_secret="asd123", 
+                             encrypt_backend=ormar.EncryptBackends.FERNET)
+
+await Filter(name='test1').save()
+await Filter(name='test1').save()
+
+# values are properly encrypted and later decrypted
+filters = await Filter.objects.all()
+assert filters[0].name == filters[1].name == 'test1'
+
+# but you cannot filter at all since part of the fernet hash is a timestamp
+# which means that even if you encrypt the same string 2 times it will be different
+with pytest.raises(NoMatch):
+    await Filter.objects.get(name='test1')
+```
+
+## Custom Backends
+
+If you wish to support other type of encryption (i.e. AES) you can provide your own `EncryptionBackend`.
+
+To setup a backend all you need to do is subclass `ormar.fields.EncryptBackend` class and provide required backend.
+
+Sample dummy backend (that does nothing) can look like following:
+
+```python
+class DummyBackend(ormar.fields.EncryptBackend):
+    def _initialize_backend(self, secret_key: bytes) -> None:
+        pass
+
+    def encrypt(self, value: Any) -> str:
+        return value
+
+    def decrypt(self, value: Any) -> str:
+        return value
+```
+
+To use this backend set `encrypt_backend` to `CUSTOM` and provide your backend as
+argument by `encrypt_custom_backend`.
+
+```python
+class Filter(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "filters"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100, 
+                             encrypt_secret="secret123", 
+                             encrypt_backend=ormar.EncryptBackends.CUSTOM,
+                             encrypt_custom_backend=DummyBackend
+                             )
+```

docs/index.md

@@ -472,6 +472,7 @@ Available Model Fields (with required args - optional ones in docs):
 * `Decimal(scale, precision)`
 * `UUID()`
 * `EnumField` - by passing `choices` to any other Field type
+* `EncryptedString` - by passing `encrypt_secret` and `encrypt_backend`
 * `ForeignKey(to)`
 * `ManyToMany(to, through)`
 

docs/releases.md

@@ -1,3 +1,21 @@
+# 0.9.8
+
+## Features
+* Add possibility to encrypt the selected field(s) in the database
+  * As minimum you need to provide `encrypt_secret` and `encrypt_backend`
+  * `encrypt_backend` can be one of the `ormar.EncryptBackends` enum (`NONE, FERNET, HASH, CUSTOM`) - default: `NONE`
+  * When custom backend is selected you need to provide your backend class that subclasses `ormar.fields.EncryptBackend`
+  * You cannot encrypt `primary_key` column and relation columns (FK and M2M).
+  * Provided are 2 backends: HASH and FERNET
+    * HASH is a one-way hash (like for password), never decrypted on retrieval
+    * FERNET is a two-way encrypt/decrypt backend
+  * Note that in FERNET backend you loose `filtering` possibility altogether as part of the encrypted value is a timestamp.
+  * Note that in HASH backend you can filter by full value but filters like `contain` will not work as comparison is make on encrypted values
+  * Note that adding `encrypt_backend` changes the database column type to `TEXT`, which needs to be reflected in db either by migration or manual change
+
+## Fixes
+* (Advanced/ Internal) Restore custom sqlalchemy types (by `types.TypeDecorator` subclass) functionality that ceased to working so `process_result_value` was never called
+
 # 0.9.7
 
 ## Features

mkdocs.yml

@@ -12,6 +12,7 @@ nav:
   - Fields:
     - Common parameters: fields/common-parameters.md
     - Fields types: fields/field-types.md
+    - Fields encryption: fields/encryption.md
   - Relations:
     - relations/index.md
     - relations/postponed-annotations.md

ormar/__init__.py

@@ -38,9 +38,12 @@ from ormar.fields import (
     BaseField,
     BigInteger,
     Boolean,
+    DECODERS_MAP,
     Date,
     DateTime,
     Decimal,
+    ENCODERS_MAP,
+    EncryptBackends,
     Float,
     ForeignKey,
     ForeignKeyField,
@@ -68,7 +71,7 @@ class UndefinedType:  # pragma no cover
 
 Undefined = UndefinedType()
 
-__version__ = "0.9.7"
+__version__ = "0.9.8"
 __all__ = [
     "Integer",
     "BigInteger",
@@ -110,4 +113,7 @@ __all__ = [
     "ExcludableItems",
     "and_",
     "or_",
+    "EncryptBackends",
+    "ENCODERS_MAP",
+    "DECODERS_MAP",
 ]

ormar/fields/__init__.py

@@ -21,6 +21,8 @@ from ormar.fields.model_fields import (
     Time,
     UUID,
 )
+from ormar.fields.parsers import DECODERS_MAP, ENCODERS_MAP
+from ormar.fields.sqlalchemy_encrypted import EncryptBackend, EncryptBackends
 from ormar.fields.through_field import Through, ThroughField
 
 __all__ = [
@@ -44,4 +46,8 @@ __all__ = [
     "ForeignKeyField",
     "ThroughField",
     "Through",
+    "EncryptBackends",
+    "EncryptBackend",
+    "DECODERS_MAP",
+    "ENCODERS_MAP",
 ]

ormar/fields/base.py

@@ -5,6 +5,12 @@ from pydantic import Field, Json, typing
 from pydantic.fields import FieldInfo, Required, Undefined
 
 import ormar  # noqa I101
+from ormar import ModelDefinitionError
+from ormar.fields.sqlalchemy_encrypted import (
+    EncryptBackend,
+    EncryptBackends,
+    EncryptedString,
+)
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar.models import Model
@@ -49,6 +55,10 @@ class BaseField(FieldInfo):
     self_reference: bool = False
     self_reference_primary: Optional[str] = None
 
+    encrypt_secret: str
+    encrypt_backend: EncryptBackends = EncryptBackends.NONE
+    encrypt_custom_backend: Optional[Type[EncryptBackend]] = None
+
     default: Any
     server_default: Any
 
@@ -256,6 +266,7 @@ class BaseField(FieldInfo):
         :return: actual definition of the database column as sqlalchemy requires.
         :rtype: sqlalchemy.Column
         """
+        if cls.encrypt_backend == EncryptBackends.NONE:
             column = sqlalchemy.Column(
                 cls.alias or name,
                 cls.column_type,
@@ -267,6 +278,38 @@ class BaseField(FieldInfo):
                 default=cls.default,
                 server_default=cls.server_default,
             )
+        else:
+            column = cls._get_encrypted_column(name=name)
+        return column
+
+    @classmethod
+    def _get_encrypted_column(cls, name: str) -> sqlalchemy.Column:
+        """
+        Returns EncryptedString column type instead of actual column.
+
+        :param name: column name
+        :type name: str
+        :return: newly defined column
+        :rtype:  sqlalchemy.Column
+        """
+        if cls.primary_key or cls.is_relation:
+            raise ModelDefinitionError(
+                "Primary key field and relations fields" "cannot be encrypted!"
+            )
+        column = sqlalchemy.Column(
+            cls.alias or name,
+            EncryptedString(
+                _field_type=cls,
+                encrypt_secret=cls.encrypt_secret,
+                encrypt_backend=cls.encrypt_backend,
+                encrypt_custom_backend=cls.encrypt_custom_backend,
+            ),
+            nullable=cls.nullable,
+            index=cls.index,
+            unique=cls.unique,
+            default=cls.default,
+            server_default=cls.server_default,
+        )
         return column
 
     @classmethod

ormar/fields/foreign_key.py

@@ -184,10 +184,23 @@ def ForeignKey(  # noqa CFQ002
 
     owner = kwargs.pop("owner", None)
     self_reference = kwargs.pop("self_reference", False)
+
     default = kwargs.pop("default", None)
-    if default is not None:
+    encrypt_secret = kwargs.pop("encrypt_secret", None)
+    encrypt_backend = kwargs.pop("encrypt_backend", None)
+    encrypt_custom_backend = kwargs.pop("encrypt_custom_backend", None)
+
+    not_supported = [
+        default,
+        encrypt_secret,
+        encrypt_backend,
+        encrypt_custom_backend,
+    ]
+    if any(x is not None for x in not_supported):
         raise ModelDefinitionError(
-            "Argument 'default' is not supported " "on relation fields!"
+            f"Argument {next((x for x in not_supported if x is not None))} "
+            f"is not supported "
+            "on relation fields!"
         )
 
     if to.__class__ == ForwardRef:
@@ -386,7 +399,7 @@ class ForeignKeyField(BaseField):
         :return: (if needed) registered Model
         :rtype: Model
         """
-        if cls.to.pk_type() == uuid.UUID and isinstance(value, str):
+        if cls.to.pk_type() == uuid.UUID and isinstance(value, str):  # pragma: nocover
             value = uuid.UUID(value)
         if not isinstance(value, cls.to.pk_type()):
             raise RelationshipInstanceError(

ormar/fields/many_to_many.py

@@ -97,9 +97,21 @@ def ManyToMany(
         forbid_through_relations(cast(Type["Model"], through))
 
     default = kwargs.pop("default", None)
-    if default is not None:
+    encrypt_secret = kwargs.pop("encrypt_secret", None)
+    encrypt_backend = kwargs.pop("encrypt_backend", None)
+    encrypt_custom_backend = kwargs.pop("encrypt_custom_backend", None)
+
+    not_supported = [
+        default,
+        encrypt_secret,
+        encrypt_backend,
+        encrypt_custom_backend,
+    ]
+    if any(x is not None for x in not_supported):
         raise ModelDefinitionError(
-            "Argument 'default' is not supported " "on relation fields!"
+            f"Argument {next((x for x in not_supported if x is not None))} "
+            f"is not supported "
+            "on relation fields!"
         )
 
     if to.__class__ == ForwardRef:

ormar/fields/model_fields.py

@@ -9,6 +9,7 @@ import sqlalchemy
 from ormar import ModelDefinitionError  # noqa I101
 from ormar.fields import sqlalchemy_uuid
 from ormar.fields.base import BaseField  # noqa I101
+from ormar.fields.sqlalchemy_encrypted import EncryptBackends
 
 
 def is_field_nullable(
@@ -73,6 +74,11 @@ class ModelFieldFactory:
         primary_key = kwargs.pop("primary_key", False)
         autoincrement = kwargs.pop("autoincrement", False)
 
+        encrypt_secret = kwargs.pop("encrypt_secret", None)
+        encrypt_backend = kwargs.pop("encrypt_backend", EncryptBackends.NONE)
+        encrypt_custom_backend = kwargs.pop("encrypt_custom_backend", None)
+        encrypt_max_length = kwargs.pop("encrypt_max_length", 5000)
+
         namespace = dict(
             __type__=cls._type,
             alias=kwargs.pop("name", None),
@@ -88,6 +94,10 @@ class ModelFieldFactory:
             autoincrement=autoincrement,
             column_type=cls.get_column_type(**kwargs),
             choices=set(kwargs.pop("choices", [])),
+            encrypt_secret=encrypt_secret,
+            encrypt_backend=encrypt_backend,
+            encrypt_custom_backend=encrypt_custom_backend,
+            encrypt_max_length=encrypt_max_length,
             **kwargs
         )
         return type(cls.__name__, cls._bases, namespace)

ormar/fields/parsers.py

@@ -0,0 +1,44 @@
+import datetime
+import decimal
+from typing import Any
+
+import pydantic
+from pydantic.datetime_parse import parse_date, parse_datetime, parse_time
+
+try:
+    import orjson as json
+except ImportError:  # pragma: no cover
+    import json  # type: ignore
+
+
+def parse_bool(value: str) -> bool:
+    return value == "true"
+
+
+def encode_bool(value: bool) -> str:
+    return "true" if value else "false"
+
+
+def encode_json(value: Any) -> str:
+    value = json.dumps(value) if not isinstance(value, str) else value
+    value = value.decode("utf-8") if isinstance(value, bytes) else value
+    return value
+
+
+ENCODERS_MAP = {
+    bool: encode_bool,
+    datetime.datetime: lambda x: x.isoformat(),
+    datetime.date: lambda x: x.isoformat(),
+    datetime.time: lambda x: x.isoformat(),
+    pydantic.Json: encode_json,
+    decimal.Decimal: float,
+}
+
+DECODERS_MAP = {
+    bool: parse_bool,
+    datetime.datetime: parse_datetime,
+    datetime.date: parse_date,
+    datetime.time: parse_time,
+    pydantic.Json: json.loads,
+    decimal.Decimal: decimal.Decimal,
+}

ormar/fields/sqlalchemy_encrypted.py

@@ -0,0 +1,182 @@
+# inspired by sqlalchemy-utils (https://github.com/kvesteri/sqlalchemy-utils)
+import abc
+import base64
+from enum import Enum
+from typing import Any, Callable, Optional, TYPE_CHECKING, Type, Union
+
+import sqlalchemy.types as types
+from pydantic.utils import lenient_issubclass
+from sqlalchemy.engine.default import DefaultDialect
+
+import ormar  # noqa: I100, I202
+from ormar import ModelDefinitionError  # noqa: I202, I100
+
+cryptography = None
+try:  # pragma: nocover
+    import cryptography  # type: ignore
+    from cryptography.fernet import Fernet
+    from cryptography.hazmat.backends import default_backend
+    from cryptography.hazmat.primitives import hashes
+except ImportError:  # pragma: nocover
+    pass
+
+if TYPE_CHECKING:  # pragma: nocover
+    from ormar import BaseField
+
+
+class EncryptBackend(abc.ABC):
+    def _refresh(self, key: Union[str, bytes]) -> None:
+        if isinstance(key, str):
+            key = key.encode()
+        digest = hashes.Hash(hashes.SHA256(), backend=default_backend())
+        digest.update(key)
+        engine_key = digest.finalize()
+        self._initialize_backend(engine_key)
+
+    @abc.abstractmethod
+    def _initialize_backend(self, secret_key: bytes) -> None:  # pragma: nocover
+        pass
+
+    @abc.abstractmethod
+    def encrypt(self, value: Any) -> str:  # pragma: nocover
+        pass
+
+    @abc.abstractmethod
+    def decrypt(self, value: Any) -> str:  # pragma: nocover
+        pass
+
+
+class HashBackend(EncryptBackend):
+    """
+    One-way hashing - in example for passwords, no way to decrypt the value!
+    """
+
+    def _initialize_backend(self, secret_key: bytes) -> None:
+        self.secret_key = base64.urlsafe_b64encode(secret_key)
+
+    def encrypt(self, value: Any) -> str:
+        if not isinstance(value, str):  # pragma: nocover
+            value = repr(value)
+        value = value.encode()
+        digest = hashes.Hash(hashes.SHA512(), backend=default_backend())
+        digest.update(self.secret_key)
+        digest.update(value)
+        hashed_value = digest.finalize()
+        return hashed_value.hex()
+
+    def decrypt(self, value: Any) -> str:
+        if not isinstance(value, str):  # pragma: nocover
+            value = str(value)
+        return value
+
+
+class FernetBackend(EncryptBackend):
+    """
+    Two-way encryption, data stored in db are encrypted but decrypted during query.
+    """
+
+    def _initialize_backend(self, secret_key: bytes) -> None:
+        self.secret_key = base64.urlsafe_b64encode(secret_key)
+        self.fernet = Fernet(self.secret_key)
+
+    def encrypt(self, value: Any) -> str:
+        if not isinstance(value, str):
+            value = repr(value)
+        value = value.encode()
+        encrypted = self.fernet.encrypt(value)
+        return encrypted.decode("utf-8")
+
+    def decrypt(self, value: Any) -> str:
+        if not isinstance(value, str):  # pragma: nocover
+            value = str(value)
+        decrypted: Union[str, bytes] = self.fernet.decrypt(value.encode())
+        if not isinstance(decrypted, str):
+            decrypted = decrypted.decode("utf-8")
+        return decrypted
+
+
+class EncryptBackends(Enum):
+    NONE = 0
+    FERNET = 1
+    HASH = 2
+    CUSTOM = 3
+
+
+BACKENDS_MAP = {
+    EncryptBackends.FERNET: FernetBackend,
+    EncryptBackends.HASH: HashBackend,
+}
+
+
+class EncryptedString(types.TypeDecorator):
+    """
+    Used to store encrypted values in a database
+    """
+
+    impl = types.TypeEngine
+
+    def __init__(
+        self,
+        encrypt_secret: Union[str, Callable],
+        encrypt_backend: EncryptBackends = EncryptBackends.FERNET,
+        encrypt_custom_backend: Type[EncryptBackend] = None,
+        **kwargs: Any,
+    ) -> None:
+        _field_type = kwargs.pop("_field_type")
+        super().__init__()
+        if not cryptography:  # pragma: nocover
+            raise ModelDefinitionError(
+                "In order to encrypt a column 'cryptography' is required!"
+            )
+        backend = BACKENDS_MAP.get(encrypt_backend, encrypt_custom_backend)
+        if not backend or not lenient_issubclass(backend, EncryptBackend):
+            raise ModelDefinitionError("Wrong or no encrypt backend provided!")
+
+        self.backend: EncryptBackend = backend()
+        self._field_type: Type["BaseField"] = _field_type
+        self._underlying_type: Any = _field_type.column_type
+        self._key: Union[str, Callable] = encrypt_secret
+        type_ = self._field_type.__type__
+        if type_ is None:  # pragma: nocover
+            raise ModelDefinitionError(
+                f"Improperly configured field " f"{self._field_type.name}"
+            )
+        self.type_: Any = type_
+
+    def __repr__(self) -> str:  # pragma: nocover
+        return "TEXT()"
+
+    def load_dialect_impl(self, dialect: DefaultDialect) -> Any:
+        return dialect.type_descriptor(types.TEXT())
+
+    def _refresh(self) -> None:
+        key = self._key() if callable(self._key) else self._key
+        self.backend._refresh(key)
+
+    def process_bind_param(self, value: Any, dialect: DefaultDialect) -> Optional[str]:
+        if value is None:
+            return value
+        self._refresh()
+        try:
+            value = self._underlying_type.process_bind_param(value, dialect)
+        except AttributeError:
+            encoder = ormar.ENCODERS_MAP.get(self.type_, None)
+            if encoder:
+                value = encoder(value)  # type: ignore
+
+        encrypted_value = self.backend.encrypt(value)
+        return encrypted_value
+
+    def process_result_value(self, value: Any, dialect: DefaultDialect) -> Any:
+        if value is None:
+            return value
+        self._refresh()
+        decrypted_value = self.backend.decrypt(value)
+        try:
+            return self._underlying_type.process_result_value(decrypted_value, dialect)
+        except AttributeError:
+            decoder = ormar.DECODERS_MAP.get(self.type_, None)
+            if decoder:
+                return decoder(decrypted_value)  # type: ignore
+
+            return self._field_type.__type__(decrypted_value)  # type: ignore

ormar/fields/sqlalchemy_uuid.py

@@ -1,12 +1,12 @@
 import uuid
-from typing import Any, Optional, Union
+from typing import Any, Optional
 
 from sqlalchemy import CHAR
 from sqlalchemy.engine.default import DefaultDialect
 from sqlalchemy.types import TypeDecorator
 
 
-class UUID(TypeDecorator):  # pragma nocover
+class UUID(TypeDecorator):
     """
     Platform-independent GUID type.
     Uses CHAR(36) if in a string mode, otherwise uses CHAR(32), to store UUID.
@@ -20,31 +20,11 @@ class UUID(TypeDecorator):  # pragma nocover
         super().__init__(*args, **kwargs)
         self.uuid_format = uuid_format
 
-    def __repr__(self) -> str:
+    def __repr__(self) -> str:  # pragma: nocover
         if self.uuid_format == "string":
             return "CHAR(36)"
         return "CHAR(32)"
 
-    def _cast_to_uuid(self, value: Union[str, int, bytes]) -> uuid.UUID:
-        """
-        Parses given value into uuid.UUID field.
-
-        :param value: value to be parsed
-        :type value: Union[str, int, bytes]
-        :return: initialized uuid
-        :rtype: uuid.UUID
-        """
-        if not isinstance(value, uuid.UUID):
-            if isinstance(value, bytes):
-                ret_value = uuid.UUID(bytes=value)
-            elif isinstance(value, int):
-                ret_value = uuid.UUID(int=value)
-            elif isinstance(value, str):
-                ret_value = uuid.UUID(value)
-        else:
-            ret_value = value
-        return ret_value
-
     def load_dialect_impl(self, dialect: DefaultDialect) -> Any:
         return (
             dialect.type_descriptor(CHAR(36))
@@ -53,12 +33,10 @@ class UUID(TypeDecorator):  # pragma nocover
         )
 
     def process_bind_param(
-        self, value: Union[str, int, bytes, uuid.UUID, None], dialect: DefaultDialect
+        self, value: uuid.UUID, dialect: DefaultDialect
     ) -> Optional[str]:
         if value is None:
             return value
-        if not isinstance(value, uuid.UUID):
-            value = self._cast_to_uuid(value)
         return str(value) if self.uuid_format == "string" else "%.32x" % value.int
 
     def process_result_value(
@@ -68,4 +46,4 @@ class UUID(TypeDecorator):  # pragma nocover
             return value
         if not isinstance(value, uuid.UUID):
             return uuid.UUID(value)
-        return value
+        return value  # pragma: nocover

ormar/models/helpers/sqlalchemy.py

@@ -289,7 +289,7 @@ def populate_meta_sqlalchemy_table_if_required(meta: "ModelMeta") -> None:
                     f'{"_".join([str(col) for col in constraint._pending_colargs])}'
                 )
         table = sqlalchemy.Table(
-            meta.tablename, meta.metadata, *meta.columns, *meta.constraints,
+            meta.tablename, meta.metadata, *meta.columns, *meta.constraints
         )
         meta.table = table
 

ormar/models/helpers/validation.py

@@ -53,6 +53,7 @@ def convert_choices_if_needed(  # noqa: CCR001
     :return: value, choices list
     :rtype: Tuple[Any, List]
     """
+    # TODO use same maps as with EncryptedString
     choices = [o.value if isinstance(o, Enum) else o for o in field.choices]
 
     if field.__type__ in [datetime.datetime, datetime.date, datetime.time]:

ormar/models/model_row.py

@@ -369,9 +369,6 @@ class ModelRow(NewBaseModel):
         and values are database values
         :rtype: Dict
         """
-        # databases does not keep aliases in Record for postgres, change to raw row
-        source = row._row if cls.db_backend_name() == "postgresql" else row
-
         selected_columns = cls.own_table_columns(
             model=cls, excludable=excludable, alias=table_prefix, use_alias=False,
         )
@@ -381,6 +378,6 @@ class ModelRow(NewBaseModel):
             alias = cls.get_column_name_from_alias(column.name)
             if alias not in item and alias in selected_columns:
                 prefixed_name = f"{column_prefix}{column.name}"
-                item[alias] = source[prefixed_name]
+                item[alias] = row[prefixed_name]
 
         return item

ormar/queryset/join.py

@@ -88,13 +88,13 @@ class SqlJoin:
         return self.main_model.Meta.alias_manager
 
     @property
-    def to_table(self) -> str:
+    def to_table(self) -> sqlalchemy.Table:
         """
         Shortcut to table name of the next model
         :return: name of the target table
         :rtype: str
         """
-        return self.next_model.Meta.table.name
+        return self.next_model.Meta.table
 
     def _on_clause(
         self, previous_alias: str, from_clause: str, to_clause: str,
@@ -282,7 +282,7 @@ class SqlJoin:
         on_clause = self._on_clause(
             previous_alias=self.own_alias,
             from_clause=f"{self.target_field.owner.Meta.tablename}.{from_key}",
-            to_clause=f"{self.to_table}.{to_key}",
+            to_clause=f"{self.to_table.name}.{to_key}",
         )
         target_table = self.alias_manager.prefixed_table_name(
             self.next_alias, self.to_table
@@ -301,7 +301,7 @@ class SqlJoin:
         )
         self.columns.extend(
             self.alias_manager.prefixed_columns(
-                self.next_alias, self.next_model.Meta.table, self_related_fields
+                self.next_alias, target_table, self_related_fields
             )
         )
         self.used_aliases.append(self.next_alias)

ormar/relations/alias_manager.py

@@ -67,24 +67,21 @@ class AliasManager:
             if not fields
             else [col for col in table.columns if col.name in fields]
         )
-        return [
+        return [column.label(f"{alias}{column.name}") for column in all_columns]
-            text(f"{alias}{table.name}.{column.name} as {alias}{column.name}")
-            for column in all_columns
-        ]
 
     @staticmethod
-    def prefixed_table_name(alias: str, name: str) -> text:
+    def prefixed_table_name(alias: str, table: sqlalchemy.Table) -> text:
         """
         Creates text clause with table name with aliased name.
 
         :param alias: alias of given table
         :type alias: str
-        :param name: table name
+        :param table: table
-        :type name: str
+        :type table: sqlalchemy.Table
         :return: sqlalchemy text clause as "table_name aliased_name"
         :rtype: sqlalchemy text clause
         """
-        return text(f"{name} {alias}_{name}")
+        return table.alias(f"{alias}_{table.name}")
 
     def add_relation_type(
         self, source_model: Type["Model"], relation_name: str, reverse_name: str = None,

requirements.txt

@@ -5,6 +5,7 @@ pydantic
 sqlalchemy
 typing_extensions
 orjson
+cryptography
 
 # Async database drivers
 aiomysql

scripts/docker-compose.yml

@@ -0,0 +1,20 @@
+version: '2.1'
+services:
+  postgres:
+    image: postgres:10.8
+    environment:
+      POSTGRES_USER: username
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: testsuite
+    ports:
+      - 5432:5432
+
+  mysql:
+    image: mysql:5.7
+    environment:
+      MYSQL_USER: username
+      MYSQL_PASSWORD: password
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_DATABASE: testsuite
+    ports:
+      - 3306:3306

setup.py

@@ -62,7 +62,8 @@ setup(
         "postgresql": ["asyncpg", "psycopg2"],
         "mysql": ["aiomysql", "pymysql"],
         "sqlite": ["aiosqlite"],
-        "orjson": ["orjson"]
+        "orjson": ["orjson"],
+        "crypto": ["cryptography"]
     },
     classifiers=[
         "Development Status :: 4 - Beta",

tests/settings.py

@@ -6,3 +6,4 @@ DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///test.db")
 database_url = databases.DatabaseURL(DATABASE_URL)
 if database_url.scheme == "postgresql+aiopg":  # pragma no cover
     DATABASE_URL = str(database_url.replace(driver=None))
+print("USED DB:", DATABASE_URL)

tests/test_encrypted_columns.py

@@ -0,0 +1,269 @@
+# type: ignore
+import base64
+import decimal
+import hashlib
+import uuid
+import datetime
+from typing import Any
+
+import databases
+import pytest
+import sqlalchemy
+
+import ormar
+from ormar import ModelDefinitionError, NoMatch
+from ormar.fields.sqlalchemy_encrypted import EncryptedString
+from tests.settings import DATABASE_URL
+
+database = databases.Database(DATABASE_URL)
+metadata = sqlalchemy.MetaData()
+
+
+class BaseMeta(ormar.ModelMeta):
+    metadata = metadata
+    database = database
+
+
+default_fernet = dict(
+    encrypt_secret="asd123", encrypt_backend=ormar.EncryptBackends.FERNET,
+)
+
+
+class DummyBackend(ormar.fields.EncryptBackend):
+    def _initialize_backend(self, secret_key: bytes) -> None:
+        pass
+
+    def encrypt(self, value: Any) -> str:
+        return value
+
+    def decrypt(self, value: Any) -> str:
+        return value
+
+
+class Author(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "authors"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100, **default_fernet)
+    uuid_test = ormar.UUID(default=uuid.uuid4, uuid_format="string")
+    uuid_test2 = ormar.UUID(nullable=True, uuid_format="string")
+    password: str = ormar.String(
+        max_length=128,
+        encrypt_secret="udxc32",
+        encrypt_backend=ormar.EncryptBackends.HASH,
+    )
+    birth_year: int = ormar.Integer(
+        nullable=True,
+        encrypt_secret="secure89key%^&psdijfipew",
+        encrypt_backend=ormar.EncryptBackends.FERNET,
+    )
+    test_text: str = ormar.Text(default="", **default_fernet)
+    test_bool: bool = ormar.Boolean(nullable=False, **default_fernet)
+    test_float: float = ormar.Float(**default_fernet)
+    test_float2: float = ormar.Float(nullable=True, **default_fernet)
+    test_datetime = ormar.DateTime(default=datetime.datetime.now, **default_fernet)
+    test_date = ormar.Date(default=datetime.date.today, **default_fernet)
+    test_time = ormar.Time(default=datetime.time, **default_fernet)
+    test_json = ormar.JSON(default={}, **default_fernet)
+    test_bigint: int = ormar.BigInteger(default=0, **default_fernet)
+    test_decimal = ormar.Decimal(scale=2, precision=10, **default_fernet)
+    test_decimal2 = ormar.Decimal(max_digits=10, decimal_places=2, **default_fernet)
+    custom_backend: str = ormar.String(
+        max_length=200,
+        encrypt_secret="asda8",
+        encrypt_backend=ormar.EncryptBackends.CUSTOM,
+        encrypt_custom_backend=DummyBackend,
+    )
+
+
+class Hash(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "hashes"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(
+        max_length=128,
+        encrypt_secret="udxc32",
+        encrypt_backend=ormar.EncryptBackends.HASH,
+    )
+
+
+class Filter(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "filters"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100, **default_fernet)
+    hash = ormar.ForeignKey(Hash)
+
+
+class Report(ormar.Model):
+    class Meta(BaseMeta):
+        tablename = "reports"
+
+    id: int = ormar.Integer(primary_key=True)
+    name: str = ormar.String(max_length=100)
+    filters = ormar.ManyToMany(Filter)
+
+
+@pytest.fixture(autouse=True, scope="module")
+def create_test_database():
+    engine = sqlalchemy.create_engine(DATABASE_URL)
+    metadata.drop_all(engine)
+    metadata.create_all(engine)
+    yield
+    metadata.drop_all(engine)
+
+
+def test_error_on_encrypted_pk():
+    with pytest.raises(ModelDefinitionError):
+
+        class Wrong(ormar.Model):
+            class Meta(BaseMeta):
+                tablename = "wrongs"
+
+            id: int = ormar.Integer(
+                primary_key=True,
+                encrypt_secret="asd123",
+                encrypt_backend=ormar.EncryptBackends.FERNET,
+            )
+
+
+def test_error_on_encrypted_relation():
+    with pytest.raises(ModelDefinitionError):
+
+        class Wrong2(ormar.Model):
+            class Meta(BaseMeta):
+                tablename = "wrongs2"
+
+            id: int = ormar.Integer(primary_key=True)
+            author = ormar.ForeignKey(
+                Author,
+                encrypt_secret="asd123",
+                encrypt_backend=ormar.EncryptBackends.FERNET,
+            )
+
+
+def test_error_on_encrypted_m2m_relation():
+    with pytest.raises(ModelDefinitionError):
+
+        class Wrong3(ormar.Model):
+            class Meta(BaseMeta):
+                tablename = "wrongs3"
+
+            id: int = ormar.Integer(primary_key=True)
+            author = ormar.ManyToMany(
+                Author,
+                encrypt_secret="asd123",
+                encrypt_backend=ormar.EncryptBackends.FERNET,
+            )
+
+
+def test_wrong_backend():
+    with pytest.raises(ModelDefinitionError):
+
+        class Wrong3(ormar.Model):
+            class Meta(BaseMeta):
+                tablename = "wrongs3"
+
+            id: int = ormar.Integer(primary_key=True)
+            author = ormar.Integer(
+                encrypt_secret="asd123",
+                encrypt_backend=ormar.EncryptBackends.CUSTOM,
+                encrypt_custom_backend="aa",
+            )
+
+
+def test_db_structure():
+    assert Author.Meta.table.c.get("name").type.__class__ == EncryptedString
+
+
+@pytest.mark.asyncio
+async def test_save_and_retrieve():
+    async with database:
+        test_uuid = uuid.uuid4()
+        await Author(
+            name="Test",
+            birth_year=1988,
+            password="test123",
+            uuid_test=test_uuid,
+            test_float=1.2,
+            test_bool=True,
+            test_decimal=decimal.Decimal(3.5),
+            test_decimal2=decimal.Decimal(5.5),
+            test_json=dict(aa=12),
+            custom_backend="test12",
+        ).save()
+        author = await Author.objects.get()
+
+        assert author.name == "Test"
+        assert author.birth_year == 1988
+        password = (
+            "03e4a4d513e99cb3fe4ee3db282c053daa3f3572b849c3868939a306944ad5c08"
+            "22b50d4886e10f4cd418c3f2df3ceb02e2e7ac6e920ae0c90f2dedfc8fa16e2"
+        )
+        assert author.password == password
+        assert author.uuid_test == test_uuid
+        assert author.uuid_test2 is None
+        assert author.test_datetime.date() == datetime.date.today()
+        assert author.test_date == datetime.date.today()
+        assert author.test_text == ""
+        assert author.test_float == 1.2
+        assert author.test_float2 is None
+        assert author.test_bigint == 0
+        assert author.test_json == {"aa": 12}
+        assert author.test_decimal == 3.5
+        assert author.test_decimal2 == 5.5
+        assert author.custom_backend == "test12"
+
+
+@pytest.mark.asyncio
+async def test_fernet_filters_nomatch():
+    async with database:
+        await Filter(name="test1").save()
+        await Filter(name="test1").save()
+
+        filters = await Filter.objects.all()
+        assert filters[0].name == filters[1].name == "test1"
+
+        with pytest.raises(NoMatch):
+            await Filter.objects.get(name="test1")
+
+
+@pytest.mark.asyncio
+async def test_hash_filters_works():
+    async with database:
+        await Hash(name="test1").save()
+        await Hash(name="test2").save()
+
+        secret = hashlib.sha256("udxc32".encode()).digest()
+        secret = base64.urlsafe_b64encode(secret)
+        hashed_test1 = hashlib.sha512(secret + "test1".encode()).hexdigest()
+
+        hash1 = await Hash.objects.get(name="test1")
+        assert hash1.name == hashed_test1
+
+        with pytest.raises(NoMatch):
+            await Filter.objects.get(name__icontains="test")
+
+
+@pytest.mark.asyncio
+async def test_related_model_fields_properly_decrypted():
+    async with database:
+        hash1 = await Hash(name="test1").save()
+        report = await Report.objects.create(name="Report1")
+        await report.filters.create(name="test1", hash=hash1)
+        await report.filters.create(name="test2")
+
+        report2 = await Report.objects.select_related("filters").get()
+        assert report2.filters[0].name == "test1"
+        assert report2.filters[1].name == "test2"
+
+        secret = hashlib.sha256("udxc32".encode()).digest()
+        secret = base64.urlsafe_b64encode(secret)
+        hashed_test1 = hashlib.sha512(secret + "test1".encode()).hexdigest()
+
+        report2 = await Report.objects.select_related("filters__hash").get()
+        assert report2.filters[0].name == "test1"
+        assert report2.filters[0].hash.name == hashed_test1