Implement re-encryption support

blackm0re · May 13, 2024 · 08a3280 · 08a3280
1 parent bb9a844
commit 08a3280
Show file tree

Hide file tree

Showing 12 changed files with 729 additions and 266 deletions.
diff --git a/.ruff.toml b/.ruff.toml
@@ -0,0 +1,73 @@
+cache-dir = "~/.cache/ruff"
+indent-width = 4
+line-length = 79
+target-version = "py312"
+
+[lint]
+select = ["ALL"]
+ignore = ["ANN", "COM812", "D105", "D202", "D203", "D205", "D211", "D212", "D400", "D401", "D403", "D415", "ERA001", "FBT001", "FBT002", "PTH111", "RUF012", "RUF013", "S101", "TRY300", "BLE001", "UP020", "C901", "D200", "D402", "EM101", "EM102", "FBT003", "INP001", "PLR0912", "PLR0913", "PLR0915", "PLR2004", "PLW2901", "S603", "T201", "TRY003", "TRY400"]
+# D105 - Missing docstring in magic method
+# D200 - One-line docstring should fit on one line
+# D203 - 1 blank line required before class docstring
+# D205 - 1 blank line required between summary line and description
+# D403 - First word of the first line should be capitalized: `str` -> `Str`
+# FBT001 - Boolean-typed positional argument in function definition
+# FBT002 - Boolean default positional argument in function definition
+# PTH111 - `os.path.expanduser()` should be replaced by `Path.expanduser()`
+# RUF012 - Mutable class attributes should be annotated with `typing.ClassVar`
+# RUF013 - PEP 484 prohibits implicit `Optional`
+# S101 - Use of `assert` detected
+# TRY300 - Consider moving this statement to an `else` block
+# TRY400 - Use `logging.exception` instead of `logging.error`
+# UP020 - Use builtin `open`
+
+# Project specific
+# C901 - `X` is too complex
+# D200 - One-line docstring should fit on one line
+# D402 - First line should not be the function's signature (bug in ruff 0.4.4)
+# EM101 - Exception must not use a string literal, assign to variable first
+# EM102 - Exception must not use an f-string literal, assign to variable first
+# FBT003 - Boolean positional value in function call
+# INP001 - File `tests/test_envtoolkit_instance_static.py` is part of an implicit namespace package. Add an `__init__.py`.
+# PLR0912 - Too many branches
+# PLR0913 - Too many arguments in function definition
+# PLR0915 - Too many statements
+# PLR2004 - Magic value used in comparison, consider replacing `X` with a constant variable
+# PLW2901 - `for` loop variable `value` overwritten by assignment target
+# S603 - `subprocess` call: check for execution of untrusted input
+# T201 - `print` found
+# TRY003 - Avoid specifying long messages outside the exception class
+
+# Allow fix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+[format]
+# Like Black, use double quotes for strings.
+quote-style = "single"
+
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = true
+
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"
+
+# Enable auto-formatting of code examples in docstrings. Markdown,
+# reStructuredText code/literal blocks and doctests are all supported.
+#
+# This is currently disabled by default, but it is planned for this
+# to be opt-out in the future.
+docstring-code-format = false
+
+# Set the line length limit used when formatting code snippets in
+# docstrings.
+#
+# This only has an effect when the `docstring-code-format` setting is
+# enabled.
+docstring-code-line-length = "dynamic"
+
+[lint.flake8-quotes]
+inline-quotes = "single"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,20 @@
 # Changelog
 
+## [2.0.0](https://github.com/blackm0re/etoolkit/tree/2.0.0) (2024-05-13)
+
+[Full Changelog](https://github.com/blackm0re/etoolkit/compare/1.2.0...2.0.0)
+
+**Changes:**
+
+- etoolkit encryption format v2, adding rnd. padding for values < 32 bytes
+
+- re-encryption support
+
+- replaced *os.system* with *subprocess*
+
+- new etoolkit.EtoolkitInstance API (not compatible with v1)
+
+
 ## [1.2.0](https://github.com/blackm0re/etoolkit/tree/1.2.0) (2022-04-04)
 
 [Full Changelog](https://github.com/blackm0re/etoolkit/compare/1.1.0...1.2.0)

diff --git a/README.md b/README.md
@@ -69,12 +69,97 @@ for processes that were not spawned by that same *etoolkit* session.
    # add sgs' custom repository using app-eselect/eselect-repository
    eselect repository add sgs
 
-   # ... or using layman (obsolete)
-   layman -a sgs
-
    emerge dev-python/etoolkit
    ```
 
+## Encryption & decryption scheme
+
+The etoolkit encryption format is currently at version 2.
+Encrypted values start with *enc-val$2$*.
+
+This new version introduces padding for values that are shorter than 32 bytes.
+The idea behind padding is to generate (32 - value length) random bytes and
+append them to the original value.
+That prevents a potential attacker from knowing the length of the encrypted
+short value (f.i. password, PIN number, username... etc).
+
+Values encrypted in the old format (*enc-val$1$*) can still be decrypted
+seamlessly.
+
+Authenticated encryption with associated data (AEAD) is implemented using
+AES-GCM.
+
+
+### Encryption
+
+Input:
+
+- plain-text value to be encrypted (P)
+
+- plain-text master-password used for key derivation (M)
+
+
+Output:
+
+- an encrypted value digest (base64) (B)
+
+
+Operation:
+
+- generate 32 bytes of random data to be used as a salt (S)
+
+- derive a 32 bytes key (K): K = scrypt(M, S, n=2**14, r=8, p=1)
+
+- use the first 12 bytes of S as nonce (NONCE)
+
+- calculate the padding length (L) as 32 - length of P, if P < 32, 0 otherwise
+
+- set the padding length bytes (N) (2bytes) to "%02d", if L > 0, "-1" otherwise
+
+- generate L bytes of random data to be used for padding (D)
+
+- encrypt and auth. P, auth.only S (E): E = AES_GCM_ENC(K, NONCE, N + P + D, S)
+
+- encrypted value digest (B) = enc-val$2$:BASE64_ENCODE(S)$BASE64_ENCODE(E)
+
+example:
+enc-val$2$uYpZM1VfAGq0CDZL2duITs076CQj+hIFEgx+F4mn80o=$UWP5YeRsh5/2vZ2J1UOS+BJti73Kbp6C1pJmCo8hFSujpe35X/XpzBegJJpo86AiCsNsUS6B6JM=
+
+
+### Decryption
+
+Input:
+
+- encrypted value digest (base64) (B)
+
+- plain-text master-password used for key derivation (M)
+
+
+Output:
+
+- plain-text password (P)
+
+
+Operation:
+
+- remove the prefix (enc-val$2$) from B and split the remaining value by '$'
+
+- base64-decode the salt (S): S = BASE64_DECODE(B1)
+
+- base64-decode the rest of the data (E): E = BASE64_DECODE(B2)
+
+- derive a 32 bytes key (K): K = scrypt(M, S, n=2**14, r=8, p=1)
+
+- use the first 12 bytes of S as nonce (NONCE)
+
+- decrypt the encrypted data (D): D = AES_GCM_DECRYPT(K, NONCE, E, S)
+
+- fetch the first 2 bytes (padding length bytes) (N): N = D[0 : 2]
+
+- calculate the padding length (L): L = INT(N) if N != "-1", 0 otherwise
+
+- fetch the plain-text (P): P = D[2 : -L] if L != 0, D[2 :] otherwise
+
 
 ## Setup and examples
 
@@ -201,6 +286,17 @@ One can also spawn a different process than an interactive shell by using the
    etoolkit --spawn /bin/othershell <instance-name>
    ```
 
+It is possible to re-encrypt all encrypted values in a specific instance or in
+all defined instances either by using the same or a new master password.
+
+   ```bash
+   etoolkit --reencrypt all
+   ```
+
+will prompt for the current master password, then for a new master password
+(with confirmation) and finally the new config file (if "all") or instance
+contents will be displayed.
+
 Contact the author for questions and suggestions! :)
 
 
@@ -223,11 +319,11 @@ or the *instances* structure being loaded from a diferent configuration file
 
 
    # using some static methods in order to create encrypted values
-   etoolkit.EtoolkitInstance.encrypt('the very secret passwd', 'secret1')
-   # Out: 'enc-val$1$Y/TBb1F3siHTw6qZg9ERzZfA8PLPf2CwGSQLpu9jYWw=$FT5tS9o+ABvsxogIXpJim16Gz5SVtV8='
+   etoolkit.EtoolkitInstance.encrypt('The very secret passwd', 'secret1')
+   # Out: 'enc-val$2$NDdp6WMbX7gdEyzGM5nI4jhyer4XL+BoQwAHtL2CXHw=$+Pztn1pfaXKjPpem5PIQrCNxR9pyE6zqgSoGg9qXvmhH6VsNQvUTmiaOvUFl35EbiYE='
 
-   etoolkit.EtoolkitInstance.encrypt('the very secret passwd', 'secret2')
-   # Out: 'enc-val$1$vIBcoCNiYrsDLtF41uLuSEnppBjhliD0B8jwcBJcj/c=$KwOGe/y1dlxktDaCnJPIVNuaQ4Q7yNo='
+   etoolkit.EtoolkitInstance.encrypt('The very secret passwd', 'secret2')
+   # Out: 'enc-val$2$H953GxW+qrYXIp+I97lJBmG1gv89wxcfmTu7PEpZzjE=$Tb3F8/izDbHAMklpIjYk73JAiav+w8ZhrMsO93FlQjGh4MTChjp2Yen5BxSBOWLvCD4='
 
 
    # The encrypted values will be used in our configuration structure
@@ -237,34 +333,33 @@ or the *instances* structure being loaded from a diferent configuration file
        },
        "instances": {
            "_default": {
-               "ETOOLKIT_PROMPT": "(%i)",
-               "PYTHONPATH": "/home/user/%i/python",
+             "ETOOLKIT_PROMPT": "(%i)",
+             "ETOOLKIT_SENSITIVE": ["DB_CONNECTION", "ETOOLKIT_TEST_PASSWORD"]
            },
            "dev": {
                "ETOOLKIT_PARENT": "_default",
-               "PYTHONPATH": "%p:/home/user/%i/.pythonpath",
+               "PYTHONPATH": ":/home/user/.pythonpath",
+               "DB_CONNECTION": "enc-val$2$RAgDei59tUvDAkrBmxROqRaV/NxNFEI2eJIOP7sG/b8=$yse7zawHCzQCU31sZj4oJYLGonz1M7oqHqCilXLHkywa9nMPALypmVzi3QekekYuLeb5XVTmmp84NHoPn1M052otoRHSp+TMPsqBPRabfriIKEK4XQ=="
            },
            "secret": {
                "ETOOLKIT_PARENT": "_default",
-               "ETOOLKIT_SENSITIVE": ["PASSWORD"],
                "GNUPGHOME": "%h/private/.gnupg",
-               "PASSWORD": "enc-val$1$vIBcoCNiYrsDLtF41uLuSEnppBjhliD0B8jwcBJcj/c=$KwOGe/y1dlxktDaCnJPIVNuaQ4Q7yNo="
+               "ETOOLKIT_TEST_PASSWORD": "enc-val$2$RCSZqq9pWrRDoCVYVHopyu1LzaJGfv8roVviqrLTBxM=$+YYrZbwTBuG0Pl+WMQrvxLUtq5j8qYuQqzoIwgoGt7AaWZCJz+E7qoDeg3wke70ST8U="
            }
        }
    }
 
-
-   dev_instance = etoolkit.EtoolkitInstance('dev', instances)
+   secret_instance = etoolkit.EtoolkitInstance('secret', instances)
 
    # fetch the variables before the processing stage (calling get_environ())
    # since raw_env_variables is a dict, it can be modified (f.i. .update())
-   dev_instance.raw_env_variables
+   secret_instance.raw_env_variables
 
-   dev_instance.master_password = 'the very secret passwd'  # or perhaps using getpass
-   env_vars = dev_instance.get_env()
-   print(env_vars['PASSWORD'])  # outputs: 'secret2'
+   secret_instance.master_password = 'The very secret passwd'  # or perhaps using getpass
+   env_vars = secret_instance.get_environ()
+   print(env_vars['ETOOLKIT_TEST_PASSWORD'])  # outputs: 'secret1'
    
-   inst.dump_env(env_vars)  # prints all values, with the exception of 'PASSWORD'
+   secret_instance.env_to_str(env_vars)  # prints all values, with the exception of 'ETOOLKIT_TEST_PASSWORD'
 
    # set the env. variables.
    os.environ.update(env_vars)

diff --git a/completion/etoolkit.bash b/completion/etoolkit.bash
@@ -30,7 +30,7 @@ _etoolkit() {
     all_params="-d --decrypt-value -e --encrypt-value -l --list -h --help
                 -P --master-password-prompt -p --generate-master-password-hash
                 -c --config-file -E --echo -m --multiple-values -q --no-output
-                -s --spawn -v --version"
+                -r --reencrypt -s --spawn -v --version"
     # if [ ${prev:0:1} == "-" ]
 
     if [ ${COMP_CWORD} -eq 1 ]; then
@@ -78,6 +78,11 @@ _etoolkit() {
             COMPREPLY=($(compgen -W "-d --decrypt-value -E -e --echo --encrypt-value -m --multiple-values" -- "$cur"))
             return
             ;;
+        "-r" | "--reencrypt")
+            COMPREPLY=($(compgen -W "all" -- "$cur"))
+            _instances "$cur"
+            return
+            ;;
         "-s" | "--spawn")
             COMPREPLY=($(compgen -c -- "$cur"))
             return

diff --git a/etoolkit_sample.json b/etoolkit_sample.json
@@ -1,21 +1,21 @@
 {
     "general": {
-        "MASTER_PASSWORD_HASH": "pbkdf2_sha256$100000$kFOQkAPtStZ/Ny/O4501ygHGQnqh5Y+ySxF9qVHriv8=$3BujuWzn3CfDnw4yiD9m3F+GjeW1MHHW40R/ThHNcn0="
+        "MASTER_PASSWORD_HASH": "pbkdf2_sha256$500000$UY3o78KUM1Btzxk3k3JCsijnwtJ2lx+hH9NewpVKxo8=$tHwDm8OVKanC4DoYTigTCb0R3lQIa/CbBYj0B3TZtHg="
     },
     "instances": {
-        "default": {
-            "ETOOLKIT_PROMPT": "(%i)"
+        "_default": {
+          "ETOOLKIT_PROMPT": "(%i)",
+          "ETOOLKIT_SENSITIVE": ["DB_CONNECTION", "ETOOLKIT_TEST_PASSWORD"]
         },
         "dev": {
-            "ETOOLKIT_PARENT": "default",
+            "ETOOLKIT_PARENT": "_default",
             "PYTHONPATH": ":/home/user/.pythonpath",
-            "DB_CONNECTION": "enc-val$1$Y/TBb1F3siHTw6qZg9ERzZfA8PLPf2CwGSQLpu9jYWw=$FT5tS9o+ABvsxogIXpJim16Gz5SVtV8="
+            "DB_CONNECTION": "enc-val$2$RAgDei59tUvDAkrBmxROqRaV/NxNFEI2eJIOP7sG/b8=$yse7zawHCzQCU31sZj4oJYLGonz1M7oqHqCilXLHkywa9nMPALypmVzi3QekekYuLeb5XVTmmp84NHoPn1M052otoRHSp+TMPsqBPRabfriIKEK4XQ=="
         },
         "secret": {
-            "ETOOLKIT_PARENT": "default",
-            "ETOOLKIT_SENSITIVE": ["PASSWORD"],
+            "ETOOLKIT_PARENT": "_default",
             "GNUPGHOME": "%h/private/.gnupg",
-            "PASSWORD": "enc-val$1$vIBcoCNiYrsDLtF41uLuSEnppBjhliD0B8jwcBJcj/c=$KwOGe/y1dlxktDaCnJPIVNuaQ4Q7yNo="
+            "ETOOLKIT_TEST_PASSWORD": "enc-val$2$RCSZqq9pWrRDoCVYVHopyu1LzaJGfv8roVviqrLTBxM=$+YYrZbwTBuG0Pl+WMQrvxLUtq5j8qYuQqzoIwgoGt7AaWZCJz+E7qoDeg3wke70ST8U="
         }
     }
 }
diff --git a/src/etoolkit/__init__.py b/src/etoolkit/__init__.py
@@ -18,7 +18,7 @@
 from .etoolkit import EtoolkitInstance, EtoolkitInstanceError
 
 __author__ = 'Simeon Simeonov'
-__version__ = '1.3.0'
+__version__ = '2.0.0'
 __license__ = 'GPL3'