Fix for App Builder and other updates (#14)

* APP-3915 - [CONFIG] Added validation to ensure displayPath is always in the install.json for API Services
* APP-4060 - [CLI] Updated proxy inputs to use environment variables
* APP-4077 - [SPEC-TOOL] Updated spec-tool to create an example app_input.py file and to display a mismatch report
* APP-4112 - [CONFIG] Updated config submodule (tcex.json model) to support legacy App Builder Apps
* APP-4113 - [CONFIG] Updated App Spec model to normalize App features

pyproject.toml

@@ -29,7 +29,7 @@ dependencies = [
   "pydantic<2.0.0",
   "python-dateutil",
   "PyYAML",
-  "redis",
+  "redis<5.0.0",
   "requests",
   "rich",
   "semantic_version",
@@ -121,6 +121,7 @@ disable = [
   "too-many-arguments",
   "too-many-branches",
   "too-many-instance-attributes",
+  "too-many-lines",
   "too-many-locals",
   "too-many-public-methods",
   "too-many-return-statements",

release_notes.md

@@ -0,0 +1,20 @@
+# Release Notes
+
+### 1.0.1
+
+-   APP-3915 - [CONFIG] Added validation to ensure displayPath is always in the install.json for API Services
+-   APP-4060 - [CLI] Updated proxy inputs to use environment variables
+-   APP-4077 - [SPEC-TOOL] Updated spec-tool to create an example app_input.py file and to display a mismatch report
+-   APP-4112 - [CONFIG] Updated config submodule (tcex.json model) to support legacy App Builder Apps
+-   APP-4113 - [CONFIG] Updated App Spec model to normalize App features
+
+
+### 1.0.0
+
+-   APP-3926 - Split CLI module of TcEx into tcex-cli project
+-   APP-3912 - [CLI] Updated `tcex` command to use "project.scripts" setting in pyproject.toml
+-   APP-3913 - [DEPS] Updated `deps` command to build **"deps"** or **"lib_"** depending on TcEx version
+-   APP-3819 - [LIST] Updated the `list` to choose the appropriate template branch depending on TcEx version
+-   APP-3820 - [DEPS] Updated the `deps` to choose the appropriate template branch depending on TcEx version
+-   APP-4053 - [CLI] Updated CLI tools to work with changes in App Builder released in ThreatConnect version 7.2
+-   APP-4059 - [CLI] Added proxy support to commands where applicable

tcex_cli/__metadata__.py

@@ -1,3 +1,3 @@
 """TcEx Framework metadata."""
 __license__ = 'Apache-2.0'
-__version__ = '1.0.0'
+__version__ = '1.0.1'

tcex_cli/cli/cli_abc.py

@@ -12,6 +12,7 @@
 
 # first-party
 from tcex_cli.app.app import App
+from tcex_cli.input.field_type.sensitive import Sensitive
 from tcex_cli.registry import registry
 from tcex_cli.util import Util
 
@@ -40,6 +41,30 @@ def __init__(self):
         # register commands
         registry.add_service(App, self.app)
 
+    def _process_proxy_host(self, proxy_host: str | None) -> str | None:
+        """Process proxy host."""
+        os_proxy_host = os.getenv('TC_PROXY_HOST')
+        return proxy_host if proxy_host else os_proxy_host
+
+    def _process_proxy_pass(self, proxy_pass: Sensitive | str | None) -> Sensitive | None:
+        """Process proxy password."""
+        os_proxy_pass = os.getenv('TC_PROXY_PASS') or os.getenv('TC_PROXY_PASSWORD')
+        proxy_pass = proxy_pass if proxy_pass else os_proxy_pass
+        if proxy_pass is not None and not isinstance(proxy_pass, Sensitive):
+            return Sensitive(proxy_pass)
+        return proxy_pass
+
+    def _process_proxy_port(self, proxy_port: int | str | None) -> int | None:
+        """Process proxy port."""
+        os_proxy_port = os.getenv('TC_PROXY_PORT')
+        port = proxy_port if proxy_port else os_proxy_port
+        return int(port) if port is not None else None
+
+    def _process_proxy_user(self, proxy_user: str | None) -> str | None:
+        """Process proxy user."""
+        os_proxy_user = os.getenv('TC_PROXY_USER') or os.getenv('TC_PROXY_USERNAME')
+        return proxy_user if proxy_user else os_proxy_user
+
     @cached_property
     def app(self) -> App:
         """Return instance of App."""

tcex_cli/cli/deploy/deploy.py

@@ -38,6 +38,12 @@ def command(
     * TC_API_PATH
     * TC_API_ACCESS_ID
     * TC_API_SECRET_KEY
+
+    Optional environment variables include:\n
+    * PROXY_HOST\n
+    * PROXY_PORT\n
+    * PROXY_USER\n
+    * PROXY_PASS\n
     """
     cli = DeployCli(
         server,

tcex_cli/cli/deploy/deploy_cli.py

@@ -31,16 +31,12 @@ def __init__(
         self._app_file = app_file
         self.allow_all_orgs = allow_all_orgs
         self.allow_distribution = allow_distribution
-        self.proxy_host = proxy_host
-        self.proxy_port = proxy_port
-        self.proxy_user = proxy_user
+        self.proxy_host = self._process_proxy_host(proxy_host)
+        self.proxy_port = self._process_proxy_port(proxy_port)
+        self.proxy_user = self._process_proxy_user(proxy_user)
         self.proxy_pass = self._process_proxy_pass(proxy_pass)
         self.server = server
 
-    def _process_proxy_pass(self, proxy_pass: str | None) -> Sensitive | None:
-        """Process proxy password."""
-        return None if proxy_pass is None else Sensitive(proxy_pass)
-
     def _check_file(self):
         """Return True if file exists."""
         if self._app_file and not os.path.isfile(self._app_file):
@@ -110,7 +106,13 @@ def deploy_app(self):
             )
 
         else:
-            response_data = response.json()[0]
+            try:
+                response_data = response.json()[0]
+            except IndexError as err:
+                Render.panel.failure(
+                    f'Unexpected response from ThreatConnect API. Failed to deploy App: {err}'
+                )
+
             Render.table.key_value(
                 'Successfully Deployed App',
                 {

tcex_cli/cli/deps/deps.py

@@ -37,7 +37,14 @@ def command(
     proxy_user: StrOrNone = typer.Option(None, help='(Advanced) Username for the proxy server.'),
     proxy_pass: StrOrNone = typer.Option(None, help='(Advanced) Password for the proxy server.'),
 ):
-    """Install dependencies defined in the requirements.txt file."""
+    r"""Install dependencies defined in the requirements.txt file.
+
+    Optional environment variables include:\n
+    * PROXY_HOST\n
+    * PROXY_PORT\n
+    * PROXY_USER\n
+    * PROXY_PASS\n
+    """
     cli = DepsCli(
         app_builder,
         branch,

tcex_cli/cli/deps/deps_cli.py

@@ -7,7 +7,7 @@
 import sys
 from functools import cached_property
 from pathlib import Path
-from urllib.parse import quote, urlsplit
+from urllib.parse import quote
 
 # third-party
 from semantic_version import Version
@@ -41,17 +41,10 @@ def __init__(
         self.branch = branch
         self.no_cache_dir = no_cache_dir
         self.pre = pre
-        self.proxy_host = proxy_host
-        self.proxy_port = proxy_port
-        self.proxy_user = proxy_user
-        self.proxy_pass = proxy_pass
-
-        if not self.proxy_host and os.environ.get('https_proxy'):
-            parsed_proxy_url = urlsplit(os.environ.get('https_proxy'))
-            self.proxy_host = parsed_proxy_url.hostname
-            self.proxy_port = parsed_proxy_url.port
-            self.proxy_user = parsed_proxy_url.username
-            self.proxy_pass = parsed_proxy_url.password
+        self.proxy_host = self._process_proxy_host(proxy_host)
+        self.proxy_port = self._process_proxy_port(proxy_port)
+        self.proxy_user = self._process_proxy_user(proxy_user)
+        self.proxy_pass = self._process_proxy_pass(proxy_pass)
 
         # properties
         self.deps_dir_tests = self.app_path / 'deps_tests'
@@ -124,13 +117,16 @@ def configure_proxy(self):
 
         if self.proxy_host is not None and self.proxy_port is not None:
             # proxy url without auth
+            proxy_pass_ = None
+            if self.proxy_pass is not None and hasattr(self.proxy_pass, 'value'):
+                proxy_pass_ = self.proxy_pass.value
             proxy_url = f'{self.proxy_host}:{self.proxy_port}'
-            if self.proxy_user is not None and self.proxy_pass is not None:
+            if self.proxy_user is not None and proxy_pass_ is not None:
                 proxy_user = quote(self.proxy_user, safe='~')
-                proxy_pass = quote(self.proxy_pass, safe='~')
+                proxy_pass_ = quote(proxy_pass_, safe='~')
 
                 # proxy url with auth
-                proxy_url = f'{proxy_user}:{proxy_pass}@{proxy_url}'
+                proxy_url = f'{proxy_user}:{proxy_pass_}@{proxy_url}'
 
             # update proxy properties
             self.proxy_enabled = True

tcex_cli/cli/package/package_cli.py

@@ -132,9 +132,12 @@ def package(self):
         # IMPORTANT:
         # The name of the folder in the zip is the *key* for an App. This
         # value must remain consistent for the App to upgrade successfully.
-        app_name_version = (
-            f'{self.app.tj.model.package.app_name}_{self.app.ij.model.package_version}'
-        )
+        # Normal behavior should be to use the major version with a "v" prefix.
+        # However, some older Apps got released with a non-standard version
+        # (e.g., v2.0). For these Apps the version can be overridden by defining
+        # the "package.app_version" field in the tcex.json file.
+        app_version = self.app.tj.model.package.app_version or self.app.ij.model.package_version
+        app_name_version = f'{self.app.tj.model.package.app_name}_{app_version}'
 
         # build app directory
         app_path_fqpn = self.build_fqpn / app_name_version

tcex_cli/cli/run/launch_abc.py

@@ -167,7 +167,6 @@ def redis_client(self) -> redis.Redis:
             )
         )
 
-    # TODO: [bcs] fix model name :(
     @cached_property
     def session(self) -> TcSession:
         """Return requests Session object for TC admin account."""

tcex_cli/cli/run/run.py

@@ -24,6 +24,10 @@ def command(
     try:
         cli.update_system_path()
 
+        # validate config.json
+        if not config_json.is_file():
+            Render.panel.failure(f'Config file not found [{config_json}]')
+
         # run in debug mode
         if debug is True:
             cli.debug(debug_port)