From 398d66f60162b71ca41b58add87f9cdcd36f8244 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 17:37:32 -0500 Subject: [PATCH 01/27] Rename __main__.py to app.py --- sdx_lc/{__main__.py => app.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sdx_lc/{__main__.py => app.py} (100%) diff --git a/sdx_lc/__main__.py b/sdx_lc/app.py similarity index 100% rename from sdx_lc/__main__.py rename to sdx_lc/app.py From d14b1bb8f51bc81bdf0a8b00370744aee810136a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 17:38:32 -0500 Subject: [PATCH 02/27] Ignore all .env* files --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3daaade..c9aac75 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ test-db # Ignore copies of env env.* -.env +.env* # Ignore files written by code coverage tools .coverage From 3b6f0f0e3737ffa7981dad1171be04ea60f3e721 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 18:25:11 -0500 Subject: [PATCH 03/27] Create a flask app --- sdx_lc/app.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 97821f7..1900e7c 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -51,7 +51,10 @@ def start_pull_topology_change(): call(["python", "sdx_lc/jobs/pull_topo_changes.py"]) -def main(): +def create_app(): + """ + Create a Flas/Connexion App. + """ if LOG_FILE: logging.basicConfig(filename=LOG_FILE, level=logging.INFO) else: @@ -78,6 +81,9 @@ def main(): thread_queue = Queue() start_consumer(thread_queue, db_instance) + return app.app + +app = create_app() if __name__ == "__main__": - main() + app.run() From f3e1873a66cfd1451e2c57eec3ea92ae58ad1d5c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 18:26:53 -0500 Subject: [PATCH 04/27] Disable thread This is unnecessary if we're using WSGI or even flask --- sdx_lc/app.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 1900e7c..47fa5e2 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -65,11 +65,6 @@ def create_app(): app.app.json_encoder = encoder.JSONEncoder app.add_api("swagger.yaml", arguments={"title": "SDX LC"}, pythonic_params=True) - # Run swagger in a thread - threading.Thread( - target=lambda: app.run(port=os.getenv("SDXLC_PORT") or 8080) - ).start() - # Start pulling topology changes from domain controller in a # separate subprocess processThread = threading.Thread(target=start_pull_topology_change) From 0f8f4cfb3bcf619b71f155350dd93925f160283b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 18:27:20 -0500 Subject: [PATCH 05/27] Use SDXLC_PORT only when launching off main --- sdx_lc/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 47fa5e2..8316093 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -81,4 +81,4 @@ def create_app(): app = create_app() if __name__ == "__main__": - app.run() + app.run(port=os.getenv("SDXLC_PORT") or 8080) From 3b075822bb39a97afe8931fbdf843c5fb55b8d02 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 18:29:10 -0500 Subject: [PATCH 06/27] Redirect to Swagger UI from / --- sdx_lc/app.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 8316093..a07687f 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -10,6 +10,7 @@ from subprocess import call import connexion +from flask import redirect from sdx_lc import encoder from sdx_lc.messaging.topic_queue_consumer import * @@ -80,5 +81,9 @@ def create_app(): app = create_app() +@app.route("/", methods=["GET"]) +def index(): + return redirect("/SDX-LC/2.0.0/ui/") + if __name__ == "__main__": app.run(port=os.getenv("SDXLC_PORT") or 8080) From f8c0b454838a501135fa8dccf58695f77c5ef74f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 18:37:48 -0500 Subject: [PATCH 07/27] Also set up an ASGI app for good measure --- sdx_lc/app.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index a07687f..5742da8 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -10,6 +10,7 @@ from subprocess import call import connexion +from asgiref.wsgi import WsgiToAsgi from flask import redirect from sdx_lc import encoder @@ -81,6 +82,13 @@ def create_app(): app = create_app() +# We use WsgiToAsgi adapter so that we can use an ASGI server (such as +# uvicorn or hypercorn), like so: +# +# $ uvicorn sdx_lc.app:asgi_app --host 0.0.0.0 --port 8080 +# +asgi_app = WsgiToAsgi(app) + @app.route("/", methods=["GET"]) def index(): return redirect("/SDX-LC/2.0.0/ui/") From 2642981c41a45ef7faf1928f42b765f35a1e201f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 18:41:33 -0500 Subject: [PATCH 08/27] Document running with gunicorn or flask --- sdx_lc/app.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 5742da8..848a137 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -80,6 +80,14 @@ def create_app(): return app.app +# We can run the app using flask, like so: +# +# $ flask --app sdx_lc.app:app run --debug +# +# Or with an WSGI server such as gunicorn: +# +# $ gunicorn --bind localhost:$SDXLC_PORT sdx_lc.app:app +# app = create_app() # We use WsgiToAsgi adapter so that we can use an ASGI server (such as From cf2ea5b476f3733529db34598fcaa9e45558f077 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 22:05:00 -0500 Subject: [PATCH 09/27] Add a comment about redirect --- sdx_lc/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 848a137..fc87136 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -97,6 +97,7 @@ def create_app(): # asgi_app = WsgiToAsgi(app) +# Set up a redirect for convenience. @app.route("/", methods=["GET"]) def index(): return redirect("/SDX-LC/2.0.0/ui/") From 29ee5f7762abd826c7faad1795c742b2db174625 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 22:06:21 -0500 Subject: [PATCH 10/27] Remove unused variables --- sdx_lc/app.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index fc87136..a4114ed 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -1,12 +1,9 @@ #!/usr/bin/env python3 -import argparse import json import logging import os import threading -import time -from optparse import OptionParser from subprocess import call import connexion From 12ca27fdc59c4df35c73d1706755ce15c619b143 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 22:07:15 -0500 Subject: [PATCH 11/27] Remove unused variable --- sdx_lc/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index a4114ed..071885d 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -22,7 +22,7 @@ def is_json(myjson): try: json.loads(myjson) - except ValueError as e: + except ValueError: return False return True From 8513433d04b944ae140d5349355b146a256a0857 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 22:08:29 -0500 Subject: [PATCH 12/27] Add asgiref dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 93f7d9b..5a04d72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "setuptools >= 21.0.0", "pika >= 1.2.0", "connexion[swagger-ui] == 2.14.2", + "asgiref >= 3.7.2", "pymongo > 3.0", ] From 92720c0576a06fb32aa7a288ea8ca72b2763bd8b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 22:09:15 -0500 Subject: [PATCH 13/27] Add uvicorn as an optional dependency --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 5a04d72..3acaece 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,9 @@ format = [ "black == 24.*", "isort == 5.*", ] +wsgi = [ + "uvicorn" +] [options.packages.find] where = "sdx_lc" From 94b7069965515fcddad854279deb37857579140e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 22:17:04 -0500 Subject: [PATCH 14/27] Reformat --- sdx_lc/app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 071885d..317135b 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -77,6 +77,7 @@ def create_app(): return app.app + # We can run the app using flask, like so: # # $ flask --app sdx_lc.app:app run --debug @@ -94,10 +95,12 @@ def create_app(): # asgi_app = WsgiToAsgi(app) + # Set up a redirect for convenience. @app.route("/", methods=["GET"]) def index(): return redirect("/SDX-LC/2.0.0/ui/") + if __name__ == "__main__": app.run(port=os.getenv("SDXLC_PORT") or 8080) From 320b0b1d35bd31c4b5b5a76e918a2078fe6bf026 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 22:17:13 -0500 Subject: [PATCH 15/27] Run with uvicorn in container --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 67fb518..7bd4975 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,10 +5,9 @@ WORKDIR /usr/src/app COPY . /usr/src/app -RUN pip3 install --no-cache-dir . +RUN pip3 install --no-cache-dir .[wsgi] EXPOSE 8080 ENTRYPOINT ["python3"] - -CMD ["-m", "sdx_lc"] +CMD ["-m", "uvicorn", "sdx_lc.app:asgi_app", "--host", "0.0.0.0", "--port", "8080"] From 0581f83db2b9cc52d77e2e811959128ec7c93421 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Jun 2024 22:22:26 -0500 Subject: [PATCH 16/27] Remove stray ` --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 52b9ae0..28cc9cd 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,6 @@ $ source ./venv/bin/activate $ pip3 install --editable .[test] $ pytest ``` -` ## Communication between SDX Controller and Local Controller From ca841f754610766e598c538dda9d8f03bb8164c9 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 18 Jun 2024 09:47:51 -0500 Subject: [PATCH 17/27] Install in a virtual environment in the container --- Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7bd4975..97cc9fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,13 @@ WORKDIR /usr/src/app COPY . /usr/src/app +# create a venv. +RUN python3 -m venv /opt/venv --upgrade-deps + +# Make sure we use the venv. +ENV PATH="/opt/venv/bin:$PATH" +ENV VIRTUAL_ENV="/opt/venv" + RUN pip3 install --no-cache-dir .[wsgi] EXPOSE 8080 From a00cbd20cd4113e740621be2b33ae120a855b9d3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 18 Jun 2024 09:50:01 -0500 Subject: [PATCH 18/27] Ignore haddolint suggestion for now --- .hadolint.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .hadolint.yml diff --git a/.hadolint.yml b/.hadolint.yml new file mode 100644 index 0000000..db5f22d --- /dev/null +++ b/.hadolint.yml @@ -0,0 +1,7 @@ +ignored: + # DL3013 recommends that Python dependencies be pinned, but we + # probably do not want to do that right now. Unpinned dependencies + # has been working okay for us so far, and pinning (using pip-tools + # would be ideal) should be done separately after a discussion, + # because that would change the workflow a bit. + - DL3013 From 67a6b70d253f1bdf72ef507b6dce23fc9a7991d0 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 18 Jun 2024 10:29:30 -0500 Subject: [PATCH 19/27] Remove unused variables --- sdx_lc/app.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 317135b..6095b5a 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -36,9 +36,6 @@ def start_consumer(thread_queue, db_instance): function. """ - MESSAGE_ID = 0 - HEARTBEAT_ID = 0 - rpc = TopicQueueConsumer(thread_queue=thread_queue, exchange_name="connection") t1 = threading.Thread(target=rpc.start_consumer, args=()) t1.start() From 6282da3bc74ac3a488b812711f82e3ffcd25f922 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 18 Jun 2024 10:30:35 -0500 Subject: [PATCH 20/27] Remove unused function --- sdx_lc/app.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 6095b5a..d849692 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import json import logging import os import threading @@ -19,14 +18,6 @@ LOG_FILE = os.environ.get("LOG_FILE") -def is_json(myjson): - try: - json.loads(myjson) - except ValueError: - return False - return True - - def start_consumer(thread_queue, db_instance): """ Accept connection (also called link) messages from SDX Controller. From 71ab5a352b58a9d26ba3ae4e577ac2fd6981d6e5 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 18 Jun 2024 10:32:42 -0500 Subject: [PATCH 21/27] Avoid wildcard imports --- sdx_lc/app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index d849692..5c15c2c 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -3,6 +3,7 @@ import logging import os import threading +from queue import Queue from subprocess import call import connexion @@ -10,8 +11,8 @@ from flask import redirect from sdx_lc import encoder -from sdx_lc.messaging.topic_queue_consumer import * -from sdx_lc.utils.db_utils import * +from sdx_lc.messaging.topic_queue_consumer import TopicQueueConsumer +from sdx_lc.utils.db_utils import DbUtils logger = logging.getLogger(__name__) logging.getLogger("pika").setLevel(logging.WARNING) From cf05cb125f7ad58cd97ad8c3f613e89281e867e0 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 18 Jun 2024 10:47:54 -0500 Subject: [PATCH 22/27] Localize logging setup --- sdx_lc/app.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 5c15c2c..c240245 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -14,10 +14,6 @@ from sdx_lc.messaging.topic_queue_consumer import TopicQueueConsumer from sdx_lc.utils.db_utils import DbUtils -logger = logging.getLogger(__name__) -logging.getLogger("pika").setLevel(logging.WARNING) -LOG_FILE = os.environ.get("LOG_FILE") - def start_consumer(thread_queue, db_instance): """ @@ -43,8 +39,14 @@ def create_app(): """ Create a Flas/Connexion App. """ - if LOG_FILE: - logging.basicConfig(filename=LOG_FILE, level=logging.INFO) + + logger = logging.getLogger(__name__) + logging.getLogger("pika").setLevel(logging.WARNING) + + log_file = os.environ.get("LOG_FILE") + + if log_file: + logging.basicConfig(filename=log_file, level=logging.INFO) else: logging.basicConfig(level=logging.INFO) From 40410fd4004a613b1e876103fbdfa199fc33ca0a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 18 Jun 2024 11:03:30 -0500 Subject: [PATCH 23/27] Update instructions on running the app --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 28cc9cd..2a9a54d 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ example -- see above), and then run the server: ```console $ source .env -$ python3 -m sdx_lc +$ flask --app sdx_lc.app:app run --debug ``` From 97d26caa2d3080e97c8ba5fa864db8dfdfe83455 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 18 Jun 2024 11:08:04 -0500 Subject: [PATCH 24/27] Add a comment --- sdx_lc/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index c240245..6565abb 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -63,6 +63,8 @@ def create_app(): # Get DB connection and tables set up. db_instance = DbUtils() db_instance.initialize_db() + + # Consume connection/link messages thread_queue = Queue() start_consumer(thread_queue, db_instance) From 695f14a2a4ed8fb4032304b85a5ad1cac0646cb8 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 18 Jun 2024 11:13:03 -0500 Subject: [PATCH 25/27] Log name and domain --- sdx_lc/app.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 6565abb..60423df 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -43,13 +43,19 @@ def create_app(): logger = logging.getLogger(__name__) logging.getLogger("pika").setLevel(logging.WARNING) - log_file = os.environ.get("LOG_FILE") + log_file = os.getenv("LOG_FILE") if log_file: logging.basicConfig(filename=log_file, level=logging.INFO) else: logging.basicConfig(level=logging.INFO) + logger.info( + f"SDX Local Controller starting up (" + f"name: {os.getenv('SDXLC_NAME')}, " + f"domain: {os.getenv('SDXLC_DOMAIN')})" + ) + # Run swagger service app = connexion.App(__name__, specification_dir="./swagger/") app.app.json_encoder = encoder.JSONEncoder From 58412e8f3b22afc734d502f0647cb646ef93045a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 18 Jun 2024 11:15:55 -0500 Subject: [PATCH 26/27] Update comment --- sdx_lc/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 60423df..636395a 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -90,7 +90,7 @@ def create_app(): # We use WsgiToAsgi adapter so that we can use an ASGI server (such as # uvicorn or hypercorn), like so: # -# $ uvicorn sdx_lc.app:asgi_app --host 0.0.0.0 --port 8080 +# $ uvicorn sdx_lc.app:asgi_app --host 0.0.0.0 --port $SDXLC_PORT # asgi_app = WsgiToAsgi(app) From d8a5169f22969e3607fa76a6beef5261239b243d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 18 Jun 2024 13:52:00 -0500 Subject: [PATCH 27/27] Run the app using a helper script All so that uvicorn will use and print the same port that we specified with SDXLC_PORT when running `docker compose up`. --- Dockerfile | 5 +---- entrypoint.sh | 10 ++++++++++ 2 files changed, 11 insertions(+), 4 deletions(-) create mode 100755 entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 97cc9fd..9cd7673 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,4 @@ ENV VIRTUAL_ENV="/opt/venv" RUN pip3 install --no-cache-dir .[wsgi] -EXPOSE 8080 - -ENTRYPOINT ["python3"] -CMD ["-m", "uvicorn", "sdx_lc.app:asgi_app", "--host", "0.0.0.0", "--port", "8080"] +CMD ["./entrypoint.sh"] diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..680265b --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,10 @@ +#! /bin/sh + +# A helper script that can be executed off docker, just so that +# uvicorn will print the right port number. Docker will not do +# environment variable substitution in CMD lines, so this script +# became necessary. + +set -eu + +python3 -m uvicorn sdx_lc.app:asgi_app --host 0.0.0.0 --port "$SDXLC_PORT"