From 76950a1756ad2ad9acf52bf2d558d1245202fa62 Mon Sep 17 00:00:00 2001 From: Rene Dohmen Date: Mon, 14 Oct 2024 10:45:28 +0200 Subject: [PATCH 1/6] 1316: Adds document describing both frontend and backend auth setup (#742) * 1316: Adds document describing both frontend and backend auth setup * 1316: Apply pr suggestions and merge auth documents --------- Co-authored-by: Ruben van Leeuwen --- .../auth-backend-and-frontend.md | 278 +++++++++++ docs/reference-docs/auth.md | 122 ----- mkdocs.yml | 440 +++++++++--------- 3 files changed, 498 insertions(+), 342 deletions(-) create mode 100644 docs/reference-docs/auth-backend-and-frontend.md delete mode 100644 docs/reference-docs/auth.md diff --git a/docs/reference-docs/auth-backend-and-frontend.md b/docs/reference-docs/auth-backend-and-frontend.md new file mode 100644 index 000000000..ddeb7548e --- /dev/null +++ b/docs/reference-docs/auth-backend-and-frontend.md @@ -0,0 +1,278 @@ +# Authentication and authorization + +The `Orchestrator-Core` application incorporates a robust security framework, utilizing OpenID Connect (OIDC) for authentication and Open Policy Agent (OPA) for authorization. This flexible system ensures secure access, allowing you to tailor the authorization components to best fit your application's specific requirements. + +WFO can be run with or without authentication. With authentication turned on authorization logic can be provided that uses - for example - user privileges to allow further access to resources. Authentication is configured using ENV variables. The frontend and backend have their own set of ENV variables and logic to be implemented to run auth(n/z). + +Note: With authentication enabled on the backend the frontend has to have authentication enabled as well. When the frontend has authentication enabled it is possible to run a backend without authentication. Please note the limitations of frontend authentication and authorization mentioned in a note under frontend authentication. + +## Definitions + +A **frontend application** refers to a web frontend based on the frontend example ui repository: [frontend repo][1] +A **backend application** refers to an application build using the orchestrator core as a base: [backend repo][2] + +## Without authentication + +Without authentication WFO allows all users access to all resources. + +##### Backend + +`OAUTH2_ACTIVE=false` + +##### Frontend: + +`OAUTH2_ACTIVE=false` + +## With Authentication + +WFO provides authentication based on an OIDC provider. The OIDC provider is presumed to be configured and to provide + +- An authentication endpoint +- A tenant +- A client id +- A client secret + +#### Frontend + +The WFO frontend uses [NextAuth](3) to handle authentication. Authentication configuration can be found in [page/api/auth/[...nextauth].ts](4) + +**ENV variables** +These variables need to be set for authentication to work on the frontend. + +``` +# Auth variables +OAUTH2_ACTIVE=true +OAUTH2_CLIENT_ID="orchestrator-client" // The oidc client id as configured in the OIDC provider +OAUTH2_CLIENT_SECRET=[SECRET] // The oidc client secret id as configured in the OIDC provider + +NEXTAUTH_PROVIDER_ID="keycloak" // String identifying the OIDC provider +NEXTAUTH_PROVIDER_NAME="Keycloak" // The name of the OIDC provider. Keycloak uses this name to display in the login screen +NEXTAUTH_AUTHORIZATION_SCOPE_OVERRIDE="openid profile" // Optional override of the scopes that are asked permission for from the OIDC provider + +# Required by the Nextauth middleware +NEXTAUTH_URL=[DOMAIN]/api/auth // The path to the [...nextauth].js file +NEXTAUTH_SECRET=[SECRET] // Used by NextAuth to encrypt the JWT token +``` + +With authentication turned on and these variables provided the frontend application will redirect unauthorized users to the login screen provided by the OIDC provider to request their credentials and return them to the page they tried to visit. + +Note: It's possible to add additional oidc providers including some that are provided by the NextAuth library like Google, Apple and others. See [NextAuthProviders](5) for more information. + +##### Authorization + +Authorization on the frontend can be used to determine if a page, action or navigation item is shown to a user. For this it uses an `isAllowedHandler` function can be passed into the WfoAuth component that wraps the page in `_app.tsx` + +```_app.tsx + +... + + ... + +... +``` + +The signature of the function should be `(routerPath: string, resource?: string) => boolean;`. The function is called on with the `routerpath` value and +the `resource`. This is the list of events the function is called on is: + +``` +export enum PolicyResource { + NAVIGATION_METADATA = '/orchestrator/metadata/', // called when determining if the metadata menuitem should be shown + NAVIGATION_SETTINGS = '/orchestrator/settings/', // called when determining if the settings menuitem should be shown + NAVIGATION_SUBSCRIPTIONS = '/orchestrator/subscriptions/', // called when determining if the subscriptions should be shown + NAVIGATION_TASKS = '/orchestrator/tasks/', // called when determining if the tasks menuitem should be shown + NAVIGATION_WORKFLOWS = '/orchestrator/processes/', // called when determining if the processes menuitem should be shown + PROCESS_ABORT = '/orchestrator/processes/abort/', // called when determining if the button to trigger a process abort should be shown + PROCESS_DELETE = '/orchestrator/processes/delete/', // called when determining if the button to trigger a process delete button should be shown + PROCESS_DETAILS = '/orchestrator/processes/details/', // called when determining if the process detail page should be displayed + PROCESS_RELATED_SUBSCRIPTIONS = '/orchestrator/subscriptions/view/from-process', // called when determining if the related subscriptions for a subscription should be shown + PROCESS_RETRY = '/orchestrator/processes/retry/', // called when determining if the button to trigger a process retry should be shown + PROCESS_USER_INPUT = '/orchestrator/processes/user-input/', // called when determining if th + SUBSCRIPTION_CREATE = '/orchestrator/processes/create/process/menu', // called when determining if create if actions that trigger a create workflow should be displayed + SUBSCRIPTION_MODIFY = '/orchestrator/subscriptions/modify/', // called when determining if create if actions that trigger a modify workflow should be displayed + SUBSCRIPTION_TERMINATE = '/orchestrator/subscriptions/terminate/', // called when determining if create if actions that trigger a terminate workflow should be displayed + SUBSCRIPTION_VALIDATE = '/orchestrator/subscriptions/validate/', // called when determining if create if actions that trigger a validate task should be displayed + TASKS_CREATE = '/orchestrator/processes/create/task', // called when determining if create if actions that trigger a task should be displayed + TASKS_RETRY_ALL = '/orchestrator/processes/all-tasks/retry', // called when determining if create if actions that trigger retry all tasks task should be displayed + SETTINGS_FLUSH_CACHE = '/orchestrator/settings/flush-cache', // called when determining if a button to flush cache should be displayed + SET_IN_SYNC = '/orchestrator/subscriptions/set-in-sync', // called when determining if a button to set a subscription in sync should be displayed +} +``` + +Note: Components that are hidden for unauthorized users are still part of the frontend application, authorization just makes sure +unauthorized users are not presented with actions they are not allowed to take. The calls these actions +make can still be made through curl calls for example. Additional authorization needs to be implemented on these calls on the backend. + +### Backend + +**ENV variables** +These variables need to be set for authentication to work on the backend + +``` +... +# OIDC settings +OAUTH2_ACTIVE: bool = True +OAUTH2_AUTHORIZATION_ACTIVE: bool = True +OAUTH2_RESOURCE_SERVER_ID: str = "" +OAUTH2_RESOURCE_SERVER_SECRET: str = "" +OAUTH2_TOKEN_URL: str = "" +OIDC_BASE_URL: str = "" +OIDC_CONF_URL: str = "" + +# OPtional OPA settings +OPA_URL: str = "" +``` + +With the variables provided, requests to endpoints will return 403 error codes for users that are not logged in and 401 error codes for users that are not authorized to do a call. + +#### Customization + +`AuthManager` serves as the central unit for managing both `authentication` and `authorization` mechanisms. +While it defaults to using `OIDCAuth` for authentication, `OPAAuthorization` for http authorization and `GraphQLOPAAuthorization` for graphql authorization , it supports customization. + +When initiating the `OrchestratorCore` class, it's [`auth_manager`][6] property is set to `AuthManager`. AuthManager is provided by [oauth2_lib][7]. + +`AuthManager` provides 3 methods that are called for authentication and authorization: `authentication`, `authentication` and `graphql_authorization`. + +`authentication`: The default method provided by Oaut2Lib implements returning the OIDC user from the OIDC introspection endpoint. + +`authorization`: A method that applies authorization decisions to HTTP requests, the decision is either true (Allowed) or false (Forbidden). Gets this payload to based decisions on. The default method provided by Oaut2Lib uses OPA and sends the payload to the opa_url specified in OPA_URL setting to get a decision. + +``` + "input": { + **(self.opa_kwargs or {}), + **(user_info or {}), + "resource": request.url.path, + "method": request_method, + "arguments": {"path": request.path_params, "query": {**request.query_params}, "json": json}, + } +``` + +Note: +The default authentication method allows for the passing in of **is_bypassable_request** method that receives the Request object +and returns a boolean. When this method returns true the request is always allowed regardless of other authorization decisions. + +`graphql_authorization`: A method that applies authorization decisions to graphql requests. Specializes OPA authorization for GraphQL operations. +GraphQl results always return a 200 response when authenticated but can return 403 results for partial results as may occur in federated scenarios. + +### Customizing + +When initializing the app we have the option to register custom authentication and authorization methods and override the default auth(n|z) logic. + +``` +... + app.register_authentication(...) + app.register_authorization(...) + app.register_graphql_authorization(...) +... +``` + +**app.register_authentication** takes an subclass of abstract class + +``` +from abc import ABC, abstractmethod + +class Authentication(ABC): + """Abstract base for authentication mechanisms. + + Requires an async authenticate method implementation. + """ + + @abstractmethod + async def authenticate(self, request: HTTPConnection, token: str | None = None) -> dict | None: + """Authenticate the user.""" + pass +``` + +Authorization decisions can be made based on request properties and the token provided + +**app.register_authorization** takes an subclass of abstract class + +``` +from abc import ABC, abstractmethod + +class Authorization(ABC): + """Defines the authorization logic interface. + + Implementations must provide an async method to authorize based on request and user info. + """ + + @abstractmethod + async def authorize(self, request: HTTPConnection, user: OIDCUserModel) -> bool | None: + pass + +``` + +Authorization decisions can be made based on request properties and user attributes + +**app.register_graphql_authorization** takes a subclass of abstract class + +``` +class GraphqlAuthorization(ABC): + """Defines the graphql authorization logic interface. + + Implementations must provide an async method to authorize based on request and user info. + """ + + @abstractmethod + async def authorize(self, request: RequestPath, user: OIDCUserModel) -> bool | None: + pass + +``` + +Graphql Authorization decisions can be made based on request properties and user attributes + +### Example + +Below is an example illustrating how to override the default configurations: + +```python +from orchestrator import OrchestratorCore, app_settings +from oauth2_lib.fastapi import OIDCAuth, OIDCUserModel, Authorization, RequestPath, GraphqlAuthorization +from oauth2_lib.settings import oauth2lib_settings +from httpx import AsyncClient +from starlette.requests import HTTPConnection +from typing import Optional + +class CustomOIDCAuth(OIDCAuth): + async def userinfo(self, async_request: AsyncClient, token: str) -> OIDCUserModel: + # Custom implementation to fetch user information + return OIDCUserModel( + sub="user-sub", + email="example-user@company.org", + # ... + ) + +class CustomAuthorization(Authorization): + async def authorize(self, request: HTTPConnection, user: OIDCUserModel) -> Optional[bool]: + # Implement custom authorization logic + return True + +class CustomGraphqlAuthorization(GraphqlAuthorization): + async def authorize(self, request: RequestPath, user: OIDCUserModel) -> Optional[bool]: + # Implement custom GraphQL authorization logic + return True + +oidc_instance = CustomOIDCAuth( + openid_url=oauth2lib_settings.OIDC_BASE_URL, + openid_config_url=oauth2lib_settings.OIDC_CONF_URL, + resource_server_id=oauth2lib_settings.OAUTH2_RESOURCE_SERVER_ID, + resource_server_secret=oauth2lib_settings.OAUTH2_RESOURCE_SERVER_SECRET, + oidc_user_model_cls=OIDCUserModel, +) + +authorization_instance = CustomAuthorization() +graphql_authorization_instance = CustomGraphqlAuthorization() + +app = OrchestratorCore(base_settings=app_settings) +app.register_authentication(oidc_instance) +app.register_authorization(authorization_instance) +app.register_graphql_authorization(graphql_authorization_instance) +``` + +[1]: https://github.com/workfloworchestrator/example-orchestrator-ui +[2]: https://github.com/workfloworchestrator/example-orchestrator +[3]: https://next-auth.js.org/ +[4]: https://github.com/workfloworchestrator/example-orchestrator-ui/blob/main/pages/api/auth/%5B...nextauth%5D.ts +[5]: https://next-auth.js.org/configuration/providers/oauth +[6]: https://github.com/workfloworchestrator/orchestrator-core/blob/70b0617049dfd25d31cbe3a7e5c8d6e48150f307/orchestrator/app.py#L95 +[7]: https://github.com/workfloworchestrator/oauth2-lib diff --git a/docs/reference-docs/auth.md b/docs/reference-docs/auth.md deleted file mode 100644 index c75ea6df8..000000000 --- a/docs/reference-docs/auth.md +++ /dev/null @@ -1,122 +0,0 @@ -# Authentication and Authorization -## Overview -The `Orchestrator-Core` application incorporates a robust security framework, utilizing OpenID Connect (OIDC) for authentication and Open Policy Agent (OPA) for authorization. -This flexible system ensures secure access, allowing you to tailor the authorization components to best fit your application's specific requirements. - -## Default Configuration -You don't need to modify any settings by default. Simply set the environment variables as needed, and the system will use these settings: - -```python -from pydantic_settings import BaseSettings - -class Oauth2LibSettings(BaseSettings): - # General settings - ENVIRONMENT: str = "local" - SERVICE_NAME: str = "" - MUTATIONS_ENABLED: bool = False - ENVIRONMENT_IGNORE_MUTATION_DISABLED: list[str] = [] - - # OIDC settings - OAUTH2_ACTIVE: bool = True - OAUTH2_AUTHORIZATION_ACTIVE: bool = True - OAUTH2_RESOURCE_SERVER_ID: str = "" - OAUTH2_RESOURCE_SERVER_SECRET: str = "" - OAUTH2_TOKEN_URL: str = "" - OIDC_BASE_URL: str = "" - OIDC_CONF_URL: str = "" - - # OPA settings - OPA_URL: str = "" -``` - -## Authentication Process -Authentication through OIDC confirms user identities and controls access to various endpoints: - -```python -from oauth2_lib.fastapi import OIDCAuth, OIDCUserModel - -# Initialize OIDC Authentication -oidc_auth = OIDCAuth( - openid_url="https://example-opendid.com/.well-known/openid-configuration", - openid_config_url="https://example-opendid.com/openid/config", - resource_server_id="your-client-id", - resource_server_secret="your-client-secret", - oidc_user_model_cls=OIDCUserModel -) -``` - -## Authorization Process -Authorization with OPA provides detailed control over user permissions: - -```python -from oauth2_lib.fastapi import OPAAuthorization, GraphQLOPAAuthorization - -# Establish OPA Authorization -opa_auth = OPAAuthorization(opa_url="https://opa.example.com/v1/data/your_policy") -graphql_opa_auth = GraphQLOPAAuthorization(opa_url="https://opa.example.com/v1/data/your_policy") -``` - -## Customizing Authentication and Authorization -### AuthManager -`AuthManager` serves as the central unit for managing both `authentication` and `authorization` mechanisms. -While it defaults to using `OIDCAuth` for authentication, `OPAAuthorization` for http authorization and `GraphQLOPAAuthorization` for graphql authorization , it supports extensive customization. - -### Implementing Custom Authentication: -To implement a custom authentication strategy, extend the abstract `Authentication` base class and implement the `authenticate` method. - -### Implementing Custom Authorization: -For custom authorization, extend the `Authorization` class and implement the `authorize` method. - -### Implementing Custom GraphQL Authorization: -To customize GraphQL authorization, extend the `GraphqlAuthorization` class and implement the `authorize` method. - -Below is an example illustrating how to override the default configurations: - -```python -from orchestrator import OrchestratorCore, app_settings -from oauth2_lib.fastapi import OIDCAuth, OIDCUserModel, Authorization, RequestPath, GraphqlAuthorization -from oauth2_lib.settings import oauth2lib_settings -from httpx import AsyncClient -from starlette.requests import HTTPConnection -from typing import Optional - -class CustomOIDCAuth(OIDCAuth): - async def userinfo(self, async_request: AsyncClient, token: str) -> OIDCUserModel: - # Custom implementation to fetch user information - return OIDCUserModel( - sub="user-sub", - email="example-user@company.org", - # ... - ) - -class CustomAuthorization(Authorization): - async def authorize(self, request: HTTPConnection, user: OIDCUserModel) -> Optional[bool]: - # Implement custom authorization logic - return True - -class CustomGraphqlAuthorization(GraphqlAuthorization): - async def authorize(self, request: RequestPath, user: OIDCUserModel) -> Optional[bool]: - # Implement custom GraphQL authorization logic - return True - -oidc_instance = CustomOIDCAuth( - openid_url=oauth2lib_settings.OIDC_BASE_URL, - openid_config_url=oauth2lib_settings.OIDC_CONF_URL, - resource_server_id=oauth2lib_settings.OAUTH2_RESOURCE_SERVER_ID, - resource_server_secret=oauth2lib_settings.OAUTH2_RESOURCE_SERVER_SECRET, - oidc_user_model_cls=OIDCUserModel, -) - -authorization_instance = CustomAuthorization() -graphql_authorization_instance = CustomGraphqlAuthorization() - -app = OrchestratorCore(base_settings=app_settings) -app.register_authentication(oidc_instance) -app.register_authorization(authorization_instance) -app.register_graphql_authorization(graphql_authorization_instance) -``` - -## Security Considerations -- Ensure secure HTTPS communications for all OIDC and OPA interactions. -- Securely store sensitive information like client secrets. -- Regularly revise OIDC and OPA configurations to align with evolving security standards and changes in external services. diff --git a/mkdocs.yml b/mkdocs.yml index 998c22006..4f2490921 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -2,237 +2,237 @@ site_name: Workflow Orchestrator site_description: An extensible workflow engine to manage customer facing resources and resource facing resources. site_url: https://workfloworchestrator.org/orchestrator-core theme: - name: material - icon: - repo: fontawesome/brands/github-alt - favicon: img/favicon.ico - logo: img/WFO-Emblem-White.png - palette: - - scheme: default - primary: teal - accent: amber - toggle: - icon: material/lightbulb-outline - name: Switch to light mode - - scheme: slate - primary: teal - accent: amber - toggle: - icon: material/lightbulb - name: Switch to dark mode - features: - - search.suggest - - search.highlight - - navigation.tabs - - navigation.tabs.sticky - - navigation.tracking - - navigation.instant - - navigation.indexes - - content.code.copy - - content.code.annotate - - content.tooltips + name: material + icon: + repo: fontawesome/brands/github-alt + favicon: img/favicon.ico + logo: img/WFO-Emblem-White.png + palette: + - scheme: default + primary: teal + accent: amber + toggle: + icon: material/lightbulb-outline + name: Switch to light mode + - scheme: slate + primary: teal + accent: amber + toggle: + icon: material/lightbulb + name: Switch to dark mode + features: + - search.suggest + - search.highlight + - navigation.tabs + - navigation.tabs.sticky + - navigation.tracking + - navigation.instant + - navigation.indexes + - content.code.copy + - content.code.annotate + - content.tooltips plugins: - - external-markdown - - search - - open-in-new-tab - - render_swagger - - macros - - include-markdown - - privacy - - social - - mkdocstrings: - default_handler: python - enable_inventory: true - handlers: - python: - options: - show_source: true - show_root_heading: true - show_root_toc_entry: true - show_symbol_type_heading: true - show_symbol_type_toc: true - docstring_style: null - docstring_section_style: list - annotations_path: full - separate_signature: true - line_length: 80 - show_signature_annotations: true - unwrap_annotated: true - docstring_options: - trim_doctest_flags: true + - external-markdown + - search + - open-in-new-tab + - render_swagger + - macros + - include-markdown + - privacy + - social + - mkdocstrings: + default_handler: python + enable_inventory: true + handlers: + python: + options: + show_source: true + show_root_heading: true + show_root_toc_entry: true + show_symbol_type_heading: true + show_symbol_type_toc: true + docstring_style: null + docstring_section_style: list + annotations_path: full + separate_signature: true + line_length: 80 + show_signature_annotations: true + unwrap_annotated: true + docstring_options: + trim_doctest_flags: true copyright: Copyright © 2018 - 2024 Workflow Orchestrator Programme extra: - generator: false + generator: false repo_name: workfloworchestrator/orchestrator-core repo_url: https://github.com/workfloworchestrator/orchestrator-core edit_uri: edit/main/docs/ markdown_extensions: - - pymdownx.emoji: - emoji_index: !!python/name:material.extensions.emoji.twemoji - emoji_generator: !!python/name:material.extensions.emoji.to_svg - - pymdownx.highlight - - pymdownx.superfences - - admonition - - pymdownx.details - - pymdownx.superfences - - pymdownx.snippets: - auto_append: - - includes/abbreviations.md - - pymdownx.keys - - pymdownx.inlinehilite - - pymdownx.tabbed: - alternate_style: true - - attr_list - - codehilite - - pymdownx.superfences: - custom_fences: - - name: mermaid - class: mermaid - format: !!python/name:pymdownx.superfences.fence_code_format - - abbr + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - pymdownx.highlight + - pymdownx.superfences + - admonition + - pymdownx.details + - pymdownx.superfences + - pymdownx.snippets: + auto_append: + - includes/abbreviations.md + - pymdownx.keys + - pymdownx.inlinehilite + - pymdownx.tabbed: + alternate_style: true + - attr_list + - codehilite + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - abbr extra_css: - - 'css/termynal.css' - - 'css/custom.css' - - 'css/style.css' + - "css/termynal.css" + - "css/custom.css" + - "css/style.css" extra_javascript: - - 'js/termynal.js' - - 'js/custom.js' + - "js/termynal.js" + - "js/custom.js" nav: - - Workflow Orchestrator: - - Workflow Orchestrator: index.md - - Orchestrator Framework: architecture/framework.md - - Orchestrator UI: - - Orchestrator UI: architecture/orchestration/orchestrator-ui.md - # - Env variables: - # - Dark Theme adjustment / dark theme env setting - # - Your own company logo - # - Component-lib - # - RBAC - # - Isallowed wrapper - # - OIDC - # - Code flow with PKCE - - User input forms: - - Generic solution: architecture/application/forms-frontend.md - # - Form page with all form field: - # - Extensibility: - # - Extra menu item - # - Add a new summary card - - Architecture: - - Architecture; TL;DR: architecture/tldr.md - - Orchestration Philosophy: architecture/orchestration/philosophy.md - - Domain Models: architecture/application/domainmodels.md - - Internals: - - How do Workflows work?: architecture/application/workflow.md - - What are tasks?: architecture/application/tasks.md - - Advanced - Product modelling: - - Introduction: architecture/product_modelling/introduction.md - - Standards: architecture/product_modelling/standards.md - - Modelling: architecture/product_modelling/modelling.md - - Context: architecture/product_modelling/context.md - - Terminology: architecture/product_modelling/terminology.md - - Example Product Models: - - Node: architecture/product_modelling/node.md - - Port: architecture/product_modelling/port.md - - L2 Point-to-Point: architecture/product_modelling/l2_point_to_point.md - - L2 VPN: architecture/product_modelling/l2_vpn.md - - IP Static: architecture/product_modelling/ip_static.md - - Product Block Graph: architecture/product_modelling/product_block_graph.md - - Importing Existing Products: architecture/product_modelling/imports.md - - Getting Started: - - Prerequisites: getting-started/versions.md - - Base Application: - - Preparing source folder: getting-started/prepare-source-folder.md - - Python Virtualenv: getting-started/base.md - - Docker: getting-started/docker.md - - Orchestrator UI: getting-started/orchestration-ui.md - - Reference Documentation: - - TL;DR: reference-docs/tldr.md - - API docs: - - Rest API: reference-docs/api.md - - GraphQL: reference-docs/graphql.md - - Auth(n|z): reference-docs/auth.md - - CLI Tools: reference-docs/cli.md - - Database: reference-docs/database.md - # - Serialization: reference-docs/serialization.md - - Domain Models: - - Overview: reference-docs/domain_models/overview.md - - Domain Model Types: - - Product Types: reference-docs/domain_models/product_types.md - - Product Blocks: reference-docs/domain_models/product_blocks.md - - Model Attributes: reference-docs/domain_models/model_attributes.md - - Helpers: - - Generator: reference-docs/domain_models/generator.md -# - Advanced Features: -# - Properties: reference-docs/domain_models/properties.md -# - Union Types: reference-docs/domain_models/union_types.md -# - Pydantic hooks: reference-docs/domain_models/pydantic_hooks.md -# - Instantiating Domain Models: reference-docs/domain_models/instantiating.md -# - Validation: reference-docs/domain_models/validation.md -# - Type Casting and Serialisation: reference-docs/domain_models/type_casting.md - - Forms: reference-docs/forms.md - - Running the App: - - App.py : reference-docs/app/app.md - - Python Version: reference-docs/python.md - - Scaling: reference-docs/app/scaling.md - # - Tasks: reference-docs/tasks.md - # - Tests: reference-docs/tests.md - - Workflows: - - Workflow Steps: reference-docs/workflows/workflow-steps.md - # - Workflow Lifecycles: reference-docs/workflows/workflow-lifecycles.md - - Callbacks: reference-docs/workflows/callbacks.md - - Websockets: reference-docs/websockets.md - - Migration guide: migration-guide/2.0.md + - Workflow Orchestrator: + - Workflow Orchestrator: index.md + - Orchestrator Framework: architecture/framework.md + - Orchestrator UI: + - Orchestrator UI: architecture/orchestration/orchestrator-ui.md + # - Env variables: + # - Dark Theme adjustment / dark theme env setting + # - Your own company logo + # - Component-lib + # - RBAC + # - Isallowed wrapper + # - OIDC + # - Code flow with PKCE + - User input forms: + - Generic solution: architecture/application/forms-frontend.md + # - Form page with all form field: + # - Extensibility: + # - Extra menu item + # - Add a new summary card + - Architecture: + - Architecture; TL;DR: architecture/tldr.md + - Orchestration Philosophy: architecture/orchestration/philosophy.md + - Domain Models: architecture/application/domainmodels.md + - Internals: + - How do Workflows work?: architecture/application/workflow.md + - What are tasks?: architecture/application/tasks.md + - Advanced - Product modelling: + - Introduction: architecture/product_modelling/introduction.md + - Standards: architecture/product_modelling/standards.md + - Modelling: architecture/product_modelling/modelling.md + - Context: architecture/product_modelling/context.md + - Terminology: architecture/product_modelling/terminology.md + - Example Product Models: + - Node: architecture/product_modelling/node.md + - Port: architecture/product_modelling/port.md + - L2 Point-to-Point: architecture/product_modelling/l2_point_to_point.md + - L2 VPN: architecture/product_modelling/l2_vpn.md + - IP Static: architecture/product_modelling/ip_static.md + - Product Block Graph: architecture/product_modelling/product_block_graph.md + - Importing Existing Products: architecture/product_modelling/imports.md + - Getting Started: + - Prerequisites: getting-started/versions.md + - Base Application: + - Preparing source folder: getting-started/prepare-source-folder.md + - Python Virtualenv: getting-started/base.md + - Docker: getting-started/docker.md + - Orchestrator UI: getting-started/orchestration-ui.md + - Reference Documentation: + - TL;DR: reference-docs/tldr.md + - API docs: + - Rest API: reference-docs/api.md + - GraphQL: reference-docs/graphql.md + - Auth(n|z): reference-docs/auth-backend-and-frontend.md + - CLI Tools: reference-docs/cli.md + - Database: reference-docs/database.md + # - Serialization: reference-docs/serialization.md + - Domain Models: + - Overview: reference-docs/domain_models/overview.md + - Domain Model Types: + - Product Types: reference-docs/domain_models/product_types.md + - Product Blocks: reference-docs/domain_models/product_blocks.md + - Model Attributes: reference-docs/domain_models/model_attributes.md + - Helpers: + - Generator: reference-docs/domain_models/generator.md + # - Advanced Features: + # - Properties: reference-docs/domain_models/properties.md + # - Union Types: reference-docs/domain_models/union_types.md + # - Pydantic hooks: reference-docs/domain_models/pydantic_hooks.md + # - Instantiating Domain Models: reference-docs/domain_models/instantiating.md + # - Validation: reference-docs/domain_models/validation.md + # - Type Casting and Serialisation: reference-docs/domain_models/type_casting.md + - Forms: reference-docs/forms.md + - Running the App: + - App.py: reference-docs/app/app.md + - Python Version: reference-docs/python.md + - Scaling: reference-docs/app/scaling.md + # - Tasks: reference-docs/tasks.md + # - Tests: reference-docs/tests.md + - Workflows: + - Workflow Steps: reference-docs/workflows/workflow-steps.md + # - Workflow Lifecycles: reference-docs/workflows/workflow-lifecycles.md + - Callbacks: reference-docs/workflows/callbacks.md + - Websockets: reference-docs/websockets.md + - Migration guide: migration-guide/2.0.md - - Workshops: -# - Beginner: -# - Overview: workshops/beginner/overview.md -# - Installation: -# - Manual: -# - Debian: workshops/beginner/debian.md -# - MacOS: workshops/beginner/macos.md -# - Docker compose: workshops/beginner/docker.md -# - Start applications: workshops/beginner/start-applications.md -# - Products: -# - Scenario: workshops/beginner/scenario.md -# - Domain models: workshops/beginner/domain-models.md -# - Database migration: workshops/beginner/database-migration.md -# - Workflows: -# - Introduction: workshops/beginner/workflow-introduction.md -# - Register workflows: workshops/beginner/register-workflows.md -# - Input forms: workshops/beginner/input-forms.md -# - Create UserGroup: workshops/beginner/create-user-group.md -# - Modify UserGroup: workshops/beginner/modify-user-group.md -# - Terminate UserGroup: workshops/beginner/terminate-user-group.md -# - Create User: workshops/beginner/create-user.md -# - Modify User: workshops/beginner/modify-user.md -# - Terminate User: workshops/beginner/terminate-user.md -# - Explore: workshops/beginner/explore.md - - Example Orchestrator Workshop: - - Overview: workshops/advanced/overview.md -# - Installation: workshops/advanced/docker-installation.md - - Bootstrapping: - - Getting started: workshops/advanced/bootstrap.md - - Seeding data: workshops/advanced/execute-workflows.md - - Products: - - Scenario: workshops/advanced/scenario.md - - Domain models: workshops/advanced/domain-models.md - - Workflows: - - Introduction: workshops/advanced/workflow-introduction.md - - Workflow Basics: workshops/advanced/workflow-basics.md - - Workflow Examples: - - Create Workflow: workshops/advanced/node-create.md - - Modify Workflow: workshops/advanced/node-modify.md - - Terminate Workflow: workshops/advanced/node-terminate.md - - Validate Workflow: workshops/advanced/node-validate.md - - Create your own product and workflows: - - L2 Point-to-Point: workshops/advanced/create-your-own.md - - Product and Workflow Generator: workshops/advanced/generator.md - - Contributing: - - Guidelines: contributing/guidelines.md - - Testing: contributing/testing.md - - Development setup: contributing/development.md + - Workshops: + # - Beginner: + # - Overview: workshops/beginner/overview.md + # - Installation: + # - Manual: + # - Debian: workshops/beginner/debian.md + # - MacOS: workshops/beginner/macos.md + # - Docker compose: workshops/beginner/docker.md + # - Start applications: workshops/beginner/start-applications.md + # - Products: + # - Scenario: workshops/beginner/scenario.md + # - Domain models: workshops/beginner/domain-models.md + # - Database migration: workshops/beginner/database-migration.md + # - Workflows: + # - Introduction: workshops/beginner/workflow-introduction.md + # - Register workflows: workshops/beginner/register-workflows.md + # - Input forms: workshops/beginner/input-forms.md + # - Create UserGroup: workshops/beginner/create-user-group.md + # - Modify UserGroup: workshops/beginner/modify-user-group.md + # - Terminate UserGroup: workshops/beginner/terminate-user-group.md + # - Create User: workshops/beginner/create-user.md + # - Modify User: workshops/beginner/modify-user.md + # - Terminate User: workshops/beginner/terminate-user.md + # - Explore: workshops/beginner/explore.md + - Example Orchestrator Workshop: + - Overview: workshops/advanced/overview.md + # - Installation: workshops/advanced/docker-installation.md + - Bootstrapping: + - Getting started: workshops/advanced/bootstrap.md + - Seeding data: workshops/advanced/execute-workflows.md + - Products: + - Scenario: workshops/advanced/scenario.md + - Domain models: workshops/advanced/domain-models.md + - Workflows: + - Introduction: workshops/advanced/workflow-introduction.md + - Workflow Basics: workshops/advanced/workflow-basics.md + - Workflow Examples: + - Create Workflow: workshops/advanced/node-create.md + - Modify Workflow: workshops/advanced/node-modify.md + - Terminate Workflow: workshops/advanced/node-terminate.md + - Validate Workflow: workshops/advanced/node-validate.md + - Create your own product and workflows: + - L2 Point-to-Point: workshops/advanced/create-your-own.md + - Product and Workflow Generator: workshops/advanced/generator.md + - Contributing: + - Guidelines: contributing/guidelines.md + - Testing: contributing/testing.md + - Development setup: contributing/development.md watch: - - includes - - orchestrator + - includes + - orchestrator From a52310bdbaa987d952b45fc35bf709c5c861c13c Mon Sep 17 00:00:00 2001 From: tjeerddie Date: Mon, 14 Oct 2024 14:42:57 +0200 Subject: [PATCH 2/6] Add depth recursion to inUseBySubscriptions and dependsOnSubscriptions (#753) * Add depth recursion to inUseBySubscription and dependsOnSubscriptions * Change root recurse filters to a recurse_filter as strawberry input - Fix recurse statuses filter to work with loader - loader does not like lists in combination with its own caching, so format list to a joined string. * Change filter_statuses to a tuple and remove before needed formatting - change filter name from recursion_filter to in_use_by_filter and depends_on_filter. - add test for `get_in_use_by_subscriptions` and `get_depends_on_subscriptions` that checks if that the subscriptions are given back in the correct order. * Fix instance relations sqlalchemy query and add tests * Add tests for graphql subscription/s recursive relations query * Change graphql subscription relation tests to parametrized tests * Change graphql get_recursive_relations in in_use_by and depends_on to a re-usable function - move unzip of subscription_ids and filter_statuses into a function to re-use for both data loaders - improve get_recursive_relations * Update test/unit_tests/graphql/conftest.py Co-authored-by: tjeerddie --------- Co-authored-by: Peter Boers --- orchestrator/graphql/loaders/subscriptions.py | 221 +------ orchestrator/graphql/schemas/subscription.py | 45 +- .../services/subscription_relations.py | 269 ++++++++ test/unit_tests/conftest.py | 6 + .../product_block_list_nested.py | 23 +- .../product_types/product_type_list_nested.py | 11 +- .../product_type_list_union_overlap.py | 4 +- .../product_types/subscription_relations.py | 300 +++++++++ test/unit_tests/graphql/conftest.py | 11 +- .../graphql/test_subscription_relations.py | 617 ++++++++++++++++++ test/unit_tests/helpers.py | 13 + .../services/test_subscription_relations.py | 190 ++++++ 12 files changed, 1494 insertions(+), 216 deletions(-) create mode 100644 orchestrator/services/subscription_relations.py create mode 100644 test/unit_tests/fixtures/products/product_types/subscription_relations.py create mode 100644 test/unit_tests/graphql/test_subscription_relations.py create mode 100644 test/unit_tests/services/test_subscription_relations.py diff --git a/orchestrator/graphql/loaders/subscriptions.py b/orchestrator/graphql/loaders/subscriptions.py index 10f2da57d..1d5e55cea 100644 --- a/orchestrator/graphql/loaders/subscriptions.py +++ b/orchestrator/graphql/loaders/subscriptions.py @@ -1,224 +1,41 @@ -from itertools import chain -from typing import Any, NamedTuple +from typing import Literal from uuid import UUID import structlog -from sqlalchemy import Row, select -from sqlalchemy import Text as SaText -from sqlalchemy import cast as sa_cast -from sqlalchemy.orm import aliased +from more_itertools import one, unique_everseen from strawberry.dataloader import DataLoader from orchestrator.db import ( - ResourceTypeTable, - SubscriptionInstanceTable, - SubscriptionInstanceValueTable, SubscriptionTable, - db, ) -from orchestrator.db.models import ( - SubscriptionInstanceRelationTable, -) -from orchestrator.services.subscriptions import RELATION_RESOURCE_TYPES +from orchestrator.services.subscription_relations import get_depends_on_subscriptions, get_in_use_by_subscriptions from orchestrator.types import SubscriptionLifecycle logger = structlog.get_logger(__name__) -class Relation(NamedTuple): - depends_on_sub_id: UUID - in_use_by_sub_id: UUID - - -def _get_instance_relations(instance_relations_query: Any) -> list[Relation]: - def to_relation(row: Row[Any]) -> Relation: - return Relation(row[0], row[1]) - - return [to_relation(row) for row in db.session.execute(instance_relations_query)] - - -async def _get_in_use_by_instance_relations(subscription_ids: list[UUID], filter_statuses: list[str]) -> list[Relation]: - """Get in_use_by by relations through subscription instance hierarchy.""" - in_use_by_subscriptions = aliased(SubscriptionTable) - in_use_by_instances = aliased(SubscriptionInstanceTable) - depends_on_instances = aliased(SubscriptionInstanceTable) - - query_get_in_use_by_ids = ( - select(depends_on_instances.subscription_id, in_use_by_instances.subscription_id) - .distinct() - .join(in_use_by_instances.subscription) - .join(in_use_by_instances.depends_on_block_relations) - .join(depends_on_instances, SubscriptionInstanceRelationTable.depends_on) - .join(in_use_by_subscriptions, depends_on_instances.subscription) - .filter(depends_on_instances.subscription_id.in_(set(subscription_ids))) - .filter(in_use_by_instances.subscription_id != depends_on_instances.subscription_id) - .filter(in_use_by_subscriptions.status.in_(filter_statuses)) - ) - - return _get_instance_relations(query_get_in_use_by_ids) - - -async def _get_depends_on_instance_relations( - subscription_ids: list[UUID], filter_statuses: list[str] -) -> list[Relation]: - """Get depends_on relations through subscription instance hierarchy.""" - in_use_by_instances = aliased(SubscriptionInstanceTable) - depends_on_instances = aliased(SubscriptionInstanceTable) - depends_on_subscriptions = aliased(SubscriptionTable) - - query_get_depends_on_ids = ( - select(depends_on_instances.subscription_id, in_use_by_instances.subscription_id) - .distinct() - .join(depends_on_instances.subscription) - .join(depends_on_instances.in_use_by_block_relations) - .join(in_use_by_instances, SubscriptionInstanceRelationTable.in_use_by) - .join(depends_on_subscriptions, in_use_by_instances.subscription) - .filter(in_use_by_instances.subscription_id.in_(set(subscription_ids))) - .filter(depends_on_instances.subscription_id != in_use_by_instances.subscription_id) - .filter(depends_on_subscriptions.status.in_(filter_statuses)) - ) - - return _get_instance_relations(query_get_depends_on_ids) - - -def _get_resource_type_relations(resource_type_relations_query: Any) -> list[Relation]: - def to_relation(row: Row[Any]) -> Relation: - return Relation(UUID(row[0]), row[1]) - - return [to_relation(row) for row in db.session.execute(resource_type_relations_query)] - - -async def _get_in_use_by_resource_type_relations( - subscription_ids: list[UUID], filter_statuses: list[str] -) -> list[Relation]: - """Get in_use_by relations through resource types.""" - logger.warning("Using legacy RELATION_RESOURCE_TYPES to find in_use_by subs") - - in_use_by_subscriptions = aliased(SubscriptionTable) - depends_on_instance_values = aliased(SubscriptionInstanceValueTable) - - # Convert UUIDs to string - unique_subscription_ids = set(map(str, subscription_ids)) - - query_get_in_use_by_ids = ( - select(depends_on_instance_values.value, in_use_by_subscriptions.subscription_id) - .select_from(depends_on_instance_values) - .join(SubscriptionInstanceTable) - .join(in_use_by_subscriptions) - .join(ResourceTypeTable) - .filter(ResourceTypeTable.resource_type.in_(RELATION_RESOURCE_TYPES)) - .filter(depends_on_instance_values.value.in_(unique_subscription_ids)) - .filter(in_use_by_subscriptions.status.in_(filter_statuses)) - ) - - return _get_resource_type_relations(query_get_in_use_by_ids) - - -async def _get_depends_on_resource_type_relations( - subscription_ids: list[UUID], filter_statuses: list[str] -) -> list[Relation]: - """Get depends_on relations through resource types.""" - logger.warning("Using legacy RELATION_RESOURCE_TYPES to find depends_on subs") - - depends_on_subscriptions = aliased(SubscriptionTable) - in_use_by_instances = aliased(SubscriptionInstanceTable) - in_use_by_instance_values = aliased(SubscriptionInstanceValueTable) - - unique_subscription_ids = set(subscription_ids) - - query_get_depends_on_ids = ( - select(in_use_by_instance_values.value, in_use_by_instances.subscription_id) - .select_from(in_use_by_instance_values) - .join(in_use_by_instances) - .join( - depends_on_subscriptions, - in_use_by_instance_values.value == sa_cast(depends_on_subscriptions.subscription_id, SaText), - ) - .join(ResourceTypeTable) - .filter(ResourceTypeTable.resource_type.in_(RELATION_RESOURCE_TYPES)) - .filter(in_use_by_instances.subscription_id.in_(unique_subscription_ids)) - .filter(depends_on_subscriptions.status.in_(filter_statuses)) +def unzip_subscription_ids_and_filter_statuses( + keys: list[tuple[UUID, tuple[str, ...]]], direction: Literal["dependsOn", "inUseBy"] +) -> tuple[list[UUID], tuple[str, ...]]: + subscription_ids = [key[0] for key in keys] + filter_statuses_values = (key[1] for key in keys) + filter_statuses = one( + unique_everseen(filter_statuses_values), + too_long=Exception(f"{direction}Filter.statuses must be set to the same value"), ) + return subscription_ids, tuple(filter_statuses or SubscriptionLifecycle.values()) - return _get_resource_type_relations(query_get_depends_on_ids) - -async def _get_in_use_by_relations(subscription_ids: list[UUID], filter_statuses: list[str]) -> list[Relation]: - if RELATION_RESOURCE_TYPES: - # Find relations through resource types - resource_type_relations = await _get_in_use_by_resource_type_relations(subscription_ids, filter_statuses) - else: - resource_type_relations = [] - # Find relations through instance hierarchy - instance_relations = await _get_in_use_by_instance_relations(subscription_ids, filter_statuses) - return list(chain(resource_type_relations, instance_relations)) - - -async def _get_depends_on_relations(subscription_ids: list[UUID], filter_statuses: list[str]) -> list[Relation]: - if RELATION_RESOURCE_TYPES: - # Find relations through resource types - resource_type_relations = await _get_depends_on_resource_type_relations(subscription_ids, filter_statuses) - else: - resource_type_relations = [] - # Find relations through instance hierarchy - instance_relations = await _get_depends_on_instance_relations(subscription_ids, filter_statuses) - return list(chain(resource_type_relations, instance_relations)) - - -async def in_use_by_subs_loader(keys: list[tuple[UUID, list[str] | None]]) -> list[list[SubscriptionTable]]: +async def in_use_by_subs_loader(keys: list[tuple[UUID, tuple[str, ...]]]) -> list[list[SubscriptionTable]]: """GraphQL dataloader to efficiently get the in_use_by SubscriptionTables for multiple subscription_ids.""" - subscription_ids = [key[0] for key in keys] - filter_statuses: list[str] = keys[0][1] or SubscriptionLifecycle.values() + subscription_ids, filter_statuses = unzip_subscription_ids_and_filter_statuses(keys, "inUseBy") + return await get_in_use_by_subscriptions(subscription_ids, filter_statuses) - in_use_by_relations = await _get_in_use_by_relations(subscription_ids, filter_statuses) - # Retrieve SubscriptionTable for all unique inuseby ids - unique_in_use_by_ids = {row.in_use_by_sub_id for row in in_use_by_relations} - _in_use_by_subs = db.session.execute( - select(SubscriptionTable).filter(SubscriptionTable.subscription_id.in_(unique_in_use_by_ids)) - ).scalars() - in_use_by_subs = {subscription.subscription_id: subscription for subscription in _in_use_by_subs} - - # group (more_itertools.bucket doesn't seem to work for tuple of uuids) - subscription_in_use_by_ids: dict[UUID, list[UUID]] = {} - for relation in in_use_by_relations: - subscription_in_use_by_ids.setdefault(relation.depends_on_sub_id, []).append(relation.in_use_by_sub_id) - - def get_in_use_by_subs(depends_on_id: UUID) -> list[SubscriptionTable]: - in_use_by_ids = subscription_in_use_by_ids.get(depends_on_id, []) - return [in_use_by_sub for id_ in in_use_by_ids if (in_use_by_sub := in_use_by_subs.get(id_))] - - # Important (as with any dataloader) - # Return the list of inuseby subs in the exact same order as the ids passed to this function - return [get_in_use_by_subs(subscription_id) for subscription_id in subscription_ids] - - -async def depends_on_subs_loader(keys: list[tuple[UUID, list[str] | None]]) -> list[list[SubscriptionTable]]: +async def depends_on_subs_loader(keys: list[tuple[UUID, tuple[str, ...]]]) -> list[list[SubscriptionTable]]: """GraphQL dataloader to efficiently get the depends_on SubscriptionTables for multiple subscription_ids.""" - subscription_ids = [key[0] for key in keys] - filter_statuses: list[str] = keys[0][1] or SubscriptionLifecycle.values() - - depends_on_relations = await _get_depends_on_relations(subscription_ids, filter_statuses) - - # Retrieve SubscriptionTable for all unique dependson ids - unique_depends_on_ids = {row.depends_on_sub_id for row in depends_on_relations} - _depends_on_subs = db.session.execute( - select(SubscriptionTable).filter(SubscriptionTable.subscription_id.in_(unique_depends_on_ids)) - ).scalars() - depends_on_subs = {subscription.subscription_id: subscription for subscription in _depends_on_subs} - - # group (more_itertools.bucket doesn't seem to work for tuple of uuids) - subscription_depends_on_ids: dict[UUID, list[UUID]] = {} - for relation in depends_on_relations: - subscription_depends_on_ids.setdefault(relation.in_use_by_sub_id, []).append(relation.depends_on_sub_id) - - def get_depends_on_subs(in_use_by_id: UUID) -> list[SubscriptionTable]: - depends_on_ids = subscription_depends_on_ids.get(in_use_by_id, []) - return [depends_on_sub for id_ in depends_on_ids if (depends_on_sub := depends_on_subs.get(id_))] - - # Important (as with any dataloader) - # Return the list of dependson subs in the exact same order as the ids passed to this function - return [get_depends_on_subs(subscription_id) for subscription_id in subscription_ids] + subscription_ids, filter_statuses = unzip_subscription_ids_and_filter_statuses(keys, "inUseBy") + return await get_depends_on_subscriptions(subscription_ids, filter_statuses) -SubsLoaderType = DataLoader[tuple[UUID, list[str] | None], list[SubscriptionTable]] +SubsLoaderType = DataLoader[tuple[UUID, tuple[str, ...]], list[SubscriptionTable]] diff --git a/orchestrator/graphql/schemas/subscription.py b/orchestrator/graphql/schemas/subscription.py index 6598a00ac..ea0bbad01 100644 --- a/orchestrator/graphql/schemas/subscription.py +++ b/orchestrator/graphql/schemas/subscription.py @@ -11,6 +11,7 @@ from oauth2_lib.strawberry import authenticated_field from orchestrator.db import FixedInputTable, ProductTable, SubscriptionTable, db from orchestrator.domain import SUBSCRIPTION_MODEL_REGISTRY +from orchestrator.graphql.loaders.subscriptions import SubsLoaderType from orchestrator.graphql.pagination import EMPTY_PAGE, Connection from orchestrator.graphql.resolvers.process import resolve_processes from orchestrator.graphql.schemas.customer import CustomerType @@ -24,6 +25,7 @@ get_subscription_product_blocks, ) from orchestrator.services.fixed_inputs import get_fixed_inputs +from orchestrator.services.subscription_relations import get_recursive_relations from orchestrator.services.subscriptions import ( get_subscription_metadata, ) @@ -36,6 +38,35 @@ static_metadata_schema = {"title": "SubscriptionMetadata", "type": "object", "properties": {}, "definitions": {}} +@strawberry.input(description="Filter recursion") +class SubscriptionRelationFilter: + statuses: list[str] | None = strawberry.field(default=None, description="Search by statusses") + recurse_depth_limit: int = strawberry.field(default=10, description="the limited depth to recurse through") + recurse_product_types: list[str] | None = strawberry.field( + default=None, description="List of product types to recurse into" + ) + + +async def _load_recursive_relations( + subscription_id: UUID, relation_filter: SubscriptionRelationFilter | None, data_loader: SubsLoaderType +) -> list[SubscriptionTable]: + sub_relation_filter = relation_filter or SubscriptionRelationFilter() + + async def get_subscriptions_from_loader( + subscription_ids: list[UUID], filter_statuses: tuple[str, ...] + ) -> list[list[SubscriptionTable]]: + load_mapping = [(sub_id, filter_statuses) for sub_id in subscription_ids] + return await data_loader.load_many(load_mapping) + + return await get_recursive_relations( + [subscription_id], + tuple(sub_relation_filter.statuses or ()), + sub_relation_filter.recurse_product_types or [], + sub_relation_filter.recurse_depth_limit, + get_subscriptions_from_loader, + ) + + @strawberry.federation.interface(description="Virtual base interface for subscriptions", keys=["subscriptionId"]) class SubscriptionInterface: subscription_id: UUID @@ -102,14 +133,18 @@ async def in_use_by_subscriptions( sort_by: list[GraphqlSort] | None = None, first: int = 10, after: int = 0, + in_use_by_filter: SubscriptionRelationFilter | None = None, ) -> Connection[Annotated["SubscriptionInterface", strawberry.lazy(".subscription")]]: from orchestrator.graphql.resolvers.subscription import resolve_subscriptions - subscriptions = await info.context.core_in_use_by_subs_loader.load((self.subscription_id, None)) - subscription_ids = [str(subscription.subscription_id) for subscription in subscriptions] + subscriptions = await _load_recursive_relations( + self.subscription_id, in_use_by_filter, info.context.core_in_use_by_subs_loader + ) + subscription_ids = [str(subscription.subscription_id) for subscription in subscriptions] if not subscription_ids: return EMPTY_PAGE + filter_by_with_related_subscriptions = (filter_by or []) + [ GraphqlFilter(field="subscriptionId", value="|".join(subscription_ids)) ] @@ -123,10 +158,14 @@ async def depends_on_subscriptions( sort_by: list[GraphqlSort] | None = None, first: int = 10, after: int = 0, + depends_on_filter: SubscriptionRelationFilter | None = None, ) -> Connection[Annotated["SubscriptionInterface", strawberry.lazy(".subscription")]]: from orchestrator.graphql.resolvers.subscription import resolve_subscriptions - subscriptions = await info.context.core_depends_on_subs_loader.load((self.subscription_id, None)) + subscriptions = await _load_recursive_relations( + self.subscription_id, depends_on_filter, info.context.core_depends_on_subs_loader + ) + subscription_ids = [str(subscription.subscription_id) for subscription in subscriptions] if not subscription_ids: return EMPTY_PAGE diff --git a/orchestrator/services/subscription_relations.py b/orchestrator/services/subscription_relations.py new file mode 100644 index 000000000..29e58266a --- /dev/null +++ b/orchestrator/services/subscription_relations.py @@ -0,0 +1,269 @@ +from itertools import chain +from typing import Any, Awaitable, Callable, NamedTuple +from uuid import UUID + +import structlog +from more_itertools import flatten, unique_everseen +from sqlalchemy import Row, select +from sqlalchemy import Text as SaText +from sqlalchemy import cast as sa_cast +from sqlalchemy.orm import aliased + +from orchestrator.db import ( + ResourceTypeTable, + SubscriptionInstanceTable, + SubscriptionInstanceValueTable, + SubscriptionTable, + db, +) +from orchestrator.db.models import ( + SubscriptionInstanceRelationTable, +) +from orchestrator.services.subscriptions import RELATION_RESOURCE_TYPES +from orchestrator.types import SubscriptionLifecycle + +logger = structlog.get_logger(__name__) + + +class Relation(NamedTuple): + depends_on_sub_id: UUID + in_use_by_sub_id: UUID + + +def _get_instance_relations(instance_relations_query: Any) -> list[Relation]: + def to_relation(row: Row[Any]) -> Relation: + return Relation(row[0], row[1]) + + return [to_relation(row) for row in db.session.execute(instance_relations_query)] + + +async def _get_in_use_by_instance_relations( + subscription_ids: list[UUID], filter_statuses: tuple[str, ...] +) -> list[Relation]: + """Get in_use_by by relations through subscription instance hierarchy.""" + in_use_by_instances = aliased(SubscriptionInstanceTable) + depends_on_instances = aliased(SubscriptionInstanceTable) + + query_get_in_use_by_ids = ( + select(depends_on_instances.subscription_id, in_use_by_instances.subscription_id) + .distinct() + .join(in_use_by_instances.subscription) + .join(in_use_by_instances.depends_on_block_relations) + .join(depends_on_instances, SubscriptionInstanceRelationTable.depends_on) + .filter(depends_on_instances.subscription_id.in_(set(subscription_ids))) + .filter(in_use_by_instances.subscription_id != depends_on_instances.subscription_id) + .filter(SubscriptionTable.status.in_(filter_statuses)) + ) + + return _get_instance_relations(query_get_in_use_by_ids) + + +async def _get_depends_on_instance_relations( + subscription_ids: list[UUID], filter_statuses: tuple[str, ...] +) -> list[Relation]: + """Get depends_on relations through subscription instance hierarchy.""" + in_use_by_instances = aliased(SubscriptionInstanceTable) + depends_on_instances = aliased(SubscriptionInstanceTable) + + query_get_depends_on_ids = ( + select(depends_on_instances.subscription_id, in_use_by_instances.subscription_id) + .distinct() + .join(depends_on_instances.subscription) + .join(depends_on_instances.in_use_by_block_relations) + .join(in_use_by_instances, SubscriptionInstanceRelationTable.in_use_by) + .filter(in_use_by_instances.subscription_id.in_(set(subscription_ids))) + .filter(depends_on_instances.subscription_id != in_use_by_instances.subscription_id) + .filter(SubscriptionTable.status.in_(filter_statuses)) + ) + + return _get_instance_relations(query_get_depends_on_ids) + + +def _get_resource_type_relations(resource_type_relations_query: Any) -> list[Relation]: + def to_relation(row: Row[Any]) -> Relation: + return Relation(UUID(row[0]), row[1]) + + return [to_relation(row) for row in db.session.execute(resource_type_relations_query)] + + +async def _get_in_use_by_resource_type_relations( + subscription_ids: list[UUID], filter_statuses: tuple[str, ...] +) -> list[Relation]: + """Get in_use_by relations through resource types.""" + logger.warning("Using legacy RELATION_RESOURCE_TYPES to find in_use_by subs") + + in_use_by_subscriptions = aliased(SubscriptionTable) + depends_on_instance_values = aliased(SubscriptionInstanceValueTable) + + # Convert UUIDs to string + unique_subscription_ids = set(map(str, subscription_ids)) + + query_get_in_use_by_ids = ( + select(depends_on_instance_values.value, in_use_by_subscriptions.subscription_id) + .select_from(depends_on_instance_values) + .join(SubscriptionInstanceTable) + .join(in_use_by_subscriptions) + .join(ResourceTypeTable) + .filter(ResourceTypeTable.resource_type.in_(RELATION_RESOURCE_TYPES)) + .filter(depends_on_instance_values.value.in_(unique_subscription_ids)) + .filter(in_use_by_subscriptions.status.in_(filter_statuses)) + ) + + return _get_resource_type_relations(query_get_in_use_by_ids) + + +async def _get_depends_on_resource_type_relations( + subscription_ids: list[UUID], filter_statuses: tuple[str, ...] +) -> list[Relation]: + """Get depends_on relations through resource types.""" + logger.warning("Using legacy RELATION_RESOURCE_TYPES to find depends_on subs") + + depends_on_subscriptions = aliased(SubscriptionTable) + in_use_by_instances = aliased(SubscriptionInstanceTable) + in_use_by_instance_values = aliased(SubscriptionInstanceValueTable) + + unique_subscription_ids = set(subscription_ids) + + query_get_depends_on_ids = ( + select(in_use_by_instance_values.value, in_use_by_instances.subscription_id) + .select_from(in_use_by_instance_values) + .join(in_use_by_instances) + .join( + depends_on_subscriptions, + in_use_by_instance_values.value == sa_cast(depends_on_subscriptions.subscription_id, SaText), + ) + .join(ResourceTypeTable) + .filter(ResourceTypeTable.resource_type.in_(RELATION_RESOURCE_TYPES)) + .filter(in_use_by_instances.subscription_id.in_(unique_subscription_ids)) + .filter(depends_on_subscriptions.status.in_(filter_statuses)) + ) + + return _get_resource_type_relations(query_get_depends_on_ids) + + +async def _get_in_use_by_relations(subscription_ids: list[UUID], filter_statuses: tuple[str, ...]) -> list[Relation]: + if RELATION_RESOURCE_TYPES: + # Find relations through resource types + resource_type_relations = await _get_in_use_by_resource_type_relations(subscription_ids, filter_statuses) + else: + resource_type_relations = [] + # Find relations through instance hierarchy + instance_relations = await _get_in_use_by_instance_relations(subscription_ids, filter_statuses) + return list(chain(resource_type_relations, instance_relations)) + + +async def _get_depends_on_relations(subscription_ids: list[UUID], filter_statuses: tuple[str, ...]) -> list[Relation]: + if RELATION_RESOURCE_TYPES: + # Find relations through resource types + resource_type_relations = await _get_depends_on_resource_type_relations(subscription_ids, filter_statuses) + else: + resource_type_relations = [] + # Find relations through instance hierarchy + instance_relations = await _get_depends_on_instance_relations(subscription_ids, filter_statuses) + return list(chain(resource_type_relations, instance_relations)) + + +async def get_in_use_by_subscriptions( + subscription_ids: list[UUID], filter_statuses: tuple[str, ...] +) -> list[list[SubscriptionTable]]: + """Function to efficiently get the in_use_by SubscriptionTables for multiple subscription_ids.""" + _filter_statuses: tuple[str, ...] = filter_statuses or tuple(SubscriptionLifecycle.values()) + in_use_by_relations = await _get_in_use_by_relations(subscription_ids, _filter_statuses) + + # Retrieve SubscriptionTable for all unique inuseby ids + unique_in_use_by_ids = {row.in_use_by_sub_id for row in in_use_by_relations} + _in_use_by_subs = db.session.execute( + select(SubscriptionTable).filter(SubscriptionTable.subscription_id.in_(unique_in_use_by_ids)) + ).scalars() + in_use_by_subs = {subscription.subscription_id: subscription for subscription in _in_use_by_subs} + + # group (more_itertools.bucket doesn't seem to work for tuple of uuids) + subscription_in_use_by_ids: dict[UUID, list[UUID]] = {} + for relation in in_use_by_relations: + subscription_in_use_by_ids.setdefault(relation.depends_on_sub_id, []).append(relation.in_use_by_sub_id) + + def get_in_use_by_subs(depends_on_id: UUID) -> list[SubscriptionTable]: + in_use_by_ids = subscription_in_use_by_ids.get(depends_on_id, []) + return [in_use_by_sub for id_ in in_use_by_ids if (in_use_by_sub := in_use_by_subs.get(id_))] + + # Important (as with any dataloader) + # Return the list of inuseby subs in the exact same order as the ids passed to this function + return [get_in_use_by_subs(subscription_id) for subscription_id in subscription_ids] + + +async def get_depends_on_subscriptions( + subscription_ids: list[UUID], filter_statuses: tuple[str, ...] +) -> list[list[SubscriptionTable]]: + """Function to efficiently get the depends_on SubscriptionTables for multiple subscription_ids.""" + _filter_statuses: tuple[str, ...] = filter_statuses or tuple(SubscriptionLifecycle.values()) + depends_on_relations = await _get_depends_on_relations(subscription_ids, _filter_statuses) + + # Retrieve SubscriptionTable for all unique dependson ids + unique_depends_on_ids = {row.depends_on_sub_id for row in depends_on_relations} + _depends_on_subs = db.session.execute( + select(SubscriptionTable).filter(SubscriptionTable.subscription_id.in_(unique_depends_on_ids)) + ).scalars() + depends_on_subs = {subscription.subscription_id: subscription for subscription in _depends_on_subs} + + # group (more_itertools.bucket doesn't seem to work for tuple of uuids) + subscription_depends_on_ids: dict[UUID, list[UUID]] = {} + for relation in depends_on_relations: + subscription_depends_on_ids.setdefault(relation.in_use_by_sub_id, []).append(relation.depends_on_sub_id) + + def get_depends_on_subs(in_use_by_id: UUID) -> list[SubscriptionTable]: + depends_on_ids = subscription_depends_on_ids.get(in_use_by_id, []) + return [depends_on_sub for id_ in depends_on_ids if (depends_on_sub := depends_on_subs.get(id_))] + + # Important (as with any dataloader) + # Return the list of dependson subs in the exact same order as the ids passed to this function + return [get_depends_on_subs(subscription_id) for subscription_id in subscription_ids] + + +async def get_recursive_relations( + subscription_ids: list[UUID], + filter_statuses: tuple[str, ...], + recurse_product_types: list[str], + recurse_depth_limit: int, + relation_fetcher: Callable[[list[UUID], tuple[str, ...]], Awaitable[list[list[SubscriptionTable]]]], + current_depth: int = 0, + used_subscription_ids: set[UUID] | None = None, +) -> list[SubscriptionTable]: + """Recursively fetches subscription relations based on a custom relation-fetching function. + + Args: + subscription_ids: List of subscription IDs to start with. + filter_statuses: List of statuses to filter the relations. + recurse_product_types: List of product types to recurse into. + recurse_depth_limit: Maximum recursion depth. + current_depth: Current recursion depth. + used_subscription_ids: List of already used subscription IDs to avoid cycles. + relation_fetcher: A function that fetches the relations for the subscriptions. + + Returns: + A flattened list of all fetched relations for the given subscriptions. + """ + + used_subscription_ids = used_subscription_ids or set(subscription_ids) + _relations = await relation_fetcher(subscription_ids, filter_statuses) + relations = list(unique_everseen(flatten(_relations), key=lambda s: s.subscription_id)) + + def get_related_subscription_ids() -> list[UUID]: + return [ + r.subscription_id + for r in relations + if r.product.product_type in recurse_product_types and r.subscription_id not in used_subscription_ids + ] + + nested_relations: list[SubscriptionTable] = [] + if recurse_depth_limit > current_depth and (related_subscription_ids := get_related_subscription_ids()): + used_subscription_ids.update(related_subscription_ids) + nested_relations = await get_recursive_relations( + related_subscription_ids, + filter_statuses, + recurse_product_types, + recurse_depth_limit, + current_depth=current_depth + 1, + used_subscription_ids=used_subscription_ids, + relation_fetcher=relation_fetcher, + ) + return list(unique_everseen(relations + nested_relations, key=lambda s: s.subscription_id)) diff --git a/test/unit_tests/conftest.py b/test/unit_tests/conftest.py index 2f8d328c9..2b7c5f834 100644 --- a/test/unit_tests/conftest.py +++ b/test/unit_tests/conftest.py @@ -113,6 +113,12 @@ test_union_product, test_union_type_product, ) +from test.unit_tests.fixtures.products.product_types.subscription_relations import ( # noqa: F401 + factory_subscription_with_nestings_depends_on, + factory_subscription_with_nestings_in_use_by, + test_product_model_list_nested_product_type_one, + test_product_model_list_nested_product_type_two, +) from test.unit_tests.fixtures.products.resource_types import ( # noqa: F401 resource_type_enum, resource_type_int, diff --git a/test/unit_tests/fixtures/products/product_blocks/product_block_list_nested.py b/test/unit_tests/fixtures/products/product_blocks/product_block_list_nested.py index 63bbc9067..32ddecc01 100644 --- a/test/unit_tests/fixtures/products/product_blocks/product_block_list_nested.py +++ b/test/unit_tests/fixtures/products/product_blocks/product_block_list_nested.py @@ -1,8 +1,12 @@ +from typing import Annotated + import pytest +import strawberry from orchestrator.db import ProductBlockTable, db from orchestrator.domain.base import ProductBlockModel from orchestrator.types import SubscriptionLifecycle +from test.unit_tests.helpers import safe_delete_product_block_id class ProductBlockListNestedForTestInactive(ProductBlockModel, product_block_name="ProductBlockListNestedForTest"): @@ -24,16 +28,31 @@ class ProductBlockListNestedForTest( int_field: int +ProductBlockListNestedForTestType = Annotated[ + "ProductBlockListNestedForTestInactiveGraphql", strawberry.lazy(".product_block_list_nested") +] + + +@strawberry.experimental.pydantic.type(model=ProductBlockListNestedForTestInactive) +class ProductBlockListNestedForTestInactiveGraphql: + sub_block_list: list[ProductBlockListNestedForTestType] + int_field: int + + @pytest.fixture -def test_product_block_list_nested(): +def test_product_block_list_nested(test_product_block_list_nested_db_in_use_by_block): # Classes defined at module level, otherwise they remain in local namespace and # `get_type_hints()` can't evaluate the ForwardRefs - return ( + yield ( ProductBlockListNestedForTestInactive, ProductBlockListNestedForTestProvisioning, ProductBlockListNestedForTest, ) + safe_delete_product_block_id(ProductBlockListNestedForTestInactive) + safe_delete_product_block_id(ProductBlockListNestedForTestProvisioning) + safe_delete_product_block_id(ProductBlockListNestedForTest) + @pytest.fixture def test_product_block_list_nested_db_in_use_by_block(resource_type_list, resource_type_int, resource_type_str): diff --git a/test/unit_tests/fixtures/products/product_types/product_type_list_nested.py b/test/unit_tests/fixtures/products/product_types/product_type_list_nested.py index 9a08cbdde..c1c4d9107 100644 --- a/test/unit_tests/fixtures/products/product_types/product_type_list_nested.py +++ b/test/unit_tests/fixtures/products/product_types/product_type_list_nested.py @@ -5,15 +5,14 @@ from orchestrator.domain.base import ProductModel, SubscriptionModel from orchestrator.domain.lifecycle import ProductLifecycle from orchestrator.types import SubscriptionLifecycle -from test.unit_tests.fixtures.products.product_blocks.product_block_list_nested import ( - ProductBlockListNestedForTest, - ProductBlockListNestedForTestInactive, - ProductBlockListNestedForTestProvisioning, -) @pytest.fixture -def test_product_type_list_nested(): +def test_product_type_list_nested(test_product_block_list_nested): + ProductBlockListNestedForTestInactive, ProductBlockListNestedForTestProvisioning, ProductBlockListNestedForTest = ( + test_product_block_list_nested + ) + class ProductTypeListNestedForTestInactive(SubscriptionModel, is_base=True): test_fixed_input: bool block: ProductBlockListNestedForTestInactive diff --git a/test/unit_tests/fixtures/products/product_types/product_type_list_union_overlap.py b/test/unit_tests/fixtures/products/product_types/product_type_list_union_overlap.py index 7cd23d3db..cdd4e2301 100644 --- a/test/unit_tests/fixtures/products/product_types/product_type_list_union_overlap.py +++ b/test/unit_tests/fixtures/products/product_types/product_type_list_union_overlap.py @@ -30,9 +30,9 @@ class ProductListUnion(ProductListUnionProvisioning, lifecycle=[SubscriptionLife test_block: ProductBlockOneForTest list_union_blocks: list_of_ports(ProductBlockOneForTest | SubBlockOneForTest) - SUBSCRIPTION_MODEL_REGISTRY["ProductListUnion"] = ProductListUnion + SUBSCRIPTION_MODEL_REGISTRY["ProductListUnionOverlap"] = ProductListUnion yield ProductListUnionInactive, ProductListUnionProvisioning, ProductListUnion - del SUBSCRIPTION_MODEL_REGISTRY["ProductListUnion"] + del SUBSCRIPTION_MODEL_REGISTRY["ProductListUnionOverlap"] @pytest.fixture diff --git a/test/unit_tests/fixtures/products/product_types/subscription_relations.py b/test/unit_tests/fixtures/products/product_types/subscription_relations.py new file mode 100644 index 000000000..ed14cc1fa --- /dev/null +++ b/test/unit_tests/fixtures/products/product_types/subscription_relations.py @@ -0,0 +1,300 @@ +from uuid import uuid4 + +import pytest + +from orchestrator.db import db +from orchestrator.db.models import FixedInputTable, ProductTable +from orchestrator.domain import SUBSCRIPTION_MODEL_REGISTRY +from orchestrator.domain.base import ProductModel, SubscriptionModel +from orchestrator.domain.lifecycle import ProductLifecycle +from orchestrator.types import SubscriptionLifecycle + + +@pytest.fixture +def test_product_model_list_nested_product_type_one( + test_product_block_list_nested_db_in_use_by_block, test_product_type_list_nested +): + product = ProductTable( + name="TestProductListNestedTypeOne", + description="Test ProductTable ProductTypeOne", + product_type="ProductTypeOne", + tag="TEST", + status="active", + ) + + fixed_input = FixedInputTable(name="test_fixed_input", value="1") + + product_block = test_product_block_list_nested_db_in_use_by_block + product.fixed_inputs = [fixed_input] + product.product_blocks = [product_block] + + db.session.add(product) + db.session.commit() + + _, _, ProductTypeListNestedForTest = test_product_type_list_nested + SUBSCRIPTION_MODEL_REGISTRY["TestProductListNestedTypeOne"] = ProductTypeListNestedForTest + yield ProductModel( + product_id=product.product_id, + name="TestProductListNestedTypeOne", + description=product.description, + product_type=product.product_type, + tag=product.tag, + status=ProductLifecycle.ACTIVE, + ) + del SUBSCRIPTION_MODEL_REGISTRY["TestProductListNestedTypeOne"] + + +@pytest.fixture +def test_product_model_list_nested_product_type_two( + test_product_block_list_nested_db_in_use_by_block, test_product_type_list_nested +): + product = ProductTable( + name="TestProductListNestedType2", + description="Test ProductTable ProductTypeTwo", + product_type="ProductTypeTwo", + tag="TEST", + status="active", + ) + + fixed_input = FixedInputTable(name="test_fixed_input", value="True") + + product_block = test_product_block_list_nested_db_in_use_by_block + product.fixed_inputs = [fixed_input] + product.product_blocks = [product_block] + + db.session.add(product) + db.session.commit() + + _, _, ProductTypeListNestedForTest = test_product_type_list_nested + SUBSCRIPTION_MODEL_REGISTRY["TestProductListNestedType2"] = ProductTypeListNestedForTest + yield ProductModel( + product_id=product.product_id, + name="TestProductListNestedType2", + description=product.description, + product_type=product.product_type, + tag=product.tag, + status=ProductLifecycle.ACTIVE, + ) + del SUBSCRIPTION_MODEL_REGISTRY["TestProductListNestedType2"] + + +@pytest.fixture +def factory_subscription_with_nestings_in_use_by( + test_product_block_list_nested, + test_product_model_list_nested, + test_product_type_list_nested, + test_product_block_list_nested_db_in_use_by_block, + test_product_model_list_nested_product_type_one, + test_product_model_list_nested_product_type_two, +): + """Fixture that creates subscriptions with multiple nestings. + + relations for in use by: + - subscription_10 + - subscription_20 + - subscription_30 + - subscription_40 + - subscription_41 - terminated + - subscription_31 - terminated + - subscription_42 + - subscription_32 - type: ProductTypeOne + - subscription_40 + - subscription_43 + + - subscription_21 - terminated + - subscription_33 + - subscription_44 + + - subscription_22 - type: ProductTypeOne + - subscription_34 - type: ProductTypeTwo + - subscription_45 + + Returns: dict with subscription ids + """ + + ProductTypeListNestedForTestInactive, _, _ = test_product_type_list_nested + ProductBlockListNestedForTestInactive, _, _ = test_product_block_list_nested + + customer_id = str(uuid4()) + + def create_subscription(*, int_value, sub_blocks=(), different_product_type=None, is_terminated=False): + product_id = test_product_model_list_nested.product_id + if different_product_type: + product_id = different_product_type.product_id + + subscription = ProductTypeListNestedForTestInactive.from_product_id( + product_id=product_id, customer_id=customer_id, insync=True + ) + subscription.block = ProductBlockListNestedForTestInactive.new( + subscription_id=subscription.subscription_id, int_field=int_value, sub_block_list=list(sub_blocks) + ) + + subscription = SubscriptionModel.from_other_lifecycle(subscription, SubscriptionLifecycle.ACTIVE) + if is_terminated: + subscription = SubscriptionModel.from_other_lifecycle(subscription, SubscriptionLifecycle.TERMINATED) + + subscription.save() + db.session.commit() + return subscription + + subscription_10 = create_subscription(int_value=10) + + subscription_20 = create_subscription(int_value=20, sub_blocks=[subscription_10.block]) + subscription_21 = create_subscription(int_value=21, sub_blocks=[subscription_10.block], is_terminated=True) + subscription_22 = create_subscription( + int_value=22, + sub_blocks=[subscription_10.block], + different_product_type=test_product_model_list_nested_product_type_one, + ) + + subscription_30 = create_subscription(int_value=30, sub_blocks=[subscription_20.block]) + subscription_31 = create_subscription(int_value=31, sub_blocks=[subscription_20.block], is_terminated=True) + subscription_32 = create_subscription( + int_value=32, + sub_blocks=[subscription_20.block], + different_product_type=test_product_model_list_nested_product_type_one, + ) + subscription_33 = create_subscription(int_value=33, sub_blocks=[subscription_21.block]) + subscription_34 = create_subscription( + int_value=34, + sub_blocks=[subscription_22.block], + different_product_type=test_product_model_list_nested_product_type_two, + ) + + subscription_40 = create_subscription(int_value=40, sub_blocks=[subscription_30.block, subscription_32.block]) + subscription_41 = create_subscription(int_value=41, sub_blocks=[subscription_30.block], is_terminated=True) + subscription_42 = create_subscription(int_value=42, sub_blocks=[subscription_31.block]) + subscription_43 = create_subscription(int_value=43, sub_blocks=[subscription_32.block]) + subscription_44 = create_subscription(int_value=44, sub_blocks=[subscription_33.block]) + subscription_45 = create_subscription(int_value=45, sub_blocks=[subscription_34.block]) + + return { + "subscription_10": subscription_10.subscription_id, + "subscription_20": subscription_20.subscription_id, + "subscription_21": subscription_21.subscription_id, + "subscription_22": subscription_22.subscription_id, + "subscription_30": subscription_30.subscription_id, + "subscription_31": subscription_31.subscription_id, + "subscription_32": subscription_32.subscription_id, + "subscription_33": subscription_33.subscription_id, + "subscription_34": subscription_34.subscription_id, + "subscription_40": subscription_40.subscription_id, + "subscription_41": subscription_41.subscription_id, + "subscription_42": subscription_42.subscription_id, + "subscription_43": subscription_43.subscription_id, + "subscription_44": subscription_44.subscription_id, + "subscription_45": subscription_45.subscription_id, + } + + +@pytest.fixture +def factory_subscription_with_nestings_depends_on( + test_product_block_list_nested, + test_product_model_list_nested, + test_product_type_list_nested, + test_product_block_list_nested_db_in_use_by_block, + test_product_model_list_nested_product_type_one, + test_product_model_list_nested_product_type_two, +): + """Fixture that creates subscriptions with multiple nestings. + + relations for depends on (default product type `Test`): + - subscription_40 + - subscription_30 + - subscription_20 + - subscription_10 + - subscription_11 - terminated + - subscription_21 - terminated + - subscription_12 + - subscription_22 - type: ProductTypeOne + - subscription_10 + - subscription_13 + + - subscription_31 - terminated + - subscription_23 + - subscription_14 + + - subscription_32 - type: ProductTypeOne + - subscription_24 - type: ProductTypeTwo + - subscription_15 + + Returns: dict with subscription ids + """ + ProductTypeListNestedForTestInactive, _, _ = test_product_type_list_nested + ProductBlockListNestedForTestInactive, _, _ = test_product_block_list_nested + + customer_id = str(uuid4()) + + def create_subscription(*, int_value, sub_blocks=(), different_product_type=None, is_terminated=False): + product_id = test_product_model_list_nested.product_id + if different_product_type: + product_id = different_product_type.product_id + + subscription = ProductTypeListNestedForTestInactive.from_product_id( + product_id=product_id, customer_id=customer_id, insync=True + ) + subscription.block = ProductBlockListNestedForTestInactive.new( + subscription_id=subscription.subscription_id, int_field=int_value, sub_block_list=list(sub_blocks) + ) + + subscription = SubscriptionModel.from_other_lifecycle(subscription, SubscriptionLifecycle.ACTIVE) + if is_terminated: + subscription = SubscriptionModel.from_other_lifecycle(subscription, SubscriptionLifecycle.TERMINATED) + + subscription.save() + db.session.commit() + return subscription + + subscription_10 = create_subscription(int_value=10) + subscription_11 = create_subscription(int_value=11, is_terminated=True) + subscription_12 = create_subscription(int_value=12) + subscription_13 = create_subscription(int_value=13) + subscription_14 = create_subscription(int_value=14) + subscription_15 = create_subscription(int_value=15) + + subscription_20 = create_subscription(int_value=20, sub_blocks=[subscription_10.block, subscription_11.block]) + subscription_21 = create_subscription(int_value=21, sub_blocks=[subscription_12.block], is_terminated=True) + subscription_22 = create_subscription( + int_value=22, + sub_blocks=[subscription_10.block, subscription_13.block], + different_product_type=test_product_model_list_nested_product_type_one, + ) + subscription_23 = create_subscription(int_value=23, sub_blocks=[subscription_14.block]) + subscription_24 = create_subscription( + int_value=24, + sub_blocks=[subscription_15.block], + different_product_type=test_product_model_list_nested_product_type_two, + ) + + subscription_30 = create_subscription( + int_value=30, sub_blocks=[subscription_20.block, subscription_21.block, subscription_22.block] + ) + subscription_31 = create_subscription(int_value=31, sub_blocks=[subscription_23.block], is_terminated=True) + subscription_32 = create_subscription( + int_value=32, + sub_blocks=[subscription_24.block], + different_product_type=test_product_model_list_nested_product_type_one, + ) + + # create subscription 40 that use subscription 30, 31 and 32 + subscription_40 = create_subscription( + int_value=40, sub_blocks=[subscription_30.block, subscription_31.block, subscription_32.block] + ) + + return { + "subscription_10": subscription_10.subscription_id, + "subscription_11": subscription_11.subscription_id, + "subscription_12": subscription_12.subscription_id, + "subscription_13": subscription_13.subscription_id, + "subscription_14": subscription_14.subscription_id, + "subscription_15": subscription_15.subscription_id, + "subscription_20": subscription_20.subscription_id, + "subscription_21": subscription_21.subscription_id, + "subscription_22": subscription_22.subscription_id, + "subscription_23": subscription_23.subscription_id, + "subscription_24": subscription_24.subscription_id, + "subscription_30": subscription_30.subscription_id, + "subscription_31": subscription_31.subscription_id, + "subscription_32": subscription_32.subscription_id, + "subscription_40": subscription_40.subscription_id, + } diff --git a/test/unit_tests/graphql/conftest.py b/test/unit_tests/graphql/conftest.py index 812322267..0809cfa0e 100644 --- a/test/unit_tests/graphql/conftest.py +++ b/test/unit_tests/graphql/conftest.py @@ -4,6 +4,9 @@ from orchestrator import app_settings from orchestrator.graphql.autoregistration import register_domain_models +from test.unit_tests.fixtures.products.product_blocks.product_block_list_nested import ( + ProductBlockListNestedForTestInactiveGraphql, +) @pytest.fixture(autouse=True) @@ -38,7 +41,13 @@ class Metadata(BaseModel): @pytest.fixture(scope="session", autouse=True) def fix_graphql_model_registration(): - internal_graphql_models = {} + # This block caches the "ProductBlockListNestedForTestInactive" model to avoid re-instantiation in each test case. + # This is necessary because this product block has a self referencing property, which strawberry can't handle correctly, + # and lead to an error expecting the `ProductBlockListNestedForTestInactive` strawberry type to already exist. + # This block caches the "ProductBlockListNestedForTestInactive" model to avoid re-instantiation in each test case. + # This is necessary because this product block has a self referencing property, which strawberry can't handle correctly, + # and lead to an error expecting the `ProductBlockListNestedForTestInactive` strawberry type to already exist. + internal_graphql_models = {"ProductBlockListNestedForTestInactive": ProductBlockListNestedForTestInactiveGraphql} def patched_register_domain_models(*args, **kwargs): graphql_models = register_domain_models(*args, **kwargs) diff --git a/test/unit_tests/graphql/test_subscription_relations.py b/test/unit_tests/graphql/test_subscription_relations.py new file mode 100644 index 000000000..e8c47a9bb --- /dev/null +++ b/test/unit_tests/graphql/test_subscription_relations.py @@ -0,0 +1,617 @@ +# Copyright 2022 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +from http import HTTPStatus + +import pytest + +from test.unit_tests.conftest import do_refresh_subscriptions_search_view + + +def assert_result_ids_against_expected_ids(result_ids, expected_ids): + assert sorted(result_ids) == sorted([str(id) for id in expected_ids]) + + +def build_subscriptions_relation_query(body: str) -> str: + return f""" +query SubscriptionQuery( + $first: Int!, + $after: Int!, + $sortBy: [GraphqlSort!], + $filterBy: [GraphqlFilter!], + $query: String, + $dependsOnFilter: SubscriptionRelationFilter, + $dependsOnSubscriptionsFilter: [GraphqlFilter!], + $inUseByFilter: SubscriptionRelationFilter, + $inUseBySubscriptionsFilter: [GraphqlFilter!], +) {{ + subscriptions(first: $first, after: $after, sortBy: $sortBy, filterBy: $filterBy, query: $query) + {body} +}} +""" + + +def get_subscriptions_query_with_processes( + first: int = 10, + after: int = 0, + filter_by: list[dict[str, str]] | None = None, + sort_by: list[dict[str, str]] | None = None, + query_string: str | None = None, +) -> bytes: + query = """ +query SubscriptionQuery( + $first: Int!, + $after: Int!, + $sortBy: [GraphqlSort!], + $filterBy: [GraphqlFilter!], + $query: String, +) { + subscriptions(first: $first, after: $after, sortBy: $sortBy, filterBy: $filterBy, query: $query) { + page { + description + subscriptionId + status + insync + note + startDate + endDate + productBlockInstances { + ownerSubscriptionId + inUseByRelations + } + processes(sortBy: [{field: "startedAt", order: ASC}]) { + page { + processId + isTask + lastStep + lastStatus + assignee + failedReason + traceback + workflowName + createdBy + startedAt + lastModifiedAt + product { + productId + name + description + productType + status + tag + createdAt + endDate + } + } + } + } + pageInfo { + startCursor + totalItems + hasPreviousPage + endCursor + hasNextPage + } + } +} + """ + return json.dumps( + { + "operationName": "SubscriptionQuery", + "query": query, + "variables": { + "first": first, + "after": after, + "sortBy": sort_by if sort_by else [], + "filterBy": filter_by if filter_by else [], + "query": query_string, + }, + } + ).encode("utf-8") + + +def get_subscriptions_query_with_relations( + first: int = 10, + after: int = 0, + filter_by: list[dict[str, str]] | None = None, + sort_by: list[dict[str, str]] | None = None, + query_string: str | None = None, + depends_on_filter: dict[str, str] | None = None, + depends_on_subscription_filter: dict[str, str] | None = None, + in_use_by_filter: dict[str, str] | None = None, + in_use_by_subscription_filter: dict[str, str] | None = None, +) -> bytes: + query = build_subscriptions_relation_query( + """{ + page { + description + subscriptionId + status + insync + note + startDate + endDate + dependsOnSubscriptions(dependsOnFilter: $dependsOnFilter, first: 20, filterBy: $dependsOnSubscriptionsFilter) { + page { + description + subscriptionId + status + insync + note + startDate + endDate + } + } + inUseBySubscriptions(inUseByFilter: $inUseByFilter, first: 20, filterBy: $inUseBySubscriptionsFilter) { + page { + description + subscriptionId + status + insync + note + startDate + endDate + } + } + } + pageInfo { + startCursor + totalItems + hasPreviousPage + endCursor + hasNextPage + } + } + """ + ) + return json.dumps( + { + "operationName": "SubscriptionQuery", + "query": query, + "variables": { + "first": first, + "after": after, + "sortBy": sort_by if sort_by else [], + "filterBy": filter_by if filter_by else [], + "query": query_string, + "dependsOnFilter": depends_on_filter, + "dependsOnSubscriptionsFilter": depends_on_subscription_filter, + "inUseByFilter": in_use_by_filter, + "inUseBySubscriptionsFilter": in_use_by_subscription_filter, + }, + } + ).encode("utf-8") + + +@pytest.mark.parametrize( + "query_args", + [ + lambda sid: {"filter_by": [{"field": "subscriptionId", "value": sid}]}, + lambda sid: {"query_string": sid}, + ], +) +def test_single_subscription_with_processes( + fastapi_app_graphql, + test_client, + product_type_1_subscriptions_factory, + mocked_processes, + mocked_processes_resumeall, # noqa: F811 + generic_subscription_2, # noqa: F811 + generic_subscription_1, + query_args, +): + # when + + product_type_1_subscriptions_factory(30) + subscription_id = generic_subscription_1 + + do_refresh_subscriptions_search_view() + + data = get_subscriptions_query_with_processes(**query_args(subscription_id)) + response = test_client.post("/api/graphql", content=data, headers={"Content-Type": "application/json"}) + + # then + + assert HTTPStatus.OK == response.status_code + result = response.json() + subscriptions_data = result["data"]["subscriptions"] + subscriptions = subscriptions_data["page"] + pageinfo = subscriptions_data["pageInfo"] + + assert "errors" not in result + assert len(subscriptions) == 1 + assert pageinfo == { + "hasPreviousPage": False, + "hasNextPage": False, + "startCursor": 0, + "endCursor": 0, + "totalItems": 1, + } + assert subscriptions[0]["subscriptionId"] == subscription_id + assert subscriptions[0]["processes"]["page"][0]["processId"] == str(mocked_processes[0]) + + +def test_single_subscription_with_depends_on_subscriptions( + fastapi_app_graphql, + test_client, + product_type_1_subscriptions_factory, + sub_one_subscription_1, + sub_two_subscription_1, + product_sub_list_union_subscription_1, +): + # when + + product_type_1_subscriptions_factory(30) + + do_refresh_subscriptions_search_view() + + subscription_id = str(product_sub_list_union_subscription_1) + data = get_subscriptions_query_with_relations(query_string=subscription_id) + response = test_client.post("/api/graphql", content=data, headers={"Content-Type": "application/json"}) + + expected_depends_on_ids = { + str(subscription.subscription_id) for subscription in [sub_one_subscription_1, sub_two_subscription_1] + } + # then + + assert HTTPStatus.OK == response.status_code + result = response.json() + subscriptions_data = result["data"]["subscriptions"] + subscriptions = subscriptions_data["page"] + pageinfo = subscriptions_data["pageInfo"] + + assert "errors" not in result + assert len(subscriptions) == 1 + assert pageinfo == { + "hasPreviousPage": False, + "hasNextPage": False, + "startCursor": 0, + "endCursor": 0, + "totalItems": 1, + } + assert subscriptions[0]["subscriptionId"] == subscription_id + depends_on_ids = subscriptions[0]["dependsOnSubscriptions"]["page"] + result_depends_on_ids = {subscription["subscriptionId"] for subscription in depends_on_ids} + assert result_depends_on_ids == expected_depends_on_ids + assert len(subscriptions[0]["inUseBySubscriptions"]["page"]) == 0 + + +def test_single_subscription_with_in_use_by_subscriptions( + fastapi_app_graphql, + test_client, + product_type_1_subscriptions_factory, + sub_one_subscription_1, + product_sub_list_union_subscription_1, +): + # when + + product_type_1_subscriptions_factory(30) + + subscription_id = str(sub_one_subscription_1.subscription_id) + subscription_query = get_subscriptions_query_with_relations( + filter_by=[{"field": "subscriptionId", "value": subscription_id}] + ) + + response = test_client.post( + "/api/graphql", content=subscription_query, headers={"Content-Type": "application/json"} + ) + + expected_in_use_by_ids = [str(product_sub_list_union_subscription_1)] + + # then + + assert HTTPStatus.OK == response.status_code + result = response.json() + subscriptions_data = result["data"]["subscriptions"] + subscriptions = subscriptions_data["page"] + pageinfo = subscriptions_data["pageInfo"] + + assert "errors" not in result + assert len(subscriptions) == 1 + assert pageinfo == { + "hasPreviousPage": False, + "hasNextPage": False, + "startCursor": 0, + "endCursor": 0, + "totalItems": 1, + } + assert subscriptions[0]["subscriptionId"] == subscription_id + assert len(subscriptions[0]["dependsOnSubscriptions"]["page"]) == 0 + result_in_use_by_ids = [ + subscription["subscriptionId"] for subscription in subscriptions[0]["inUseBySubscriptions"]["page"] + ] + assert result_in_use_by_ids == expected_in_use_by_ids + + +in_use_by_recurse_all_ids = [ + "subscription_20", + "subscription_21", + "subscription_22", + "subscription_30", + "subscription_31", + "subscription_32", + "subscription_33", + "subscription_34", + "subscription_40", + "subscription_41", + "subscription_42", + "subscription_43", + "subscription_44", + "subscription_45", +] +in_use_by_recurse_all_with_filter_by_ids = [ + "subscription_22", + "subscription_32", +] +in_use_by_recurse_status_active_ids = [ + "subscription_20", + "subscription_22", + "subscription_30", + "subscription_32", + "subscription_34", + "subscription_40", + "subscription_43", + "subscription_45", +] +in_use_by_recurse_depth_limit_ids = [ + "subscription_20", + "subscription_21", + "subscription_22", + "subscription_30", + "subscription_31", + "subscription_32", + "subscription_33", + "subscription_34", +] +in_use_by_recurse_only_product_type_test_ids = [ + "subscription_20", + "subscription_21", + "subscription_22", + "subscription_30", + "subscription_31", + "subscription_32", + "subscription_33", + "subscription_40", + "subscription_41", + "subscription_42", + "subscription_44", +] + + +@pytest.mark.parametrize( + "in_use_by_filter,in_use_by_subscription_filter,subscription_ids,subscription_statuses", + [ + ( + {"recurseProductTypes": ["Test", "ProductTypeOne", "ProductTypeTwo"]}, + None, + in_use_by_recurse_all_ids, + {"ACTIVE", "TERMINATED"}, + ), + ( + {"recurseProductTypes": ["Test", "ProductTypeOne", "ProductTypeTwo"]}, + [{"field": "product", "value": "TestProductListNestedTypeOne"}], + in_use_by_recurse_all_with_filter_by_ids, + {"ACTIVE"}, + ), + ( + {"statuses": ["active"], "recurseProductTypes": ["Test", "ProductTypeOne", "ProductTypeTwo"]}, + None, + in_use_by_recurse_status_active_ids, + {"ACTIVE"}, + ), + ( + {"recurseProductTypes": ["Test", "ProductTypeOne", "ProductTypeTwo"], "recurseDepthLimit": 1}, + None, + in_use_by_recurse_depth_limit_ids, + {"ACTIVE", "TERMINATED"}, + ), + ( + {"recurseProductTypes": ["Test"]}, + None, + in_use_by_recurse_only_product_type_test_ids, + {"ACTIVE", "TERMINATED"}, + ), + ], + ids=[ + "recurse_all", + "recurse_all_with_filter_by", + "recurse_status_active", + "recurse_depth_limit", + "recurse_only_product_type_test", + ], +) +def test_single_subscription_with_in_use_by_subscriptions_recurse( + fastapi_app_graphql, + test_client, + factory_subscription_with_nestings_in_use_by, + in_use_by_filter, + in_use_by_subscription_filter, + subscription_ids, + subscription_statuses, +): + # when + + all_ids = factory_subscription_with_nestings_in_use_by + + subscription_id = str(all_ids["subscription_10"]) + subscription_query = get_subscriptions_query_with_relations( + filter_by=[{"field": "subscriptionId", "value": subscription_id}], + in_use_by_filter=in_use_by_filter, + in_use_by_subscription_filter=in_use_by_subscription_filter, + ) + + response = test_client.post( + "/api/graphql", content=subscription_query, headers={"Content-Type": "application/json"} + ) + + expected_relation_ids = [all_ids[sub_id] for sub_id in subscription_ids] + + # then + + assert HTTPStatus.OK == response.status_code + result = response.json() + subscriptions_data = result["data"]["subscriptions"] + subscriptions = subscriptions_data["page"] + + assert "errors" not in result + assert subscriptions[0]["subscriptionId"] == subscription_id + result_related_ids = [ + subscription["subscriptionId"] for subscription in subscriptions[0]["inUseBySubscriptions"]["page"] + ] + assert_result_ids_against_expected_ids(result_related_ids, expected_relation_ids) + result_related_statuses = { + subscription["status"] for subscription in subscriptions[0]["inUseBySubscriptions"]["page"] + } + assert result_related_statuses == subscription_statuses + + +depends_on_recurse_all_ids = [ + "subscription_10", + "subscription_11", + "subscription_12", + "subscription_13", + "subscription_14", + "subscription_15", + "subscription_20", + "subscription_21", + "subscription_22", + "subscription_23", + "subscription_24", + "subscription_30", + "subscription_31", + "subscription_32", +] +depends_on_recurse_all_with_filter_by_ids = [ + "subscription_22", + "subscription_32", +] +depends_on_recurse_status_active_ids = [ + "subscription_10", + "subscription_13", + "subscription_15", + "subscription_20", + "subscription_22", + "subscription_24", + "subscription_30", + "subscription_32", +] +depends_on_recurse_depth_limit_ids = [ + "subscription_20", + "subscription_21", + "subscription_22", + "subscription_23", + "subscription_24", + "subscription_30", + "subscription_31", + "subscription_32", +] +depends_on_recurse_only_product_type_test_ids = [ + "subscription_10", + "subscription_11", + "subscription_12", + "subscription_14", + "subscription_20", + "subscription_21", + "subscription_22", + "subscription_23", + "subscription_30", + "subscription_31", + "subscription_32", +] + + +@pytest.mark.parametrize( + "depends_on_filter,depends_on_subscription_filter,subscription_ids,subscription_statuses", + [ + ( + {"recurseProductTypes": ["Test", "ProductTypeOne", "ProductTypeTwo"]}, + None, + depends_on_recurse_all_ids, + {"ACTIVE", "TERMINATED"}, + ), + ( + {"recurseProductTypes": ["Test", "ProductTypeOne", "ProductTypeTwo"]}, + [{"field": "product", "value": "TestProductListNestedTypeOne"}], + depends_on_recurse_all_with_filter_by_ids, + {"ACTIVE"}, + ), + ( + {"statuses": ["active"], "recurseProductTypes": ["Test", "ProductTypeOne", "ProductTypeTwo"]}, + None, + depends_on_recurse_status_active_ids, + {"ACTIVE"}, + ), + ( + {"recurseProductTypes": ["Test", "ProductTypeOne", "ProductTypeTwo"], "recurseDepthLimit": 1}, + None, + depends_on_recurse_depth_limit_ids, + {"ACTIVE", "TERMINATED"}, + ), + ( + {"recurseProductTypes": ["Test"]}, + None, + depends_on_recurse_only_product_type_test_ids, + {"ACTIVE", "TERMINATED"}, + ), + ], + ids=[ + "recurse_all", + "recurse_all_with_filter_by", + "recurse_status_active", + "recurse_depth_limit", + "recurse_only_product_type_test", + ], +) +def test_single_subscription_with_depends_on_subscriptions_recurse( + fastapi_app_graphql, + test_client, + factory_subscription_with_nestings_depends_on, + depends_on_filter, + depends_on_subscription_filter, + subscription_ids, + subscription_statuses, +): + # when + + all_ids = factory_subscription_with_nestings_depends_on + + subscription_id = str(all_ids["subscription_40"]) + subscription_query = get_subscriptions_query_with_relations( + filter_by=[{"field": "subscriptionId", "value": subscription_id}], + depends_on_filter=depends_on_filter, + depends_on_subscription_filter=depends_on_subscription_filter, + ) + + response = test_client.post( + "/api/graphql", content=subscription_query, headers={"Content-Type": "application/json"} + ) + + expected_relation_ids = [all_ids[sub_id] for sub_id in subscription_ids] + + # then + + assert HTTPStatus.OK == response.status_code + result = response.json() + subscriptions_data = result["data"]["subscriptions"] + subscriptions = subscriptions_data["page"] + + assert "errors" not in result + assert subscriptions[0]["subscriptionId"] == subscription_id + result_related_ids = [ + subscription["subscriptionId"] for subscription in subscriptions[0]["dependsOnSubscriptions"]["page"] + ] + assert_result_ids_against_expected_ids(result_related_ids, expected_relation_ids) + result_related_statuses = { + subscription["status"] for subscription in subscriptions[0]["dependsOnSubscriptions"]["page"] + } + assert result_related_statuses == subscription_statuses diff --git a/test/unit_tests/helpers.py b/test/unit_tests/helpers.py index 325d0f371..58df5283e 100644 --- a/test/unit_tests/helpers.py +++ b/test/unit_tests/helpers.py @@ -11,6 +11,19 @@ def assert_no_diff(expected, actual, exclude_paths=None): assert diff == {}, f"Difference between expected and actual output\n{prettydiff}" +def safe_delete_product_block_id(product_block_class): + """Safely delete product_block_id from product block class if its defined. + + When a product block is not defined within a fixture function, the product_block_id + is stored inside the class and is kept through multiple tests, + which results in a foreign key error product block does not exist. + """ + try: + del product_block_class.product_block_id + except AttributeError: + pass + + # By default Pydantic v2 includes documentation urls in the errors. # Update these urls when upgrading Pydantic. URL_MISSING = {"url": mock.ANY} diff --git a/test/unit_tests/services/test_subscription_relations.py b/test/unit_tests/services/test_subscription_relations.py new file mode 100644 index 000000000..c8bc642b6 --- /dev/null +++ b/test/unit_tests/services/test_subscription_relations.py @@ -0,0 +1,190 @@ +from orchestrator.db import db +from orchestrator.domain.base import SubscriptionModel +from orchestrator.services.subscription_relations import get_depends_on_subscriptions, get_in_use_by_subscriptions +from orchestrator.types import SubscriptionLifecycle + + +def terminate_subscription(subscription_id): + subscription = SubscriptionModel.from_subscription(subscription_id) + terminated_subscription = SubscriptionModel.from_other_lifecycle(subscription, SubscriptionLifecycle.TERMINATED) + terminated_subscription.save() + db.session.commit() + return terminated_subscription + + +async def test_get_in_use_by_subscriptions( + sub_one_subscription_1, + sub_two_subscription_1, + product_sub_list_union_subscription_1, + sub_list_union_overlap_subscription_1, +): + # when + subscription_ids = [sub_one_subscription_1.subscription_id, sub_two_subscription_1.subscription_id] + + terminate_subscription(product_sub_list_union_subscription_1) + terminate_subscription(sub_two_subscription_1.subscription_id) + + result = await get_in_use_by_subscriptions(subscription_ids, ()) + + # then + expected_result = [ + # sub_one_subscription_1 in_use_by_subscriptions + sorted([product_sub_list_union_subscription_1, sub_list_union_overlap_subscription_1.subscription_id]), + # sub_two_subscription_1 in_use_by_subscriptions + [product_sub_list_union_subscription_1], + ] + + assert [sorted([sub.subscription_id for sub in r_list]) for r_list in result] == expected_result + + +async def test_get_in_use_by_subscriptions_only_active( + sub_one_subscription_1, + sub_two_subscription_1, + product_sub_list_union_subscription_1, + sub_list_union_overlap_subscription_1, +): + # when + subscription_ids = [sub_one_subscription_1.subscription_id, sub_two_subscription_1.subscription_id] + + terminate_subscription(product_sub_list_union_subscription_1) + terminate_subscription(sub_two_subscription_1.subscription_id) + + result = await get_in_use_by_subscriptions(subscription_ids, ("active",)) + + # then + expected_result = [ + # sub_one_subscription_1 in_use_by_subscriptions + sorted( + [ + sub_list_union_overlap_subscription_1.subscription_id, # ACTIVE + ] + ), + # sub_two_subscription_1 in_use_by_subscriptions - terminated + [], + ] + + assert [sorted([sub.subscription_id for sub in r_list]) for r_list in result] == expected_result + + +async def test_get_in_use_by_subscriptions_only_terminated( + sub_one_subscription_1, + sub_two_subscription_1, + product_sub_list_union_subscription_1, + sub_list_union_overlap_subscription_1, +): + # when + subscription_ids = [sub_one_subscription_1.subscription_id, sub_two_subscription_1.subscription_id] + + terminate_subscription(product_sub_list_union_subscription_1) + terminate_subscription(sub_two_subscription_1.subscription_id) + + result = await get_in_use_by_subscriptions(subscription_ids, ("terminated",)) + + # then + expected_result = [ + # sub_one_subscription_1 in_use_by_subscriptions + [product_sub_list_union_subscription_1], + # sub_two_subscription_1 in_use_by_subscriptions - terminated + [product_sub_list_union_subscription_1], + ] + + assert [sorted([sub.subscription_id for sub in r_list]) for r_list in result] == expected_result + + +async def test_get_in_use_by_subscriptions_empty(): + # when + subscription_ids = [] + + result = await get_in_use_by_subscriptions(subscription_ids, ()) + + # then + expected_result = [] + + assert result == expected_result + + +async def test_get_depends_on_subscriptions( + sub_one_subscription_1, + sub_two_subscription_1, + product_sub_list_union_subscription_1, + sub_list_union_overlap_subscription_1, +): + # when + subscription_ids = [product_sub_list_union_subscription_1, sub_list_union_overlap_subscription_1.subscription_id] + + result = await get_depends_on_subscriptions(subscription_ids, ()) + + # then + expected_result = [ + # product_sub_list_union_subscription_1 in_use_by_subscriptions + sorted([sub_two_subscription_1.subscription_id, sub_one_subscription_1.subscription_id]), + # sub_list_union_overlap_subscription_1 in_use_by_subscriptions + [sub_one_subscription_1.subscription_id], + ] + + assert [sorted([sub.subscription_id for sub in r_list]) for r_list in result] == expected_result + + +async def test_get_depends_on_subscriptions_only_active( + sub_one_subscription_1, + sub_two_subscription_1, + product_sub_list_union_subscription_1, + sub_list_union_overlap_subscription_1, +): + # when + subscription_ids = [product_sub_list_union_subscription_1, sub_list_union_overlap_subscription_1.subscription_id] + terminate_subscription(product_sub_list_union_subscription_1) + terminate_subscription(sub_two_subscription_1.subscription_id) + + result = await get_depends_on_subscriptions(subscription_ids, ("active",)) + + # then + expected_result = [ + # product_sub_list_union_subscription_1 in_use_by_subscriptions - terminated + [sub_one_subscription_1.subscription_id], + # sub_list_union_overlap_subscription_1 in_use_by_subscriptions + [sub_one_subscription_1.subscription_id], + ] + + assert [sorted([sub.subscription_id for sub in r_list]) for r_list in result] == expected_result + + +async def test_get_depends_on_subscriptions_only_terminated( + sub_one_subscription_1, + sub_two_subscription_1, + product_sub_list_union_subscription_1, + sub_list_union_overlap_subscription_1, +): + # when + subscription_ids = [product_sub_list_union_subscription_1, sub_list_union_overlap_subscription_1.subscription_id] + terminate_subscription(product_sub_list_union_subscription_1) + terminate_subscription(sub_two_subscription_1.subscription_id) + + result = await get_depends_on_subscriptions(subscription_ids, ("terminated",)) + + # then + expected_result = [ + # product_sub_list_union_subscription_1 in_use_by_subscriptions - terminated + sorted( + [ + sub_two_subscription_1.subscription_id, # TERMINATED + # sub_one_subscription_1.subscription_id # ACTIVE + ] + ), + # sub_list_union_overlap_subscription_1 in_use_by_subscriptions sub_one_subscription_1 + [], + ] + + assert [sorted([sub.subscription_id for sub in r_list]) for r_list in result] == expected_result + + +async def test_get_depends_on_subscriptions_empty(): + # when + subscription_ids = [] + + result = await get_depends_on_subscriptions(subscription_ids, ()) + + # then + expected_result = [] + + assert result == expected_result From 5e98018e2bc12260da8db6fc51b5a4de3c20342f Mon Sep 17 00:00:00 2001 From: Peter Boers Date: Mon, 14 Oct 2024 08:44:39 -0400 Subject: [PATCH 3/6] Update .bumpversion.cfg (#761) --- .bumpversion.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 52d33c8f7..81a0a83c4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.7.6 +current_version = 2.8.0rc1 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(rc(?P\d+))? From cf0d944466f7834ff9fa77e8623a6583e63628c4 Mon Sep 17 00:00:00 2001 From: Peter Boers Date: Mon, 14 Oct 2024 08:44:56 -0400 Subject: [PATCH 4/6] Update __init__.py (#762) --- orchestrator/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestrator/__init__.py b/orchestrator/__init__.py index 1c8739eb3..96a1b5979 100644 --- a/orchestrator/__init__.py +++ b/orchestrator/__init__.py @@ -13,7 +13,7 @@ """This is the orchestrator workflow engine.""" -__version__ = "2.7.6" +__version__ = "2.8.0rc1" from orchestrator.app import OrchestratorCore from orchestrator.settings import app_settings From 0d7f799db34dc679aa8e504606d73bc77628f73f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 08:45:34 -0400 Subject: [PATCH 5/6] Bump typer from 0.12.3 to 0.12.5 (#760) Bumps [typer](https://github.com/fastapi/typer) from 0.12.3 to 0.12.5. - [Release notes](https://github.com/fastapi/typer/releases) - [Changelog](https://github.com/fastapi/typer/blob/master/docs/release-notes.md) - [Commits](https://github.com/fastapi/typer/compare/0.12.3...0.12.5) --- updated-dependencies: - dependency-name: typer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Peter Boers --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e824c9a53..72d837878 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ dependencies = [ "SQLAlchemy==2.0.31", "SQLAlchemy-Utils==0.41.2", "structlog", - "typer==0.12.3", + "typer==0.12.5", "uvicorn[standard]~=0.30.1", "nwa-stdlib~=1.7.3", "oauth2-lib~=2.1.0", From 17694c123a7bb89f0f6c08db13cac95b0b9e5911 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 08:46:03 -0400 Subject: [PATCH 6/6] Update pydantic-settings requirement from ~=2.4.0 to ~=2.5.2 (#748) Updates the requirements on [pydantic-settings](https://github.com/pydantic/pydantic-settings) to permit the latest version. - [Release notes](https://github.com/pydantic/pydantic-settings/releases) - [Commits](https://github.com/pydantic/pydantic-settings/compare/v2.4.0...v2.5.2) --- updated-dependencies: - dependency-name: pydantic-settings dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Peter Boers --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 72d837878..f2f0183d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "orjson==3.10.7", "psycopg2-binary==2.9.9", "pydantic[email]~=2.7.4", - "pydantic-settings~=2.4.0", + "pydantic-settings~=2.5.2", "python-dateutil==2.8.2", "python-rapidjson>=1.18,<1.20", "pytz==2024.1",