feat: import resources from git repo or archive file

dioptra.toml

@@ -0,0 +1,38 @@
+[[plugins]]
+path = "plugins/hello_world"
+description = "A simple plugin used for testing and demonstration purposes."
+
+
+ [[plugins.tasks]]
+ filename = "tasks.py"
+ name = "hello"
+ input_params = [ { name = "name", type = "string", required = true} ]
+ output_params = [ { name = "message", type = "string" } ]
+
+ [[plugins.tasks]]
+ filename = "tasks.py"
+ name = "greet"
+ input_params = [
+ { name = "greeting", type = "string", required = true },
+ { name = "name", type = "string", required = true },
+ ]
+ output_params = [ { name = "message", type = "string" } ]
+
+ [[plugins.tasks]]
+ filename = "tasks.py"
+ name = "shout"
+ input_params = [ { name = "message", type = "string", required = true} ]
+ output_params = [ { name = "message", type = "string" } ]
+
+[[plugin_param_types]]
+name = "image"
+structure = { list = ["int", "int", "int"] }
+
+[[entrypoints]]
+path = "examples/hello-world.yaml"
+name = "Hello World"
+description = "A simple example using the hello_world plugin."
+params = [
+ { name = "name", type = "string", default_value = "World" }
+]
+plugins = [ "hello_world" ]

entrypoints/hello-world.yaml

@@ -0,0 +1,3 @@
+message:
+ greet:
+ name: $name

examples/hello-world.yaml

@@ -0,0 +1,10 @@
+hello_step:
+ hello:
+ name: $name
+goodbye_step:
+ greet:
+ greeting: Goodbye
+ name: $name
+shout_step:
+ shout:
+ message: $goodbye_step.message

plugins/hello_world/__init__.py

@@ -0,0 +1,16 @@
+# This Software (Dioptra) is being made available as a public service by the
+# National Institute of Standards and Technology (NIST), an Agency of the United
+# States Department of Commerce. This software was developed in part by employees of
+# NIST and in part by NIST contractors. Copyright in portions of this software that
+# were developed by NIST contractors has been licensed or assigned to NIST. Pursuant
+# to Title 17 United States Code Section 105, works of NIST employees are not
+# subject to copyright protection in the United States. However, NIST may hold
+# international copyright in software created by its employees and domestic
+# copyright (or licensing rights) in portions of software that were assigned or
+# licensed to NIST. To the extent that NIST holds copyright in this software, it is
+# being made available under the Creative Commons Attribution 4.0 International
+# license (CC BY 4.0). The disclaimers of the CC BY 4.0 license apply to all parts
+# of the software developed or licensed by NIST.
+#
+# ACCESS THE FULL CC BY 4.0 LICENSE HERE:
+# https://creativecommons.org/licenses/by/4.0/legalcode

plugins/hello_world/tasks.py

@@ -0,0 +1,41 @@
+# This Software (Dioptra) is being made available as a public service by the
+# National Institute of Standards and Technology (NIST), an Agency of the United
+# States Department of Commerce. This software was developed in part by employees of
+# NIST and in part by NIST contractors. Copyright in portions of this software that
+# were developed by NIST contractors has been licensed or assigned to NIST. Pursuant
+# to Title 17 United States Code Section 105, works of NIST employees are not
+# subject to copyright protection in the United States. However, NIST may hold
+# international copyright in software created by its employees and domestic
+# copyright (or licensing rights) in portions of software that were assigned or
+# licensed to NIST. To the extent that NIST holds copyright in this software, it is
+# being made available under the Creative Commons Attribution 4.0 International
+# license (CC BY 4.0). The disclaimers of the CC BY 4.0 license apply to all parts
+# of the software developed or licensed by NIST.
+#
+# ACCESS THE FULL CC BY 4.0 LICENSE HERE:
+# https://creativecommons.org/licenses/by/4.0/legalcode
+import structlog
+from dioptra import pyplugs
+
+LOGGER = structlog.get_logger()
+
+
+@pyplugs.register()
+def hello(name: str) -> str:
+ message = f"Hello, {name}"
+ LOGGER.info(message)
+ return message
+
+
+@pyplugs.register()
+def greet(greeting: str, name: str) -> str:
+ message = f"{greeting}, {name}"
+ LOGGER.info(message)
+ return message
+
+
+@pyplugs.register()
+def shout(message: str) -> str:
+ message = message.upper()
+ LOGGER.info(message)
+ return message

src/dioptra/restapi/utils.py

@@ -305,6 +305,7 @@ def setup_injection(api: Api, injector: Injector) -> None:
  ma.Decimal: float,
  ma.Dict: dict,
  ma.Email: str,
+ ma.Enum: str,
  FileUpload: FileStorage,
  ma.Float: float,
  ma.Function: str,

src/dioptra/restapi/v1/plugin_parameter_types/service.py

@@ -580,6 +580,56 @@ def __init__(
  """
  self._group_id_service = group_id_service
 
+ def get(
+ self,
+ group_id: int,
+ error_if_not_found: bool = False,
+ **kwargs,
+ ) -> models.PluginTaskParameterType | None:
+ """Fetch a list of plugin parameter types by their names.
+
+ Args:
+ group_id: The the group id of the plugin parameter type.
+ error_if_not_found: If True, raise an error if the plugin parameter
+ type is not found. Defaults to False.
+
+ Returns:
+ The plugin parameter type object if found, otherwise None.
+
+ Raises:
+ PluginParameterTypeDoesNotExistError: If the plugin parameter type
+ is not found and `error_if_not_found` is True.
+ """
+ log: BoundLogger = kwargs.get("log", LOGGER.new())
+ log.debug(
+ "Get builtin plugin parameter types",
+ group_id=group_id,
+ )
+
+ builtin_types = list(BUILTIN_TYPES.keys())
+
+ stmt = (
+ select(models.PluginTaskParameterType)
+ .join(models.Resource)
+ .where(
+ models.PluginTaskParameterType.name.in_(builtin_types),
+ models.Resource.group_id == group_id,
+ models.Resource.is_deleted == False, # noqa: E712
+ models.Resource.latest_snapshot_id
+ == models.PluginTaskParameterType.resource_snapshot_id,
+ )
+ )
+ plugin_parameter_types = list(db.session.scalars(stmt).all())
+
+ if len(plugin_parameter_types) != len(builtin_types):
+ retrieved_names = {param_type.name for param_type in plugin_parameter_types}
+ missing_names = set(builtin_types) - retrieved_names
+ if error_if_not_found:
+ log.debug("Plugin Parameter Type(s) not found", names=missing_names)
+ raise PluginParameterTypeDoesNotExistError
+
+ return plugin_parameter_types
+
  def create_all(
  self,
  user: models.User,

src/dioptra/restapi/v1/workflows/controller.py

@@ -25,8 +25,14 @@
 from injector import inject
 from structlog.stdlib import BoundLogger
 
-from .schema import FileTypes, JobFilesDownloadQueryParametersSchema
-from .service import JobFilesDownloadService
+from .schema import (
+ FileTypes,
+ JobFilesDownloadQueryParametersSchema,
+ ResourceImportSchema,
+)
+from .service import JobFilesDownloadService, ResourceImportService
+
+from dioptra.restapi.utils import as_api_parser, as_parameters_schema_list
 
 LOGGER: BoundLogger = structlog.stdlib.get_logger()
 
@@ -78,3 +84,51 @@ def get(self):
  mimetype=mimetype[parsed_query_params["file_type"]],
  download_name=download_name[parsed_query_params["file_type"]],
  )
+
+
+@api.route("/resourceImport")
+class ResourceImport(Resource):
+ @inject
+ def __init__(
+ self, resource_import_service: ResourceImportService, *args, **kwargs
+ ) -> None:
+ """Initialize the workflow resource.
+
+ All arguments are provided via dependency injection.
+
+ Args:
+ resource_import_service: A ResourceImportService object.
+ """
+ self._resource_import_service = resource_import_service
+ super().__init__(*args, **kwargs)
+
+ @login_required
+ @api.expect(
+ as_api_parser(
+ api,
+ as_parameters_schema_list(
+ ResourceImportSchema, operation="load", location="form"
+ ),
+ )
+ )
+ @accepts(form_schema=ResourceImportSchema, api=api)
+ def post(self):
+ """Import resources from an external source.""" # noqa: B950
+ log = LOGGER.new( # noqa: F841
+ request_id=str(uuid.uuid4()), resource="ResourceImport", request_type="POST"
+ )
+ parsed_form = request.parsed_form
+
+ log.info("HERE")
+ return self._resource_import_service.import_resources(
+ group_id=parsed_form["group_id"],
+ source_type=parsed_form["source_type"],
+ git_url=parsed_form.get("git_url", None),
+ archive_file=request.files.get("archiveFile", None),
+ config_path=parsed_form["config_path"],
+ read_only=parsed_form["read_only"],
+ resolve_name_conflicts_strategy=parsed_form[
+ "resolve_name_conflicts_strategy"
+ ],
+ log=log,
+ )

src/dioptra/restapi/v1/workflows/lib/__init__.py

@@ -15,12 +15,14 @@
 # ACCESS THE FULL CC BY 4.0 LICENSE HERE:
 # https://creativecommons.org/licenses/by/4.0/legalcode
 from . import views
+from .clone_git_repository import clone_git_repository
 from .export_plugin_files import export_plugin_files
 from .export_task_engine_yaml import export_task_engine_yaml
 from .package_job_files import package_job_files
 
 __all__ = [
  "views",
+ "clone_git_repository",
  "export_plugin_files",
  "export_task_engine_yaml",
  "package_job_files",

src/dioptra/restapi/v1/workflows/lib/clone_git_repository.py

@@ -0,0 +1,60 @@
+from urllib.parse import urlparse
+from pathlib import Path
+import subprocess
+
+
+def clone_git_repository(url: str, dir: Path):
+ parsed_url = urlparse(url)
+ git_branch = parsed_url.fragment or None
+ git_paths = parsed_url.params or None
+ # note: username and password being used for authentication is currently not
+ # supported. we need to consider how this info needs to be stored
+ git_url = parsed_url._replace(
+ username=None, password=None, fragment="", params=""
+ ).geturl()
+
+ git_sparse_args = ["--filter=blob:none", "--no-checkout", "--depth=1"]
+ git_branch_args = ["-b", git_branch] if git_branch else []
+ clone_cmd = ["git", "clone", *git_sparse_args, *git_branch_args, git_url, dir]
+ clone_result = subprocess.run(clone_cmd, capture_output=True, text=True)
+
+ if clone_result.returncode != 0:
+ raise subprocess.CalledProcessError(
+ clone_result.returncode, clone_result.stderr
+ )
+
+ if git_paths is not None:
+ paths = git_paths.split(",")
+ sparse_checkout_cmd = ["git", "sparse-checkout", "set", "--cone", *paths]
+ sparse_checkout_result = subprocess.run(
+ sparse_checkout_cmd, cwd=dir, capture_output=True, text=True
+ )
+
+ if sparse_checkout_result.returncode != 0:
+ raise subprocess.CalledProcessError(
+ sparse_checkout_result.returncode, sparse_checkout_result.stderr
+ )
+
+ checkout_cmd = ["git", "checkout"]
+ checkout_result = subprocess.run(
+ checkout_cmd, cwd=dir, capture_output=True, text=True
+ )
+
+ if checkout_result.returncode != 0:
+ raise subprocess.CalledProcessError(
+ checkout_result.returncode, checkout_result.stderr
+ )
+
+ hash_cmd = ["git", "rev-parse", "HEAD"]
+ hash_result = subprocess.run(hash_cmd, cwd=dir, capture_output=True, text=True)
+
+ if hash_result.returncode != 0:
+ raise subprocess.CalledProcessError
+
+ return hash
+
+
+if __name__ == "__main__":
+ clone_git_repository(
+ "https://github.com/usnistgov/dioptra.git;plugins#dev", Path("dioptra-plugins")
+ )