feat: import resources from git repo or archive file

dioptra.toml

@@ -0,0 +1,24 @@
+[plugins]
+
+[plugins.hello_world]
+description = "The simplest possible plugin"
+
+[[plugins.hello_world.tasks]]
+name = "greet"
+description = "Print a greeting"
+
+input_params = [
+ { name = "greeting", type = "string" }
+ { name = "name", type = "string" }
+]
+output_params = [ { name = "message", type = "string" } ]
+
+[entrypoints]
+
+[entrypoints.hello-world]
+description = "Use the hello_world plugin to print a message."
+params = [
+ { name = "greeting", type = "string", default = "hello" },
+ { name = "name", type = "string", default = "world" }
+]
+plugins = [ "hello_world" ]

entrypoints/hello-world.yaml

@@ -0,0 +1,3 @@
+message:
+ greet:
+ name: $name

plugins/hello_world/__init__.py

@@ -0,0 +1,16 @@
+# This Software (Dioptra) is being made available as a public service by the
+# National Institute of Standards and Technology (NIST), an Agency of the United
+# States Department of Commerce. This software was developed in part by employees of
+# NIST and in part by NIST contractors. Copyright in portions of this software that
+# were developed by NIST contractors has been licensed or assigned to NIST. Pursuant
+# to Title 17 United States Code Section 105, works of NIST employees are not
+# subject to copyright protection in the United States. However, NIST may hold
+# international copyright in software created by its employees and domestic
+# copyright (or licensing rights) in portions of software that were assigned or
+# licensed to NIST. To the extent that NIST holds copyright in this software, it is
+# being made available under the Creative Commons Attribution 4.0 International
+# license (CC BY 4.0). The disclaimers of the CC BY 4.0 license apply to all parts
+# of the software developed or licensed by NIST.
+#
+# ACCESS THE FULL CC BY 4.0 LICENSE HERE:
+# https://creativecommons.org/licenses/by/4.0/legalcode

plugins/hello_world/tasks.py

@@ -0,0 +1,27 @@
+# This Software (Dioptra) is being made available as a public service by the
+# National Institute of Standards and Technology (NIST), an Agency of the United
+# States Department of Commerce. This software was developed in part by employees of
+# NIST and in part by NIST contractors. Copyright in portions of this software that
+# were developed by NIST contractors has been licensed or assigned to NIST. Pursuant
+# to Title 17 United States Code Section 105, works of NIST employees are not
+# subject to copyright protection in the United States. However, NIST may hold
+# international copyright in software created by its employees and domestic
+# copyright (or licensing rights) in portions of software that were assigned or
+# licensed to NIST. To the extent that NIST holds copyright in this software, it is
+# being made available under the Creative Commons Attribution 4.0 International
+# license (CC BY 4.0). The disclaimers of the CC BY 4.0 license apply to all parts
+# of the software developed or licensed by NIST.
+#
+# ACCESS THE FULL CC BY 4.0 LICENSE HERE:
+# https://creativecommons.org/licenses/by/4.0/legalcode
+import structlog
+from dioptra import pyplugs
+
+LOGGER = structlog.get_logger()
+
+
+@pyplugs.register()
+def greet(greeting: str, name: str) -> str:
+ message = f"Hello, {name}"
+ LOGGER.info(message)
+ return message

src/dioptra/restapi/utils.py

@@ -305,6 +305,7 @@ def setup_injection(api: Api, injector: Injector) -> None:
  ma.Decimal: float,
  ma.Dict: dict,
  ma.Email: str,
+ ma.Enum: str,
  FileUpload: FileStorage,
  ma.Float: float,
  ma.Function: str,

src/dioptra/restapi/v1/plugin_parameter_types/service.py

@@ -555,6 +555,56 @@ def __init__(
  """
  self._group_id_service = group_id_service
 
+ def get(
+ self,
+ group_id: int,
+ error_if_not_found: bool = False,
+ **kwargs,
+ ) -> models.PluginTaskParameterType | None:
+ """Fetch a list of plugin parameter types by their names.
+
+ Args:
+ group_id: The the group id of the plugin parameter type.
+ error_if_not_found: If True, raise an error if the plugin parameter
+ type is not found. Defaults to False.
+
+ Returns:
+ The plugin parameter type object if found, otherwise None.
+
+ Raises:
+ PluginParameterTypeDoesNotExistError: If the plugin parameter type
+ is not found and `error_if_not_found` is True.
+ """
+ log: BoundLogger = kwargs.get("log", LOGGER.new())
+ log.debug(
+ "Get builtin plugin parameter types",
+ group_id=group_id,
+ )
+
+ builtin_types = list(BUILTIN_TYPES.keys())
+
+ stmt = (
+ select(models.PluginTaskParameterType)
+ .join(models.Resource)
+ .where(
+ models.PluginTaskParameterType.name.in_(builtin_types),
+ models.Resource.group_id == group_id,
+ models.Resource.is_deleted == False, # noqa: E712
+ models.Resource.latest_snapshot_id
+ == models.PluginTaskParameterType.resource_snapshot_id,
+ )
+ )
+ plugin_parameter_types = list(db.session.scalars(stmt).all())
+
+ if len(plugin_parameter_types) != len(builtin_types):
+ retrieved_names = {param_type.name for param_type in plugin_parameter_types}
+ missing_names = set(builtin_types) - retrieved_names
+ if error_if_not_found:
+ log.debug("Plugin Parameter Type(s) not found", names=missing_names)
+ raise PluginParameterTypeDoesNotExistError
+
+ return plugin_parameter_types
+
  def create_all(
  self,
  user: models.User,

src/dioptra/restapi/v1/plugins/service.py

@@ -656,12 +656,11 @@ def get(
  return plugin_file
 
 
-class PluginIdFileService(object):
+class PluginFileService(object):
  @inject
  def __init__(
  self,
  plugin_file_name_service: PluginFileNameService,
- plugin_id_service: PluginIdService,
  group_id_service: GroupIdService,
  ) -> None:
  """Initialize the plugin file service.
@@ -670,11 +669,9 @@ def __init__(
 
  Args:
  plugin_file_name_service: A PluginFileNameService object.
- plugin_id_service: A PluginIdService object.
  group_id_service: A GroupIdService object.
  """
  self._plugin_file_name_service = plugin_file_name_service
- self._plugin_id_service = plugin_id_service
  self._group_id_service = group_id_service
 
  def create(
@@ -683,7 +680,7 @@ def create(
  contents: str,
  description: str,
  tasks: list[dict[str, Any]],
- plugin_id: int,
+ plugin: models.Plugin,
  commit: bool = True,
  **kwargs,
  ) -> utils.PluginFileDict:
@@ -709,13 +706,7 @@ def create(
  """
 
  log: BoundLogger = kwargs.get("log", LOGGER.new())
- log.debug("Create a new plugin file", plugin_id=plugin_id, filename=filename)
-
- # Check if plugin_id points to a Plugin that exists, and if so, retrieve it.
- plugin_dict = cast(
- utils.PluginWithFilesDict,
- self._plugin_id_service.get(plugin_id, error_if_not_found=True),
- )
+ plugin_id = plugin.resource_id
 
  # Validate that the proposed filename hasn't already been used in the plugin.
  if (
@@ -729,7 +720,6 @@ def create(
 
  # The owner of the PluginFile resource must match the owner of the Plugin
  # resource.
- plugin = plugin_dict["plugin"]
  resource = models.Resource(
  resource_type=PLUGIN_FILE_RESOURCE_TYPE, owner=plugin.resource.owner
  )
@@ -758,6 +748,79 @@ def create(
  plugin_file=new_plugin_file, plugin=plugin, has_draft=False
  )
 
+
+class PluginIdFileService(object):
+ @inject
+ def __init__(
+ self,
+ plugin_id_service: PluginIdService,
+ plugin_file_service: PluginFileService,
+ group_id_service: GroupIdService,
+ ) -> None:
+ """Initialize the plugin file service.
+
+ All arguments are provided via dependency injection.
+
+ Args:
+ plugin_id_service: A PluginIdService object.
+ plugin_file_service: A PluginFileService object.
+ group_id_service: A GroupIdService object.
+ """
+ self._plugin_id_service = plugin_id_service
+ self._plugin_file_service = plugin_file_service
+ self._group_id_service = group_id_service
+
+ def create(
+ self,
+ filename: str,
+ contents: str,
+ description: str,
+ tasks: list[dict[str, Any]],
+ plugin_id: int,
+ commit: bool = True,
+ **kwargs,
+ ) -> utils.PluginFileDict:
+ """Creates a new PluginFile object.
+
+ The PluginFile object will belong to the group that owns the plugin associated
+ with the PluginFile. The creator will be the current user.
+
+ Args:
+ filename: The name of the plugin file.
+ contents: The contents of the plugin file.
+ description: The description of the plugin file.
+ tasks: The tasks associated with the plugin file.
+ plugin_id: The unique id of the plugin containing the plugin file.
+ commit: If True, commit the transaction. Defaults to True.
+
+ Returns:
+ The newly created plugin file object.
+
+ Raises:
+ PluginFileAlreadyExistsError: If a plugin file with the given filename
+ already exists.
+ """
+
+ log: BoundLogger = kwargs.get("log", LOGGER.new())
+ log.debug("Create a new plugin file", plugin_id=plugin_id, filename=filename)
+
+ # Check if plugin_id points to a Plugin that exists, and if so, retrieve it.
+ plugin_dict = cast(
+ utils.PluginWithFilesDict,
+ self._plugin_id_service.get(plugin_id, error_if_not_found=True),
+ )
+ plugin = plugin_dict["plugin"]
+
+ return self._plugin_file_service.create(
+ filename=filename,
+ contents=contents,
+ description=description,
+ tasks=tasks,
+ plugin=plugin,
+ commit=commit,
+ log=log,
+ )
+
  def get(
  self,
  plugin_id: int,

src/dioptra/restapi/v1/workflows/controller.py

@@ -25,8 +25,14 @@
 from injector import inject
 from structlog.stdlib import BoundLogger
 
-from .schema import FileTypes, JobFilesDownloadQueryParametersSchema
-from .service import JobFilesDownloadService
+from .schema import (
+ FileTypes,
+ JobFilesDownloadQueryParametersSchema,
+ ResourceImportSchema,
+)
+from .service import JobFilesDownloadService, ResourceImportService
+
+from dioptra.restapi.utils import as_api_parser, as_parameters_schema_list
 
 LOGGER: BoundLogger = structlog.stdlib.get_logger()
 
@@ -78,3 +84,49 @@ def get(self):
  mimetype=mimetype[parsed_query_params["file_type"]],
  download_name=download_name[parsed_query_params["file_type"]],
  )
+
+
+@api.route("/resourceImport")
+class ResourceImport(Resource):
+ @inject
+ def __init__(
+ self, resource_import_service: ResourceImportService, *args, **kwargs
+ ) -> None:
+ """Initialize the workflow resource.
+
+ All arguments are provided via dependency injection.
+
+ Args:
+ resource_import_service: A ResourceImportService object.
+ """
+ self._resource_import_service = resource_import_service
+ super().__init__(*args, **kwargs)
+
+ @login_required
+ @api.expect(
+ as_api_parser(
+ api,
+ as_parameters_schema_list(
+ ResourceImportSchema, operation="load", location="form"
+ ),
+ )
+ )
+ @accepts(form_schema=ResourceImportSchema, api=api)
+ def post(self):
+ """Import resources from an external source.""" # noqa: B950
+ log = LOGGER.new( # noqa: F841
+ request_id=str(uuid.uuid4()), resource="ResourceImport", request_type="POST"
+ )
+ parsed_form = request.parsed_form
+
+ return self._resource_import_service.import_resources(
+ group_id=parsed_form["group_id"],
+ source_type=parsed_form["source_type"],
+ git_url=parsed_form.get("git_url", None),
+ archive_file=request.files.get("archiveFile", None),
+ read_only=parsed_form["read_only"],
+ resolve_name_conflicts_strategy=parsed_form[
+ "resolve_name_conflicts_strategy"
+ ],
+ log=log,
+ )

src/dioptra/restapi/v1/workflows/lib/__init__.py

@@ -15,12 +15,14 @@
 # ACCESS THE FULL CC BY 4.0 LICENSE HERE:
 # https://creativecommons.org/licenses/by/4.0/legalcode
 from . import views
+from .clone_git_repository import clone_git_repository
 from .export_plugin_files import export_plugin_files
 from .export_task_engine_yaml import export_task_engine_yaml
 from .package_job_files import package_job_files
 
 __all__ = [
  "views",
+ "clone_git_repository",
  "export_plugin_files",
  "export_task_engine_yaml",
  "package_job_files",