diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eab2d0452..bb22cd71a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,4 +11,4 @@ repos: rev: 24.8.0 hooks: - id: black - types: [file, python] + types_or: [python, pyi] diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..3a413de1e --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include perun *.pyi +include perun/py.typed \ No newline at end of file diff --git a/docs/_static/templates/degradation_api.py b/docs/_static/templates/degradation_api.py index c53f86c05..083539335 100644 --- a/docs/_static/templates/degradation_api.py +++ b/docs/_static/templates/degradation_api.py @@ -1,6 +1,6 @@ """...""" -from perun.utils.structs import DegradationInfo +from perun.utils.structs.common_structs import DegradationInfo def my_degradation_checker(baseline_profile, target_profile): diff --git a/docs/cli.rst b/docs/cli.rst index 8e036c16f..2706d43ef 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -69,7 +69,7 @@ Perun Commands Collect Commands ---------------- -.. click:: perun.cli:collect +.. click:: perun.cli_groups.collect_cli:collect :prog: perun collect .. _cli-collect-units-ref: diff --git a/docs/degradation.rst b/docs/degradation.rst index 2eb8038b5..cc098046d 100644 --- a/docs/degradation.rst +++ b/docs/degradation.rst @@ -233,10 +233,10 @@ just small requirements and have to `yield` the reports about degradation as a i ``DegradationInfo`` objects specified as follows: .. currentmodule: perun.utils.structs -.. autoclass:: perun.utils.structs.DegradationInfo +.. autoclass:: perun.utils.structs.common_structs.DegradationInfo :members: -.. autoclass:: perun.utils.structs.PerformanceChange +.. autoclass:: perun.utils.structs.common_structs.PerformanceChange :members: You can register your new performance change checker as follows: diff --git a/meson.build b/meson.build index 2569ef13a..d56077f1e 100644 --- a/meson.build +++ b/meson.build @@ -18,6 +18,7 @@ perun_files = files( 'LICENSE', 'pyproject.toml', 'tox.ini', + 'MANIFEST.in', ) perun_dir = 'perun' diff --git a/perun/check/__init__.py b/perun/check/__init__.py index f25953a46..109e5d57b 100644 --- a/perun/check/__init__.py +++ b/perun/check/__init__.py @@ -2,3 +2,7 @@ Contains the actual methods in isolate modules, and then a factory module, containing helper and generic stuff.""" + +import lazy_loader as lazy + +__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__) diff --git a/perun/check/__init__.pyi b/perun/check/__init__.pyi new file mode 100644 index 000000000..a2e21c14d --- /dev/null +++ b/perun/check/__init__.pyi @@ -0,0 +1,21 @@ +from .detection_kit import ( + create_filter_by_model as create_filter_by_model, + create_model_record as create_model_record, + get_filtered_best_models_of as get_filtered_best_models_of, + get_function_values as get_function_values, + general_detection as general_detection, +) +from .factory import ( + pre_collect_profiles as pre_collect_profiles, + degradation_in_minor as degradation_in_minor, + degradation_in_history as degradation_in_history, + degradation_between_profiles as degradation_between_profiles, + run_degradation_check as run_degradation_check, + degradation_between_files as degradation_between_files, + is_rule_applicable_for as is_rule_applicable_for, + run_detection_with_strategy as run_detection_with_strategy, +) +from .nonparam_kit import ( + classify_change as classify_change, + preprocess_nonparam_models as preprocess_nonparam_models, +) diff --git a/perun/check/detection_kit.py b/perun/check/detection_kit.py index ef10a2119..fd14bc273 100644 --- a/perun/check/detection_kit.py +++ b/perun/check/detection_kit.py @@ -21,7 +21,7 @@ from perun.postprocess.regression_analysis import regression_models from perun.profile import query from perun.utils.common import common_kit -from perun.utils.structs import ( +from perun.utils.structs.common_structs import ( PerformanceChange, DegradationInfo, ModelRecord, diff --git a/perun/check/factory.py b/perun/check/factory.py index 99aef55ed..219a9404a 100644 --- a/perun/check/factory.py +++ b/perun/check/factory.py @@ -27,7 +27,7 @@ polynomial_regression, ) from perun.utils import decorators, log -from perun.utils.structs import ( +from perun.utils.structs.common_structs import ( DetectionChangeResult, DegradationInfo, PerformanceChange, @@ -61,34 +61,6 @@ def __call__( """Call Function""" -def get_supported_detection_models_strategies() -> list[str]: - """ - Provides supported detection models strategies to execute - the degradation check between two profiles with different kinds - of models. The individual strategies represent the way of - executing the detection between profiles and their models: - - - best-param: best parametric models from both profiles - - best-non-param: best non-parametric models from both profiles - - best-model: best models from both profiles - - all-param: all parametric models pair from both profiles - - all-non-param: all non-parametric models pair from both profiles - - all-models: all models pair from both profiles - - best-both: best parametric and non-parametric models from both profiles - - :return: the names of all supported degradation models strategies - """ - return [ - "best-model", - "best-param", - "best-nonparam", - "all-param", - "all-nonparam", - "all-models", - "best-both", - ] - - def profiles_to_queue( minor_version: str, ) -> dict[tuple[str, str, str, str], ProfileInfo]: diff --git a/perun/check/meson.build b/perun/check/meson.build index 07acc152b..84a4f1098 100644 --- a/perun/check/meson.build +++ b/perun/check/meson.build @@ -2,6 +2,7 @@ perun_check_dir = perun_dir / 'check' perun_check_files = files( '__init__.py', + '__init__.pyi', 'factory.py', 'detection_kit.py', 'nonparam_kit.py', diff --git a/perun/check/methods/abstract_base_checker.py b/perun/check/methods/abstract_base_checker.py index 7f98427a9..9d81f4098 100755 --- a/perun/check/methods/abstract_base_checker.py +++ b/perun/check/methods/abstract_base_checker.py @@ -16,7 +16,7 @@ # Perun Imports if TYPE_CHECKING: from perun.profile.factory import Profile - from perun.utils.structs import DegradationInfo + from perun.utils.structs.common_structs import DegradationInfo class AbstractBaseChecker(ABC): diff --git a/perun/check/methods/average_amount_threshold.py b/perun/check/methods/average_amount_threshold.py index 7296915f8..956e1098f 100644 --- a/perun/check/methods/average_amount_threshold.py +++ b/perun/check/methods/average_amount_threshold.py @@ -46,7 +46,7 @@ from perun.check.methods.abstract_base_checker import AbstractBaseChecker from perun.profile import convert from perun.utils.common import common_kit -from perun.utils.structs import DegradationInfo, PerformanceChange +from perun.utils.structs.common_structs import DegradationInfo, PerformanceChange if TYPE_CHECKING: from perun.profile.factory import Profile diff --git a/perun/check/methods/best_model_order_equality.py b/perun/check/methods/best_model_order_equality.py index a9011b3d6..d89eb6567 100644 --- a/perun/check/methods/best_model_order_equality.py +++ b/perun/check/methods/best_model_order_equality.py @@ -48,9 +48,9 @@ # Third-Party Imports # Perun Imports +from perun import check as check from perun.check.methods.abstract_base_checker import AbstractBaseChecker -from perun.utils.structs import DegradationInfo, PerformanceChange -import perun.check.detection_kit as detection +from perun.utils.structs.common_structs import DegradationInfo, PerformanceChange if TYPE_CHECKING: from perun.profile.factory import Profile @@ -81,10 +81,8 @@ def check( :param _: unification with other detection methods (unused in this method) :returns: tuple (degradation result, degradation location, degradation rate) """ - best_baseline_models = detection.get_filtered_best_models_of( - baseline_profile, group="param" - ) - best_target_models = detection.get_filtered_best_models_of(target_profile, group="param") + best_baseline_models = check.get_filtered_best_models_of(baseline_profile, group="param") + best_target_models = check.get_filtered_best_models_of(target_profile, group="param") for uid, best_model in best_target_models.items(): best_baseline_model = best_baseline_models.get(uid) diff --git a/perun/check/methods/exclusive_time_outliers.py b/perun/check/methods/exclusive_time_outliers.py index e31b4a15d..e7440eac6 100644 --- a/perun/check/methods/exclusive_time_outliers.py +++ b/perun/check/methods/exclusive_time_outliers.py @@ -79,7 +79,7 @@ from perun.check.methods.abstract_base_checker import AbstractBaseChecker from perun.logic import config from perun.profile import convert -from perun.utils.structs import DegradationInfo, PerformanceChange +from perun.utils.structs.common_structs import DegradationInfo, PerformanceChange if TYPE_CHECKING: from perun.profile.factory import Profile diff --git a/perun/check/methods/fast_check.py b/perun/check/methods/fast_check.py index b86f998c2..08a2c6fcd 100644 --- a/perun/check/methods/fast_check.py +++ b/perun/check/methods/fast_check.py @@ -14,10 +14,10 @@ import numpy as np # Perun Imports +from perun import check as check from perun.check.methods.abstract_base_checker import AbstractBaseChecker from perun.logic import runner -from perun.utils.structs import DegradationInfo, ClassificationMethod -import perun.check.detection_kit as detect +from perun.utils.structs.common_structs import DegradationInfo, ClassificationMethod if TYPE_CHECKING: from perun.profile.factory import Profile @@ -35,7 +35,7 @@ def check( :param _: unification with other detection methods (unused in this method) :returns: tuple (degradation result, degradation location, degradation rate, confidence) """ - return detect.general_detection( + return check.general_detection( baseline_profile, target_profile, ClassificationMethod.FastCheck ) diff --git a/perun/check/methods/integral_comparison.py b/perun/check/methods/integral_comparison.py index f07017162..da216267c 100644 --- a/perun/check/methods/integral_comparison.py +++ b/perun/check/methods/integral_comparison.py @@ -14,10 +14,10 @@ # Perun Imports from perun.check.methods.abstract_base_checker import AbstractBaseChecker -from perun.check import factory, nonparam_kit as nparam_helpers +from perun import check as check from perun.postprocess.regression_analysis import regression_models from perun.utils.common import common_kit -from perun.utils.structs import DegradationInfo, ModelRecord, DetectionChangeResult +from perun.utils.structs.common_structs import DegradationInfo, ModelRecord, DetectionChangeResult if TYPE_CHECKING: from perun.profile.factory import Profile @@ -95,7 +95,7 @@ def execute_analysis( :return: tuple with degradation info between a pair of models: (deg. result, deg. location, deg. rate, confidence type and rate, etc.) """ - x_pts, baseline_y_pts, target_y_pts = nparam_helpers.preprocess_nonparam_models( + x_pts, baseline_y_pts, target_y_pts = check.preprocess_nonparam_models( uid, baseline_model, target_profile, target_model ) @@ -114,7 +114,7 @@ def execute_analysis( float(target_integral - baseline_integral), float(baseline_integral) ) - change_info = nparam_helpers.classify_change( + change_info = check.classify_change( rel_error if np.isfinite(rel_error) else 0, _INTEGRATE_DIFF_NO_CHANGE, _INTEGRATE_DIFF_CHANGE, @@ -142,7 +142,7 @@ def check( :param _: other kwgargs :returns: tuple - degradation result (structure DegradationInfo) """ - for degradation_info in factory.run_detection_with_strategy( + for degradation_info in check.run_detection_with_strategy( execute_analysis, baseline_profile, target_profile, models_strategy ): yield degradation_info diff --git a/perun/check/methods/linear_regression.py b/perun/check/methods/linear_regression.py index 00457a4c5..c5e16a1ae 100644 --- a/perun/check/methods/linear_regression.py +++ b/perun/check/methods/linear_regression.py @@ -14,11 +14,11 @@ from scipy import stats # Perun Imports -from perun.check import detection_kit as detect +from perun import check as check from perun.check.methods import fast_check from perun.check.methods.abstract_base_checker import AbstractBaseChecker from perun.utils.common import common_kit -from perun.utils.structs import DegradationInfo, ModelRecord, ClassificationMethod +from perun.utils.structs.common_structs import DegradationInfo, ModelRecord, ClassificationMethod if TYPE_CHECKING: import numpy @@ -40,7 +40,7 @@ def check( :returns: tuple (degradation result, degradation location, degradation rate, confidence) """ - return detect.general_detection( + return check.general_detection( baseline_profile, target_profile, ClassificationMethod.LinearRegression ) @@ -107,15 +107,15 @@ def exec_linear_regression( uid, baseline_profile, baseline_x_pts, lin_abs_error ) # obtaining the models (linear and quadratic) from the new regressed profile - quad_err_model = detect.get_filtered_best_models_of( + quad_err_model = check.get_filtered_best_models_of( std_err_profile, group="param", - model_filter=detect.create_filter_by_model("quadratic"), + model_filter=check.create_filter_by_model("quadratic"), ) - linear_err_model = detect.get_filtered_best_models_of( + linear_err_model = check.get_filtered_best_models_of( std_err_profile, group="param", - model_filter=detect.create_filter_by_model("linear"), + model_filter=check.create_filter_by_model("linear"), ) # check the last quadratic type of change @@ -127,7 +127,7 @@ def exec_linear_regression( # We did not classify the change if not change_type: - std_err_model = detect.get_filtered_best_models_of(std_err_profile, group="param") + std_err_model = check.get_filtered_best_models_of(std_err_profile, group="param") change_type = std_err_model[uid].type return change_type diff --git a/perun/check/methods/local_statistics.py b/perun/check/methods/local_statistics.py index 9a997b0ba..1f6aa42b6 100644 --- a/perun/check/methods/local_statistics.py +++ b/perun/check/methods/local_statistics.py @@ -13,12 +13,11 @@ from scipy import integrate # Perun Imports -from perun.check import factory +from perun import check as check from perun.check.methods.abstract_base_checker import AbstractBaseChecker from perun.profile.factory import Profile from perun.utils.common import common_kit -from perun.utils.structs import DegradationInfo, ModelRecord, DetectionChangeResult -import perun.check.nonparam_kit as nparam_helpers +from perun.utils.structs.common_structs import DegradationInfo, ModelRecord, DetectionChangeResult if TYPE_CHECKING: import numpy.typing as npt @@ -141,7 +140,7 @@ def classify_stats_diff( """ # create vectorized functions which take a np.arrays as inputs and perform actions over it compare_diffs = np.vectorize(compare_diff_values) - classify_change = np.vectorize(nparam_helpers.classify_change) + classify_change = np.vectorize(check.classify_change) stat_no = len(baseline_stats.keys()) stat_size = baseline_stats.get(list(baseline_stats.keys())[0]) @@ -227,7 +226,7 @@ def execute_analysis( original_x_pts, baseline_y_pts, target_y_pts, - ) = nparam_helpers.preprocess_nonparam_models(uid, baseline_model, target_profile, target_model) + ) = check.preprocess_nonparam_models(uid, baseline_model, target_profile, target_model) baseline_window_stats, _ = compute_window_stats(original_x_pts, baseline_y_pts) target_window_stats, x_pts = compute_window_stats(original_x_pts, target_y_pts) @@ -238,7 +237,7 @@ def execute_analysis( x_pts_odd = x_pts[:, 1::2].reshape(-1, x_pts.size // 2)[0].round(2) partial_intervals = list(np.array((change_info, partial_rel_error, x_pts_even, x_pts_odd)).T) - change_info_enum = nparam_helpers.classify_change( + change_info_enum = check.classify_change( common_kit.safe_division(float(np.sum(partial_rel_error)), partial_rel_error.size), _STATS_DIFF_NO_CHANGE, _STATS_DIFF_CHANGE, @@ -268,7 +267,7 @@ def check( :param models_strategy: detection model strategy for obtains the relevant kind of models :returns: tuple - degradation result """ - for degradation_info in factory.run_detection_with_strategy( + for degradation_info in check.run_detection_with_strategy( execute_analysis, baseline_profile, target_profile, models_strategy ): yield degradation_info diff --git a/perun/check/methods/polynomial_regression.py b/perun/check/methods/polynomial_regression.py index e49916523..8cca36dde 100644 --- a/perun/check/methods/polynomial_regression.py +++ b/perun/check/methods/polynomial_regression.py @@ -13,9 +13,9 @@ import numpy as np # Perun Imports +from perun import check as check from perun.check.methods.abstract_base_checker import AbstractBaseChecker -from perun.utils.structs import DegradationInfo, ClassificationMethod -import perun.check.detection_kit as detect +from perun.utils.structs.common_structs import DegradationInfo, ClassificationMethod if TYPE_CHECKING: import numpy.typing as npt @@ -37,7 +37,7 @@ def check( :param _: unification with other detection methods (unused in this method) :returns: tuple (degradation result, degradation location, degradation rate, confidence) """ - return detect.general_detection( + return check.general_detection( baseline_profile, target_profile, ClassificationMethod.PolynomialRegression, diff --git a/perun/check/nonparam_kit.py b/perun/check/nonparam_kit.py index 14d02d6ba..fd64184dc 100644 --- a/perun/check/nonparam_kit.py +++ b/perun/check/nonparam_kit.py @@ -8,10 +8,10 @@ import numpy as np # Perun Imports +from perun import check as check from perun.postprocess.regression_analysis import data_provider from perun.utils import log -from perun.utils.structs import PerformanceChange, ModelRecord -import perun.check.detection_kit as methods +from perun.utils.structs.common_structs import PerformanceChange, ModelRecord import perun.postprocess.regressogram.methods as rg_methods if TYPE_CHECKING: @@ -107,7 +107,7 @@ def unify_buckets_in_regressogram( # match the regressogram model with the right 'uid' model = [model for model in new_regressogram_models if model["uid"] == uid][0] - return methods.create_model_record(model) + return check.create_model_record(model) def preprocess_nonparam_models( @@ -146,7 +146,7 @@ def get_model_coordinates(model: ModelRecord) -> tuple[list[float], list[float]] :return: obtained x and y coordinates - x-points, y-points """ if model.b1 is not None: - x_pts, y_pts = methods.get_function_values(model) + x_pts, y_pts = check.get_function_values(model) else: x_pts = np.linspace(model.x_start, model.x_end, num=len(model.b0)) y_pts = model.b0 diff --git a/perun/cli.py b/perun/cli.py index cb412ac1b..14c36a544 100644 --- a/perun/cli.py +++ b/perun/cli.py @@ -42,7 +42,7 @@ from __future__ import annotations # Standard Imports -from typing import Optional, Any +from typing import Optional, Any, TYPE_CHECKING import os import sys @@ -50,18 +50,12 @@ import click # Perun Imports -from perun.cli_groups import check_cli, config_cli, run_cli, utils_cli, import_cli +from perun.cli_groups import check_cli, collect_cli, config_cli, run_cli, utils_cli, import_cli +import perun.collect from perun.logic import commands, pcs, config as perun_config from perun.utils import exceptions, log as perun_log from perun.utils.common import cli_kit, common_kit from perun.utils.external import commands as external_commands -from perun.collect.trace.optimizations.structs import ( - Pipeline, - Optimizations, - CallGraphTypes, -) -from perun.collect.trace.optimizations.structs import Parameters -from perun.profile.factory import Profile from perun.utils.exceptions import ( UnsupportedModuleException, NotPerunRepositoryException, @@ -69,9 +63,8 @@ MissingConfigSectionException, ExternalEditorErrorException, ) -from perun.utils.structs import Executable -import perun.collect -import perun.fuzz.factory as fuzz +from perun.utils.structs.common_structs import Executable +from perun import fuzz as fuzz import perun.postprocess import perun.profile.helpers as profiles import perun.view @@ -79,6 +72,10 @@ import perun.deltadebugging.factory as delta +if TYPE_CHECKING: + from perun.profile.factory import Profile + + DEV_MODE = False @@ -831,176 +828,6 @@ def postprocessby(ctx: click.Context, profile: Profile, **_: Any) -> None: ctx.obj = profile -@cli.group() -@click.option( - "--output-file", - "-o", - nargs=1, - required=False, - multiple=False, - type=click.Path(writable=True), - help="Specifies the full path to where the profile will be stored.", -) -@click.option( - "--profile-name", - "-pn", - nargs=1, - required=False, - multiple=False, - type=str, - help="Specifies the name of the profile, which will be collected, e.g. profile.perf. The profile will be stored in .perun/jobs", -) -@click.option( - "--minor-version", - "-m", - "minor_version_list", - nargs=1, - multiple=True, - callback=cli_kit.minor_version_list_callback, - default=["HEAD"], - help="Specifies the head minor version, for which the profiles will be collected.", -) -@click.option( - "--crawl-parents", - "-cp", - is_flag=True, - default=False, - is_eager=True, - help=( - "If set to true, then for each specified minor versions, profiles for parents" - " will be collected as well" - ), -) -@click.option( - "--cmd", - "-c", - nargs=1, - required=False, - multiple=True, - default=[""], - help=( - "Command that is being profiled. Either corresponds to some" - " script, binary or command, e.g. ``./mybin`` or ``perun``." - ), -) -@click.option( - "--args", - "-a", - nargs=1, - required=False, - multiple=True, - help="Additional parameters for . E.g. ``status`` or ``-al`` is command parameter.", -) -@click.option( - "--workload", - "-w", - nargs=1, - required=False, - multiple=True, - default=[""], - help="Inputs for . E.g. ``./subdir`` is possible workload for ``ls`` command.", -) -@click.option( - "--params", - "-p", - nargs=1, - required=False, - multiple=True, - callback=cli_kit.single_yaml_param_callback, - help="Additional parameters for called collector read from file in YAML format.", -) -@click.option( - "--output-filename-template", - "-ot", - default=None, - callback=cli_kit.set_config_option_from_flag( - perun_config.runtime, "format.output_profile_template", str - ), - help=( - "Specifies the template for automatic generation of output filename" - " This way the file with collected data will have a resulting filename w.r.t " - " to this parameter. Refer to :ckey:`format.output_profile_template` for more" - " details about the format of the template." - ), -) -@click.option( - "--optimization-pipeline", - "-op", - type=click.Choice(Pipeline.supported()), - default=Pipeline.default(), - callback=cli_kit.set_optimization, - help="Pre-configured combinations of collection optimization methods.", -) -@click.option( - "--optimization-on", - "-on", - type=click.Choice(Optimizations.supported()), - multiple=True, - callback=cli_kit.set_optimization, - help="Enable the specified collection optimization method.", -) -@click.option( - "--optimization-off", - "-off", - type=click.Choice(Optimizations.supported()), - multiple=True, - callback=cli_kit.set_optimization, - help="Disable the specified collection optimization method.", -) -@click.option( - "--optimization-args", - "-oa", - type=(click.Choice(Parameters.supported()), str), - multiple=True, - callback=cli_kit.set_optimization_param, - help="Set parameter values for various optimizations.", -) -@click.option( - "--optimization-cache-off", - is_flag=True, - callback=cli_kit.set_optimization_cache, - help="Ignore cached optimization data (e.g., cached call graph).", -) -@click.option( - "--optimization-reset-cache", - is_flag=True, - default=False, - callback=cli_kit.reset_optimization_cache, - help="Remove the cached optimization resources and data.", -) -@click.option( - "--use-cg-type", - "-cg", - type=(click.Choice(CallGraphTypes.supported())), - default=CallGraphTypes.default(), - callback=cli_kit.set_call_graph_type, -) -@click.pass_context -def collect(ctx: click.Context, **kwargs: Any) -> None: - """Generates performance profile using selected collector. - - Runs the single collector unit (registered in Perun) on given profiled - command (optionally with given arguments and workloads) and generates - performance profile. The generated profile is then stored in - ``.perun/jobs/`` directory as a file, by default with filename in form of:: - - bin-collector-workload-timestamp.perf - - Generated profiles will not be postprocessed in any way. Consult ``perun - postprocessby --help`` in order to postprocess the resulting profile. - - The configuration of collector can be specified in external YAML file given - by the ``-p``/``--params`` argument. - - For a thorough list and description of supported collectors refer to - :ref:`collectors-list`. For a more subtle running of profiling jobs and - more complex configuration consult either ``perun run matrix --help`` or - ``perun run job --help``. - """ - commands.try_init() - ctx.obj = kwargs - - @cli.command("fuzz") @click.option("--cmd", "-b", nargs=1, required=True, help="The command which will be fuzzed.") @click.option( @@ -1319,7 +1146,7 @@ def init_unit_commands(lazy_init: bool = True) -> None: (perun.view_diff, showdiff, "showdiff"), (perun.view, show, "show"), (perun.postprocess, postprocessby, "postprocessby"), - (perun.collect, collect, "collect"), + (perun.collect, collect_cli.collect, "collect"), ]: if lazy_init and cli_arg not in sys.argv: continue @@ -1330,6 +1157,7 @@ def init_unit_commands(lazy_init: bool = True) -> None: # Initialization of other stuff init_unit_commands() cli.add_command(check_cli.check_group) +cli.add_command(collect_cli.collect) cli.add_command(config_cli.config) cli.add_command(run_cli.run) cli.add_command(utils_cli.utils_group) diff --git a/perun/cli_groups/check_cli.py b/perun/cli_groups/check_cli.py index 2a7b17621..774df90b6 100644 --- a/perun/cli_groups/check_cli.py +++ b/perun/cli_groups/check_cli.py @@ -9,10 +9,11 @@ import click # Perun Imports +from perun import check as check from perun.logic import pcs, config as perun_config, commands from perun.utils import log from perun.utils.common import cli_kit, common_kit -import perun.check.factory as check +from perun.utils.structs import check_structs if TYPE_CHECKING: from perun.profile.factory import Profile @@ -50,8 +51,8 @@ nargs=1, required=False, multiple=False, - type=click.Choice(check.get_supported_detection_models_strategies()), - default=check.get_supported_detection_models_strategies()[0], + type=click.Choice(check_structs.get_supported_detection_models_strategies()), + default=check_structs.get_supported_detection_models_strategies()[0], help=( "The detection models strategies predict the way of executing " "the detection between two profiles, respectively between relevant " diff --git a/perun/cli_groups/collect_cli.py b/perun/cli_groups/collect_cli.py new file mode 100644 index 000000000..0fa938160 --- /dev/null +++ b/perun/cli_groups/collect_cli.py @@ -0,0 +1,183 @@ +# Standard Imports +from typing import Any + +# Third-Party Imports +import click + +# Perun Imports +from perun.utils.structs import collect_structs +from perun.utils.common import cli_kit +from perun.logic import commands, config as perun_config + + +@click.group() +@click.option( + "--output-file", + "-o", + nargs=1, + required=False, + multiple=False, + type=click.Path(writable=True), + help="Specifies the full path to where the profile will be stored.", +) +@click.option( + "--profile-name", + "-pn", + nargs=1, + required=False, + multiple=False, + type=str, + help=( + "Specifies the name of the profile, which will be collected, e.g. profile.perf. The profile will be stored in" + " .perun/jobs" + ), +) +@click.option( + "--minor-version", + "-m", + "minor_version_list", + nargs=1, + multiple=True, + callback=cli_kit.minor_version_list_callback, + default=["HEAD"], + help="Specifies the head minor version, for which the profiles will be collected.", +) +@click.option( + "--crawl-parents", + "-cp", + is_flag=True, + default=False, + is_eager=True, + help=( + "If set to true, then for each specified minor versions, profiles for parents" + " will be collected as well" + ), +) +@click.option( + "--cmd", + "-c", + nargs=1, + required=False, + multiple=True, + default=[""], + help=( + "Command that is being profiled. Either corresponds to some" + " script, binary or command, e.g. ``./mybin`` or ``perun``." + ), +) +@click.option( + "--args", + "-a", + nargs=1, + required=False, + multiple=True, + help="Additional parameters for . E.g. ``status`` or ``-al`` is command parameter.", +) +@click.option( + "--workload", + "-w", + nargs=1, + required=False, + multiple=True, + default=[""], + help="Inputs for . E.g. ``./subdir`` is possible workload for ``ls`` command.", +) +@click.option( + "--params", + "-p", + nargs=1, + required=False, + multiple=True, + callback=cli_kit.single_yaml_param_callback, + help="Additional parameters for called collector read from file in YAML format.", +) +@click.option( + "--output-filename-template", + "-ot", + default=None, + callback=cli_kit.set_config_option_from_flag( + perun_config.runtime, "format.output_profile_template", str + ), + help=( + "Specifies the template for automatic generation of output filename" + " This way the file with collected data will have a resulting filename w.r.t " + " to this parameter. Refer to :ckey:`format.output_profile_template` for more" + " details about the format of the template." + ), +) +@click.option( + "--optimization-pipeline", + "-op", + type=click.Choice(collect_structs.Pipeline.supported()), + default=collect_structs.Pipeline.default(), + callback=cli_kit.set_optimization, + help="Pre-configured combinations of collection optimization methods.", +) +@click.option( + "--optimization-on", + "-on", + type=click.Choice(collect_structs.Optimizations.supported()), + multiple=True, + callback=cli_kit.set_optimization, + help="Enable the specified collection optimization method.", +) +@click.option( + "--optimization-off", + "-off", + type=click.Choice(collect_structs.Optimizations.supported()), + multiple=True, + callback=cli_kit.set_optimization, + help="Disable the specified collection optimization method.", +) +@click.option( + "--optimization-args", + "-oa", + type=(click.Choice(collect_structs.Parameters.supported()), str), + multiple=True, + callback=cli_kit.set_optimization_param, + help="Set parameter values for various optimizations.", +) +@click.option( + "--optimization-cache-off", + is_flag=True, + callback=cli_kit.set_optimization_cache, + help="Ignore cached optimization data (e.g., cached call graph).", +) +@click.option( + "--optimization-reset-cache", + is_flag=True, + default=False, + callback=cli_kit.reset_optimization_cache, + help="Remove the cached optimization resources and data.", +) +@click.option( + "--use-cg-type", + "-cg", + type=(click.Choice(collect_structs.CallGraphTypes.supported())), + default=collect_structs.CallGraphTypes.default(), + callback=cli_kit.set_call_graph_type, +) +@click.pass_context +def collect(ctx: click.Context, **kwargs: Any) -> None: + """Generates performance profile using selected collector. + + Runs the single collector unit (registered in Perun) on given profiled + command (optionally with given arguments and workloads) and generates + performance profile. The generated profile is then stored in + ``.perun/jobs/`` directory as a file, by default with filename in form of:: + + bin-collector-workload-timestamp.perf + + Generated profiles will not be postprocessed in any way. Consult ``perun + postprocessby --help`` in order to postprocess the resulting profile. + + The configuration of collector can be specified in external YAML file given + by the ``-p``/``--params`` argument. + + For a thorough list and description of supported collectors refer to + :ref:`collectors-list`. For a more subtle running of profiling jobs and + more complex configuration consult either ``perun run matrix --help`` or + ``perun run job --help``. + """ + commands.try_init() + ctx.obj = kwargs diff --git a/perun/cli_groups/import_cli.py b/perun/cli_groups/import_cli.py index eb54f35d2..5714d38bd 100755 --- a/perun/cli_groups/import_cli.py +++ b/perun/cli_groups/import_cli.py @@ -10,7 +10,7 @@ # Perun Imports from perun.logic import commands, config -from perun.profile import imports +from perun import profile as profile from perun.utils.common import cli_kit @@ -144,7 +144,7 @@ def from_binary(ctx: click.Context, import_entries: list[str], **kwargs: Any) -> that combines the --stats-headers option and profile entries. """ kwargs.update(ctx.obj) - imports.import_perf_from_record(import_entries, **kwargs) + profile.import_perf_from_record(import_entries, **kwargs) @perf_group.command("script") @@ -171,7 +171,7 @@ def from_text(ctx: click.Context, import_entries: list[str], **kwargs: Any) -> N that combines the --stats-headers option and profile entries. """ kwargs.update(ctx.obj) - imports.import_perf_from_script(import_entries, **kwargs) + profile.import_perf_from_script(import_entries, **kwargs) @perf_group.command("stack") @@ -199,7 +199,7 @@ def from_stacks(ctx: click.Context, import_entries: list[str], **kwargs: Any) -> that combines the --stats-headers option and profile entries. """ kwargs.update(ctx.obj) - imports.import_perf_from_stack(import_entries, **kwargs) + profile.import_perf_from_stack(import_entries, **kwargs) @import_group.group("elk") @@ -229,4 +229,4 @@ def from_json(ctx: click.Context, import_entries: list[str], **kwargs: Any) -> N Each import entry may specify a JSON path 'file_path.json'. """ kwargs.update(ctx.obj) - imports.import_elk_from_json(import_entries, **kwargs) + profile.import_elk_from_json(import_entries, **kwargs) diff --git a/perun/cli_groups/meson.build b/perun/cli_groups/meson.build index 5757b2840..95af87045 100644 --- a/perun/cli_groups/meson.build +++ b/perun/cli_groups/meson.build @@ -3,6 +3,7 @@ perun_cli_groups_dir = perun_dir / 'cli_groups' perun_cli_groups_files = files( '__init__.py', 'check_cli.py', + 'collect_cli.py', 'config_cli.py', 'import_cli.py', 'run_cli.py', diff --git a/perun/cli_groups/run_cli.py b/perun/cli_groups/run_cli.py index 4aeb1fab4..c25438af2 100644 --- a/perun/cli_groups/run_cli.py +++ b/perun/cli_groups/run_cli.py @@ -12,7 +12,7 @@ from perun.logic import config as perun_config, runner, commands from perun.utils import log as perun_log from perun.utils.common import cli_kit -from perun.utils.structs import CollectStatus +from perun.utils.structs.common_structs import CollectStatus @click.group() diff --git a/perun/collect/bounds/run.py b/perun/collect/bounds/run.py index ff4990e52..e956bfcb6 100644 --- a/perun/collect/bounds/run.py +++ b/perun/collect/bounds/run.py @@ -22,7 +22,7 @@ from perun.logic import runner from perun.utils import log from perun.utils.external import commands -from perun.utils.structs import CollectStatus +from perun.utils.structs.common_structs import CollectStatus _CLANG_COMPILER = "clang-3.5" _CLANG_COMPILATION_PARAMS = ["-g", "-emit-llvm", "-c"] diff --git a/perun/collect/complexity/run.py b/perun/collect/complexity/run.py index fb6f50ba9..69d5a5546 100644 --- a/perun/collect/complexity/run.py +++ b/perun/collect/complexity/run.py @@ -22,7 +22,7 @@ from perun.logic import runner from perun.utils import exceptions, log from perun.utils.external import commands -from perun.utils.structs import Executable, CollectStatus +from perun.utils.structs.common_structs import Executable, CollectStatus # The profiling record template diff --git a/perun/collect/kperf/run.py b/perun/collect/kperf/run.py index 1cf13cc6b..f608597e1 100755 --- a/perun/collect/kperf/run.py +++ b/perun/collect/kperf/run.py @@ -16,7 +16,7 @@ from perun.logic import runner from perun.utils import log from perun.utils.common import script_kit -from perun.utils.structs import Executable, CollectStatus +from perun.utils.structs.common_structs import Executable, CollectStatus from perun.utils.external import commands from perun.utils.exceptions import SuppressedExceptions diff --git a/perun/collect/memory/parsing.py b/perun/collect/memory/parsing.py index d9f33291e..9a00bb1c4 100644 --- a/perun/collect/memory/parsing.py +++ b/perun/collect/memory/parsing.py @@ -16,7 +16,7 @@ from perun.utils.common import common_kit if TYPE_CHECKING: - from perun.utils.structs import Executable + from perun.utils.structs.common_structs import Executable PATTERN_WORD: re.Pattern[str] = re.compile(r"(\w+|[?])") diff --git a/perun/collect/memory/run.py b/perun/collect/memory/run.py index 734875e9d..f9ebefd8f 100644 --- a/perun/collect/memory/run.py +++ b/perun/collect/memory/run.py @@ -13,7 +13,7 @@ from perun.collect.memory import filter as filters, parsing as parser, syscalls from perun.logic import runner from perun.utils import log -from perun.utils.structs import CollectStatus, Executable +from perun.utils.structs.common_structs import CollectStatus, Executable _lib_name: str = "malloc.so" diff --git a/perun/collect/memory/syscalls.py b/perun/collect/memory/syscalls.py index c3b8ab3a1..9dff9fdf4 100644 --- a/perun/collect/memory/syscalls.py +++ b/perun/collect/memory/syscalls.py @@ -14,7 +14,7 @@ from perun.utils.exceptions import SuppressedExceptions if TYPE_CHECKING: - from perun.utils.structs import Executable + from perun.utils.structs.common_structs import Executable PATTERN_WORD = re.compile(r"(\w+)|[?]") PATTERN_HEXADECIMAL = re.compile(r"0x[0-9a-fA-F]+") diff --git a/perun/collect/time/run.py b/perun/collect/time/run.py index e25814e12..52a913a11 100644 --- a/perun/collect/time/run.py +++ b/perun/collect/time/run.py @@ -18,7 +18,7 @@ from perun.utils import log from perun.utils.common import common_kit from perun.utils.external import commands -from perun.utils.structs import CollectStatus, Executable +from perun.utils.structs.common_structs import CollectStatus, Executable TIME_TYPES = ("real", "user", "sys") diff --git a/perun/collect/trace/ebpf/program.py b/perun/collect/trace/ebpf/program.py index b5b8700ce..2f866edb0 100644 --- a/perun/collect/trace/ebpf/program.py +++ b/perun/collect/trace/ebpf/program.py @@ -5,7 +5,7 @@ """ from perun.collect.trace.watchdog import WATCH_DOG -from perun.collect.trace.optimizations.structs import Optimizations +from perun.utils.structs.collect_structs import Optimizations def assemble_ebpf_program(src_file, probes, config, **_): diff --git a/perun/collect/trace/optimizations/optimization.py b/perun/collect/trace/optimizations/optimization.py index 6b17fa1d9..176a47711 100644 --- a/perun/collect/trace/optimizations/optimization.py +++ b/perun/collect/trace/optimizations/optimization.py @@ -6,13 +6,10 @@ from perun.utils.common.common_kit import sanitize_filepart from perun.utils.exceptions import SuppressedExceptions from perun.collect.trace.optimizations.structs import ( - Optimizations, - Pipeline, - Parameters, - CallGraphTypes, ParametersManager, CGShapingMode, ) +from perun.utils.structs.collect_structs import Optimizations, Pipeline, Parameters, CallGraphTypes import perun.collect.trace.optimizations.resources.manager as resources from perun.collect.trace.optimizations.call_graph import CallGraphResource import perun.collect.trace.optimizations.cg_projection as proj diff --git a/perun/collect/trace/optimizations/structs.py b/perun/collect/trace/optimizations/structs.py index daa2a6b50..2abc9d83b 100644 --- a/perun/collect/trace/optimizations/structs.py +++ b/perun/collect/trace/optimizations/structs.py @@ -7,137 +7,10 @@ from functools import partial from enum import Enum -from perun.utils.structs import OrderedEnum +from perun.utils.structs.common_structs import OrderedEnum from perun.utils.common import common_kit import perun.utils.metrics as metrics - - -class Optimizations(Enum): - """Enumeration of the implemented methods and their CLI name.""" - - BASELINE_STATIC = "baseline-static" - BASELINE_DYNAMIC = "baseline-dynamic" - CALL_GRAPH_SHAPING = "cg-shaping" - DYNAMIC_SAMPLING = "dynamic-sampling" - DIFF_TRACING = "diff-tracing" - DYNAMIC_PROBING = "dynamic-probing" - TIMED_SAMPLING = "timed-sampling" - - @staticmethod - def supported(): - """List the currently supported optimization methods. - - :return: CLI names of the supported optimizations - """ - return [optimization.value for optimization in Optimizations] - - -class Pipeline(Enum): - """Enumeration of the implemented pipelines and their CLI name. - Custom represents a defualt pipeline that has no pre-configured methods or parameters - """ - - CUSTOM = "custom" - BASIC = "basic" - ADVANCED = "advanced" - FULL = "full" - - @staticmethod - def supported(): - """List the currently supported optimization pipelines. - - :return: CLI names of the supported pipelines - """ - return [pipeline.value for pipeline in Pipeline] - - @staticmethod - def default(): - """Name of the default pipeline. - - :return: the CLI name of the default pipeline - """ - return Pipeline.CUSTOM.value - - def map_to_optimizations(self): - """Map the selected optimization pipeline to the set of employed optimization methods. - - :return: list of the Optimizations enumeration objects - """ - if self == Pipeline.BASIC: - return [Optimizations.CALL_GRAPH_SHAPING, Optimizations.BASELINE_DYNAMIC] - if self == Pipeline.ADVANCED: - return [ - Optimizations.DIFF_TRACING, - Optimizations.CALL_GRAPH_SHAPING, - Optimizations.BASELINE_DYNAMIC, - Optimizations.DYNAMIC_SAMPLING, - ] - if self == Pipeline.FULL: - return [ - Optimizations.DIFF_TRACING, - Optimizations.CALL_GRAPH_SHAPING, - Optimizations.BASELINE_STATIC, - Optimizations.BASELINE_DYNAMIC, - Optimizations.DYNAMIC_SAMPLING, - Optimizations.DYNAMIC_PROBING, - ] - return [] - - -class CallGraphTypes(Enum): - """Enumeration of the implemented call graph types and their CLI names.""" - - STATIC = "static" - DYNAMIC = "dynamic" - MIXED = "mixed" - - @staticmethod - def supported(): - """List the currently supported call graph types. - - :return: CLI names of the supported cg types - """ - return [cg.value for cg in CallGraphTypes] - - @staticmethod - def default(): - """Name of the default cg type. - - :return: the CLI name of the default cg type - """ - return CallGraphTypes.STATIC.value - - -class Parameters(Enum): - """Enumeration of the currently supported CLI options for optimization methods and pipelines.""" - - DIFF_VERSION = "diff-version" - DIFF_KEEP_LEAF = "diff-keep-leaf" - DIFF_INSPECT_ALL = "diff-inspect-all" - DIFF_CG_MODE = "diff-cfg-mode" - SOURCE_FILES = "source-files" - SOURCE_DIRS = "source-dirs" - STATIC_COMPLEXITY = "static-complexity" - STATIC_KEEP_TOP = "static-keep-top" - CG_SHAPING_MODE = "cg-mode" - CG_PROJ_LEVELS = "cg-proj-levels" - CG_PROJ_KEEP_LEAF = "cg-proj-keep-leaf" - DYNSAMPLE_STEP = "dyn-sample-step" - DYNSAMPLE_THRESHOLD = "dyn-sample-threshold" - PROBING_THRESHOLD = "probing-threshold" - PROBING_REATTACH = "probing-reattach" - TIMEDSAMPLE_FREQ = "timed-sample-freq" - DYNBASE_SOFT_THRESHOLD = "dyn-base-soft-threshold" - DYNBASE_HARD_THRESHOLD = "dyn-base-hard-threshold" - THRESHOLD_MODE = "threshold-mode" - - @staticmethod - def supported(): - """List the currently supported optimization parameters. - - :return: CLI names of the supported parameters - """ - return [parameter.value for parameter in Parameters] +from perun.utils.structs import collect_structs class DiffCfgMode(Enum): @@ -278,67 +151,70 @@ def __init__(self): self.cli_params = [] self.param_map = { # TODO: add proper check - Parameters.DIFF_VERSION: {"value": None, "validate": lambda x: x}, - Parameters.DIFF_KEEP_LEAF: { + collect_structs.Parameters.DIFF_VERSION: {"value": None, "validate": lambda x: x}, + collect_structs.Parameters.DIFF_KEEP_LEAF: { "value": False, "validate": self._validate_bool, }, - Parameters.DIFF_INSPECT_ALL: { + collect_structs.Parameters.DIFF_INSPECT_ALL: { "value": True, "validate": self._validate_bool, }, - Parameters.DIFF_CG_MODE: { + collect_structs.Parameters.DIFF_CG_MODE: { "value": DiffCfgMode.SEMISTRICT, "validate": partial(self._validate_enum, DiffCfgMode), }, - Parameters.SOURCE_FILES: {"value": [], "validate": self._validate_path}, - Parameters.SOURCE_DIRS: {"value": [], "validate": self._validate_path}, - Parameters.STATIC_COMPLEXITY: { + collect_structs.Parameters.SOURCE_FILES: {"value": [], "validate": self._validate_path}, + collect_structs.Parameters.SOURCE_DIRS: {"value": [], "validate": self._validate_path}, + collect_structs.Parameters.STATIC_COMPLEXITY: { "value": Complexity.CONSTANT, "validate": partial(self._validate_enum, Complexity), }, - Parameters.STATIC_KEEP_TOP: { + collect_structs.Parameters.STATIC_KEEP_TOP: { "value": self._default_keep_top, "validate": self._validate_uint, }, - Parameters.CG_SHAPING_MODE: { + collect_structs.Parameters.CG_SHAPING_MODE: { "value": CGShapingMode.MATCH, "validate": partial(self._validate_enum, CGShapingMode), }, - Parameters.CG_PROJ_LEVELS: { + collect_structs.Parameters.CG_PROJ_LEVELS: { "value": self._default_chain_length, "validate": self._validate_uint, }, - Parameters.CG_PROJ_KEEP_LEAF: { + collect_structs.Parameters.CG_PROJ_KEEP_LEAF: { "value": False, "validate": self._validate_bool, }, - Parameters.DYNSAMPLE_STEP: { + collect_structs.Parameters.DYNSAMPLE_STEP: { "value": self._default_sampling_step, "validate": self._validate_ufloat, }, - Parameters.DYNSAMPLE_THRESHOLD: { + collect_structs.Parameters.DYNSAMPLE_THRESHOLD: { "value": self._threshold_soft_base, "validate": self._validate_uint, }, - Parameters.PROBING_THRESHOLD: { + collect_structs.Parameters.PROBING_THRESHOLD: { "value": self._probing_threshold, "validate": self._validate_uint, }, - Parameters.PROBING_REATTACH: { + collect_structs.Parameters.PROBING_REATTACH: { "value": False, "validate": self._validate_bool, }, - Parameters.TIMEDSAMPLE_FREQ: {"value": 1, "validate": self._validate_uint}, - Parameters.DYNBASE_SOFT_THRESHOLD: { + collect_structs.Parameters.TIMEDSAMPLE_FREQ: { + "value": 1, + "validate": self._validate_uint, + }, + collect_structs.Parameters.DYNBASE_SOFT_THRESHOLD: { "value": self._threshold_soft_base, "validate": self._validate_uint, }, - Parameters.DYNBASE_HARD_THRESHOLD: { + collect_structs.Parameters.DYNBASE_HARD_THRESHOLD: { "value": self._threshold_soft_base * self._hard_threshold_coefficient, "validate": self._validate_uint, }, - Parameters.THRESHOLD_MODE: { + collect_structs.Parameters.THRESHOLD_MODE: { "value": ThresholdMode.SOFT, "validate": partial(self._validate_enum, ThresholdMode), }, @@ -369,7 +245,7 @@ def add_cli_parameter(self, name, value): :return: the parameter value if the validation is successful, else None """ - param = Parameters(name) + param = collect_structs.Parameters(name) validated = self.param_map[param]["validate"](value) if validated is not None: self.cli_params.append((param, validated)) @@ -393,9 +269,9 @@ def infer_params(self, call_graph, pipeline, binary): self._default_keep_top = call_graph.coverage_max_cut()[1] + 1 # Extract the user-supplied modes and parameters modes = [ - Parameters.DIFF_CG_MODE, - Parameters.CG_SHAPING_MODE, - Parameters.THRESHOLD_MODE, + collect_structs.Parameters.DIFF_CG_MODE, + collect_structs.Parameters.CG_SHAPING_MODE, + collect_structs.Parameters.THRESHOLD_MODE, ] cli_modes, cli_params = common_kit.partition_list( self.cli_params, lambda param: param[0] in modes @@ -433,11 +309,11 @@ def _infer_general_parameters(self, func_count, level_count): return # Keep the leaf functions if the total number of profiled functions is low if func_count <= self._functions_keep_leaves: - self[Parameters.DIFF_KEEP_LEAF] = True - self[Parameters.CG_PROJ_KEEP_LEAF] = True + self[collect_structs.Parameters.DIFF_KEEP_LEAF] = True + self[collect_structs.Parameters.CG_PROJ_KEEP_LEAF] = True # Keep-top: 10% of levels, minimum is default keep_top = max(math.ceil(level_count * self._keep_top_ratio), self._default_keep_top) - self[Parameters.STATIC_KEEP_TOP] = keep_top + self[collect_structs.Parameters.STATIC_KEEP_TOP] = keep_top def _infer_modes(self, selected_pipeline, user_modes): """Predicts the mode parameters based on the used pipeline. @@ -445,13 +321,13 @@ def _infer_modes(self, selected_pipeline, user_modes): :param selected_pipeline: the currently selected pipeline :param user_modes: list of pairs with user-specified modes """ - self[Parameters.DIFF_CG_MODE] = DiffCfgMode.COLORING - self[Parameters.CG_SHAPING_MODE] = CGShapingMode.TOP_DOWN + self[collect_structs.Parameters.DIFF_CG_MODE] = DiffCfgMode.COLORING + self[collect_structs.Parameters.CG_SHAPING_MODE] = CGShapingMode.TOP_DOWN # The selected pipeline determines the used modes - if selected_pipeline == Pipeline.BASIC: - self[Parameters.THRESHOLD_MODE] = ThresholdMode.STRICT + if selected_pipeline == collect_structs.Pipeline.BASIC: + self[collect_structs.Parameters.THRESHOLD_MODE] = ThresholdMode.STRICT else: - self[Parameters.THRESHOLD_MODE] = ThresholdMode.SOFT + self[collect_structs.Parameters.THRESHOLD_MODE] = ThresholdMode.SOFT # Apply the user-supplied modes for mode_type, mode_value in user_modes: self[mode_type] = mode_value @@ -467,18 +343,20 @@ def _infer_cg_shaping_parameters(self, func_count, level_count): # Determine the number of trimmed levels trim_levels = round(level_count * self._levels_ratio) # Set the trim levels - self[Parameters.CG_PROJ_LEVELS] = max(trim_levels, self._default_min_levels) + self[collect_structs.Parameters.CG_PROJ_LEVELS] = max(trim_levels, self._default_min_levels) def _infer_thresholds(self): """Infer the threshold values based on the selected modes.""" # Determine the thresholds based on the mode base = self._threshold_soft_base - if self[Parameters.THRESHOLD_MODE] == ThresholdMode.STRICT: + if self[collect_structs.Parameters.THRESHOLD_MODE] == ThresholdMode.STRICT: base = self._threshold_strict_base # Set the threshold - self[Parameters.DYNSAMPLE_THRESHOLD] = base - self[Parameters.DYNBASE_SOFT_THRESHOLD] = base - self[Parameters.DYNBASE_HARD_THRESHOLD] = base * self._hard_threshold_coefficient + self[collect_structs.Parameters.DYNSAMPLE_THRESHOLD] = base + self[collect_structs.Parameters.DYNBASE_SOFT_THRESHOLD] = base + self[collect_structs.Parameters.DYNBASE_HARD_THRESHOLD] = ( + base * self._hard_threshold_coefficient + ) def _infer_dynamic_probing(self, cli_params): """Predict parameters and threshold values for Dynamic Probing . @@ -486,17 +364,22 @@ def _infer_dynamic_probing(self, cli_params): :param cli_params: a collection of user-supplied parameters """ # Update the probing threshold if reattach is enabled and probing threshold is not set - probing_threshold_set = Parameters.PROBING_THRESHOLD in [param for param, _ in cli_params] - if self[Parameters.PROBING_REATTACH] and not probing_threshold_set: + probing_threshold_set = collect_structs.Parameters.PROBING_THRESHOLD in [ + param for param, _ in cli_params + ] + if self[collect_structs.Parameters.PROBING_REATTACH] and not probing_threshold_set: probing_threshold = self._probing_threshold * self._probing_reattach_coefficient - self[Parameters.PROBING_THRESHOLD] = probing_threshold + self[collect_structs.Parameters.PROBING_THRESHOLD] = probing_threshold def _extract_sources(self, binary): """Search for source files of the project in the binary directory, if none are given. :param binary: path to the binary executable """ - files, dirs = self[Parameters.SOURCE_FILES], self[Parameters.SOURCE_DIRS] + files, dirs = ( + self[collect_structs.Parameters.SOURCE_FILES], + self[collect_structs.Parameters.SOURCE_DIRS], + ) # No need to extract if only source files are supplied if files and not dirs: return @@ -505,7 +388,7 @@ def _extract_sources(self, binary): dirs.append(os.path.dirname(binary)) # Save the sources - self[Parameters.SOURCE_FILES] = list(set(_get_source_files(dirs, files))) + self[collect_structs.Parameters.SOURCE_FILES] = list(set(_get_source_files(dirs, files))) @staticmethod def _validate_bool(value): diff --git a/perun/collect/trace/run.py b/perun/collect/trace/run.py index 022b5fcbb..57b996d92 100644 --- a/perun/collect/trace/run.py +++ b/perun/collect/trace/run.py @@ -25,7 +25,7 @@ import perun.utils.log as stdout import perun.utils.metrics as metrics from perun.profile.factory import Profile -from perun.utils.structs import CollectStatus +from perun.utils.structs.common_structs import CollectStatus def before(executable, **kwargs): diff --git a/perun/collect/trace/systemtap/script_compact.py b/perun/collect/trace/systemtap/script_compact.py index e300e6df8..457adedfd 100644 --- a/perun/collect/trace/systemtap/script_compact.py +++ b/perun/collect/trace/systemtap/script_compact.py @@ -4,7 +4,7 @@ from perun.collect.trace.watchdog import WATCH_DOG from perun.collect.trace.values import RecordType -from perun.collect.trace.optimizations.structs import Optimizations, Parameters +from perun.utils.structs.collect_structs import Optimizations, Parameters # Names of the global arrays used throughout the script diff --git a/perun/deltadebugging/factory.py b/perun/deltadebugging/factory.py index 01d20990b..73d0a3fcf 100644 --- a/perun/deltadebugging/factory.py +++ b/perun/deltadebugging/factory.py @@ -12,7 +12,7 @@ from perun.utils.common import common_kit if TYPE_CHECKING: - from perun.utils.structs import Executable + from perun.utils.structs.common_structs import Executable def run_delta_debugging_for_command( diff --git a/perun/fuzz/__init__.py b/perun/fuzz/__init__.py index ed2dd5b59..0be2db06f 100644 --- a/perun/fuzz/__init__.py +++ b/perun/fuzz/__init__.py @@ -4,3 +4,7 @@ the fuzzing loop and strategies/heuristics for enqueueing newly discovered workloads for further fuzzing. """ + +import lazy_loader as lazy + +__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__) diff --git a/perun/fuzz/__init__.pyi b/perun/fuzz/__init__.pyi new file mode 100644 index 000000000..1d99baf33 --- /dev/null +++ b/perun/fuzz/__init__.pyi @@ -0,0 +1 @@ +from .factory import run_fuzzing_for_command as run_fuzzing_for_command diff --git a/perun/fuzz/evaluate/by_coverage.py b/perun/fuzz/evaluate/by_coverage.py index 183160ed0..10a8fba93 100644 --- a/perun/fuzz/evaluate/by_coverage.py +++ b/perun/fuzz/evaluate/by_coverage.py @@ -27,7 +27,7 @@ FuzzingProgress, CoverageConfiguration, ) - from perun.utils.structs import Executable + from perun.utils.structs.common_structs import Executable def prepare_workspace(source_path: str) -> None: diff --git a/perun/fuzz/evaluate/by_perun.py b/perun/fuzz/evaluate/by_perun.py index b7c34c581..ed8729206 100644 --- a/perun/fuzz/evaluate/by_perun.py +++ b/perun/fuzz/evaluate/by_perun.py @@ -12,15 +12,15 @@ # Third-Party Imports # Perun Imports -import perun.check.factory as check +from perun import check as check import perun.logic.runner as run -from perun.utils.structs import PerformanceChange +from perun.utils.structs.common_structs import PerformanceChange from perun.utils import log if TYPE_CHECKING: from perun.fuzz.structs import Mutation from perun.profile.factory import Profile - from perun.utils.structs import Executable, MinorVersion, CollectStatus, Job + from perun.utils.structs.common_structs import Executable, MinorVersion, CollectStatus, Job DEGRADATION_RATIO_THRESHOLD = 0.0 diff --git a/perun/fuzz/factory.py b/perun/fuzz/factory.py index 3b9434e6b..0533290af 100644 --- a/perun/fuzz/factory.py +++ b/perun/fuzz/factory.py @@ -29,7 +29,7 @@ RuleSet, TimeSeries, ) -from perun.utils import decorators, log +from perun.utils import log from perun.utils.exceptions import SuppressedExceptions import perun.fuzz.evaluate.by_perun as evaluate_workloads_by_perun import perun.fuzz.evaluate.by_coverage as evaluate_workloads_by_coverage @@ -37,7 +37,7 @@ if TYPE_CHECKING: import types - from perun.utils.structs import Executable, MinorVersion + from perun.utils.structs.common_structs import Executable, MinorVersion # to ignore numpy division warnings np.seterr(divide="ignore", invalid="ignore") diff --git a/perun/fuzz/meson.build b/perun/fuzz/meson.build index fba291f20..b35dc0f7a 100644 --- a/perun/fuzz/meson.build +++ b/perun/fuzz/meson.build @@ -2,6 +2,7 @@ perun_fuzz_dir = perun_dir / 'fuzz' perun_fuzz_files = files( '__init__.py', + '__init__.pyi', 'factory.py', 'filesystem.py', 'filetype.py', diff --git a/perun/logic/commands.py b/perun/logic/commands.py index 6f5ada390..19bcc16fc 100644 --- a/perun/logic/commands.py +++ b/perun/logic/commands.py @@ -43,7 +43,7 @@ PROFILE_DELIMITER, ColorChoiceType, ) -from perun.utils.structs import ProfileListConfig, MinorVersion +from perun.utils.structs.common_structs import ProfileListConfig, MinorVersion from perun.vcs import vcs_kit from perun.vcs.git_repository import GitRepository import perun.profile.helpers as profile diff --git a/perun/logic/runner.py b/perun/logic/runner.py index c13d6d6e6..1cc3ffd4d 100644 --- a/perun/logic/runner.py +++ b/perun/logic/runner.py @@ -19,7 +19,7 @@ from perun.utils.common import common_kit from perun.utils.exceptions import SignalReceivedException from perun.utils.external import commands as external_commands -from perun.utils.structs import ( +from perun.utils.structs.common_structs import ( CollectStatus, Executable, GeneratorSpec, @@ -31,7 +31,6 @@ Unit, ) from perun.workload.singleton_generator import SingletonGenerator -import perun.collect.trace.optimizations.optimization as optimizations import perun.profile.helpers as profile import perun.workload as workloads @@ -286,6 +285,8 @@ def run_all_phases_for( :param runner_params: dictionary of arguments for runner :return: report about the run phase adn profile """ + import perun.collect.trace.optimizations.optimization as optimizations + runner_verb = runner_type[:-2] # Create immutable list of resource that should hold even in case of problems runner_params["opened_resources"] = [] diff --git a/perun/logic/store.py b/perun/logic/store.py index 03a80afb4..a3cf11b1f 100644 --- a/perun/logic/store.py +++ b/perun/logic/store.py @@ -19,11 +19,11 @@ # Third-Party Imports # Perun Imports -from perun.profile.factory import Profile +from perun import profile as profile from perun.utils import log from perun.utils.common import common_kit from perun.utils.exceptions import IncorrectProfileFormatException -from perun.utils.structs import PerformanceChange, DegradationInfo +from perun.utils.structs.common_structs import PerformanceChange, DegradationInfo INDEX_TAG_REGEX = re.compile(r"^(\d+)@i$") @@ -295,7 +295,7 @@ def load_degradation_list_for( def load_profile_from_file( file_name: str, is_raw_profile: bool, unsafe_load: bool = False -) -> Profile: +) -> profile.Profile: """Loads profile w.r.t :ref:`profile-spec` from file. :param file_name: file path, where the profile is stored @@ -317,7 +317,7 @@ def load_profile_from_file( def load_profile_from_handle( file_name: str, file_handle: BinaryIO, is_raw_profile: bool -) -> Profile: +) -> profile.Profile: """ Fixme: Add check that the loaded profile is in valid format!!! TODO: This should be broken into two parts @@ -348,7 +348,7 @@ def load_profile_from_handle( # Try to load the json, if there is issue with the profile try: with common_kit.disposable_resources(json.loads(body)) as json_profile: - return Profile(json_profile) + return profile.Profile(json_profile) except ValueError: raise IncorrectProfileFormatException( file_name, f"profile '{file_name}' is not in profile format" diff --git a/perun/meson.build b/perun/meson.build index df16d5412..459cfb9d9 100644 --- a/perun/meson.build +++ b/perun/meson.build @@ -1,6 +1,7 @@ perun_files = files( 'cli.py', '__init__.py', + 'py.typed', ) py3.install_sources( diff --git a/perun/postprocess/kernel_regression/run.py b/perun/postprocess/kernel_regression/run.py index 377a97d88..e64fec671 100644 --- a/perun/postprocess/kernel_regression/run.py +++ b/perun/postprocess/kernel_regression/run.py @@ -15,7 +15,7 @@ from perun.postprocess.kernel_regression import methods from perun.postprocess.regression_analysis import data_provider, tools from perun.utils.common import cli_kit -from perun.utils.structs import PostprocessStatus +from perun.utils.structs.common_structs import PostprocessStatus if TYPE_CHECKING: from perun.profile.factory import Profile diff --git a/perun/postprocess/moving_average/run.py b/perun/postprocess/moving_average/run.py index e08fea890..8251cd621 100644 --- a/perun/postprocess/moving_average/run.py +++ b/perun/postprocess/moving_average/run.py @@ -16,7 +16,7 @@ from perun.postprocess.moving_average import methods from perun.postprocess.regression_analysis import data_provider, tools from perun.utils.common import cli_kit -from perun.utils.structs import PostprocessStatus +from perun.utils.structs.common_structs import PostprocessStatus if TYPE_CHECKING: from perun.profile.factory import Profile diff --git a/perun/postprocess/regression_analysis/regression_models.py b/perun/postprocess/regression_analysis/regression_models.py index 8bab81090..6c5d1b9ce 100644 --- a/perun/postprocess/regression_analysis/regression_models.py +++ b/perun/postprocess/regression_analysis/regression_models.py @@ -19,6 +19,7 @@ from perun.utils import exceptions from perun.utils.common import common_kit import perun.postprocess.regression_analysis.extensions.plot_models as plot +from perun.utils.structs.postprocess_structs import get_supported_models def get_formula_of(model: str) -> Callable[..., float]: @@ -32,17 +33,6 @@ def get_formula_of(model: str) -> Callable[..., float]: return MODEL_MAP[model]["transformations"]["plot_model"]["formula"] -def get_supported_models() -> list[str]: - """Provides all currently supported models as a list of their names. - - The 'all' specifier is used in reverse mapping as it enables to easily specify all models - - :return: the names of all supported models and 'all' specifier - """ - # Disable quadratic model, but allow to process already existing profiles with quad model - return [key for key in sorted(MODEL_MAP.keys())] - - def get_supported_transformations(model_key: str) -> list[str]: """Provides all currently supported transformations for given model as a list of their names. @@ -289,3 +279,5 @@ def filter_derived(regression_models_keys: tuple[str]) -> tuple[tuple[str], tupl }, }, } + +SUPPORTED_MODELS = MODEL_MAP.keys() diff --git a/perun/postprocess/regression_analysis/run.py b/perun/postprocess/regression_analysis/run.py index 18604c6f3..f118c3171 100644 --- a/perun/postprocess/regression_analysis/run.py +++ b/perun/postprocess/regression_analysis/run.py @@ -10,11 +10,11 @@ # Perun Imports from perun.logic import runner -from perun.postprocess.regression_analysis import data_provider, methods, regression_models, tools +from perun.postprocess.regression_analysis import data_provider, methods, tools from perun.profile.factory import pass_profile, Profile from perun.utils import metrics from perun.utils.common import cli_kit -from perun.utils.structs import PostprocessStatus +from perun.utils.structs import common_structs, postprocess_structs _DEFAULT_STEPS = 3 @@ -22,7 +22,7 @@ def postprocess( profile: Profile, **configuration: Any -) -> tuple[PostprocessStatus, str, dict[str, Any]]: +) -> tuple[common_structs.PostprocessStatus, str, dict[str, Any]]: """Invoked from perun core, handles the postprocess actions :param profile: the profile to analyze @@ -42,7 +42,7 @@ def postprocess( # Store the results new_profile = tools.add_models_to_profile(profile, analysis) - return PostprocessStatus.OK, "", {"profile": new_profile} + return common_structs.PostprocessStatus.OK, "", {"profile": new_profile} def store_model_counts(analysis: list[dict[str, Any]]) -> None: @@ -70,7 +70,7 @@ def store_model_counts(analysis: list[dict[str, Any]]) -> None: metrics.save_separate(f"details/{metrics.Metrics.metrics_id}.json", func_summary) # Count the number of respective models - models = {model: 0 for model in regression_models.get_supported_models() if model != "all"} + models = {model: 0 for model in postprocess_structs.get_supported_models() if model != "all"} models["undefined"] = 0 for func_record in funcs.values(): models["undefined" if (func_record["r_square"] <= 0.5) else func_record["model"]] += 1 @@ -94,7 +94,7 @@ def store_model_counts(analysis: list[dict[str, Any]]) -> None: @click.option( "--regression_models", "-r", - type=click.Choice(regression_models.get_supported_models()), + type=click.Choice(postprocess_structs.get_supported_models()), required=False, multiple=True, help=( diff --git a/perun/postprocess/regressogram/methods.py b/perun/postprocess/regressogram/methods.py index 498e29afc..7f918a8d9 100644 --- a/perun/postprocess/regressogram/methods.py +++ b/perun/postprocess/regressogram/methods.py @@ -23,15 +23,6 @@ _REQUIRED_KEYS = ["bucket_method", "statistic_function"] -def get_supported_nparam_methods() -> list[str]: - """Provides all currently supported computational methods, to - estimate the optimal number of buckets, as a list of their names. - - :return: the names of all supported methods - """ - return _METHODS - - def get_supported_selectors() -> list[str]: """Provides all currently supported computational methods, to estimate the optimal number of buckets, as a list of their names. @@ -146,6 +137,3 @@ def regressogram( "sqrt": numpy_bucket_selectors._hist_bin_sqrt, # type: ignore "sturges": numpy_bucket_selectors._hist_bin_sturges, # type: ignore } - -# supported non-parametric methods -_METHODS = ["regressogram", "moving_average", "kernel_regression"] diff --git a/perun/postprocess/regressogram/run.py b/perun/postprocess/regressogram/run.py index 3ef001bd7..d7f466e95 100644 --- a/perun/postprocess/regressogram/run.py +++ b/perun/postprocess/regressogram/run.py @@ -16,7 +16,7 @@ from perun.postprocess.regressogram import methods from perun.profile.factory import pass_profile, Profile from perun.utils.common import cli_kit -from perun.utils.structs import PostprocessStatus +from perun.utils.structs.common_structs import PostprocessStatus _DEFAULT_BUCKETS_METHOD = "doane" diff --git a/perun/profile/__init__.py b/perun/profile/__init__.py index 474a3a2d1..e48b2db44 100644 --- a/perun/profile/__init__.py +++ b/perun/profile/__init__.py @@ -3,3 +3,7 @@ Contains queries over profiles, storage and loading of the profile in the filesystem, transforming the profiles, and converting profiles to different formats. """ + +import lazy_loader as lazy + +__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__) diff --git a/perun/profile/__init__.pyi b/perun/profile/__init__.pyi new file mode 100644 index 000000000..2f2ad20f6 --- /dev/null +++ b/perun/profile/__init__.pyi @@ -0,0 +1,59 @@ +from .factory import ( + Profile as Profile, + pass_profile as pass_profile, +) + +from .convert import ( + resources_to_pandas_dataframe as resources_to_pandas_dataframe, + models_to_pandas_dataframe as models_to_pandas_dataframe, + to_flame_graph_format as to_flame_graph_format, + to_uid as to_uid, + to_string_line as to_string_line, + plot_data_from_coefficients_of as plot_data_from_coefficients_of, + flatten as flatten, +) + +from .helpers import ( + generate_profile_name as generate_profile_name, + load_list_for_minor_version as load_list_for_minor_version, + get_nth_profile_of as get_nth_profile_of, + find_profile_entry as find_profile_entry, + finalize_profile_for_job as finalize_profile_for_job, + to_string as to_string, + to_config_tuple as to_config_tuple, + config_tuple_to_cmdstr as config_tuple_to_cmdstr, + extract_job_from_profile as extract_job_from_profile, + is_key_aggregatable_by as is_key_aggregatable_by, + sort_profiles as sort_profiles, + merge_resources_of as merge_resources_of, + get_default_independent_variable as get_default_independent_variable, + get_default_dependent_variable as get_default_dependent_variable, + ProfileInfo as ProfileInfo, + ProfileMetadata as ProfileMetadata, +) + +from .imports import ( + import_perf_from_record as import_perf_from_record, + import_perf_from_script as import_perf_from_script, + import_perf_from_stack as import_perf_from_stack, + import_elk_from_json as import_elk_from_json, +) + +from .query import ( + flattened_values as flattened_values, + all_items_of as all_items_of, + all_model_fields_of as all_model_fields_of, + all_numerical_resource_fields_of as all_numerical_resource_fields_of, + unique_resource_values_of as unique_resource_values_of, + all_key_values_of as all_key_values_of, + unique_model_values_of as unique_model_values_of, +) + +from .stats import ( + ProfileStatComparison as ProfileStatComparison, + StatComparisonResult as StatComparisonResult, + ProfileStat as ProfileStat, + ProfileStatAggregation as ProfileStatAggregation, + aggregate_stats as aggregate_stats, + compare_stats as compare_stats, +) diff --git a/perun/profile/factory.py b/perun/profile/factory.py index 52d2dceeb..17bd0ed25 100644 --- a/perun/profile/factory.py +++ b/perun/profile/factory.py @@ -19,16 +19,15 @@ import click # Perun Imports +from perun import check as check from perun.logic import config -from perun.postprocess.regression_analysis import regression_models from perun.profile import convert, query, stats, helpers from perun.utils import log +from perun.utils.structs import postprocess_structs from perun.utils.common import common_kit -import perun.check.detection_kit as detection -import perun.postprocess.regressogram.methods as nparam_methods if TYPE_CHECKING: - from perun.utils.structs import ModelRecord + from perun.utils.structs.common_structs import ModelRecord class Profile(MutableMapping[str, Any]): @@ -61,7 +60,7 @@ class Profile(MutableMapping[str, Any]): } persistent = {"trace", "type", "subtype", "uid", "location"} - independent = [ + independent: list[str] = [ "structure-unit-size", "snapshot", "order", @@ -70,7 +69,7 @@ class Profile(MutableMapping[str, Any]): "timestamp", "exclusive", ] - dependent = ["amount"] + dependent: list[str] = ["amount"] def __init__(self, *args: Any, **kwargs: Any) -> None: """Initializes the internal storage @@ -374,9 +373,9 @@ def all_filtered_models(self, models_strategy: str) -> dict[str, ModelRecord]: """ group = models_strategy.rsplit("-")[1] if models_strategy in ("all-param", "all-nonparam"): - return detection.get_filtered_best_models_of(self, group=group, model_filter=None) + return check.get_filtered_best_models_of(self, group=group, model_filter=None) elif models_strategy in ("best-nonparam", "best-model", "best-param"): - return detection.get_filtered_best_models_of(self, group=group) + return check.get_filtered_best_models_of(self, group=group) else: return {} @@ -415,11 +414,11 @@ def all_models(self, group: str = "model") -> Iterable[tuple[int, dict[str, Any] group == "model" or ( group == "param" - and model.get("model") in regression_models.get_supported_models() + and model.get("model") in postprocess_structs.get_supported_models() ) or ( group == "nonparam" - and model.get("model") in nparam_methods.get_supported_nparam_methods() + and model.get("model") in postprocess_structs.get_supported_nparam_methods() ) ): yield model_idx, model diff --git a/perun/profile/helpers.py b/perun/profile/helpers.py index d2dc67b43..14c8da65f 100644 --- a/perun/profile/helpers.py +++ b/perun/profile/helpers.py @@ -29,7 +29,7 @@ # Perun Imports from perun.logic import config, index, pcs, store -from perun.profile import factory as profiles, query +from perun import profile as profiles from perun.utils import decorators, log as perun_log from perun.utils.common import common_kit from perun.utils.external import environment @@ -38,7 +38,7 @@ MissingConfigSectionException, TagOutOfRangeException, ) -from perun.utils.structs import Unit, Executable, Job +from perun.utils.structs.common_structs import Unit, Executable, Job from perun.vcs import vcs_kit if TYPE_CHECKING: @@ -79,7 +79,7 @@ def lookup_param(profile: profiles.Profile, unit: str, param: str) -> str: unit_params = unit_param_map.get(unit) if unit_params: return ( - common_kit.sanitize_filepart(list(query.all_key_values_of(unit_params, param))[0]) + common_kit.sanitize_filepart(list(profiles.all_key_values_of(unit_params, param))[0]) or "_" ) else: @@ -397,7 +397,7 @@ def is_key_aggregatable_by(profile: profiles.Profile, func: str, key: str, keyna return True # Get all valid numeric keys and validate - valid_keys = set(query.all_numerical_resource_fields_of(profile)) + valid_keys = set(profiles.all_numerical_resource_fields_of(profile)) if key not in valid_keys: choices = "(choose either count/nunique as aggregation function;" choices += f" or from the following keys: {', '.join(map(str, valid_keys))})" diff --git a/perun/profile/imports.py b/perun/profile/imports.py index e7fff622f..5ba984fd3 100755 --- a/perun/profile/imports.py +++ b/perun/profile/imports.py @@ -16,14 +16,13 @@ # Third-Party Imports # Perun Imports +from perun import profile as profile from perun.collect.kperf import parser from perun.logic import commands, config, index, pcs -from perun.profile import query, helpers as profile_helpers, stats as profile_stats -from perun.profile.factory import Profile from perun.utils import log, streams from perun.utils.common import script_kit, common_kit from perun.utils.external import commands as external_commands, environment -from perun.utils.structs import MinorVersion +from perun.utils.structs.common_structs import MinorVersion from perun.vcs import vcs_kit @@ -31,7 +30,7 @@ class _PerfProfileSpec: """A representation of a perf profile record to import. - :ivar path: the absolute path to the perf profile. + :ivar path: the absolute path to the perf :ivar exit_code: the exit code of the profile collection process. """ @@ -162,7 +161,7 @@ def import_elk_from_json( import_dir = Path(config.lookup_key_recursively("import.dir", os.getcwd())) resources: list[dict[str, Any]] = [] # Load the CLI-supplied metadata, if any - elk_metadata: dict[str, profile_helpers.ProfileMetadata] = { + elk_metadata: dict[str, profile.ProfileMetadata] = { data.name: data for data in _import_metadata(metadata, import_dir) } @@ -184,7 +183,7 @@ def import_elk_from_json( def import_perf_profile( profiles: list[_PerfProfileSpec], - stats: list[profile_stats.ProfileStat], + stats: list[profile.ProfileStat], resources: list[dict[str, Any]], minor_version: MinorVersion, **kwargs: Any, @@ -198,7 +197,7 @@ def import_perf_profile( :param kwargs: rest of the parameters. """ import_dir = Path(config.lookup_key_recursively("import.dir", os.getcwd())) - prof = Profile( + prof = profile.Profile( { "global": { "time": "???", @@ -214,7 +213,7 @@ def import_perf_profile( "header": { "type": "time", "cmd": kwargs.get("cmd", ""), - "exitcode": [profile.exit_code for profile in profiles], + "exitcode": [p.exit_code for p in profiles], "workload": kwargs.get("workload", ""), "units": {"time": "sample"}, }, @@ -234,7 +233,7 @@ def import_perf_profile( def import_elk_profile( resources: list[dict[str, Any]], - metadata: dict[str, profile_helpers.ProfileMetadata], + metadata: dict[str, profile.ProfileMetadata], minor_version: MinorVersion, save_to_index: bool = False, **kwargs: Any, @@ -247,7 +246,7 @@ def import_elk_profile( :param save_to_index: indication whether we should save the imported profiles to index. :param kwargs: rest of the parameters. """ - prof = Profile( + prof = profile.Profile( { "global": { "time": "???", @@ -273,14 +272,16 @@ def import_elk_profile( save_imported_profile(prof, save_to_index, minor_version) -def save_imported_profile(prof: Profile, save_to_index: bool, minor_version: MinorVersion) -> None: +def save_imported_profile( + prof: profile.Profile, save_to_index: bool, minor_version: MinorVersion +) -> None: """Saves the imported profile either to index or to pending jobs. :param prof: imported profile :param minor_version: minor version corresponding to the imported profiles. :param save_to_index: indication whether we should save the imported profiles to index. """ - full_profile_name = profile_helpers.generate_profile_name(prof) + full_profile_name = profile.generate_profile_name(prof) profile_directory = pcs.get_job_directory() full_profile_path = os.path.join(profile_directory, full_profile_name) @@ -318,7 +319,7 @@ def load_perf_file(filepath: Path) -> str: def extract_from_elk( elk_query: list[dict[str, Any]] -) -> tuple[list[dict[str, Any]], dict[str, profile_helpers.ProfileMetadata]]: +) -> tuple[list[dict[str, Any]], dict[str, profile.ProfileMetadata]]: """For the given elk query, extracts resources and metadata. For metadata, we consider any key that has only single value through the profile, @@ -339,7 +340,7 @@ def extract_from_elk( if not k.startswith("metric") and not k.startswith("benchmarking") and len(v) == 1 } - metadata = {k: profile_helpers.ProfileMetadata(k, res_counter[k].pop()) for k in metadata_keys} + metadata = {k: profile.ProfileMetadata(k, res_counter[k].pop()) for k in metadata_keys} resources = [ { k: common_kit.try_convert(v, [int, float, str]) @@ -383,7 +384,7 @@ def get_machine_info(machine_info: str, import_dir: Path) -> dict[str, Any]: def extract_machine_info_from_elk_metadata( - metadata: dict[str, profile_helpers.ProfileMetadata] + metadata: dict[str, profile.ProfileMetadata] ) -> dict[str, Any]: """Extracts the parts of the profile that correspond to machine info. @@ -395,27 +396,19 @@ def extract_machine_info_from_elk_metadata( :return: machine info extracted from the profiles. """ machine_info: dict[str, Any] = { - "architecture": metadata.get( - "machine.arch", profile_helpers.ProfileMetadata("", "?") - ).value, + "architecture": metadata.get("machine.arch", profile.ProfileMetadata("", "?")).value, "system": str( - metadata.get("machine.os", profile_helpers.ProfileMetadata("", "?")).value + metadata.get("machine.os", profile.ProfileMetadata("", "?")).value ).capitalize(), - "release": metadata.get( - "extra.machine.platform", profile_helpers.ProfileMetadata("", "?") - ).value, - "host": metadata.get("machine.hostname", profile_helpers.ProfileMetadata("", "?")).value, + "release": metadata.get("extra.machine.platform", profile.ProfileMetadata("", "?")).value, + "host": metadata.get("machine.hostname", profile.ProfileMetadata("", "?")).value, "cpu": { "physical": "?", - "total": metadata.get( - "machine.cpu-cores", profile_helpers.ProfileMetadata("", "?") - ).value, + "total": metadata.get("machine.cpu-cores", profile.ProfileMetadata("", "?")).value, "frequency": "?", }, "memory": { - "total_ram": metadata.get( - "machine.ram", profile_helpers.ProfileMetadata("", "?") - ).value, + "total_ram": metadata.get("machine.ram", profile.ProfileMetadata("", "?")).value, "swap": "?", }, "boot_info": "?", @@ -426,9 +419,7 @@ def extract_machine_info_from_elk_metadata( return machine_info -def _import_metadata( - metadata: tuple[str, ...], import_dir: Path -) -> list[profile_helpers.ProfileMetadata]: +def _import_metadata(metadata: tuple[str, ...], import_dir: Path) -> list[profile.ProfileMetadata]: """Parse the metadata entries from CLI and convert them to our internal representation. :param import_dir: the import directory to use for relative metadata file paths. @@ -436,7 +427,7 @@ def _import_metadata( :return: a collection of parsed and converted metadata objects """ - p_metadata: list[profile_helpers.ProfileMetadata] = [] + p_metadata: list[profile.ProfileMetadata] = [] # Normalize the metadata string for parsing and/or opening the file for metadata_str in map(str.strip, metadata): if metadata_str.lower().endswith(".json"): @@ -445,13 +436,13 @@ def _import_metadata( else: # Add a single metadata entry parsed from its string representation try: - p_metadata.append(profile_helpers.ProfileMetadata.from_string(metadata_str)) + p_metadata.append(profile.ProfileMetadata.from_string(metadata_str)) except TypeError: log.warn(f"Ignoring invalid profile metadata string '{metadata_str}'.") return p_metadata -def _parse_metadata_json(metadata_path: Path) -> list[profile_helpers.ProfileMetadata]: +def _parse_metadata_json(metadata_path: Path) -> list[profile.ProfileMetadata]: """Parse a metadata JSON file into the metadata objects. If the JSON file contains nested dictionaries, the hierarchical keys will be flattened. @@ -467,8 +458,8 @@ def _parse_metadata_json(metadata_path: Path) -> list[profile_helpers.ProfileMet return [] # Make sure we flatten the input metadata_list = [ - profile_helpers.ProfileMetadata(k, v) - for k, v in query.all_items_of(json.load(metadata_handle)) + profile.ProfileMetadata(k, v) + for k, v in profile.all_items_of(json.load(metadata_handle)) ] log.minor_success(log.path_style(str(metadata_path)), "parsed") return metadata_list @@ -476,7 +467,7 @@ def _parse_metadata_json(metadata_path: Path) -> list[profile_helpers.ProfileMet def _parse_perf_import_entries( import_entries: list[str], cli_stats_headers: str -) -> tuple[list[_PerfProfileSpec], list[profile_stats.ProfileStat]]: +) -> tuple[list[_PerfProfileSpec], list[profile.ProfileStat]]: """Parses perf import entries and stats. An import entry is either a profile entry @@ -504,8 +495,7 @@ def _parse_perf_import_entries( :return: parsed profiles and stats. """ stats = [ - profile_stats.ProfileStat.from_string(*stat.split("|")) - for stat in cli_stats_headers.split(",") + profile.ProfileStat.from_string(*stat.split("|")) for stat in cli_stats_headers.split(",") ] cli_stats_len = len(stats) @@ -528,7 +518,7 @@ def _parse_perf_import_csv( csv_file: str, import_dir: Path, profiles: list[_PerfProfileSpec], - stats: list[profile_stats.ProfileStat], + stats: list[profile.ProfileStat], ) -> None: """Parse stats headers and perf import entries in a CSV file. @@ -547,8 +537,8 @@ def _parse_perf_import_csv( log.warn(f"Empty import file {csv_path}. Skipping.") return # Parse the stats headers - csv_stats: list[profile_stats.ProfileStat] = [ - profile_stats.ProfileStat.from_string(*stat_definition.split("|")) + csv_stats: list[profile.ProfileStat] = [ + profile.ProfileStat.from_string(*stat_definition.split("|")) for stat_definition in header[2:] ] # Parse the remaining rows that represent profile specifications and filter invalid ones @@ -564,7 +554,7 @@ def _parse_perf_import_csv( def _parse_perf_entry( - entry: list[str], import_dir: Path, stats: list[profile_stats.ProfileStat] + entry: list[str], import_dir: Path, stats: list[profile.ProfileStat] ) -> _PerfProfileSpec | None: """Parse a single perf profile import entry. @@ -596,9 +586,7 @@ def _parse_perf_entry( return profile_info -def _merge_stats( - new_stat: profile_stats.ProfileStat, into_stats: list[profile_stats.ProfileStat] -) -> None: +def _merge_stats(new_stat: profile.ProfileStat, into_stats: list[profile.ProfileStat]) -> None: """Merge a new profile stat values into the current profile stats. If an existing stat with the same name exists, the values of both stats are merged. If no such diff --git a/perun/profile/meson.build b/perun/profile/meson.build index ba4baac8a..505e51ac2 100644 --- a/perun/profile/meson.build +++ b/perun/profile/meson.build @@ -2,6 +2,7 @@ perun_profile_dir = perun_dir / 'profile' perun_profile_files = files( '__init__.py', + '__init__.pyi', 'convert.py', 'factory.py', 'helpers.py', diff --git a/perun/profile/query.py b/perun/profile/query.py index f8ded281a..9fa137510 100644 --- a/perun/profile/query.py +++ b/perun/profile/query.py @@ -26,7 +26,7 @@ from perun.utils.common import common_kit if TYPE_CHECKING: - from perun.profile.factory import Profile + from perun.profile import Profile def flattened_values(root_key: Any, root_value: Any) -> Iterable[tuple[str, str | float]]: diff --git a/perun/py.typed b/perun/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/perun/select/abstract_base_selection.py b/perun/select/abstract_base_selection.py index 54c7e3126..a9879526a 100755 --- a/perun/select/abstract_base_selection.py +++ b/perun/select/abstract_base_selection.py @@ -21,7 +21,7 @@ if TYPE_CHECKING: - from perun.utils.structs import MinorVersion + from perun.utils.structs.common_structs import MinorVersion from perun.profile.factory import Profile from perun.profile.helpers import ProfileInfo diff --git a/perun/select/whole_repository_selection.py b/perun/select/whole_repository_selection.py index 243b626e9..625cb9e42 100755 --- a/perun/select/whole_repository_selection.py +++ b/perun/select/whole_repository_selection.py @@ -17,7 +17,7 @@ from perun.profile.factory import Profile from perun.profile.helpers import ProfileInfo from perun.select.abstract_base_selection import AbstractBaseSelection -from perun.utils.structs import MinorVersion +from perun.utils.structs.common_structs import MinorVersion class WholeRepositorySelection(AbstractBaseSelection): diff --git a/perun/templates/check.jinja2 b/perun/templates/check.jinja2 index c35f542bb..436cea0a9 100644 --- a/perun/templates/check.jinja2 +++ b/perun/templates/check.jinja2 @@ -3,7 +3,7 @@ TODO: Write long dosctring of the new degradation check """ -import perun.check as check +from perun import check as check def {{ unit_name }}(baseline_profile, target_profile): diff --git a/perun/testing/mock_results.py b/perun/testing/mock_results.py index 457624b36..cb00cbfa9 100644 --- a/perun/testing/mock_results.py +++ b/perun/testing/mock_results.py @@ -5,8 +5,8 @@ # Third-Party Imports # Perun Imports -from perun.utils.structs import DegradationInfo -from perun.utils.structs import PerformanceChange as pc +from perun.utils.structs.common_structs import DegradationInfo +from perun.utils.structs.common_structs import PerformanceChange as pc _PREG_EXPECTED_RESULTS = [ diff --git a/perun/utils/common/cli_kit.py b/perun/utils/common/cli_kit.py index a45a81b2c..acd97ea74 100644 --- a/perun/utils/common/cli_kit.py +++ b/perun/utils/common/cli_kit.py @@ -24,11 +24,9 @@ import jinja2 # Perun Imports -from perun.collect.trace.optimizations.optimization import Optimization -from perun.collect.trace.optimizations.structs import CallGraphTypes +import perun +from perun import profile as profile from perun.logic import commands, store, stats, config, pcs -from perun.profile import helpers as profile_helpers, query -from perun.profile.factory import Profile from perun.utils import exceptions, streams, timestamps, log, metrics from perun.utils.common import common_kit from perun.utils.exceptions import ( @@ -37,10 +35,10 @@ StatsFileNotFoundException, NotPerunRepositoryException, ) -import perun +from perun.utils.structs import collect_structs if TYPE_CHECKING: - from perun.utils.structs import MinorVersion + from perun.utils.structs.common_structs import MinorVersion def print_version(_: click.Context, __: click.Option, value: bool) -> None: @@ -105,7 +103,7 @@ def process_resource_key_param( assert ( hasattr(ctx, "parent") and ctx.parent is not None ), "The function expects `ctx` has parent" - valid_keys = set(ctx.parent.params.get("profile", Profile()).all_resource_fields()) + valid_keys = set(ctx.parent.params.get("profile", profile.Profile()).all_resource_fields()) if value not in valid_keys: valid_keys_str = ", ".join(f"'{vk}'" for vk in valid_keys) raise click.BadParameter(f"'{value}' is not one of {valid_keys_str}.") @@ -129,7 +127,9 @@ def process_continuous_key( if value != "snapshots" and ctx.parent is not None: # If the requested value is not 'snapshots', then get all the numerical keys valid_numeric_keys = set( - query.all_numerical_resource_fields_of(ctx.parent.params.get("profile", Profile())) + profile.all_numerical_resource_fields_of( + ctx.parent.params.get("profile", profile.Profile()) + ) ) # Check if the value is valid numeric key if value not in valid_numeric_keys: @@ -298,7 +298,7 @@ def lookup_nth_pending_filename(position: int) -> str: :return: pending profile at given position """ pending = commands.get_untracked_profiles() - profile_helpers.sort_profiles(pending) + profile.sort_profiles(pending) if 0 <= position < len(pending): return pending[position].realpath else: @@ -370,7 +370,7 @@ def add_to_removed_from_index(index: int) -> None: :param index: index we are looking up and registering to massaged values """ try: - index_filename = profile_helpers.get_nth_profile_of(index, ctx.params["minor"]) + index_filename = profile.get_nth_profile_of(index, ctx.params["minor"]) start = index_filename.rfind("objects") + len("objects") # Remove the .perun/objects/... prefix and merge the directory and file to sha ctx.params["from_index_generator"].add("".join(index_filename[start:].split("/"))) @@ -471,7 +471,7 @@ def lookup_minor_version_callback(_: click.Context, __: click.Option, value: str def lookup_list_of_profiles_callback( ctx: click.Context, arg: click.Argument, value: tuple[str] -) -> list[Profile]: +) -> list[profile.Profile]: """Callback for lookup up list of profiles anywhere :param ctx: context of the CLI @@ -481,14 +481,16 @@ def lookup_list_of_profiles_callback( """ profiles = [] aggregation_function = config.lookup_key_recursively("profile.aggregation", default="median") - for profile in value: - loaded_profile = lookup_any_profile_callback(ctx, arg, profile) + for prof in value: + loaded_profile = lookup_any_profile_callback(ctx, arg, prof) loaded_profile.apply(aggregation_function) profiles.append(loaded_profile) return profiles -def lookup_any_profile_callback(_: click.Context, __: click.Argument, value: str) -> Profile: +def lookup_any_profile_callback( + _: click.Context, __: click.Argument, value: str +) -> profile.Profile: """Callback for looking up any profile, i.e. anywhere (in index, in pending, etc.) :param _: context @@ -508,7 +510,7 @@ def lookup_any_profile_callback(_: click.Context, __: click.Argument, value: str index_tag_match = store.INDEX_TAG_REGEX.match(value) if index_tag_match: try: - index_profile = profile_helpers.get_nth_profile_of(int(index_tag_match.group(1)), rev) + index_profile = profile.get_nth_profile_of(int(index_tag_match.group(1)), rev) return store.load_profile_from_file(index_profile, is_raw_profile=False) except TagOutOfRangeException as exc: raise click.BadParameter(str(exc)) @@ -832,6 +834,8 @@ def set_optimization(_: click.Context, param: click.Argument, value: str) -> str :param value: value of the parameter :return: the value """ + from perun.collect.trace.optimizations.optimization import Optimization + # Set the optimization pipeline if param.human_readable_name == "optimization_pipeline": Optimization.set_pipeline(value) @@ -854,6 +858,9 @@ def set_optimization_param(_: click.Context, __: click.Argument, value: str) -> :param value: value of the parameter :return: the value """ + # TODO: to be fixed when lazy-loader is used in the entire codebase + from perun.collect.trace.optimizations.optimization import Optimization + for param in value: # Process all parameters as 'parameter: value' tuples opt_name, opt_value = param[0], param[1] @@ -871,6 +878,8 @@ def set_optimization_cache(_: click.Context, __: click.Option, value: str) -> No :param __: the click parameter :param value: value of the parameter """ + from perun.collect.trace.optimizations.optimization import Optimization + Optimization.resource_cache = not value @@ -882,6 +891,8 @@ def reset_optimization_cache(_: click.Context, __: click.Option, value: str) -> :param __: the click parameter :param value: value of the parameter """ + from perun.collect.trace.optimizations.optimization import Optimization + Optimization.reset_cache = value @@ -892,7 +903,9 @@ def set_call_graph_type(_: click.Context, __: click.Argument, value: str) -> Non :param __: the click parameter :param value: value of the parameter """ - Optimization.call_graph_type = CallGraphTypes(value) + from perun.collect.trace.optimizations.optimization import Optimization + + Optimization.call_graph_type = collect_structs.CallGraphTypes(value) def configure_metrics(_: click.Context, __: click.Option, value: tuple[str, str]) -> None: diff --git a/perun/utils/log.py b/perun/utils/log.py index 4814bfa6d..297a73f96 100644 --- a/perun/utils/log.py +++ b/perun/utils/log.py @@ -32,7 +32,7 @@ AttrChoiceType, ColorChoiceType, ) -from perun.utils.structs import ( +from perun.utils.structs.common_structs import ( PerformanceChange, DegradationInfo, MinorVersion, diff --git a/perun/utils/meson.build b/perun/utils/meson.build index 419f52fa9..11a4270f9 100644 --- a/perun/utils/meson.build +++ b/perun/utils/meson.build @@ -8,7 +8,6 @@ perun_utils_files = files( 'mapping.py', 'metrics.py', 'streams.py', - 'structs.py', 'timestamps.py', ) @@ -19,3 +18,4 @@ py3.install_sources( subdir('common') subdir('external') +subdir('structs') diff --git a/perun/utils/structs/__init__.py b/perun/utils/structs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/perun/utils/structs/check_structs.py b/perun/utils/structs/check_structs.py new file mode 100644 index 000000000..b6a5cb50d --- /dev/null +++ b/perun/utils/structs/check_structs.py @@ -0,0 +1,29 @@ +from __future__ import annotations + + +def get_supported_detection_models_strategies() -> list[str]: + """ + Provides supported detection models strategies to execute + the degradation check between two profiles with different kinds + of models. The individual strategies represent the way of + executing the detection between profiles and their models: + + - best-param: best parametric models from both profiles + - best-non-param: best non-parametric models from both profiles + - best-model: best models from both profiles + - all-param: all parametric models pair from both profiles + - all-non-param: all non-parametric models pair from both profiles + - all-models: all models pair from both profiles + - best-both: best parametric and non-parametric models from both profiles + + :return: the names of all supported degradation models strategies + """ + return [ + "best-model", + "best-param", + "best-nonparam", + "all-param", + "all-nonparam", + "all-models", + "best-both", + ] diff --git a/perun/utils/structs/collect_structs.py b/perun/utils/structs/collect_structs.py new file mode 100644 index 000000000..03f3be409 --- /dev/null +++ b/perun/utils/structs/collect_structs.py @@ -0,0 +1,133 @@ +# Standard Imports +from enum import Enum + +# Third-Party Imports +# Perun Imports + + +class Optimizations(Enum): + """Enumeration of the implemented methods and their CLI name.""" + + BASELINE_STATIC = "baseline-static" + BASELINE_DYNAMIC = "baseline-dynamic" + CALL_GRAPH_SHAPING = "cg-shaping" + DYNAMIC_SAMPLING = "dynamic-sampling" + DIFF_TRACING = "diff-tracing" + DYNAMIC_PROBING = "dynamic-probing" + TIMED_SAMPLING = "timed-sampling" + + @staticmethod + def supported(): + """List the currently supported optimization methods. + + :return: CLI names of the supported optimizations + """ + return [optimization.value for optimization in Optimizations] + + +class Pipeline(Enum): + """Enumeration of the implemented pipelines and their CLI name. + Custom represents a defualt pipeline that has no pre-configured methods or parameters + """ + + CUSTOM = "custom" + BASIC = "basic" + ADVANCED = "advanced" + FULL = "full" + + @staticmethod + def supported(): + """List the currently supported optimization pipelines. + + :return: CLI names of the supported pipelines + """ + return [pipeline.value for pipeline in Pipeline] + + @staticmethod + def default(): + """Name of the default pipeline. + + :return: the CLI name of the default pipeline + """ + return Pipeline.CUSTOM.value + + def map_to_optimizations(self): + """Map the selected optimization pipeline to the set of employed optimization methods. + + :return: list of the Optimizations enumeration objects + """ + if self == Pipeline.BASIC: + return [Optimizations.CALL_GRAPH_SHAPING, Optimizations.BASELINE_DYNAMIC] + if self == Pipeline.ADVANCED: + return [ + Optimizations.DIFF_TRACING, + Optimizations.CALL_GRAPH_SHAPING, + Optimizations.BASELINE_DYNAMIC, + Optimizations.DYNAMIC_SAMPLING, + ] + if self == Pipeline.FULL: + return [ + Optimizations.DIFF_TRACING, + Optimizations.CALL_GRAPH_SHAPING, + Optimizations.BASELINE_STATIC, + Optimizations.BASELINE_DYNAMIC, + Optimizations.DYNAMIC_SAMPLING, + Optimizations.DYNAMIC_PROBING, + ] + return [] + + +class CallGraphTypes(Enum): + """Enumeration of the implemented call graph types and their CLI names.""" + + STATIC = "static" + DYNAMIC = "dynamic" + MIXED = "mixed" + + @staticmethod + def supported(): + """List the currently supported call graph types. + + :return: CLI names of the supported cg types + """ + return [cg.value for cg in CallGraphTypes] + + @staticmethod + def default(): + """Name of the default cg type. + + :return: the CLI name of the default cg type + """ + return CallGraphTypes.STATIC.value + + +class Parameters(Enum): + """Enumeration of the currently supported CLI options for optimization methods and pipelines.""" + + DIFF_VERSION = "diff-version" + DIFF_KEEP_LEAF = "diff-keep-leaf" + DIFF_INSPECT_ALL = "diff-inspect-all" + DIFF_CG_MODE = "diff-cfg-mode" + SOURCE_FILES = "source-files" + SOURCE_DIRS = "source-dirs" + STATIC_COMPLEXITY = "static-complexity" + STATIC_KEEP_TOP = "static-keep-top" + CG_SHAPING_MODE = "cg-mode" + CG_PROJ_LEVELS = "cg-proj-levels" + CG_PROJ_KEEP_LEAF = "cg-proj-keep-leaf" + DYNSAMPLE_STEP = "dyn-sample-step" + DYNSAMPLE_THRESHOLD = "dyn-sample-threshold" + PROBING_THRESHOLD = "probing-threshold" + PROBING_REATTACH = "probing-reattach" + TIMEDSAMPLE_FREQ = "timed-sample-freq" + DYNBASE_SOFT_THRESHOLD = "dyn-base-soft-threshold" + DYNBASE_HARD_THRESHOLD = "dyn-base-hard-threshold" + THRESHOLD_MODE = "threshold-mode" + + @staticmethod + def supported(): + """List the currently supported optimization parameters. + + :return: CLI names of the supported parameters + """ + return [parameter.value for parameter in Parameters] diff --git a/perun/utils/structs.py b/perun/utils/structs/common_structs.py similarity index 99% rename from perun/utils/structs.py rename to perun/utils/structs/common_structs.py index 92c5eeefb..448d1362e 100644 --- a/perun/utils/structs.py +++ b/perun/utils/structs/common_structs.py @@ -24,6 +24,8 @@ import numpy.typing as npt import numpy +# TODO: think about breaking this into more modules and/or renaming it to something better + class SignalCallback(Protocol): def __call__(self, *args: Any, **kwargs: Any) -> Any: ... diff --git a/perun/utils/structs/meson.build b/perun/utils/structs/meson.build new file mode 100644 index 000000000..bc4271f6c --- /dev/null +++ b/perun/utils/structs/meson.build @@ -0,0 +1,14 @@ +perun_utils_structs_dir = perun_utils_dir / 'structs' + +perun_utils_structs_files = files( + '__init__.py', + 'check_structs.py', + 'collect_structs.py', + 'common_structs.py', + 'postprocess_structs.py', +) + +py3.install_sources( + perun_utils_structs_files, + subdir: perun_utils_structs_dir +) diff --git a/perun/utils/structs/postprocess_structs.py b/perun/utils/structs/postprocess_structs.py new file mode 100644 index 000000000..ea2ef632d --- /dev/null +++ b/perun/utils/structs/postprocess_structs.py @@ -0,0 +1,21 @@ +from __future__ import annotations + + +def get_supported_models() -> list[str]: + """Provides all currently supported models as a list of their names. + + The 'all' specifier is used in reverse mapping as it enables to easily specify all models + + :return: the names of all supported models and 'all' specifier + """ + # Disable quadratic model, but allow to process already existing profiles with quad model + return ["all", "constant", "linear", "logarithmic", "quadratic", "power", "exponential"] + + +def get_supported_nparam_methods() -> list[str]: + """Provides all currently supported computational methods, to + estimate the optimal number of buckets, as a list of their names. + + :return: the names of all supported methods + """ + return ["regressogram", "moving_average", "kernel_regression"] diff --git a/perun/vcs/abstract_repository.py b/perun/vcs/abstract_repository.py index abf745d20..93daf8ac1 100755 --- a/perun/vcs/abstract_repository.py +++ b/perun/vcs/abstract_repository.py @@ -10,7 +10,7 @@ # Perun Imports if TYPE_CHECKING: - from perun.utils.structs import MinorVersion, MajorVersion + from perun.utils.structs.common_structs import MinorVersion, MajorVersion class AbstractRepository(ABC): diff --git a/perun/vcs/git_repository.py b/perun/vcs/git_repository.py index 1fe9db3f1..c9e46cee0 100644 --- a/perun/vcs/git_repository.py +++ b/perun/vcs/git_repository.py @@ -20,7 +20,7 @@ from perun.vcs.abstract_repository import AbstractRepository from perun.utils import log as perun_log, timestamps from perun.utils.exceptions import VersionControlSystemException -from perun.utils.structs import MinorVersion, MajorVersion +from perun.utils.structs.common_structs import MinorVersion, MajorVersion class GitRepository(AbstractRepository): diff --git a/perun/vcs/svs_repository.py b/perun/vcs/svs_repository.py index 2a429f31c..84b73bc80 100755 --- a/perun/vcs/svs_repository.py +++ b/perun/vcs/svs_repository.py @@ -18,7 +18,7 @@ # Third-Party Imports # Perun Imports -from perun.utils.structs import MajorVersion, MinorVersion +from perun.utils.structs.common_structs import MajorVersion, MinorVersion from perun.vcs.abstract_repository import AbstractRepository SINGLE_VERSION_BRANCH: str = "master" diff --git a/perun/view_diff/report/run.py b/perun/view_diff/report/run.py index 725be79dd..aaff0c25e 100755 --- a/perun/view_diff/report/run.py +++ b/perun/view_diff/report/run.py @@ -34,7 +34,7 @@ from perun.templates import filters, factory as templates from perun.utils import log, mapping from perun.utils.common import diff_kit, common_kit -from perun.utils.structs import WebColorPalette +from perun.utils.structs.common_structs import WebColorPalette from perun.view_diff.flamegraph import run as flamegraph_run diff --git a/perun/view_diff/sankey/run.py b/perun/view_diff/sankey/run.py index f6f4d5e00..839903a52 100755 --- a/perun/view_diff/sankey/run.py +++ b/perun/view_diff/sankey/run.py @@ -30,7 +30,7 @@ from perun.profile.factory import Profile from perun.utils import log from perun.utils.common import diff_kit, common_kit -from perun.utils.structs import WebColorPalette +from perun.utils.structs.common_structs import WebColorPalette @dataclass diff --git a/perun/workload/__init__.py b/perun/workload/__init__.py index 429ac0140..a61b31d78 100644 --- a/perun/workload/__init__.py +++ b/perun/workload/__init__.py @@ -18,7 +18,7 @@ from perun.logic import config from perun.utils import log, decorators from perun.utils.common import common_kit -from perun.utils.structs import GeneratorSpec +from perun.utils.structs.common_structs import GeneratorSpec @decorators.resetable_always_singleton diff --git a/perun/workload/external_generator.py b/perun/workload/external_generator.py index af3511d82..5cb9e28f0 100644 --- a/perun/workload/external_generator.py +++ b/perun/workload/external_generator.py @@ -47,7 +47,7 @@ from perun.workload.generator import WorkloadGenerator if TYPE_CHECKING: - from perun.utils.structs import Job + from perun.utils.structs.common_structs import Job class ExternalGenerator(WorkloadGenerator): diff --git a/perun/workload/generator.py b/perun/workload/generator.py index 5d27555eb..f02982017 100644 --- a/perun/workload/generator.py +++ b/perun/workload/generator.py @@ -15,10 +15,10 @@ # Third-Party Imports # Perun Imports +from perun import profile from perun.logic import config -from perun.profile import helpers as profile_helpers, factory as profile_factory from perun.utils import log -from perun.utils.structs import CollectStatus, Job, Unit +from perun.utils.structs.common_structs import CollectStatus, Job, Unit from perun.utils.common import common_kit if TYPE_CHECKING: @@ -53,7 +53,7 @@ def generate( :return: tuple of collection status and collected profile """ - collective_profile, collective_status = profile_factory.Profile(), CollectStatus.OK + collective_profile, collective_status = profile.Profile(), CollectStatus.OK for workload, workload_ctx in self._generate_next_workload(): config.runtime().set("context.workload", workload_ctx) @@ -70,7 +70,7 @@ def generate( collective_status = ( CollectStatus.ERROR if collective_status == CollectStatus.ERROR else c_status ) - collective_profile = profile_helpers.merge_resources_of(collective_profile, prof) + collective_profile = profile.merge_resources_of(collective_profile, prof) if not self.for_each: yield collective_status, collective_profile diff --git a/perun/workload/integer_generator.py b/perun/workload/integer_generator.py index c7e96e767..f0856baf1 100644 --- a/perun/workload/integer_generator.py +++ b/perun/workload/integer_generator.py @@ -32,7 +32,7 @@ # Third-Party Imports # Perun Imports -from perun.utils.structs import Job +from perun.utils.structs.common_structs import Job from perun.workload.generator import WorkloadGenerator diff --git a/perun/workload/singleton_generator.py b/perun/workload/singleton_generator.py index 6307b3412..d62f54990 100644 --- a/perun/workload/singleton_generator.py +++ b/perun/workload/singleton_generator.py @@ -19,7 +19,7 @@ # Third-Party Imports # Perun Imports -from perun.utils.structs import Job +from perun.utils.structs.common_structs import Job from perun.workload.generator import WorkloadGenerator diff --git a/perun/workload/string_generator.py b/perun/workload/string_generator.py index 1b7902c7c..fe4bcb45b 100644 --- a/perun/workload/string_generator.py +++ b/perun/workload/string_generator.py @@ -35,7 +35,7 @@ # Third-Party Imports # Perun Imports -from perun.utils.structs import Job +from perun.utils.structs.common_structs import Job from perun.workload.generator import WorkloadGenerator diff --git a/perun/workload/textfile_generator.py b/perun/workload/textfile_generator.py index 4c5289481..f44610813 100644 --- a/perun/workload/textfile_generator.py +++ b/perun/workload/textfile_generator.py @@ -43,7 +43,7 @@ import faker # Perun Imports -from perun.utils.structs import Job +from perun.utils.structs.common_structs import Job from perun.utils.common import common_kit from perun.workload.generator import WorkloadGenerator diff --git a/pyproject.toml b/pyproject.toml index cbc5f5c9f..79e83a7df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,9 @@ dependencies = [ "ninja>=1.11", "wheel>=0.43.0", + # Lazy loading + "lazy-loader>=0.4", + # Other "psutil>=6.0.0", @@ -140,6 +143,7 @@ module = [ "statsmodels.*", "holoviews.*", "bcc.*", + "lazy_loader.*", ] ignore_missing_imports = true diff --git a/tests/test_cli.py b/tests/test_cli.py index fbb06fdf6..376b1ae1f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -19,15 +19,14 @@ # Perun Imports from perun import cli -from perun.cli_groups import utils_cli, config_cli, run_cli, check_cli +from perun.cli_groups import utils_cli, config_cli, run_cli, check_cli, collect_cli from perun.logic import config, pcs, stats, temp from perun.testing import asserts from perun.utils import exceptions, log from perun.utils.common import common_kit -from perun.utils.exceptions import NotPerunRepositoryException from perun.utils.external import commands -from perun.utils.structs import CollectStatus, RunnerReport -import perun.check.factory as check +from perun.utils.structs.common_structs import CollectStatus, RunnerReport +from perun import check as check import perun.testing.utils as test_utils @@ -1401,7 +1400,8 @@ def test_collect_correct(pcs_with_root): """ runner = CliRunner() result = runner.invoke( - cli.collect, ["-c echo", "-w hello", "-o", "prof.perf", "time", "--repeat=1", "--warmup=1"] + collect_cli.collect, + ["-c echo", "-w hello", "-o", "prof.perf", "time", "--repeat=1", "--warmup=1"], ) asserts.predicate_from_cli(result, result.exit_code == 0) assert "prof.perf" in os.listdir(".") @@ -1409,10 +1409,14 @@ def test_collect_correct(pcs_with_root): current_dir = os.path.split(__file__)[0] src_dir = os.path.join(current_dir, "sources", "collect_bounds") src_file = os.path.join(src_dir, "partitioning.c") - result = runner.invoke(cli.collect, ["-c echo", "-w hello", "bounds", "-d", f"{src_dir}"]) + result = runner.invoke( + collect_cli.collect, ["-c echo", "-w hello", "bounds", "-d", f"{src_dir}"] + ) asserts.predicate_from_cli(result, result.exit_code == 0) - result = runner.invoke(cli.collect, ["-c echo", "-w hello", "bounds", "-s", f"{src_file}"]) + result = runner.invoke( + collect_cli.collect, ["-c echo", "-w hello", "bounds", "-s", f"{src_file}"] + ) asserts.predicate_from_cli(result, result.exit_code == 0) assert len(os.listdir(os.path.join(".perun", "logs"))) == 0 @@ -2620,11 +2624,13 @@ def test_svs(): result = runner.invoke(cli.init, [dst]) asserts.predicate_from_cli(result, result.exit_code == 0) - result = runner.invoke(cli.collect, ["-c echo", "-w hello", "-o", "prof.perf", "kperf"]) + result = runner.invoke(collect_cli.collect, ["-c echo", "-w hello", "-o", "prof.perf", "kperf"]) asserts.predicate_from_cli(result, result.exit_code == 0) assert "prof.perf" in os.listdir(".") - result = runner.invoke(cli.collect, ["-c echo", "-w world", "-o", "prof2.perf", "kperf"]) + result = runner.invoke( + collect_cli.collect, ["-c echo", "-w world", "-o", "prof2.perf", "kperf"] + ) asserts.predicate_from_cli(result, result.exit_code == 0) assert "prof2.perf" in os.listdir(".") diff --git a/tests/test_collect.py b/tests/test_collect.py index c1c9314f0..060851670 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -10,7 +10,7 @@ from click.testing import CliRunner # Perun Imports -from perun import cli +from perun.cli_groups import collect_cli from perun.collect.complexity import makefiles, symbols, run as complexity, configurator from perun.logic import pcs, runner as run from perun.profile.factory import Profile @@ -18,7 +18,7 @@ from perun.utils import log from perun.utils.common import common_kit from perun.utils.external import commands -from perun.utils.structs import Unit, Executable, CollectStatus, RunnerReport, Job +from perun.utils.structs.common_structs import Unit, Executable, CollectStatus, RunnerReport, Job from perun.workload.integer_generator import IntegerGenerator @@ -99,7 +99,7 @@ def test_collect_complexity(monkeypatch, pcs_with_root, complexity_collect_job): ) runner = CliRunner() result = runner.invoke( - cli.collect, + collect_cli.collect, [ f"-c{job_params['target_dir']}", "-w input", @@ -123,7 +123,7 @@ def test_collect_complexity(monkeypatch, pcs_with_root, complexity_collect_job): ] rules.extend([f"-r{rule}" for rule in more_rules]) result = runner.invoke( - cli.collect, + collect_cli.collect, [f"-c{job_params['target_dir']}", "complexity", f"-t{job_params['target_dir']}"] + files + rules @@ -161,17 +161,17 @@ def mocked_safe_external(*_, **__): runner = CliRunner() # Try missing parameters --target-dir and --files - result = runner.invoke(cli.collect, ["complexity"]) + result = runner.invoke(collect_cli.collect, ["complexity"]) asserts.predicate_from_cli(result, result.exit_code == 1) asserts.predicate_from_cli(result, "--target-dir parameter must be supplied" in result.output) - result = runner.invoke(cli.collect, ["complexity", f"-t{job_params['target_dir']}"]) + result = runner.invoke(collect_cli.collect, ["complexity", f"-t{job_params['target_dir']}"]) asserts.predicate_from_cli(result, result.exit_code == 1) asserts.predicate_from_cli(result, "--files parameter must be supplied" in result.output) # Try supplying invalid directory path, which is a file instead invalid_target = os.path.join(os.path.dirname(script_dir), "job.yml") - result = runner.invoke(cli.collect, ["complexity", f"-t{invalid_target}"]) + result = runner.invoke(collect_cli.collect, ["complexity", f"-t{invalid_target}"]) asserts.predicate_from_cli(result, result.exit_code == 1) asserts.predicate_from_cli(result, "already exists" in result.output) @@ -189,12 +189,12 @@ def mocked_make(cmd, *_, **__): + rules + samplings ) - result = runner.invoke(cli.collect, command) + result = runner.invoke(collect_cli.collect, command) asserts.predicate_from_cli( result, "Command 'cmake' returned non-zero exit status 1" in result.output ) monkeypatch.setattr(commands, "run_external_command", mocked_make) - result = runner.invoke(cli.collect, command) + result = runner.invoke(collect_cli.collect, command) asserts.predicate_from_cli( result, "Command 'make' returned non-zero exit status 1" in result.output ) @@ -203,12 +203,12 @@ def mocked_make(cmd, *_, **__): # Simulate that some required library is missing old_libs_existence = makefiles._libraries_exist monkeypatch.setattr(makefiles, "_libraries_exist", _mocked_libs_existence_fails) - result = runner.invoke(cli.collect, command) + result = runner.invoke(collect_cli.collect, command) asserts.predicate_from_cli(result, "libraries are missing" in result.output) # Simulate that the libraries directory path cannot be found monkeypatch.setattr(makefiles, "_libraries_exist", _mocked_libs_existence_exception) - result = runner.invoke(cli.collect, command) + result = runner.invoke(collect_cli.collect, command) asserts.predicate_from_cli(result, "Unable to locate" in result.output) monkeypatch.setattr(makefiles, "_libraries_exist", old_libs_existence) @@ -219,7 +219,7 @@ def mocked_make(cmd, *_, **__): mock_handle.write("a b c d\na b c d") old_record_processing = complexity._process_file_record monkeypatch.setattr(complexity, "_process_file_record", _mocked_record_processing) - result = runner.invoke(cli.collect, command) + result = runner.invoke(collect_cli.collect, command) asserts.predicate_from_cli(result, "Call stack error" in result.output) monkeypatch.setattr(complexity, "_process_file_record", old_record_processing) @@ -230,13 +230,13 @@ def mock_find_all_braces(s, b, e): return [1] if b == "(" and e == ")" else old_find_braces(s, b, e) monkeypatch.setattr(symbols, "_find_all_braces", mock_find_all_braces) - result = runner.invoke(cli.collect, command) + result = runner.invoke(collect_cli.collect, command) asserts.predicate_from_cli(result, "wrong prototype of function" in result.output) monkeypatch.setattr(symbols, "_find_all_braces", old_find_braces) # Simulate missing dependencies monkeypatch.setattr("shutil.which", lambda *_: False) - result = runner.invoke(cli.collect, command) + result = runner.invoke(collect_cli.collect, command) asserts.predicate_from_cli(result, "Could not find 'make'" in result.output) asserts.predicate_from_cli(result, "Could not find 'cmake'" in result.output) @@ -329,7 +329,7 @@ def test_collect_memory(capsys, pcs_with_root, memory_collect_job, memory_collec # Try running memory from CLI runner = CliRunner() - result = runner.invoke(cli.collect, [f"-c{job.executable.cmd}", "memory"]) + result = runner.invoke(collect_cli.collect, [f"-c{job.executable.cmd}", "memory"]) assert result.exit_code == 0 @@ -561,7 +561,9 @@ def test_collect_kperf(monkeypatch, pcs_with_root, capsys): before_object_count = test_utils.count_contents_on_path(pcs_with_root.get_path())[0] runner = CliRunner() - result = runner.invoke(cli.collect, ["-c", "ls", "-w", ".", "kperf", "-w", "1", "-r", "1"]) + result = runner.invoke( + collect_cli.collect, ["-c", "ls", "-w", ".", "kperf", "-w", "1", "-r", "1"] + ) assert result.exit_code == 0 after_object_count = test_utils.count_contents_on_path(pcs_with_root.get_path())[0] assert before_object_count + 2 == after_object_count @@ -573,7 +575,7 @@ def mocked_safe_external(*_, **__): old_run = commands.run_external_command monkeypatch.setattr(commands, "run_safely_external_command", mocked_safe_external) result = runner.invoke( - cli.collect, ["-c", "ls", "-w", ".", "kperf", "-w", "1", "-r", "1", "--with-sudo"] + collect_cli.collect, ["-c", "ls", "-w", ".", "kperf", "-w", "1", "-r", "1", "--with-sudo"] ) assert result.exit_code == 0 @@ -587,7 +589,7 @@ def mocked_fail_external(cmd, *args, **kwargs): monkeypatch.setattr(commands, "run_safely_external_command", mocked_fail_external) monkeypatch.setattr("perun.utils.external.commands.is_executable", lambda command: True) result = runner.invoke( - cli.collect, ["-c", "ls", "-w", ".", "kperf", "-w", "1", "-r", "1", "--with-sudo"] + collect_cli.collect, ["-c", "ls", "-w", ".", "kperf", "-w", "1", "-r", "1", "--with-sudo"] ) assert result.exit_code == 0 monkeypatch.setattr(commands, "run_safely_external_command", old_run) @@ -600,7 +602,9 @@ def mocked_is_executable_sc(command): return True monkeypatch.setattr("perun.utils.external.commands.is_executable", mocked_is_executable_sc) - result = runner.invoke(cli.collect, ["-c", "ls", "-w", ".", "kperf", "-w", "0", "-r", "1"]) + result = runner.invoke( + collect_cli.collect, ["-c", "ls", "-w", ".", "kperf", "-w", "0", "-r", "1"] + ) assert result.exit_code != 0 assert "not-executable" in result.output @@ -611,6 +615,8 @@ def mocked_is_executable_perf(command): return True monkeypatch.setattr("perun.utils.external.commands.is_executable", mocked_is_executable_perf) - result = runner.invoke(cli.collect, ["-c", "ls", "-w", ".", "kperf", "-w", "0", "-r", "1"]) + result = runner.invoke( + collect_cli.collect, ["-c", "ls", "-w", ".", "kperf", "-w", "0", "-r", "1"] + ) assert result.exit_code != 0 assert "not-executable" in result.output diff --git a/tests/test_degradations.py b/tests/test_degradations.py index aaad0e868..9bd9a4541 100644 --- a/tests/test_degradations.py +++ b/tests/test_degradations.py @@ -10,11 +10,12 @@ import pytest # Perun Imports +from perun import check as check from perun.check.methods.abstract_base_checker import AbstractBaseChecker from perun.logic import config, store from perun.utils import log +from perun.utils.structs.common_structs import PerformanceChange from perun.utils.exceptions import UnsupportedModuleException -import perun.check.factory as check def test_degradation_precollect(monkeypatch, pcs_with_degradations, capsys): @@ -85,7 +86,7 @@ def test_degradation_in_history(pcs_with_degradations): head = str(git_repo.head.commit) result = check.degradation_in_history(head) - assert check.PerformanceChange.Degradation in [r[0].result for r in result] + assert PerformanceChange.Degradation in [r[0].result for r in result] def test_degradation_between_profiles(pcs_with_root, capsys): @@ -114,8 +115,8 @@ def test_degradation_between_profiles(pcs_with_root, capsys): ) ) expected_changes = { - check.PerformanceChange.TotalDegradation, - check.PerformanceChange.NoChange, + PerformanceChange.TotalDegradation, + PerformanceChange.NoChange, } assert expected_changes & set(r.result for r in result) @@ -127,9 +128,9 @@ def test_degradation_between_profiles(pcs_with_root, capsys): ) # We allow TotalDegradation and TotalOptimization since one them is always reported allowed = { - check.PerformanceChange.NoChange, - check.PerformanceChange.TotalDegradation, - check.PerformanceChange.TotalOptimization, + PerformanceChange.NoChange, + PerformanceChange.TotalDegradation, + PerformanceChange.TotalOptimization, } # No other result should be present here assert not set(r.result for r in result) - allowed @@ -139,33 +140,33 @@ def test_degradation_between_profiles(pcs_with_root, capsys): result = list( check.run_degradation_check("best_model_order_equality", profiles[0], profiles[1]) ) - assert check.PerformanceChange.NoChange in [r.result for r in result] + assert PerformanceChange.NoChange in [r.result for r in result] # Can detect degradation using BMOE strategy betwen these pairs of profiles result = list( check.run_degradation_check("best_model_order_equality", profiles[1], profiles[2]) ) - assert check.PerformanceChange.Degradation in [r.result for r in result] + assert PerformanceChange.Degradation in [r.result for r in result] result = list( check.run_degradation_check("best_model_order_equality", profiles[0], profiles[2]) ) - assert check.PerformanceChange.Degradation in [r.result for r in result] + assert PerformanceChange.Degradation in [r.result for r in result] result = list(check.run_degradation_check("average_amount_threshold", profiles[1], profiles[2])) - assert check.PerformanceChange.Degradation in [r.result for r in result] + assert PerformanceChange.Degradation in [r.result for r in result] # Can detect optimizations both using BMOE and AAT and Fast result = list(check.run_degradation_check("average_amount_threshold", profiles[2], profiles[1])) - assert check.PerformanceChange.Optimization in [r.result for r in result] + assert PerformanceChange.Optimization in [r.result for r in result] result = list(check.run_degradation_check("fast_check", profiles[2], profiles[1])) - assert check.PerformanceChange.MaybeOptimization in [r.result for r in result] + assert PerformanceChange.MaybeOptimization in [r.result for r in result] result = list( check.run_degradation_check("best_model_order_equality", profiles[2], profiles[1]) ) - assert check.PerformanceChange.Optimization in [r.result for r in result] + assert PerformanceChange.Optimization in [r.result for r in result] # Try that we printed confidence deg_list = [(res, "", "") for res in result] @@ -178,7 +179,7 @@ def test_degradation_between_profiles(pcs_with_root, capsys): # Assert that DegradationInfo was yield assert result # Assert there was no change - assert check.PerformanceChange.NoChange in [r.result for r in result] + assert PerformanceChange.NoChange in [r.result for r in result] # Test incompatible profiles pool_path = os.path.join(os.path.split(__file__)[0], "profiles", "full_profiles") diff --git a/tests/test_detection_methods.py b/tests/test_detection_methods.py index 494cff16a..ea0dc658a 100644 --- a/tests/test_detection_methods.py +++ b/tests/test_detection_methods.py @@ -13,11 +13,11 @@ # Third-Party Imports # Perun Imports +from perun import check from perun.logic import store from perun.testing.mock_results import PARAM_EXPECTED_RESULTS, NONPARAM_EXPECTED_RESULTS from perun.utils.log import aggregate_intervals -from perun.utils.structs import PerformanceChange -import perun.check.factory as check_factory +from perun.utils.structs.common_structs import PerformanceChange def load_profiles(param): @@ -74,7 +74,7 @@ def load_profiles(param): def check_degradation_result(baseline_profile, target_profile, expected_result, function): - result = list(check_factory.run_degradation_check(function, baseline_profile, target_profile)) + result = list(check.run_degradation_check(function, baseline_profile, target_profile)) assert expected_result["result"] & {r.result for r in result} assert expected_result["type"] & {r.type for r in result} assert expected_result["rate"] & {round(r.rate_degradation) for r in result} @@ -108,7 +108,7 @@ def test_complex_detection_methods(): for expected_results in NONPARAM_EXPECTED_RESULTS: degradation_list = list( - check_factory.run_degradation_check( + check.run_degradation_check( expected_results["function"], profiles[0], profiles[1], models_strategy="all-models" ) ) diff --git a/tests/test_generators.py b/tests/test_generators.py index 5305f30ed..8757c3077 100644 --- a/tests/test_generators.py +++ b/tests/test_generators.py @@ -14,7 +14,7 @@ from perun.logic import config, runner from perun.utils import decorators from perun.utils.common import common_kit -from perun.utils.structs import Unit, Executable, CollectStatus, Job +from perun.utils.structs.common_structs import Unit, Executable, CollectStatus, Job from perun.workload.external_generator import ExternalGenerator from perun.workload.generator import WorkloadGenerator from perun.workload.integer_generator import IntegerGenerator diff --git a/tests/test_log.py b/tests/test_log.py index 074b3b662..0f5c9573f 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -23,7 +23,7 @@ NotPerunRepositoryException, UnsupportedModuleException, ) -from perun.utils.structs import MinorVersion, ProfileListConfig +from perun.utils.structs.common_structs import MinorVersion, ProfileListConfig @pytest.mark.usefixtures("cleandir") diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 04aaac934..abcce432c 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -13,11 +13,12 @@ # Perun Imports from perun import cli +from perun.cli_groups import collect_cli from perun.collect.trace.values import TraceRecord, RecordType, FileSize from perun.logic import config, locks, temp, pcs from perun.utils import decorators from perun.utils.exceptions import SystemTapStartupException -from perun.utils.structs import CollectStatus +from perun.utils.structs.common_structs import CollectStatus import perun.collect.trace.run as trace_run import perun.collect.trace.systemtap.engine as stap import perun.testing.utils as test_utils @@ -265,7 +266,7 @@ def test_collect_trace_cli_no_stap(monkeypatch, pcs_full): # Test that non-existing command is not accepted monkeypatch.setattr(shutil, "which", lambda *_: None) result = runner.invoke( - cli.collect, + collect_cli.collect, [ f"-c{os.path.join('invalid', 'executable', 'path')}", "trace", @@ -278,7 +279,7 @@ def test_collect_trace_cli_no_stap(monkeypatch, pcs_full): # Test that non-elf files are not accepted not_elf = os.path.join(target_dir, "job.yml") - result = runner.invoke(cli.collect, [f"-c{not_elf}", "trace", "-f", "main"]) + result = runner.invoke(collect_cli.collect, [f"-c{not_elf}", "trace", "-f", "main"]) assert result.exit_code == 1 assert "Supplied binary" in result.output @@ -309,7 +310,7 @@ def test_collect_trace_utils(pcs_with_root): temp.touch_temp_dir(locks_dir) temp.touch_temp_file(os.path.join(locks_dir, lock_file), protect=True) - result = runner.invoke(cli.collect, [f"-c{target}", "trace", "-f", "main", "-w"]) + result = runner.invoke(collect_cli.collect, [f"-c{target}", "trace", "-f", "main", "-w"]) assert result.exit_code == 0 @@ -331,7 +332,7 @@ def test_collect_trace(pcs_with_root, trace_collect_job): # Test the suppress output handling and that missing stap actually terminates the collection result = runner.invoke( - cli.collect, + collect_cli.collect, [f"-c{target}", "trace", "-o", "suppress"] + func + usdt + binary, ) @@ -350,20 +351,20 @@ def test_collect_trace(pcs_with_root, trace_collect_job): # script_dir = os.path.split(__file__)[0] # source_dir = os.path.join(script_dir, 'sources', 'collect_trace') # job_config_file = os.path.join(source_dir, 'job.yml') - # result = runner.invoke(cli.collect, ['-c{}'.format(target), '-p{}'.format(job_config_file), + # result = runner.invoke(collect_collect_cli.collect, ['-c{}'.format(target), '-p{}'.format(job_config_file), # 'trace']) # assert result.exit_code == 0 # Test running the job from the params using the yaml string result = runner.invoke( - cli.collect, + collect_cli.collect, [f"-c{target}", '-p"global_sampling: 2"', "trace"] + func + usdt + binary, ) assert result.exit_code == 0 # Try different template result = runner.invoke( - cli.collect, + collect_cli.collect, [ "-ot", "%collector%-profile", @@ -383,7 +384,7 @@ def test_collect_trace(pcs_with_root, trace_collect_job): # Test duplicity detection and pairing result = runner.invoke( - cli.collect, + collect_cli.collect, [ f"-c{target}", "trace", @@ -412,7 +413,7 @@ def test_collect_trace(pcs_with_root, trace_collect_job): # Test negative global sampling, it should be converted to no sampling # Also test that watchdog and quiet works result = runner.invoke( - cli.collect, + collect_cli.collect, [f"-c{target}", "trace", "-g", "-2", "-w", "-q", "-k"] + binary + func, ) log_path = os.path.join(pcs.get_log_directory(), "trace") @@ -424,12 +425,14 @@ def test_collect_trace(pcs_with_root, trace_collect_job): # Try missing parameter -c but with 'binary' present # This should use the binary parameter as executable - result = runner.invoke(cli.collect, ["trace", "-i"] + binary + func) + result = runner.invoke(collect_cli.collect, ["trace", "-i"] + binary + func) archives = glob.glob(os.path.join(log_path, "collect_files_*.zip.lzma")) assert len(archives) == 1 assert result.exit_code == 0 # Try it the other way around - result = runner.invoke(cli.collect, [f"-c{target}", "trace", "-o", "capture", "-k"] + func) + result = runner.invoke( + collect_cli.collect, [f"-c{target}", "trace", "-o", "capture", "-k"] + func + ) files_path = os.path.join(pcs.get_tmp_directory(), "trace", "files") capture = glob.glob(os.path.join(files_path, "collect_capture_*.txt")) # Two previous tests and this one kept the capture files @@ -438,7 +441,9 @@ def test_collect_trace(pcs_with_root, trace_collect_job): # Try timeout parameter which actually interrupts a running program wait_target = os.path.join(target_dir, "tst_waiting") - result = runner.invoke(cli.collect, ["-c", wait_target, "trace", "-w", "-f", "main", "-t", 1]) + result = runner.invoke( + collect_cli.collect, ["-c", wait_target, "trace", "-w", "-f", "main", "-t", 1] + ) assert result.exit_code == 0 @@ -458,25 +463,27 @@ def test_collect_trace_strategies(monkeypatch, pcs_full): target = os.path.join(target_dir, "tst") # Test simple userspace strategy without external modification or sampling - result = runner.invoke(cli.collect, [f"-c{target}", "trace", "-s", "userspace", "-k"]) + result = runner.invoke(collect_cli.collect, [f"-c{target}", "trace", "-s", "userspace", "-k"]) assert result.exit_code == 0 assert _compare_collect_scripts( _get_latest_collect_script(), os.path.join(target_dir, "strategy1_script.txt") ) # Test simple u_sampled strategy without external modification - result = runner.invoke(cli.collect, [f"-c{target}", "trace", "-s", "u_sampled", "-k"]) + result = runner.invoke(collect_cli.collect, [f"-c{target}", "trace", "-s", "u_sampled", "-k"]) assert result.exit_code == 0 assert _compare_collect_scripts( _get_latest_collect_script(), os.path.join(target_dir, "strategy2_script.txt") ) # Test simple all strategy without external modification or sampling - result = runner.invoke(cli.collect, [f"-c{target}", "trace", "-s", "all", "-k"]) + result = runner.invoke(collect_cli.collect, [f"-c{target}", "trace", "-s", "all", "-k"]) assert result.exit_code == 0 assert _compare_collect_scripts( _get_latest_collect_script(), os.path.join(target_dir, "strategy3_script.txt") ) # Test simple a_sampled strategy with verbose trace and without external modification - result = runner.invoke(cli.collect, [f"-c{target}", "trace", "-s", "a_sampled", "-vt", "-k"]) + result = runner.invoke( + collect_cli.collect, [f"-c{target}", "trace", "-s", "a_sampled", "-vt", "-k"] + ) assert result.exit_code == 0 assert _compare_collect_scripts( _get_latest_collect_script(), os.path.join(target_dir, "strategy4_script.txt") @@ -486,7 +493,7 @@ def test_collect_trace_strategies(monkeypatch, pcs_full): monkeypatch.setattr(stap, "_extract_usdt_probes", _mocked_stap_extraction_empty) # Test userspace strategy without static probes and added global_sampling result = runner.invoke( - cli.collect, + collect_cli.collect, [ f"-c{target}", "trace", @@ -503,7 +510,7 @@ def test_collect_trace_strategies(monkeypatch, pcs_full): # Test u_sampled strategy without static probes and overriden global_sampling # The output should be exactly the same as the previous result = runner.invoke( - cli.collect, + collect_cli.collect, [ f"-c{target}", "trace", @@ -519,7 +526,7 @@ def test_collect_trace_strategies(monkeypatch, pcs_full): ) # Test userspace strategy with overridden function, respecified function and invalid function result = runner.invoke( - cli.collect, + collect_cli.collect, [ f"-c{target}", "trace", @@ -536,7 +543,7 @@ def test_collect_trace_strategies(monkeypatch, pcs_full): ) # Test userspace strategy with invalid static probe (won't be detected as --no-static is used) result = runner.invoke( - cli.collect, + collect_cli.collect, [ f"-c{target}", "trace", @@ -552,7 +559,7 @@ def test_collect_trace_strategies(monkeypatch, pcs_full): ) # Test u_sampled strategy with more static probes to check correct pairing monkeypatch.setattr(stap, "_extract_usdt_probes", _mocked_stap_extraction2) - result = runner.invoke(cli.collect, [f"-c{target}", "trace", "-s", "u_sampled", "-k"]) + result = runner.invoke(collect_cli.collect, [f"-c{target}", "trace", "-s", "u_sampled", "-k"]) assert result.exit_code == 0 assert _compare_collect_scripts( _get_latest_collect_script(), os.path.join(target_dir, "strategy8_script.txt") @@ -575,28 +582,28 @@ def test_collect_trace_fail(monkeypatch, pcs_full, trace_collect_job): target = os.path.join(target_dir, "tst") # Try missing 'command' and 'binary' - result = runner.invoke(cli.collect, ["trace"]) + result = runner.invoke(collect_cli.collect, ["trace"]) assert result.exit_code == 1 assert "does not exist or is not an executable ELF file" in result.output # Try missing probe points - result = runner.invoke(cli.collect, ["trace", f"-b{target}"]) + result = runner.invoke(collect_cli.collect, ["trace", f"-b{target}"]) assert result.exit_code == 1 assert "No profiling probes created" in result.output # Try invalid parameter --strategy - result = runner.invoke(cli.collect, [f"-c{target}", "trace", "-sinvalid", "-b", target]) + result = runner.invoke(collect_cli.collect, [f"-c{target}", "trace", "-sinvalid", "-b", target]) assert result.exit_code == 2 # Try binary parameter that is actually not executable ELF invalid_target = os.path.join(target_dir, "cpp_sources", "tst.cpp") - result = runner.invoke(cli.collect, [f"-c{invalid_target}", "trace"]) + result = runner.invoke(collect_cli.collect, [f"-c{invalid_target}", "trace"]) assert result.exit_code == 1 assert "is not an executable ELF file." in result.output monkeypatch.setattr(stap.SystemTapEngine, "collect", _mocked_stap2) # Test malformed file that ends in unexpected way _mocked_stap_file = "record_malformed.txt" - result = runner.invoke(cli.collect, [f"-c{target}", "-w 1", "trace", "-s", "userspace"]) + result = runner.invoke(collect_cli.collect, [f"-c{target}", "-w 1", "trace", "-s", "userspace"]) # However, the collector should still be able to correctly process it assert result.exit_code == 0 after_object_count = test_utils.count_contents_on_path(pcs_full.get_path())[0] @@ -606,7 +613,7 @@ def test_collect_trace_fail(monkeypatch, pcs_full, trace_collect_job): # Test malformed file that ends in another unexpected way _mocked_stap_file = "record_malformed2.txt" - result = runner.invoke(cli.collect, [f"-c{target}", "-w 2", "trace", "-s", "userspace"]) + result = runner.invoke(collect_cli.collect, [f"-c{target}", "-w 2", "trace", "-s", "userspace"]) # Check if the collector managed to process the file assert result.exit_code == 0 after_object_count = test_utils.count_contents_on_path(pcs_full.get_path())[0] @@ -615,7 +622,7 @@ def test_collect_trace_fail(monkeypatch, pcs_full, trace_collect_job): # Test malformed file that has corrupted record _mocked_stap_file = "record_malformed3.txt" - result = runner.invoke(cli.collect, [f"-c{target}", "-w 3", "trace", "-s", "userspace"]) + result = runner.invoke(collect_cli.collect, [f"-c{target}", "-w 3", "trace", "-s", "userspace"]) # Check if the collector managed to process the file assert result.exit_code == 0 after_object_count = test_utils.count_contents_on_path(pcs_full.get_path())[0] @@ -625,7 +632,7 @@ def test_collect_trace_fail(monkeypatch, pcs_full, trace_collect_job): # Test malformed file that has misplaced data chunk _mocked_stap_file = "record_malformed4.txt" - result = runner.invoke(cli.collect, [f"-c{target}", "-w 4", "trace", "-s", "userspace"]) + result = runner.invoke(collect_cli.collect, [f"-c{target}", "-w 4", "trace", "-s", "userspace"]) # Check if the collector managed to process the file assert result.exit_code == 0 after_object_count = test_utils.count_contents_on_path(pcs_full.get_path())[0] @@ -634,7 +641,7 @@ def test_collect_trace_fail(monkeypatch, pcs_full, trace_collect_job): # Simulate the failure of the systemTap _mocked_stap_code = 1 - runner.invoke(cli.collect, [f"-c{target}", "trace", "-s", "userspace"]) + runner.invoke(collect_cli.collect, [f"-c{target}", "trace", "-s", "userspace"]) # Assert that nothing was added after_object_count = test_utils.count_contents_on_path(pcs_full.get_path())[0] assert before_object_count == after_object_count @@ -645,7 +652,7 @@ def test_collect_trace_fail(monkeypatch, pcs_full, trace_collect_job): # monkeypatch.setattr(parse, '_init_stack_and_map', _mocked_trace_stack) # monkeypatch.setattr(parse, '_parse_record', _mocked_parse_record) # result = runner.invoke( - # cli.collect, ['-c{}'.format(target), '-w 4', 'trace', '-s', 'userspace', '-w'] + # collect_cli.collect, ['-c{}'.format(target), '-w 4', 'trace', '-s', 'userspace', '-w'] # ) # assert result.exit_code == 1 # assert 'Error while parsing the raw trace record' in result.output diff --git a/tests/test_utils.py b/tests/test_utils.py index 24df49250..6f124282a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -29,7 +29,7 @@ SystemTapStartupException, ResourceLockedException, ) -from perun.utils.structs import Unit, OrderedEnum, HandledSignals +from perun.utils.structs.common_structs import Unit, OrderedEnum, HandledSignals from perun.utils.external import environment, commands as external_commands, processes, executable from perun.view_diff.datatables.run import TraceInfo @@ -112,6 +112,7 @@ def test_get_supported_modules(): """ # Check that all of the CLI units (collectors, postprocessors and visualizations) are properly # registered. + collect.lazy_get_cli_commands() assert_all_registered_cli_units("collect", collect, ["collect"]) assert_all_registered_cli_units("postprocess", postprocess, ["postprocess"]) assert_all_registered_cli_units("view", view, [])