Merge branch 'main' into aupimo/speedup

CHANGELOG.md

@@ -15,10 +15,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 
+- Add duration of experiments in seconds in the benchmark CSV result by [mzweilin](https://github.com/mzweilin) in https://github.com/openvinotoolkit/anomalib/pull/2392
+- Export flat configurations in benchmark CSV results by [mzweilin](https://github.com/mzweilin) in https://github.com/openvinotoolkit/anomalib/pull/2391
+
 ### Deprecated
 
 ### Fixed
 
+- Make single GPU benchmarking 5x more efficient by [mzweilin](https://github.com/mzweilin) in https://github.com/openvinotoolkit/anomalib/pull/2390
+
 ### New Contributors
 
 **Full Changelog**:

pyproject.toml

@@ -34,6 +34,7 @@ dependencies = [
     "jsonargparse[signatures]>=4.27.7",
     "docstring_parser",     # CLI help-formatter
     "rich_argparse",        # CLI help-formatter
+    "lightning-utilities",
 ]
 
 [project.optional-dependencies]
@@ -293,11 +294,15 @@ pythonpath = "src"
 # COVERAGE CONFIGURATION                                                      #
 [tool.coverage.report]
 exclude_lines = [
-    "except ImportError",
+    "pragma: no cover",
+    "def __repr__",
+    "raise NotImplementedError",
+    "if TYPE_CHECKING:",
+    "@abstractmethod",
+    "pass",
     "raise ImportError",
-    "except ApiException",
-    "raise ApiException",
     "raise ValueError",
+    "except ImportError:",
 ]
 
 [tool.coverage.paths]

src/anomalib/models/__init__.py

@@ -58,9 +58,9 @@ class UnknownModelError(ModuleNotFoundError):
     "Rkde",
     "Stfpm",
     "Uflow",
-    "AiVad",
     "VlmAd",
     "WinClip",
+    "AiVad",
 ]
 
 logger = logging.getLogger(__name__)

src/anomalib/models/image/vlm_ad/backends/huggingface.py

@@ -5,19 +5,21 @@
 
 import logging
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from lightning_utilities.core.imports import package_available
 from PIL import Image
-from transformers.modeling_utils import PreTrainedModel
 
 from anomalib.models.image.vlm_ad.utils import Prompt
 
 from .base import Backend
 
-if package_available("transformers"):
-    import transformers
+if TYPE_CHECKING:
     from transformers.modeling_utils import PreTrainedModel
     from transformers.processing_utils import ProcessorMixin
+
+if package_available("transformers"):
+    import transformers
 else:
     transformers = None
 
@@ -39,7 +41,7 @@ def __init__(
         self._model: PreTrainedModel | None = None
 
     @property
-    def processor(self) -> ProcessorMixin:
+    def processor(self) -> "ProcessorMixin":
         """Get the Huggingface processor."""
         if self._processor is None:
             if transformers is None:
@@ -49,7 +51,7 @@ def processor(self) -> ProcessorMixin:
         return self._processor
 
     @property
-    def model(self) -> PreTrainedModel:
+    def model(self) -> "PreTrainedModel":
         """Get the Huggingface model."""
         if self._model is None:
             if transformers is None:

src/anomalib/pipelines/benchmark/generator.py

@@ -10,6 +10,7 @@
 from anomalib.pipelines.components import JobGenerator
 from anomalib.pipelines.components.utils import get_iterator_from_grid_dict
 from anomalib.pipelines.types import PREV_STAGE_RESULT
+from anomalib.utils.config import flatten_dict
 from anomalib.utils.logging import hide_output
 
 from .job import BenchmarkJob
@@ -39,9 +40,12 @@ def generate_jobs(
         """Return iterator based on the arguments."""
         del previous_stage_result  # Not needed for this job
         for _container in get_iterator_from_grid_dict(args):
+            # Pass experimental configs as a flatten dictionary to the job runner.
+            flat_cfg = flatten_dict(_container)
             yield BenchmarkJob(
                 accelerator=self.accelerator,
                 seed=_container["seed"],
                 model=get_model(_container["model"]),
                 datamodule=get_datamodule(_container["data"]),
+                flat_cfg=flat_cfg,
             )

src/anomalib/pipelines/benchmark/job.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
+import time
 from datetime import datetime
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -31,23 +32,33 @@ class BenchmarkJob(Job):
         model (AnomalyModule): The model to use.
         datamodule (AnomalibDataModule): The data module to use.
         seed (int): The seed to use.
+        flat_cfg (dict): The flat dictionary of configs with dotted keys.
     """
 
     name = "benchmark"
 
-    def __init__(self, accelerator: str, model: AnomalyModule, datamodule: AnomalibDataModule, seed: int) -> None:
+    def __init__(
+        self,
+        accelerator: str,
+        model: AnomalyModule,
+        datamodule: AnomalibDataModule,
+        seed: int,
+        flat_cfg: dict,
+    ) -> None:
         super().__init__()
         self.accelerator = accelerator
         self.model = model
         self.datamodule = datamodule
         self.seed = seed
+        self.flat_cfg = flat_cfg
 
     @hide_output
     def run(
         self,
         task_id: int | None = None,
     ) -> dict[str, Any]:
         """Run the benchmark."""
+        job_start_time = time.time()
         devices: str | list[int] = "auto"
         if task_id is not None:
             devices = [task_id]
@@ -59,16 +70,22 @@ def run(
                 devices=devices,
                 default_root_dir=temp_dir,
             )
+            fit_start_time = time.time()
             engine.fit(self.model, self.datamodule)
+            test_start_time = time.time()
             test_results = engine.test(self.model, self.datamodule)
+        job_end_time = time.time()
+        durations = {
+            "job_duration": job_end_time - job_start_time,
+            "fit_duration": test_start_time - fit_start_time,
+            "test_duration": job_end_time - test_start_time,
+        }
         # TODO(ashwinvaidya17): Restore throughput
         # https://github.com/openvinotoolkit/anomalib/issues/2054
         output = {
-            "seed": self.seed,
             "accelerator": self.accelerator,
-            "model": self.model.__class__.__name__,
-            "data": self.datamodule.__class__.__name__,
-            "category": self.datamodule.category,
+            **durations,
+            **self.flat_cfg,
             **test_results[0],
         }
         logger.info(f"Completed with result {output}")

src/anomalib/pipelines/benchmark/pipeline.py

@@ -20,11 +20,12 @@ def _setup_runners(args: dict) -> list[Runner]:
         accelerators = args["accelerator"] if isinstance(args["accelerator"], list) else [args["accelerator"]]
         runners: list[Runner] = []
         for accelerator in accelerators:
-            if accelerator == "cpu":
-                runners.append(SerialRunner(BenchmarkJobGenerator("cpu")))
-            elif accelerator == "cuda":
-                runners.append(ParallelRunner(BenchmarkJobGenerator("cuda"), n_jobs=torch.cuda.device_count()))
-            else:
+            if accelerator not in {"cpu", "cuda"}:
                 msg = f"Unsupported accelerator: {accelerator}"
                 raise ValueError(msg)
+            device_count = torch.cuda.device_count()
+            if device_count <= 1:
+                runners.append(SerialRunner(BenchmarkJobGenerator(accelerator)))
+            else:
+                runners.append(ParallelRunner(BenchmarkJobGenerator(accelerator), n_jobs=device_count))
         return runners

src/anomalib/utils/logging.py

@@ -74,10 +74,8 @@ def redirect_logs(log_file: str) -> None:
     """
     Path(log_file).parent.mkdir(exist_ok=True, parents=True)
     logger_file_handler = logging.FileHandler(log_file)
-    root_logger = logging.getLogger()
-    root_logger.setLevel(logging.DEBUG)
     format_string = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-    logging.basicConfig(format=format_string, level=logging.DEBUG, handlers=[logger_file_handler])
+    logging.basicConfig(format=format_string, handlers=[logger_file_handler])
     logging.captureWarnings(capture=True)
     # remove other handlers from all loggers
     loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict]