From 17efdb56fab4a94a74bb9ba9f85d2234fda52800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20Rolih?= <61357777+blaz-r@users.noreply.github.com> Date: Fri, 30 Jun 2023 16:05:43 +0200 Subject: [PATCH] Fix benchmark (#1155) * Fixed broken links in readme * Fixed inference command in readme * Fix benchmark torch throughput * Add tests to cover throughput * Add fixture path to conftest * Format changed files * Add implementation to conftest --- tests/pre_merge/utils/sweep/conftest.py | 68 +++++++++++++++++++ .../pre_merge/utils/sweep/test_throughput.py | 62 +++++++++++++++++ tools/benchmarking/benchmark.py | 16 ++++- 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 tests/pre_merge/utils/sweep/conftest.py create mode 100644 tests/pre_merge/utils/sweep/test_throughput.py diff --git a/tests/pre_merge/utils/sweep/conftest.py b/tests/pre_merge/utils/sweep/conftest.py new file mode 100644 index 0000000000..f8e3b78044 --- /dev/null +++ b/tests/pre_merge/utils/sweep/conftest.py @@ -0,0 +1,68 @@ +"""Fixtures for the sweep tests.""" + +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from tempfile import TemporaryDirectory +from typing import Optional, Union + +import pytest +from omegaconf import DictConfig, ListConfig +from pytorch_lightning import Trainer + +from anomalib.config import get_configurable_parameters +from anomalib.data import get_datamodule +from anomalib.models import get_model +from anomalib.utils.callbacks import get_callbacks + + +def get_model_config( + project_path: str, + model_name: str, + dataset_path: str, + category: str, + task: str = "classification", + export_mode: Optional[str] = None, +): + model_config = get_configurable_parameters(model_name=model_name) + model_config.project.path = project_path + model_config.dataset.task = task + model_config.dataset.path = dataset_path + model_config.dataset.category = category + model_config.trainer.fast_dev_run = True + model_config.trainer.max_epochs = 1 + model_config.trainer.devices = 1 + model_config.trainer.accelerator = "gpu" + model_config.optimization.export_mode = export_mode + return model_config + + +@pytest.fixture(scope="package") +def generate_results_dir(): + with TemporaryDirectory() as project_path: + + def make( + model_name: str, + dataset_path: str, + category: str, + task: str = "classification", + export_mode: Optional[str] = None, + ) -> Union[DictConfig, ListConfig]: + # then train the model + model_config = get_model_config( + project_path=project_path, + model_name=model_name, + dataset_path=dataset_path, + category=category, + task=task, + export_mode=export_mode, + ) + model = get_model(model_config) + datamodule = get_datamodule(model_config) + callbacks = get_callbacks(model_config) + trainer = Trainer(**model_config.trainer, logger=False, callbacks=callbacks) + trainer.fit(model=model, datamodule=datamodule) + + return model_config + + yield make diff --git a/tests/pre_merge/utils/sweep/test_throughput.py b/tests/pre_merge/utils/sweep/test_throughput.py new file mode 100644 index 0000000000..89e5140886 --- /dev/null +++ b/tests/pre_merge/utils/sweep/test_throughput.py @@ -0,0 +1,62 @@ +"""Tests for Torch and OpenVINO inferencer throughput used in sweep.""" + +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import albumentations as A +from albumentations.pytorch import ToTensorV2 + +from anomalib.data import TaskType +from anomalib.data.folder import FolderDataset +from anomalib.deploy import ExportMode + +from anomalib.utils.sweep.helpers import get_torch_throughput, get_openvino_throughput + +from tests.helpers.dataset import TestDataset + + +transforms = A.Compose([A.ToFloat(max_value=255), ToTensorV2()]) + + +@TestDataset(num_train=20, num_test=10) +def test_torch_throughput(generate_results_dir, path: str = None, category: str = "shapes"): + """Test get_torch_throughput from utils/sweep/inference.py""" + # generate results with torch model exported + model_config = generate_results_dir( + model_name="padim", + dataset_path=path, + task=TaskType.CLASSIFICATION, + category=category, + export_mode=ExportMode.TORCH, + ) + + # create Dataset from generated TestDataset + dataset = FolderDataset( + task=TaskType.CLASSIFICATION, transform=transforms, root=path, normal_dir=f"{category}/test/good" + ) + dataset.setup() + + # run procedure using torch inferencer + get_torch_throughput(model_config.project.path, dataset, device=model_config.trainer.accelerator) + + +@TestDataset(num_train=20, num_test=10) +def test_openvino_throughput(generate_results_dir, path: str = None, category: str = "shapes"): + """Test get_openvino_throughput from utils/sweep/inference.py""" + # generate results with torch model exported + model_config = generate_results_dir( + model_name="padim", + dataset_path=path, + task=TaskType.CLASSIFICATION, + category=category, + export_mode=ExportMode.OPENVINO, + ) + + # create Dataset from generated TestDataset + dataset = FolderDataset( + task=TaskType.CLASSIFICATION, transform=transforms, root=path, normal_dir=f"{category}/test/good" + ) + dataset.setup() + + # run procedure using openvino inferencer + get_openvino_throughput(model_config.project.path, dataset) diff --git a/tools/benchmarking/benchmark.py b/tools/benchmarking/benchmark.py index dfdec0954c..d55f6c9eaf 100644 --- a/tools/benchmarking/benchmark.py +++ b/tools/benchmarking/benchmark.py @@ -111,7 +111,21 @@ def get_single_model_metrics(model_config: DictConfig | ListConfig, openvino_met # get testing time testing_time = time.time() - start_time - throughput = get_torch_throughput(model_config, model, datamodule.test_dataloader().dataset) + # Create dirs for torch export (as default only lighting model is produced) + export( + task=model_config.dataset.task, + transform=trainer.datamodule.test_data.transform.to_dict(), + input_size=model_config.model.input_size, + model=model, + export_mode=ExportMode.TORCH, + export_root=project_path, + ) + + throughput = get_torch_throughput( + model_path=project_path, + test_dataset=datamodule.test_dataloader().dataset, + device=model_config.trainer.accelerator, + ) # Get OpenVINO metrics openvino_throughput = float("nan")