์ฝ˜ํ…์ธ ๋กœ ๊ฑด๋„ˆ๋›ฐ๊ธฐ

์ฐธ์กฐ ultralytics/utils/benchmarks.py

์ฐธ๊ณ 

์ด ํŒŒ์ผ์€ https://github.com/ultralytics/ ultralytics/blob/main/ ultralytics/utils/benchmarks .py์—์„œ ํ™•์ธํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. ๋ฌธ์ œ๋ฅผ ๋ฐœ๊ฒฌํ•˜๋ฉด ํ’€ ๋ฆฌํ€˜์ŠคํŠธ (๐Ÿ› ๏ธ)๋ฅผ ์ œ์ถœํ•˜์—ฌ ๋ฌธ์ œ๋ฅผ ํ•ด๊ฒฐํ•˜๋„๋ก ๋„์™€์ฃผ์„ธ์š”. ๊ฐ์‚ฌํ•ฉ๋‹ˆ๋‹ค ๐Ÿ™!



ultralytics.utils.benchmarks.RF100Benchmark

์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
class RF100Benchmark:
    def __init__(self):
        """Function for initialization of RF100Benchmark."""
        self.ds_names = []
        self.ds_cfg_list = []
        self.rf = None
        self.val_metrics = ["class", "images", "targets", "precision", "recall", "map50", "map95"]

    def set_key(self, api_key):
        """
        Set Roboflow API key for processing.

        Args:
            api_key (str): The API key.
        """

        check_requirements("roboflow")
        from roboflow import Roboflow

        self.rf = Roboflow(api_key=api_key)

    def parse_dataset(self, ds_link_txt="datasets_links.txt"):
        """
        Parse dataset links and downloads datasets.

        Args:
            ds_link_txt (str): Path to dataset_links file.
        """

        (shutil.rmtree("rf-100"), os.mkdir("rf-100")) if os.path.exists("rf-100") else os.mkdir("rf-100")
        os.chdir("rf-100")
        os.mkdir("ultralytics-benchmarks")
        safe_download("https://ultralytics.com/assets/datasets_links.txt")

        with open(ds_link_txt, "r") as file:
            for line in file:
                try:
                    _, url, workspace, project, version = re.split("/+", line.strip())
                    self.ds_names.append(project)
                    proj_version = f"{project}-{version}"
                    if not Path(proj_version).exists():
                        self.rf.workspace(workspace).project(project).version(version).download("yolov8")
                    else:
                        print("Dataset already downloaded.")
                    self.ds_cfg_list.append(Path.cwd() / proj_version / "data.yaml")
                except Exception:
                    continue

        return self.ds_names, self.ds_cfg_list

    @staticmethod
    def fix_yaml(path):
        """
        Function to fix YAML train and val path.

        Args:
            path (str): YAML file path.
        """

        with open(path, "r") as file:
            yaml_data = yaml.safe_load(file)
        yaml_data["train"] = "train/images"
        yaml_data["val"] = "valid/images"
        with open(path, "w") as file:
            yaml.safe_dump(yaml_data, file)

    def evaluate(self, yaml_path, val_log_file, eval_log_file, list_ind):
        """
        Model evaluation on validation results.

        Args:
            yaml_path (str): YAML file path.
            val_log_file (str): val_log_file path.
            eval_log_file (str): eval_log_file path.
            list_ind (int): Index for current dataset.
        """
        skip_symbols = ["๐Ÿš€", "โš ๏ธ", "๐Ÿ’ก", "โŒ"]
        with open(yaml_path) as stream:
            class_names = yaml.safe_load(stream)["names"]
        with open(val_log_file, "r", encoding="utf-8") as f:
            lines = f.readlines()
            eval_lines = []
            for line in lines:
                if any(symbol in line for symbol in skip_symbols):
                    continue
                entries = line.split(" ")
                entries = list(filter(lambda val: val != "", entries))
                entries = [e.strip("\n") for e in entries]
                eval_lines.extend(
                    {
                        "class": entries[0],
                        "images": entries[1],
                        "targets": entries[2],
                        "precision": entries[3],
                        "recall": entries[4],
                        "map50": entries[5],
                        "map95": entries[6],
                    }
                    for e in entries
                    if e in class_names or (e == "all" and "(AP)" not in entries and "(AR)" not in entries)
                )
        map_val = 0.0
        if len(eval_lines) > 1:
            print("There's more dicts")
            for lst in eval_lines:
                if lst["class"] == "all":
                    map_val = lst["map50"]
        else:
            print("There's only one dict res")
            map_val = [res["map50"] for res in eval_lines][0]

        with open(eval_log_file, "a") as f:
            f.write(f"{self.ds_names[list_ind]}: {map_val}\n")

__init__()

RF100 ๋ฒค์น˜๋งˆํฌ ์ดˆ๊ธฐํ™”๋ฅผ ์œ„ํ•œ ๊ธฐ๋Šฅ์ž…๋‹ˆ๋‹ค.

์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
def __init__(self):
    """Function for initialization of RF100Benchmark."""
    self.ds_names = []
    self.ds_cfg_list = []
    self.rf = None
    self.val_metrics = ["class", "images", "targets", "precision", "recall", "map50", "map95"]

evaluate(yaml_path, val_log_file, eval_log_file, list_ind)

์œ ํšจ์„ฑ ๊ฒ€์‚ฌ ๊ฒฐ๊ณผ์— ๋Œ€ํ•œ ๋ชจ๋ธ ํ‰๊ฐ€.

๋งค๊ฐœ๋ณ€์ˆ˜:

์ด๋ฆ„ ์œ ํ˜• ์„ค๋ช… ๊ธฐ๋ณธ๊ฐ’
yaml_path str

YAML ํŒŒ์ผ ๊ฒฝ๋กœ์ž…๋‹ˆ๋‹ค.

ํ•„์ˆ˜
val_log_file str

VAL_LOG_FILE ๊ฒฝ๋กœ.

ํ•„์ˆ˜
eval_log_file str

eval_log_file ๊ฒฝ๋กœ.

ํ•„์ˆ˜
list_ind int

ํ˜„์žฌ ๋ฐ์ดํ„ฐ ์ง‘ํ•ฉ์˜ ์ธ๋ฑ์Šค์ž…๋‹ˆ๋‹ค.

ํ•„์ˆ˜
์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
def evaluate(self, yaml_path, val_log_file, eval_log_file, list_ind):
    """
    Model evaluation on validation results.

    Args:
        yaml_path (str): YAML file path.
        val_log_file (str): val_log_file path.
        eval_log_file (str): eval_log_file path.
        list_ind (int): Index for current dataset.
    """
    skip_symbols = ["๐Ÿš€", "โš ๏ธ", "๐Ÿ’ก", "โŒ"]
    with open(yaml_path) as stream:
        class_names = yaml.safe_load(stream)["names"]
    with open(val_log_file, "r", encoding="utf-8") as f:
        lines = f.readlines()
        eval_lines = []
        for line in lines:
            if any(symbol in line for symbol in skip_symbols):
                continue
            entries = line.split(" ")
            entries = list(filter(lambda val: val != "", entries))
            entries = [e.strip("\n") for e in entries]
            eval_lines.extend(
                {
                    "class": entries[0],
                    "images": entries[1],
                    "targets": entries[2],
                    "precision": entries[3],
                    "recall": entries[4],
                    "map50": entries[5],
                    "map95": entries[6],
                }
                for e in entries
                if e in class_names or (e == "all" and "(AP)" not in entries and "(AR)" not in entries)
            )
    map_val = 0.0
    if len(eval_lines) > 1:
        print("There's more dicts")
        for lst in eval_lines:
            if lst["class"] == "all":
                map_val = lst["map50"]
    else:
        print("There's only one dict res")
        map_val = [res["map50"] for res in eval_lines][0]

    with open(eval_log_file, "a") as f:
        f.write(f"{self.ds_names[list_ind]}: {map_val}\n")

fix_yaml(path) staticmethod

Function to fix YAML train and val path.

๋งค๊ฐœ๋ณ€์ˆ˜:

์ด๋ฆ„ ์œ ํ˜• ์„ค๋ช… ๊ธฐ๋ณธ๊ฐ’
path str

YAML ํŒŒ์ผ ๊ฒฝ๋กœ์ž…๋‹ˆ๋‹ค.

ํ•„์ˆ˜
์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
@staticmethod
def fix_yaml(path):
    """
    Function to fix YAML train and val path.

    Args:
        path (str): YAML file path.
    """

    with open(path, "r") as file:
        yaml_data = yaml.safe_load(file)
    yaml_data["train"] = "train/images"
    yaml_data["val"] = "valid/images"
    with open(path, "w") as file:
        yaml.safe_dump(yaml_data, file)

parse_dataset(ds_link_txt='datasets_links.txt')

๋ฐ์ดํ„ฐ ์„ธํŠธ ๋งํฌ๋ฅผ ๊ตฌ๋ฌธ ๋ถ„์„ํ•˜๊ณ  ๋ฐ์ดํ„ฐ ์„ธํŠธ๋ฅผ ๋‹ค์šด๋กœ๋“œํ•ฉ๋‹ˆ๋‹ค.

๋งค๊ฐœ๋ณ€์ˆ˜:

์ด๋ฆ„ ์œ ํ˜• ์„ค๋ช… ๊ธฐ๋ณธ๊ฐ’
ds_link_txt str

๋ฐ์ดํ„ฐ์„ธํŠธ_๋งํฌ ํŒŒ์ผ์˜ ๊ฒฝ๋กœ์ž…๋‹ˆ๋‹ค.

'datasets_links.txt'
์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
def parse_dataset(self, ds_link_txt="datasets_links.txt"):
    """
    Parse dataset links and downloads datasets.

    Args:
        ds_link_txt (str): Path to dataset_links file.
    """

    (shutil.rmtree("rf-100"), os.mkdir("rf-100")) if os.path.exists("rf-100") else os.mkdir("rf-100")
    os.chdir("rf-100")
    os.mkdir("ultralytics-benchmarks")
    safe_download("https://ultralytics.com/assets/datasets_links.txt")

    with open(ds_link_txt, "r") as file:
        for line in file:
            try:
                _, url, workspace, project, version = re.split("/+", line.strip())
                self.ds_names.append(project)
                proj_version = f"{project}-{version}"
                if not Path(proj_version).exists():
                    self.rf.workspace(workspace).project(project).version(version).download("yolov8")
                else:
                    print("Dataset already downloaded.")
                self.ds_cfg_list.append(Path.cwd() / proj_version / "data.yaml")
            except Exception:
                continue

    return self.ds_names, self.ds_cfg_list

set_key(api_key)

์ฒ˜๋ฆฌํ•  Roboflow API ํ‚ค๋ฅผ ์„ค์ •ํ•ฉ๋‹ˆ๋‹ค.

๋งค๊ฐœ๋ณ€์ˆ˜:

์ด๋ฆ„ ์œ ํ˜• ์„ค๋ช… ๊ธฐ๋ณธ๊ฐ’
api_key str

API ํ‚ค์ž…๋‹ˆ๋‹ค.

ํ•„์ˆ˜
์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
def set_key(self, api_key):
    """
    Set Roboflow API key for processing.

    Args:
        api_key (str): The API key.
    """

    check_requirements("roboflow")
    from roboflow import Roboflow

    self.rf = Roboflow(api_key=api_key)



ultralytics.utils.benchmarks.ProfileModels

ONNX ๋ฐ TensorRT ์—์„œ ๋‹ค์–‘ํ•œ ๋ชจ๋ธ์„ ํ”„๋กœํŒŒ์ผ๋งํ•˜๊ธฐ ์œ„ํ•œ ProfileModels ํด๋ž˜์Šค.

์ด ํด๋ž˜์Šค๋Š” ๋‹ค์–‘ํ•œ ๋ชจ๋ธ์˜ ์„ฑ๋Šฅ์„ ํ”„๋กœํŒŒ์ผ๋งํ•˜์—ฌ ๋ชจ๋ธ ์†๋„ ๋ฐ FLOP๊ณผ ๊ฐ™์€ ๊ฒฐ๊ณผ๋ฅผ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค.

์†์„ฑ:

์ด๋ฆ„ ์œ ํ˜• ์„ค๋ช…
paths list

ํ”„๋กœํŒŒ์ผ๋งํ•  ๋ชจ๋ธ์˜ ๊ฒฝ๋กœ์ž…๋‹ˆ๋‹ค.

num_timed_runs int

ํ”„๋กœํŒŒ์ผ๋ง์˜ ์‹œ๊ฐ„ ์ œํ•œ ์‹คํ–‰ ํšŸ์ˆ˜์ž…๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ 100์ž…๋‹ˆ๋‹ค.

num_warmup_runs int

ํ”„๋กœํŒŒ์ผ๋ง ์ „ ์›Œ๋ฐ์—… ์‹คํ–‰ ํšŸ์ˆ˜์ž…๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ 10์ž…๋‹ˆ๋‹ค.

min_time float

ํ”„๋กœํŒŒ์ผ๋งํ•  ์ตœ์†Œ ์‹œ๊ฐ„(์ดˆ)์ž…๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ 60์ดˆ์ž…๋‹ˆ๋‹ค.

imgsz int

๋ชจ๋ธ์— ์‚ฌ์šฉ๋˜๋Š” ์ด๋ฏธ์ง€ ํฌ๊ธฐ์ž…๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ 640์ž…๋‹ˆ๋‹ค.

๋ฐฉ๋ฒ•:

์ด๋ฆ„ ์„ค๋ช…
profile

๋ชจ๋ธ์„ ํ”„๋กœํŒŒ์ผ๋งํ•˜๊ณ  ๊ฒฐ๊ณผ๋ฅผ ์ธ์‡„ํ•ฉ๋‹ˆ๋‹ค.

์˜ˆ
from ultralytics.utils.benchmarks import ProfileModels

ProfileModels(['yolov8n.yaml', 'yolov8s.yaml'], imgsz=640).profile()
์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
class ProfileModels:
    """
    ProfileModels class for profiling different models on ONNX and TensorRT.

    This class profiles the performance of different models, returning results such as model speed and FLOPs.

    Attributes:
        paths (list): Paths of the models to profile.
        num_timed_runs (int): Number of timed runs for the profiling. Default is 100.
        num_warmup_runs (int): Number of warmup runs before profiling. Default is 10.
        min_time (float): Minimum number of seconds to profile for. Default is 60.
        imgsz (int): Image size used in the models. Default is 640.

    Methods:
        profile(): Profiles the models and prints the result.

    Example:
        ```python
        from ultralytics.utils.benchmarks import ProfileModels

        ProfileModels(['yolov8n.yaml', 'yolov8s.yaml'], imgsz=640).profile()
        ```
    """

    def __init__(
        self,
        paths: list,
        num_timed_runs=100,
        num_warmup_runs=10,
        min_time=60,
        imgsz=640,
        half=True,
        trt=True,
        device=None,
    ):
        """
        Initialize the ProfileModels class for profiling models.

        Args:
            paths (list): List of paths of the models to be profiled.
            num_timed_runs (int, optional): Number of timed runs for the profiling. Default is 100.
            num_warmup_runs (int, optional): Number of warmup runs before the actual profiling starts. Default is 10.
            min_time (float, optional): Minimum time in seconds for profiling a model. Default is 60.
            imgsz (int, optional): Size of the image used during profiling. Default is 640.
            half (bool, optional): Flag to indicate whether to use half-precision floating point for profiling.
            trt (bool, optional): Flag to indicate whether to profile using TensorRT. Default is True.
            device (torch.device, optional): Device used for profiling. If None, it is determined automatically.
        """
        self.paths = paths
        self.num_timed_runs = num_timed_runs
        self.num_warmup_runs = num_warmup_runs
        self.min_time = min_time
        self.imgsz = imgsz
        self.half = half
        self.trt = trt  # run TensorRT profiling
        self.device = device or torch.device(0 if torch.cuda.is_available() else "cpu")

    def profile(self):
        """Logs the benchmarking results of a model, checks metrics against floor and returns the results."""
        files = self.get_files()

        if not files:
            print("No matching *.pt or *.onnx files found.")
            return

        table_rows = []
        output = []
        for file in files:
            engine_file = file.with_suffix(".engine")
            if file.suffix in {".pt", ".yaml", ".yml"}:
                model = YOLO(str(file))
                model.fuse()  # to report correct params and GFLOPs in model.info()
                model_info = model.info()
                if self.trt and self.device.type != "cpu" and not engine_file.is_file():
                    engine_file = model.export(
                        format="engine", half=self.half, imgsz=self.imgsz, device=self.device, verbose=False
                    )
                onnx_file = model.export(
                    format="onnx", half=self.half, imgsz=self.imgsz, simplify=True, device=self.device, verbose=False
                )
            elif file.suffix == ".onnx":
                model_info = self.get_onnx_model_info(file)
                onnx_file = file
            else:
                continue

            t_engine = self.profile_tensorrt_model(str(engine_file))
            t_onnx = self.profile_onnx_model(str(onnx_file))
            table_rows.append(self.generate_table_row(file.stem, t_onnx, t_engine, model_info))
            output.append(self.generate_results_dict(file.stem, t_onnx, t_engine, model_info))

        self.print_table(table_rows)
        return output

    def get_files(self):
        """Returns a list of paths for all relevant model files given by the user."""
        files = []
        for path in self.paths:
            path = Path(path)
            if path.is_dir():
                extensions = ["*.pt", "*.onnx", "*.yaml"]
                files.extend([file for ext in extensions for file in glob.glob(str(path / ext))])
            elif path.suffix in {".pt", ".yaml", ".yml"}:  # add non-existing
                files.append(str(path))
            else:
                files.extend(glob.glob(str(path)))

        print(f"Profiling: {sorted(files)}")
        return [Path(file) for file in sorted(files)]

    def get_onnx_model_info(self, onnx_file: str):
        """Retrieves the information including number of layers, parameters, gradients and FLOPs for an ONNX model
        file.
        """
        return 0.0, 0.0, 0.0, 0.0  # return (num_layers, num_params, num_gradients, num_flops)

    @staticmethod
    def iterative_sigma_clipping(data, sigma=2, max_iters=3):
        """Applies an iterative sigma clipping algorithm to the given data times number of iterations."""
        data = np.array(data)
        for _ in range(max_iters):
            mean, std = np.mean(data), np.std(data)
            clipped_data = data[(data > mean - sigma * std) & (data < mean + sigma * std)]
            if len(clipped_data) == len(data):
                break
            data = clipped_data
        return data

    def profile_tensorrt_model(self, engine_file: str, eps: float = 1e-3):
        """Profiles the TensorRT model, measuring average run time and standard deviation among runs."""
        if not self.trt or not Path(engine_file).is_file():
            return 0.0, 0.0

        # Model and input
        model = YOLO(engine_file)
        input_data = np.random.rand(self.imgsz, self.imgsz, 3).astype(np.float32)  # must be FP32

        # Warmup runs
        elapsed = 0.0
        for _ in range(3):
            start_time = time.time()
            for _ in range(self.num_warmup_runs):
                model(input_data, imgsz=self.imgsz, verbose=False)
            elapsed = time.time() - start_time

        # Compute number of runs as higher of min_time or num_timed_runs
        num_runs = max(round(self.min_time / (elapsed + eps) * self.num_warmup_runs), self.num_timed_runs * 50)

        # Timed runs
        run_times = []
        for _ in TQDM(range(num_runs), desc=engine_file):
            results = model(input_data, imgsz=self.imgsz, verbose=False)
            run_times.append(results[0].speed["inference"])  # Convert to milliseconds

        run_times = self.iterative_sigma_clipping(np.array(run_times), sigma=2, max_iters=3)  # sigma clipping
        return np.mean(run_times), np.std(run_times)

    def profile_onnx_model(self, onnx_file: str, eps: float = 1e-3):
        """Profiles an ONNX model by executing it multiple times and returns the mean and standard deviation of run
        times.
        """
        check_requirements("onnxruntime")
        import onnxruntime as ort

        # Session with either 'TensorrtExecutionProvider', 'CUDAExecutionProvider', 'CPUExecutionProvider'
        sess_options = ort.SessionOptions()
        sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
        sess_options.intra_op_num_threads = 8  # Limit the number of threads
        sess = ort.InferenceSession(onnx_file, sess_options, providers=["CPUExecutionProvider"])

        input_tensor = sess.get_inputs()[0]
        input_type = input_tensor.type
        dynamic = not all(isinstance(dim, int) and dim >= 0 for dim in input_tensor.shape)  # dynamic input shape
        input_shape = (1, 3, self.imgsz, self.imgsz) if dynamic else input_tensor.shape

        # Mapping ONNX datatype to numpy datatype
        if "float16" in input_type:
            input_dtype = np.float16
        elif "float" in input_type:
            input_dtype = np.float32
        elif "double" in input_type:
            input_dtype = np.float64
        elif "int64" in input_type:
            input_dtype = np.int64
        elif "int32" in input_type:
            input_dtype = np.int32
        else:
            raise ValueError(f"Unsupported ONNX datatype {input_type}")

        input_data = np.random.rand(*input_shape).astype(input_dtype)
        input_name = input_tensor.name
        output_name = sess.get_outputs()[0].name

        # Warmup runs
        elapsed = 0.0
        for _ in range(3):
            start_time = time.time()
            for _ in range(self.num_warmup_runs):
                sess.run([output_name], {input_name: input_data})
            elapsed = time.time() - start_time

        # Compute number of runs as higher of min_time or num_timed_runs
        num_runs = max(round(self.min_time / (elapsed + eps) * self.num_warmup_runs), self.num_timed_runs)

        # Timed runs
        run_times = []
        for _ in TQDM(range(num_runs), desc=onnx_file):
            start_time = time.time()
            sess.run([output_name], {input_name: input_data})
            run_times.append((time.time() - start_time) * 1000)  # Convert to milliseconds

        run_times = self.iterative_sigma_clipping(np.array(run_times), sigma=2, max_iters=5)  # sigma clipping
        return np.mean(run_times), np.std(run_times)

    def generate_table_row(self, model_name, t_onnx, t_engine, model_info):
        """Generates a formatted string for a table row that includes model performance and metric details."""
        layers, params, gradients, flops = model_info
        return (
            f"| {model_name:18s} | {self.imgsz} | - | {t_onnx[0]:.2f} ยฑ {t_onnx[1]:.2f} ms | {t_engine[0]:.2f} ยฑ "
            f"{t_engine[1]:.2f} ms | {params / 1e6:.1f} | {flops:.1f} |"
        )

    @staticmethod
    def generate_results_dict(model_name, t_onnx, t_engine, model_info):
        """Generates a dictionary of model details including name, parameters, GFLOPS and speed metrics."""
        layers, params, gradients, flops = model_info
        return {
            "model/name": model_name,
            "model/parameters": params,
            "model/GFLOPs": round(flops, 3),
            "model/speed_ONNX(ms)": round(t_onnx[0], 3),
            "model/speed_TensorRT(ms)": round(t_engine[0], 3),
        }

    @staticmethod
    def print_table(table_rows):
        """Formats and prints a comparison table for different models with given statistics and performance data."""
        gpu = torch.cuda.get_device_name(0) if torch.cuda.is_available() else "GPU"
        header = (
            f"| Model | size<br><sup>(pixels) | mAP<sup>val<br>50-95 | Speed<br><sup>CPU ONNX<br>(ms) | "
            f"Speed<br><sup>{gpu} TensorRT<br>(ms) | params<br><sup>(M) | FLOPs<br><sup>(B) |"
        )
        separator = (
            "|-------------|---------------------|--------------------|------------------------------|"
            "-----------------------------------|------------------|-----------------|"
        )

        print(f"\n\n{header}")
        print(separator)
        for row in table_rows:
            print(row)

__init__(paths, num_timed_runs=100, num_warmup_runs=10, min_time=60, imgsz=640, half=True, trt=True, device=None)

ํ”„๋กœํŒŒ์ผ๋ง ๋ชจ๋ธ์— ๋Œ€ํ•œ ProfileModels ํด๋ž˜์Šค๋ฅผ ์ดˆ๊ธฐํ™”ํ•ฉ๋‹ˆ๋‹ค.

๋งค๊ฐœ๋ณ€์ˆ˜:

์ด๋ฆ„ ์œ ํ˜• ์„ค๋ช… ๊ธฐ๋ณธ๊ฐ’
paths list

ํ”„๋กœํŒŒ์ผ๋งํ•  ๋ชจ๋ธ์˜ ๊ฒฝ๋กœ ๋ชฉ๋ก์ž…๋‹ˆ๋‹ค.

ํ•„์ˆ˜
num_timed_runs int

ํ”„๋กœํŒŒ์ผ๋ง์˜ ์‹œ๊ฐ„ ์ œํ•œ ์‹คํ–‰ ํšŸ์ˆ˜์ž…๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ 100์ž…๋‹ˆ๋‹ค.

100
num_warmup_runs int

์‹ค์ œ ํ”„๋กœํŒŒ์ผ๋ง์ด ์‹œ์ž‘๋˜๊ธฐ ์ „ ์›Œ๋ฐ์—… ์‹คํ–‰ ํšŸ์ˆ˜์ž…๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ 10์ž…๋‹ˆ๋‹ค.

10
min_time float

๋ชจ๋ธ ํ”„๋กœํŒŒ์ผ๋ง์— ์†Œ์š”๋˜๋Š” ์ตœ์†Œ ์‹œ๊ฐ„(์ดˆ)์ž…๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ 60์ž…๋‹ˆ๋‹ค.

60
imgsz int

ํ”„๋กœํŒŒ์ผ๋ง ์ค‘์— ์‚ฌ์šฉ๋˜๋Š” ์ด๋ฏธ์ง€์˜ ํฌ๊ธฐ์ž…๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ 640์ž…๋‹ˆ๋‹ค.

640
half bool

ํ”„๋กœํŒŒ์ผ๋ง์— ๋ฐ˜์ •๋ฐ€๋„ ๋ถ€๋™ ์†Œ์ˆ˜์ ์„ ์‚ฌ์šฉํ• ์ง€ ์—ฌ๋ถ€๋ฅผ ๋‚˜ํƒ€๋‚ด๋Š” ํ”Œ๋ž˜๊ทธ์ž…๋‹ˆ๋‹ค.

True
trt bool

TensorRT ์„ ์‚ฌ์šฉํ•˜์—ฌ ํ”„๋กœํŒŒ์ผ๋งํ• ์ง€ ์—ฌ๋ถ€๋ฅผ ๋‚˜ํƒ€๋‚ด๋Š” ํ”Œ๋ž˜๊ทธ์ž…๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ True์ž…๋‹ˆ๋‹ค.

True
device device

ํ”„๋กœํŒŒ์ผ๋ง์— ์‚ฌ์šฉ๋˜๋Š” ์žฅ์น˜์ž…๋‹ˆ๋‹ค. ์—†์Œ์ธ ๊ฒฝ์šฐ ์ž๋™์œผ๋กœ ๊ฒฐ์ •๋ฉ๋‹ˆ๋‹ค.

None
์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
def __init__(
    self,
    paths: list,
    num_timed_runs=100,
    num_warmup_runs=10,
    min_time=60,
    imgsz=640,
    half=True,
    trt=True,
    device=None,
):
    """
    Initialize the ProfileModels class for profiling models.

    Args:
        paths (list): List of paths of the models to be profiled.
        num_timed_runs (int, optional): Number of timed runs for the profiling. Default is 100.
        num_warmup_runs (int, optional): Number of warmup runs before the actual profiling starts. Default is 10.
        min_time (float, optional): Minimum time in seconds for profiling a model. Default is 60.
        imgsz (int, optional): Size of the image used during profiling. Default is 640.
        half (bool, optional): Flag to indicate whether to use half-precision floating point for profiling.
        trt (bool, optional): Flag to indicate whether to profile using TensorRT. Default is True.
        device (torch.device, optional): Device used for profiling. If None, it is determined automatically.
    """
    self.paths = paths
    self.num_timed_runs = num_timed_runs
    self.num_warmup_runs = num_warmup_runs
    self.min_time = min_time
    self.imgsz = imgsz
    self.half = half
    self.trt = trt  # run TensorRT profiling
    self.device = device or torch.device(0 if torch.cuda.is_available() else "cpu")

generate_results_dict(model_name, t_onnx, t_engine, model_info) staticmethod

์ด๋ฆ„, ๋งค๊ฐœ๋ณ€์ˆ˜, GFLOPS ๋ฐ ์†๋„ ๋ฉ”ํŠธ๋ฆญ์„ ํฌํ•จํ•œ ๋ชจ๋ธ ์„ธ๋ถ€ ์ •๋ณด ์‚ฌ์ „์„ ์ƒ์„ฑํ•ฉ๋‹ˆ๋‹ค.

์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
@staticmethod
def generate_results_dict(model_name, t_onnx, t_engine, model_info):
    """Generates a dictionary of model details including name, parameters, GFLOPS and speed metrics."""
    layers, params, gradients, flops = model_info
    return {
        "model/name": model_name,
        "model/parameters": params,
        "model/GFLOPs": round(flops, 3),
        "model/speed_ONNX(ms)": round(t_onnx[0], 3),
        "model/speed_TensorRT(ms)": round(t_engine[0], 3),
    }

generate_table_row(model_name, t_onnx, t_engine, model_info)

๋ชจ๋ธ ์„ฑ๋Šฅ ๋ฐ ์ง€ํ‘œ ์„ธ๋ถ€ ์ •๋ณด๊ฐ€ ํฌํ•จ๋œ ํ…Œ์ด๋ธ” ํ–‰์— ๋Œ€ํ•ด ํ˜•์‹์ด ์ง€์ •๋œ ๋ฌธ์ž์—ด์„ ์ƒ์„ฑํ•ฉ๋‹ˆ๋‹ค.

์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
def generate_table_row(self, model_name, t_onnx, t_engine, model_info):
    """Generates a formatted string for a table row that includes model performance and metric details."""
    layers, params, gradients, flops = model_info
    return (
        f"| {model_name:18s} | {self.imgsz} | - | {t_onnx[0]:.2f} ยฑ {t_onnx[1]:.2f} ms | {t_engine[0]:.2f} ยฑ "
        f"{t_engine[1]:.2f} ms | {params / 1e6:.1f} | {flops:.1f} |"
    )

get_files()

์‚ฌ์šฉ์ž๊ฐ€ ์ œ๊ณตํ•œ ๋ชจ๋“  ๊ด€๋ จ ๋ชจ๋ธ ํŒŒ์ผ์˜ ๊ฒฝ๋กœ ๋ชฉ๋ก์„ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค.

์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
def get_files(self):
    """Returns a list of paths for all relevant model files given by the user."""
    files = []
    for path in self.paths:
        path = Path(path)
        if path.is_dir():
            extensions = ["*.pt", "*.onnx", "*.yaml"]
            files.extend([file for ext in extensions for file in glob.glob(str(path / ext))])
        elif path.suffix in {".pt", ".yaml", ".yml"}:  # add non-existing
            files.append(str(path))
        else:
            files.extend(glob.glob(str(path)))

    print(f"Profiling: {sorted(files)}")
    return [Path(file) for file in sorted(files)]

get_onnx_model_info(onnx_file)

ONNX ๋ชจ๋ธ์˜ ๋ ˆ์ด์–ด ์ˆ˜, ๋งค๊ฐœ๋ณ€์ˆ˜, ๊ทธ๋ผ๋ฐ์ด์…˜ ๋ฐ FLOP์„ ํฌํ•จํ•œ ์ •๋ณด๋ฅผ ๊ฒ€์ƒ‰ํ•ฉ๋‹ˆ๋‹ค. ํŒŒ์ผ์— ๋Œ€ํ•œ ์ •๋ณด๋ฅผ ๊ฒ€์ƒ‰ํ•ฉ๋‹ˆ๋‹ค.

์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
def get_onnx_model_info(self, onnx_file: str):
    """Retrieves the information including number of layers, parameters, gradients and FLOPs for an ONNX model
    file.
    """
    return 0.0, 0.0, 0.0, 0.0  # return (num_layers, num_params, num_gradients, num_flops)

iterative_sigma_clipping(data, sigma=2, max_iters=3) staticmethod

์ฃผ์–ด์ง„ ๋ฐ์ดํ„ฐ์— ๋ฐ˜๋ณต ์‹œ๊ทธ๋งˆ ํด๋ฆฌํ•‘ ์•Œ๊ณ ๋ฆฌ์ฆ˜์„ ๋ฐ˜๋ณต ํšŸ์ˆ˜๋งŒํผ ์ ์šฉํ•ฉ๋‹ˆ๋‹ค.

์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
@staticmethod
def iterative_sigma_clipping(data, sigma=2, max_iters=3):
    """Applies an iterative sigma clipping algorithm to the given data times number of iterations."""
    data = np.array(data)
    for _ in range(max_iters):
        mean, std = np.mean(data), np.std(data)
        clipped_data = data[(data > mean - sigma * std) & (data < mean + sigma * std)]
        if len(clipped_data) == len(data):
            break
        data = clipped_data
    return data

print_table(table_rows) staticmethod

์ฃผ์–ด์ง„ ํ†ต๊ณ„ ๋ฐ ์„ฑ๋Šฅ ๋ฐ์ดํ„ฐ๋กœ ์—ฌ๋Ÿฌ ๋ชจ๋ธ์— ๋Œ€ํ•œ ๋น„๊ต ํ…Œ์ด๋ธ”์˜ ์„œ์‹์„ ์ง€์ •ํ•˜๊ณ  ์ธ์‡„ํ•ฉ๋‹ˆ๋‹ค.

์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
@staticmethod
def print_table(table_rows):
    """Formats and prints a comparison table for different models with given statistics and performance data."""
    gpu = torch.cuda.get_device_name(0) if torch.cuda.is_available() else "GPU"
    header = (
        f"| Model | size<br><sup>(pixels) | mAP<sup>val<br>50-95 | Speed<br><sup>CPU ONNX<br>(ms) | "
        f"Speed<br><sup>{gpu} TensorRT<br>(ms) | params<br><sup>(M) | FLOPs<br><sup>(B) |"
    )
    separator = (
        "|-------------|---------------------|--------------------|------------------------------|"
        "-----------------------------------|------------------|-----------------|"
    )

    print(f"\n\n{header}")
    print(separator)
    for row in table_rows:
        print(row)

profile()

๋ชจ๋ธ์˜ ๋ฒค์น˜๋งˆํ‚น ๊ฒฐ๊ณผ๋ฅผ ๊ธฐ๋กํ•˜๊ณ  ๋ฐ”๋‹ฅ๊ณผ ๋น„๊ตํ•˜์—ฌ ๋ฉ”ํŠธ๋ฆญ์„ ํ™•์ธํ•œ ํ›„ ๊ฒฐ๊ณผ๋ฅผ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค.

์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
def profile(self):
    """Logs the benchmarking results of a model, checks metrics against floor and returns the results."""
    files = self.get_files()

    if not files:
        print("No matching *.pt or *.onnx files found.")
        return

    table_rows = []
    output = []
    for file in files:
        engine_file = file.with_suffix(".engine")
        if file.suffix in {".pt", ".yaml", ".yml"}:
            model = YOLO(str(file))
            model.fuse()  # to report correct params and GFLOPs in model.info()
            model_info = model.info()
            if self.trt and self.device.type != "cpu" and not engine_file.is_file():
                engine_file = model.export(
                    format="engine", half=self.half, imgsz=self.imgsz, device=self.device, verbose=False
                )
            onnx_file = model.export(
                format="onnx", half=self.half, imgsz=self.imgsz, simplify=True, device=self.device, verbose=False
            )
        elif file.suffix == ".onnx":
            model_info = self.get_onnx_model_info(file)
            onnx_file = file
        else:
            continue

        t_engine = self.profile_tensorrt_model(str(engine_file))
        t_onnx = self.profile_onnx_model(str(onnx_file))
        table_rows.append(self.generate_table_row(file.stem, t_onnx, t_engine, model_info))
        output.append(self.generate_results_dict(file.stem, t_onnx, t_engine, model_info))

    self.print_table(table_rows)
    return output

profile_onnx_model(onnx_file, eps=0.001)

ONNX ๋ชจ๋ธ์„ ์—ฌ๋Ÿฌ ๋ฒˆ ์‹คํ–‰ํ•˜์—ฌ ํ”„๋กœํŒŒ์ผ๋งํ•˜๊ณ  ์‹คํ–‰ ์‹œ๊ฐ„์˜ ํ‰๊ท ๊ณผ ํ‘œ์ค€ ํŽธ์ฐจ๋ฅผ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค. ํšŸ์ˆ˜๋ฅผ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค.

์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
def profile_onnx_model(self, onnx_file: str, eps: float = 1e-3):
    """Profiles an ONNX model by executing it multiple times and returns the mean and standard deviation of run
    times.
    """
    check_requirements("onnxruntime")
    import onnxruntime as ort

    # Session with either 'TensorrtExecutionProvider', 'CUDAExecutionProvider', 'CPUExecutionProvider'
    sess_options = ort.SessionOptions()
    sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
    sess_options.intra_op_num_threads = 8  # Limit the number of threads
    sess = ort.InferenceSession(onnx_file, sess_options, providers=["CPUExecutionProvider"])

    input_tensor = sess.get_inputs()[0]
    input_type = input_tensor.type
    dynamic = not all(isinstance(dim, int) and dim >= 0 for dim in input_tensor.shape)  # dynamic input shape
    input_shape = (1, 3, self.imgsz, self.imgsz) if dynamic else input_tensor.shape

    # Mapping ONNX datatype to numpy datatype
    if "float16" in input_type:
        input_dtype = np.float16
    elif "float" in input_type:
        input_dtype = np.float32
    elif "double" in input_type:
        input_dtype = np.float64
    elif "int64" in input_type:
        input_dtype = np.int64
    elif "int32" in input_type:
        input_dtype = np.int32
    else:
        raise ValueError(f"Unsupported ONNX datatype {input_type}")

    input_data = np.random.rand(*input_shape).astype(input_dtype)
    input_name = input_tensor.name
    output_name = sess.get_outputs()[0].name

    # Warmup runs
    elapsed = 0.0
    for _ in range(3):
        start_time = time.time()
        for _ in range(self.num_warmup_runs):
            sess.run([output_name], {input_name: input_data})
        elapsed = time.time() - start_time

    # Compute number of runs as higher of min_time or num_timed_runs
    num_runs = max(round(self.min_time / (elapsed + eps) * self.num_warmup_runs), self.num_timed_runs)

    # Timed runs
    run_times = []
    for _ in TQDM(range(num_runs), desc=onnx_file):
        start_time = time.time()
        sess.run([output_name], {input_name: input_data})
        run_times.append((time.time() - start_time) * 1000)  # Convert to milliseconds

    run_times = self.iterative_sigma_clipping(np.array(run_times), sigma=2, max_iters=5)  # sigma clipping
    return np.mean(run_times), np.std(run_times)

profile_tensorrt_model(engine_file, eps=0.001)

TensorRT ๋ชจ๋ธ์„ ํ”„๋กœํŒŒ์ผ๋งํ•˜์—ฌ ํ‰๊ท  ์‹คํ–‰ ์‹œ๊ฐ„ ๋ฐ ์‹คํ–‰ ๊ฐ„ ํ‘œ์ค€ ํŽธ์ฐจ๋ฅผ ์ธก์ •ํ•ฉ๋‹ˆ๋‹ค.

์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
def profile_tensorrt_model(self, engine_file: str, eps: float = 1e-3):
    """Profiles the TensorRT model, measuring average run time and standard deviation among runs."""
    if not self.trt or not Path(engine_file).is_file():
        return 0.0, 0.0

    # Model and input
    model = YOLO(engine_file)
    input_data = np.random.rand(self.imgsz, self.imgsz, 3).astype(np.float32)  # must be FP32

    # Warmup runs
    elapsed = 0.0
    for _ in range(3):
        start_time = time.time()
        for _ in range(self.num_warmup_runs):
            model(input_data, imgsz=self.imgsz, verbose=False)
        elapsed = time.time() - start_time

    # Compute number of runs as higher of min_time or num_timed_runs
    num_runs = max(round(self.min_time / (elapsed + eps) * self.num_warmup_runs), self.num_timed_runs * 50)

    # Timed runs
    run_times = []
    for _ in TQDM(range(num_runs), desc=engine_file):
        results = model(input_data, imgsz=self.imgsz, verbose=False)
        run_times.append(results[0].speed["inference"])  # Convert to milliseconds

    run_times = self.iterative_sigma_clipping(np.array(run_times), sigma=2, max_iters=3)  # sigma clipping
    return np.mean(run_times), np.std(run_times)



ultralytics.utils.benchmarks.benchmark(model=WEIGHTS_DIR / 'yolov8n.pt', data=None, imgsz=160, half=False, int8=False, device='cpu', verbose=False)

์†๋„์™€ ์ •ํ™•์„ฑ์„ ์œ„ํ•ด ๋‹ค์–‘ํ•œ ํ˜•์‹์˜ YOLO ๋ชจ๋ธ์„ ๋ฒค์น˜๋งˆํ‚นํ•˜์„ธ์š”.

๋งค๊ฐœ๋ณ€์ˆ˜:

์ด๋ฆ„ ์œ ํ˜• ์„ค๋ช… ๊ธฐ๋ณธ๊ฐ’
model str | Path | optional

๋ชจ๋ธ ํŒŒ์ผ ๋˜๋Š” ๋””๋ ‰ํ† ๋ฆฌ์˜ ๊ฒฝ๋กœ์ž…๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ Path(SETTINGS['weights_dir']) / 'yolov8n.pt'.

WEIGHTS_DIR / 'yolov8n.pt'
data str

ํ‰๊ฐ€ํ•  ๋ฐ์ดํ„ฐ ์ง‘ํ•ฉ์œผ๋กœ, ์ „๋‹ฌ๋˜์ง€ ์•Š์€ ๊ฒฝ์šฐ TASK2DATA์—์„œ ์ƒ์†๋ฉ๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ ์—†์Œ์ž…๋‹ˆ๋‹ค.

None
imgsz int

๋ฒค์น˜๋งˆํฌ์šฉ ์ด๋ฏธ์ง€ ํฌ๊ธฐ์ž…๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ 160์ž…๋‹ˆ๋‹ค.

160
half bool

True์ธ ๊ฒฝ์šฐ ๋ชจ๋ธ์— ๋ฐ˜์ •ํ™•๋„๋ฅผ ์‚ฌ์šฉํ•ฉ๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ False์ž…๋‹ˆ๋‹ค.

False
int8 bool

True์ธ ๊ฒฝ์šฐ ๋ชจ๋ธ์— int8-precision์„ ์‚ฌ์šฉํ•ฉ๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ False์ž…๋‹ˆ๋‹ค.

False
device str

๋ฒค์น˜๋งˆํฌ๋ฅผ ์‹คํ–‰ํ•  ๋””๋ฐ”์ด์Šค, 'cpu' ๋˜๋Š” 'cuda' ์ค‘ ํ•˜๋‚˜์ž…๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ 'cpu'์ž…๋‹ˆ๋‹ค.

'cpu'
verbose bool | float | optional

True ๋˜๋Š” ์‹ค์ˆ˜์ธ ๊ฒฝ์šฐ ๋ฒค์น˜๋งˆํฌ๊ฐ€ ์ง€์ •๋œ ๋ฉ”ํŠธ๋ฆญ์„ ํ†ต๊ณผํ–ˆ๋‹ค๊ณ  ์ฃผ์žฅํ•ฉ๋‹ˆ๋‹ค. ๊ธฐ๋ณธ๊ฐ’์€ False์ž…๋‹ˆ๋‹ค.

False

๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค:

์ด๋ฆ„ ์œ ํ˜• ์„ค๋ช…
df DataFrame

ํŒŒ์ผ ํฌ๊ธฐ๋ฅผ ํฌํ•จํ•œ ๊ฐ ํ˜•์‹์— ๋Œ€ํ•œ ๋ฒค์น˜๋งˆํฌ ๊ฒฐ๊ณผ๊ฐ€ ํฌํ•จ๋œ ํŒ๋‹ค์Šค ๋ฐ์ดํ„ฐ ํ”„๋ ˆ์ž„์ž…๋‹ˆ๋‹ค, ๋ฉ”ํŠธ๋ฆญ ๋ฐ ์ถ”๋ก  ์‹œ๊ฐ„์„ ํฌํ•จํ•œ ๊ฐ ํ˜•์‹์˜ ๋ฒค์น˜๋งˆํฌ ๊ฒฐ๊ณผ๊ฐ€ ํฌํ•จ๋œ ๋ฐ์ดํ„ฐ ํ”„๋ ˆ์ž„์ž…๋‹ˆ๋‹ค.

์˜ˆ
from ultralytics.utils.benchmarks import benchmark

benchmark(model='yolov8n.pt', imgsz=640)
์˜ ์†Œ์Šค ์ฝ”๋“œ ultralytics/utils/benchmarks.py
def benchmark(
    model=WEIGHTS_DIR / "yolov8n.pt", data=None, imgsz=160, half=False, int8=False, device="cpu", verbose=False
):
    """
    Benchmark a YOLO model across different formats for speed and accuracy.

    Args:
        model (str | Path | optional): Path to the model file or directory. Default is
            Path(SETTINGS['weights_dir']) / 'yolov8n.pt'.
        data (str, optional): Dataset to evaluate on, inherited from TASK2DATA if not passed. Default is None.
        imgsz (int, optional): Image size for the benchmark. Default is 160.
        half (bool, optional): Use half-precision for the model if True. Default is False.
        int8 (bool, optional): Use int8-precision for the model if True. Default is False.
        device (str, optional): Device to run the benchmark on, either 'cpu' or 'cuda'. Default is 'cpu'.
        verbose (bool | float | optional): If True or a float, assert benchmarks pass with given metric.
            Default is False.

    Returns:
        df (pandas.DataFrame): A pandas DataFrame with benchmark results for each format, including file size,
            metric, and inference time.

    Example:
        ```python
        from ultralytics.utils.benchmarks import benchmark

        benchmark(model='yolov8n.pt', imgsz=640)
        ```
    """
    import pandas as pd  # scope for faster 'import ultralytics'

    pd.options.display.max_columns = 10
    pd.options.display.width = 120
    device = select_device(device, verbose=False)
    if isinstance(model, (str, Path)):
        model = YOLO(model)
    is_end2end = getattr(model.model.model[-1], "end2end", False)

    y = []
    t0 = time.time()
    for i, (name, format, suffix, cpu, gpu) in export_formats().iterrows():  # index, (name, format, suffix, CPU, GPU)
        emoji, filename = "โŒ", None  # export defaults
        try:
            # Checks
            if i == 7:  # TF GraphDef
                assert model.task != "obb", "TensorFlow GraphDef not supported for OBB task"
            elif i == 9:  # Edge TPU
                assert LINUX and not ARM64, "Edge TPU export only supported on non-aarch64 Linux"
            elif i in {5, 10}:  # CoreML and TF.js
                assert MACOS or LINUX, "CoreML and TF.js export only supported on macOS and Linux"
                assert not IS_RASPBERRYPI, "CoreML and TF.js export not supported on Raspberry Pi"
                assert not IS_JETSON, "CoreML and TF.js export not supported on NVIDIA Jetson"
                assert not is_end2end, "End-to-end models not supported by CoreML and TF.js yet"
            if i in {3, 5}:  # CoreML and OpenVINO
                assert not IS_PYTHON_3_12, "CoreML and OpenVINO not supported on Python 3.12"
            if i in {6, 7, 8, 9, 10}:  # All TF formats
                assert not isinstance(model, YOLOWorld), "YOLOWorldv2 TensorFlow exports not supported by onnx2tf yet"
                assert not is_end2end, "End-to-end models not supported by onnx2tf yet"
            if i in {11}:  # Paddle
                assert not isinstance(model, YOLOWorld), "YOLOWorldv2 Paddle exports not supported yet"
                assert not is_end2end, "End-to-end models not supported by PaddlePaddle yet"
            if i in {12}:  # NCNN
                assert not isinstance(model, YOLOWorld), "YOLOWorldv2 NCNN exports not supported yet"
                assert not is_end2end, "End-to-end models not supported by NCNN yet"
            if "cpu" in device.type:
                assert cpu, "inference not supported on CPU"
            if "cuda" in device.type:
                assert gpu, "inference not supported on GPU"

            # Export
            if format == "-":
                filename = model.ckpt_path or model.cfg
                exported_model = model  # PyTorch format
            else:
                filename = model.export(imgsz=imgsz, format=format, half=half, int8=int8, device=device, verbose=False)
                exported_model = YOLO(filename, task=model.task)
                assert suffix in str(filename), "export failed"
            emoji = "โŽ"  # indicates export succeeded

            # Predict
            assert model.task != "pose" or i != 7, "GraphDef Pose inference is not supported"
            assert i not in {9, 10}, "inference not supported"  # Edge TPU and TF.js are unsupported
            assert i != 5 or platform.system() == "Darwin", "inference only supported on macOS>=10.13"  # CoreML
            exported_model.predict(ASSETS / "bus.jpg", imgsz=imgsz, device=device, half=half)

            # Validate
            data = data or TASK2DATA[model.task]  # task to dataset, i.e. coco8.yaml for task=detect
            key = TASK2METRIC[model.task]  # task to metric, i.e. metrics/mAP50-95(B) for task=detect
            results = exported_model.val(
                data=data, batch=1, imgsz=imgsz, plots=False, device=device, half=half, int8=int8, verbose=False
            )
            metric, speed = results.results_dict[key], results.speed["inference"]
            fps = round((1000 / speed), 2)  # frames per second
            y.append([name, "โœ…", round(file_size(filename), 1), round(metric, 4), round(speed, 2), fps])
        except Exception as e:
            if verbose:
                assert type(e) is AssertionError, f"Benchmark failure for {name}: {e}"
            LOGGER.warning(f"ERROR โŒ๏ธ Benchmark failure for {name}: {e}")
            y.append([name, emoji, round(file_size(filename), 1), None, None, None])  # mAP, t_inference

    # Print results
    check_yolo(device=device)  # print system info
    df = pd.DataFrame(y, columns=["Format", "Statusโ”", "Size (MB)", key, "Inference time (ms/im)", "FPS"])

    name = Path(model.ckpt_path).name
    s = f"\nBenchmarks complete for {name} on {data} at imgsz={imgsz} ({time.time() - t0:.2f}s)\n{df}\n"
    LOGGER.info(s)
    with open("benchmarks.log", "a", errors="ignore", encoding="utf-8") as f:
        f.write(s)

    if verbose and isinstance(verbose, float):
        metrics = df[key].array  # values to compare to floor
        floor = verbose  # minimum metric floor to pass, i.e. = 0.29 mAP for YOLOv5n
        assert all(x > floor for x in metrics if pd.notna(x)), f"Benchmark failure: metric(s) < floor {floor}"

    return df





Created 2023-11-12, Updated 2024-06-02
Authors: glenn-jocher (6), Burhan-Q (1)