Reference for ultralytics/utils/metrics.py

def matrix(self):
    """Return the confusion matrix."""
    return self.matrix

@TryExcept(msg="ConfusionMatrix plot failure")
@plt_settings()
def plot(self, normalize: bool = True, save_dir: str = "", on_plot=None):
    """
    Plot the confusion matrix using matplotlib and save it to a file.

    Args:
        normalize (bool, optional): Whether to normalize the confusion matrix.
        save_dir (str, optional): Directory where the plot will be saved.
        on_plot (callable, optional): An optional callback to pass plots path and data when they are rendered.
    """
    import matplotlib.pyplot as plt  # scope for faster 'import ultralytics'

    array = self.matrix / ((self.matrix.sum(0).reshape(1, -1) + 1e-9) if normalize else 1)  # normalize columns
    array[array < 0.005] = np.nan  # don't annotate (would appear as 0.00)

    fig, ax = plt.subplots(1, 1, figsize=(12, 9))
    names, n = list(self.names.values()), self.nc
    if self.nc >= 100:  # downsample for large class count
        k = max(2, self.nc // 60)  # step size for downsampling, always > 1
        keep_idx = slice(None, None, k)  # create slice instead of array
        names = names[keep_idx]  # slice class names
        array = array[keep_idx, :][:, keep_idx]  # slice matrix rows and cols
        n = (self.nc + k - 1) // k  # number of retained classes
    nc = nn = n if self.task == "classify" else n + 1  # adjust for background if needed
    ticklabels = (names + ["background"]) if (0 < nn < 99) and (nn == nc) else "auto"
    xy_ticks = np.arange(len(ticklabels))
    tick_fontsize = max(6, 15 - 0.1 * nc)  # Minimum size is 6
    label_fontsize = max(6, 12 - 0.1 * nc)
    title_fontsize = max(6, 12 - 0.1 * nc)
    btm = max(0.1, 0.25 - 0.001 * nc)  # Minimum value is 0.1
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")  # suppress empty matrix RuntimeWarning: All-NaN slice encountered
        im = ax.imshow(array, cmap="Blues", vmin=0.0, interpolation="none")
        ax.xaxis.set_label_position("bottom")
        if nc < 30:  # Add score for each cell of confusion matrix
            color_threshold = 0.45 * (1 if normalize else np.nanmax(array))  # text color threshold
            for i, row in enumerate(array[:nc]):
                for j, val in enumerate(row[:nc]):
                    val = array[i, j]
                    if np.isnan(val):
                        continue
                    ax.text(
                        j,
                        i,
                        f"{val:.2f}" if normalize else f"{int(val)}",
                        ha="center",
                        va="center",
                        fontsize=10,
                        color="white" if val > color_threshold else "black",
                    )
        cbar = fig.colorbar(im, ax=ax, fraction=0.046, pad=0.05)
    title = "Confusion Matrix" + " Normalized" * normalize
    ax.set_xlabel("True", fontsize=label_fontsize, labelpad=10)
    ax.set_ylabel("Predicted", fontsize=label_fontsize, labelpad=10)
    ax.set_title(title, fontsize=title_fontsize, pad=20)
    ax.set_xticks(xy_ticks)
    ax.set_yticks(xy_ticks)
    ax.tick_params(axis="x", bottom=True, top=False, labelbottom=True, labeltop=False)
    ax.tick_params(axis="y", left=True, right=False, labelleft=True, labelright=False)
    if ticklabels != "auto":
        ax.set_xticklabels(ticklabels, fontsize=tick_fontsize, rotation=90, ha="center")
        ax.set_yticklabels(ticklabels, fontsize=tick_fontsize)
    for s in {"left", "right", "bottom", "top", "outline"}:
        if s != "outline":
            ax.spines[s].set_visible(False)  # Confusion matrix plot don't have outline
        cbar.ax.spines[s].set_visible(False)
    fig.subplots_adjust(left=0, right=0.84, top=0.94, bottom=btm)  # Adjust layout to ensure equal margins
    plot_fname = Path(save_dir) / f"{title.lower().replace(' ', '_')}.png"
    fig.savefig(plot_fname, dpi=250)
    plt.close(fig)
    if on_plot:
        on_plot(plot_fname)

def plot_matches(self, img: torch.Tensor, im_file: str, save_dir: Path) -> None:
    """
    Plot grid of GT, TP, FP, FN for each image.

    Args:
        img (torch.Tensor): Image to plot onto.
        im_file (str): Image filename to save visualizations.
        save_dir (Path): Location to save the visualizations to.
    """
    if not self.matches:
        return
    from .ops import xyxy2xywh
    from .plotting import plot_images

    # Create batch of 4 (GT, TP, FP, FN)
    labels = defaultdict(list)
    for i, mtype in enumerate(["GT", "FP", "TP", "FN"]):
        mbatch = self.matches[mtype]
        if "conf" not in mbatch:
            mbatch["conf"] = torch.tensor([1.0] * len(mbatch["bboxes"]), device=img.device)
        mbatch["batch_idx"] = torch.ones(len(mbatch["bboxes"]), device=img.device) * i
        for k in mbatch.keys():
            labels[k] += mbatch[k]

    labels = {k: torch.stack(v, 0) if len(v) else v for k, v in labels.items()}
    if self.task != "obb" and len(labels["bboxes"]):
        labels["bboxes"] = xyxy2xywh(labels["bboxes"])
    (save_dir / "visualizations").mkdir(parents=True, exist_ok=True)
    plot_images(
        labels,
        img.repeat(4, 1, 1, 1),
        paths=["Ground Truth", "False Positives", "True Positives", "False Negatives"],
        fname=save_dir / "visualizations" / Path(im_file).name,
        names=self.names,
        max_subplots=4,
        conf_thres=0.001,
    )

def print(self):
    """Print the confusion matrix to the console."""
    for i in range(self.matrix.shape[0]):
        LOGGER.info(" ".join(map(str, self.matrix[i])))

def process_batch(
    self,
    detections: Dict[str, torch.Tensor],
    batch: Dict[str, Any],
    conf: float = 0.25,
    iou_thres: float = 0.45,
) -> None:
    """
    Update confusion matrix for object detection task.

    Args:
        detections (Dict[str, torch.Tensor]): Dictionary containing detected bounding boxes and their associated information.
                                   Should contain 'cls', 'conf', and 'bboxes' keys, where 'bboxes' can be
                                   Array[N, 4] for regular boxes or Array[N, 5] for OBB with angle.
        batch (Dict[str, Any]): Batch dictionary containing ground truth data with 'bboxes' (Array[M, 4]| Array[M, 5]) and
            'cls' (Array[M]) keys, where M is the number of ground truth objects.
        conf (float, optional): Confidence threshold for detections.
        iou_thres (float, optional): IoU threshold for matching detections to ground truth.
    """
    gt_cls, gt_bboxes = batch["cls"], batch["bboxes"]
    if self.matches is not None:  # only if visualization is enabled
        self.matches = {k: defaultdict(list) for k in {"TP", "FP", "FN", "GT"}}
        for i in range(len(gt_cls)):
            self._append_matches("GT", batch, i)  # store GT
    is_obb = gt_bboxes.shape[1] == 5  # check if boxes contains angle for OBB
    conf = 0.25 if conf in {None, 0.01 if is_obb else 0.001} else conf  # apply 0.25 if default val conf is passed
    no_pred = len(detections["cls"]) == 0
    if gt_cls.shape[0] == 0:  # Check if labels is empty
        if not no_pred:
            detections = {k: detections[k][detections["conf"] > conf] for k in detections}
            detection_classes = detections["cls"].int().tolist()
            for i, dc in enumerate(detection_classes):
                self.matrix[dc, self.nc] += 1  # FP
                self._append_matches("FP", detections, i)
        return
    if no_pred:
        gt_classes = gt_cls.int().tolist()
        for i, gc in enumerate(gt_classes):
            self.matrix[self.nc, gc] += 1  # FN
            self._append_matches("FN", batch, i)
        return

    detections = {k: detections[k][detections["conf"] > conf] for k in detections}
    gt_classes = gt_cls.int().tolist()
    detection_classes = detections["cls"].int().tolist()
    bboxes = detections["bboxes"]
    iou = batch_probiou(gt_bboxes, bboxes) if is_obb else box_iou(gt_bboxes, bboxes)

    x = torch.where(iou > iou_thres)
    if x[0].shape[0]:
        matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy()
        if x[0].shape[0] > 1:
            matches = matches[matches[:, 2].argsort()[::-1]]
            matches = matches[np.unique(matches[:, 1], return_index=True)[1]]
            matches = matches[matches[:, 2].argsort()[::-1]]
            matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
    else:
        matches = np.zeros((0, 3))

    n = matches.shape[0] > 0
    m0, m1, _ = matches.transpose().astype(int)
    for i, gc in enumerate(gt_classes):
        j = m0 == i
        if n and sum(j) == 1:
            dc = detection_classes[m1[j].item()]
            self.matrix[dc, gc] += 1  # TP if class is correct else both an FP and an FN
            if dc == gc:
                self._append_matches("TP", detections, m1[j].item())
            else:
                self._append_matches("FP", detections, m1[j].item())
                self._append_matches("FN", batch, i)
        else:
            self.matrix[self.nc, gc] += 1  # FN
            self._append_matches("FN", batch, i)

    for i, dc in enumerate(detection_classes):
        if not any(m1 == i):
            self.matrix[dc, self.nc] += 1  # FP
            self._append_matches("FP", detections, i)

def process_cls_preds(self, preds: List[torch.Tensor], targets: List[torch.Tensor]) -> None:
    """
    Update confusion matrix for classification task.

    Args:
        preds (List[N, min(nc,5)]): Predicted class labels.
        targets (List[N, 1]): Ground truth class labels.
    """
    preds, targets = torch.cat(preds)[:, 0], torch.cat(targets)
    for p, t in zip(preds.cpu().numpy(), targets.cpu().numpy()):
        self.matrix[p][t] += 1

def summary(self, normalize: bool = False, decimals: int = 5) -> List[Dict[str, float]]:
    """
    Generate a summarized representation of the confusion matrix as a list of dictionaries, with optional
    normalization. This is useful for exporting the matrix to various formats such as CSV, XML, HTML, JSON, or SQL.

    Args:
        normalize (bool): Whether to normalize the confusion matrix values.
        decimals (int): Number of decimal places to round the output values to.

    Returns:
        (List[Dict[str, float]]): A list of dictionaries, each representing one predicted class with corresponding values for all actual classes.

    Examples:
        >>> results = model.val(data="coco8.yaml", plots=True)
        >>> cm_dict = results.confusion_matrix.summary(normalize=True, decimals=5)
        >>> print(cm_dict)
    """
    import re

    names = list(self.names.values()) if self.task == "classify" else list(self.names.values()) + ["background"]
    clean_names, seen = [], set()
    for name in names:
        clean_name = re.sub(r"[^a-zA-Z0-9_]", "_", name)
        original_clean = clean_name
        counter = 1
        while clean_name.lower() in seen:
            clean_name = f"{original_clean}_{counter}"
            counter += 1
        seen.add(clean_name.lower())
        clean_names.append(clean_name)
    array = (self.matrix / ((self.matrix.sum(0).reshape(1, -1) + 1e-9) if normalize else 1)).round(decimals)
    return [
        dict({"Predicted": clean_names[i]}, **{clean_names[j]: array[i, j] for j in range(len(clean_names))})
        for i in range(len(clean_names))
    ]

def tp_fp(self) -> Tuple[np.ndarray, np.ndarray]:
    """
    Return true positives and false positives.

    Returns:
        tp (np.ndarray): True positives.
        fp (np.ndarray): False positives.
    """
    tp = self.matrix.diagonal()  # true positives
    fp = self.matrix.sum(1) - tp  # false positives
    # fn = self.matrix.sum(0) - tp  # false negatives (missed detections)
    return (tp, fp) if self.task == "classify" else (tp[:-1], fp[:-1])  # remove background class if task=detect

def class_result(self, i: int) -> Tuple[float, float, float, float]:
    """Return class-aware result, p[i], r[i], ap50[i], ap[i]."""
    return self.p[i], self.r[i], self.ap50[i], self.ap[i]

def fitness(self) -> float:
    """Return model fitness as a weighted combination of metrics."""
    w = [0.0, 0.0, 0.1, 0.9]  # weights for [P, R, mAP@0.5, mAP@0.5:0.95]
    return (np.nan_to_num(np.array(self.mean_results())) * w).sum()

def mean_results(self) -> List[float]:
    """Return mean of results, mp, mr, map50, map."""
    return [self.mp, self.mr, self.map50, self.map]

def update(self, results: tuple):
    """
    Update the evaluation metrics with a new set of results.

    Args:
        results (tuple): A tuple containing evaluation metrics:
            - p (list): Precision for each class.
            - r (list): Recall for each class.
            - f1 (list): F1 score for each class.
            - all_ap (list): AP scores for all classes and all IoU thresholds.
            - ap_class_index (list): Index of class for each AP score.
            - p_curve (list): Precision curve for each class.
            - r_curve (list): Recall curve for each class.
            - f1_curve (list): F1 curve for each class.
            - px (list): X values for the curves.
            - prec_values (list): Precision values for each class.
    """
    (
        self.p,
        self.r,
        self.f1,
        self.all_ap,
        self.ap_class_index,
        self.p_curve,
        self.r_curve,
        self.f1_curve,
        self.px,
        self.prec_values,
    ) = results

def class_result(self, i: int) -> Tuple[float, float, float, float]:
    """Return the result of evaluating the performance of an object detection model on a specific class."""
    return self.box.class_result(i)

def clear_stats(self):
    """Clear the stored statistics."""
    for v in self.stats.values():
        v.clear()

def mean_results(self) -> List[float]:
    """Calculate mean of detected objects & return precision, recall, mAP50, and mAP50-95."""
    return self.box.mean_results()

def process(self, save_dir: Path = Path("."), plot: bool = False, on_plot=None) -> Dict[str, np.ndarray]:
    """
    Process predicted results for object detection and update metrics.

    Args:
        save_dir (Path): Directory to save plots. Defaults to Path(".").
        plot (bool): Whether to plot precision-recall curves. Defaults to False.
        on_plot (callable, optional): Function to call after plots are generated. Defaults to None.

    Returns:
        (Dict[str, np.ndarray]): Dictionary containing concatenated statistics arrays.
    """
    stats = {k: np.concatenate(v, 0) for k, v in self.stats.items()}  # to numpy
    if not stats:
        return stats
    results = ap_per_class(
        stats["tp"],
        stats["conf"],
        stats["pred_cls"],
        stats["target_cls"],
        plot=plot,
        save_dir=save_dir,
        names=self.names,
        on_plot=on_plot,
        prefix="Box",
    )[2:]
    self.box.nc = len(self.names)
    self.box.update(results)
    self.nt_per_class = np.bincount(stats["target_cls"].astype(int), minlength=len(self.names))
    self.nt_per_image = np.bincount(stats["target_img"].astype(int), minlength=len(self.names))
    return stats

def summary(self, normalize: bool = True, decimals: int = 5) -> List[Dict[str, Any]]:
    """
    Generate a summarized representation of per-class detection metrics as a list of dictionaries. Includes shared
    scalar metrics (mAP, mAP50, mAP75) alongside precision, recall, and F1-score for each class.

    Args:
       normalize (bool): For Detect metrics, everything is normalized  by default [0-1].
       decimals (int): Number of decimal places to round the metrics values to.

    Returns:
       (List[Dict[str, Any]]): A list of dictionaries, each representing one class with corresponding metric values.

    Examples:
       >>> results = model.val(data="coco8.yaml")
       >>> detection_summary = results.summary()
       >>> print(detection_summary)
    """
    per_class = {
        "Box-P": self.box.p,
        "Box-R": self.box.r,
        "Box-F1": self.box.f1,
    }
    return [
        {
            "Class": self.names[self.ap_class_index[i]],
            "Images": self.nt_per_image[self.ap_class_index[i]],
            "Instances": self.nt_per_class[self.ap_class_index[i]],
            **{k: round(v[i], decimals) for k, v in per_class.items()},
            "mAP50": round(self.class_result(i)[2], decimals),
            "mAP50-95": round(self.class_result(i)[3], decimals),
        }
        for i in range(len(per_class["Box-P"]))
    ]

def update_stats(self, stat: Dict[str, Any]) -> None:
    """
    Update statistics by appending new values to existing stat collections.

    Args:
        stat (Dict[str, any]): Dictionary containing new statistical values to append.
                     Keys should match existing keys in self.stats.
    """
    for k in self.stats.keys():
        self.stats[k].append(stat[k])

def class_result(self, i: int) -> List[float]:
    """Return classification results for a specified class index."""
    return DetMetrics.class_result(self, i) + self.seg.class_result(i)

def mean_results(self) -> List[float]:
    """Return the mean metrics for bounding box and segmentation results."""
    return DetMetrics.mean_results(self) + self.seg.mean_results()

def process(self, save_dir: Path = Path("."), plot: bool = False, on_plot=None) -> Dict[str, np.ndarray]:
    """
    Process the detection and segmentation metrics over the given set of predictions.

    Args:
        save_dir (Path): Directory to save plots. Defaults to Path(".").
        plot (bool): Whether to plot precision-recall curves. Defaults to False.
        on_plot (callable, optional): Function to call after plots are generated. Defaults to None.

    Returns:
        (Dict[str, np.ndarray]): Dictionary containing concatenated statistics arrays.
    """
    stats = DetMetrics.process(self, save_dir, plot, on_plot=on_plot)  # process box stats
    results_mask = ap_per_class(
        stats["tp_m"],
        stats["conf"],
        stats["pred_cls"],
        stats["target_cls"],
        plot=plot,
        on_plot=on_plot,
        save_dir=save_dir,
        names=self.names,
        prefix="Mask",
    )[2:]
    self.seg.nc = len(self.names)
    self.seg.update(results_mask)
    return stats

def summary(self, normalize: bool = True, decimals: int = 5) -> List[Dict[str, Any]]:
    """
    Generate a summarized representation of per-class segmentation metrics as a list of dictionaries. Includes both
    box and mask scalar metrics (mAP, mAP50, mAP75) alongside precision, recall, and F1-score for each class.

    Args:
        normalize (bool): For Segment metrics, everything is normalized  by default [0-1].
        decimals (int): Number of decimal places to round the metrics values to.

    Returns:
        (List[Dict[str, Any]]): A list of dictionaries, each representing one class with corresponding metric values.

    Examples:
        >>> results = model.val(data="coco8-seg.yaml")
        >>> seg_summary = results.summary(decimals=4)
        >>> print(seg_summary)
    """
    per_class = {
        "Mask-P": self.seg.p,
        "Mask-R": self.seg.r,
        "Mask-F1": self.seg.f1,
    }
    summary = DetMetrics.summary(self, normalize, decimals)  # get box summary
    for i, s in enumerate(summary):
        s.update({**{k: round(v[i], decimals) for k, v in per_class.items()}})
    return summary

def class_result(self, i: int) -> List[float]:
    """Return the class-wise detection results for a specific class i."""
    return DetMetrics.class_result(self, i) + self.pose.class_result(i)

def mean_results(self) -> List[float]:
    """Return the mean results of box and pose."""
    return DetMetrics.mean_results(self) + self.pose.mean_results()

def process(self, save_dir: Path = Path("."), plot: bool = False, on_plot=None) -> Dict[str, np.ndarray]:
    """
    Process the detection and pose metrics over the given set of predictions.

    Args:
        save_dir (Path): Directory to save plots. Defaults to Path(".").
        plot (bool): Whether to plot precision-recall curves. Defaults to False.
        on_plot (callable, optional): Function to call after plots are generated.

    Returns:
        (Dict[str, np.ndarray]): Dictionary containing concatenated statistics arrays.
    """
    stats = DetMetrics.process(self, save_dir, plot, on_plot=on_plot)  # process box stats
    results_pose = ap_per_class(
        stats["tp_p"],
        stats["conf"],
        stats["pred_cls"],
        stats["target_cls"],
        plot=plot,
        on_plot=on_plot,
        save_dir=save_dir,
        names=self.names,
        prefix="Pose",
    )[2:]
    self.pose.nc = len(self.names)
    self.pose.update(results_pose)
    return stats