Skip to content

Reference for ultralytics/trackers/byte_tracker.py

Note

This file is available at https://github.com/ultralytics/ultralytics/blob/main/ultralytics/trackers/byte_tracker.py. If you spot a problem please help fix it by contributing a Pull Request 🛠️. Thank you 🙏!


ultralytics.trackers.byte_tracker.STrack

STrack(xywh, score, cls)

Bases: BaseTrack

Single object tracking representation that uses Kalman filtering for state estimation.

This class is responsible for storing all the information regarding individual tracklets and performs state updates and predictions based on Kalman filter.

Attributes:

Name Type Description
shared_kalman KalmanFilterXYAH

Shared Kalman filter used across all STrack instances for prediction.

_tlwh ndarray

Private attribute to store top-left corner coordinates and width and height of bounding box.

kalman_filter KalmanFilterXYAH

Instance of Kalman filter used for this particular object track.

mean ndarray

Mean state estimate vector.

covariance ndarray

Covariance of state estimate.

is_activated bool

Boolean flag indicating if the track has been activated.

score float

Confidence score of the track.

tracklet_len int

Length of the tracklet.

cls Any

Class label for the object.

idx int

Index or identifier for the object.

frame_id int

Current frame ID.

start_frame int

Frame where the object was first detected.

Methods:

Name Description
predict

Predict the next state of the object using Kalman filter.

multi_predict

Predict the next states for multiple tracks.

multi_gmc

Update multiple track states using a homography matrix.

activate

Activate a new tracklet.

re_activate

Reactivate a previously lost tracklet.

update

Update the state of a matched track.

convert_coords

Convert bounding box to x-y-aspect-height format.

tlwh_to_xyah

Convert tlwh bounding box to xyah format.

Examples:

Initialize and activate a new track

>>> track = STrack(xywh=[100, 200, 50, 80, 0], score=0.9, cls="person")
>>> track.activate(kalman_filter=KalmanFilterXYAH(), frame_id=1)

Parameters:

Name Type Description Default
xywh List[float]

Bounding box coordinates and dimensions in the format (x, y, w, h, [a], idx), where (x, y) is the center, (w, h) are width and height, [a] is optional aspect ratio, and idx is the id.

required
score float

Confidence score of the detection.

required
cls Any

Class label for the detected object.

required

Examples:

>>> xywh = [100.0, 150.0, 50.0, 75.0, 1]
>>> score = 0.9
>>> cls = "person"
>>> track = STrack(xywh, score, cls)
Source code in ultralytics/trackers/byte_tracker.py
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
def __init__(self, xywh, score, cls):
    """
    Initialize a new STrack instance.

    Args:
        xywh (List[float]): Bounding box coordinates and dimensions in the format (x, y, w, h, [a], idx), where
            (x, y) is the center, (w, h) are width and height, [a] is optional aspect ratio, and idx is the id.
        score (float): Confidence score of the detection.
        cls (Any): Class label for the detected object.

    Examples:
        >>> xywh = [100.0, 150.0, 50.0, 75.0, 1]
        >>> score = 0.9
        >>> cls = "person"
        >>> track = STrack(xywh, score, cls)
    """
    super().__init__()
    # xywh+idx or xywha+idx
    assert len(xywh) in {5, 6}, f"expected 5 or 6 values but got {len(xywh)}"
    self._tlwh = np.asarray(xywh2ltwh(xywh[:4]), dtype=np.float32)
    self.kalman_filter = None
    self.mean, self.covariance = None, None
    self.is_activated = False

    self.score = score
    self.tracklet_len = 0
    self.cls = cls
    self.idx = xywh[-1]
    self.angle = xywh[4] if len(xywh) == 6 else None

result property

result

Returns the current tracking results in the appropriate bounding box format.

tlwh property

tlwh

Returns the bounding box in top-left-width-height format from the current state estimate.

xywh property

xywh

Returns the current position of the bounding box in (center x, center y, width, height) format.

xywha property

xywha

Returns position in (center x, center y, width, height, angle) format, warning if angle is missing.

xyxy property

xyxy

Converts bounding box from (top left x, top left y, width, height) to (min x, min y, max x, max y) format.

__repr__

__repr__()

Returns a string representation of the STrack object including start frame, end frame, and track ID.

Source code in ultralytics/trackers/byte_tracker.py
230
231
232
def __repr__(self):
    """Returns a string representation of the STrack object including start frame, end frame, and track ID."""
    return f"OT_{self.track_id}_({self.start_frame}-{self.end_frame})"

activate

activate(kalman_filter, frame_id)

Activate a new tracklet using the provided Kalman filter and initialize its state and covariance.

Source code in ultralytics/trackers/byte_tracker.py
122
123
124
125
126
127
128
129
130
131
132
133
def activate(self, kalman_filter, frame_id):
    """Activate a new tracklet using the provided Kalman filter and initialize its state and covariance."""
    self.kalman_filter = kalman_filter
    self.track_id = self.next_id()
    self.mean, self.covariance = self.kalman_filter.initiate(self.convert_coords(self._tlwh))

    self.tracklet_len = 0
    self.state = TrackState.Tracked
    if frame_id == 1:
        self.is_activated = True
    self.frame_id = frame_id
    self.start_frame = frame_id

convert_coords

convert_coords(tlwh)

Convert a bounding box's top-left-width-height format to its x-y-aspect-height equivalent.

Source code in ultralytics/trackers/byte_tracker.py
180
181
182
def convert_coords(self, tlwh):
    """Convert a bounding box's top-left-width-height format to its x-y-aspect-height equivalent."""
    return self.tlwh_to_xyah(tlwh)

multi_gmc staticmethod

multi_gmc(stracks, H=np.eye(2, 3))

Update state tracks positions and covariances using a homography matrix for multiple tracks.

Source code in ultralytics/trackers/byte_tracker.py
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
@staticmethod
def multi_gmc(stracks, H=np.eye(2, 3)):
    """Update state tracks positions and covariances using a homography matrix for multiple tracks."""
    if len(stracks) > 0:
        multi_mean = np.asarray([st.mean.copy() for st in stracks])
        multi_covariance = np.asarray([st.covariance for st in stracks])

        R = H[:2, :2]
        R8x8 = np.kron(np.eye(4, dtype=float), R)
        t = H[:2, 2]

        for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)):
            mean = R8x8.dot(mean)
            mean[:2] += t
            cov = R8x8.dot(cov).dot(R8x8.transpose())

            stracks[i].mean = mean
            stracks[i].covariance = cov

multi_predict staticmethod

multi_predict(stracks)

Perform multi-object predictive tracking using Kalman filter for the provided list of STrack instances.

Source code in ultralytics/trackers/byte_tracker.py
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
@staticmethod
def multi_predict(stracks):
    """Perform multi-object predictive tracking using Kalman filter for the provided list of STrack instances."""
    if len(stracks) <= 0:
        return
    multi_mean = np.asarray([st.mean.copy() for st in stracks])
    multi_covariance = np.asarray([st.covariance for st in stracks])
    for i, st in enumerate(stracks):
        if st.state != TrackState.Tracked:
            multi_mean[i][7] = 0
    multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(multi_mean, multi_covariance)
    for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)):
        stracks[i].mean = mean
        stracks[i].covariance = cov

predict

predict()

Predicts the next state (mean and covariance) of the object using the Kalman filter.

Source code in ultralytics/trackers/byte_tracker.py
81
82
83
84
85
86
def predict(self):
    """Predicts the next state (mean and covariance) of the object using the Kalman filter."""
    mean_state = self.mean.copy()
    if self.state != TrackState.Tracked:
        mean_state[7] = 0
    self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance)

re_activate

re_activate(new_track, frame_id, new_id=False)

Reactivates a previously lost track using new detection data and updates its state and attributes.

Source code in ultralytics/trackers/byte_tracker.py
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def re_activate(self, new_track, frame_id, new_id=False):
    """Reactivates a previously lost track using new detection data and updates its state and attributes."""
    self.mean, self.covariance = self.kalman_filter.update(
        self.mean, self.covariance, self.convert_coords(new_track.tlwh)
    )
    self.tracklet_len = 0
    self.state = TrackState.Tracked
    self.is_activated = True
    self.frame_id = frame_id
    if new_id:
        self.track_id = self.next_id()
    self.score = new_track.score
    self.cls = new_track.cls
    self.angle = new_track.angle
    self.idx = new_track.idx

tlwh_to_xyah staticmethod

tlwh_to_xyah(tlwh)

Convert bounding box from tlwh format to center-x-center-y-aspect-height (xyah) format.

Source code in ultralytics/trackers/byte_tracker.py
201
202
203
204
205
206
207
@staticmethod
def tlwh_to_xyah(tlwh):
    """Convert bounding box from tlwh format to center-x-center-y-aspect-height (xyah) format."""
    ret = np.asarray(tlwh).copy()
    ret[:2] += ret[2:] / 2
    ret[2] /= ret[3]
    return ret

update

update(new_track, frame_id)

Update the state of a matched track.

Parameters:

Name Type Description Default
new_track STrack

The new track containing updated information.

required
frame_id int

The ID of the current frame.

required

Examples:

Update the state of a track with new detection information

>>> track = STrack([100, 200, 50, 80, 0.9, 1])
>>> new_track = STrack([105, 205, 55, 85, 0.95, 1])
>>> track.update(new_track, 2)
Source code in ultralytics/trackers/byte_tracker.py
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
def update(self, new_track, frame_id):
    """
    Update the state of a matched track.

    Args:
        new_track (STrack): The new track containing updated information.
        frame_id (int): The ID of the current frame.

    Examples:
        Update the state of a track with new detection information
        >>> track = STrack([100, 200, 50, 80, 0.9, 1])
        >>> new_track = STrack([105, 205, 55, 85, 0.95, 1])
        >>> track.update(new_track, 2)
    """
    self.frame_id = frame_id
    self.tracklet_len += 1

    new_tlwh = new_track.tlwh
    self.mean, self.covariance = self.kalman_filter.update(
        self.mean, self.covariance, self.convert_coords(new_tlwh)
    )
    self.state = TrackState.Tracked
    self.is_activated = True

    self.score = new_track.score
    self.cls = new_track.cls
    self.angle = new_track.angle
    self.idx = new_track.idx





ultralytics.trackers.byte_tracker.BYTETracker

BYTETracker(args, frame_rate=30)

BYTETracker: A tracking algorithm built on top of YOLOv8 for object detection and tracking.

This class encapsulates the functionality for initializing, updating, and managing the tracks for detected objects in a video sequence. It maintains the state of tracked, lost, and removed tracks over frames, utilizes Kalman filtering for predicting the new object locations, and performs data association.

Attributes:

Name Type Description
tracked_stracks List[STrack]

List of successfully activated tracks.

lost_stracks List[STrack]

List of lost tracks.

removed_stracks List[STrack]

List of removed tracks.

frame_id int

The current frame ID.

args Namespace

Command-line arguments.

max_time_lost int

The maximum frames for a track to be considered as 'lost'.

kalman_filter KalmanFilterXYAH

Kalman Filter object.

Methods:

Name Description
update

Updates object tracker with new detections.

get_kalmanfilter

Returns a Kalman filter object for tracking bounding boxes.

init_track

Initialize object tracking with detections.

get_dists

Calculates the distance between tracks and detections.

multi_predict

Predicts the location of tracks.

reset_id

Resets the ID counter of STrack.

joint_stracks

Combines two lists of stracks.

sub_stracks

Filters out the stracks present in the second list from the first list.

remove_duplicate_stracks

Removes duplicate stracks based on IoU.

Examples:

Initialize BYTETracker and update with detection results

>>> tracker = BYTETracker(args, frame_rate=30)
>>> results = yolo_model.detect(image)
>>> tracked_objects = tracker.update(results)

Parameters:

Name Type Description Default
args Namespace

Command-line arguments containing tracking parameters.

required
frame_rate int

Frame rate of the video sequence.

30

Examples:

Initialize BYTETracker with command-line arguments and a frame rate of 30

>>> args = Namespace(track_buffer=30)
>>> tracker = BYTETracker(args, frame_rate=30)
Source code in ultralytics/trackers/byte_tracker.py
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
def __init__(self, args, frame_rate=30):
    """
    Initialize a BYTETracker instance for object tracking.

    Args:
        args (Namespace): Command-line arguments containing tracking parameters.
        frame_rate (int): Frame rate of the video sequence.

    Examples:
        Initialize BYTETracker with command-line arguments and a frame rate of 30
        >>> args = Namespace(track_buffer=30)
        >>> tracker = BYTETracker(args, frame_rate=30)
    """
    self.tracked_stracks = []  # type: list[STrack]
    self.lost_stracks = []  # type: list[STrack]
    self.removed_stracks = []  # type: list[STrack]

    self.frame_id = 0
    self.args = args
    self.max_time_lost = int(frame_rate / 30.0 * args.track_buffer)
    self.kalman_filter = self.get_kalmanfilter()
    self.reset_id()

get_dists

get_dists(tracks, detections)

Calculates the distance between tracks and detections using IoU and optionally fuses scores.

Source code in ultralytics/trackers/byte_tracker.py
415
416
417
418
419
420
def get_dists(self, tracks, detections):
    """Calculates the distance between tracks and detections using IoU and optionally fuses scores."""
    dists = matching.iou_distance(tracks, detections)
    if self.args.fuse_score:
        dists = matching.fuse_score(dists, detections)
    return dists

get_kalmanfilter

get_kalmanfilter()

Returns a Kalman filter object for tracking bounding boxes using KalmanFilterXYAH.

Source code in ultralytics/trackers/byte_tracker.py
407
408
409
def get_kalmanfilter(self):
    """Returns a Kalman filter object for tracking bounding boxes using KalmanFilterXYAH."""
    return KalmanFilterXYAH()

init_track

init_track(dets, scores, cls, img=None)

Initializes object tracking with given detections, scores, and class labels using the STrack algorithm.

Source code in ultralytics/trackers/byte_tracker.py
411
412
413
def init_track(self, dets, scores, cls, img=None):
    """Initializes object tracking with given detections, scores, and class labels using the STrack algorithm."""
    return [STrack(xyxy, s, c) for (xyxy, s, c) in zip(dets, scores, cls)] if len(dets) else []  # detections

joint_stracks staticmethod

joint_stracks(tlista, tlistb)

Combines two lists of STrack objects into a single list, ensuring no duplicates based on track IDs.

Source code in ultralytics/trackers/byte_tracker.py
440
441
442
443
444
445
446
447
448
449
450
451
452
453
@staticmethod
def joint_stracks(tlista, tlistb):
    """Combines two lists of STrack objects into a single list, ensuring no duplicates based on track IDs."""
    exists = {}
    res = []
    for t in tlista:
        exists[t.track_id] = 1
        res.append(t)
    for t in tlistb:
        tid = t.track_id
        if not exists.get(tid, 0):
            exists[tid] = 1
            res.append(t)
    return res

multi_predict

multi_predict(tracks)

Predict the next states for multiple tracks using Kalman filter.

Source code in ultralytics/trackers/byte_tracker.py
422
423
424
def multi_predict(self, tracks):
    """Predict the next states for multiple tracks using Kalman filter."""
    STrack.multi_predict(tracks)

remove_duplicate_stracks staticmethod

remove_duplicate_stracks(stracksa, stracksb)

Removes duplicate stracks from two lists based on Intersection over Union (IoU) distance.

Source code in ultralytics/trackers/byte_tracker.py
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
@staticmethod
def remove_duplicate_stracks(stracksa, stracksb):
    """Removes duplicate stracks from two lists based on Intersection over Union (IoU) distance."""
    pdist = matching.iou_distance(stracksa, stracksb)
    pairs = np.where(pdist < 0.15)
    dupa, dupb = [], []
    for p, q in zip(*pairs):
        timep = stracksa[p].frame_id - stracksa[p].start_frame
        timeq = stracksb[q].frame_id - stracksb[q].start_frame
        if timep > timeq:
            dupb.append(q)
        else:
            dupa.append(p)
    resa = [t for i, t in enumerate(stracksa) if i not in dupa]
    resb = [t for i, t in enumerate(stracksb) if i not in dupb]
    return resa, resb

reset

reset()

Resets the tracker by clearing all tracked, lost, and removed tracks and reinitializing the Kalman filter.

Source code in ultralytics/trackers/byte_tracker.py
431
432
433
434
435
436
437
438
def reset(self):
    """Resets the tracker by clearing all tracked, lost, and removed tracks and reinitializing the Kalman filter."""
    self.tracked_stracks = []  # type: list[STrack]
    self.lost_stracks = []  # type: list[STrack]
    self.removed_stracks = []  # type: list[STrack]
    self.frame_id = 0
    self.kalman_filter = self.get_kalmanfilter()
    self.reset_id()

reset_id staticmethod

reset_id()

Resets the ID counter for STrack instances to ensure unique track IDs across tracking sessions.

Source code in ultralytics/trackers/byte_tracker.py
426
427
428
429
@staticmethod
def reset_id():
    """Resets the ID counter for STrack instances to ensure unique track IDs across tracking sessions."""
    STrack.reset_id()

sub_stracks staticmethod

sub_stracks(tlista, tlistb)

Filters out the stracks present in the second list from the first list.

Source code in ultralytics/trackers/byte_tracker.py
455
456
457
458
459
@staticmethod
def sub_stracks(tlista, tlistb):
    """Filters out the stracks present in the second list from the first list."""
    track_ids_b = {t.track_id for t in tlistb}
    return [t for t in tlista if t.track_id not in track_ids_b]

update

update(results, img=None, feats=None)

Updates the tracker with new detections and returns the current list of tracked objects.

Source code in ultralytics/trackers/byte_tracker.py
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
def update(self, results, img=None, feats=None):
    """Updates the tracker with new detections and returns the current list of tracked objects."""
    self.frame_id += 1
    activated_stracks = []
    refind_stracks = []
    lost_stracks = []
    removed_stracks = []

    scores = results.conf
    bboxes = results.xywhr if hasattr(results, "xywhr") else results.xywh
    # Add index
    bboxes = np.concatenate([bboxes, np.arange(len(bboxes)).reshape(-1, 1)], axis=-1)
    cls = results.cls

    remain_inds = scores >= self.args.track_high_thresh
    inds_low = scores > self.args.track_low_thresh
    inds_high = scores < self.args.track_high_thresh

    inds_second = inds_low & inds_high
    dets_second = bboxes[inds_second]
    dets = bboxes[remain_inds]
    scores_keep = scores[remain_inds]
    scores_second = scores[inds_second]
    cls_keep = cls[remain_inds]
    cls_second = cls[inds_second]

    detections = self.init_track(dets, scores_keep, cls_keep, img if feats is None else feats)
    # Add newly detected tracklets to tracked_stracks
    unconfirmed = []
    tracked_stracks = []  # type: list[STrack]
    for track in self.tracked_stracks:
        if not track.is_activated:
            unconfirmed.append(track)
        else:
            tracked_stracks.append(track)
    # Step 2: First association, with high score detection boxes
    strack_pool = self.joint_stracks(tracked_stracks, self.lost_stracks)
    # Predict the current location with KF
    self.multi_predict(strack_pool)
    if hasattr(self, "gmc") and img is not None:
        warp = self.gmc.apply(img, dets)
        STrack.multi_gmc(strack_pool, warp)
        STrack.multi_gmc(unconfirmed, warp)

    dists = self.get_dists(strack_pool, detections)
    matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.args.match_thresh)

    for itracked, idet in matches:
        track = strack_pool[itracked]
        det = detections[idet]
        if track.state == TrackState.Tracked:
            track.update(det, self.frame_id)
            activated_stracks.append(track)
        else:
            track.re_activate(det, self.frame_id, new_id=False)
            refind_stracks.append(track)
    # Step 3: Second association, with low score detection boxes association the untrack to the low score detections
    detections_second = self.init_track(dets_second, scores_second, cls_second, img if feats is None else feats)
    r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked]
    # TODO
    dists = matching.iou_distance(r_tracked_stracks, detections_second)
    matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.5)
    for itracked, idet in matches:
        track = r_tracked_stracks[itracked]
        det = detections_second[idet]
        if track.state == TrackState.Tracked:
            track.update(det, self.frame_id)
            activated_stracks.append(track)
        else:
            track.re_activate(det, self.frame_id, new_id=False)
            refind_stracks.append(track)

    for it in u_track:
        track = r_tracked_stracks[it]
        if track.state != TrackState.Lost:
            track.mark_lost()
            lost_stracks.append(track)
    # Deal with unconfirmed tracks, usually tracks with only one beginning frame
    detections = [detections[i] for i in u_detection]
    dists = self.get_dists(unconfirmed, detections)
    matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7)
    for itracked, idet in matches:
        unconfirmed[itracked].update(detections[idet], self.frame_id)
        activated_stracks.append(unconfirmed[itracked])
    for it in u_unconfirmed:
        track = unconfirmed[it]
        track.mark_removed()
        removed_stracks.append(track)
    # Step 4: Init new stracks
    for inew in u_detection:
        track = detections[inew]
        if track.score < self.args.new_track_thresh:
            continue
        track.activate(self.kalman_filter, self.frame_id)
        activated_stracks.append(track)
    # Step 5: Update state
    for track in self.lost_stracks:
        if self.frame_id - track.end_frame > self.max_time_lost:
            track.mark_removed()
            removed_stracks.append(track)

    self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked]
    self.tracked_stracks = self.joint_stracks(self.tracked_stracks, activated_stracks)
    self.tracked_stracks = self.joint_stracks(self.tracked_stracks, refind_stracks)
    self.lost_stracks = self.sub_stracks(self.lost_stracks, self.tracked_stracks)
    self.lost_stracks.extend(lost_stracks)
    self.lost_stracks = self.sub_stracks(self.lost_stracks, self.removed_stracks)
    self.tracked_stracks, self.lost_stracks = self.remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks)
    self.removed_stracks.extend(removed_stracks)
    if len(self.removed_stracks) > 1000:
        self.removed_stracks = self.removed_stracks[-999:]  # clip remove stracks to 1000 maximum

    return np.asarray([x.result for x in self.tracked_stracks if x.is_activated], dtype=np.float32)





📅 Created 1 year ago ✏️ Updated 7 months ago