Skip to content

Add object detection, tracking, mobile notifications, and search to any security camera.

second bytetrack

rorycleard98122b76c08
diff --git a/test/test.py b/test/test.py
index 2316508..49273c0 100644
--- a/test/test.py
+++ b/test/test.py
@@ -46,6 +46,7 @@ def test_yolo_infer():
import pickle
from yolox.tracker.byte_tracker import BYTETracker
+from yolox.tracker.byte_tracker2 import BYTETracker2
def test_bytetracker():
class Args:
@@ -54,14 +55,27 @@ def test_bytetracker():
self.mot20 = False
self.match_thresh = 0.9
tracker = BYTETracker(Args())
+ tracker2 = BYTETracker2(Args())
inputs = pickle.load(open("test/tracker_inputs.pkl", "rb"))
outputs = pickle.load(open("test/tracker_outputs.pkl", "rb"))
+ total_time = 0
+ total_time2 = 0
for i in range(len(inputs)):
+ st = time.perf_counter()
output = tracker.update(inputs[i], [1280,1280], [1280,1280], threshold=0.5)
+ total_time += (time.perf_counter() - st)
+ st = time.perf_counter()
+ output2 = tracker2.update(inputs[i], [1280,1280], [1280,1280], threshold=0.5)
+ total_time2 += (time.perf_counter() - st)
assert len(outputs[i]) == len(output)
+ assert len(outputs[i]) == len(output2)
for j in range(len(output)):
np.testing.assert_allclose(output[j]._tlwh, outputs[i][j]._tlwh)
+ np.testing.assert_allclose(output2[j]._tlwh, outputs[i][j]._tlwh)
+
+ assert total_time2 < (total_time * 1.1), "too slow"
+
def setup_test_bytetracker():
depth, width, ratio = get_variant_multiples("s")
diff --git a/yolox/tracker/byte_tracker2.py b/yolox/tracker/byte_tracker2.py
new file mode 100644
index 0000000..3f1b55f
--- /dev/null
+++ b/yolox/tracker/byte_tracker2.py
@@ -0,0 +1,325 @@
+import numpy as np
+
+from .kalman_filter import KalmanFilter
+from yolox.tracker import matching
+from .basetrack import BaseTrack, TrackState
+from collections import defaultdict
+
+class STrack(BaseTrack):
+ shared_kalman = KalmanFilter()
+ def __init__(self, tlwh, score, class_id):
+ self.occurrences = defaultdict(float)
+ # wait activate
+ self._tlwh = np.asarray(tlwh, dtype=np.float64)
+ self.kalman_filter = None
+ self.mean, self.covariance = None, None
+ self.is_activated = False
+
+ self.score = score
+ self.class_id = class_id
+ self.tracklet_len = 0
+
+ def predict(self):
+ mean_state = self.mean.copy()
+ if self.state != TrackState.Tracked:
+ mean_state[7] = 0
+ self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance)
+
+ @staticmethod
+ def multi_predict(stracks):
+ if len(stracks) > 0:
+ multi_mean = np.asarray([st.mean.copy() for st in stracks])
+ multi_covariance = np.asarray([st.covariance for st in stracks])
+ for i, st in enumerate(stracks):
+ if st.state != TrackState.Tracked:
+ multi_mean[i][7] = 0
+ multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(multi_mean, multi_covariance)
+ for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)):
+ stracks[i].mean = mean
+ stracks[i].covariance = cov
+
+ def activate(self, kalman_filter, frame_id):
+ """Start a new tracklet"""
+ self.kalman_filter = kalman_filter
+ self.track_id = self.next_id()
+ self.mean, self.covariance = self.kalman_filter.initiate(self.tlwh_to_xyah(self._tlwh))
+
+ self.tracklet_len = 0
+ self.state = TrackState.Tracked
+ if frame_id == 1:
+ self.is_activated = True
+ # self.is_activated = True
+ self.frame_id = frame_id
+ self.start_frame = frame_id
+
+ def re_activate(self, new_track, frame_id, new_id=False):
+ self.mean, self.covariance = self.kalman_filter.update(
+ self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh)
+ )
+ self.tracklet_len = 0
+ self.state = TrackState.Tracked
+ self.is_activated = True
+ self.frame_id = frame_id
+ if new_id:
+ self.track_id = self.next_id()
+ self.score = new_track.score
+
+ def update(self, new_track, frame_id):
+ """
+ Update a matched track
+ :type new_track: STrack
+ :type frame_id: int
+ :type update_feature: bool
+ :return:
+ """
+ self.frame_id = frame_id
+ self.tracklet_len += 1
+
+ new_tlwh = new_track.tlwh
+ self.mean, self.covariance = self.kalman_filter.update(
+ self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh))
+ self.state = TrackState.Tracked
+ self.is_activated = True
+
+ self.score = new_track.score
+ self.occurrences[new_track.class_id] += new_track.score
+ self.class_id = max(self.occurrences, key=self.occurrences.get) # use highest combined score
+
+ @property
+ # @jit(nopython=True)
+ def tlwh(self):
+ """Get current position in bounding box format `(top left x, top left y,
+ width, height)`.
+ """
+ if self.mean is None:
+ return self._tlwh.copy()
+ ret = self.mean[:4].copy()
+ ret[2] *= ret[3]
+ ret[:2] -= ret[2:] / 2
+ return ret
+
+ @property
+ # @jit(nopython=True)
+ def tlbr(self):
+ """Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,
+ `(top left, bottom right)`.
+ """
+ ret = self.tlwh.copy()
+ ret[2:] += ret[:2]
+ return ret
+
+ @staticmethod
+ # @jit(nopython=True)
+ def tlwh_to_xyah(tlwh):
+ """Convert bounding box to format `(center x, center y, aspect ratio,
+ height)`, where the aspect ratio is `width / height`.
+ """
+ ret = np.asarray(tlwh).copy()
+ ret[:2] += ret[2:] / 2
+ ret[2] /= ret[3]
+ return ret
+
+ def to_xyah(self):
+ return self.tlwh_to_xyah(self.tlwh)
+
+ @staticmethod
+ # @jit(nopython=True)
+ def tlbr_to_tlwh(tlbr):
+ ret = np.asarray(tlbr).copy()
+ ret[2:] -= ret[:2]
+ return ret
+
+ @staticmethod
+ # @jit(nopython=True)
+ def tlwh_to_tlbr(tlwh):
+ ret = np.asarray(tlwh).copy()
+ ret[2:] += ret[:2]
+ return ret
+
+ def __repr__(self):
+ return 'OT_{}_({}-{})'.format(self.track_id, self.start_frame, self.end_frame)
+
+
+class BYTETracker2(object):
+ def __init__(self, args, frame_rate=30):
+ self.tracked_stracks = [] # type: list[STrack]
+ self.lost_stracks = [] # type: list[STrack]
+ self.removed_stracks = [] # type: list[STrack]
+
+ self.frame_id = 0
+ self.args = args
+ self.buffer_size = int(frame_rate / 30.0 * args.track_buffer)
+ self.max_time_lost = self.buffer_size
+ self.kalman_filter = KalmanFilter()
+
+ def update(self, output_results, img_info, img_size, threshold=0.5):
+ self.frame_id += 1
+ activated_starcks = []
+ refind_stracks = []
+ lost_stracks = []
+ removed_stracks = []
+
+ classes = output_results[:, 5]
+ scores = output_results[:, 4]
+ bboxes = output_results[:, :4]
+
+ img_h, img_w = img_info[0], img_info[1]
+ scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w))
+ bboxes /= scale
+
+ remain_inds = scores > threshold # prev track thresh
+ inds_low = scores > 0.1
+ inds_high = scores < threshold # prev track thresh
+
+ inds_second = np.logical_and(inds_low, inds_high)
+ dets_second = bboxes[inds_second]
+ dets_keep = bboxes[remain_inds]
+ scores_keep = scores[remain_inds]
+ scores_second = scores[inds_second]
+ classes_keep = classes[remain_inds]
+ classes_second = classes[inds_second]
+
+ if len(dets_keep) > 0:
+ '''Detections'''
+ detections = [STrack(STrack.tlbr_to_tlwh(tlbr), s, c) for
+ (tlbr, s, c) in zip(dets_keep, scores_keep, classes_keep)]
+ else:
+ detections = []
+
+ ''' Add newly detected tracklets to tracked_stracks'''
+ unconfirmed = []
+ tracked_stracks = [] # type: list[STrack]
+ for track in self.tracked_stracks:
+ if not track.is_activated:
+ unconfirmed.append(track)
+ else:
+ tracked_stracks.append(track)
+
+ ''' Step 2: First association, with high score detection boxes'''
+ strack_pool = joint_stracks(tracked_stracks, self.lost_stracks)
+ # Predict the current location with KF
+ STrack.multi_predict(strack_pool)
+ dists = matching.iou_distance(strack_pool, detections)
+ if not self.args.mot20:
+ dists = matching.fuse_score(dists, detections)
+ matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.args.match_thresh)
+
+ for itracked, idet in matches:
+ track = strack_pool[itracked]
+ det = detections[idet]
+ if track.state == TrackState.Tracked:
+ track.update(detections[idet], self.frame_id)
+ activated_starcks.append(track)
+ else:
+ track.re_activate(det, self.frame_id, new_id=False)
+ refind_stracks.append(track)
+
+ ''' Step 3: Second association, with low score detection boxes'''
+ # association the untrack to the low score detections
+ if len(dets_second) > 0:
+ '''Detections'''
+ detections_second = [STrack(STrack.tlbr_to_tlwh(tlbr), s, c) for
+ (tlbr, s, c) in zip(dets_second, scores_second, classes_second)]
+ else:
+ detections_second = []
+ r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked]
+ dists = matching.iou_distance(r_tracked_stracks, detections_second)
+ matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=threshold) # prev 0.5
+ for itracked, idet in matches:
+ track = r_tracked_stracks[itracked]
+ det = detections_second[idet]
+ if track.state == TrackState.Tracked:
+ track.update(det, self.frame_id)
+ activated_starcks.append(track)
+ else:
+ track.re_activate(det, self.frame_id, new_id=False)
+ refind_stracks.append(track)
+
+ for it in u_track:
+ track = r_tracked_stracks[it]
+ if not track.state == TrackState.Lost:
+ track.mark_lost()
+ lost_stracks.append(track)
+
+ '''Deal with unconfirmed tracks, usually tracks with only one beginning frame'''
+ detections = [detections[i] for i in u_detection]
+ dists = matching.iou_distance(unconfirmed, detections)
+ if not self.args.mot20:
+ dists = matching.fuse_score(dists, detections)
+ matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7)
+ for itracked, idet in matches:
+ unconfirmed[itracked].update(detections[idet], self.frame_id)
+ activated_starcks.append(unconfirmed[itracked])
+ for it in u_unconfirmed:
+ track = unconfirmed[it]
+ track.mark_removed()
+ removed_stracks.append(track)
+
+ """ Step 4: Init new stracks"""
+ for inew in u_detection:
+ track = detections[inew]
+ if track.score < threshold:
+ continue
+ track.activate(self.kalman_filter, self.frame_id)
+ activated_starcks.append(track)
+ """ Step 5: Update state"""
+ for track in self.lost_stracks:
+ if self.frame_id - track.end_frame > self.max_time_lost:
+ track.mark_removed()
+ removed_stracks.append(track)
+
+ # print('Ramained match {} s'.format(t4-t3))
+
+ self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked]
+ self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks)
+ self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks)
+ self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks)
+ self.lost_stracks.extend(lost_stracks)
+ self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks)
+ self.removed_stracks.extend(removed_stracks)
+ self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks)
+ # get scores of lost tracks
+ output_stracks = [track for track in self.tracked_stracks if track.is_activated]
+
+ return output_stracks
+
+
+def joint_stracks(tlista, tlistb):
+ exists = {}
+ res = []
+ for t in tlista:
+ exists[t.track_id] = 1
+ res.append(t)
+ for t in tlistb:
+ tid = t.track_id
+ if not exists.get(tid, 0):
+ exists[tid] = 1
+ res.append(t)
+ return res
+
+
+def sub_stracks(tlista, tlistb):
+ stracks = {}
+ for t in tlista:
+ stracks[t.track_id] = t
+ for t in tlistb:
+ tid = t.track_id
+ if stracks.get(tid, 0):
+ del stracks[tid]
+ return list(stracks.values())
+
+
+def remove_duplicate_stracks(stracksa, stracksb):
+ pdist = matching.iou_distance(stracksa, stracksb)
+ pairs = np.where(pdist < 0.15)
+ dupa, dupb = list(), list()
+ for p, q in zip(*pairs):
+ timep = stracksa[p].frame_id - stracksa[p].start_frame
+ timeq = stracksb[q].frame_id - stracksb[q].start_frame
+ if timep > timeq:
+ dupb.append(q)
+ else:
+ dupa.append(p)
+ resa = [t for i, t in enumerate(stracksa) if not i in dupa]
+ resb = [t for i, t in enumerate(stracksb) if not i in dupb]
+ return resa, resb
\ No newline at end of file