Add object detection, tracking, mobile notifications, and search to any security camera.
second bytetrack
rorycleard98122b76c08
diff --git a/test/test.py b/test/test.pyindex 2316508..49273c0 100644--- a/test/test.py+++ b/test/test.py@@ -46,6 +46,7 @@ def test_yolo_infer():import picklefrom yolox.tracker.byte_tracker import BYTETracker+from yolox.tracker.byte_tracker2 import BYTETracker2def test_bytetracker():class Args:@@ -54,14 +55,27 @@ def test_bytetracker():self.mot20 = Falseself.match_thresh = 0.9tracker = BYTETracker(Args())+ tracker2 = BYTETracker2(Args())inputs = pickle.load(open("test/tracker_inputs.pkl", "rb"))outputs = pickle.load(open("test/tracker_outputs.pkl", "rb"))+ total_time = 0+ total_time2 = 0for i in range(len(inputs)):+ st = time.perf_counter()output = tracker.update(inputs[i], [1280,1280], [1280,1280], threshold=0.5)+ total_time += (time.perf_counter() - st)+ st = time.perf_counter()+ output2 = tracker2.update(inputs[i], [1280,1280], [1280,1280], threshold=0.5)+ total_time2 += (time.perf_counter() - st)assert len(outputs[i]) == len(output)+ assert len(outputs[i]) == len(output2)for j in range(len(output)):np.testing.assert_allclose(output[j]._tlwh, outputs[i][j]._tlwh)+ np.testing.assert_allclose(output2[j]._tlwh, outputs[i][j]._tlwh)++ assert total_time2 < (total_time * 1.1), "too slow"+def setup_test_bytetracker():depth, width, ratio = get_variant_multiples("s")diff --git a/yolox/tracker/byte_tracker2.py b/yolox/tracker/byte_tracker2.pynew file mode 100644index 0000000..3f1b55f--- /dev/null+++ b/yolox/tracker/byte_tracker2.py@@ -0,0 +1,325 @@+import numpy as np++from .kalman_filter import KalmanFilter+from yolox.tracker import matching+from .basetrack import BaseTrack, TrackState+from collections import defaultdict++class STrack(BaseTrack):+ shared_kalman = KalmanFilter()+ def __init__(self, tlwh, score, class_id):+ self.occurrences = defaultdict(float)+ # wait activate+ self._tlwh = np.asarray(tlwh, dtype=np.float64)+ self.kalman_filter = None+ self.mean, self.covariance = None, None+ self.is_activated = False++ self.score = score+ self.class_id = class_id+ self.tracklet_len = 0++ def predict(self):+ mean_state = self.mean.copy()+ if self.state != TrackState.Tracked:+ mean_state[7] = 0+ self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance)++ @staticmethod+ def multi_predict(stracks):+ if len(stracks) > 0:+ multi_mean = np.asarray([st.mean.copy() for st in stracks])+ multi_covariance = np.asarray([st.covariance for st in stracks])+ for i, st in enumerate(stracks):+ if st.state != TrackState.Tracked:+ multi_mean[i][7] = 0+ multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(multi_mean, multi_covariance)+ for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)):+ stracks[i].mean = mean+ stracks[i].covariance = cov++ def activate(self, kalman_filter, frame_id):+ """Start a new tracklet"""+ self.kalman_filter = kalman_filter+ self.track_id = self.next_id()+ self.mean, self.covariance = self.kalman_filter.initiate(self.tlwh_to_xyah(self._tlwh))++ self.tracklet_len = 0+ self.state = TrackState.Tracked+ if frame_id == 1:+ self.is_activated = True+ # self.is_activated = True+ self.frame_id = frame_id+ self.start_frame = frame_id++ def re_activate(self, new_track, frame_id, new_id=False):+ self.mean, self.covariance = self.kalman_filter.update(+ self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh)+ )+ self.tracklet_len = 0+ self.state = TrackState.Tracked+ self.is_activated = True+ self.frame_id = frame_id+ if new_id:+ self.track_id = self.next_id()+ self.score = new_track.score++ def update(self, new_track, frame_id):+ """+ Update a matched track+ :type new_track: STrack+ :type frame_id: int+ :type update_feature: bool+ :return:+ """+ self.frame_id = frame_id+ self.tracklet_len += 1++ new_tlwh = new_track.tlwh+ self.mean, self.covariance = self.kalman_filter.update(+ self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh))+ self.state = TrackState.Tracked+ self.is_activated = True++ self.score = new_track.score+ self.occurrences[new_track.class_id] += new_track.score+ self.class_id = max(self.occurrences, key=self.occurrences.get) # use highest combined score++ @property+ # @jit(nopython=True)+ def tlwh(self):+ """Get current position in bounding box format `(top left x, top left y,+ width, height)`.+ """+ if self.mean is None:+ return self._tlwh.copy()+ ret = self.mean[:4].copy()+ ret[2] *= ret[3]+ ret[:2] -= ret[2:] / 2+ return ret++ @property+ # @jit(nopython=True)+ def tlbr(self):+ """Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,+ `(top left, bottom right)`.+ """+ ret = self.tlwh.copy()+ ret[2:] += ret[:2]+ return ret++ @staticmethod+ # @jit(nopython=True)+ def tlwh_to_xyah(tlwh):+ """Convert bounding box to format `(center x, center y, aspect ratio,+ height)`, where the aspect ratio is `width / height`.+ """+ ret = np.asarray(tlwh).copy()+ ret[:2] += ret[2:] / 2+ ret[2] /= ret[3]+ return ret++ def to_xyah(self):+ return self.tlwh_to_xyah(self.tlwh)++ @staticmethod+ # @jit(nopython=True)+ def tlbr_to_tlwh(tlbr):+ ret = np.asarray(tlbr).copy()+ ret[2:] -= ret[:2]+ return ret++ @staticmethod+ # @jit(nopython=True)+ def tlwh_to_tlbr(tlwh):+ ret = np.asarray(tlwh).copy()+ ret[2:] += ret[:2]+ return ret++ def __repr__(self):+ return 'OT_{}_({}-{})'.format(self.track_id, self.start_frame, self.end_frame)+++class BYTETracker2(object):+ def __init__(self, args, frame_rate=30):+ self.tracked_stracks = [] # type: list[STrack]+ self.lost_stracks = [] # type: list[STrack]+ self.removed_stracks = [] # type: list[STrack]++ self.frame_id = 0+ self.args = args+ self.buffer_size = int(frame_rate / 30.0 * args.track_buffer)+ self.max_time_lost = self.buffer_size+ self.kalman_filter = KalmanFilter()++ def update(self, output_results, img_info, img_size, threshold=0.5):+ self.frame_id += 1+ activated_starcks = []+ refind_stracks = []+ lost_stracks = []+ removed_stracks = []++ classes = output_results[:, 5]+ scores = output_results[:, 4]+ bboxes = output_results[:, :4]++ img_h, img_w = img_info[0], img_info[1]+ scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w))+ bboxes /= scale++ remain_inds = scores > threshold # prev track thresh+ inds_low = scores > 0.1+ inds_high = scores < threshold # prev track thresh++ inds_second = np.logical_and(inds_low, inds_high)+ dets_second = bboxes[inds_second]+ dets_keep = bboxes[remain_inds]+ scores_keep = scores[remain_inds]+ scores_second = scores[inds_second]+ classes_keep = classes[remain_inds]+ classes_second = classes[inds_second]++ if len(dets_keep) > 0:+ '''Detections'''+ detections = [STrack(STrack.tlbr_to_tlwh(tlbr), s, c) for+ (tlbr, s, c) in zip(dets_keep, scores_keep, classes_keep)]+ else:+ detections = []++ ''' Add newly detected tracklets to tracked_stracks'''+ unconfirmed = []+ tracked_stracks = [] # type: list[STrack]+ for track in self.tracked_stracks:+ if not track.is_activated:+ unconfirmed.append(track)+ else:+ tracked_stracks.append(track)++ ''' Step 2: First association, with high score detection boxes'''+ strack_pool = joint_stracks(tracked_stracks, self.lost_stracks)+ # Predict the current location with KF+ STrack.multi_predict(strack_pool)+ dists = matching.iou_distance(strack_pool, detections)+ if not self.args.mot20:+ dists = matching.fuse_score(dists, detections)+ matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.args.match_thresh)++ for itracked, idet in matches:+ track = strack_pool[itracked]+ det = detections[idet]+ if track.state == TrackState.Tracked:+ track.update(detections[idet], self.frame_id)+ activated_starcks.append(track)+ else:+ track.re_activate(det, self.frame_id, new_id=False)+ refind_stracks.append(track)++ ''' Step 3: Second association, with low score detection boxes'''+ # association the untrack to the low score detections+ if len(dets_second) > 0:+ '''Detections'''+ detections_second = [STrack(STrack.tlbr_to_tlwh(tlbr), s, c) for+ (tlbr, s, c) in zip(dets_second, scores_second, classes_second)]+ else:+ detections_second = []+ r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked]+ dists = matching.iou_distance(r_tracked_stracks, detections_second)+ matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=threshold) # prev 0.5+ for itracked, idet in matches:+ track = r_tracked_stracks[itracked]+ det = detections_second[idet]+ if track.state == TrackState.Tracked:+ track.update(det, self.frame_id)+ activated_starcks.append(track)+ else:+ track.re_activate(det, self.frame_id, new_id=False)+ refind_stracks.append(track)++ for it in u_track:+ track = r_tracked_stracks[it]+ if not track.state == TrackState.Lost:+ track.mark_lost()+ lost_stracks.append(track)++ '''Deal with unconfirmed tracks, usually tracks with only one beginning frame'''+ detections = [detections[i] for i in u_detection]+ dists = matching.iou_distance(unconfirmed, detections)+ if not self.args.mot20:+ dists = matching.fuse_score(dists, detections)+ matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7)+ for itracked, idet in matches:+ unconfirmed[itracked].update(detections[idet], self.frame_id)+ activated_starcks.append(unconfirmed[itracked])+ for it in u_unconfirmed:+ track = unconfirmed[it]+ track.mark_removed()+ removed_stracks.append(track)++ """ Step 4: Init new stracks"""+ for inew in u_detection:+ track = detections[inew]+ if track.score < threshold:+ continue+ track.activate(self.kalman_filter, self.frame_id)+ activated_starcks.append(track)+ """ Step 5: Update state"""+ for track in self.lost_stracks:+ if self.frame_id - track.end_frame > self.max_time_lost:+ track.mark_removed()+ removed_stracks.append(track)++ # print('Ramained match {} s'.format(t4-t3))++ self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked]+ self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks)+ self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks)+ self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks)+ self.lost_stracks.extend(lost_stracks)+ self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks)+ self.removed_stracks.extend(removed_stracks)+ self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks)+ # get scores of lost tracks+ output_stracks = [track for track in self.tracked_stracks if track.is_activated]++ return output_stracks+++def joint_stracks(tlista, tlistb):+ exists = {}+ res = []+ for t in tlista:+ exists[t.track_id] = 1+ res.append(t)+ for t in tlistb:+ tid = t.track_id+ if not exists.get(tid, 0):+ exists[tid] = 1+ res.append(t)+ return res+++def sub_stracks(tlista, tlistb):+ stracks = {}+ for t in tlista:+ stracks[t.track_id] = t+ for t in tlistb:+ tid = t.track_id+ if stracks.get(tid, 0):+ del stracks[tid]+ return list(stracks.values())+++def remove_duplicate_stracks(stracksa, stracksb):+ pdist = matching.iou_distance(stracksa, stracksb)+ pairs = np.where(pdist < 0.15)+ dupa, dupb = list(), list()+ for p, q in zip(*pairs):+ timep = stracksa[p].frame_id - stracksa[p].start_frame+ timeq = stracksb[q].frame_id - stracksb[q].start_frame+ if timep > timeq:+ dupb.append(q)+ else:+ dupa.append(p)+ resa = [t for i, t in enumerate(stracksa) if not i in dupa]+ resb = [t for i, t in enumerate(stracksb) if not i in dupb]+ return resa, resb\ No newline at end of file