Add object detection, tracking, mobile notifications, and search to any security camera.
add tinybytetrack prog
rorycleareaedb7af06c0
diff --git a/experimental/bytetrack.py b/experimental/bytetrack.pynew file mode 100644index 0000000..cad097f--- /dev/null+++ b/experimental/bytetrack.py@@ -0,0 +1,1277 @@+from tinygrad.nn import Conv2d, BatchNorm2d+from tinygrad.tensor import Tensor+from tinygrad.device import is_dtype_supported+from tinygrad import dtypes+import numpy as np+from itertools import chain+from pathlib import Path+import cv2+from collections import defaultdict+import time, sys+from tinygrad.helpers import fetch+from tinygrad.nn.state import safe_load, load_state_dict+import json+from tinygrad import TinyJit+import lap+from collections import OrderedDict+import pickle+++class KalmanFilter(object):+ def __init__(self):+ ndim, dt = 4, 1.+ self._motion_mat = np.eye(2 * ndim, 2 * ndim)+ for i in range(ndim):+ self._motion_mat[i, ndim + i] = dt+ self._update_mat = np.eye(ndim, 2 * ndim)+ self._std_weight_position = 1. / 20+ self._std_weight_velocity = 1. / 160++ def initiate(self, measurement):+ mean_pos = measurement+ mean_vel = np.zeros_like(mean_pos)+ mean = np.r_[mean_pos, mean_vel]++ std = [+ 2 * self._std_weight_position * measurement[3],+ 2 * self._std_weight_position * measurement[3],+ 1e-2,+ 2 * self._std_weight_position * measurement[3],+ 10 * self._std_weight_velocity * measurement[3],+ 10 * self._std_weight_velocity * measurement[3],+ 1e-5,+ 10 * self._std_weight_velocity * measurement[3]]+ covariance = np.diag(np.square(std))+ return mean, covariance++ def project(self, mean, covariance):+ std = [+ self._std_weight_position * mean[3],+ self._std_weight_position * mean[3],+ 1e-1,+ self._std_weight_position * mean[3]]+ innovation_cov = np.diag(np.square(std))++ mean = np.dot(self._update_mat, mean)+ covariance = np.linalg.multi_dot((+ self._update_mat, covariance, self._update_mat.T))+ return mean, covariance + innovation_cov++ def multi_predict(self, mean, covariance):+ std_pos = [+ self._std_weight_position * mean[:, 3],+ self._std_weight_position * mean[:, 3],+ 1e-2 * np.ones_like(mean[:, 3]),+ self._std_weight_position * mean[:, 3]]+ std_vel = [+ self._std_weight_velocity * mean[:, 3],+ self._std_weight_velocity * mean[:, 3],+ 1e-5 * np.ones_like(mean[:, 3]),+ self._std_weight_velocity * mean[:, 3]]+ sqr = np.square(np.r_[std_pos, std_vel]).T++ motion_cov = []+ for i in range(len(mean)):+ motion_cov.append(np.diag(sqr[i]))+ motion_cov = np.asarray(motion_cov)++ mean = np.dot(mean, self._motion_mat.T)+ left = np.dot(self._motion_mat, covariance).transpose((1, 0, 2))+ covariance = np.dot(left, self._motion_mat.T) + motion_cov++ return mean, covariance++ def update(self, mean, covariance, measurement):+ projected_mean, projected_cov = self.project(mean, covariance)++ chol_factor = np.linalg.cholesky(projected_cov)+ y = np.linalg.solve(chol_factor, np.dot(covariance, self._update_mat.T).T)+ kalman_gain = np.linalg.solve(chol_factor.T, y).T+ innovation = measurement - projected_mean+ new_mean = mean + np.dot(innovation, kalman_gain.T)+ new_covariance = covariance - np.linalg.multi_dot((+ kalman_gain, projected_cov, kalman_gain.T))+ return new_mean, new_covariance++class TrackState(object):+ New = 1+ Tracked = 2+ Lost = 3+ Removed = 4+ Replaced = 5++def tlbr_np(values, mean):+ """Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,+ `(top left, bottom right)`."""+ if mean is None:+ ret = values[:4].copy()+ else:+ ret = mean[:4].copy()+ ret[2] *= ret[3]+ ret[:2] -= ret[2:] / 2+ ret[2:] += ret[:2]+ return ret++def tlbr_np_batch(tracked_stracks_values, tracked_stracks_means):+ ret = np.empty((len(tracked_stracks_values), 4))+ use_mean = np.array([m is not None for m in tracked_stracks_means], dtype=bool)+ if np.any(use_mean):+ means = np.array([m for m in tracked_stracks_means if m is not None])+ ret[use_mean, :] = means[:, :4].copy()+ ret[use_mean, 2] *= ret[use_mean, 3]+ ret[use_mean, :2] -= ret[use_mean, 2:] / 2+ if not np.all(use_mean):+ ret[~use_mean, :] = tracked_stracks_values[~use_mean, :4].copy()+ ret[:, 2:] += ret[:, :2]+ return ret++def tlwh_np(values,mean):+ """Get current position in bounding box format `(top left x, top left y,+ width, height)`.+ """+ if mean is None:+ return values[:4].copy()+ ret = mean[:4].copy()+ ret[2] *= ret[3]+ ret[:2] -= ret[2:] / 2+ return ret++def tlwh_to_xyah(tlwh):+ """Convert bounding box to format `(center x, center y, aspect ratio,+ height)`, where the aspect ratio is `width / height`.+ """+ ret = np.asarray(tlwh).copy()+ ret[:2] += ret[2:] / 2+ ret[2] /= ret[3]+ return ret++def bbox_ious(boxes, query_boxes):+ N = boxes.shape[0]+ K = query_boxes.shape[0]++ boxes_area = (+ (boxes[:, 2] - boxes[:, 0] + 1) *+ (boxes[:, 3] - boxes[:, 1] + 1)+ ).reshape(N, 1)++ query_areas = (+ (query_boxes[:, 2] - query_boxes[:, 0] + 1) *+ (query_boxes[:, 3] - query_boxes[:, 1] + 1)+ ).reshape(1, K)++ ixmin = np.maximum(boxes[:, 0].reshape(N, 1), query_boxes[:, 0].reshape(1, K))+ iymin = np.maximum(boxes[:, 1].reshape(N, 1), query_boxes[:, 1].reshape(1, K))+ ixmax = np.minimum(boxes[:, 2].reshape(N, 1), query_boxes[:, 2].reshape(1, K))+ iymax = np.minimum(boxes[:, 3].reshape(N, 1), query_boxes[:, 3].reshape(1, K))++ iw = np.maximum(ixmax - ixmin + 1, 0)+ ih = np.maximum(iymax - iymin + 1, 0)+ intersection = iw * ih++ union = boxes_area + query_areas - intersection++ overlaps = np.where(+ (iw > 0) & (ih > 0),+ intersection / union,+ np.zeros_like(intersection)+ )+ return overlaps++class BYTETracker(object):+ def __init__(self, args, frame_rate=30):+ self._count = 0++ self.lost_stracks_ids_tg = Tensor.empty()+ self.lost_stracks_fids_tg = Tensor.empty()+ self.lost_stracks_startframes_tg = Tensor.empty()+ self.lost_stracks_states_tg = Tensor.empty()+ self.lost_stracks_bools_tg = Tensor.empty()+ self.lost_stracks_values_tg = Tensor.empty()+ self.lost_stracks_means_tg = Tensor.empty()+ self.lost_stracks_covs_tg = Tensor.empty()++ self.tracked_stracks_values = []+ self.lost_stracks_values = []++ self.tracked_stracks_means = []+ self.tracked_stracks_bools = []+ self.tracked_stracks_covs = []+ self.tracked_stracks_ids = []+ self.tracked_stracks_fids = []+ self.tracked_stracks_startframes = []+ self.tracked_stracks_states = []+ self.lost_stracks_means = []+ self.lost_stracks_bools = []+ self.lost_stracks_covs = []+ self.lost_stracks_ids = []+ self.lost_stracks_fids = []+ self.lost_stracks_startframes = []+ self.lost_stracks_states = []++ self.frame_id = 0+ self.args = args+ self.det_thresh = args.track_thresh + 0.1+ self.buffer_size = int(frame_rate / 30.0 * args.track_buffer)+ self.max_time_lost = self.buffer_size+ self.kalman_filter = KalmanFilter()++ def update(self, output_results, img_info, img_size):+ self.frame_id += 1+ activated_stracks_values = []+ activated_stracks_means = []+ activated_stracks_bools = []+ activated_stracks_covs = []+ activated_stracks_ids = []+ activated_stracks_fids = []+ activated_stracks_startframes = []+ activated_stracks_states = []+ refind_stracks_values = []+ refind_stracks_means = []+ refind_stracks_bools = []+ refind_stracks_covs = []+ refind_stracks_ids = []+ refind_stracks_fids = []+ refind_stracks_startframes = []+ refind_stracks_states = []+ lost_stracks_means = []+ lost_stracks_bools = []+ lost_stracks_covs = []+ lost_stracks_values = []+ lost_stracks_startframes = []+ lost_stracks_states = []+ lost_stracks_ids = []+ lost_stracks_fids = []+ removed_stracks_ids = []++ classes = output_results[:, 5]+ scores = output_results[:, 4]+ bboxes = output_results[:, :4]++ img_h, img_w = img_info[0], img_info[1]+ scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w))+ bboxes = bboxes / scale++ remain_inds = scores > self.args.track_thresh+ inds_low = scores > 0.1+ inds_high = scores < self.args.track_thresh+ inds_second = inds_low & inds_high++ dets = bboxes * remain_inds.unsqueeze(1)+ dets_second = bboxes * inds_second.unsqueeze(1)+ dets[:, 2:] -= dets[:, :2] #tlbr to tlwh+ dets_second[:, 2:] -= dets_second[:, :2]+ dets_score_classes = dets.cat(scores.reshape(-1,1), dim=1).cat(classes.reshape(-1,1), dim=1)+ dets_score_classes = dets_score_classes.numpy()+ dets_score_classes_second = dets_second.cat(scores.reshape(-1,1), dim=1).cat(classes.reshape(-1,1), dim=1)+ dets_score_classes_second = dets_score_classes_second.numpy()+ detections_means = [None for _ in dets_score_classes]+ detections_bools = [False for _ in dets_score_classes_second]+ detections_cov = [None for _ in dets_score_classes]+ detections_ids = [None for _ in dets_score_classes]+ detections_fids = [None for _ in dets_score_classes]+ detections_startframes = [0 for _ in dets_score_classes]+ detections_states = [TrackState.New for _ in dets_score_classes]++ tracked_stracks_means = []+ tracked_stracks_covs = []++ mask = np.array(self.tracked_stracks_bools).astype(bool)+ mask_tg = Tensor(mask)++ for i in range(len(self.tracked_stracks_ids)):+ mean = self.tracked_stracks_means[i]+ cov = self.tracked_stracks_covs[i]+ if self.tracked_stracks_bools[i]:+ tracked_stracks_means.append(mean)+ tracked_stracks_covs.append(cov)++ self.tracked_stracks_ids_tg = Tensor(self.tracked_stracks_ids)+ self.tracked_stracks_fids_tg = Tensor(self.tracked_stracks_fids)+ self.tracked_stracks_bools_tg = Tensor(self.tracked_stracks_bools)+ self.tracked_stracks_startframes_tg = Tensor(self.tracked_stracks_startframes)+ self.tracked_stracks_states_tg = Tensor(self.tracked_stracks_states)+ self.tracked_stracks_values_tg = Tensor(self.tracked_stracks_values)+ self.tracked_stracks_covs_tg = Tensor(self.tracked_stracks_covs)+ self.tracked_stracks_means_tg = Tensor(self.tracked_stracks_means)++ tracked_stracks_ids_tg = (self.tracked_stracks_ids_tg * mask_tg)+ tracked_stracks_fids_tg = (self.tracked_stracks_fids_tg * mask_tg)+ tracked_stracks_bools_tg = (self.tracked_stracks_bools_tg * mask_tg)+ tracked_stracks_startframes_tg = (self.tracked_stracks_startframes_tg * mask_tg)+ tracked_stracks_states_tg = (self.tracked_stracks_states_tg * mask_tg)+ tracked_stracks_values_tg = self.tracked_stracks_values_tg * mask_tg.view(-1,1)++ unconfirmed_ids_tg = self.tracked_stracks_ids_tg * ~mask_tg+ unconfirmed_values_tg = self.tracked_stracks_values_tg * ~mask_tg.view(-1,1)+ unconfirmed_bools_tg = self.tracked_stracks_bools_tg * ~mask_tg+ unconfirmed_states_tg = self.tracked_stracks_states_tg * ~mask_tg+ unconfirmed_fids_tg = self.tracked_stracks_fids_tg * ~mask_tg+ unconfirmed_startframes_tg = self.tracked_stracks_startframes_tg * ~mask_tg+ unconfirmed_covs_tg = self.tracked_stracks_covs_tg * ~mask_tg.view(-1,1,1)+ unconfirmed_means_tg = self.tracked_stracks_means_tg * ~mask_tg.view(-1,1)++ if len(self.lost_stracks_values_tg.shape) > 1 and self.lost_stracks_values_tg.shape[0] > 0:+ tracked_stracks_ids_tg = tracked_stracks_ids_tg.cat(self.lost_stracks_ids_tg)+ tracked_stracks_fids_tg = tracked_stracks_fids_tg.cat(self.lost_stracks_fids_tg)+ tracked_stracks_bools_tg = tracked_stracks_bools_tg.cat(self.lost_stracks_bools_tg)+ tracked_stracks_startframes_tg = tracked_stracks_startframes_tg.cat(self.lost_stracks_startframes_tg)+ tracked_stracks_states_tg = tracked_stracks_states_tg.cat(self.lost_stracks_states_tg)+ tracked_stracks_values_tg = tracked_stracks_values_tg.cat(self.lost_stracks_values_tg)++ tracked_stracks_ids = tracked_stracks_ids_tg.numpy()+ id_mask = tracked_stracks_ids != 0+ tracked_stracks_ids = tracked_stracks_ids[id_mask].tolist()+ tracked_stracks_fids = tracked_stracks_fids_tg.numpy()+ tracked_stracks_fids = tracked_stracks_fids[id_mask].tolist()+ tracked_stracks_bools = tracked_stracks_bools_tg.numpy()+ tracked_stracks_bools = tracked_stracks_bools[id_mask].tolist()+ tracked_stracks_startframes = tracked_stracks_startframes_tg.numpy()+ tracked_stracks_startframes = tracked_stracks_startframes[id_mask].tolist()+ tracked_stracks_states = tracked_stracks_states_tg.numpy()+ tracked_stracks_states = tracked_stracks_states[id_mask].tolist()+ tracked_stracks_values = tracked_stracks_values_tg.numpy()+ tracked_stracks_values = tracked_stracks_values[id_mask].tolist()++ unconfirmed_ids = unconfirmed_ids_tg.numpy()+ id_mask = unconfirmed_ids != 0+ unconfirmed_ids = unconfirmed_ids[id_mask].tolist()+ unconfirmed_values = unconfirmed_values_tg.numpy()+ unconfirmed_values = unconfirmed_values[id_mask].tolist()+ unconfirmed_bools = unconfirmed_bools_tg.numpy()+ unconfirmed_bools = unconfirmed_bools[id_mask].tolist()+ unconfirmed_states = unconfirmed_states_tg.numpy()+ unconfirmed_states = unconfirmed_states[id_mask].tolist()+ unconfirmed_covs = unconfirmed_covs_tg.numpy()+ unconfirmed_covs = unconfirmed_covs[id_mask].tolist()+ unconfirmed_means = unconfirmed_means_tg.numpy()+ unconfirmed_means = unconfirmed_means[id_mask].tolist()+ unconfirmed_fids = unconfirmed_fids_tg.numpy()+ unconfirmed_fids = unconfirmed_fids[id_mask].tolist()+ unconfirmed_startframes = unconfirmed_startframes_tg.numpy()+ unconfirmed_startframes = unconfirmed_startframes[id_mask].tolist()+ tracked_stracks_means = tracked_stracks_means + list(self.lost_stracks_means)+ tracked_stracks_covs = tracked_stracks_covs + list(self.lost_stracks_covs)++ # Predict the current location with KF+ if len(tracked_stracks_ids) > 0:+ multi_mean = np.asarray([st for st in tracked_stracks_means])+ multi_covariance = np.asarray([st for st in tracked_stracks_covs])+ for i in range(len(tracked_stracks_ids)):+ if tracked_stracks_states[i] != TrackState.Tracked:+ multi_mean[i][7] = 0++ multi_mean, multi_covariance = self.kalman_filter.multi_predict(multi_mean, multi_covariance)+ for i in range(len(tracked_stracks_ids)):+ tracked_stracks_means[i][:] = multi_mean[i].astype(np.float32)+ tracked_stracks_covs[i][:] = multi_covariance[i]++ for i, t in enumerate(self.tracked_stracks_covs):+ for j, p in enumerate(tracked_stracks_covs):+ if t is p:+ self.tracked_stracks_covs[i] = tracked_stracks_covs[j]++ atlbrs = tlbr_np_batch(tracked_stracks_values, tracked_stracks_means)+ btlbrs = tlbr_np_batch(dets_score_classes,detections_means)+ dists = iou_distance(atlbrs, btlbrs)+ dists = fuse_score(dists, dets_score_classes)+ matches, u_track, u_detection = linear_assignment(dists, thresh=self.args.match_thresh)++ det_values_arr = [dets_score_classes[i] for _, i in matches]++ for idx, (itracked, idet) in enumerate(matches):+ det_xyah = tlwh_to_xyah(tlwh_np(det_values_arr[idx], detections_means[idx]))+ x, y = self.kalman_filter.update(tracked_stracks_means[itracked], tracked_stracks_covs[itracked], det_xyah)+ tracked_stracks_covs[itracked][:] = y+ tracked_stracks_means[itracked][:] = x+ tracked_stracks_fids[itracked] = self.frame_id+ if itracked < len(self.tracked_stracks_fids): self.tracked_stracks_fids[itracked] = self.frame_id++ if tracked_stracks_states[itracked] == TrackState.Tracked:+ activated_stracks_values.append(tracked_stracks_values[itracked])+ activated_stracks_means.append(tracked_stracks_means[itracked])+ activated_stracks_bools.append(tracked_stracks_bools[itracked])+ activated_stracks_covs.append(tracked_stracks_covs[itracked])+ activated_stracks_ids.append(tracked_stracks_ids[itracked])+ activated_stracks_fids.append(tracked_stracks_fids[itracked])+ activated_stracks_startframes.append(tracked_stracks_startframes[itracked])+ activated_stracks_states.append(tracked_stracks_states[itracked])+ else:+ tracked_stracks_states[itracked] = TrackState.Tracked+ tracked_stracks_bools[itracked] = True+ refind_stracks_values.append(tracked_stracks_values[itracked])+ refind_stracks_means.append(tracked_stracks_means[itracked])+ refind_stracks_bools.append(True)+ refind_stracks_covs.append(tracked_stracks_covs[itracked])+ refind_stracks_ids.append(tracked_stracks_ids[itracked])+ refind_stracks_fids.append(tracked_stracks_fids[itracked])+ refind_stracks_startframes.append(tracked_stracks_startframes[itracked])+ refind_stracks_states.append(tracked_stracks_states[itracked])++ r_tracked_stracks_values = []+ r_tracked_stracks_means = []+ r_tracked_stracks_bools = []+ r_tracked_stracks_covs = []+ r_tracked_stracks_ids = []+ r_tracked_stracks_fids = []+ r_tracked_stracks_startframes = []+ r_tracked_stracks_states = []++ for i in range(len(u_track)):+ if tracked_stracks_states[u_track[i]] == TrackState.Tracked:+ r_tracked_stracks_values.append(tracked_stracks_values[u_track[i]])+ r_tracked_stracks_means.append(tracked_stracks_means[u_track[i]])+ r_tracked_stracks_bools.append(tracked_stracks_bools[u_track[i]])+ r_tracked_stracks_covs.append(tracked_stracks_covs[u_track[i]])+ r_tracked_stracks_ids.append(tracked_stracks_ids[u_track[i]])+ r_tracked_stracks_fids.append(tracked_stracks_fids[u_track[i]])+ r_tracked_stracks_startframes.append(tracked_stracks_startframes[u_track[i]])+ r_tracked_stracks_states.append(tracked_stracks_states[u_track[i]])++ det_values = dets_score_classes_second+ det_means = [None] * len(det_values) # all means are None initially++ atlbrs = tlbr_np_batch(r_tracked_stracks_values, r_tracked_stracks_means)+ btlbrs = tlbr_np_batch(det_values, det_means)+ dists = iou_distance(atlbrs, btlbrs)++ matches, u_track, _ = linear_assignment(dists, thresh=0.5)++ for itracked, idet in matches:+ mean = r_tracked_stracks_means[itracked]+ bool_val = r_tracked_stracks_bools[itracked]+ cov = r_tracked_stracks_covs[itracked]+ startframe = r_tracked_stracks_startframes[itracked]+ state = r_tracked_stracks_states[itracked]+ id_val = r_tracked_stracks_ids[itracked]+ fid = r_tracked_stracks_fids[itracked]+ values = r_tracked_stracks_values[itracked]+ t_val = r_tracked_stracks_values[itracked]+ d_val = det_values[idet]+ d_mean = det_means[idet]++ xyah = tlwh_to_xyah(tlwh_np(d_val, d_mean))+ r_tracked_stracks_means[itracked][:], x = self.kalman_filter.update(mean, cov, xyah)+ r_tracked_stracks_covs[itracked][:] = x+ t_val = list(t_val)+ t_val[4] = d_val[4] # Update score+ r_tracked_stracks_values[itracked] = tuple(t_val)+ r_tracked_stracks_fids[itracked] = self.frame_id++ # Update tracked_stracks attributes if track exists in tracked_stracks+ for i, t in enumerate(self.tracked_stracks_ids):+ if t is id_val:+ self.tracked_stracks_fids[i] = self.frame_id+ self.tracked_stracks_values[i] = tuple(t_val)+ self.tracked_stracks_means[i] = r_tracked_stracks_means[itracked]+ self.tracked_stracks_covs[i] = r_tracked_stracks_covs[itracked]+ self.tracked_stracks_states[i] = TrackState.Tracked+ break++ if state == TrackState.Tracked:+ activated_stracks_values.append(r_tracked_stracks_values[itracked])+ activated_stracks_means.append(r_tracked_stracks_means[itracked])+ activated_stracks_bools.append(bool_val)+ activated_stracks_covs.append(r_tracked_stracks_covs[itracked])+ activated_stracks_ids.append(id_val)+ activated_stracks_fids.append(self.frame_id)+ activated_stracks_startframes.append(startframe)+ activated_stracks_states.append(state)+ else:+ r_tracked_stracks_bools[itracked] = True+ r_tracked_stracks_states[itracked] = TrackState.Tracked+ refind_stracks_values.append(r_tracked_stracks_values[itracked])+ refind_stracks_means.append(r_tracked_stracks_means[itracked])+ refind_stracks_bools.append(True)+ refind_stracks_covs.append(r_tracked_stracks_covs[itracked])+ refind_stracks_ids.append(id_val)+ refind_stracks_fids.append(self.frame_id)+ refind_stracks_startframes.append(startframe)+ refind_stracks_states.append(state)++ for i in range(len(u_track)):+ values = r_tracked_stracks_values[u_track[i]]+ mean = r_tracked_stracks_means[u_track[i]]+ bool_val = r_tracked_stracks_bools[u_track[i]]+ cov = r_tracked_stracks_covs[u_track[i]]+ id_val = r_tracked_stracks_ids[u_track[i]]+ fid = r_tracked_stracks_fids[u_track[i]]+ startframe = r_tracked_stracks_startframes[u_track[i]]+ state = r_tracked_stracks_states[u_track[i]]+ if state != TrackState.Lost:+ for j, t in enumerate(self.tracked_stracks_ids):+ if t is id_val:+ self.tracked_stracks_states[j] = TrackState.Lost+ break+ lost_stracks_values.append(values)+ lost_stracks_means.append(mean)+ lost_stracks_bools.append(bool_val)+ lost_stracks_covs.append(cov)+ lost_stracks_ids.append(id_val)+ lost_stracks_fids.append(fid)+ lost_stracks_startframes.append(startframe)+ lost_stracks_states.append(TrackState.Lost)++ u_detection_np = np.array(u_detection)+ detections_ids = np.array(detections_ids)[u_detection_np]+ detections_fids = np.array(detections_fids)[u_detection_np]+ detections_bools = np.array(detections_bools)[u_detection_np]+ detections_means = np.array(detections_means)[u_detection_np]+ detections_cov = np.array(detections_cov)[u_detection_np]+ detections_startframes = np.array(detections_startframes)[u_detection_np]+ detections_states = np.array(detections_states)[u_detection_np]+ dets_score_classes_second = np.array(dets_score_classes)[u_detection_np]++ atlbrs = tlbr_np_batch(unconfirmed_values, unconfirmed_means)+ btlbrs = tlbr_np_batch(dets_score_classes_second, detections_means)+ dists = iou_distance(atlbrs, btlbrs)+ dists = fuse_score(dists, dets_score_classes_second)+ matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.7)++ updated_means = []+ tracks_values = []+ updated_bools = []+ updated_covs = []+ updated_ids = []+ updated_fids = []+ updated_startframes = []+ updated_states = []++ if len(matches) > 0:+ matches_arr = np.array(matches)+ itracked_arr = matches_arr[:, 0]+ idet_arr = matches_arr[:, 1]+ ids = [unconfirmed_ids[i] for i in itracked_arr]+ fids = [unconfirmed_fids[i] for i in itracked_arr]+ startframes = [unconfirmed_startframes[i] for i in itracked_arr]+ states = [unconfirmed_states[i] for i in itracked_arr]+ updated_bools = [unconfirmed_bools[i] for i in itracked_arr]+ means = [unconfirmed_means[i] for i in itracked_arr]+ tracks_values = [unconfirmed_values[i] for i in itracked_arr]+++ covariances = [unconfirmed_covs[i] for i in itracked_arr]+ det_values = dets_score_classes_second[idet_arr]+ tlwhs = det_values[:, :4]+ scores = det_values[:, 4]++ for mean, cov, tlwh, id, sf, fid, state in zip(means, covariances, tlwhs, ids, startframes, fids, states):+ new_mean, new_cov = self.kalman_filter.update(mean, cov, tlwh_to_xyah(tlwh))+ updated_means.append(new_mean)+ updated_covs.append(new_cov)+ updated_ids.append(id)+ updated_fids.append(fid)+ updated_startframes.append(sf)+ updated_states.append(state)++ updated_scores = scores+ frame_id_val = self.frame_id+ for i, (mean, cov, score, values, fids, id) in enumerate(zip(updated_means, updated_covs, updated_scores, tracks_values, updated_fids, updated_ids)):+ covariances[i][:] = cov+ values = list(values)+ values[4] = score+ values = tuple(values)+ updated_fids[i] = frame_id_val+ updated_states[i] = TrackState.Tracked++ if id in self.tracked_stracks_ids:+ idx = self.tracked_stracks_ids.index(id)+ self.tracked_stracks_bools[idx] = True+ self.tracked_stracks_states[idx] = TrackState.Tracked++ activated_stracks_bools.extend(updated_bools)+ activated_stracks_values.extend(tracks_values)+ activated_stracks_means.extend(updated_means)+ activated_stracks_covs.extend(updated_covs)+ activated_stracks_ids.extend(updated_ids)+ activated_stracks_fids.extend(updated_fids)+ activated_stracks_startframes.extend(updated_startframes)+ activated_stracks_states.extend(updated_states)++ u_unconfirmed_np = np.asarray(u_unconfirmed)+ ids = np.fromiter((unconfirmed_ids[key] for key in u_unconfirmed_np), dtype=object)+ if ids.size > 0:+ removed_stracks_ids.extend(ids.tolist())++ u_detection = np.asarray(u_detection)+ track_scores = dets_score_classes_second[u_detection, 4] # Direct score access+ valid_mask = track_scores >= self.det_thresh+ valid_indices = u_detection[valid_mask].tolist() # Convert to list of integers++ # Get tracks using proper list indexing++ valid_values = dets_score_classes_second[valid_indices] # Get corresponding values+ valid_means = [detections_means[i] for i in valid_indices]+ valid_bools = [detections_bools[i] for i in valid_indices]+ valid_covs = [detections_cov[i] for i in valid_indices]+ valid_ids = [detections_ids[i] for i in valid_indices]+ valid_fids = [detections_fids[i] for i in valid_indices]+ valid_startframes = [detections_startframes[i] for i in valid_indices]+ valid_states = [detections_states[i] for i in valid_indices]++ for i, (vals, mean) in enumerate(zip(valid_values, valid_means)):+ y = self._count = self._count + 1+ valid_ids[i] = y+ valid_means[i], x = self.kalman_filter.initiate(+ tlwh_to_xyah(vals[:4]))+ valid_covs[i] = x+ if self.frame_id == 1:+ valid_bools[i] = True+ valid_fids[i] = self.frame_id++ activated_stracks_means.extend(valid_means)+ activated_stracks_values.extend(valid_values)+ activated_stracks_bools.extend(valid_bools)+ activated_stracks_covs.extend(valid_covs)+ activated_stracks_ids.extend(valid_ids)+ activated_stracks_fids.extend(valid_fids)+ activated_stracks_startframes.extend(valid_startframes)+ activated_stracks_states.extend(valid_states)+ remove_mask = (self.frame_id - np.array(self.lost_stracks_fids)) > self.max_time_lost+ for t in np.array(self.lost_stracks_states)[remove_mask]: t = TrackState.Removed+ self.lost_stracks_means = np.array(self.lost_stracks_means)[~remove_mask]+ self.lost_stracks_bools = np.array(self.lost_stracks_bools)[~remove_mask]+ self.lost_stracks_values = (np.array(self.lost_stracks_values)[~remove_mask])+ self.lost_stracks_covs = np.array(self.lost_stracks_covs)[~remove_mask]+ self.lost_stracks_ids = np.array(self.lost_stracks_ids)[~remove_mask]+ self.lost_stracks_fids = np.array(self.lost_stracks_fids)[~remove_mask]+ self.lost_stracks_startframes = np.array(self.lost_stracks_startframes)[~remove_mask]+ self.lost_stracks_states = np.array(self.lost_stracks_states)[~remove_mask]++ self.tracked_stracks_states_tg = Tensor(self.tracked_stracks_states)+ mask_tg = self.tracked_stracks_states_tg == TrackState.Tracked+ self.tracked_stracks_ids_tg = Tensor(self.tracked_stracks_ids)+ self.tracked_stracks_ids_tg *= mask_tg+ self.tracked_stracks_ids = self.tracked_stracks_ids_tg.numpy()+ mask = self.tracked_stracks_ids != 0++ self.tracked_stracks_values = np.array(self.tracked_stracks_values)[mask]+ self.tracked_stracks_means = np.array(self.tracked_stracks_means)[mask]+ self.tracked_stracks_bools = np.array(self.tracked_stracks_bools)[mask]+ self.tracked_stracks_covs = np.array(self.tracked_stracks_covs)[mask]+ self.tracked_stracks_fids = np.array(self.tracked_stracks_fids)[mask]+ self.tracked_stracks_ids = np.array(self.tracked_stracks_ids)[mask]+ self.tracked_stracks_startframes = np.array(self.tracked_stracks_startframes)[mask]+ self.tracked_stracks_states = np.array(self.tracked_stracks_states)[mask]++ keep_activated = np.where(~np.isin(activated_stracks_ids, self.tracked_stracks_ids))[0]+ keep_activated_tg = Tensor(keep_activated)++ self.tracked_stracks_ids_tg = Tensor(self.tracked_stracks_ids,dtype=dtypes.int)+ activated_stracks_ids_tg = Tensor(activated_stracks_ids,dtype=dtypes.int)+ self.tracked_stracks_fids_tg = Tensor(self.tracked_stracks_fids,dtype=dtypes.int)+ activated_stracks_fids_tg = Tensor(activated_stracks_fids,dtype=dtypes.int)+ self.tracked_stracks_states_tg = Tensor(self.tracked_stracks_states,dtype=dtypes.int)+ activated_stracks_states_tg = Tensor(activated_stracks_states,dtype=dtypes.int)+ self.tracked_stracks_startframes_tg = Tensor(self.tracked_stracks_startframes,dtype=dtypes.int)+ activated_stracks_startframes_tg = Tensor(activated_stracks_startframes)+ self.tracked_stracks_bools_tg = Tensor(self.tracked_stracks_bools,dtype=dtypes.bool)+ activated_stracks_bools_tg = Tensor(activated_stracks_bools,dtype=dtypes.bool)+ self.tracked_stracks_values_tg = Tensor(self.tracked_stracks_values,dtype=dtypes.float32)+ activated_stracks_values_tg = Tensor(activated_stracks_values,dtype=dtypes.float32)+ self.tracked_stracks_means_tg = Tensor(self.tracked_stracks_means,dtype=dtypes.float32)+ activated_stracks_means_tg = Tensor(activated_stracks_means,dtype=dtypes.float32)+ self.tracked_stracks_covs_tg = Tensor(self.tracked_stracks_covs,dtype=dtypes.float32)+ activated_stracks_covs_tg = Tensor(activated_stracks_covs,dtype=dtypes.float32)+++ self.tracked_stracks_ids_tg = self.tracked_stracks_ids_tg.cat(activated_stracks_ids_tg[keep_activated_tg])+ self.tracked_stracks_fids_tg = self.tracked_stracks_fids_tg.cat(activated_stracks_fids_tg[keep_activated_tg])+ self.tracked_stracks_states_tg = self.tracked_stracks_states_tg.cat(activated_stracks_states_tg[keep_activated_tg])+ self.tracked_stracks_startframes_tg = self.tracked_stracks_startframes_tg.cat(activated_stracks_startframes_tg[keep_activated_tg])+ self.tracked_stracks_bools_tg = self.tracked_stracks_bools_tg.cat(activated_stracks_bools_tg[keep_activated_tg])+ if self.tracked_stracks_values_tg.shape[0] > 0:+ self.tracked_stracks_values_tg = self.tracked_stracks_values_tg.cat(activated_stracks_values_tg[keep_activated_tg])+ self.tracked_stracks_means_tg = self.tracked_stracks_means_tg.cat(activated_stracks_means_tg[keep_activated_tg])+ self.tracked_stracks_covs_tg = self.tracked_stracks_covs_tg.cat(activated_stracks_covs_tg[keep_activated_tg])+ else:+ self.tracked_stracks_values_tg = activated_stracks_values_tg[keep_activated_tg]+ self.tracked_stracks_means_tg = activated_stracks_means_tg[keep_activated_tg]+ self.tracked_stracks_covs_tg = activated_stracks_covs_tg[keep_activated_tg]++ self.tracked_stracks_ids = self.tracked_stracks_ids_tg.numpy()+ self.tracked_stracks_covs = self.tracked_stracks_covs_tg.numpy()++ refind_stracks_fids_tg = Tensor(refind_stracks_fids)+ refind_stracks_bools_tg = Tensor(refind_stracks_bools)+ refind_stracks_states_tg = Tensor(refind_stracks_states,dtype=dtypes.int)+ refind_stracks_startframes_tg = Tensor(refind_stracks_startframes)+ refind_stracks_values_tg = Tensor(refind_stracks_values)+ refind_stracks_means_tg = Tensor(refind_stracks_means)+ refind_stracks_covs_tg = Tensor(refind_stracks_covs)++ self.tracked_stracks_fids_tg = self.tracked_stracks_fids_tg.cat(refind_stracks_fids_tg)+ self.tracked_stracks_bools_tg = self.tracked_stracks_bools_tg.cat(refind_stracks_bools_tg)+ self.tracked_stracks_states_tg = self.tracked_stracks_states_tg.cat(refind_stracks_states_tg)+ self.tracked_stracks_startframes_tg = self.tracked_stracks_startframes_tg.cat(refind_stracks_startframes_tg)+ if refind_stracks_means_tg.shape[0] > 0:+ self.tracked_stracks_means_tg = self.tracked_stracks_means_tg.cat(refind_stracks_means_tg)+ self.tracked_stracks_values_tg = self.tracked_stracks_values_tg.cat(refind_stracks_values_tg)+ self.tracked_stracks_covs_tg = self.tracked_stracks_covs_tg.cat(refind_stracks_covs_tg)++ self.tracked_stracks_ids = np.array(self.tracked_stracks_ids).tolist() + np.array(refind_stracks_ids).tolist()++ self.tracked_stracks_fids = self.tracked_stracks_fids_tg.numpy()+ self.tracked_stracks_bools = self.tracked_stracks_bools_tg.numpy()+ self.tracked_stracks_states = self.tracked_stracks_states_tg.numpy()+ self.tracked_stracks_startframes = self.tracked_stracks_startframes_tg.numpy()+ self.tracked_stracks_values = self.tracked_stracks_values_tg.numpy()+ self.tracked_stracks_means = self.tracked_stracks_means_tg.numpy()+ self.tracked_stracks_covs = self.tracked_stracks_covs_tg.numpy()++ mask = ~np.isin(self.lost_stracks_ids, self.tracked_stracks_ids)+ mask_tg = Tensor(mask)++ self.lost_stracks_ids_tg = Tensor(self.lost_stracks_ids)+ self.lost_stracks_fids_tg = Tensor(self.lost_stracks_fids)+ self.lost_stracks_startframes_tg = Tensor(self.lost_stracks_startframes)+ self.lost_stracks_states_tg = Tensor(self.lost_stracks_states)+ self.lost_stracks_bools_tg = Tensor(self.lost_stracks_bools)+ self.lost_stracks_values_tg = Tensor(self.lost_stracks_values)+ self.lost_stracks_means_tg = Tensor(self.lost_stracks_means)+ self.lost_stracks_covs_tg = Tensor(self.lost_stracks_covs)+++ self.lost_stracks_ids_tg *= mask_tg+ self.lost_stracks_fids_tg *= mask_tg+ self.lost_stracks_startframes_tg *= mask_tg+ self.lost_stracks_states_tg *= mask_tg+ self.lost_stracks_bools_tg *= mask_tg+ self.lost_stracks_values_tg = self.lost_stracks_values_tg * mask_tg.view(-1, 1)+ self.lost_stracks_means_tg = self.lost_stracks_means_tg * mask_tg.view(-1, 1)+ self.lost_stracks_covs_tg = self.lost_stracks_covs_tg * mask_tg.view(-1, 1, 1)++ lost_stracks_values_tg = Tensor(lost_stracks_values)+ lost_stracks_means_tg = Tensor(lost_stracks_means)+ lost_stracks_bools_tg = Tensor(lost_stracks_bools)+ lost_stracks_covs_tg = Tensor(lost_stracks_covs)+ lost_stracks_ids_tg = Tensor(lost_stracks_ids)+ lost_stracks_fids_tg = Tensor(lost_stracks_fids)+ lost_stracks_startframes_tg = Tensor(lost_stracks_startframes)+ lost_stracks_states_tg = Tensor(lost_stracks_states)++ if self.lost_stracks_values_tg.shape[0] == 0:+ self.lost_stracks_values_tg = lost_stracks_values_tg+ self.lost_stracks_means_tg = lost_stracks_means_tg+ self.lost_stracks_bools_tg = lost_stracks_bools_tg+ self.lost_stracks_covs_tg = lost_stracks_covs_tg+ self.lost_stracks_ids_tg = lost_stracks_ids_tg+ self.lost_stracks_fids_tg = lost_stracks_fids_tg+ self.lost_stracks_startframes_tg = lost_stracks_startframes_tg+ self.lost_stracks_states_tg = lost_stracks_states_tg+ self.lost_stracks_states_tg = lost_stracks_states_tg+ elif lost_stracks_values_tg.shape[0] != 0:+ self.lost_stracks_values_tg = self.lost_stracks_values_tg.cat(lost_stracks_values_tg)+ self.lost_stracks_means_tg = self.lost_stracks_means_tg.cat(lost_stracks_means_tg)+ self.lost_stracks_bools_tg = self.lost_stracks_bools_tg.cat(lost_stracks_bools_tg)+ self.lost_stracks_covs_tg = self.lost_stracks_covs_tg.cat(lost_stracks_covs_tg)+ self.lost_stracks_ids_tg = self.lost_stracks_ids_tg.cat(lost_stracks_ids_tg)+ self.lost_stracks_fids_tg = self.lost_stracks_fids_tg.cat(lost_stracks_fids_tg)+ self.lost_stracks_startframes_tg = self.lost_stracks_startframes_tg.cat(lost_stracks_startframes_tg)+ self.lost_stracks_states_tg = self.lost_stracks_states_tg.cat(lost_stracks_states_tg)++ self.lost_stracks_values = self.lost_stracks_values_tg.numpy()+ self.lost_stracks_means = self.lost_stracks_means_tg.numpy()+ self.lost_stracks_bools = self.lost_stracks_bools_tg.numpy()+ self.lost_stracks_covs = self.lost_stracks_covs_tg.numpy()+ self.lost_stracks_ids = self.lost_stracks_ids_tg.numpy()+ self.lost_stracks_fids = self.lost_stracks_fids_tg.numpy()+ self.lost_stracks_startframes = self.lost_stracks_startframes_tg.numpy()+ self.lost_stracks_states = self.lost_stracks_states_tg.numpy()++ self.tracked_stracks_bools_tg = Tensor(self.tracked_stracks_bools).unsqueeze(-1)+ self.tracked_stracks_means_tg = Tensor(self.tracked_stracks_means)+ self.tracked_stracks_ids_tg = Tensor(self.tracked_stracks_ids)+ self.tracked_stracks_values_tg = Tensor(self.tracked_stracks_values)+++ output_stracks_means_tg = self.tracked_stracks_means_tg * self.tracked_stracks_bools_tg+ output_stracks_values_tg = self.tracked_stracks_values_tg * self.tracked_stracks_bools_tg+ output_stracks_ids_tg = self.tracked_stracks_ids_tg * self.tracked_stracks_bools_tg.squeeze(-1)++ zeros = self.lost_stracks_ids != 0+ self.lost_stracks_ids = self.lost_stracks_ids[zeros]+ self.lost_stracks_fids = self.lost_stracks_fids[zeros]+ self.lost_stracks_startframes = self.lost_stracks_startframes[zeros]+ self.lost_stracks_states = self.lost_stracks_states[zeros]+ self.lost_stracks_bools = self.lost_stracks_bools[zeros]+ self.lost_stracks_values = self.lost_stracks_values[zeros]+ self.lost_stracks_means = self.lost_stracks_means[zeros]+ self.lost_stracks_covs = self.lost_stracks_covs[zeros]+ v,m,i = output_stracks_values_tg.numpy(), output_stracks_means_tg.numpy(), output_stracks_ids_tg.numpy()+ return v,m,i+++def ious(atlbrs, btlbrs):+ """+ Compute cost based on IoU+ :type atlbrs: list[tlbr] | np.ndarray+ :type atlbrs: list[tlbr] | np.ndarray++ :rtype ious np.ndarray+ """+ ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float)+ if ious.size == 0:+ return ious++ ious = bbox_ious(np.ascontiguousarray(atlbrs, dtype=np.float), np.ascontiguousarray(btlbrs, dtype=np.float))+ return ious++def iou_distance(atlbrs, btlbrs):+ _ious = ious(atlbrs, btlbrs)+ cost_matrix = 1 - _ious+ return cost_matrix++def fuse_score(cost_matrix, det_values):+ if cost_matrix.size == 0:+ return cost_matrix+ iou_sim = 1 - cost_matrix+ det_scores = np.array([det[4] for det in det_values])+ det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0)+ fuse_sim = iou_sim * det_scores+ fuse_cost = 1 - fuse_sim+ return fuse_cost++def linear_assignment(cost_matrix, thresh):+ if cost_matrix.size == 0:+ return np.empty((0, 2), dtype=int), np.arange(cost_matrix.shape[0]), np.arange(cost_matrix.shape[1])+ _, x, y = lap.lapjv(cost_matrix, extend_cost=True, cost_limit=thresh)+ matched_mask = x >= 0+ matches = np.column_stack((np.arange(len(x))[matched_mask],x[matched_mask]))+ unmatched_a = np.where(~matched_mask)[0]+ unmatched_b = np.where(y < 0)[0]+ return matches, unmatched_a, unmatched_b+++#Model architecture from https://github.com/ultralytics/ultralytics/issues/189+#The upsampling class has been taken from this pull request https://github.com/tinygrad/tinygrad/pull/784 by dc-dc-dc. Now 2(?) models use upsampling. (retinet and this)++#Pre processing image functions.+def compute_transform(image, new_shape=(1280, 1280), auto=False, scaleFill=False, scaleup=True, stride=32) -> Tensor:+ shape = image.shape[:2] # current shape [height, width]+ new_shape = (new_shape, new_shape) if isinstance(new_shape, int) else new_shape+ r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])+ r = min(r, 1.0) if not scaleup else r+ new_unpad = (int(round(shape[1] * r)), int(round(shape[0] * r)))+ dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1]+ dw, dh = (np.mod(dw, stride), np.mod(dh, stride)) if auto else (0.0, 0.0)+ new_unpad = (new_shape[1], new_shape[0]) if scaleFill else new_unpad+ dw /= 2+ dh /= 2+ image = cv2.resize(image, new_unpad, interpolation=cv2.INTER_LINEAR) if shape[::-1] != new_unpad else image+ top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))+ left, right = int(round(dw - 0.1)), int(round(dw + 0.1))+ image = cv2.copyMakeBorder(image, top, bottom, left, right, cv2.BORDER_CONSTANT, value=(114, 114, 114))+ return Tensor(image)++def preprocess(im, imgsz=1280, model_stride=32, model_pt=True):+ im = compute_transform(im, new_shape=imgsz, auto=True, stride=model_stride)+ im = im.unsqueeze(0)+ im = im[..., ::-1].permute(0, 3, 1, 2)+ im = im / 255.0+ return im++# utility functions for forward pass.+def dist2bbox(distance, anchor_points, xywh=True, dim=-1):+ lt, rb = distance.chunk(2, dim)+ x1y1 = anchor_points - lt+ x2y2 = anchor_points + rb+ if xywh:+ c_xy = (x1y1 + x2y2) / 2+ wh = x2y2 - x1y1+ return c_xy.cat(wh, dim=1)+ return x1y1.cat(x2y2, dim=1)++def make_anchors(feats, strides, grid_cell_offset=0.5):+ anchor_points, stride_tensor = [], []+ assert feats is not None+ for i, stride in enumerate(strides):+ _, _, h, w = feats[i].shape+ sx = Tensor.arange(w) + grid_cell_offset+ sy = Tensor.arange(h) + grid_cell_offset++ # this is np.meshgrid but in tinygrad+ sx = sx.reshape(1, -1).repeat([h, 1]).reshape(-1)+ sy = sy.reshape(-1, 1).repeat([1, w]).reshape(-1)++ anchor_points.append(Tensor.stack(sx, sy, dim=-1).reshape(-1, 2))+ stride_tensor.append(Tensor.full((h * w), stride))+ anchor_points = anchor_points[0].cat(anchor_points[1], anchor_points[2])+ stride_tensor = stride_tensor[0].cat(stride_tensor[1], stride_tensor[2]).unsqueeze(1)+ return anchor_points, stride_tensor++# this function is from the original implementation+def autopad(k, p=None, d=1): # kernel, padding, dilation+ if d > 1:+ k = d * (k - 1) + 1 if isinstance(k, int) else [d * (x - 1) + 1 for x in k] # actual kernel-size+ if p is None:+ p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad+ return p++def clip_boxes(boxes, shape):+ boxes[..., [0, 2]] = np.clip(boxes[..., [0, 2]], 0, shape[1]) # x1, x2+ boxes[..., [1, 3]] = np.clip(boxes[..., [1, 3]], 0, shape[0]) # y1, y2+ return boxes++def scale_boxes(img1_shape, predictions, img0_shape, ratio_pad=None):+ gain = ratio_pad if ratio_pad else min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1])+ pad = ((img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2)+ for pred in predictions:+ boxes_np = pred[:4].numpy() if isinstance(pred[:4], Tensor) else pred[:4]+ boxes_np[..., [0, 2]] -= pad[0]+ boxes_np[..., [1, 3]] -= pad[1]+ boxes_np[..., :4] /= gain+ boxes_np = clip_boxes(boxes_np, img0_shape)+ pred[:4] = boxes_np+ return predictions++def get_variant_multiples(variant):+ return {'n':(0.33, 0.25, 2.0), 's':(0.33, 0.50, 2.0), 'm':(0.67, 0.75, 1.5), 'l':(1.0, 1.0, 1.0), 'x':(1, 1.25, 1.0) }.get(variant, None)++def label_predictions(all_predictions):+ class_index_count = defaultdict(int)+ for pred in all_predictions:+ class_id = int(pred[-1])+ if pred[-2] != 0: class_index_count[class_id] += 1++ return dict(class_index_count)++#this is taken from https://github.com/tinygrad/tinygrad/pull/784/files by dc-dc-dc (Now 2 models use upsampling)+class Upsample:+ def __init__(self, scale_factor:int, mode: str = "nearest") -> None:+ assert mode == "nearest" # only mode supported for now+ self.mode = mode+ self.scale_factor = scale_factor++ def __call__(self, x: Tensor) -> Tensor:+ assert len(x.shape) > 2 and len(x.shape) <= 5+ (b, c), _lens = x.shape[:2], len(x.shape[2:])+ tmp = x.reshape([b, c, -1] + [1] * _lens) * Tensor.ones(*[1, 1, 1] + [self.scale_factor] * _lens)+ return tmp.reshape(list(x.shape) + [self.scale_factor] * _lens).permute([0, 1] + list(chain.from_iterable([[y+2, y+2+_lens] for y in range(_lens)]))).reshape([b, c] + [x * self.scale_factor for x in x.shape[2:]])++class Conv_Block:+ def __init__(self, c1, c2, kernel_size=1, stride=1, groups=1, dilation=1, padding=None):+ self.conv = Conv2d(c1,c2, kernel_size, stride, padding=autopad(kernel_size, padding, dilation), bias=False, groups=groups, dilation=dilation)+ self.bn = BatchNorm2d(c2, eps=0.001)++ def __call__(self, x):+ return self.bn(self.conv(x)).silu()++class Bottleneck:+ def __init__(self, c1, c2 , shortcut: bool, g=1, kernels: list = (3,3), channel_factor=0.5):+ c_ = int(c2 * channel_factor)+ self.cv1 = Conv_Block(c1, c_, kernel_size=kernels[0], stride=1, padding=None)+ self.cv2 = Conv_Block(c_, c2, kernel_size=kernels[1], stride=1, padding=None, groups=g)+ self.residual = c1 == c2 and shortcut++ def __call__(self, x):+ return x + self.cv2(self.cv1(x)) if self.residual else self.cv2(self.cv1(x))++class C2f:+ def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5):+ self.c = int(c2 * e)+ self.cv1 = Conv_Block(c1, 2 * self.c, 1,)+ self.cv2 = Conv_Block((2 + n) * self.c, c2, 1)+ self.bottleneck = [Bottleneck(self.c, self.c, shortcut, g, kernels=[(3, 3), (3, 3)], channel_factor=1.0) for _ in range(n)]++ def __call__(self, x):+ y= list(self.cv1(x).chunk(2, 1))+ y.extend(m(y[-1]) for m in self.bottleneck)+ z = y[0]+ for i in y[1:]: z = z.cat(i, dim=1)+ return self.cv2(z)++class SPPF:+ def __init__(self, c1, c2, k=5):+ c_ = c1 // 2 # hidden channels+ self.cv1 = Conv_Block(c1, c_, 1, 1, padding=None)+ self.cv2 = Conv_Block(c_ * 4, c2, 1, 1, padding=None)++ # TODO: this pads with 0s, whereas torch function pads with -infinity. This results in a < 2% difference in prediction which does not make a difference visually.+ self.maxpool = lambda x : x.pad((k // 2, k // 2, k // 2, k // 2)).max_pool2d(kernel_size=k, stride=1)++ def __call__(self, x):+ x = self.cv1(x)+ x2 = self.maxpool(x)+ x3 = self.maxpool(x2)+ x4 = self.maxpool(x3)+ return self.cv2(x.cat(x2, x3, x4, dim=1))++class DFL:+ def __init__(self, c1=16):+ self.conv = Conv2d(c1, 1, 1, bias=False)+ x = Tensor.arange(c1)+ self.conv.weight.replace(x.reshape(1, c1, 1, 1))+ self.c1 = c1++ def __call__(self, x):+ b, c, a = x.shape # batch, channels, anchors+ return self.conv(x.reshape(b, 4, self.c1, a).transpose(2, 1).softmax(1)).reshape(b, 4, a)++#backbone+class Darknet:+ def __init__(self, w, r, d):+ self.b1 = [Conv_Block(c1=3, c2= int(64*w), kernel_size=3, stride=2, padding=1), Conv_Block(int(64*w), int(128*w), kernel_size=3, stride=2, padding=1)]+ self.b2 = [C2f(c1=int(128*w), c2=int(128*w), n=round(3*d), shortcut=True), Conv_Block(int(128*w), int(256*w), 3, 2, 1), C2f(int(256*w), int(256*w), round(6*d), True)]+ self.b3 = [Conv_Block(int(256*w), int(512*w), kernel_size=3, stride=2, padding=1), C2f(int(512*w), int(512*w), round(6*d), True)]+ self.b4 = [Conv_Block(int(512*w), int(512*w*r), kernel_size=3, stride=2, padding=1), C2f(int(512*w*r), int(512*w*r), round(3*d), True)]+ self.b5 = [SPPF(int(512*w*r), int(512*w*r), 5)]++ def return_modules(self):+ return [*self.b1, *self.b2, *self.b3, *self.b4, *self.b5]++ def __call__(self, x):+ x1 = x.sequential(self.b1)+ x2 = x1.sequential(self.b2)+ x3 = x2.sequential(self.b3)+ x4 = x3.sequential(self.b4)+ x5 = x4.sequential(self.b5)+ return (x2, x3, x5)++#yolo fpn (neck)+class Yolov8NECK:+ def __init__(self, w, r, d): #width_multiple, ratio_multiple, depth_multiple+ self.up = Upsample(2, mode='nearest')+ self.n1 = C2f(c1=int(512*w*(1+r)), c2=int(512*w), n=round(3*d), shortcut=False)+ self.n2 = C2f(c1=int(768*w), c2=int(256*w), n=round(3*d), shortcut=False)+ self.n3 = Conv_Block(c1=int(256*w), c2=int(256*w), kernel_size=3, stride=2, padding=1)+ self.n4 = C2f(c1=int(768*w), c2=int(512*w), n=round(3*d), shortcut=False)+ self.n5 = Conv_Block(c1=int(512* w), c2=int(512 * w), kernel_size=3, stride=2, padding=1)+ self.n6 = C2f(c1=int(512*w*(1+r)), c2=int(512*w*r), n=round(3*d), shortcut=False)++ def return_modules(self):+ return [self.n1, self.n2, self.n3, self.n4, self.n5, self.n6]++ def __call__(self, p3, p4, p5):+ x = self.n1(self.up(p5).cat(p4, dim=1))+ head_1 = self.n2(self.up(x).cat(p3, dim=1))+ head_2 = self.n4(self.n3(head_1).cat(x, dim=1))+ head_3 = self.n6(self.n5(head_2).cat(p5, dim=1))+ return [head_1, head_2, head_3]++#task specific head.+class DetectionHead:+ def __init__(self, nc=80, filters=()):+ self.ch = 16+ self.nc = nc # number of classes+ self.nl = len(filters)+ self.no = nc + self.ch * 4 #+ self.stride = [8, 16, 32]+ c1 = max(filters[0], self.nc)+ c2 = max((filters[0] // 4, self.ch * 4))+ self.dfl = DFL(self.ch)+ self.cv3 = [[Conv_Block(x, c1, 3), Conv_Block(c1, c1, 3), Conv2d(c1, self.nc, 1)] for x in filters]+ self.cv2 = [[Conv_Block(x, c2, 3), Conv_Block(c2, c2, 3), Conv2d(c2, 4 * self.ch, 1)] for x in filters]++ def __call__(self, x):+ for i in range(self.nl):+ x[i] = (x[i].sequential(self.cv2[i]).cat(x[i].sequential(self.cv3[i]), dim=1))+ self.anchors, self.strides = (x.transpose(0, 1) for x in make_anchors(x, self.stride, 0.5))+ y = [(i.reshape(x[0].shape[0], self.no, -1)) for i in x]+ x_cat = y[0].cat(y[1], y[2], dim=2)+ box, cls = x_cat[:, :self.ch * 4], x_cat[:, self.ch * 4:]+ dbox = dist2bbox(self.dfl(box), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides+ z = dbox.cat(cls.sigmoid(), dim=1)+ return z++class YOLOv8:+ def __init__(self, w, r, d, num_classes): #width_multiple, ratio_multiple, depth_multiple+ self.net = Darknet(w, r, d)+ self.fpn = Yolov8NECK(w, r, d)+ self.head = DetectionHead(num_classes, filters=(int(256*w), int(512*w), int(512*w*r)))++ def __call__(self, x):+ x = self.net(x)+ x = self.fpn(*x)+ x = self.head(x)+ # TODO: postprocess needs to be in the model to be compiled to webgpu+ return postprocess(x)++ def return_all_trainable_modules(self):+ backbone_modules = [*range(10)]+ yolov8neck_modules = [12, 15, 16, 18, 19, 21]+ yolov8_head_weights = [(22, self.head)]+ return [*zip(backbone_modules, self.net.return_modules()), *zip(yolov8neck_modules, self.fpn.return_modules()), *yolov8_head_weights]++def convert_f16_safetensor_to_f32(input_file: Path, output_file: Path):+ with open(input_file, 'rb') as f:+ metadata_length = int.from_bytes(f.read(8), 'little')+ metadata = json.loads(f.read(metadata_length).decode())+ float32_values = np.fromfile(f, dtype=np.float16).astype(np.float32)++ for v in metadata.values():+ if v["dtype"] == "F16": v.update({"dtype": "F32", "data_offsets": [offset * 2 for offset in v["data_offsets"]]})++ with open(output_file, 'wb') as f:+ new_metadata_bytes = json.dumps(metadata).encode()+ f.write(len(new_metadata_bytes).to_bytes(8, 'little'))+ f.write(new_metadata_bytes)+ float32_values.tofile(f)++def compute_iou_matrix(boxes):+ x1, y1, x2, y2 = boxes[:, 0], boxes[:, 1], boxes[:, 2], boxes[:, 3]+ areas = (x2 - x1) * (y2 - y1)+ x1 = Tensor.maximum(x1[:, None], x1[None, :])+ y1 = Tensor.maximum(y1[:, None], y1[None, :])+ x2 = Tensor.minimum(x2[:, None], x2[None, :])+ y2 = Tensor.minimum(y2[:, None], y2[None, :])+ w = Tensor.maximum(Tensor(0), x2 - x1)+ h = Tensor.maximum(Tensor(0), y2 - y1)+ intersection = w * h+ union = areas[:, None] + areas[None, :] - intersection+ return intersection / union++def postprocess(output, max_det=300, conf_threshold=0.25, iou_threshold=0.45):+ xc, yc, w, h, class_scores = output[0][0], output[0][1], output[0][2], output[0][3], output[0][4:]+ class_ids = Tensor.argmax(class_scores, axis=0)+ probs = Tensor.max(class_scores, axis=0)+ probs = Tensor.where(probs >= conf_threshold, probs, 0)+ x1 = xc - w / 2+ y1 = yc - h / 2+ x2 = xc + w / 2+ y2 = yc + h / 2+ boxes = Tensor.stack(x1, y1, x2, y2, probs, class_ids, dim=1)+ order = Tensor.topk(probs, max_det)[1]+ boxes = boxes[order]+ iou = compute_iou_matrix(boxes[:, :4])+ iou = Tensor.triu(iou, diagonal=1)+ same_class_mask = boxes[:, -1][:, None] == boxes[:, -1][None, :]+ high_iou_mask = (iou > iou_threshold) & same_class_mask+ no_overlap_mask = high_iou_mask.sum(axis=0) == 0+ boxes = boxes * no_overlap_mask.unsqueeze(-1)+ return boxes++def get_weights_location(yolo_variant: str) -> Path:+ weights_location = Path(__file__).parents[1] / "weights" / f'yolov8{yolo_variant}.safetensors'+ fetch(f'https://gitlab.com/r3sist/yolov8_weights/-/raw/master/yolov8{yolo_variant}.safetensors', weights_location)+ f32_weights = weights_location.with_name(f"{weights_location.stem}_f32.safetensors")+ if not f32_weights.exists(): convert_f16_safetensor_to_f32(weights_location, f32_weights)+ return f32_weights++def draw_predictions_on_frame(frame, predictions, class_labels, color_dict):+ font = cv2.FONT_HERSHEY_SIMPLEX++ def is_bright_color(color):+ r, g, b = color+ brightness = (r * 299 + g * 587 + b * 114) / 1000+ return brightness > 127++ h, w, _ = frame.shape+ box_thickness = int((h + w) / 400)+ font_scale = (h + w) / 2500++ for pred in predictions:+ x1, y1, x2, y2, id, class_id = pred+ x1, y1, x2, y2, class_id = map(int, (x1, y1, x2, y2, class_id))+ color = color_dict[class_labels[class_id]]+ label = f"{class_labels[class_id]} {int(id)}"+ cv2.rectangle(frame, (x1, y1), (x2, y2), color, box_thickness)+ text_size, _ = cv2.getTextSize(label, font, font_scale, 1)+ label_y, bg_y = (y1 - 4, y1 - text_size[1] - 4) if y1 - text_size[1] - 4 > 0 else (y1 + text_size[1], y1)+ cv2.rectangle(frame, (x1, bg_y), (x1 + text_size[0], bg_y + text_size[1]), color, -1)+ font_color = (0, 0, 0) if is_bright_color(color) else (255, 255, 255)+ cv2.putText(frame, label, (x1, label_y), font, font_scale, font_color, 1, cv2.LINE_AA)+++class Args:+ def __init__(self):+ self.track_thresh = 0.6+ self.track_buffer = 60 #frames, was 30+ self.mot20 = False+ self.match_thresh = 0.9++tracker = BYTETracker(Args())++@TinyJit+def do_inf(image):+ predictions = yolo_infer(image)+ return predictions++from urllib.request import urlopen, urlretrieve++if __name__ == '__main__':+ if len(sys.argv) < 2:+ print("Error: Video URL or path not provided.")+ sys.exit(1)++ video_path = sys.argv[1]+ yolo_variant = sys.argv[2] if len(sys.argv) >= 3 else (print("No variant given, so choosing 'n' as default.") or 'n')+ print(f'Running inference for YOLOv8 variant: {yolo_variant}')++ output_folder_path = Path('./video_outputs')+ output_folder_path.mkdir(parents=True, exist_ok=True)++ # Download and open the video+ local_video_path = fetch(video_path).as_posix()+ cap = cv2.VideoCapture(local_video_path)++ if not cap.isOpened():+ print("Error: Could not open video.")+ sys.exit(1)++ fps = int(cap.get(cv2.CAP_PROP_FPS))+ width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))+ height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))++ out_path = (output_folder_path / f"{Path(local_video_path).stem}_output.mp4").as_posix()+ out_writer = cv2.VideoWriter(out_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (width, height))++ # Load YOLOv8 model+ depth, width_mult, ratio = get_variant_multiples(yolo_variant)+ yolo_infer = YOLOv8(w=width_mult, r=ratio, d=depth, num_classes=80)+ state_dict = safe_load(get_weights_location(yolo_variant))+ load_state_dict(yolo_infer, state_dict)++ class_labels = fetch('https://raw.githubusercontent.com/pjreddie/darknet/master/data/coco.names').read_text().split("\n")+ color_dict = {label: tuple((((i+1) * 50) % 256, ((i+1) * 100) % 256, ((i+1) * 150) % 256)) for i, label in enumerate(class_labels)}+++ frame_count = 0+ people = set()+ #outs = []+ expected_values = pickle.load(open('values.pkl', 'rb'))+ while True:+ ret, frame = cap.read()+ if not ret:+ break+ frame_count += 1++ pre_processed = preprocess(frame)+ predictions = do_inf(pre_processed)+ values, means, track_ids = tracker.update(predictions, [1280,1280], [1280,1280])+ pred_track = np.array([np.append(tlbr_np(v,m), [tid,v[5]]) for v,m,tid in zip(values,means,track_ids)], dtype=np.float32)++ # sanity check print people+ for v,tid in zip(values,track_ids):+ if v[5] == 0 and tid != 0:+ people.add(tid)++ pred_track = scale_boxes(pre_processed.shape[2:], pred_track, frame.shape)+ predictions = predictions.numpy()+ predictions = scale_boxes(pre_processed.shape[2:], predictions, frame.shape)++ # Draw predictions+ #draw_predictions_on_frame(frame, predictions, class_labels, color_dict)+ draw_predictions_on_frame(frame, pred_track, class_labels, color_dict)++ out_writer.write(frame)+++ if sys.argv[1] == "https://motchallenge.net/sequenceVideos/MOT17-08-DPM-raw.mp4":+ if not np.array_equal(np.array(expected_values[frame_count - 1]), values):+ print("wrong output")+ exit()+ #outs.append(values)+++ if frame_count % 10 == 0:+ print(f"Processed frame {frame_count}")+ print(len(people))++ #pickle.dump(outs, open('values.pkl', 'wb'))+ cap.release()+ out_writer.release()+ print(f"Saved processed video to {out_path}")++#https://motchallenge.net/sequenceVideos/MOT17-08-DPM-raw.mp4 73+#https://motchallenge.net/sequenceVideos/MOT17-03-FRCNN-raw.mp4 173+