Add object detection, tracking, mobile notifications, and search to any security camera.
export model that doesnt threshold
rorycleare7a366b7a374
diff --git a/clearcam/batch_data b/clearcam/batch_dataindex 035a45c..de65d65 100644Binary files a/clearcam/batch_data and b/clearcam/batch_data differdiff --git a/clearcam/yolov8.py b/clearcam/yolov8.pyindex 9eb6f39..72ee4bc 100644--- a/clearcam/yolov8.py+++ b/clearcam/yolov8.py@@ -8,15 +8,10 @@ from pathlib import Pathimport cv2from collections import defaultdictimport time, sys-from tinygrad.helpers import fetch, argsort+from tinygrad.helpers import fetchfrom tinygrad.nn.state import safe_load, load_state_dictimport json-#640x640 imgs to use:-#https://media.tycsports.com/files/2022/12/03/512501/seleccion-argentina-vs-australia-el-gol-de-lionel-messi-1-1_640x640.webp-#https://media.tycsports.com/files/2025/04/09/830424/jorge-mas-lionel-messi_640x640.webp-#+tie in second-#Model architecture from https://github.com/ultralytics/ultralytics/issues/189#The upsampling class has been taken from this pull request https://github.com/tinygrad/tinygrad/pull/784 by dc-dc-dc. Now 2(?) models use upsampling. (retinet and this)@@ -47,69 +42,9 @@ def preprocess(im, imgsz=640, model_stride=32, model_pt=True):im = im / 255.0 # 0 - 255 to 0.0 - 1.0return im-# Post Processing functions-def box_area(box):- return (box[:, 2] - box[:, 0]) * (box[:, 3] - box[:, 1])--def box_iou(box1, box2):- lt = np.maximum(box1[:, None, :2], box2[:, :2])- rb = np.minimum(box1[:, None, 2:], box2[:, 2:])- wh = np.clip(rb - lt, 0, None)- inter = wh[:, :, 0] * wh[:, :, 1]- area1 = box_area(box1)[:, None]- area2 = box_area(box2)[None, :]- iou = inter / (area1 + area2 - inter)- return iou-def compute_nms(boxes, scores, iou_threshold):- order, keep = scores.argsort()[::-1], []- while order.size > 0:- i = order[0]- keep.append(i)- if order.size == 1:- break- iou = box_iou(boxes[i][None, :], boxes[order[1:]])- inds = np.where(np.atleast_1d(iou.squeeze()) <= iou_threshold)[0]- order = order[inds + 1]- return np.array(keep)--def non_max_suppression(prediction, conf_thres=0.25, iou_thres=0.45, agnostic=False, max_det=300, nc=0, max_wh=7680):- prediction = prediction[0] if isinstance(prediction, (list, tuple)) else prediction- bs, nc = prediction.shape[0], nc or (prediction.shape[1] - 4)- xc = np.amax(prediction[:, 4:4 + nc], axis=1) > conf_thres- nm = prediction.shape[1] - nc - 4- output = [np.zeros((0, 6 + nm))] * bs-- for xi, x in enumerate(prediction):- x = x.swapaxes(0, -1)[xc[xi]]- if not x.shape[0]: continue- box, cls, mask = np.split(x, [4, 4 + nc], axis=1)- conf, j = np.max(cls, axis=1, keepdims=True), np.argmax(cls, axis=1, keepdims=True)- x = np.concatenate((xywh2xyxy(box), conf, j.astype(np.float32), mask), axis=1)- x = x[conf.ravel() > conf_thres]- if not x.shape[0]: continue- x = x[np.argsort(-x[:, 4])]- c = x[:, 5:6] * (0 if agnostic else max_wh)- boxes, scores = x[:, :4] + c, x[:, 4]- i = compute_nms(boxes, scores, iou_thres)[:max_det]- output[xi] = x[i]- return output--def postprocess(preds, img, orig_imgs):- print('copying to CPU now for post processing')- #if you are on CPU, this causes an overflow runtime error. doesn't "seem" to make any difference in the predictions though.- # TODO: make non_max_suppression in tinygrad - to make this faster- preds = preds.numpy() if isinstance(preds, Tensor) else preds- preds = non_max_suppression(prediction=preds, conf_thres=0.25, iou_thres=0.7, agnostic=False, max_det=300)- all_preds = []- for i, pred in enumerate(preds):- orig_img = orig_imgs[i] if isinstance(orig_imgs, list) else orig_imgs- if not isinstance(orig_imgs, Tensor):- pred[:, :4] = scale_boxes(img.shape[2:], pred[:, :4], orig_img.shape)- all_preds.append(pred)- return all_preds--def draw_bounding_boxes_and_save(orig_img_paths, output_img_paths, all_predictions, class_labels, iou_threshold=0.5):+def draw_bounding_boxes_and_save(orig_img_paths, output_img_paths, all_predictions, class_labels,size=640, conf_threshold=0.25):+ all_predictions = [all_predictions]color_dict = {label: tuple((((i+1) * 50) % 256, ((i+1) * 100) % 256, ((i+1) * 150) % 256)) for i, label in enumerate(class_labels)}font = cv2.FONT_HERSHEY_SIMPLEX@@ -131,31 +66,19 @@ def draw_bounding_boxes_and_save(orig_img_paths, output_img_paths, all_predictiofor pred_np in predictions:grouped_preds[int(pred_np[-1])].append(pred_np)- def draw_box_and_label(pred, color):- x1, y1, x2, y2, conf, _ = pred+ for pred in predictions:+ x1, y1, x2, y2, conf, class_id = pred+ if conf < conf_threshold: continue+ color = color_dict[class_labels[int(class_id)]]x1, y1, x2, y2 = map(int, (x1, y1, x2, y2))cv2.rectangle(orig_img, (x1, y1), (x2, y2), color, box_thickness)- label = f"{class_labels[class_id]} {conf:.2f}"+ label = f"{class_labels[int(class_id)]} {conf:.2f}"text_size, _ = cv2.getTextSize(label, font, font_scale, 1)label_y, bg_y = (y1 - 4, y1 - text_size[1] - 4) if y1 - text_size[1] - 4 > 0 else (y1 + text_size[1], y1)cv2.rectangle(orig_img, (x1, bg_y), (x1 + text_size[0], bg_y + text_size[1]), color, -1)font_color = (0, 0, 0) if is_bright_color(color) else (255, 255, 255)cv2.putText(orig_img, label, (x1, label_y), font, font_scale, font_color, 1, cv2.LINE_AA)-- for class_id, pred_list in grouped_preds.items():- pred_list = np.array(pred_list)- while len(pred_list) > 0:- max_conf_idx = np.argmax(pred_list[:, 4])- max_conf_pred = pred_list[max_conf_idx]- pred_list = np.delete(pred_list, max_conf_idx, axis=0)- color = color_dict[class_labels[class_id]]- draw_box_and_label(max_conf_pred, color)- object_count[class_labels[class_id]] += 1- iou_scores = box_iou(np.array([max_conf_pred[:4]]), pred_list[:, :4])- low_iou_indices = np.where(iou_scores[0] < iou_threshold)[0]- pred_list = pred_list[low_iou_indices]- for low_conf_pred in pred_list:- draw_box_and_label(low_conf_pred, color)+ object_count[class_labels[int(class_id)]] += 1print(f"Image {img_idx + 1}:")print("Objects detected:")@@ -202,41 +125,17 @@ def autopad(k, p=None, d=1): # kernel, padding, dilationp = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-padreturn p-def clip_boxes(boxes, shape):- boxes[..., [0, 2]] = np.clip(boxes[..., [0, 2]], 0, shape[1]) # x1, x2- boxes[..., [1, 3]] = np.clip(boxes[..., [1, 3]], 0, shape[0]) # y1, y2- return boxes--def scale_boxes(img1_shape, boxes, img0_shape, ratio_pad=None):- gain = ratio_pad if ratio_pad else min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1])- pad = ((img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2)- boxes_np = boxes.numpy() if isinstance(boxes, Tensor) else boxes- boxes_np[..., [0, 2]] -= pad[0]- boxes_np[..., [1, 3]] -= pad[1]- boxes_np[..., :4] /= gain- boxes_np = clip_boxes(boxes_np, img0_shape)- return boxes_np--def xywh2xyxy(x):- xy = x[..., :2] # center x, y- wh = x[..., 2:4] # width, height- xy1 = xy - wh / 2 # top left x, y- xy2 = xy + wh / 2 # bottom right x, y- result = np.concatenate((xy1, xy2), axis=-1)- return Tensor(result) if isinstance(x, Tensor) else result-def get_variant_multiples(variant):return {'n':(0.33, 0.25, 2.0), 's':(0.33, 0.50, 2.0), 'm':(0.67, 0.75, 1.5), 'l':(1.0, 1.0, 1.0), 'x':(1, 1.25, 1.0) }.get(variant, None)def label_predictions(all_predictions):- class_index_count = defaultdict(int)- for predictions in all_predictions:- predictions = np.array(predictions)- for pred_np in predictions:- class_id = int(pred_np[-1])- class_index_count[class_id] += 1-- return dict(class_index_count)+ predictions_list = all_predictions.numpy()+ class_counts = defaultdict(int)+ for pred in predictions_list:+ prob = pred[4]+ class_id = int(pred[5])+ if prob != 0: class_counts[class_id] += 1+ return dict(class_counts)#this is taken from https://github.com/tinygrad/tinygrad/pull/784/files by dc-dc-dc (Now 2 models use upsampling)class Upsample:@@ -382,11 +281,11 @@ class YOLOv8:self.fpn = Yolov8NECK(w, r, d)self.head = DetectionHead(num_classes, filters=(int(256*w), int(512*w), int(512*w*r)))- def __call__(self, x):+ def __call__(self, x, do_postprocess=True):x = self.net(x)x = self.fpn(*x)x = self.head(x)- return process_output(x)+ return postprocess(x) if do_postprocess else xdef return_all_trainable_modules(self):backbone_modules = [*range(10)]@@ -409,43 +308,53 @@ def convert_f16_safetensor_to_f32(input_file: Path, output_file: Path):f.write(new_metadata_bytes)float32_values.tofile(f)-def intersection_between_box(box1, box2):- x1 = max(box1[0], box2[0])- y1 = max(box1[1], box2[1])- x2 = min(box1[2], box2[2])- y2 = min(box1[3], box2[3])- return max(0, x2 - x1) * max(0, y2 - y1)-def compute_iou_matrix(boxes):- x1 = Tensor.maximum(boxes[:, None, 0],boxes[:, None, 0])- y1 = Tensor.maximum(boxes[:, None, 1], boxes[None, :, 1])- x2 = Tensor.minimum(boxes[:, None, 2], boxes[None, :, 2])- y2 = Tensor.minimum(boxes[:, None, 3], boxes[None, :, 3])- inter = Tensor.maximum(Tensor(0), x2 - x1) * Tensor.maximum(Tensor(0), y2 - y1)- area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])- union = area[:, None] + area[None, :] - inter- return inter / union--def process_output(output, num_predictions=8400, threshold=0.25): #todo 3549 for 416x416, 8400 for 640x640- xc, yc, w, h, class_scores = output[0][0], output[0][1], output[0][2], output[0][3], output[0][4:]- class_ids = Tensor.argmax(class_scores,axis=0)- probs = Tensor.max(class_scores, axis=0)- probs = Tensor.where(probs>=0.25,probs,0)- x1 = xc - w / 2- y1 = yc - h / 2- x2 = xc + w / 2- y2 = yc + h / 2- boxes = Tensor.stack(x1, y1, x2, y2, probs, class_ids, dim=1)- order = Tensor.topk(probs,300)[1]- boxes = boxes[order]- iou = compute_iou_matrix(boxes)- iou_mask = (iou > 0.45)- suppressed = Tensor.triu(iou_mask,diagonal=1)- suppressed = suppressed.any(axis=0)- suppressed = suppressed.reshape(suppressed.shape[0],1)- boxes *= ~suppressed- return boxes+ x1, y1, x2, y2 = boxes[:, 0], boxes[:, 1], boxes[:, 2], boxes[:, 3]+ areas = (x2 - x1) * (y2 - y1)+ x1 = Tensor.maximum(x1[:, None], x1[None, :])+ y1 = Tensor.maximum(y1[:, None], y1[None, :])+ x2 = Tensor.minimum(x2[:, None], x2[None, :])+ y2 = Tensor.minimum(y2[:, None], y2[None, :])+ w = Tensor.maximum(Tensor(0),x2-x1)+ h = Tensor.maximum(Tensor(0),y2-y1)+ intersection = w * h+ iou = intersection / (areas[:, None] + areas[None, :] - intersection)+ return iou+def postprocess(output, max_det=300, conf_threshold=0.25):+ xc, yc, w, h, class_scores = output[0][0], output[0][1], output[0][2], output[0][3], output[0][4:]+ class_ids = Tensor.argmax(class_scores, axis=0)+ probs = Tensor.max(class_scores, axis=0)+ #probs = Tensor.where(probs >= conf_threshold, probs, 0) #dont use line in ios export+ x1 = xc - w / 2+ y1 = yc - h / 2+ x2 = xc + w / 2+ y2 = yc + h / 2+ boxes = Tensor.stack(x1, y1, x2, y2, probs, class_ids, dim=1)+ order = Tensor.topk(probs, max_det)[1]+ boxes = boxes[order]+ iou = compute_iou_matrix(boxes[:, :4])+ iou = Tensor.triu(iou, diagonal=1)+ same_class_mask = boxes[:, -1][:, None] == boxes[:, -1][None, :]+ high_iou_mask = (iou > 0.45) & same_class_mask+ no_overlap_mask = high_iou_mask.sum(axis=0) == 0+ boxes = boxes * no_overlap_mask.unsqueeze(-1)+ return boxes++def clip_boxes(boxes, shape):+ boxes[..., [0, 2]] = np.clip(boxes[..., [0, 2]], 0, shape[1]) # x1, x2+ boxes[..., [1, 3]] = np.clip(boxes[..., [1, 3]], 0, shape[0]) # y1, y2+ return boxes++def scale_boxes(img1_shape, boxes, img0_shape, ratio_pad=None):+ gain = ratio_pad if ratio_pad else min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1])+ pad = ((img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2)+ boxes_np = boxes.numpy() if isinstance(boxes, Tensor) else boxes+ boxes_np[..., [0, 2]] -= pad[0]+ boxes_np[..., [1, 3]] -= pad[1]+ boxes_np[..., :4] /= gain+ boxes_np = clip_boxes(boxes_np, img0_shape)+ return boxes_npdef get_weights_location(yolo_variant: str) -> Path:weights_location = Path(__file__).parents[1] / "weights" / f'yolov8{yolo_variant}.safetensors'@@ -479,7 +388,6 @@ if __name__ == '__main__':print('Error in image loading. Check your image file.')sys.exit(1)pre_processed_image = preprocess(image)-# Different YOLOv8 variants use different w , r, and d multiples. For a list , refer to this yaml file (the scales section) https://github.com/ultralytics/ultralytics/blob/main/ultralytics/cfg/models/v8/yolov8.yamldepth, width, ratio = get_variant_multiples(yolo_variant)yolo_infer = YOLOv8(w=width, r=ratio, d=depth, num_classes=80)@@ -487,12 +395,12 @@ if __name__ == '__main__':load_state_dict(yolo_infer, state_dict)st = time.time()- predictions = yolo_infer(pre_processed_image)+ predictions = yolo_infer(pre_processed_image).numpy()+print(f'did inference in {int(round(((time.time() - st) * 1000)))}ms')- predictions = predictions.numpy()- predictions = [x for x in predictions if x[-2] >= 0.25]- predictions = [predictions]+ #v8 and v3 have same 80 class names for Object Detectionclass_labels = fetch('https://raw.githubusercontent.com/pjreddie/darknet/master/data/coco.names').read_text().split("\n")+ for pred in predictions: pred[:4] = scale_boxes(pre_processed_image.shape[2:],pred[:4],image[0].shape)draw_bounding_boxes_and_save(orig_img_paths=image_location, output_img_paths=out_paths, all_predictions=predictions, class_labels=class_labels)# TODO for later: