Skip to content

Add object detection, tracking, mobile notifications, and search to any security camera.

export model that doesnt threshold

rorycleare7a366b7a374
diff --git a/clearcam/batch_data b/clearcam/batch_data
index 035a45c..de65d65 100644
Binary files a/clearcam/batch_data and b/clearcam/batch_data differ
diff --git a/clearcam/yolov8.py b/clearcam/yolov8.py
index 9eb6f39..72ee4bc 100644
--- a/clearcam/yolov8.py
+++ b/clearcam/yolov8.py
@@ -8,15 +8,10 @@ from pathlib import Path
import cv2
from collections import defaultdict
import time, sys
-from tinygrad.helpers import fetch, argsort
+from tinygrad.helpers import fetch
from tinygrad.nn.state import safe_load, load_state_dict
import json
-#640x640 imgs to use:
-#https://media.tycsports.com/files/2022/12/03/512501/seleccion-argentina-vs-australia-el-gol-de-lionel-messi-1-1_640x640.webp
-#https://media.tycsports.com/files/2025/04/09/830424/jorge-mas-lionel-messi_640x640.webp
-#+tie in second
-
#Model architecture from https://github.com/ultralytics/ultralytics/issues/189
#The upsampling class has been taken from this pull request https://github.com/tinygrad/tinygrad/pull/784 by dc-dc-dc. Now 2(?) models use upsampling. (retinet and this)
@@ -47,69 +42,9 @@ def preprocess(im, imgsz=640, model_stride=32, model_pt=True):
im = im / 255.0 # 0 - 255 to 0.0 - 1.0
return im
-# Post Processing functions
-def box_area(box):
- return (box[:, 2] - box[:, 0]) * (box[:, 3] - box[:, 1])
-
-def box_iou(box1, box2):
- lt = np.maximum(box1[:, None, :2], box2[:, :2])
- rb = np.minimum(box1[:, None, 2:], box2[:, 2:])
- wh = np.clip(rb - lt, 0, None)
- inter = wh[:, :, 0] * wh[:, :, 1]
- area1 = box_area(box1)[:, None]
- area2 = box_area(box2)[None, :]
- iou = inter / (area1 + area2 - inter)
- return iou
-def compute_nms(boxes, scores, iou_threshold):
- order, keep = scores.argsort()[::-1], []
- while order.size > 0:
- i = order[0]
- keep.append(i)
- if order.size == 1:
- break
- iou = box_iou(boxes[i][None, :], boxes[order[1:]])
- inds = np.where(np.atleast_1d(iou.squeeze()) <= iou_threshold)[0]
- order = order[inds + 1]
- return np.array(keep)
-
-def non_max_suppression(prediction, conf_thres=0.25, iou_thres=0.45, agnostic=False, max_det=300, nc=0, max_wh=7680):
- prediction = prediction[0] if isinstance(prediction, (list, tuple)) else prediction
- bs, nc = prediction.shape[0], nc or (prediction.shape[1] - 4)
- xc = np.amax(prediction[:, 4:4 + nc], axis=1) > conf_thres
- nm = prediction.shape[1] - nc - 4
- output = [np.zeros((0, 6 + nm))] * bs
-
- for xi, x in enumerate(prediction):
- x = x.swapaxes(0, -1)[xc[xi]]
- if not x.shape[0]: continue
- box, cls, mask = np.split(x, [4, 4 + nc], axis=1)
- conf, j = np.max(cls, axis=1, keepdims=True), np.argmax(cls, axis=1, keepdims=True)
- x = np.concatenate((xywh2xyxy(box), conf, j.astype(np.float32), mask), axis=1)
- x = x[conf.ravel() > conf_thres]
- if not x.shape[0]: continue
- x = x[np.argsort(-x[:, 4])]
- c = x[:, 5:6] * (0 if agnostic else max_wh)
- boxes, scores = x[:, :4] + c, x[:, 4]
- i = compute_nms(boxes, scores, iou_thres)[:max_det]
- output[xi] = x[i]
- return output
-
-def postprocess(preds, img, orig_imgs):
- print('copying to CPU now for post processing')
- #if you are on CPU, this causes an overflow runtime error. doesn't "seem" to make any difference in the predictions though.
- # TODO: make non_max_suppression in tinygrad - to make this faster
- preds = preds.numpy() if isinstance(preds, Tensor) else preds
- preds = non_max_suppression(prediction=preds, conf_thres=0.25, iou_thres=0.7, agnostic=False, max_det=300)
- all_preds = []
- for i, pred in enumerate(preds):
- orig_img = orig_imgs[i] if isinstance(orig_imgs, list) else orig_imgs
- if not isinstance(orig_imgs, Tensor):
- pred[:, :4] = scale_boxes(img.shape[2:], pred[:, :4], orig_img.shape)
- all_preds.append(pred)
- return all_preds
-
-def draw_bounding_boxes_and_save(orig_img_paths, output_img_paths, all_predictions, class_labels, iou_threshold=0.5):
+def draw_bounding_boxes_and_save(orig_img_paths, output_img_paths, all_predictions, class_labels,size=640, conf_threshold=0.25):
+ all_predictions = [all_predictions]
color_dict = {label: tuple((((i+1) * 50) % 256, ((i+1) * 100) % 256, ((i+1) * 150) % 256)) for i, label in enumerate(class_labels)}
font = cv2.FONT_HERSHEY_SIMPLEX
@@ -131,31 +66,19 @@ def draw_bounding_boxes_and_save(orig_img_paths, output_img_paths, all_predictio
for pred_np in predictions:
grouped_preds[int(pred_np[-1])].append(pred_np)
- def draw_box_and_label(pred, color):
- x1, y1, x2, y2, conf, _ = pred
+ for pred in predictions:
+ x1, y1, x2, y2, conf, class_id = pred
+ if conf < conf_threshold: continue
+ color = color_dict[class_labels[int(class_id)]]
x1, y1, x2, y2 = map(int, (x1, y1, x2, y2))
cv2.rectangle(orig_img, (x1, y1), (x2, y2), color, box_thickness)
- label = f"{class_labels[class_id]} {conf:.2f}"
+ label = f"{class_labels[int(class_id)]} {conf:.2f}"
text_size, _ = cv2.getTextSize(label, font, font_scale, 1)
label_y, bg_y = (y1 - 4, y1 - text_size[1] - 4) if y1 - text_size[1] - 4 > 0 else (y1 + text_size[1], y1)
cv2.rectangle(orig_img, (x1, bg_y), (x1 + text_size[0], bg_y + text_size[1]), color, -1)
font_color = (0, 0, 0) if is_bright_color(color) else (255, 255, 255)
cv2.putText(orig_img, label, (x1, label_y), font, font_scale, font_color, 1, cv2.LINE_AA)
-
- for class_id, pred_list in grouped_preds.items():
- pred_list = np.array(pred_list)
- while len(pred_list) > 0:
- max_conf_idx = np.argmax(pred_list[:, 4])
- max_conf_pred = pred_list[max_conf_idx]
- pred_list = np.delete(pred_list, max_conf_idx, axis=0)
- color = color_dict[class_labels[class_id]]
- draw_box_and_label(max_conf_pred, color)
- object_count[class_labels[class_id]] += 1
- iou_scores = box_iou(np.array([max_conf_pred[:4]]), pred_list[:, :4])
- low_iou_indices = np.where(iou_scores[0] < iou_threshold)[0]
- pred_list = pred_list[low_iou_indices]
- for low_conf_pred in pred_list:
- draw_box_and_label(low_conf_pred, color)
+ object_count[class_labels[int(class_id)]] += 1
print(f"Image {img_idx + 1}:")
print("Objects detected:")
@@ -202,41 +125,17 @@ def autopad(k, p=None, d=1): # kernel, padding, dilation
p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad
return p
-def clip_boxes(boxes, shape):
- boxes[..., [0, 2]] = np.clip(boxes[..., [0, 2]], 0, shape[1]) # x1, x2
- boxes[..., [1, 3]] = np.clip(boxes[..., [1, 3]], 0, shape[0]) # y1, y2
- return boxes
-
-def scale_boxes(img1_shape, boxes, img0_shape, ratio_pad=None):
- gain = ratio_pad if ratio_pad else min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1])
- pad = ((img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2)
- boxes_np = boxes.numpy() if isinstance(boxes, Tensor) else boxes
- boxes_np[..., [0, 2]] -= pad[0]
- boxes_np[..., [1, 3]] -= pad[1]
- boxes_np[..., :4] /= gain
- boxes_np = clip_boxes(boxes_np, img0_shape)
- return boxes_np
-
-def xywh2xyxy(x):
- xy = x[..., :2] # center x, y
- wh = x[..., 2:4] # width, height
- xy1 = xy - wh / 2 # top left x, y
- xy2 = xy + wh / 2 # bottom right x, y
- result = np.concatenate((xy1, xy2), axis=-1)
- return Tensor(result) if isinstance(x, Tensor) else result
-
def get_variant_multiples(variant):
return {'n':(0.33, 0.25, 2.0), 's':(0.33, 0.50, 2.0), 'm':(0.67, 0.75, 1.5), 'l':(1.0, 1.0, 1.0), 'x':(1, 1.25, 1.0) }.get(variant, None)
def label_predictions(all_predictions):
- class_index_count = defaultdict(int)
- for predictions in all_predictions:
- predictions = np.array(predictions)
- for pred_np in predictions:
- class_id = int(pred_np[-1])
- class_index_count[class_id] += 1
-
- return dict(class_index_count)
+ predictions_list = all_predictions.numpy()
+ class_counts = defaultdict(int)
+ for pred in predictions_list:
+ prob = pred[4]
+ class_id = int(pred[5])
+ if prob != 0: class_counts[class_id] += 1
+ return dict(class_counts)
#this is taken from https://github.com/tinygrad/tinygrad/pull/784/files by dc-dc-dc (Now 2 models use upsampling)
class Upsample:
@@ -382,11 +281,11 @@ class YOLOv8:
self.fpn = Yolov8NECK(w, r, d)
self.head = DetectionHead(num_classes, filters=(int(256*w), int(512*w), int(512*w*r)))
- def __call__(self, x):
+ def __call__(self, x, do_postprocess=True):
x = self.net(x)
x = self.fpn(*x)
x = self.head(x)
- return process_output(x)
+ return postprocess(x) if do_postprocess else x
def return_all_trainable_modules(self):
backbone_modules = [*range(10)]
@@ -409,43 +308,53 @@ def convert_f16_safetensor_to_f32(input_file: Path, output_file: Path):
f.write(new_metadata_bytes)
float32_values.tofile(f)
-def intersection_between_box(box1, box2):
- x1 = max(box1[0], box2[0])
- y1 = max(box1[1], box2[1])
- x2 = min(box1[2], box2[2])
- y2 = min(box1[3], box2[3])
- return max(0, x2 - x1) * max(0, y2 - y1)
-
def compute_iou_matrix(boxes):
- x1 = Tensor.maximum(boxes[:, None, 0],boxes[:, None, 0])
- y1 = Tensor.maximum(boxes[:, None, 1], boxes[None, :, 1])
- x2 = Tensor.minimum(boxes[:, None, 2], boxes[None, :, 2])
- y2 = Tensor.minimum(boxes[:, None, 3], boxes[None, :, 3])
- inter = Tensor.maximum(Tensor(0), x2 - x1) * Tensor.maximum(Tensor(0), y2 - y1)
- area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
- union = area[:, None] + area[None, :] - inter
- return inter / union
-
-def process_output(output, num_predictions=8400, threshold=0.25): #todo 3549 for 416x416, 8400 for 640x640
- xc, yc, w, h, class_scores = output[0][0], output[0][1], output[0][2], output[0][3], output[0][4:]
- class_ids = Tensor.argmax(class_scores,axis=0)
- probs = Tensor.max(class_scores, axis=0)
- probs = Tensor.where(probs>=0.25,probs,0)
- x1 = xc - w / 2
- y1 = yc - h / 2
- x2 = xc + w / 2
- y2 = yc + h / 2
- boxes = Tensor.stack(x1, y1, x2, y2, probs, class_ids, dim=1)
- order = Tensor.topk(probs,300)[1]
- boxes = boxes[order]
- iou = compute_iou_matrix(boxes)
- iou_mask = (iou > 0.45)
- suppressed = Tensor.triu(iou_mask,diagonal=1)
- suppressed = suppressed.any(axis=0)
- suppressed = suppressed.reshape(suppressed.shape[0],1)
- boxes *= ~suppressed
- return boxes
+ x1, y1, x2, y2 = boxes[:, 0], boxes[:, 1], boxes[:, 2], boxes[:, 3]
+ areas = (x2 - x1) * (y2 - y1)
+ x1 = Tensor.maximum(x1[:, None], x1[None, :])
+ y1 = Tensor.maximum(y1[:, None], y1[None, :])
+ x2 = Tensor.minimum(x2[:, None], x2[None, :])
+ y2 = Tensor.minimum(y2[:, None], y2[None, :])
+ w = Tensor.maximum(Tensor(0),x2-x1)
+ h = Tensor.maximum(Tensor(0),y2-y1)
+ intersection = w * h
+ iou = intersection / (areas[:, None] + areas[None, :] - intersection)
+ return iou
+def postprocess(output, max_det=300, conf_threshold=0.25):
+ xc, yc, w, h, class_scores = output[0][0], output[0][1], output[0][2], output[0][3], output[0][4:]
+ class_ids = Tensor.argmax(class_scores, axis=0)
+ probs = Tensor.max(class_scores, axis=0)
+ #probs = Tensor.where(probs >= conf_threshold, probs, 0) #dont use line in ios export
+ x1 = xc - w / 2
+ y1 = yc - h / 2
+ x2 = xc + w / 2
+ y2 = yc + h / 2
+ boxes = Tensor.stack(x1, y1, x2, y2, probs, class_ids, dim=1)
+ order = Tensor.topk(probs, max_det)[1]
+ boxes = boxes[order]
+ iou = compute_iou_matrix(boxes[:, :4])
+ iou = Tensor.triu(iou, diagonal=1)
+ same_class_mask = boxes[:, -1][:, None] == boxes[:, -1][None, :]
+ high_iou_mask = (iou > 0.45) & same_class_mask
+ no_overlap_mask = high_iou_mask.sum(axis=0) == 0
+ boxes = boxes * no_overlap_mask.unsqueeze(-1)
+ return boxes
+
+def clip_boxes(boxes, shape):
+ boxes[..., [0, 2]] = np.clip(boxes[..., [0, 2]], 0, shape[1]) # x1, x2
+ boxes[..., [1, 3]] = np.clip(boxes[..., [1, 3]], 0, shape[0]) # y1, y2
+ return boxes
+
+def scale_boxes(img1_shape, boxes, img0_shape, ratio_pad=None):
+ gain = ratio_pad if ratio_pad else min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1])
+ pad = ((img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2)
+ boxes_np = boxes.numpy() if isinstance(boxes, Tensor) else boxes
+ boxes_np[..., [0, 2]] -= pad[0]
+ boxes_np[..., [1, 3]] -= pad[1]
+ boxes_np[..., :4] /= gain
+ boxes_np = clip_boxes(boxes_np, img0_shape)
+ return boxes_np
def get_weights_location(yolo_variant: str) -> Path:
weights_location = Path(__file__).parents[1] / "weights" / f'yolov8{yolo_variant}.safetensors'
@@ -479,7 +388,6 @@ if __name__ == '__main__':
print('Error in image loading. Check your image file.')
sys.exit(1)
pre_processed_image = preprocess(image)
-
# Different YOLOv8 variants use different w , r, and d multiples. For a list , refer to this yaml file (the scales section) https://github.com/ultralytics/ultralytics/blob/main/ultralytics/cfg/models/v8/yolov8.yaml
depth, width, ratio = get_variant_multiples(yolo_variant)
yolo_infer = YOLOv8(w=width, r=ratio, d=depth, num_classes=80)
@@ -487,12 +395,12 @@ if __name__ == '__main__':
load_state_dict(yolo_infer, state_dict)
st = time.time()
- predictions = yolo_infer(pre_processed_image)
+ predictions = yolo_infer(pre_processed_image).numpy()
+
print(f'did inference in {int(round(((time.time() - st) * 1000)))}ms')
- predictions = predictions.numpy()
- predictions = [x for x in predictions if x[-2] >= 0.25]
- predictions = [predictions]
+ #v8 and v3 have same 80 class names for Object Detection
class_labels = fetch('https://raw.githubusercontent.com/pjreddie/darknet/master/data/coco.names').read_text().split("\n")
+ for pred in predictions: pred[:4] = scale_boxes(pre_processed_image.shape[2:],pred[:4],image[0].shape)
draw_bounding_boxes_and_save(orig_img_paths=image_location, output_img_paths=out_paths, all_predictions=predictions, class_labels=class_labels)
# TODO for later: