Skip to content

Add object detection, tracking, mobile notifications, and search to any security camera.

11.0 KB
Raw Blame
from tinygrad import Tensor, nn, TinyJit
from tinygrad.nn.state import safe_save, safe_load, get_state_dict, load_state_dict
from utils.helpers import resize

class BlazeBlock():
    def __init__(self, c=None, channel_pad=0):
        if c is not None:
            self.stride = c.stride
            self.channel_pad = c.channel_pad
            self.conv0_tiny = c.conv0_tiny
            self.conv1_tiny = c.conv1_tiny
            return
        
        self.channel_pad = channel_pad
    
    def __call__(self, x):
        if self.stride == 2:
            h = x.pad(((0, 0), (0, 0), (0, 2), (0, 2)))
            x = x.max_pool2d(self.stride, self.stride)
        else:
            h = x

        if self.channel_pad > 0:
            x = x.pad(((0, 0), (0, self.channel_pad), (0, 0), (0, 0)))


        h = self.conv0_tiny(h)
        h = self.conv1_tiny(h)
        x += h
        x = x.relu()
        return x

class Seq():
    def __init__(self, size=0):
        super().__init__()
        self.list = [None] * size
    def __len__(self): return len(self.list)
    def __setitem__(self, key, value): self.list[key] = value
    def __getitem__(self, idx): return self.list[idx]
    def __call__(self, x):
        for y in self.list: x = y(x)
        return x

class FinalBlazeBlock():
    def __init__(self, f=None):
        if f is not None:
            self.act = f.act
            self.convs = f.convs
            self.conv0_tiny = f.conv0_tiny
            self.conv1_tiny = f.conv1_tiny
            return
        
        self.conv0_tiny = nn.Conv2d(96, 96, kernel_size=3, stride=2, padding=0, groups=96, bias=True)
        self.conv1_tiny = nn.Conv2d(96, 96, kernel_size=1, stride=1, padding=0, bias=True)

    def __call__(self, x):
        x = x.pad(((0, 0), (0, 0), (0, 2), (0, 2)))
        x = self.conv0_tiny(x)
        x = self.conv1_tiny(x)
        x = x.relu()
        return x

class BlazeFace():
    def __init__(self, m=None):
        if m is not None:
            self.backbone_tiny = m.backbone_tiny
            self.conv_tiny = m.conv_tiny
            self.classifier_8 = m.classifier_8
            self.classifier_16 = m.classifier_16
            self.regressor_8 = m.regressor_8
            self.regressor_16 = m.regressor_16
            self.anchors = m.anchors
            self.x_scale = m.x_scale
            self.y_scale = m.y_scale
            self.w_scale = m.w_scale
            self.h_scale = m.h_scale
            self.score_clipping_thresh = m.score_clipping_thresh
            self.min_score_thresh = m.min_score_thresh
            self.min_suppression_threshold = m.min_suppression_threshold
            self.final = m.final
            return
        
        self.conv_tiny = nn.Conv2d(in_channels=3, out_channels=24, kernel_size=5, stride=2, padding=0, bias=True)
        self.classifier_8_tiny = nn.Conv2d(in_channels=96, out_channels=2, kernel_size=1, groups=1, bias=True)
        self.classifier_16_tiny = nn.Conv2d(in_channels=96, out_channels=6, kernel_size=1, groups=1, bias=True)
        self.regressor_8_tiny = nn.Conv2d(in_channels=96, out_channels=32, kernel_size=1, groups=1, bias=True)
        self.regressor_16_tiny = nn.Conv2d(in_channels=96, out_channels=96, kernel_size=1, groups=1, bias=True)

        self.final = FinalBlazeBlock()
        self.backbone_tiny = Seq(31)
        for i in range(7):
            self.backbone_tiny[i] = BlazeBlock()
            self.backbone_tiny[i].conv0_tiny = nn.Conv2d(24, 24, kernel_size=3, stride=1, padding=1, groups=24, bias=True)
            self.backbone_tiny[i].conv1_tiny = nn.Conv2d(24, 24, kernel_size=1, stride=1, padding=0, groups=1, bias=True)
            self.backbone_tiny[i].stride = 1
        self.backbone_tiny[7] = BlazeBlock()
        self.backbone_tiny[7].conv0_tiny = nn.Conv2d(24, 24, kernel_size=3, stride=2, padding=0, groups=24, bias=True)
        self.backbone_tiny[7].conv1_tiny = nn.Conv2d(24, 24, kernel_size=1, stride=1, padding=0, groups=1, bias=True)
        self.backbone_tiny[7].stride = 2
        for i in range(8, 15):
            self.backbone_tiny[i] = BlazeBlock()
            self.backbone_tiny[i].conv0_tiny = nn.Conv2d(24, 24, kernel_size=3, stride=1, padding=1, groups=24, bias=True)
            self.backbone_tiny[i].conv1_tiny = nn.Conv2d(24, 24, kernel_size=1, stride=1, padding=0, groups=1, bias=True)
            self.backbone_tiny[i].stride = 1
        self.backbone_tiny[15] = BlazeBlock(channel_pad=24)
        self.backbone_tiny[15].conv0_tiny = nn.Conv2d(24, 24, kernel_size=3, stride=2, padding=0, groups=24, bias=True)
        self.backbone_tiny[15].conv1_tiny = nn.Conv2d(24, 48, kernel_size=1, stride=1, padding=0, groups=1, bias=True)
        self.backbone_tiny[15].stride = 2
        for i in range(16, 23):
            self.backbone_tiny[i] = BlazeBlock()
            self.backbone_tiny[i].conv0_tiny = nn.Conv2d(48, 48, kernel_size=3, stride=1, padding=1, groups=48, bias=True)
            self.backbone_tiny[i].conv1_tiny = nn.Conv2d(48, 48, kernel_size=1, stride=1, padding=0, groups=1, bias=True)
            self.backbone_tiny[i].stride = 1
        self.backbone_tiny[23] = BlazeBlock(channel_pad=48)
        self.backbone_tiny[23].conv0_tiny = nn.Conv2d(48, 48, kernel_size=3, stride=2, padding=0, groups=48, bias=True)
        self.backbone_tiny[23].conv1_tiny = nn.Conv2d(48, 96, kernel_size=1, stride=1, padding=0, groups=1, bias=True)
        self.backbone_tiny[23].stride = 2
        for i in range(24, 31):
            self.backbone_tiny[i] = BlazeBlock()
            self.backbone_tiny[i].conv0_tiny = nn.Conv2d(96, 96, kernel_size=3, stride=1, padding=1, groups=96, bias=True)
            self.backbone_tiny[i].conv1_tiny = nn.Conv2d(96, 96, kernel_size=1, stride=1, padding=0, groups=1, bias=True)
            self.backbone_tiny[i].stride = 1

        self.anchors = Tensor.empty(896, 4)
        self.num_classes = 1
        self.num_anchors = 896
        self.num_coords = 16
        self.score_clipping_thresh = 100.0

        self.x_scale = 256.0
        self.y_scale = 256.0
        self.h_scale = 256.0
        self.w_scale = 256.0
        self.min_score_thresh = 0.85
        self.min_suppression_threshold = 0.3

        load_state_dict(self, safe_load("models/blazeface.safetensors"))


    def forward(self, x):
        x = x.pad(((0, 0), (0, 0), (1, 2), (1, 2)))
        b = x.shape[0]      # batch size, needed for reshaping later
        x = self.conv_tiny(x)
        x = x.relu()
        x = self.backbone_tiny(x)           # (b, 16, 16, 96)
        h = self.final(x)              # (b, 8, 8, 96)
        c1 = self.classifier_8_tiny(x)       # (b, 2, 16, 16)
        c1 = c1.permute(0, 2, 3, 1)     # (b, 16, 16, 2)
        c1 = c1.reshape(b, -1, 1)       # (b, 512, 1)

        c2 = self.classifier_16_tiny(h)      # (b, 6, 8, 8)
        c2 = c2.permute(0, 2, 3, 1)     # (b, 8, 8, 6)
        c2 = c2.reshape(b, -1, 1)       # (b, 384, 1)

        c = Tensor.cat(c1, c2, dim=1)
        r1 = self.regressor_8_tiny(x)        # (b, 32, 16, 16)
        r1 = r1.permute(0, 2, 3, 1)     # (b, 16, 16, 32)
        r1 = r1.reshape(b, -1, 16)      # (b, 512, 16)
        
        r2 = self.regressor_16_tiny(h)       # (b, 96, 8, 8)
        r2 = r2.permute(0, 2, 3, 1)     # (b, 8, 8, 96)
        r2 = r2.reshape(b, -1, 16)      # (b, 384, 16)

        r = Tensor.cat(r1, r2, dim=1)
        return [r, c]

    @TinyJit
    def __call__(self, img):
        h0, w0 = img.shape[:2]

        scale = min(256/w0, 256/h0)
        new_w, new_h = int(w0*scale), int(h0*scale)
        img = resize(img, [new_w, new_h])
        pad_top = (256 - new_h) // 2
        pad_bottom = 256 - new_h - pad_top
        pad_left = (256 - new_w) // 2
        pad_right = 256 - new_w - pad_left
        x = img.pad(((pad_top, pad_bottom), (pad_left, pad_right), (0,0)), value=0)
        x = x.permute((2, 0, 1))
        x = x.unsqueeze(0)
        x = x / 127.5 - 1.0
        out = self.forward(x)

        detections = self._tensors_to_detections(out[0], out[1], self.anchors)

        detections = self.postprocess(detections)[0]

        detections = detections * 256
        detections[:, [0, 2]] -= pad_top   # ymin, ymax
        detections[:, [1, 3]] -= pad_left  # xmin, xmax
        detections /= scale
        return detections

    def _tensors_to_detections(self, raw_box_tensor, raw_score_tensor, anchors):
        detection_boxes = self._decode_boxes(raw_box_tensor, anchors)  # (B, N, 16)
        thresh = self.score_clipping_thresh
        scores = raw_score_tensor.clip(-thresh, thresh).sigmoid().squeeze(-1)
        mask = scores >= self.min_score_thresh  # (B, N)
        scores = scores.unsqueeze(-1)  # (B, N, 1)
        detections = Tensor.cat(detection_boxes, scores, dim=-1)  # (B, N, 17)
        detections *= mask.unsqueeze(-1)
        return detections[0]
    
    def _decode_boxes(self, raw_boxes, anchors):
        boxes = Tensor.zeros_like(raw_boxes).contiguous()
        ax = anchors[:, 0]
        ay = anchors[:, 1]
        aw = anchors[:, 2]
        ah = anchors[:, 3]
        x_center = raw_boxes[..., 0] / self.x_scale * aw + ax
        y_center = raw_boxes[..., 1] / self.y_scale * ah + ay
        w = raw_boxes[..., 2] / self.w_scale * aw
        h = raw_boxes[..., 3] / self.h_scale * ah
        boxes[..., 0] = y_center - h / 2.  # ymin
        boxes[..., 1] = x_center - w / 2.  # xmin
        boxes[..., 2] = y_center + h / 2.  # ymax
        boxes[..., 3] = x_center + w / 2.  # xmax
        keypoints = raw_boxes[..., 4:].view(*raw_boxes.shape[:-1], 6, 2)
        kp_x = keypoints[..., 0] / self.x_scale * aw.unsqueeze(0).unsqueeze(-1) + ax.unsqueeze(0).unsqueeze(-1)
        kp_y = keypoints[..., 1] / self.y_scale * ah.unsqueeze(0).unsqueeze(-1) + ay.unsqueeze(0).unsqueeze(-1)
        keypoints_decoded = Tensor.stack((kp_x, kp_y), dim=-1)  # (B, N, 6, 2)
        boxes[..., 4:] = keypoints_decoded.view(*raw_boxes.shape[:-1], -1)
        return boxes

    def postprocess(self, boxes):
        boxes = boxes.unsqueeze(0)
        max_det = 896
        iou_threshold = 0.3
        probs = boxes[:, :, 16] 
        order_all = Tensor.topk(probs, min(max_det, probs.shape[1]))[1]
        batch_idx = Tensor.arange(order_all.shape[0]).reshape(-1, 1)
        boxes = boxes[batch_idx, order_all]
        ious = compute_iou_matrix(boxes[:, :, :4])
        ious = Tensor.triu(ious, diagonal=1)
        high_iou_mask = (ious > iou_threshold)
        no_overlap_mask = high_iou_mask.sum(axis=1) == 0
        conf_mask = (boxes[:, :, 16] >= self.min_score_thresh)
        final_mask = no_overlap_mask * conf_mask
        return boxes * final_mask.unsqueeze(-1)

def compute_iou_matrix(boxes):
  x1s = boxes[:, :, 0]
  y1s = boxes[:, :, 1]
  x2s = boxes[:, :, 2]
  y2s = boxes[:, :, 3]
  areas = (x2s - x1s) * (y2s - y1s)
  x1 = Tensor.maximum(x1s[:, :, None], x1s[:, None, :])
  y1 = Tensor.maximum(y1s[:, :, None], y1s[:, None, :])
  x2 = Tensor.minimum(x2s[:, :, None], x2s[:, None, :])
  y2 = Tensor.minimum(y2s[:, :, None], y2s[:, None, :])
  w = Tensor.maximum(Tensor(0), x2 - x1)
  h = Tensor.maximum(Tensor(0), y2 - y1)
  intersection = w * h
  union = areas[:, :, None] + areas[:, None, :] - intersection
  return intersection / union