Skip to content

Add object detection, tracking, mobile notifications, and search to any security camera.

remove detr

roryclearb67f3d53a7d8
diff --git a/clearcam.py b/clearcam.py
index 6ee5bfa..d4d990f 100644
--- a/clearcam.py
+++ b/clearcam.py
@@ -2,7 +2,6 @@ from tinygrad.tensor import Tensor
from tinygrad.helpers import fetch
from detection.yolov9 import YOLOv9
from llm.qwen3vl import Qwen3VL
-from detection.rfdetr import RFDETR, detr_to_yolo
import numpy as np
from pathlib import Path
import cv2
@@ -585,8 +584,6 @@ class VideoCapture:
thresh = (self.settings[cam_name].get("threshold") if self.settings[cam_name] else 0.5) or 0.5 #todo clean!
online_targets = self.tracker[cam_name].update(preds, thresh)
online_targets = [p for p in online_targets if (classes is None or str(int(p.class_id)) in classes)]
- if type(model) == RFDETR: # RF-DETR has different class_ids
- for j in range(len(online_targets)): online_targets[j].class_id = detr_to_yolo[int(online_targets[j].class_id)]
preds = []
for x in online_targets:
if x.tracklet_len < 1: continue # dont alert for 1 frame, too many false positives.
diff --git a/detection/rfdetr.py b/detection/rfdetr.py
deleted file mode 100644
index 56f7534..0000000
--- a/detection/rfdetr.py
+++ /dev/null
@@ -1,707 +0,0 @@
-import numpy as np
-import math
-from tinygrad.dtype import dtypes
-from tinygrad.nn.state import get_state_dict, load_state_dict, safe_save, safe_load
-from tinygrad.helpers import fetch
-from tinygrad import Tensor, nn
-from tinygrad import TinyJit
-from utils.helpers import resize
-
-COCO_CLASSES = ["","person","bicycle","car","motorcycle","airplane","bus","train","truck","boat","traffic light","fire hydrant","","stop sign","parking meter","bench","bird","cat","dog","horse","sheep","cow","elephant","bear","zebra","giraffe","","backpack","umbrella","","","handbag","tie","suitcase","frisbee","skis","snowboard","sports ball","kite","baseball bat","baseball glove","skateboard","surfboard","tennis racket","bottle","","wine glass","cup","fork","knife","spoon","bowl","banana","apple","sandwich","orange","broccoli","carrot","hot dog","pizza","donut","cake","chair","couch","potted plant","bed","","dining table","","","toilet","","tv","laptop","mouse","remote","keyboard","cell phone","microwave","oven","toaster","sink","refrigerator","","book","clock","vase","scissors","teddy bear","hair drier"]
-detr_to_yolo = [80, 0, 1, 2, -1, -1, 5, 6, 7, 8, 9, 10, 80, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 80, 24, 25, 80, 80, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 80, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, -1, -1, 59, 80, -1, 80, 80, 61, 80, -1, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 80, 73, 74, 75, 76, 77, 78]
-
-class WindowedDinov2WithRegistersEmbeddings():
- def __call__(self, pixel_values):
- batch_size, _, height, width = pixel_values.shape
- embeddings = self.projection(pixel_values).flatten(2).transpose(1, 2)
-
- cls_tokens = self.cls_token.expand(batch_size, -1, -1)
- embeddings = Tensor.cat(cls_tokens, embeddings, dim=1)
- embeddings = embeddings + self.position_embeddings
- num_h_patches = height // 16
- num_w_patches = width // 16
- cls_token_with_pos_embed = embeddings[:, :1]
- pixel_tokens_with_pos_embed = embeddings[:, 1:]
- pixel_tokens_with_pos_embed = pixel_tokens_with_pos_embed.view(batch_size, num_h_patches, num_w_patches, -1)
-
- num_w_patches_per_window = num_w_patches // self.num_windows
- num_h_patches_per_window = num_h_patches // self.num_windows
- windowed_pixel_tokens = pixel_tokens_with_pos_embed.reshape(batch_size, self.num_windows, num_h_patches_per_window, self.num_windows, num_w_patches_per_window, -1)
- windowed_pixel_tokens = windowed_pixel_tokens.permute(0,1,3,2,4,5)
- windowed_pixel_tokens = windowed_pixel_tokens.reshape(batch_size * self.num_windows**2, num_h_patches_per_window * num_w_patches_per_window, -1)
-
- windowed_cls_token_with_pos_embed = cls_token_with_pos_embed.repeat(self.num_windows ** 2, 1, 1)
- embeddings = Tensor.cat(windowed_cls_token_with_pos_embed, windowed_pixel_tokens, dim=1)
- return embeddings
-
-class Dinov2WithRegistersSdpaSelfAttention():
- def transpose_for_scores(self, x):
- new_x_shape = x.size()[:-1] + (6, 64)
- x = x.view(new_x_shape)
- return x.permute(0, 2, 1, 3)
-
- def __call__(self, hidden_states, head_mask, output_attentions):
- mixed_query_layer = self.query(hidden_states)
- key_layer = self.transpose_for_scores(self.key(hidden_states))
- value_layer = self.transpose_for_scores(self.value(hidden_states))
- query_layer = self.transpose_for_scores(mixed_query_layer)
-
- d_k = query_layer.size(-1)
- attn_scores = Tensor.matmul(query_layer, key_layer.transpose(-2, -1)) / math.sqrt(d_k)
- attn_probs = Tensor.softmax(attn_scores, axis=-1)
- context_layer = Tensor.matmul(attn_probs, value_layer)
- context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
- new_context_layer_shape = context_layer.size()[:-2] + (384,)
- context_layer = context_layer.view(new_context_layer_shape)
- return context_layer, None
-
-class Dinov2WithRegistersSdpaAttention():
- def __call__(self, hidden_states, head_mask=None, output_attentions= False):
- self_outputs = self.attention(hidden_states, head_mask, output_attentions)
- attention_output = self.dense(self_outputs[0])
- return (attention_output,) + self_outputs[1:]
-
-class Dinov2WithRegistersMLP():
- def __call__(self, hidden_state):
- hidden_state = self.fc1(hidden_state)
- hidden_state = hidden_state * 0.5 * (1.0 + Tensor.erf(hidden_state / math.sqrt(2.0)))
- hidden_state = self.fc2(hidden_state)
- return hidden_state
-
-class WindowedDinov2WithRegistersLayer():
- def __call__(self, hidden_states, head_mask=None, output_attentions= False, run_full_attention= False):
- shortcut = hidden_states
- if run_full_attention:
- B, HW, C = hidden_states.shape
- num_windows_squared = self.num_windows ** 2
- hidden_states = hidden_states.view(B // num_windows_squared, num_windows_squared * HW, C)
- x = self.norm1(hidden_states)
-
- # todo
- self_attention_outputs = self.attention(x, head_mask, output_attentions=output_attentions,)
- attention_output = self_attention_outputs[0]
-
- if run_full_attention:
- B, HW, C = hidden_states.shape
- num_windows_squared = self.num_windows ** 2
- attention_output = attention_output.view(B * num_windows_squared, HW // num_windows_squared, C)
- attention_output = (attention_output) * self.lambda1
- outputs = self_attention_outputs[1:]
- hidden_states = attention_output + shortcut
-
- # in Dinov2WithRegisters, layernorm is also applied after self-attention
- layer_output = self.norm2(hidden_states)
- layer_output = self.mlp(layer_output)
- layer_output = layer_output * self.lambda2
- layer_output = layer_output + hidden_states
- return (layer_output,) + outputs
-
-class WindowedDinov2WithRegistersEncoder():
- def __call__(self, hidden_states, head_mask=None, output_attentions=False, output_hidden_states=False, return_dict=True,):
- all_hidden_states = () if output_hidden_states else None
- all_self_attentions = () if output_attentions else None
-
- for i, layer_module in enumerate(self.layer):
- all_hidden_states = all_hidden_states + (hidden_states,)
- run_full_attention = i not in [0, 1, 2, 4, 5, 7, 8, 10, 11]
- layer_head_mask = None
- layer_outputs = layer_module(hidden_states, layer_head_mask, output_attentions, run_full_attention)
- hidden_states = layer_outputs[0]
-
- all_hidden_states = all_hidden_states + (hidden_states,)
- return tuple(v for v in [hidden_states, all_hidden_states, all_self_attentions] if v is not None)
-
-class WindowedDinov2WithRegistersBackbone():
- def __init__(self):
- self.config = {}
- self.stage_names = ['stem', 'stage1', 'stage2', 'stage3', 'stage4', 'stage5', 'stage6', 'stage7', 'stage8', 'stage9', 'stage10', 'stage11', 'stage12']
- self.out_features = ['stage3', 'stage6', 'stage9', 'stage12']
-
- def __call__(self, pixel_values, output_hidden_states=None, output_attentions=None, return_dict=None,):
- embedding_output = self.embeddings(pixel_values)
-
- outputs = self.encoder(embedding_output, output_hidden_states=True, output_attentions=output_attentions, return_dict=return_dict)
-
- hidden_states = outputs[1]
-
- feature_maps = ()
- for stage, hidden_state in zip(self.stage_names, hidden_states):
- if stage in self.out_features:
- hidden_state = self.layernorm(hidden_state)
- hidden_state = hidden_state[:, 1 :]
- # this was actually a bug in the original implementation that we copied here,
- # cause normally the order is height, width
- batch_size, _, height, width = pixel_values.shape
- patch_size = 16
-
-
- num_h_patches = height // patch_size
- num_w_patches = width // patch_size
-
- # undo windowing
- num_windows_squared = self.num_windows ** 2
- B, HW, C = hidden_state.shape
-
- num_h_patches_per_window = num_h_patches // self.num_windows
- num_w_patches_per_window = num_w_patches // self.num_windows
-
- hidden_state = hidden_state.reshape(B // num_windows_squared, self.num_windows, self.num_windows, num_h_patches_per_window, num_w_patches_per_window, C)
- hidden_state = hidden_state.permute(0,1,3,2,4,5)
- hidden_state = hidden_state.reshape(batch_size, num_h_patches, num_w_patches, C)
- hidden_state = hidden_state.permute(0,3,1,2).contiguous()
- feature_maps += (hidden_state,)
-
- output = (feature_maps,) + outputs[2:]
- return output
-
-def ms_deform_attn_core(value, value_spatial_shapes, sampling_locations, attention_weights):
- B, n_heads, head_dim, _ = value.shape
- _, Len_q, n_heads, L, P, _ = sampling_locations.shape
- sampling_grids = 2 * sampling_locations - 1
- value_l_ = value.view(B * n_heads, head_dim, int(value_spatial_shapes), int(value_spatial_shapes))
- sampling_grid_l_ = sampling_grids[:, :, :, 0].transpose(1, 2).flatten(0, 1)
-
- N, C, H, W = value_l_.shape
- _, H_out, W_out, _ = sampling_grid_l_.shape
- x = (sampling_grid_l_[..., 0] + 1) * W / 2 - 0.5
- y = (sampling_grid_l_[..., 1] + 1) * H / 2 - 0.5
- x0 = x.floor().cast(dtype=dtypes.int64)
- y0 = y.floor().cast(dtype=dtypes.int64)
- x1 = x0 + 1
- y1 = y0 + 1
-
- wx = x - x0.float()
-
- wy = y - y0.cast(dtype=dtypes.float)
- w00 = (1 - wx) * (1 - wy)
- w01 = (1 - wx) * wy
- w10 = wx * (1 - wy)
- w11 = wx * wy
-
-
- v00 = (x0 >= 0) & (x0 < W) & (y0 >= 0) & (y0 < H)
-
- v01 = (x0 >= 0) & (x0 < W) & (y1 >= 0) & (y1 < H)
-
- v10 = (x1 >= 0) & (x1 < W) & (y0 >= 0) & (y0 < H)
- v11 = (x1 >= 0) & (x1 < W) & (y1 >= 0) & (y1 < H)
-
- x0c = x0.clamp(0, W - 1)
- x1c = x1.clamp(0, W - 1)
- y0c = y0.clamp(0, H - 1)
- y1c = y1.clamp(0, H - 1)
-
- value_flat = value_l_.view(N, C, H * W)
- def gather(xi, yi):
- idx = (yi * W + xi).view(N, 1, -1).expand(-1, C, -1)
- return Tensor.gather(value_flat, 2, idx).view(N, C, H_out, W_out)
-
- g00 = gather(x0c, y0c)
- g01 = gather(x0c, y1c)
- g10 = gather(x1c, y0c)
- g11 = gather(x1c, y1c)
- sampling_value_l_ = (g00 * (w00 * v00).unsqueeze(1) + g01 * (w01 * v01).unsqueeze(1) +
- g10 * (w10 * v10).unsqueeze(1) + g11 * (w11 * v11).unsqueeze(1))
- attention_weights = attention_weights.transpose(1, 2).reshape(B * n_heads, 1, Len_q, L * P)
- output = (sampling_value_l_ * attention_weights).sum(-1).view(B, n_heads * head_dim, Len_q)
- ret = output.transpose(1, 2).contiguous()
- return ret
-
-class TransformerDecoderLayer(): # todo, remove unused
- def __call__(self, tgt, memory, memory_key_padding_mask, query_pos,
- reference_points=None, spatial_shapes=None, level_start_index=None):
- q = k = tgt + query_pos
- v = tgt
-
- C = 256
- B, T, C = q.shape
- H = 8
- D = C // H
- w = self.in_proj_weight
- b = self.in_proj_bias
- wo = self.out_proj_weight
- bo = self.out_proj_bias
- wq, wk, wv = w.chunk(3, dim=0)
- bq, bk, bv = b.chunk(3, dim=0)
-
- q = q @ wq.T + bq
- k = k @ wk.T + bk
- v = v @ wv.T + bv
-
- q = q.view(B, T, H, D).transpose(1, 2)
- k = k.view(B, T, H, D).transpose(1, 2)
- v = v.view(B, T, H, D).transpose(1, 2)
-
- attn = Tensor.scaled_dot_product_attention(q,k,v)
- attn = attn.transpose(1, 2).contiguous().view(B, T, C)
- tgt2 = attn @ wo.T + bo
- tgt = tgt + tgt2
- tgt = self.norm1(tgt)
- tgt2 = self.cross_attn(tgt+query_pos, reference_points, memory, spatial_shapes, memory_key_padding_mask)
- tgt = tgt + tgt2
- tgt = self.norm2(tgt)
- x = self.linear1(tgt)
- x = x.relu()
- tgt2 = self.linear2(x)
- tgt += tgt2
- tgt = self.norm3(tgt)
- return tgt
-
-def gen_sineembed_for_position(pos_tensor, dim=128):
- scale = 2 * math.pi
- dim_t = Tensor.arange(dim)
- dim_t = 10000 ** (2 * (dim_t // 2) / dim)
- x_embed = pos_tensor[:, :, 0] * scale
- y_embed = pos_tensor[:, :, 1] * scale
- pos_x = x_embed[:, :, None] / dim_t
- pos_y = y_embed[:, :, None] / dim_t
- pos_x = Tensor.stack(pos_x[:, :, 0::2].sin(), pos_x[:, :, 1::2].cos(), dim=3).flatten(2)
- pos_y = Tensor.stack(pos_y[:, :, 0::2].sin(), pos_y[:, :, 1::2].cos(), dim=3).flatten(2)
- w_embed = pos_tensor[:, :, 2] * scale
- pos_w = w_embed[:, :, None] / dim_t
- pos_w = Tensor.stack(pos_w[:, :, 0::2].sin(), pos_w[:, :, 1::2].cos(), dim=3).flatten(2)
-
- h_embed = pos_tensor[:, :, 3] * scale
- pos_h = h_embed[:, :, None] / dim_t
- pos_h = Tensor.stack(pos_h[:, :, 0::2].sin(), pos_h[:, :, 1::2].cos(), dim=3).flatten(2)
- pos = Tensor.cat(pos_y, pos_x, pos_w, pos_h, dim=2)
- return pos
-
-class TransformerDecoder():
- def __call__(self, tgt, memory, memory_key_padding_mask=None,
- refpoints_unsigmoid=None, level_start_index=None, spatial_shapes=None):
- intermediate = []
- def get_reference(refpoints_unsigmoid):
- obj_center = refpoints_unsigmoid[..., :4]
- refpoints_input = obj_center[:, :, None]
- query_sine_embed = gen_sineembed_for_position(refpoints_input[:, :, 0, :], 256 / 2)
- query_pos = self.ref_point_head(query_sine_embed)
- return refpoints_input, query_pos
-
- refpoints_input, query_pos = get_reference(refpoints_unsigmoid)
- for layer in self.layers:
- tgt = layer(tgt, memory,
- memory_key_padding_mask=memory_key_padding_mask,
- query_pos=query_pos,
- reference_points=refpoints_input,
- spatial_shapes=spatial_shapes,
- level_start_index=level_start_index)
-
- x = self.norm(tgt)
- intermediate.append(x)
- return [(Tensor.stack(intermediate)), (refpoints_unsigmoid.unsqueeze(0))]
-
-def gen_encoder_output_proposals(memory, memory_padding_mask, spatial_shape, unsigmoid=True):
- memory_padding_mask = memory_padding_mask.cast(dtype=dtypes.bool)
-
- H_, W_ = spatial_shape, spatial_shape
- mask = memory_padding_mask.reshape(1, H_, W_)
-
- valid_H = (~mask[:, :, 0]).sum(axis=1).unsqueeze(-1)
- valid_W = (~mask[:, 0, :]).sum(axis=1).unsqueeze(-1)
-
- x = Tensor.linspace(0, H_ - 1, H_)
- y = Tensor.linspace(0, W_ - 1, W_)
- grid_y, grid_x = Tensor.meshgrid(y, x)
-
- grid = Tensor.cat(grid_x.unsqueeze(-1), grid_y.unsqueeze(-1), dim=-1)
- scale = Tensor.cat(valid_W, valid_H, dim=1).view(1, 1, 1, 2)
- grid = (grid.unsqueeze(0).expand(1, -1, -1, -1) + 0.5) / scale
-
- wh = Tensor.ones_like(grid) * 0.05
- output_proposals = Tensor.cat(grid, wh, dim=-1).view(1, -1, 4)
-
- output_proposals_valid = ((output_proposals > 0.01) & (output_proposals < 0.99)).all(-1, keepdim=True)
- output_proposals = output_proposals.masked_fill(memory_padding_mask.unsqueeze(-1), float(0))
- output_proposals = output_proposals.masked_fill(~output_proposals_valid, float(0))
- output_memory = memory
- output_memory = output_memory.masked_fill(memory_padding_mask.unsqueeze(-1), float(0))
- output_memory = output_memory.masked_fill(~output_proposals_valid, float(0))
- return output_memory, output_proposals
-
-class MSDeformAttn():
- def __call__(self, query, reference_points, input_flatten, input_spatial_shapes, input_padding_mask=None):
- N, Len_q, _ = query.shape
- N, Len_in, _ = input_flatten.shape
-
- value = self.value_proj(input_flatten)
- value = value.masked_fill(input_padding_mask[..., None], float(0))
- sampling_offsets = self.sampling_offsets(query).view(N, Len_q, 16, 1, 2, 2)
- attention_weights = self.attention_weights(query).view(N, Len_q, 16, 1 * 2)
- sampling_locations = reference_points[:, :, None, :, None, :2] \
- + sampling_offsets / 2 * reference_points[:, :, None, :, None, 2:] * 0.5
- attention_weights = attention_weights.softmax(-1)
- value = value.transpose(1, 2).contiguous().view(N, 16, 256 // 16, Len_in)
- output = ms_deform_attn_core(
- value, input_spatial_shapes, sampling_locations, attention_weights)
- output = self.output_proj(output)
- return output
-
-class Transformer():
- def __call__(self, srcs, masks, refpoint_embed, query_feat):
-
- self.enc_out_class_embed_w = self.enc_out_class_embed[0].weight
- self.enc_out_class_embed_b = self.enc_out_class_embed[0].bias
-
- src = srcs[0] if type(srcs) == list else srcs
- bs, _, h, w = src.shape
- src = src.flatten(2).transpose(1, 2) # bs, hw, c
- mask = masks[0].flatten(1) if type(masks) == list else masks.flatten(1)
- level_start_index = Tensor([0])
- output_memory, output_proposals = gen_encoder_output_proposals(
- src, mask, h, unsigmoid=True)
-
- output_memory_gidx = self.enc_output_norm(self.enc_output(output_memory))
- enc_outputs_class_unselected_gidx = output_memory_gidx @ self.enc_out_class_embed_w.T + self.enc_out_class_embed_b
-
-
- enc_outputs_coord_delta_gidx = self.enc_out_bbox_embed[0](output_memory_gidx)
-
- enc_outputs_coord_cxcy_gidx = enc_outputs_coord_delta_gidx[...,
- :2] * output_proposals[..., 2:] + output_proposals[..., :2]
- enc_outputs_coord_wh_gidx = enc_outputs_coord_delta_gidx[..., 2:].exp() * output_proposals[..., 2:]
- enc_outputs_coord_unselected_gidx = Tensor.cat(enc_outputs_coord_cxcy_gidx, enc_outputs_coord_wh_gidx, dim=-1)
- topk = min(300, enc_outputs_class_unselected_gidx.shape[-2])
- x = enc_outputs_class_unselected_gidx.max(-1)
- topk_proposals_gidx = Tensor.topk(x, topk, dim=1)[1]
- boxes_ts = enc_outputs_coord_unselected_gidx.gather(dim=1, index=topk_proposals_gidx.unsqueeze(-1).repeat(1, 1, 4))
- tgt = query_feat.unsqueeze(0).repeat(bs, 1, 1)
- refpoint_embed = refpoint_embed.unsqueeze(0).repeat(bs, 1, 1)
-
- ts_len = boxes_ts.shape[-2]
- refpoint_embed_ts_subset = refpoint_embed[..., :ts_len, :]
- refpoint_embed_subset = refpoint_embed[..., ts_len:, :]
-
-
- refpoint_embed_cxcy = refpoint_embed_ts_subset[..., :2] * boxes_ts[..., 2:]
- refpoint_embed_cxcy = refpoint_embed_cxcy + boxes_ts[..., :2]
- refpoint_embed_wh = refpoint_embed_ts_subset[..., 2:].exp() * boxes_ts[..., 2:]
- refpoint_embed_ts_subset = Tensor.cat(refpoint_embed_cxcy, refpoint_embed_wh, dim=-1)
- refpoint_embed = Tensor.cat(refpoint_embed_ts_subset, refpoint_embed_subset, dim=-2)
- hs, references = self.decoder(tgt, src, memory_key_padding_mask=mask,
- refpoints_unsigmoid=refpoint_embed,
- level_start_index=level_start_index,
- spatial_shapes=h)
-
- return hs, references
-
-
-class ConvX():
- def __call__(self, x):
- x = self.conv(x)
- x = self.bn(x)
- return Tensor.silu(x)
-
-class Bottleneck():
- def __call__(self, x): return self.cv2(self.cv1(x))
-
-class C2f():
- def __init__(self): self.c = 128
- def __call__(self, x):
- y = list(self.cv1(x).split((self.c, self.c), 1))
- y.extend(m(y[-1]) for m in self.m)
- y = Tensor.cat(*y, dim=1)
- y = self.cv2(y)
- return y
-
-class LayerNorm():
- def __init__(self): self.eps = 1e-6
- def __call__(self, x):
- x = x.permute(0, 2, 3, 1)
- x -= x.mean(axis=-1, keepdim=True)
- var = (x ** 2).mean(axis=-1, keepdim=True) + self.eps
- var = Tensor.sqrt(var)
- x_norm = x / var
- x_norm = x_norm * self.weight
- x_norm = x_norm + self.bias
- x = x_norm
- x = x.permute(0, 3, 1, 2)
- return x
-
-class MultiScaleProjector():
- def __call__(self, x):
- feat_fuse = Tensor.cat(*x, dim=1)
- stage_output = self.stages[0](feat_fuse)
- return [stage_output]
-
-class PositionEmbeddingSine():
- def __init__(self):
- self.scale = 6.283185307179586
- self.num_pos_feats = 128
- self.temperature = 10000
-
- def __call__(self, tensors, mask, align_dim_orders = True):
- not_mask = ~mask
- y_embed = not_mask.cumsum(1)
- x_embed = not_mask.cumsum(2)
- eps = 1e-6
- y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale
- x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale
- dim_t = Tensor.arange(self.num_pos_feats)
- dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats)
- pos_x = x_embed[:, :, :, None] / dim_t
- pos_y = y_embed[:, :, :, None] / dim_t
- pos_x = Tensor.stack((pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4).flatten(3)
- pos_y = Tensor.stack((pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4).flatten(3)
- pos = Tensor.cat(pos_y, pos_x, dim=3).permute(0, 3, 1, 2)
- return pos
-
-class Backbone():
- def __call__(self, tensors ,mask):
- feats = list(self.encoder(tensors)[0])
- feats = self.projector(feats)
- m = mask
- mask = ~Tensor.interpolate(m.unsqueeze(0), size=feats[0].shape[-2:])[0]
- return feats[0], mask
-
-class MLP():
- def __call__(self, x):
- for i, layer in enumerate(self.layers): x = Tensor.relu(layer(x)) if i < self.num_layers - 1 else layer(x)
- return x
-
-class RFDETR():
- def __init__(self, name, res=None):
- self.means = Tensor.empty(1,1,3)
- self.stds = Tensor.empty(1,1,3)
- self.num_queries = 300
- config = {"nano":{"n_layers":2, "size": 577, "res": 384}, "small":{"n_layers":3, "size":1025, "res":512},
- "medium":{"n_layers":4, "size":1297, "res":576}, "large":{"n_layers":4, "size":1937, "res":704}}
- self.res = config[name]["res"] if res is None else res
-
- num_windows = 2
- self.position_embedding = PositionEmbeddingSine()
- self.query_feat = Tensor.empty((3900, 256))
- self.refpoint_embed = Tensor.empty((3900, 4))
- self.class_embed = nn.Linear(256, 91)
- self.bbox_embed = MLP()
- self.bbox_embed.num_layers = 3
- self.bbox_embed.layers = seq(size=3)
- self.bbox_embed.layers[0] = nn.Linear(256, 256)
- self.bbox_embed.layers[1] = nn.Linear(256, 256)
- self.bbox_embed.layers[2] = nn.Linear(256, 4)
-
- self.bbox_embed.layers = seq(size=3)
- self.bbox_embed.layers[0] = nn.Linear(256, 256)
- self.bbox_embed.layers[1] = nn.Linear(256, 256)
- self.bbox_embed.layers[2] = nn.Linear(256, 4)
-
-
- self.transformer = Transformer()
- self.transformer.enc_output = nn.Linear(256, 256)
- self.transformer.enc_output_norm = nn.LayerNorm(256)
- self.transformer.decoder = TransformerDecoder()
- self.transformer.decoder.norm = nn.LayerNorm(256)
- self.transformer.decoder.layers = seq(config[name]["n_layers"])
- for i in range(config[name]["n_layers"]):
- self.transformer.decoder.layers[i] = TransformerDecoderLayer()
- self.transformer.decoder.layers[i].cross_attn = MSDeformAttn()
- self.transformer.decoder.layers[i].linear1 = nn.Linear(256, 2048)
- self.transformer.decoder.layers[i].linear2 = nn.Linear(2048, 256)
-
- self.transformer.decoder.layers[i].in_proj_weight = Tensor.empty(768, 256)
- self.transformer.decoder.layers[i].in_proj_bias = Tensor.empty(768)
- self.transformer.decoder.layers[i].out_proj_weight = Tensor.empty(256, 256)
- self.transformer.decoder.layers[i].out_proj_bias = Tensor.empty(256)
-
- self.transformer.decoder.layers[i].norm1 = nn.LayerNorm(256)
- self.transformer.decoder.layers[i].norm2 = nn.LayerNorm(256)
- self.transformer.decoder.layers[i].norm3 = nn.LayerNorm(256)
-
- self.transformer.decoder.layers[i].cross_attn.value_proj = nn.Linear(256, 256)
- self.transformer.decoder.layers[i].cross_attn.output_proj = nn.Linear(256, 256)
- self.transformer.decoder.layers[i].cross_attn.sampling_offsets = nn.Linear(256, 64)
- self.transformer.decoder.layers[i].cross_attn.attention_weights = nn.Linear(256, 32)
-
- self.transformer.enc_out_bbox_embed = seq(13)
- self.transformer.enc_out_class_embed = seq(13)
- for i in range(13):
- self.transformer.enc_out_bbox_embed[i] = MLP()
- self.transformer.enc_out_bbox_embed[i].num_layers = 3
- self.transformer.enc_out_bbox_embed[i].layers = seq(3)
- self.transformer.enc_out_bbox_embed[i].layers[0] = nn.Linear(256, 256)
- self.transformer.enc_out_bbox_embed[i].layers[1] = nn.Linear(256, 256)
- self.transformer.enc_out_bbox_embed[i].layers[2] = nn.Linear(256, 4)
- self.transformer.enc_out_bbox_embed[i].layers = seq(3)
- self.transformer.enc_out_bbox_embed[i].layers[0] = nn.Linear(256, 256)
- self.transformer.enc_out_bbox_embed[i].layers[1] = nn.Linear(256, 256)
- self.transformer.enc_out_bbox_embed[i].layers[2] = nn.Linear(256, 4)
-
- self.transformer.enc_out_class_embed[i] = nn.Linear(256, 91)
-
- self.transformer.decoder.ref_point_head = MLP()
- self.transformer.decoder.ref_point_head.layers = seq(size=2)
- self.transformer.decoder.ref_point_head.num_layers = 2
- self.transformer.decoder.ref_point_head.layers = seq(size=2)
- self.transformer.decoder.ref_point_head.layers[0] = nn.Linear(512, 256)
- self.transformer.decoder.ref_point_head.layers[1] = nn.Linear(256, 256)
- self.transformer.decoder.ref_point_head.layers[0] = nn.Linear(512, 256) # todo remove
- self.transformer.decoder.ref_point_head.layers[1] = nn.Linear(256, 256)
-
- self.backbone = Backbone()
- self.backbone.projector = MultiScaleProjector()
- self.backbone.projector.stages = seq(1) # todo, this is dumb
- self.backbone.projector.stages[0] = seq(2)
- self.backbone.projector.stages[0][0] = C2f()
- self.backbone.projector.stages[0][1] = LayerNorm()
- self.backbone.projector.stages[0][1].weight = Tensor.empty((256))
- self.backbone.projector.stages[0][1].bias = Tensor.empty((256))
- self.backbone.projector.stages[0][0].cv1 = ConvX()
- self.backbone.projector.stages[0][0].cv2 = ConvX()
- self.backbone.projector.stages[0][0].cv1.conv = nn.Conv2d(in_channels=1536, out_channels=256, kernel_size=1, bias=False, padding=(0, 0))
- self.backbone.projector.stages[0][0].cv2.conv = nn.Conv2d(in_channels=640, out_channels=256, kernel_size=1, bias=False, padding=(0, 0))
-
- self.backbone.projector.stages[0][0].cv1.bn = LayerNorm()
- self.backbone.projector.stages[0][0].cv1.bn.weight = Tensor.empty((256))
- self.backbone.projector.stages[0][0].cv1.bn.bias = Tensor.empty((256))
- self.backbone.projector.stages[0][0].cv2.bn = LayerNorm()
- self.backbone.projector.stages[0][0].cv2.bn.weight = Tensor.empty((256))
- self.backbone.projector.stages[0][0].cv2.bn.bias = Tensor.empty((256))
-
- self.backbone.projector.stages[0][0].m = seq(size=3)
- for i in range(3):
- self.backbone.projector.stages[0][0].m[i] = Bottleneck()
- self.backbone.projector.stages[0][0].m[i].cv1 = ConvX()
- self.backbone.projector.stages[0][0].m[i].cv1.conv = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, bias=False, padding=(1, 1))
- self.backbone.projector.stages[0][0].m[i].cv2 = ConvX()
- self.backbone.projector.stages[0][0].m[i].cv2.conv = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, bias=False, padding=(1, 1))
- self.backbone.projector.stages[0][0].m[i].cv1.bn = LayerNorm()
- self.backbone.projector.stages[0][0].m[i].cv1.bn.weight = Tensor.empty((128))
- self.backbone.projector.stages[0][0].m[i].cv1.bn.bias = Tensor.empty((128))
- self.backbone.projector.stages[0][0].m[i].cv2.bn = LayerNorm()
- self.backbone.projector.stages[0][0].m[i].cv2.bn.weight = Tensor.empty((128))
- self.backbone.projector.stages[0][0].m[i].cv2.bn.bias = Tensor.empty((128))
-
-
- self.backbone.patch_size = 16 # todo, not in state_dict?
- self.backbone.num_windows = num_windows # todo, not in state_dict?
- self.backbone.encoder = WindowedDinov2WithRegistersBackbone()
- self.backbone.encoder.num_windows = num_windows
- self.backbone.encoder.layernorm = nn.LayerNorm(384)
- self.backbone.encoder.encoder = WindowedDinov2WithRegistersEncoder()
- self.backbone.encoder.embeddings = WindowedDinov2WithRegistersEmbeddings()
- self.backbone.encoder.embeddings.num_windows = num_windows
- self.backbone.encoder.embeddings.projection = nn.Conv2d(in_channels=3, out_channels=384, kernel_size=16, stride=(16, 16))
- self.backbone.encoder.embeddings.cls_token = Tensor.empty((1, 1, 384))
- self.backbone.encoder.embeddings.position_embeddings = Tensor.empty((1, config[name]["size"], 384))
- self.backbone.encoder.encoder.layer = seq(size=13)
- for i in range(12):
- self.backbone.encoder.encoder.layer[i] = WindowedDinov2WithRegistersLayer()
- self.backbone.encoder.encoder.layer[i].num_windows = num_windows
- self.backbone.encoder.encoder.layer[i].norm1 = nn.LayerNorm(384)
- self.backbone.encoder.encoder.layer[i].norm2 = nn.LayerNorm(384)
- self.backbone.encoder.encoder.layer[i].attention = Dinov2WithRegistersSdpaAttention()
- self.backbone.encoder.encoder.layer[i].attention.attention = Dinov2WithRegistersSdpaSelfAttention()
- self.backbone.encoder.encoder.layer[i].attention.attention.query = nn.Linear(384, 384)
- self.backbone.encoder.encoder.layer[i].attention.attention.key = nn.Linear(384, 384)
- self.backbone.encoder.encoder.layer[i].attention.attention.value = nn.Linear(384, 384)
- self.backbone.encoder.encoder.layer[i].attention.dense = nn.Linear(384, 384)
- self.backbone.encoder.encoder.layer[i].lambda1 = Tensor.empty((384))
- self.backbone.encoder.encoder.layer[i].lambda2 = Tensor.empty((384))
- self.backbone.encoder.encoder.layer[i].mlp = Dinov2WithRegistersMLP()
- self.backbone.encoder.encoder.layer[i].mlp.fc1 = nn.Linear(384, 1536)
- self.backbone.encoder.encoder.layer[i].mlp.fc2 = nn.Linear(1536, 384)
- state_dict = safe_load(fetch(f'https://huggingface.co/roryclear/rf-detr/resolve/main/{name}.safetensors'))
- load_state_dict(self, state_dict)
-
- def __call__(self, frame):
- pre = self.preprocess(frame)
- predictions = self.predict(pre)
- out_logits, out_bbox = predictions
- prob = out_logits.sigmoid()
- topk_values, topk_indexes = Tensor.topk(prob.view(out_logits.shape[0], -1), 300, dim=1)
- topk_boxes = topk_indexes // out_logits.shape[2]
- labels = topk_indexes % out_logits.shape[2]
- boxes = box_cxcywh_to_xyxy(out_bbox)
- boxes = Tensor.gather(boxes, 1, topk_boxes.unsqueeze(-1).repeat(1,1,4))
- ret = Tensor.cat(boxes.squeeze(0), topk_values.squeeze(0).unsqueeze(1), labels.squeeze(0).unsqueeze(1), dim=1)
- ret = self.scale_boxes(pre.shape[2:], ret, frame.shape[:2])
- return ret
-
- def predict(self, samples, targets=None):
- _, _, h, w = samples.shape
- mask = Tensor.zeros((1, h, w), dtype=dtypes.bool)
- feature, mask = self.backbone(samples, mask)
- refpoint_embed_weight = self.refpoint_embed[:self.num_queries]
- query_feat_weight = self.query_feat[:self.num_queries]
- hs, ref_unsigmoid = self.transformer(feature, mask, refpoint_embed_weight, query_feat_weight)
- outputs_coord_delta = self.bbox_embed(hs)
-
- outputs_coord_cxcy = outputs_coord_delta[..., :2] * ref_unsigmoid[..., 2:] + ref_unsigmoid[..., :2]
- outputs_coord_wh = outputs_coord_delta[..., 2:].exp() * ref_unsigmoid[..., 2:]
- outputs_coord = Tensor.cat(outputs_coord_cxcy, outputs_coord_wh, dim=-1)
- outputs_class = self.class_embed(hs)[-1]
- return outputs_class, outputs_coord[-1]
-
- def preprocess(self, img):
- img = img.cast(dtypes.float32)
- img = img[:, :, ::-1]
- img /= 255.0
- h, w = img.shape[:2]
- scale = self.res / max(h, w)
- new_w = int(w * scale)
- new_h = int(h * scale)
- img = resize(img, (new_w, new_h))
- canvas = Tensor.zeros((self.res, self.res, 3))
- pad_x = (self.res - new_w) // 2
- pad_y = (self.res - new_h) // 2
- canvas[pad_y:pad_y+new_h, pad_x:pad_x+new_w] = img
- img = canvas
- img = (img - self.means) / self.stds
- img = img.permute(2, 0, 1).unsqueeze(0)
- return img
-
- def scale_boxes(self, img1_shape, predictions, img0_shape):
- orig_h, orig_w = img0_shape[0], img0_shape[1]
- proc_h, proc_w = img1_shape[0], img1_shape[1]
- scale = min(proc_w / orig_w, proc_h / orig_h)
- new_w = int(orig_w * scale)
- new_h = int(orig_h * scale)
- pad_x = (proc_w - new_w) // 2
- pad_y = (proc_h - new_h) // 2
- predictions[:, 0] = (predictions[:, 0] * proc_w - pad_x) / scale
- predictions[:, 1] = (predictions[:, 1] * proc_h - pad_y) / scale
- predictions[:, 2] = (predictions[:, 2] * proc_w - pad_x) / scale
- predictions[:, 3] = (predictions[:, 3] * proc_h - pad_y) / scale
- predictions[:, 0] = Tensor.clip(predictions[:, 0], 0, orig_w)
- predictions[:, 1] = Tensor.clip(predictions[:, 1], 0, orig_h)
- predictions[:, 2] = Tensor.clip(predictions[:, 2], 0, orig_w)
- predictions[:, 3] = Tensor.clip(predictions[:, 3], 0, orig_h)
- return predictions
-
-def compute_iou_matrix(boxes):
- x1s = boxes[:, :, 0]
- y1s = boxes[:, :, 1]
- x2s = boxes[:, :, 2]
- y2s = boxes[:, :, 3]
- areas = (x2s - x1s) * (y2s - y1s)
- x1 = Tensor.maximum(x1s[:, :, None], x1s[:, None, :])
- y1 = Tensor.maximum(y1s[:, :, None], y1s[:, None, :])
- x2 = Tensor.minimum(x2s[:, :, None], x2s[:, None, :])
- y2 = Tensor.minimum(y2s[:, :, None], y2s[:, None, :])
- w = Tensor.maximum(Tensor(0), x2 - x1)
- h = Tensor.maximum(Tensor(0), y2 - y1)
- intersection = w * h
- union = areas[:, :, None] + areas[:, None, :] - intersection
- return intersection / union
-
-def box_cxcywh_to_xyxy(x):
- x_c, y_c, w, h = [t.squeeze(-1) for t in x.split(1, dim=-1)]
- w_pos = w.clip(0.0, float("inf"))
- h_pos = h.clip(0.0, float("inf"))
- b = [x_c - 0.5 * w_pos, y_c - 0.5 * h_pos, x_c + 0.5 * w_pos, y_c + 0.5 * h_pos]
- return Tensor.stack(b, dim=-1)
-
-class seq:
- def __init__(self, size=0): self.size = size
- def __setitem__(self, key, value): setattr(self, str(key), value)
- def __getitem__(self, idx):
- try:
- return getattr(self, str(idx))
- except AttributeError:
- raise IndexError(idx)
- def __len__(self): return self.size
- def __call__(self, x):
- for i in range(self.size):
- layer = getattr(self, str(i))
- x = layer(x)
- return x
\ No newline at end of file