Add object detection, tracking, mobile notifications, and search to any security camera.
add tracking stuff
roryclearf6dbfcc63a9c
diff --git a/experimental/rtsp_yolov8.py b/experimental/rtsp_yolov8.pyindex ec41145..2ec043c 100644--- a/experimental/rtsp_yolov8.py+++ b/experimental/rtsp_yolov8.py@@ -20,6 +20,17 @@ import threadingimport shutilfrom datetime import datetime, time as time_obj+from yolox.tracker.byte_tracker import BYTETracker++class Args:+ def __init__(self):+ self.track_thresh = 0.6+ self.track_buffer = 60 #frames, was 30+ self.mot20 = False+ self.match_thresh = 0.9++tracker = BYTETracker(Args())+#Model architecture from https://github.com/ultralytics/ultralytics/issues/189#The upsampling class has been taken from this pull request https://github.com/tinygrad/tinygrad/pull/784 by dc-dc-dc. Now 2(?) models use upsampling. (retinet and this)@@ -460,10 +471,15 @@ class VideoCapture:frame = self.raw_frame.copy() if self.raw_frame is not None else Noneif frame is not None:pre = preprocess(frame)- preds = do_inf(pre).numpy()- preds = scale_boxes(pre.shape[:2], preds, frame.shape)+ preds2 = do_inf(pre).numpy()+ online_targets = tracker.update(preds2, [1280,1280], [1280,1280])+ preds2 = []+ for x in online_targets:+ preds2.append(np.array([x.tlwh[0],x.tlwh[1],(x.tlwh[0]+x.tlwh[2]),(x.tlwh[1]+x.tlwh[3]),x.score,x.class_id]))+ preds2 = np.array(preds2)+ preds2 = scale_boxes(pre.shape[:2], preds2, frame.shape)with self.lock:- self.last_preds = preds+ self.last_preds = preds2curr_time = time.time()fps = 1 / (curr_time - prev_time)prev_time = curr_timediff --git a/experimental/yolox/.DS_Store b/experimental/yolox/.DS_Storenew file mode 100644index 0000000..ab82fe8Binary files /dev/null and b/experimental/yolox/.DS_Store differdiff --git a/experimental/yolox/__init__.py b/experimental/yolox/__init__.pynew file mode 100644index 0000000..1cbc411--- /dev/null+++ b/experimental/yolox/__init__.py@@ -0,0 +1,8 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-++from .utils import configure_module++configure_module()++__version__ = "0.1.0"diff --git a/experimental/yolox/__pycache__/__init__.cpython-310.pyc b/experimental/yolox/__pycache__/__init__.cpython-310.pycnew file mode 100644index 0000000..025cd56Binary files /dev/null and b/experimental/yolox/__pycache__/__init__.cpython-310.pyc differdiff --git a/experimental/yolox/core/__init__.py b/experimental/yolox/core/__init__.pynew file mode 100644index 0000000..c2379c7--- /dev/null+++ b/experimental/yolox/core/__init__.py@@ -0,0 +1,6 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) Megvii, Inc. and its affiliates.++from .launch import launch+from .trainer import Trainerdiff --git a/experimental/yolox/core/launch.py b/experimental/yolox/core/launch.pynew file mode 100644index 0000000..2fd5eaa--- /dev/null+++ b/experimental/yolox/core/launch.py@@ -0,0 +1,219 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Code are based on+# https://github.com/facebookresearch/detectron2/blob/master/detectron2/engine/launch.py+# Copyright (c) Facebook, Inc. and its affiliates.+# Copyright (c) Megvii, Inc. and its affiliates.++from loguru import logger++import torch+import torch.distributed as dist+import torch.multiprocessing as mp++import yolox.utils.dist as comm+from yolox.utils import configure_nccl++import os+import subprocess+import sys+import time++__all__ = ["launch"]+++def _find_free_port():+ """+ Find an available port of current machine / node.+ """+ import socket++ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)+ # Binding to port 0 will cause the OS to find an available port for us+ sock.bind(("", 0))+ port = sock.getsockname()[1]+ sock.close()+ # NOTE: there is still a chance the port could be taken by other processes.+ return port+++def launch(+ main_func,+ num_gpus_per_machine,+ num_machines=1,+ machine_rank=0,+ backend="nccl",+ dist_url=None,+ args=(),+):+ """+ Args:+ main_func: a function that will be called by `main_func(*args)`+ num_machines (int): the total number of machines+ machine_rank (int): the rank of this machine (one per machine)+ dist_url (str): url to connect to for distributed training, including protocol+ e.g. "tcp://127.0.0.1:8686".+ Can be set to auto to automatically select a free port on localhost+ args (tuple): arguments passed to main_func+ """+ world_size = num_machines * num_gpus_per_machine+ if world_size > 1:+ if int(os.environ.get("WORLD_SIZE", "1")) > 1:+ dist_url = "{}:{}".format(+ os.environ.get("MASTER_ADDR", None),+ os.environ.get("MASTER_PORT", "None"),+ )+ local_rank = int(os.environ.get("LOCAL_RANK", "0"))+ world_size = int(os.environ.get("WORLD_SIZE", "1"))+ _distributed_worker(+ local_rank,+ main_func,+ world_size,+ num_gpus_per_machine,+ num_machines,+ machine_rank,+ backend,+ dist_url,+ args,+ )+ exit()+ launch_by_subprocess(+ sys.argv,+ world_size,+ num_machines,+ machine_rank,+ num_gpus_per_machine,+ dist_url,+ args,+ )+ else:+ main_func(*args)+++def launch_by_subprocess(+ raw_argv,+ world_size,+ num_machines,+ machine_rank,+ num_gpus_per_machine,+ dist_url,+ args,+):+ assert (+ world_size > 1+ ), "subprocess mode doesn't support single GPU, use spawn mode instead"++ if dist_url is None:+ # ------------------------hack for multi-machine training -------------------- #+ if num_machines > 1:+ master_ip = subprocess.check_output(["hostname", "--fqdn"]).decode("utf-8")+ master_ip = str(master_ip).strip()+ dist_url = "tcp://{}".format(master_ip)+ ip_add_file = "./" + args[1].experiment_name + "_ip_add.txt"+ if machine_rank == 0:+ port = _find_free_port()+ with open(ip_add_file, "w") as ip_add:+ ip_add.write(dist_url+'\n')+ ip_add.write(str(port))+ else:+ while not os.path.exists(ip_add_file):+ time.sleep(0.5)++ with open(ip_add_file, "r") as ip_add:+ dist_url = ip_add.readline().strip()+ port = ip_add.readline()+ else:+ dist_url = "tcp://127.0.0.1"+ port = _find_free_port()++ # set PyTorch distributed related environmental variables+ current_env = os.environ.copy()+ current_env["MASTER_ADDR"] = dist_url+ current_env["MASTER_PORT"] = str(port)+ current_env["WORLD_SIZE"] = str(world_size)+ assert num_gpus_per_machine <= torch.cuda.device_count()++ if "OMP_NUM_THREADS" not in os.environ and num_gpus_per_machine > 1:+ current_env["OMP_NUM_THREADS"] = str(1)+ logger.info(+ "\n*****************************************\n"+ "Setting OMP_NUM_THREADS environment variable for each process "+ "to be {} in default, to avoid your system being overloaded, "+ "please further tune the variable for optimal performance in "+ "your application as needed. \n"+ "*****************************************".format(+ current_env["OMP_NUM_THREADS"]+ )+ )++ processes = []+ for local_rank in range(0, num_gpus_per_machine):+ # each process's rank+ dist_rank = machine_rank * num_gpus_per_machine + local_rank+ current_env["RANK"] = str(dist_rank)+ current_env["LOCAL_RANK"] = str(local_rank)++ # spawn the processes+ cmd = ["python3", *raw_argv]++ process = subprocess.Popen(cmd, env=current_env)+ processes.append(process)++ for process in processes:+ process.wait()+ if process.returncode != 0:+ raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)+++def _distributed_worker(+ local_rank,+ main_func,+ world_size,+ num_gpus_per_machine,+ num_machines,+ machine_rank,+ backend,+ dist_url,+ args,+):+ assert (+ torch.cuda.is_available()+ ), "cuda is not available. Please check your installation."+ configure_nccl()+ global_rank = machine_rank * num_gpus_per_machine + local_rank+ logger.info("Rank {} initialization finished.".format(global_rank))+ try:+ dist.init_process_group(+ backend=backend,+ init_method=dist_url,+ world_size=world_size,+ rank=global_rank,+ )+ except Exception:+ logger.error("Process group URL: {}".format(dist_url))+ raise+ # synchronize is needed here to prevent a possible timeout after calling init_process_group+ # See: https://github.com/facebookresearch/maskrcnn-benchmark/issues/172+ comm.synchronize()++ if global_rank == 0 and os.path.exists(+ "./" + args[1].experiment_name + "_ip_add.txt"+ ):+ os.remove("./" + args[1].experiment_name + "_ip_add.txt")++ assert num_gpus_per_machine <= torch.cuda.device_count()+ torch.cuda.set_device(local_rank)++ args[1].local_rank = local_rank+ args[1].num_machines = num_machines++ # Setup the local process group (which contains ranks within the same machine)+ # assert comm._LOCAL_PROCESS_GROUP is None+ # num_machines = world_size // num_gpus_per_machine+ # for i in range(num_machines):+ # ranks_on_i = list(range(i * num_gpus_per_machine, (i + 1) * num_gpus_per_machine))+ # pg = dist.new_group(ranks_on_i)+ # if i == machine_rank:+ # comm._LOCAL_PROCESS_GROUP = pg++ main_func(*args)diff --git a/experimental/yolox/core/trainer.py b/experimental/yolox/core/trainer.pynew file mode 100644index 0000000..5ab1006--- /dev/null+++ b/experimental/yolox/core/trainer.py@@ -0,0 +1,329 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) Megvii, Inc. and its affiliates.++from loguru import logger++import torch++from torch.nn.parallel import DistributedDataParallel as DDP+from torch.utils.tensorboard import SummaryWriter++from yolox.data import DataPrefetcher+from yolox.utils import (+ MeterBuffer,+ ModelEMA,+ all_reduce_norm,+ get_model_info,+ get_rank,+ get_world_size,+ gpu_mem_usage,+ load_ckpt,+ occupy_mem,+ save_checkpoint,+ setup_logger,+ synchronize+)++import datetime+import os+import time+++class Trainer:+ def __init__(self, exp, args):+ # init function only defines some basic attr, other attrs like model, optimizer are built in+ # before_train methods.+ self.exp = exp+ self.args = args++ # training related attr+ self.max_epoch = exp.max_epoch+ self.amp_training = args.fp16+ self.scaler = torch.cuda.amp.GradScaler(enabled=args.fp16)+ self.is_distributed = get_world_size() > 1+ self.rank = get_rank()+ self.local_rank = args.local_rank+ self.device = "cuda:{}".format(self.local_rank)+ self.use_model_ema = exp.ema++ # data/dataloader related attr+ self.data_type = torch.float16 if args.fp16 else torch.float32+ self.input_size = exp.input_size+ self.best_ap = 0++ # metric record+ self.meter = MeterBuffer(window_size=exp.print_interval)+ self.file_name = os.path.join(exp.output_dir, args.experiment_name)++ if self.rank == 0:+ os.makedirs(self.file_name, exist_ok=True)++ setup_logger(+ self.file_name,+ distributed_rank=self.rank,+ filename="train_log.txt",+ mode="a",+ )++ def train(self):+ self.before_train()+ try:+ self.train_in_epoch()+ except Exception:+ raise+ finally:+ self.after_train()++ def train_in_epoch(self):+ for self.epoch in range(self.start_epoch, self.max_epoch):+ self.before_epoch()+ self.train_in_iter()+ self.after_epoch()++ def train_in_iter(self):+ for self.iter in range(self.max_iter):+ self.before_iter()+ self.train_one_iter()+ self.after_iter()++ def train_one_iter(self):+ iter_start_time = time.time()++ inps, targets = self.prefetcher.next()+ track_ids = targets[:, :, 5]+ targets = targets[:, :, :5]+ inps = inps.to(self.data_type)+ targets = targets.to(self.data_type)+ targets.requires_grad = False+ data_end_time = time.time()++ with torch.cuda.amp.autocast(enabled=self.amp_training):+ outputs = self.model(inps, targets)+ loss = outputs["total_loss"]++ self.optimizer.zero_grad()+ self.scaler.scale(loss).backward()+ self.scaler.step(self.optimizer)+ self.scaler.update()++ if self.use_model_ema:+ self.ema_model.update(self.model)++ lr = self.lr_scheduler.update_lr(self.progress_in_iter + 1)+ for param_group in self.optimizer.param_groups:+ param_group["lr"] = lr++ iter_end_time = time.time()+ self.meter.update(+ iter_time=iter_end_time - iter_start_time,+ data_time=data_end_time - iter_start_time,+ lr=lr,+ **outputs,+ )++ def before_train(self):+ logger.info("args: {}".format(self.args))+ logger.info("exp value:\n{}".format(self.exp))++ # model related init+ torch.cuda.set_device(self.local_rank)+ model = self.exp.get_model()+ logger.info(+ "Model Summary: {}".format(get_model_info(model, self.exp.test_size))+ )+ model.to(self.device)++ # solver related init+ self.optimizer = self.exp.get_optimizer(self.args.batch_size)++ # value of epoch will be set in `resume_train`+ model = self.resume_train(model)++ # data related init+ self.no_aug = self.start_epoch >= self.max_epoch - self.exp.no_aug_epochs+ self.train_loader = self.exp.get_data_loader(+ batch_size=self.args.batch_size,+ is_distributed=self.is_distributed,+ no_aug=self.no_aug,+ )+ logger.info("init prefetcher, this might take one minute or less...")+ self.prefetcher = DataPrefetcher(self.train_loader)+ # max_iter means iters per epoch+ self.max_iter = len(self.train_loader)++ self.lr_scheduler = self.exp.get_lr_scheduler(+ self.exp.basic_lr_per_img * self.args.batch_size, self.max_iter+ )+ if self.args.occupy:+ occupy_mem(self.local_rank)++ if self.is_distributed:+ model = DDP(model, device_ids=[self.local_rank], broadcast_buffers=False)++ if self.use_model_ema:+ self.ema_model = ModelEMA(model, 0.9998)+ self.ema_model.updates = self.max_iter * self.start_epoch++ self.model = model+ self.model.train()++ self.evaluator = self.exp.get_evaluator(+ batch_size=self.args.batch_size, is_distributed=self.is_distributed+ )+ # Tensorboard logger+ if self.rank == 0:+ self.tblogger = SummaryWriter(self.file_name)++ logger.info("Training start...")+ #logger.info("\n{}".format(model))++ def after_train(self):+ logger.info(+ "Training of experiment is done and the best AP is {:.2f}".format(+ self.best_ap * 100+ )+ )++ def before_epoch(self):+ logger.info("---> start train epoch{}".format(self.epoch + 1))++ if self.epoch + 1 == self.max_epoch - self.exp.no_aug_epochs or self.no_aug:++ logger.info("--->No mosaic aug now!")+ self.train_loader.close_mosaic()+ logger.info("--->Add additional L1 loss now!")+ if self.is_distributed:+ self.model.module.head.use_l1 = True+ else:+ self.model.head.use_l1 = True++ self.exp.eval_interval = 1+ if not self.no_aug:+ self.save_ckpt(ckpt_name="last_mosaic_epoch")++ def after_epoch(self):+ if self.use_model_ema:+ self.ema_model.update_attr(self.model)++ self.save_ckpt(ckpt_name="latest")++ if (self.epoch + 1) % self.exp.eval_interval == 0:+ all_reduce_norm(self.model)+ self.evaluate_and_save_model()++ def before_iter(self):+ pass++ def after_iter(self):+ """+ `after_iter` contains two parts of logic:+ * log information+ * reset setting of resize+ """+ # log needed information+ if (self.iter + 1) % self.exp.print_interval == 0:+ # TODO check ETA logic+ left_iters = self.max_iter * self.max_epoch - (self.progress_in_iter + 1)+ eta_seconds = self.meter["iter_time"].global_avg * left_iters+ eta_str = "ETA: {}".format(datetime.timedelta(seconds=int(eta_seconds)))++ progress_str = "epoch: {}/{}, iter: {}/{}".format(+ self.epoch + 1, self.max_epoch, self.iter + 1, self.max_iter+ )+ loss_meter = self.meter.get_filtered_meter("loss")+ loss_str = ", ".join(+ ["{}: {:.3f}".format(k, v.latest) for k, v in loss_meter.items()]+ )++ time_meter = self.meter.get_filtered_meter("time")+ time_str = ", ".join(+ ["{}: {:.3f}s".format(k, v.avg) for k, v in time_meter.items()]+ )++ logger.info(+ "{}, mem: {:.0f}Mb, {}, {}, lr: {:.3e}".format(+ progress_str,+ gpu_mem_usage(),+ time_str,+ loss_str,+ self.meter["lr"].latest,+ )+ + (", size: {:d}, {}".format(self.input_size[0], eta_str))+ )+ self.meter.clear_meters()++ # random resizing+ if self.exp.random_size is not None and (self.progress_in_iter + 1) % 10 == 0:+ self.input_size = self.exp.random_resize(+ self.train_loader, self.epoch, self.rank, self.is_distributed+ )++ @property+ def progress_in_iter(self):+ return self.epoch * self.max_iter + self.iter++ def resume_train(self, model):+ if self.args.resume:+ logger.info("resume training")+ if self.args.ckpt is None:+ ckpt_file = os.path.join(self.file_name, "latest" + "_ckpt.pth.tar")+ else:+ ckpt_file = self.args.ckpt++ ckpt = torch.load(ckpt_file, map_location=self.device)+ # resume the model/optimizer state dict+ model.load_state_dict(ckpt["model"])+ self.optimizer.load_state_dict(ckpt["optimizer"])+ start_epoch = (+ self.args.start_epoch - 1+ if self.args.start_epoch is not None+ else ckpt["start_epoch"]+ )+ self.start_epoch = start_epoch+ logger.info(+ "loaded checkpoint '{}' (epoch {})".format(+ self.args.resume, self.start_epoch+ )+ ) # noqa+ else:+ if self.args.ckpt is not None:+ logger.info("loading checkpoint for fine tuning")+ ckpt_file = self.args.ckpt+ ckpt = torch.load(ckpt_file, map_location=self.device)["model"]+ model = load_ckpt(model, ckpt)+ self.start_epoch = 0++ return model++ def evaluate_and_save_model(self):+ evalmodel = self.ema_model.ema if self.use_model_ema else self.model+ ap50_95, ap50, summary = self.exp.eval(+ evalmodel, self.evaluator, self.is_distributed+ )+ self.model.train()+ if self.rank == 0:+ self.tblogger.add_scalar("val/COCOAP50", ap50, self.epoch + 1)+ self.tblogger.add_scalar("val/COCOAP50_95", ap50_95, self.epoch + 1)+ logger.info("\n" + summary)+ synchronize()++ #self.best_ap = max(self.best_ap, ap50_95)+ self.save_ckpt("last_epoch", ap50 > self.best_ap)+ self.best_ap = max(self.best_ap, ap50)++ def save_ckpt(self, ckpt_name, update_best_ckpt=False):+ if self.rank == 0:+ save_model = self.ema_model.ema if self.use_model_ema else self.model+ logger.info("Save weights to {}".format(self.file_name))+ ckpt_state = {+ "start_epoch": self.epoch + 1,+ "model": save_model.state_dict(),+ "optimizer": self.optimizer.state_dict(),+ }+ save_checkpoint(+ ckpt_state,+ update_best_ckpt,+ self.file_name,+ ckpt_name,+ )diff --git a/experimental/yolox/data/__init__.py b/experimental/yolox/data/__init__.pynew file mode 100644index 0000000..258effe--- /dev/null+++ b/experimental/yolox/data/__init__.py@@ -0,0 +1,9 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) Megvii, Inc. and its affiliates.++from .data_augment import TrainTransform, ValTransform+from .data_prefetcher import DataPrefetcher+from .dataloading import DataLoader, get_yolox_datadir+from .datasets import *+from .samplers import InfiniteSampler, YoloBatchSamplerdiff --git a/experimental/yolox/data/data_augment.py b/experimental/yolox/data/data_augment.pynew file mode 100644index 0000000..99fb30a--- /dev/null+++ b/experimental/yolox/data/data_augment.py@@ -0,0 +1,299 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) Megvii, Inc. and its affiliates.+"""+Data augmentation functionality. Passed as callable transformations to+Dataset classes.++The data augmentation procedures were interpreted from @weiliu89's SSD paper+http://arxiv.org/abs/1512.02325+"""++import cv2+import numpy as np++import torch++from yolox.utils import xyxy2cxcywh++import math+import random+++def augment_hsv(img, hgain=0.015, sgain=0.7, vgain=0.4):+ r = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] + 1 # random gains+ hue, sat, val = cv2.split(cv2.cvtColor(img, cv2.COLOR_BGR2HSV))+ dtype = img.dtype # uint8++ x = np.arange(0, 256, dtype=np.int16)+ lut_hue = ((x * r[0]) % 180).astype(dtype)+ lut_sat = np.clip(x * r[1], 0, 255).astype(dtype)+ lut_val = np.clip(x * r[2], 0, 255).astype(dtype)++ img_hsv = cv2.merge(+ (cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val))+ ).astype(dtype)+ cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR, dst=img) # no return needed+++def box_candidates(box1, box2, wh_thr=2, ar_thr=20, area_thr=0.2):+ # box1(4,n), box2(4,n)+ # Compute candidate boxes which include follwing 5 things:+ # box1 before augment, box2 after augment, wh_thr (pixels), aspect_ratio_thr, area_ratio+ w1, h1 = box1[2] - box1[0], box1[3] - box1[1]+ w2, h2 = box2[2] - box2[0], box2[3] - box2[1]+ ar = np.maximum(w2 / (h2 + 1e-16), h2 / (w2 + 1e-16)) # aspect ratio+ return (+ (w2 > wh_thr)+ & (h2 > wh_thr)+ & (w2 * h2 / (w1 * h1 + 1e-16) > area_thr)+ & (ar < ar_thr)+ ) # candidates+++def random_perspective(+ img,+ targets=(),+ degrees=10,+ translate=0.1,+ scale=0.1,+ shear=10,+ perspective=0.0,+ border=(0, 0),+):+ # targets = [cls, xyxy]+ height = img.shape[0] + border[0] * 2 # shape(h,w,c)+ width = img.shape[1] + border[1] * 2++ # Center+ C = np.eye(3)+ C[0, 2] = -img.shape[1] / 2 # x translation (pixels)+ C[1, 2] = -img.shape[0] / 2 # y translation (pixels)++ # Rotation and Scale+ R = np.eye(3)+ a = random.uniform(-degrees, degrees)+ # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations+ s = random.uniform(scale[0], scale[1])+ # s = 2 ** random.uniform(-scale, scale)+ R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s)++ # Shear+ S = np.eye(3)+ S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # x shear (deg)+ S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # y shear (deg)++ # Translation+ T = np.eye(3)+ T[0, 2] = (+ random.uniform(0.5 - translate, 0.5 + translate) * width+ ) # x translation (pixels)+ T[1, 2] = (+ random.uniform(0.5 - translate, 0.5 + translate) * height+ ) # y translation (pixels)++ # Combined rotation matrix+ M = T @ S @ R @ C # order of operations (right to left) is IMPORTANT++ ###########################+ # For Aug out of Mosaic+ # s = 1.+ # M = np.eye(3)+ ###########################++ if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed+ if perspective:+ img = cv2.warpPerspective(+ img, M, dsize=(width, height), borderValue=(114, 114, 114)+ )+ else: # affine+ img = cv2.warpAffine(+ img, M[:2], dsize=(width, height), borderValue=(114, 114, 114)+ )++ # Transform label coordinates+ n = len(targets)+ if n:+ # warp points+ xy = np.ones((n * 4, 3))+ xy[:, :2] = targets[:, [0, 1, 2, 3, 0, 3, 2, 1]].reshape(+ n * 4, 2+ ) # x1y1, x2y2, x1y2, x2y1+ xy = xy @ M.T # transform+ if perspective:+ xy = (xy[:, :2] / xy[:, 2:3]).reshape(n, 8) # rescale+ else: # affine+ xy = xy[:, :2].reshape(n, 8)++ # create new boxes+ x = xy[:, [0, 2, 4, 6]]+ y = xy[:, [1, 3, 5, 7]]+ xy = np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T++ # clip boxes+ #xy[:, [0, 2]] = xy[:, [0, 2]].clip(0, width)+ #xy[:, [1, 3]] = xy[:, [1, 3]].clip(0, height)++ # filter candidates+ i = box_candidates(box1=targets[:, :4].T * s, box2=xy.T)+ targets = targets[i]+ targets[:, :4] = xy[i]++ targets = targets[targets[:, 0] < width]+ targets = targets[targets[:, 2] > 0]+ targets = targets[targets[:, 1] < height]+ targets = targets[targets[:, 3] > 0]++ return img, targets+++def _distort(image):+ def _convert(image, alpha=1, beta=0):+ tmp = image.astype(float) * alpha + beta+ tmp[tmp < 0] = 0+ tmp[tmp > 255] = 255+ image[:] = tmp++ image = image.copy()++ if random.randrange(2):+ _convert(image, beta=random.uniform(-32, 32))++ if random.randrange(2):+ _convert(image, alpha=random.uniform(0.5, 1.5))++ image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)++ if random.randrange(2):+ tmp = image[:, :, 0].astype(int) + random.randint(-18, 18)+ tmp %= 180+ image[:, :, 0] = tmp++ if random.randrange(2):+ _convert(image[:, :, 1], alpha=random.uniform(0.5, 1.5))++ image = cv2.cvtColor(image, cv2.COLOR_HSV2BGR)++ return image+++def _mirror(image, boxes):+ _, width, _ = image.shape+ if random.randrange(2):+ image = image[:, ::-1]+ boxes = boxes.copy()+ boxes[:, 0::2] = width - boxes[:, 2::-2]+ return image, boxes+++def preproc(image, input_size, mean, std, swap=(2, 0, 1)):+ if len(image.shape) == 3:+ padded_img = np.ones((input_size[0], input_size[1], 3)) * 114.0+ else:+ padded_img = np.ones(input_size) * 114.0+ img = np.array(image)+ r = min(input_size[0] / img.shape[0], input_size[1] / img.shape[1])+ resized_img = cv2.resize(+ img,+ (int(img.shape[1] * r), int(img.shape[0] * r)),+ interpolation=cv2.INTER_LINEAR,+ ).astype(np.float32)+ padded_img[: int(img.shape[0] * r), : int(img.shape[1] * r)] = resized_img++ padded_img = padded_img[:, :, ::-1]+ padded_img /= 255.0+ if mean is not None:+ padded_img -= mean+ if std is not None:+ padded_img /= std+ padded_img = padded_img.transpose(swap)+ padded_img = np.ascontiguousarray(padded_img, dtype=np.float32)+ return padded_img, r+++class TrainTransform:+ def __init__(self, p=0.5, rgb_means=None, std=None, max_labels=100):+ self.means = rgb_means+ self.std = std+ self.p = p+ self.max_labels = max_labels++ def __call__(self, image, targets, input_dim):+ boxes = targets[:, :4].copy()+ labels = targets[:, 4].copy()+ ids = targets[:, 5].copy()+ if len(boxes) == 0:+ targets = np.zeros((self.max_labels, 6), dtype=np.float32)+ image, r_o = preproc(image, input_dim, self.means, self.std)+ image = np.ascontiguousarray(image, dtype=np.float32)+ return image, targets++ image_o = image.copy()+ targets_o = targets.copy()+ height_o, width_o, _ = image_o.shape+ boxes_o = targets_o[:, :4]+ labels_o = targets_o[:, 4]+ ids_o = targets_o[:, 5]+ # bbox_o: [xyxy] to [c_x,c_y,w,h]+ boxes_o = xyxy2cxcywh(boxes_o)++ image_t = _distort(image)+ image_t, boxes = _mirror(image_t, boxes)+ height, width, _ = image_t.shape+ image_t, r_ = preproc(image_t, input_dim, self.means, self.std)+ # boxes [xyxy] 2 [cx,cy,w,h]+ boxes = xyxy2cxcywh(boxes)+ boxes *= r_++ mask_b = np.minimum(boxes[:, 2], boxes[:, 3]) > 1+ boxes_t = boxes[mask_b]+ labels_t = labels[mask_b]+ ids_t = ids[mask_b]++ if len(boxes_t) == 0:+ image_t, r_o = preproc(image_o, input_dim, self.means, self.std)+ boxes_o *= r_o+ boxes_t = boxes_o+ labels_t = labels_o+ ids_t = ids_o++ labels_t = np.expand_dims(labels_t, 1)+ ids_t = np.expand_dims(ids_t, 1)++ targets_t = np.hstack((labels_t, boxes_t, ids_t))+ padded_labels = np.zeros((self.max_labels, 6))+ padded_labels[range(len(targets_t))[: self.max_labels]] = targets_t[+ : self.max_labels+ ]+ padded_labels = np.ascontiguousarray(padded_labels, dtype=np.float32)+ image_t = np.ascontiguousarray(image_t, dtype=np.float32)+ return image_t, padded_labels+++class ValTransform:+ """+ Defines the transformations that should be applied to test PIL image+ for input into the network++ dimension -> tensorize -> color adj++ Arguments:+ resize (int): input dimension to SSD+ rgb_means ((int,int,int)): average RGB of the dataset+ (104,117,123)+ swap ((int,int,int)): final order of channels++ Returns:+ transform (transform) : callable transform to be applied to test/val+ data+ """++ def __init__(self, rgb_means=None, std=None, swap=(2, 0, 1)):+ self.means = rgb_means+ self.swap = swap+ self.std = std++ # assume input is cv2 img for now+ def __call__(self, img, res, input_size):+ img, _ = preproc(img, input_size, self.means, self.std, self.swap)+ return img, np.zeros((1, 5))diff --git a/experimental/yolox/data/data_prefetcher.py b/experimental/yolox/data/data_prefetcher.pynew file mode 100644index 0000000..0f5d2b5--- /dev/null+++ b/experimental/yolox/data/data_prefetcher.py@@ -0,0 +1,77 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) Megvii, Inc. and its affiliates.++import torch+import torch.distributed as dist++from yolox.utils import synchronize++import random+++class DataPrefetcher:+ """+ DataPrefetcher is inspired by code of following file:+ https://github.com/NVIDIA/apex/blob/master/examples/imagenet/main_amp.py+ It could speedup your pytorch dataloader. For more information, please check+ https://github.com/NVIDIA/apex/issues/304#issuecomment-493562789.+ """++ def __init__(self, loader):+ self.loader = iter(loader)+ self.stream = torch.cuda.Stream()+ self.input_cuda = self._input_cuda_for_image+ self.record_stream = DataPrefetcher._record_stream_for_image+ self.preload()++ def preload(self):+ try:+ self.next_input, self.next_target, _, _ = next(self.loader)+ except StopIteration:+ self.next_input = None+ self.next_target = None+ return++ with torch.cuda.stream(self.stream):+ self.input_cuda()+ self.next_target = self.next_target.cuda(non_blocking=True)++ def next(self):+ torch.cuda.current_stream().wait_stream(self.stream)+ input = self.next_input+ target = self.next_target+ if input is not None:+ self.record_stream(input)+ if target is not None:+ target.record_stream(torch.cuda.current_stream())+ self.preload()+ return input, target++ def _input_cuda_for_image(self):+ self.next_input = self.next_input.cuda(non_blocking=True)++ @staticmethod+ def _record_stream_for_image(input):+ input.record_stream(torch.cuda.current_stream())+++def random_resize(data_loader, exp, epoch, rank, is_distributed):+ tensor = torch.LongTensor(1).cuda()+ if is_distributed:+ synchronize()++ if rank == 0:+ if epoch > exp.max_epoch - 10:+ size = exp.input_size+ else:+ size = random.randint(*exp.random_size)+ size = int(32 * size)+ tensor.fill_(size)++ if is_distributed:+ synchronize()+ dist.broadcast(tensor, 0)++ input_size = data_loader.change_input_dim(multiple=tensor.item(), random_range=None)+ return input_sizediff --git a/experimental/yolox/data/dataloading.py b/experimental/yolox/data/dataloading.pynew file mode 100644index 0000000..2756f2f--- /dev/null+++ b/experimental/yolox/data/dataloading.py@@ -0,0 +1,178 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) Megvii, Inc. and its affiliates.++import torch+from torch.utils.data.dataloader import DataLoader as torchDataLoader+from torch.utils.data.dataloader import default_collate++import os+import random++from .samplers import YoloBatchSampler+++def get_yolox_datadir():+ """+ get dataset dir of YOLOX. If environment variable named `YOLOX_DATADIR` is set,+ this function will return value of the environment variable. Otherwise, use data+ """+ yolox_datadir = os.getenv("YOLOX_DATADIR", None)+ if yolox_datadir is None:+ import yolox++ yolox_path = os.path.dirname(os.path.dirname(yolox.__file__))+ yolox_datadir = os.path.join(yolox_path, "datasets")+ return yolox_datadir+++class DataLoader(torchDataLoader):+ """+ Lightnet dataloader that enables on the fly resizing of the images.+ See :class:`torch.utils.data.DataLoader` for more information on the arguments.+ Check more on the following website:+ https://gitlab.com/EAVISE/lightnet/-/blob/master/lightnet/data/_dataloading.py++ Note:+ This dataloader only works with :class:`lightnet.data.Dataset` based datasets.++ Example:+ >>> class CustomSet(ln.data.Dataset):+ ... def __len__(self):+ ... return 4+ ... @ln.data.Dataset.resize_getitem+ ... def __getitem__(self, index):+ ... # Should return (image, anno) but here we return (input_dim,)+ ... return (self.input_dim,)+ >>> dl = ln.data.DataLoader(+ ... CustomSet((200,200)),+ ... batch_size = 2,+ ... collate_fn = ln.data.list_collate # We want the data to be grouped as a list+ ... )+ >>> dl.dataset.input_dim # Default input_dim+ (200, 200)+ >>> for d in dl:+ ... d+ [[(200, 200), (200, 200)]]+ [[(200, 200), (200, 200)]]+ >>> dl.change_input_dim(320, random_range=None)+ (320, 320)+ >>> for d in dl:+ ... d+ [[(320, 320), (320, 320)]]+ [[(320, 320), (320, 320)]]+ >>> dl.change_input_dim((480, 320), random_range=None)+ (480, 320)+ >>> for d in dl:+ ... d+ [[(480, 320), (480, 320)]]+ [[(480, 320), (480, 320)]]+ """++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self.__initialized = False+ shuffle = False+ batch_sampler = None+ if len(args) > 5:+ shuffle = args[2]+ sampler = args[3]+ batch_sampler = args[4]+ elif len(args) > 4:+ shuffle = args[2]+ sampler = args[3]+ if "batch_sampler" in kwargs:+ batch_sampler = kwargs["batch_sampler"]+ elif len(args) > 3:+ shuffle = args[2]+ if "sampler" in kwargs:+ sampler = kwargs["sampler"]+ if "batch_sampler" in kwargs:+ batch_sampler = kwargs["batch_sampler"]+ else:+ if "shuffle" in kwargs:+ shuffle = kwargs["shuffle"]+ if "sampler" in kwargs:+ sampler = kwargs["sampler"]+ if "batch_sampler" in kwargs:+ batch_sampler = kwargs["batch_sampler"]++ # Use custom BatchSampler+ if batch_sampler is None:+ if sampler is None:+ if shuffle:+ sampler = torch.utils.data.sampler.RandomSampler(self.dataset)+ # sampler = torch.utils.data.DistributedSampler(self.dataset)+ else:+ sampler = torch.utils.data.sampler.SequentialSampler(self.dataset)+ batch_sampler = YoloBatchSampler(+ sampler,+ self.batch_size,+ self.drop_last,+ input_dimension=self.dataset.input_dim,+ )+ # batch_sampler = IterationBasedBatchSampler(batch_sampler, num_iterations =++ self.batch_sampler = batch_sampler++ self.__initialized = True++ def close_mosaic(self):+ self.batch_sampler.mosaic = False++ def change_input_dim(self, multiple=32, random_range=(10, 19)):+ """This function will compute a new size and update it on the next mini_batch.++ Args:+ multiple (int or tuple, optional): values to multiply the randomly generated range by.+ Default **32**+ random_range (tuple, optional): This (min, max) tuple sets the range+ for the randomisation; Default **(10, 19)**++ Return:+ tuple: width, height tuple with new dimension++ Note:+ The new size is generated as follows: |br|+ First we compute a random integer inside ``[random_range]``.+ We then multiply that number with the ``multiple`` argument,+ which gives our final new input size. |br|+ If ``multiple`` is an integer we generate a square size. If you give a tuple+ of **(width, height)**, the size is computed+ as :math:`rng * multiple[0], rng * multiple[1]`.++ Note:+ You can set the ``random_range`` argument to **None** to set+ an exact size of multiply. |br|+ See the example above for how this works.+ """+ if random_range is None:+ size = 1+ else:+ size = random.randint(*random_range)++ if isinstance(multiple, int):+ size = (size * multiple, size * multiple)+ else:+ size = (size * multiple[0], size * multiple[1])++ self.batch_sampler.new_input_dim = size++ return size+++def list_collate(batch):+ """+ Function that collates lists or tuples together into one list (of lists/tuples).+ Use this as the collate function in a Dataloader, if you want to have a list of+ items as an output, as opposed to tensors (eg. Brambox.boxes).+ """+ items = list(zip(*batch))++ for i in range(len(items)):+ if isinstance(items[i][0], (list, tuple)):+ items[i] = list(items[i])+ else:+ items[i] = default_collate(items[i])++ return itemsdiff --git a/experimental/yolox/data/datasets/__init__.py b/experimental/yolox/data/datasets/__init__.pynew file mode 100644index 0000000..61065a8--- /dev/null+++ b/experimental/yolox/data/datasets/__init__.py@@ -0,0 +1,7 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) Megvii, Inc. and its affiliates.++from .datasets_wrapper import ConcatDataset, Dataset, MixConcatDataset+from .mosaicdetection import MosaicDetection+from .mot import MOTDatasetdiff --git a/experimental/yolox/data/datasets/datasets_wrapper.py b/experimental/yolox/data/datasets/datasets_wrapper.pynew file mode 100644index 0000000..a262e6a--- /dev/null+++ b/experimental/yolox/data/datasets/datasets_wrapper.py@@ -0,0 +1,128 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) Megvii, Inc. and its affiliates.++from torch.utils.data.dataset import ConcatDataset as torchConcatDataset+from torch.utils.data.dataset import Dataset as torchDataset++import bisect+from functools import wraps+++class ConcatDataset(torchConcatDataset):+ def __init__(self, datasets):+ super(ConcatDataset, self).__init__(datasets)+ if hasattr(self.datasets[0], "input_dim"):+ self._input_dim = self.datasets[0].input_dim+ self.input_dim = self.datasets[0].input_dim++ def pull_item(self, idx):+ if idx < 0:+ if -idx > len(self):+ raise ValueError(+ "absolute value of index should not exceed dataset length"+ )+ idx = len(self) + idx+ dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx)+ if dataset_idx == 0:+ sample_idx = idx+ else:+ sample_idx = idx - self.cumulative_sizes[dataset_idx - 1]+ return self.datasets[dataset_idx].pull_item(sample_idx)+++class MixConcatDataset(torchConcatDataset):+ def __init__(self, datasets):+ super(MixConcatDataset, self).__init__(datasets)+ if hasattr(self.datasets[0], "input_dim"):+ self._input_dim = self.datasets[0].input_dim+ self.input_dim = self.datasets[0].input_dim++ def __getitem__(self, index):++ if not isinstance(index, int):+ idx = index[1]+ if idx < 0:+ if -idx > len(self):+ raise ValueError(+ "absolute value of index should not exceed dataset length"+ )+ idx = len(self) + idx+ dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx)+ if dataset_idx == 0:+ sample_idx = idx+ else:+ sample_idx = idx - self.cumulative_sizes[dataset_idx - 1]+ if not isinstance(index, int):+ index = (index[0], sample_idx, index[2])++ return self.datasets[dataset_idx][index]+++class Dataset(torchDataset):+ """ This class is a subclass of the base :class:`torch.utils.data.Dataset`,+ that enables on the fly resizing of the ``input_dim``.++ Args:+ input_dimension (tuple): (width,height) tuple with default dimensions of the network+ """++ def __init__(self, input_dimension, mosaic=True):+ super().__init__()+ self.__input_dim = input_dimension[:2]+ self.enable_mosaic = mosaic++ @property+ def input_dim(self):+ """+ Dimension that can be used by transforms to set the correct image size, etc.+ This allows transforms to have a single source of truth+ for the input dimension of the network.++ Return:+ list: Tuple containing the current width,height+ """+ if hasattr(self, "_input_dim"):+ return self._input_dim+ return self.__input_dim++ @staticmethod+ def resize_getitem(getitem_fn):+ """+ Decorator method that needs to be used around the ``__getitem__`` method. |br|+ This decorator enables the on the fly resizing of+ the ``input_dim`` with our :class:`~lightnet.data.DataLoader` class.++ Example:+ >>> class CustomSet(ln.data.Dataset):+ ... def __len__(self):+ ... return 10+ ... @ln.data.Dataset.resize_getitem+ ... def __getitem__(self, index):+ ... # Should return (image, anno) but here we return input_dim+ ... return self.input_dim+ >>> data = CustomSet((200,200))+ >>> data[0]+ (200, 200)+ >>> data[(480,320), 0]+ (480, 320)+ """++ @wraps(getitem_fn)+ def wrapper(self, index):+ if not isinstance(index, int):+ has_dim = True+ self._input_dim = index[0]+ self.enable_mosaic = index[2]+ index = index[1]+ else:+ has_dim = False++ ret_val = getitem_fn(self, index)++ if has_dim:+ del self._input_dim++ return ret_val++ return wrapperdiff --git a/experimental/yolox/data/datasets/mosaicdetection.py b/experimental/yolox/data/datasets/mosaicdetection.pynew file mode 100644index 0000000..d2bf39f--- /dev/null+++ b/experimental/yolox/data/datasets/mosaicdetection.py@@ -0,0 +1,242 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) Megvii, Inc. and its affiliates.++import cv2+import numpy as np++from yolox.utils import adjust_box_anns++import random++from ..data_augment import box_candidates, random_perspective, augment_hsv+from .datasets_wrapper import Dataset+++def get_mosaic_coordinate(mosaic_image, mosaic_index, xc, yc, w, h, input_h, input_w):+ # TODO update doc+ # index0 to top left part of image+ if mosaic_index == 0:+ x1, y1, x2, y2 = max(xc - w, 0), max(yc - h, 0), xc, yc+ small_coord = w - (x2 - x1), h - (y2 - y1), w, h+ # index1 to top right part of image+ elif mosaic_index == 1:+ x1, y1, x2, y2 = xc, max(yc - h, 0), min(xc + w, input_w * 2), yc+ small_coord = 0, h - (y2 - y1), min(w, x2 - x1), h+ # index2 to bottom left part of image+ elif mosaic_index == 2:+ x1, y1, x2, y2 = max(xc - w, 0), yc, xc, min(input_h * 2, yc + h)+ small_coord = w - (x2 - x1), 0, w, min(y2 - y1, h)+ # index2 to bottom right part of image+ elif mosaic_index == 3:+ x1, y1, x2, y2 = xc, yc, min(xc + w, input_w * 2), min(input_h * 2, yc + h) # noqa+ small_coord = 0, 0, min(w, x2 - x1), min(y2 - y1, h)+ return (x1, y1, x2, y2), small_coord+++class MosaicDetection(Dataset):+ """Detection dataset wrapper that performs mixup for normal dataset."""++ def __init__(+ self, dataset, img_size, mosaic=True, preproc=None,+ degrees=10.0, translate=0.1, scale=(0.5, 1.5), mscale=(0.5, 1.5),+ shear=2.0, perspective=0.0, enable_mixup=True, *args+ ):+ """++ Args:+ dataset(Dataset) : Pytorch dataset object.+ img_size (tuple):+ mosaic (bool): enable mosaic augmentation or not.+ preproc (func):+ degrees (float):+ translate (float):+ scale (tuple):+ mscale (tuple):+ shear (float):+ perspective (float):+ enable_mixup (bool):+ *args(tuple) : Additional arguments for mixup random sampler.+ """+ super().__init__(img_size, mosaic=mosaic)+ self._dataset = dataset+ self.preproc = preproc+ self.degrees = degrees+ self.translate = translate+ self.scale = scale+ self.shear = shear+ self.perspective = perspective+ self.mixup_scale = mscale+ self.enable_mosaic = mosaic+ self.enable_mixup = enable_mixup++ def __len__(self):+ return len(self._dataset)++ @Dataset.resize_getitem+ def __getitem__(self, idx):+ if self.enable_mosaic:+ mosaic_labels = []+ input_dim = self._dataset.input_dim+ input_h, input_w = input_dim[0], input_dim[1]++ # yc, xc = s, s # mosaic center x, y+ yc = int(random.uniform(0.5 * input_h, 1.5 * input_h))+ xc = int(random.uniform(0.5 * input_w, 1.5 * input_w))++ # 3 additional image indices+ indices = [idx] + [random.randint(0, len(self._dataset) - 1) for _ in range(3)]++ for i_mosaic, index in enumerate(indices):+ img, _labels, _, _ = self._dataset.pull_item(index)+ h0, w0 = img.shape[:2] # orig hw+ scale = min(1. * input_h / h0, 1. * input_w / w0)+ img = cv2.resize(+ img, (int(w0 * scale), int(h0 * scale)), interpolation=cv2.INTER_LINEAR+ )+ # generate output mosaic image+ (h, w, c) = img.shape[:3]+ if i_mosaic == 0:+ mosaic_img = np.full((input_h * 2, input_w * 2, c), 114, dtype=np.uint8)++ # suffix l means large image, while s means small image in mosaic aug.+ (l_x1, l_y1, l_x2, l_y2), (s_x1, s_y1, s_x2, s_y2) = get_mosaic_coordinate(+ mosaic_img, i_mosaic, xc, yc, w, h, input_h, input_w+ )++ mosaic_img[l_y1:l_y2, l_x1:l_x2] = img[s_y1:s_y2, s_x1:s_x2]+ padw, padh = l_x1 - s_x1, l_y1 - s_y1++ labels = _labels.copy()+ # Normalized xywh to pixel xyxy format+ if _labels.size > 0:+ labels[:, 0] = scale * _labels[:, 0] + padw+ labels[:, 1] = scale * _labels[:, 1] + padh+ labels[:, 2] = scale * _labels[:, 2] + padw+ labels[:, 3] = scale * _labels[:, 3] + padh+ mosaic_labels.append(labels)++ if len(mosaic_labels):+ mosaic_labels = np.concatenate(mosaic_labels, 0)+ '''+ np.clip(mosaic_labels[:, 0], 0, 2 * input_w, out=mosaic_labels[:, 0])+ np.clip(mosaic_labels[:, 1], 0, 2 * input_h, out=mosaic_labels[:, 1])+ np.clip(mosaic_labels[:, 2], 0, 2 * input_w, out=mosaic_labels[:, 2])+ np.clip(mosaic_labels[:, 3], 0, 2 * input_h, out=mosaic_labels[:, 3])+ '''++ mosaic_labels = mosaic_labels[mosaic_labels[:, 0] < 2 * input_w]+ mosaic_labels = mosaic_labels[mosaic_labels[:, 2] > 0]+ mosaic_labels = mosaic_labels[mosaic_labels[:, 1] < 2 * input_h]+ mosaic_labels = mosaic_labels[mosaic_labels[:, 3] > 0]++ #augment_hsv(mosaic_img)+ mosaic_img, mosaic_labels = random_perspective(+ mosaic_img,+ mosaic_labels,+ degrees=self.degrees,+ translate=self.translate,+ scale=self.scale,+ shear=self.shear,+ perspective=self.perspective,+ border=[-input_h // 2, -input_w // 2],+ ) # border to remove++ # -----------------------------------------------------------------+ # CopyPaste: https://arxiv.org/abs/2012.07177+ # -----------------------------------------------------------------+ if self.enable_mixup and not len(mosaic_labels) == 0:+ mosaic_img, mosaic_labels = self.mixup(mosaic_img, mosaic_labels, self.input_dim)++ mix_img, padded_labels = self.preproc(mosaic_img, mosaic_labels, self.input_dim)+ img_info = (mix_img.shape[1], mix_img.shape[0])++ return mix_img, padded_labels, img_info, np.array([idx])++ else:+ self._dataset._input_dim = self.input_dim+ img, label, img_info, id_ = self._dataset.pull_item(idx)+ img, label = self.preproc(img, label, self.input_dim)+ return img, label, img_info, id_++ def mixup(self, origin_img, origin_labels, input_dim):+ jit_factor = random.uniform(*self.mixup_scale)+ FLIP = random.uniform(0, 1) > 0.5+ cp_labels = []+ while len(cp_labels) == 0:+ cp_index = random.randint(0, self.__len__() - 1)+ cp_labels = self._dataset.load_anno(cp_index)+ img, cp_labels, _, _ = self._dataset.pull_item(cp_index)++ if len(img.shape) == 3:+ cp_img = np.ones((input_dim[0], input_dim[1], 3)) * 114.0+ else:+ cp_img = np.ones(input_dim) * 114.0+ cp_scale_ratio = min(input_dim[0] / img.shape[0], input_dim[1] / img.shape[1])+ resized_img = cv2.resize(+ img,+ (int(img.shape[1] * cp_scale_ratio), int(img.shape[0] * cp_scale_ratio)),+ interpolation=cv2.INTER_LINEAR,+ ).astype(np.float32)+ cp_img[+ : int(img.shape[0] * cp_scale_ratio), : int(img.shape[1] * cp_scale_ratio)+ ] = resized_img+ cp_img = cv2.resize(+ cp_img,+ (int(cp_img.shape[1] * jit_factor), int(cp_img.shape[0] * jit_factor)),+ )+ cp_scale_ratio *= jit_factor+ if FLIP:+ cp_img = cp_img[:, ::-1, :]++ origin_h, origin_w = cp_img.shape[:2]+ target_h, target_w = origin_img.shape[:2]+ padded_img = np.zeros(+ (max(origin_h, target_h), max(origin_w, target_w), 3)+ ).astype(np.uint8)+ padded_img[:origin_h, :origin_w] = cp_img++ x_offset, y_offset = 0, 0+ if padded_img.shape[0] > target_h:+ y_offset = random.randint(0, padded_img.shape[0] - target_h - 1)+ if padded_img.shape[1] > target_w:+ x_offset = random.randint(0, padded_img.shape[1] - target_w - 1)+ padded_cropped_img = padded_img[+ y_offset: y_offset + target_h, x_offset: x_offset + target_w+ ]++ cp_bboxes_origin_np = adjust_box_anns(+ cp_labels[:, :4].copy(), cp_scale_ratio, 0, 0, origin_w, origin_h+ )+ if FLIP:+ cp_bboxes_origin_np[:, 0::2] = (+ origin_w - cp_bboxes_origin_np[:, 0::2][:, ::-1]+ )+ cp_bboxes_transformed_np = cp_bboxes_origin_np.copy()+ '''+ cp_bboxes_transformed_np[:, 0::2] = np.clip(+ cp_bboxes_transformed_np[:, 0::2] - x_offset, 0, target_w+ )+ cp_bboxes_transformed_np[:, 1::2] = np.clip(+ cp_bboxes_transformed_np[:, 1::2] - y_offset, 0, target_h+ )+ '''+ cp_bboxes_transformed_np[:, 0::2] = cp_bboxes_transformed_np[:, 0::2] - x_offset+ cp_bboxes_transformed_np[:, 1::2] = cp_bboxes_transformed_np[:, 1::2] - y_offset+ keep_list = box_candidates(cp_bboxes_origin_np.T, cp_bboxes_transformed_np.T, 5)++ if keep_list.sum() >= 1.0:+ cls_labels = cp_labels[keep_list, 4:5].copy()+ id_labels = cp_labels[keep_list, 5:6].copy()+ box_labels = cp_bboxes_transformed_np[keep_list]+ labels = np.hstack((box_labels, cls_labels, id_labels))+ # remove outside bbox+ labels = labels[labels[:, 0] < target_w]+ labels = labels[labels[:, 2] > 0]+ labels = labels[labels[:, 1] < target_h]+ labels = labels[labels[:, 3] > 0]+ origin_labels = np.vstack((origin_labels, labels))+ origin_img = origin_img.astype(np.float32)+ origin_img = 0.5 * origin_img + 0.5 * padded_cropped_img.astype(np.float32)++ return origin_img, origin_labelsdiff --git a/experimental/yolox/data/datasets/mot.py b/experimental/yolox/data/datasets/mot.pynew file mode 100644index 0000000..d52febc--- /dev/null+++ b/experimental/yolox/data/datasets/mot.py@@ -0,0 +1,132 @@+import cv2+import numpy as np+from pycocotools.coco import COCO++import os++from ..dataloading import get_yolox_datadir+from .datasets_wrapper import Dataset+++class MOTDataset(Dataset):+ """+ COCO dataset class.+ """++ def __init__(+ self,+ data_dir=None,+ json_file="train_half.json",+ name="train",+ img_size=(608, 1088),+ preproc=None,+ ):+ """+ COCO dataset initialization. Annotation data are read into memory by COCO API.+ Args:+ data_dir (str): dataset root directory+ json_file (str): COCO json file name+ name (str): COCO data name (e.g. 'train2017' or 'val2017')+ img_size (int): target image size after pre-processing+ preproc: data augmentation strategy+ """+ super().__init__(img_size)+ if data_dir is None:+ data_dir = os.path.join(get_yolox_datadir(), "mot")+ self.data_dir = data_dir+ self.json_file = json_file++ self.coco = COCO(os.path.join(self.data_dir, "annotations", self.json_file))+ self.ids = self.coco.getImgIds()+ self.class_ids = sorted(self.coco.getCatIds())+ cats = self.coco.loadCats(self.coco.getCatIds())+ self._classes = tuple([c["name"] for c in cats])+ self.annotations = self._load_coco_annotations()+ self.name = name+ self.img_size = img_size+ self.preproc = preproc++ def __len__(self):+ return len(self.ids)++ def _load_coco_annotations(self):+ return [self.load_anno_from_ids(_ids) for _ids in self.ids]++ def load_anno_from_ids(self, id_):+ im_ann = self.coco.loadImgs(id_)[0]+ width = im_ann["width"]+ height = im_ann["height"]+ frame_id = im_ann["frame_id"]+ video_id = im_ann["video_id"]+ anno_ids = self.coco.getAnnIds(imgIds=[int(id_)], iscrowd=False)+ annotations = self.coco.loadAnns(anno_ids)+ objs = []+ for obj in annotations:+ x1 = obj["bbox"][0]+ y1 = obj["bbox"][1]+ x2 = x1 + obj["bbox"][2]+ y2 = y1 + obj["bbox"][3]+ if obj["area"] > 0 and x2 >= x1 and y2 >= y1:+ obj["clean_bbox"] = [x1, y1, x2, y2]+ objs.append(obj)++ num_objs = len(objs)++ res = np.zeros((num_objs, 6))++ for ix, obj in enumerate(objs):+ cls = self.class_ids.index(obj["category_id"])+ res[ix, 0:4] = obj["clean_bbox"]+ res[ix, 4] = cls+ res[ix, 5] = obj["track_id"]++ file_name = im_ann["file_name"] if "file_name" in im_ann else "{:012}".format(id_) + ".jpg"+ img_info = (height, width, frame_id, video_id, file_name)++ del im_ann, annotations++ return (res, img_info, file_name)++ def load_anno(self, index):+ return self.annotations[index][0]++ def pull_item(self, index):+ id_ = self.ids[index]++ res, img_info, file_name = self.annotations[index]+ # load image and preprocess+ img_file = os.path.join(+ self.data_dir, self.name, file_name+ )+ img = cv2.imread(img_file)+ assert img is not None++ return img, res.copy(), img_info, np.array([id_])++ @Dataset.resize_getitem+ def __getitem__(self, index):+ """+ One image / label pair for the given index is picked up and pre-processed.++ Args:+ index (int): data index++ Returns:+ img (numpy.ndarray): pre-processed image+ padded_labels (torch.Tensor): pre-processed label data.+ The shape is :math:`[max_labels, 5]`.+ each label consists of [class, xc, yc, w, h]:+ class (float): class index.+ xc, yc (float) : center of bbox whose values range from 0 to 1.+ w, h (float) : size of bbox whose values range from 0 to 1.+ info_img : tuple of h, w, nh, nw, dx, dy.+ h, w (int): original shape of the image+ nh, nw (int): shape of the resized image without padding+ dx, dy (int): pad size+ img_id (int): same as the input index. Used for evaluation.+ """+ img, target, img_info, img_id = self.pull_item(index)++ if self.preproc is not None:+ img, target = self.preproc(img, target, self.input_dim)+ return img, target, img_info, img_iddiff --git a/experimental/yolox/data/samplers.py b/experimental/yolox/data/samplers.pynew file mode 100644index 0000000..064b133--- /dev/null+++ b/experimental/yolox/data/samplers.py@@ -0,0 +1,95 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) Megvii, Inc. and its affiliates.++import torch+import torch.distributed as dist+from torch.utils.data.sampler import BatchSampler as torchBatchSampler+from torch.utils.data.sampler import Sampler++import itertools+from typing import Optional+++class YoloBatchSampler(torchBatchSampler):+ """+ This batch sampler will generate mini-batches of (dim, index) tuples from another sampler.+ It works just like the :class:`torch.utils.data.sampler.BatchSampler`,+ but it will prepend a dimension, whilst ensuring it stays the same across one mini-batch.+ """++ def __init__(self, *args, input_dimension=None, mosaic=True, **kwargs):+ super().__init__(*args, **kwargs)+ self.input_dim = input_dimension+ self.new_input_dim = None+ self.mosaic = mosaic++ def __iter__(self):+ self.__set_input_dim()+ for batch in super().__iter__():+ yield [(self.input_dim, idx, self.mosaic) for idx in batch]+ self.__set_input_dim()++ def __set_input_dim(self):+ """ This function randomly changes the the input dimension of the dataset. """+ if self.new_input_dim is not None:+ self.input_dim = (self.new_input_dim[0], self.new_input_dim[1])+ self.new_input_dim = None+++class InfiniteSampler(Sampler):+ """+ In training, we only care about the "infinite stream" of training data.+ So this sampler produces an infinite stream of indices and+ all workers cooperate to correctly shuffle the indices and sample different indices.+ The samplers in each worker effectively produces `indices[worker_id::num_workers]`+ where `indices` is an infinite stream of indices consisting of+ `shuffle(range(size)) + shuffle(range(size)) + ...` (if shuffle is True)+ or `range(size) + range(size) + ...` (if shuffle is False)+ """++ def __init__(+ self,+ size: int,+ shuffle: bool = True,+ seed: Optional[int] = 0,+ rank=0,+ world_size=1,+ ):+ """+ Args:+ size (int): the total number of data of the underlying dataset to sample from+ shuffle (bool): whether to shuffle the indices or not+ seed (int): the initial seed of the shuffle. Must be the same+ across all workers. If None, will use a random seed shared+ among workers (require synchronization among all workers).+ """+ self._size = size+ assert size > 0+ self._shuffle = shuffle+ self._seed = int(seed)++ if dist.is_available() and dist.is_initialized():+ self._rank = dist.get_rank()+ self._world_size = dist.get_world_size()+ else:+ self._rank = rank+ self._world_size = world_size++ def __iter__(self):+ start = self._rank+ yield from itertools.islice(+ self._infinite_indices(), start, None, self._world_size+ )++ def _infinite_indices(self):+ g = torch.Generator()+ g.manual_seed(self._seed)+ while True:+ if self._shuffle:+ yield from torch.randperm(self._size, generator=g)+ else:+ yield from torch.arange(self._size)++ def __len__(self):+ return self._size // self._world_sizediff --git a/experimental/yolox/deepsort_tracker/deepsort.py b/experimental/yolox/deepsort_tracker/deepsort.pynew file mode 100644index 0000000..198f3b0--- /dev/null+++ b/experimental/yolox/deepsort_tracker/deepsort.py@@ -0,0 +1,295 @@+import numpy as np+import torch+import cv2+import os++from .reid_model import Extractor+from yolox.deepsort_tracker import kalman_filter, linear_assignment, iou_matching+from yolox.data.dataloading import get_yolox_datadir+from .detection import Detection+from .track import Track+++def _cosine_distance(a, b, data_is_normalized=False):+ if not data_is_normalized:+ a = np.asarray(a) / np.linalg.norm(a, axis=1, keepdims=True)+ b = np.asarray(b) / np.linalg.norm(b, axis=1, keepdims=True)+ return 1. - np.dot(a, b.T)+++def _nn_cosine_distance(x, y):+ distances = _cosine_distance(x, y)+ return distances.min(axis=0)+++class Tracker:+ def __init__(self, metric, max_iou_distance=0.7, max_age=70, n_init=3):+ self.metric = metric+ self.max_iou_distance = max_iou_distance+ self.max_age = max_age+ self.n_init = n_init++ self.kf = kalman_filter.KalmanFilter()+ self.tracks = []+ self._next_id = 1++ def predict(self):+ """Propagate track state distributions one time step forward.+ This function should be called once every time step, before `update`.+ """+ for track in self.tracks:+ track.predict(self.kf)++ def increment_ages(self):+ for track in self.tracks:+ track.increment_age()+ track.mark_missed()++ def update(self, detections, classes):+ """Perform measurement update and track management.+ Parameters+ ----------+ detections : List[deep_sort.detection.Detection]+ A list of detections at the current time step.+ """+ # Run matching cascade.+ matches, unmatched_tracks, unmatched_detections = \+ self._match(detections)++ # Update track set.+ for track_idx, detection_idx in matches:+ self.tracks[track_idx].update(+ self.kf, detections[detection_idx])+ for track_idx in unmatched_tracks:+ self.tracks[track_idx].mark_missed()+ for detection_idx in unmatched_detections:+ self._initiate_track(detections[detection_idx], classes[detection_idx].item())+ self.tracks = [t for t in self.tracks if not t.is_deleted()]++ # Update distance metric.+ active_targets = [t.track_id for t in self.tracks if t.is_confirmed()]+ features, targets = [], []+ for track in self.tracks:+ if not track.is_confirmed():+ continue+ features += track.features+ targets += [track.track_id for _ in track.features]+ track.features = []+ self.metric.partial_fit(+ np.asarray(features), np.asarray(targets), active_targets)++ def _match(self, detections):++ def gated_metric(tracks, dets, track_indices, detection_indices):+ features = np.array([dets[i].feature for i in detection_indices])+ targets = np.array([tracks[i].track_id for i in track_indices])+ cost_matrix = self.metric.distance(features, targets)+ cost_matrix = linear_assignment.gate_cost_matrix(+ self.kf, cost_matrix, tracks, dets, track_indices,+ detection_indices)++ return cost_matrix++ # Split track set into confirmed and unconfirmed tracks.+ confirmed_tracks = [+ i for i, t in enumerate(self.tracks) if t.is_confirmed()]+ unconfirmed_tracks = [+ i for i, t in enumerate(self.tracks) if not t.is_confirmed()]++ # Associate confirmed tracks using appearance features.+ matches_a, unmatched_tracks_a, unmatched_detections = \+ linear_assignment.matching_cascade(+ gated_metric, self.metric.matching_threshold, self.max_age,+ self.tracks, detections, confirmed_tracks)++ # Associate remaining tracks together with unconfirmed tracks using IOU.+ iou_track_candidates = unconfirmed_tracks + [+ k for k in unmatched_tracks_a if+ self.tracks[k].time_since_update == 1]+ unmatched_tracks_a = [+ k for k in unmatched_tracks_a if+ self.tracks[k].time_since_update != 1]+ matches_b, unmatched_tracks_b, unmatched_detections = \+ linear_assignment.min_cost_matching(+ iou_matching.iou_cost, self.max_iou_distance, self.tracks,+ detections, iou_track_candidates, unmatched_detections)++ matches = matches_a + matches_b+ unmatched_tracks = list(set(unmatched_tracks_a + unmatched_tracks_b))+ return matches, unmatched_tracks, unmatched_detections++ def _initiate_track(self, detection, class_id):+ mean, covariance = self.kf.initiate(detection.to_xyah())+ self.tracks.append(Track(+ mean, covariance, self._next_id, class_id, self.n_init, self.max_age,+ detection.feature))+ self._next_id += 1+++class NearestNeighborDistanceMetric(object):+ def __init__(self, metric, matching_threshold, budget=None):++ if metric == "cosine":+ self._metric = _nn_cosine_distance+ else:+ raise ValueError(+ "Invalid metric; must be either 'euclidean' or 'cosine'")+ self.matching_threshold = matching_threshold+ self.budget = budget+ self.samples = {}++ def partial_fit(self, features, targets, active_targets):+ for feature, target in zip(features, targets):+ self.samples.setdefault(target, []).append(feature)+ if self.budget is not None:+ self.samples[target] = self.samples[target][-self.budget:]+ self.samples = {k: self.samples[k] for k in active_targets}++ def distance(self, features, targets):+ cost_matrix = np.zeros((len(targets), len(features)))+ for i, target in enumerate(targets):+ cost_matrix[i, :] = self._metric(self.samples[target], features)+ return cost_matrix+++class DeepSort(object):+ def __init__(self, model_path, max_dist=0.1, min_confidence=0.3, nms_max_overlap=1.0, max_iou_distance=0.7, max_age=30, n_init=3, nn_budget=100, use_cuda=True):+ self.min_confidence = min_confidence+ self.nms_max_overlap = nms_max_overlap++ self.extractor = Extractor(model_path, use_cuda=use_cuda)++ max_cosine_distance = max_dist+ metric = NearestNeighborDistanceMetric(+ "cosine", max_cosine_distance, nn_budget)+ self.tracker = Tracker(+ metric, max_iou_distance=max_iou_distance, max_age=max_age, n_init=n_init)++ def update(self, output_results, img_info, img_size, img_file_name):+ img_file_name = os.path.join(get_yolox_datadir(), 'mot', 'train', img_file_name)+ ori_img = cv2.imread(img_file_name)+ self.height, self.width = ori_img.shape[:2]+ # post process detections+ output_results = output_results.cpu().numpy()+ confidences = output_results[:, 4] * output_results[:, 5]++ bboxes = output_results[:, :4] # x1y1x2y2+ img_h, img_w = img_info[0], img_info[1]+ scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w))+ bboxes /= scale+ bbox_xyxy = bboxes+ bbox_tlwh = self._xyxy_to_tlwh_array(bbox_xyxy)+ remain_inds = confidences > self.min_confidence+ bbox_tlwh = bbox_tlwh[remain_inds]+ confidences = confidences[remain_inds]++ # generate detections+ features = self._get_features(bbox_tlwh, ori_img)+ detections = [Detection(bbox_tlwh[i], conf, features[i]) for i, conf in enumerate(+ confidences) if conf > self.min_confidence]+ classes = np.zeros((len(detections), ))++ # run on non-maximum supression+ boxes = np.array([d.tlwh for d in detections])+ scores = np.array([d.confidence for d in detections])++ # update tracker+ self.tracker.predict()+ self.tracker.update(detections, classes)++ # output bbox identities+ outputs = []+ for track in self.tracker.tracks:+ if not track.is_confirmed() or track.time_since_update > 1:+ continue+ box = track.to_tlwh()+ x1, y1, x2, y2 = self._tlwh_to_xyxy_noclip(box)+ track_id = track.track_id+ class_id = track.class_id+ outputs.append(np.array([x1, y1, x2, y2, track_id, class_id], dtype=np.int))+ if len(outputs) > 0:+ outputs = np.stack(outputs, axis=0)+ return outputs++ """+ TODO:+ Convert bbox from xc_yc_w_h to xtl_ytl_w_h+ Thanks [email protected] for reporting this bug!+ """+ @staticmethod+ def _xywh_to_tlwh(bbox_xywh):+ if isinstance(bbox_xywh, np.ndarray):+ bbox_tlwh = bbox_xywh.copy()+ elif isinstance(bbox_xywh, torch.Tensor):+ bbox_tlwh = bbox_xywh.clone()+ bbox_tlwh[:, 0] = bbox_xywh[:, 0] - bbox_xywh[:, 2] / 2.+ bbox_tlwh[:, 1] = bbox_xywh[:, 1] - bbox_xywh[:, 3] / 2.+ return bbox_tlwh++ @staticmethod+ def _xyxy_to_tlwh_array(bbox_xyxy):+ if isinstance(bbox_xyxy, np.ndarray):+ bbox_tlwh = bbox_xyxy.copy()+ elif isinstance(bbox_xyxy, torch.Tensor):+ bbox_tlwh = bbox_xyxy.clone()+ bbox_tlwh[:, 2] = bbox_xyxy[:, 2] - bbox_xyxy[:, 0]+ bbox_tlwh[:, 3] = bbox_xyxy[:, 3] - bbox_xyxy[:, 1]+ return bbox_tlwh++ def _xywh_to_xyxy(self, bbox_xywh):+ x, y, w, h = bbox_xywh+ x1 = max(int(x - w / 2), 0)+ x2 = min(int(x + w / 2), self.width - 1)+ y1 = max(int(y - h / 2), 0)+ y2 = min(int(y + h / 2), self.height - 1)+ return x1, y1, x2, y2++ def _tlwh_to_xyxy(self, bbox_tlwh):+ """+ TODO:+ Convert bbox from xtl_ytl_w_h to xc_yc_w_h+ Thanks [email protected] for reporting this bug!+ """+ x, y, w, h = bbox_tlwh+ x1 = max(int(x), 0)+ x2 = min(int(x+w), self.width - 1)+ y1 = max(int(y), 0)+ y2 = min(int(y+h), self.height - 1)+ return x1, y1, x2, y2++ def _tlwh_to_xyxy_noclip(self, bbox_tlwh):+ """+ TODO:+ Convert bbox from xtl_ytl_w_h to xc_yc_w_h+ Thanks [email protected] for reporting this bug!+ """+ x, y, w, h = bbox_tlwh+ x1 = x+ x2 = x + w+ y1 = y+ y2 = y + h+ return x1, y1, x2, y2++ def increment_ages(self):+ self.tracker.increment_ages()++ def _xyxy_to_tlwh(self, bbox_xyxy):+ x1, y1, x2, y2 = bbox_xyxy++ t = x1+ l = y1+ w = int(x2 - x1)+ h = int(y2 - y1)+ return t, l, w, h++ def _get_features(self, bbox_xywh, ori_img):+ im_crops = []+ for box in bbox_xywh:+ x1, y1, x2, y2 = self._tlwh_to_xyxy(box)+ im = ori_img[y1:y2, x1:x2]+ im_crops.append(im)+ if im_crops:+ features = self.extractor(im_crops)+ else:+ features = np.array([])+ return featuresdiff --git a/experimental/yolox/deepsort_tracker/detection.py b/experimental/yolox/deepsort_tracker/detection.pynew file mode 100644index 0000000..9bd8497--- /dev/null+++ b/experimental/yolox/deepsort_tracker/detection.py@@ -0,0 +1,46 @@+# vim: expandtab:ts=4:sw=4+import numpy as np+++class Detection(object):+ """+ This class represents a bounding box detection in a single image.+ Parameters+ ----------+ tlwh : array_like+ Bounding box in format `(x, y, w, h)`.+ confidence : float+ Detector confidence score.+ feature : array_like+ A feature vector that describes the object contained in this image.+ Attributes+ ----------+ tlwh : ndarray+ Bounding box in format `(top left x, top left y, width, height)`.+ confidence : ndarray+ Detector confidence score.+ feature : ndarray | NoneType+ A feature vector that describes the object contained in this image.+ """++ def __init__(self, tlwh, confidence, feature):+ self.tlwh = np.asarray(tlwh, dtype=np.float)+ self.confidence = float(confidence)+ self.feature = np.asarray(feature, dtype=np.float32)++ def to_tlbr(self):+ """Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,+ `(top left, bottom right)`.+ """+ ret = self.tlwh.copy()+ ret[2:] += ret[:2]+ return ret++ def to_xyah(self):+ """Convert bounding box to format `(center x, center y, aspect ratio,+ height)`, where the aspect ratio is `width / height`.+ """+ ret = self.tlwh.copy()+ ret[:2] += ret[2:] / 2+ ret[2] /= ret[3]+ return ret\ No newline at end of filediff --git a/experimental/yolox/deepsort_tracker/iou_matching.py b/experimental/yolox/deepsort_tracker/iou_matching.pynew file mode 100644index 0000000..7a27a4d--- /dev/null+++ b/experimental/yolox/deepsort_tracker/iou_matching.py@@ -0,0 +1,76 @@+# vim: expandtab:ts=4:sw=4+from __future__ import absolute_import+import numpy as np+from yolox.deepsort_tracker import linear_assignment+++def iou(bbox, candidates):+ """Computer intersection over union.+ Parameters+ ----------+ bbox : ndarray+ A bounding box in format `(top left x, top left y, width, height)`.+ candidates : ndarray+ A matrix of candidate bounding boxes (one per row) in the same format+ as `bbox`.+ Returns+ -------+ ndarray+ The intersection over union in [0, 1] between the `bbox` and each+ candidate. A higher score means a larger fraction of the `bbox` is+ occluded by the candidate.+ """+ bbox_tl, bbox_br = bbox[:2], bbox[:2] + bbox[2:]+ candidates_tl = candidates[:, :2]+ candidates_br = candidates[:, :2] + candidates[:, 2:]++ tl = np.c_[np.maximum(bbox_tl[0], candidates_tl[:, 0])[:, np.newaxis],+ np.maximum(bbox_tl[1], candidates_tl[:, 1])[:, np.newaxis]]+ br = np.c_[np.minimum(bbox_br[0], candidates_br[:, 0])[:, np.newaxis],+ np.minimum(bbox_br[1], candidates_br[:, 1])[:, np.newaxis]]+ wh = np.maximum(0., br - tl)++ area_intersection = wh.prod(axis=1)+ area_bbox = bbox[2:].prod()+ area_candidates = candidates[:, 2:].prod(axis=1)+ return area_intersection / (area_bbox + area_candidates - area_intersection)+++def iou_cost(tracks, detections, track_indices=None,+ detection_indices=None):+ """An intersection over union distance metric.+ Parameters+ ----------+ tracks : List[deep_sort.track.Track]+ A list of tracks.+ detections : List[deep_sort.detection.Detection]+ A list of detections.+ track_indices : Optional[List[int]]+ A list of indices to tracks that should be matched. Defaults to+ all `tracks`.+ detection_indices : Optional[List[int]]+ A list of indices to detections that should be matched. Defaults+ to all `detections`.+ Returns+ -------+ ndarray+ Returns a cost matrix of shape+ len(track_indices), len(detection_indices) where entry (i, j) is+ `1 - iou(tracks[track_indices[i]], detections[detection_indices[j]])`.+ """+ if track_indices is None:+ track_indices = np.arange(len(tracks))+ if detection_indices is None:+ detection_indices = np.arange(len(detections))++ cost_matrix = np.zeros((len(track_indices), len(detection_indices)))+ for row, track_idx in enumerate(track_indices):+ if tracks[track_idx].time_since_update > 1:+ cost_matrix[row, :] = linear_assignment.INFTY_COST+ continue++ bbox = tracks[track_idx].to_tlwh()+ candidates = np.asarray(+ [detections[i].tlwh for i in detection_indices])+ cost_matrix[row, :] = 1. - iou(bbox, candidates)+ return cost_matrix\ No newline at end of filediff --git a/experimental/yolox/deepsort_tracker/kalman_filter.py b/experimental/yolox/deepsort_tracker/kalman_filter.pynew file mode 100644index 0000000..1d4bcc3--- /dev/null+++ b/experimental/yolox/deepsort_tracker/kalman_filter.py@@ -0,0 +1,208 @@+# vim: expandtab:ts=4:sw=4+import numpy as np+import scipy.linalg+++"""+Table for the 0.95 quantile of the chi-square distribution with N degrees of+freedom (contains values for N=1, ..., 9). Taken from MATLAB/Octave's chi2inv+function and used as Mahalanobis gating threshold.+"""+chi2inv95 = {+ 1: 3.8415,+ 2: 5.9915,+ 3: 7.8147,+ 4: 9.4877,+ 5: 11.070,+ 6: 12.592,+ 7: 14.067,+ 8: 15.507,+ 9: 16.919}+++class KalmanFilter(object):+ """+ A simple Kalman filter for tracking bounding boxes in image space.+ The 8-dimensional state space+ x, y, a, h, vx, vy, va, vh+ contains the bounding box center position (x, y), aspect ratio a, height h,+ and their respective velocities.+ Object motion follows a constant velocity model. The bounding box location+ (x, y, a, h) is taken as direct observation of the state space (linear+ observation model).+ """++ def __init__(self):+ ndim, dt = 4, 1.++ # Create Kalman filter model matrices.+ self._motion_mat = np.eye(2 * ndim, 2 * ndim)+ for i in range(ndim):+ self._motion_mat[i, ndim + i] = dt+ self._update_mat = np.eye(ndim, 2 * ndim)++ # Motion and observation uncertainty are chosen relative to the current+ # state estimate. These weights control the amount of uncertainty in+ # the model. This is a bit hacky.+ self._std_weight_position = 1. / 20+ self._std_weight_velocity = 1. / 160++ def initiate(self, measurement):+ """Create track from unassociated measurement.+ Parameters+ ----------+ measurement : ndarray+ Bounding box coordinates (x, y, a, h) with center position (x, y),+ aspect ratio a, and height h.+ Returns+ -------+ (ndarray, ndarray)+ Returns the mean vector (8 dimensional) and covariance matrix (8x8+ dimensional) of the new track. Unobserved velocities are initialized+ to 0 mean.+ """+ mean_pos = measurement+ mean_vel = np.zeros_like(mean_pos)+ mean = np.r_[mean_pos, mean_vel]++ std = [+ 2 * self._std_weight_position * measurement[3],+ 2 * self._std_weight_position * measurement[3],+ 1e-2,+ 2 * self._std_weight_position * measurement[3],+ 10 * self._std_weight_velocity * measurement[3],+ 10 * self._std_weight_velocity * measurement[3],+ 1e-5,+ 10 * self._std_weight_velocity * measurement[3]]+ covariance = np.diag(np.square(std))+ return mean, covariance++ def predict(self, mean, covariance):+ """Run Kalman filter prediction step.+ Parameters+ ----------+ mean : ndarray+ The 8 dimensional mean vector of the object state at the previous+ time step.+ covariance : ndarray+ The 8x8 dimensional covariance matrix of the object state at the+ previous time step.+ Returns+ -------+ (ndarray, ndarray)+ Returns the mean vector and covariance matrix of the predicted+ state. Unobserved velocities are initialized to 0 mean.+ """+ std_pos = [+ self._std_weight_position * mean[3],+ self._std_weight_position * mean[3],+ 1e-2,+ self._std_weight_position * mean[3]]+ std_vel = [+ self._std_weight_velocity * mean[3],+ self._std_weight_velocity * mean[3],+ 1e-5,+ self._std_weight_velocity * mean[3]]+ motion_cov = np.diag(np.square(np.r_[std_pos, std_vel]))++ mean = np.dot(self._motion_mat, mean)+ covariance = np.linalg.multi_dot((+ self._motion_mat, covariance, self._motion_mat.T)) + motion_cov++ return mean, covariance++ def project(self, mean, covariance):+ """Project state distribution to measurement space.+ Parameters+ ----------+ mean : ndarray+ The state's mean vector (8 dimensional array).+ covariance : ndarray+ The state's covariance matrix (8x8 dimensional).+ Returns+ -------+ (ndarray, ndarray)+ Returns the projected mean and covariance matrix of the given state+ estimate.+ """+ std = [+ self._std_weight_position * mean[3],+ self._std_weight_position * mean[3],+ 1e-1,+ self._std_weight_position * mean[3]]+ innovation_cov = np.diag(np.square(std))++ mean = np.dot(self._update_mat, mean)+ covariance = np.linalg.multi_dot((+ self._update_mat, covariance, self._update_mat.T))+ return mean, covariance + innovation_cov++ def update(self, mean, covariance, measurement):+ """Run Kalman filter correction step.+ Parameters+ ----------+ mean : ndarray+ The predicted state's mean vector (8 dimensional).+ covariance : ndarray+ The state's covariance matrix (8x8 dimensional).+ measurement : ndarray+ The 4 dimensional measurement vector (x, y, a, h), where (x, y)+ is the center position, a the aspect ratio, and h the height of the+ bounding box.+ Returns+ -------+ (ndarray, ndarray)+ Returns the measurement-corrected state distribution.+ """+ projected_mean, projected_cov = self.project(mean, covariance)++ chol_factor, lower = scipy.linalg.cho_factor(+ projected_cov, lower=True, check_finite=False)+ kalman_gain = scipy.linalg.cho_solve(+ (chol_factor, lower), np.dot(covariance, self._update_mat.T).T,+ check_finite=False).T+ innovation = measurement - projected_mean++ new_mean = mean + np.dot(innovation, kalman_gain.T)+ new_covariance = covariance - np.linalg.multi_dot((+ kalman_gain, projected_cov, kalman_gain.T))+ return new_mean, new_covariance++ def gating_distance(self, mean, covariance, measurements,+ only_position=False):+ """Compute gating distance between state distribution and measurements.+ A suitable distance threshold can be obtained from `chi2inv95`. If+ `only_position` is False, the chi-square distribution has 4 degrees of+ freedom, otherwise 2.+ Parameters+ ----------+ mean : ndarray+ Mean vector over the state distribution (8 dimensional).+ covariance : ndarray+ Covariance of the state distribution (8x8 dimensional).+ measurements : ndarray+ An Nx4 dimensional matrix of N measurements, each in+ format (x, y, a, h) where (x, y) is the bounding box center+ position, a the aspect ratio, and h the height.+ only_position : Optional[bool]+ If True, distance computation is done with respect to the bounding+ box center position only.+ Returns+ -------+ ndarray+ Returns an array of length N, where the i-th element contains the+ squared Mahalanobis distance between (mean, covariance) and+ `measurements[i]`.+ """+ mean, covariance = self.project(mean, covariance)+ if only_position:+ mean, covariance = mean[:2], covariance[:2, :2]+ measurements = measurements[:, :2]++ cholesky_factor = np.linalg.cholesky(covariance)+ d = measurements - mean+ z = scipy.linalg.solve_triangular(+ cholesky_factor, d.T, lower=True, check_finite=False,+ overwrite_b=True)+ squared_maha = np.sum(z * z, axis=0)+ return squared_maha\ No newline at end of filediff --git a/experimental/yolox/deepsort_tracker/linear_assignment.py b/experimental/yolox/deepsort_tracker/linear_assignment.pynew file mode 100644index 0000000..5651893--- /dev/null+++ b/experimental/yolox/deepsort_tracker/linear_assignment.py@@ -0,0 +1,182 @@+from __future__ import absolute_import+import numpy as np+# from sklearn.utils.linear_assignment_ import linear_assignment+from scipy.optimize import linear_sum_assignment as linear_assignment+from yolox.deepsort_tracker import kalman_filter+++INFTY_COST = 1e+5+++def min_cost_matching(+ distance_metric, max_distance, tracks, detections, track_indices=None,+ detection_indices=None):+ """Solve linear assignment problem.+ Parameters+ ----------+ distance_metric : Callable[List[Track], List[Detection], List[int], List[int]) -> ndarray+ The distance metric is given a list of tracks and detections as well as+ a list of N track indices and M detection indices. The metric should+ return the NxM dimensional cost matrix, where element (i, j) is the+ association cost between the i-th track in the given track indices and+ the j-th detection in the given detection_indices.+ max_distance : float+ Gating threshold. Associations with cost larger than this value are+ disregarded.+ tracks : List[track.Track]+ A list of predicted tracks at the current time step.+ detections : List[detection.Detection]+ A list of detections at the current time step.+ track_indices : List[int]+ List of track indices that maps rows in `cost_matrix` to tracks in+ `tracks` (see description above).+ detection_indices : List[int]+ List of detection indices that maps columns in `cost_matrix` to+ detections in `detections` (see description above).+ Returns+ -------+ (List[(int, int)], List[int], List[int])+ Returns a tuple with the following three entries:+ * A list of matched track and detection indices.+ * A list of unmatched track indices.+ * A list of unmatched detection indices.+ """+ if track_indices is None:+ track_indices = np.arange(len(tracks))+ if detection_indices is None:+ detection_indices = np.arange(len(detections))++ if len(detection_indices) == 0 or len(track_indices) == 0:+ return [], track_indices, detection_indices # Nothing to match.++ cost_matrix = distance_metric(+ tracks, detections, track_indices, detection_indices)+ cost_matrix[cost_matrix > max_distance] = max_distance + 1e-5++ row_indices, col_indices = linear_assignment(cost_matrix)++ matches, unmatched_tracks, unmatched_detections = [], [], []+ for col, detection_idx in enumerate(detection_indices):+ if col not in col_indices:+ unmatched_detections.append(detection_idx)+ for row, track_idx in enumerate(track_indices):+ if row not in row_indices:+ unmatched_tracks.append(track_idx)+ for row, col in zip(row_indices, col_indices):+ track_idx = track_indices[row]+ detection_idx = detection_indices[col]+ if cost_matrix[row, col] > max_distance:+ unmatched_tracks.append(track_idx)+ unmatched_detections.append(detection_idx)+ else:+ matches.append((track_idx, detection_idx))+ return matches, unmatched_tracks, unmatched_detections+++def matching_cascade(+ distance_metric, max_distance, cascade_depth, tracks, detections,+ track_indices=None, detection_indices=None):+ """Run matching cascade.+ Parameters+ ----------+ distance_metric : Callable[List[Track], List[Detection], List[int], List[int]) -> ndarray+ The distance metric is given a list of tracks and detections as well as+ a list of N track indices and M detection indices. The metric should+ return the NxM dimensional cost matrix, where element (i, j) is the+ association cost between the i-th track in the given track indices and+ the j-th detection in the given detection indices.+ max_distance : float+ Gating threshold. Associations with cost larger than this value are+ disregarded.+ cascade_depth: int+ The cascade depth, should be se to the maximum track age.+ tracks : List[track.Track]+ A list of predicted tracks at the current time step.+ detections : List[detection.Detection]+ A list of detections at the current time step.+ track_indices : Optional[List[int]]+ List of track indices that maps rows in `cost_matrix` to tracks in+ `tracks` (see description above). Defaults to all tracks.+ detection_indices : Optional[List[int]]+ List of detection indices that maps columns in `cost_matrix` to+ detections in `detections` (see description above). Defaults to all+ detections.+ Returns+ -------+ (List[(int, int)], List[int], List[int])+ Returns a tuple with the following three entries:+ * A list of matched track and detection indices.+ * A list of unmatched track indices.+ * A list of unmatched detection indices.+ """+ if track_indices is None:+ track_indices = list(range(len(tracks)))+ if detection_indices is None:+ detection_indices = list(range(len(detections)))++ unmatched_detections = detection_indices+ matches = []+ for level in range(cascade_depth):+ if len(unmatched_detections) == 0: # No detections left+ break++ track_indices_l = [+ k for k in track_indices+ if tracks[k].time_since_update == 1 + level+ ]+ if len(track_indices_l) == 0: # Nothing to match at this level+ continue++ matches_l, _, unmatched_detections = \+ min_cost_matching(+ distance_metric, max_distance, tracks, detections,+ track_indices_l, unmatched_detections)+ matches += matches_l+ unmatched_tracks = list(set(track_indices) - set(k for k, _ in matches))+ return matches, unmatched_tracks, unmatched_detections+++def gate_cost_matrix(+ kf, cost_matrix, tracks, detections, track_indices, detection_indices,+ gated_cost=INFTY_COST, only_position=False):+ """Invalidate infeasible entries in cost matrix based on the state+ distributions obtained by Kalman filtering.+ Parameters+ ----------+ kf : The Kalman filter.+ cost_matrix : ndarray+ The NxM dimensional cost matrix, where N is the number of track indices+ and M is the number of detection indices, such that entry (i, j) is the+ association cost between `tracks[track_indices[i]]` and+ `detections[detection_indices[j]]`.+ tracks : List[track.Track]+ A list of predicted tracks at the current time step.+ detections : List[detection.Detection]+ A list of detections at the current time step.+ track_indices : List[int]+ List of track indices that maps rows in `cost_matrix` to tracks in+ `tracks` (see description above).+ detection_indices : List[int]+ List of detection indices that maps columns in `cost_matrix` to+ detections in `detections` (see description above).+ gated_cost : Optional[float]+ Entries in the cost matrix corresponding to infeasible associations are+ set this value. Defaults to a very large value.+ only_position : Optional[bool]+ If True, only the x, y position of the state distribution is considered+ during gating. Defaults to False.+ Returns+ -------+ ndarray+ Returns the modified cost matrix.+ """+ gating_dim = 2 if only_position else 4+ gating_threshold = kalman_filter.chi2inv95[gating_dim]+ measurements = np.asarray(+ [detections[i].to_xyah() for i in detection_indices])+ for row, track_idx in enumerate(track_indices):+ track = tracks[track_idx]+ gating_distance = kf.gating_distance(+ track.mean, track.covariance, measurements, only_position)+ cost_matrix[row, gating_distance > gating_threshold] = gated_cost+ return cost_matrix\ No newline at end of filediff --git a/experimental/yolox/deepsort_tracker/reid_model.py b/experimental/yolox/deepsort_tracker/reid_model.pynew file mode 100644index 0000000..6aff885--- /dev/null+++ b/experimental/yolox/deepsort_tracker/reid_model.py@@ -0,0 +1,145 @@+import torch+import torch.nn as nn+import torch.nn.functional as F+import numpy as np+import cv2+import logging+import torchvision.transforms as transforms+++class BasicBlock(nn.Module):+ def __init__(self, c_in, c_out, is_downsample=False):+ super(BasicBlock, self).__init__()+ self.is_downsample = is_downsample+ if is_downsample:+ self.conv1 = nn.Conv2d(+ c_in, c_out, 3, stride=2, padding=1, bias=False)+ else:+ self.conv1 = nn.Conv2d(+ c_in, c_out, 3, stride=1, padding=1, bias=False)+ self.bn1 = nn.BatchNorm2d(c_out)+ self.relu = nn.ReLU(True)+ self.conv2 = nn.Conv2d(c_out, c_out, 3, stride=1,+ padding=1, bias=False)+ self.bn2 = nn.BatchNorm2d(c_out)+ if is_downsample:+ self.downsample = nn.Sequential(+ nn.Conv2d(c_in, c_out, 1, stride=2, bias=False),+ nn.BatchNorm2d(c_out)+ )+ elif c_in != c_out:+ self.downsample = nn.Sequential(+ nn.Conv2d(c_in, c_out, 1, stride=1, bias=False),+ nn.BatchNorm2d(c_out)+ )+ self.is_downsample = True++ def forward(self, x):+ y = self.conv1(x)+ y = self.bn1(y)+ y = self.relu(y)+ y = self.conv2(y)+ y = self.bn2(y)+ if self.is_downsample:+ x = self.downsample(x)+ return F.relu(x.add(y), True)+++def make_layers(c_in, c_out, repeat_times, is_downsample=False):+ blocks = []+ for i in range(repeat_times):+ if i == 0:+ blocks += [BasicBlock(c_in, c_out, is_downsample=is_downsample), ]+ else:+ blocks += [BasicBlock(c_out, c_out), ]+ return nn.Sequential(*blocks)+++class Net(nn.Module):+ def __init__(self, num_classes=751, reid=False):+ super(Net, self).__init__()+ # 3 128 64+ self.conv = nn.Sequential(+ nn.Conv2d(3, 64, 3, stride=1, padding=1),+ nn.BatchNorm2d(64),+ nn.ReLU(inplace=True),+ # nn.Conv2d(32,32,3,stride=1,padding=1),+ # nn.BatchNorm2d(32),+ # nn.ReLU(inplace=True),+ nn.MaxPool2d(3, 2, padding=1),+ )+ # 32 64 32+ self.layer1 = make_layers(64, 64, 2, False)+ # 32 64 32+ self.layer2 = make_layers(64, 128, 2, True)+ # 64 32 16+ self.layer3 = make_layers(128, 256, 2, True)+ # 128 16 8+ self.layer4 = make_layers(256, 512, 2, True)+ # 256 8 4+ self.avgpool = nn.AvgPool2d((8, 4), 1)+ # 256 1 1+ self.reid = reid+ self.classifier = nn.Sequential(+ nn.Linear(512, 256),+ nn.BatchNorm1d(256),+ nn.ReLU(inplace=True),+ nn.Dropout(),+ nn.Linear(256, num_classes),+ )++ def forward(self, x):+ x = self.conv(x)+ x = self.layer1(x)+ x = self.layer2(x)+ x = self.layer3(x)+ x = self.layer4(x)+ x = self.avgpool(x)+ x = x.view(x.size(0), -1)+ # B x 128+ if self.reid:+ x = x.div(x.norm(p=2, dim=1, keepdim=True))+ return x+ # classifier+ x = self.classifier(x)+ return x+++class Extractor(object):+ def __init__(self, model_path, use_cuda=True):+ self.net = Net(reid=True)+ self.device = "cuda" if torch.cuda.is_available() and use_cuda else "cpu"+ state_dict = torch.load(model_path, map_location=torch.device(self.device))[+ 'net_dict']+ self.net.load_state_dict(state_dict)+ logger = logging.getLogger("root.tracker")+ logger.info("Loading weights from {}... Done!".format(model_path))+ self.net.to(self.device)+ self.size = (64, 128)+ self.norm = transforms.Compose([+ transforms.ToTensor(),+ transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),+ ])++ def _preprocess(self, im_crops):+ """+ TODO:+ 1. to float with scale from 0 to 1+ 2. resize to (64, 128) as Market1501 dataset did+ 3. concatenate to a numpy array+ 3. to torch Tensor+ 4. normalize+ """+ def _resize(im, size):+ return cv2.resize(im.astype(np.float32)/255., size)++ im_batch = torch.cat([self.norm(_resize(im, self.size)).unsqueeze(+ 0) for im in im_crops], dim=0).float()+ return im_batch++ def __call__(self, im_crops):+ im_batch = self._preprocess(im_crops)+ with torch.no_grad():+ im_batch = im_batch.to(self.device)+ features = self.net(im_batch)+ return features.cpu().numpy()\ No newline at end of filediff --git a/experimental/yolox/deepsort_tracker/track.py b/experimental/yolox/deepsort_tracker/track.pynew file mode 100644index 0000000..6867441--- /dev/null+++ b/experimental/yolox/deepsort_tracker/track.py@@ -0,0 +1,158 @@+# vim: expandtab:ts=4:sw=4+++class TrackState:+ """+ Enumeration type for the single target track state. Newly created tracks are+ classified as `tentative` until enough evidence has been collected. Then,+ the track state is changed to `confirmed`. Tracks that are no longer alive+ are classified as `deleted` to mark them for removal from the set of active+ tracks.+ """++ Tentative = 1+ Confirmed = 2+ Deleted = 3+++class Track:+ """+ A single target track with state space `(x, y, a, h)` and associated+ velocities, where `(x, y)` is the center of the bounding box, `a` is the+ aspect ratio and `h` is the height.+ Parameters+ ----------+ mean : ndarray+ Mean vector of the initial state distribution.+ covariance : ndarray+ Covariance matrix of the initial state distribution.+ track_id : int+ A unique track identifier.+ n_init : int+ Number of consecutive detections before the track is confirmed. The+ track state is set to `Deleted` if a miss occurs within the first+ `n_init` frames.+ max_age : int+ The maximum number of consecutive misses before the track state is+ set to `Deleted`.+ feature : Optional[ndarray]+ Feature vector of the detection this track originates from. If not None,+ this feature is added to the `features` cache.+ Attributes+ ----------+ mean : ndarray+ Mean vector of the initial state distribution.+ covariance : ndarray+ Covariance matrix of the initial state distribution.+ track_id : int+ A unique track identifier.+ hits : int+ Total number of measurement updates.+ age : int+ Total number of frames since first occurance.+ time_since_update : int+ Total number of frames since last measurement update.+ state : TrackState+ The current track state.+ features : List[ndarray]+ A cache of features. On each measurement update, the associated feature+ vector is added to this list.+ """++ def __init__(self, mean, covariance, track_id, class_id, n_init, max_age,+ feature=None):+ self.mean = mean+ self.covariance = covariance+ self.track_id = track_id+ self.class_id = class_id+ self.hits = 1+ self.age = 1+ self.time_since_update = 0++ self.state = TrackState.Tentative+ self.features = []+ if feature is not None:+ self.features.append(feature)++ self._n_init = n_init+ self._max_age = max_age++ def to_tlwh(self):+ """Get current position in bounding box format `(top left x, top left y,+ width, height)`.+ Returns+ -------+ ndarray+ The bounding box.+ """+ ret = self.mean[:4].copy()+ ret[2] *= ret[3]+ ret[:2] -= ret[2:] / 2+ return ret++ def to_tlbr(self):+ """Get current position in bounding box format `(min x, miny, max x,+ max y)`.+ Returns+ -------+ ndarray+ The bounding box.+ """+ ret = self.to_tlwh()+ ret[2:] = ret[:2] + ret[2:]+ return ret++ def increment_age(self):+ self.age += 1+ self.time_since_update += 1++ def predict(self, kf):+ """Propagate the state distribution to the current time step using a+ Kalman filter prediction step.+ Parameters+ ----------+ kf : kalman_filter.KalmanFilter+ The Kalman filter.+ """+ self.mean, self.covariance = kf.predict(self.mean, self.covariance)+ self.increment_age()++ def update(self, kf, detection):+ """Perform Kalman filter measurement update step and update the feature+ cache.+ Parameters+ ----------+ kf : kalman_filter.KalmanFilter+ The Kalman filter.+ detection : Detection+ The associated detection.+ """+ self.mean, self.covariance = kf.update(+ self.mean, self.covariance, detection.to_xyah())+ self.features.append(detection.feature)++ self.hits += 1+ self.time_since_update = 0+ if self.state == TrackState.Tentative and self.hits >= self._n_init:+ self.state = TrackState.Confirmed++ def mark_missed(self):+ """Mark this track as missed (no association at the current time step).+ """+ if self.state == TrackState.Tentative:+ self.state = TrackState.Deleted+ elif self.time_since_update > self._max_age:+ self.state = TrackState.Deleted++ def is_tentative(self):+ """Returns True if this track is tentative (unconfirmed).+ """+ return self.state == TrackState.Tentative++ def is_confirmed(self):+ """Returns True if this track is confirmed."""+ return self.state == TrackState.Confirmed++ def is_deleted(self):+ """Returns True if this track is dead and should be deleted."""+ return self.state == TrackState.Deleted\ No newline at end of filediff --git a/experimental/yolox/evaluators/__init__.py b/experimental/yolox/evaluators/__init__.pynew file mode 100644index 0000000..5d704e0--- /dev/null+++ b/experimental/yolox/evaluators/__init__.py@@ -0,0 +1,6 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) Megvii, Inc. and its affiliates.++from .coco_evaluator import COCOEvaluator+from .mot_evaluator import MOTEvaluatordiff --git a/experimental/yolox/evaluators/coco_evaluator.py b/experimental/yolox/evaluators/coco_evaluator.pynew file mode 100644index 0000000..24dce23--- /dev/null+++ b/experimental/yolox/evaluators/coco_evaluator.py@@ -0,0 +1,224 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) Megvii, Inc. and its affiliates.++from loguru import logger+from tqdm import tqdm++import torch++from yolox.utils import (+ gather,+ is_main_process,+ postprocess,+ synchronize,+ time_synchronized,+ xyxy2xywh+)++import contextlib+import io+import itertools+import json+import tempfile+import time+++class COCOEvaluator:+ """+ COCO AP Evaluation class. All the data in the val2017 dataset are processed+ and evaluated by COCO API.+ """++ def __init__(+ self, dataloader, img_size, confthre, nmsthre, num_classes, testdev=False+ ):+ """+ Args:+ dataloader (Dataloader): evaluate dataloader.+ img_size (int): image size after preprocess. images are resized+ to squares whose shape is (img_size, img_size).+ confthre (float): confidence threshold ranging from 0 to 1, which+ is defined in the config file.+ nmsthre (float): IoU threshold of non-max supression ranging from 0 to 1.+ """+ self.dataloader = dataloader+ self.img_size = img_size+ self.confthre = confthre+ self.nmsthre = nmsthre+ self.num_classes = num_classes+ self.testdev = testdev++ def evaluate(+ self,+ model,+ distributed=False,+ half=False,+ trt_file=None,+ decoder=None,+ test_size=None,+ ):+ """+ COCO average precision (AP) Evaluation. Iterate inference on the test dataset+ and the results are evaluated by COCO API.++ NOTE: This function will change training mode to False, please save states if needed.++ Args:+ model : model to evaluate.++ Returns:+ ap50_95 (float) : COCO AP of IoU=50:95+ ap50 (float) : COCO AP of IoU=50+ summary (sr): summary info of evaluation.+ """+ # TODO half to amp_test+ tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor+ model = model.eval()+ if half:+ model = model.half()+ ids = []+ data_list = []+ progress_bar = tqdm if is_main_process() else iter++ inference_time = 0+ nms_time = 0+ n_samples = len(self.dataloader) - 1++ if trt_file is not None:+ from torch2trt import TRTModule++ model_trt = TRTModule()+ model_trt.load_state_dict(torch.load(trt_file))++ x = torch.ones(1, 3, test_size[0], test_size[1]).cuda()+ model(x)+ model = model_trt++ for cur_iter, (imgs, _, info_imgs, ids) in enumerate(+ progress_bar(self.dataloader)+ ):+ with torch.no_grad():+ imgs = imgs.type(tensor_type)++ # skip the the last iters since batchsize might be not enough for batch inference+ is_time_record = cur_iter < len(self.dataloader) - 1+ if is_time_record:+ start = time.time()++ outputs = model(imgs)+ if decoder is not None:+ outputs = decoder(outputs, dtype=outputs.type())++ if is_time_record:+ infer_end = time_synchronized()+ inference_time += infer_end - start++ outputs = postprocess(+ outputs, self.num_classes, self.confthre, self.nmsthre+ )+ if is_time_record:+ nms_end = time_synchronized()+ nms_time += nms_end - infer_end++ data_list.extend(self.convert_to_coco_format(outputs, info_imgs, ids))++ statistics = torch.cuda.FloatTensor([inference_time, nms_time, n_samples])+ if distributed:+ data_list = gather(data_list, dst=0)+ data_list = list(itertools.chain(*data_list))+ torch.distributed.reduce(statistics, dst=0)++ eval_results = self.evaluate_prediction(data_list, statistics)+ synchronize()+ return eval_results++ def convert_to_coco_format(self, outputs, info_imgs, ids):+ data_list = []+ for (output, img_h, img_w, img_id) in zip(+ outputs, info_imgs[0], info_imgs[1], ids+ ):+ if output is None:+ continue+ output = output.cpu()++ bboxes = output[:, 0:4]++ # preprocessing: resize+ scale = min(+ self.img_size[0] / float(img_h), self.img_size[1] / float(img_w)+ )+ bboxes /= scale+ bboxes = xyxy2xywh(bboxes)++ cls = output[:, 6]+ scores = output[:, 4] * output[:, 5]+ for ind in range(bboxes.shape[0]):+ label = self.dataloader.dataset.class_ids[int(cls[ind])]+ pred_data = {+ "image_id": int(img_id),+ "category_id": label,+ "bbox": bboxes[ind].numpy().tolist(),+ "score": scores[ind].numpy().item(),+ "segmentation": [],+ } # COCO json format+ data_list.append(pred_data)+ return data_list++ def evaluate_prediction(self, data_dict, statistics):+ if not is_main_process():+ return 0, 0, None++ logger.info("Evaluate in main process...")++ annType = ["segm", "bbox", "keypoints"]++ inference_time = statistics[0].item()+ nms_time = statistics[1].item()+ n_samples = statistics[2].item()++ a_infer_time = 1000 * inference_time / (n_samples * self.dataloader.batch_size)+ a_nms_time = 1000 * nms_time / (n_samples * self.dataloader.batch_size)++ time_info = ", ".join(+ [+ "Average {} time: {:.2f} ms".format(k, v)+ for k, v in zip(+ ["forward", "NMS", "inference"],+ [a_infer_time, a_nms_time, (a_infer_time + a_nms_time)],+ )+ ]+ )++ info = time_info + "\n"++ # Evaluate the Dt (detection) json comparing with the ground truth+ if len(data_dict) > 0:+ cocoGt = self.dataloader.dataset.coco+ # TODO: since pycocotools can't process dict in py36, write data to json file.+ if self.testdev:+ json.dump(data_dict, open("./yolox_testdev_2017.json", "w"))+ cocoDt = cocoGt.loadRes("./yolox_testdev_2017.json")+ else:+ _, tmp = tempfile.mkstemp()+ json.dump(data_dict, open(tmp, "w"))+ cocoDt = cocoGt.loadRes(tmp)+ '''+ try:+ from yolox.layers import COCOeval_opt as COCOeval+ except ImportError:+ from pycocotools import cocoeval as COCOeval+ logger.warning("Use standard COCOeval.")+ '''+ #from pycocotools.cocoeval import COCOeval+ from yolox.layers import COCOeval_opt as COCOeval+ cocoEval = COCOeval(cocoGt, cocoDt, annType[1])+ cocoEval.evaluate()+ cocoEval.accumulate()+ redirect_string = io.StringIO()+ with contextlib.redirect_stdout(redirect_string):+ cocoEval.summarize()+ info += redirect_string.getvalue()+ return cocoEval.stats[0], cocoEval.stats[1], info+ else:+ return 0, 0, infodiff --git a/experimental/yolox/evaluators/evaluation.py b/experimental/yolox/evaluators/evaluation.pynew file mode 100644index 0000000..fd72f82--- /dev/null+++ b/experimental/yolox/evaluators/evaluation.py@@ -0,0 +1,200 @@+import os+import numpy as np+import copy+import motmetrics as mm+mm.lap.default_solver = 'lap'+++class Evaluator(object):++ def __init__(self, data_root, seq_name, data_type):+ self.data_root = data_root+ self.seq_name = seq_name+ self.data_type = data_type++ self.load_annotations()+ self.reset_accumulator()++ def load_annotations(self):+ assert self.data_type == 'mot'++ gt_filename = os.path.join(self.data_root, self.seq_name, 'gt', 'gt.txt')+ self.gt_frame_dict = read_results(gt_filename, self.data_type, is_gt=True)+ self.gt_ignore_frame_dict = read_results(gt_filename, self.data_type, is_ignore=True)++ def reset_accumulator(self):+ self.acc = mm.MOTAccumulator(auto_id=True)++ def eval_frame(self, frame_id, trk_tlwhs, trk_ids, rtn_events=False):+ # results+ trk_tlwhs = np.copy(trk_tlwhs)+ trk_ids = np.copy(trk_ids)++ # gts+ gt_objs = self.gt_frame_dict.get(frame_id, [])+ gt_tlwhs, gt_ids = unzip_objs(gt_objs)[:2]++ # ignore boxes+ ignore_objs = self.gt_ignore_frame_dict.get(frame_id, [])+ ignore_tlwhs = unzip_objs(ignore_objs)[0]++ # remove ignored results+ keep = np.ones(len(trk_tlwhs), dtype=bool)+ iou_distance = mm.distances.iou_matrix(ignore_tlwhs, trk_tlwhs, max_iou=0.5)+ if len(iou_distance) > 0:+ match_is, match_js = mm.lap.linear_sum_assignment(iou_distance)+ match_is, match_js = map(lambda a: np.asarray(a, dtype=int), [match_is, match_js])+ match_ious = iou_distance[match_is, match_js]++ match_js = np.asarray(match_js, dtype=int)+ match_js = match_js[np.logical_not(np.isnan(match_ious))]+ keep[match_js] = False+ trk_tlwhs = trk_tlwhs[keep]+ trk_ids = trk_ids[keep]+ #match_is, match_js = mm.lap.linear_sum_assignment(iou_distance)+ #match_is, match_js = map(lambda a: np.asarray(a, dtype=int), [match_is, match_js])+ #match_ious = iou_distance[match_is, match_js]++ #match_js = np.asarray(match_js, dtype=int)+ #match_js = match_js[np.logical_not(np.isnan(match_ious))]+ #keep[match_js] = False+ #trk_tlwhs = trk_tlwhs[keep]+ #trk_ids = trk_ids[keep]++ # get distance matrix+ iou_distance = mm.distances.iou_matrix(gt_tlwhs, trk_tlwhs, max_iou=0.5)++ # acc+ self.acc.update(gt_ids, trk_ids, iou_distance)++ if rtn_events and iou_distance.size > 0 and hasattr(self.acc, 'last_mot_events'):+ events = self.acc.last_mot_events # only supported by https://github.com/longcw/py-motmetrics+ else:+ events = None+ return events++ def eval_file(self, filename):+ self.reset_accumulator()++ result_frame_dict = read_results(filename, self.data_type, is_gt=False)+ #frames = sorted(list(set(self.gt_frame_dict.keys()) | set(result_frame_dict.keys())))+ frames = sorted(list(set(result_frame_dict.keys())))+ for frame_id in frames:+ trk_objs = result_frame_dict.get(frame_id, [])+ trk_tlwhs, trk_ids = unzip_objs(trk_objs)[:2]+ self.eval_frame(frame_id, trk_tlwhs, trk_ids, rtn_events=False)++ return self.acc++ @staticmethod+ def get_summary(accs, names, metrics=('mota', 'num_switches', 'idp', 'idr', 'idf1', 'precision', 'recall')):+ names = copy.deepcopy(names)+ if metrics is None:+ metrics = mm.metrics.motchallenge_metrics+ metrics = copy.deepcopy(metrics)++ mh = mm.metrics.create()+ summary = mh.compute_many(+ accs,+ metrics=metrics,+ names=names,+ generate_overall=True+ )++ return summary++ @staticmethod+ def save_summary(summary, filename):+ import pandas as pd+ writer = pd.ExcelWriter(filename)+ summary.to_excel(writer)+ writer.save()++++++def read_results(filename, data_type: str, is_gt=False, is_ignore=False):+ if data_type in ('mot', 'lab'):+ read_fun = read_mot_results+ else:+ raise ValueError('Unknown data type: {}'.format(data_type))++ return read_fun(filename, is_gt, is_ignore)+++"""+labels={'ped', ... % 1+'person_on_vhcl', ... % 2+'car', ... % 3+'bicycle', ... % 4+'mbike', ... % 5+'non_mot_vhcl', ... % 6+'static_person', ... % 7+'distractor', ... % 8+'occluder', ... % 9+'occluder_on_grnd', ... %10+'occluder_full', ... % 11+'reflection', ... % 12+'crowd' ... % 13+};+"""+++def read_mot_results(filename, is_gt, is_ignore):+ valid_labels = {1}+ ignore_labels = {2, 7, 8, 12}+ results_dict = dict()+ if os.path.isfile(filename):+ with open(filename, 'r') as f:+ for line in f.readlines():+ linelist = line.split(',')+ if len(linelist) < 7:+ continue+ fid = int(linelist[0])+ if fid < 1:+ continue+ results_dict.setdefault(fid, list())++ box_size = float(linelist[4]) * float(linelist[5])++ if is_gt:+ if 'MOT16-' in filename or 'MOT17-' in filename:+ label = int(float(linelist[7]))+ mark = int(float(linelist[6]))+ if mark == 0 or label not in valid_labels:+ continue+ score = 1+ elif is_ignore:+ if 'MOT16-' in filename or 'MOT17-' in filename:+ label = int(float(linelist[7]))+ vis_ratio = float(linelist[8])+ if label not in ignore_labels and vis_ratio >= 0:+ continue+ else:+ continue+ score = 1+ else:+ score = float(linelist[6])++ #if box_size > 7000:+ #if box_size <= 7000 or box_size >= 15000:+ #if box_size < 15000:+ #continue++ tlwh = tuple(map(float, linelist[2:6]))+ target_id = int(linelist[1])++ results_dict[fid].append((tlwh, target_id, score))++ return results_dict+++def unzip_objs(objs):+ if len(objs) > 0:+ tlwhs, ids, scores = zip(*objs)+ else:+ tlwhs, ids, scores = [], [], []+ tlwhs = np.asarray(tlwhs, dtype=float).reshape(-1, 4)++ return tlwhs, ids, scores\ No newline at end of filediff --git a/experimental/yolox/evaluators/mot_evaluator.py b/experimental/yolox/evaluators/mot_evaluator.pynew file mode 100644index 0000000..f8c2d83--- /dev/null+++ b/experimental/yolox/evaluators/mot_evaluator.py@@ -0,0 +1,676 @@+from collections import defaultdict+from loguru import logger+from tqdm import tqdm++import torch++from yolox.utils import (+ gather,+ is_main_process,+ postprocess,+ synchronize,+ time_synchronized,+ xyxy2xywh+)+from yolox.tracker.byte_tracker import BYTETracker+from yolox.sort_tracker.sort import Sort+from yolox.deepsort_tracker.deepsort import DeepSort+from yolox.motdt_tracker.motdt_tracker import OnlineTracker++import contextlib+import io+import os+import itertools+import json+import tempfile+import time+++def write_results(filename, results):+ save_format = '{frame},{id},{x1},{y1},{w},{h},{s},-1,-1,-1\n'+ with open(filename, 'w') as f:+ for frame_id, tlwhs, track_ids, scores in results:+ for tlwh, track_id, score in zip(tlwhs, track_ids, scores):+ if track_id < 0:+ continue+ x1, y1, w, h = tlwh+ line = save_format.format(frame=frame_id, id=track_id, x1=round(x1, 1), y1=round(y1, 1), w=round(w, 1), h=round(h, 1), s=round(score, 2))+ f.write(line)+ logger.info('save results to {}'.format(filename))+++def write_results_no_score(filename, results):+ save_format = '{frame},{id},{x1},{y1},{w},{h},-1,-1,-1,-1\n'+ with open(filename, 'w') as f:+ for frame_id, tlwhs, track_ids in results:+ for tlwh, track_id in zip(tlwhs, track_ids):+ if track_id < 0:+ continue+ x1, y1, w, h = tlwh+ line = save_format.format(frame=frame_id, id=track_id, x1=round(x1, 1), y1=round(y1, 1), w=round(w, 1), h=round(h, 1))+ f.write(line)+ logger.info('save results to {}'.format(filename))+++class MOTEvaluator:+ """+ COCO AP Evaluation class. All the data in the val2017 dataset are processed+ and evaluated by COCO API.+ """++ def __init__(+ self, args, dataloader, img_size, confthre, nmsthre, num_classes):+ """+ Args:+ dataloader (Dataloader): evaluate dataloader.+ img_size (int): image size after preprocess. images are resized+ to squares whose shape is (img_size, img_size).+ confthre (float): confidence threshold ranging from 0 to 1, which+ is defined in the config file.+ nmsthre (float): IoU threshold of non-max supression ranging from 0 to 1.+ """+ self.dataloader = dataloader+ self.img_size = img_size+ self.confthre = confthre+ self.nmsthre = nmsthre+ self.num_classes = num_classes+ self.args = args++ def evaluate(+ self,+ model,+ distributed=False,+ half=False,+ trt_file=None,+ decoder=None,+ test_size=None,+ result_folder=None+ ):+ """+ COCO average precision (AP) Evaluation. Iterate inference on the test dataset+ and the results are evaluated by COCO API.++ NOTE: This function will change training mode to False, please save states if needed.++ Args:+ model : model to evaluate.++ Returns:+ ap50_95 (float) : COCO AP of IoU=50:95+ ap50 (float) : COCO AP of IoU=50+ summary (sr): summary info of evaluation.+ """+ # TODO half to amp_test+ tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor+ model = model.eval()+ if half:+ model = model.half()+ ids = []+ data_list = []+ results = []+ video_names = defaultdict()+ progress_bar = tqdm if is_main_process() else iter++ inference_time = 0+ track_time = 0+ n_samples = len(self.dataloader) - 1++ if trt_file is not None:+ from torch2trt import TRTModule++ model_trt = TRTModule()+ model_trt.load_state_dict(torch.load(trt_file))++ x = torch.ones(1, 3, test_size[0], test_size[1]).cuda()+ model(x)+ model = model_trt++ tracker = BYTETracker(self.args)+ ori_thresh = self.args.track_thresh+ for cur_iter, (imgs, _, info_imgs, ids) in enumerate(+ progress_bar(self.dataloader)+ ):+ with torch.no_grad():+ # init tracker+ frame_id = info_imgs[2].item()+ video_id = info_imgs[3].item()+ img_file_name = info_imgs[4]+ video_name = img_file_name[0].split('/')[0]+ if video_name == 'MOT17-05-FRCNN' or video_name == 'MOT17-06-FRCNN':+ self.args.track_buffer = 14+ elif video_name == 'MOT17-13-FRCNN' or video_name == 'MOT17-14-FRCNN':+ self.args.track_buffer = 25+ else:+ self.args.track_buffer = 30++ if video_name == 'MOT17-01-FRCNN':+ self.args.track_thresh = 0.65+ elif video_name == 'MOT17-06-FRCNN':+ self.args.track_thresh = 0.65+ elif video_name == 'MOT17-12-FRCNN':+ self.args.track_thresh = 0.7+ elif video_name == 'MOT17-14-FRCNN':+ self.args.track_thresh = 0.67+ elif video_name in ['MOT20-06', 'MOT20-08']:+ self.args.track_thresh = 0.3+ else:+ self.args.track_thresh = ori_thresh++ if video_name not in video_names:+ video_names[video_id] = video_name+ if frame_id == 1:+ tracker = BYTETracker(self.args)+ if len(results) != 0:+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id - 1]))+ write_results(result_filename, results)+ results = []++ imgs = imgs.type(tensor_type)++ # skip the the last iters since batchsize might be not enough for batch inference+ is_time_record = cur_iter < len(self.dataloader) - 1+ if is_time_record:+ start = time.time()++ outputs = model(imgs)+ if decoder is not None:+ outputs = decoder(outputs, dtype=outputs.type())++ outputs = postprocess(outputs, self.num_classes, self.confthre, self.nmsthre)++ if is_time_record:+ infer_end = time_synchronized()+ inference_time += infer_end - start++ output_results = self.convert_to_coco_format(outputs, info_imgs, ids)+ data_list.extend(output_results)++ # run tracking+ if outputs[0] is not None:+ online_targets = tracker.update(outputs[0], info_imgs, self.img_size)+ online_tlwhs = []+ online_ids = []+ online_scores = []+ for t in online_targets:+ tlwh = t.tlwh+ tid = t.track_id+ vertical = tlwh[2] / tlwh[3] > 1.6+ if tlwh[2] * tlwh[3] > self.args.min_box_area and not vertical:+ online_tlwhs.append(tlwh)+ online_ids.append(tid)+ online_scores.append(t.score)+ # save results+ results.append((frame_id, online_tlwhs, online_ids, online_scores))++ if is_time_record:+ track_end = time_synchronized()+ track_time += track_end - infer_end++ if cur_iter == len(self.dataloader) - 1:+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id]))+ write_results(result_filename, results)++ statistics = torch.cuda.FloatTensor([inference_time, track_time, n_samples])+ if distributed:+ data_list = gather(data_list, dst=0)+ data_list = list(itertools.chain(*data_list))+ torch.distributed.reduce(statistics, dst=0)++ eval_results = self.evaluate_prediction(data_list, statistics)+ synchronize()+ return eval_results++ def evaluate_sort(+ self,+ model,+ distributed=False,+ half=False,+ trt_file=None,+ decoder=None,+ test_size=None,+ result_folder=None+ ):+ """+ COCO average precision (AP) Evaluation. Iterate inference on the test dataset+ and the results are evaluated by COCO API.++ NOTE: This function will change training mode to False, please save states if needed.++ Args:+ model : model to evaluate.++ Returns:+ ap50_95 (float) : COCO AP of IoU=50:95+ ap50 (float) : COCO AP of IoU=50+ summary (sr): summary info of evaluation.+ """+ # TODO half to amp_test+ tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor+ model = model.eval()+ if half:+ model = model.half()+ ids = []+ data_list = []+ results = []+ video_names = defaultdict()+ progress_bar = tqdm if is_main_process() else iter++ inference_time = 0+ track_time = 0+ n_samples = len(self.dataloader) - 1++ if trt_file is not None:+ from torch2trt import TRTModule++ model_trt = TRTModule()+ model_trt.load_state_dict(torch.load(trt_file))++ x = torch.ones(1, 3, test_size[0], test_size[1]).cuda()+ model(x)+ model = model_trt++ tracker = Sort(self.args.track_thresh)++ for cur_iter, (imgs, _, info_imgs, ids) in enumerate(+ progress_bar(self.dataloader)+ ):+ with torch.no_grad():+ # init tracker+ frame_id = info_imgs[2].item()+ video_id = info_imgs[3].item()+ img_file_name = info_imgs[4]+ video_name = img_file_name[0].split('/')[0]++ if video_name not in video_names:+ video_names[video_id] = video_name+ if frame_id == 1:+ tracker = Sort(self.args.track_thresh)+ if len(results) != 0:+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id - 1]))+ write_results_no_score(result_filename, results)+ results = []++ imgs = imgs.type(tensor_type)++ # skip the the last iters since batchsize might be not enough for batch inference+ is_time_record = cur_iter < len(self.dataloader) - 1+ if is_time_record:+ start = time.time()++ outputs = model(imgs)+ if decoder is not None:+ outputs = decoder(outputs, dtype=outputs.type())++ outputs = postprocess(outputs, self.num_classes, self.confthre, self.nmsthre)++ if is_time_record:+ infer_end = time_synchronized()+ inference_time += infer_end - start++ output_results = self.convert_to_coco_format(outputs, info_imgs, ids)+ data_list.extend(output_results)++ # run tracking+ online_targets = tracker.update(outputs[0], info_imgs, self.img_size)+ online_tlwhs = []+ online_ids = []+ for t in online_targets:+ tlwh = [t[0], t[1], t[2] - t[0], t[3] - t[1]]+ tid = t[4]+ vertical = tlwh[2] / tlwh[3] > 1.6+ if tlwh[2] * tlwh[3] > self.args.min_box_area and not vertical:+ online_tlwhs.append(tlwh)+ online_ids.append(tid)+ # save results+ results.append((frame_id, online_tlwhs, online_ids))++ if is_time_record:+ track_end = time_synchronized()+ track_time += track_end - infer_end++ if cur_iter == len(self.dataloader) - 1:+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id]))+ write_results_no_score(result_filename, results)++ statistics = torch.cuda.FloatTensor([inference_time, track_time, n_samples])+ if distributed:+ data_list = gather(data_list, dst=0)+ data_list = list(itertools.chain(*data_list))+ torch.distributed.reduce(statistics, dst=0)++ eval_results = self.evaluate_prediction(data_list, statistics)+ synchronize()+ return eval_results++ def evaluate_deepsort(+ self,+ model,+ distributed=False,+ half=False,+ trt_file=None,+ decoder=None,+ test_size=None,+ result_folder=None,+ model_folder=None+ ):+ """+ COCO average precision (AP) Evaluation. Iterate inference on the test dataset+ and the results are evaluated by COCO API.++ NOTE: This function will change training mode to False, please save states if needed.++ Args:+ model : model to evaluate.++ Returns:+ ap50_95 (float) : COCO AP of IoU=50:95+ ap50 (float) : COCO AP of IoU=50+ summary (sr): summary info of evaluation.+ """+ # TODO half to amp_test+ tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor+ model = model.eval()+ if half:+ model = model.half()+ ids = []+ data_list = []+ results = []+ video_names = defaultdict()+ progress_bar = tqdm if is_main_process() else iter++ inference_time = 0+ track_time = 0+ n_samples = len(self.dataloader) - 1++ if trt_file is not None:+ from torch2trt import TRTModule++ model_trt = TRTModule()+ model_trt.load_state_dict(torch.load(trt_file))++ x = torch.ones(1, 3, test_size[0], test_size[1]).cuda()+ model(x)+ model = model_trt++ tracker = DeepSort(model_folder, min_confidence=self.args.track_thresh)++ for cur_iter, (imgs, _, info_imgs, ids) in enumerate(+ progress_bar(self.dataloader)+ ):+ with torch.no_grad():+ # init tracker+ frame_id = info_imgs[2].item()+ video_id = info_imgs[3].item()+ img_file_name = info_imgs[4]+ video_name = img_file_name[0].split('/')[0]++ if video_name not in video_names:+ video_names[video_id] = video_name+ if frame_id == 1:+ tracker = DeepSort(model_folder, min_confidence=self.args.track_thresh)+ if len(results) != 0:+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id - 1]))+ write_results_no_score(result_filename, results)+ results = []++ imgs = imgs.type(tensor_type)++ # skip the the last iters since batchsize might be not enough for batch inference+ is_time_record = cur_iter < len(self.dataloader) - 1+ if is_time_record:+ start = time.time()++ outputs = model(imgs)+ if decoder is not None:+ outputs = decoder(outputs, dtype=outputs.type())++ outputs = postprocess(outputs, self.num_classes, self.confthre, self.nmsthre)++ if is_time_record:+ infer_end = time_synchronized()+ inference_time += infer_end - start++ output_results = self.convert_to_coco_format(outputs, info_imgs, ids)+ data_list.extend(output_results)++ # run tracking+ online_targets = tracker.update(outputs[0], info_imgs, self.img_size, img_file_name[0])+ online_tlwhs = []+ online_ids = []+ for t in online_targets:+ tlwh = [t[0], t[1], t[2] - t[0], t[3] - t[1]]+ tid = t[4]+ vertical = tlwh[2] / tlwh[3] > 1.6+ if tlwh[2] * tlwh[3] > self.args.min_box_area and not vertical:+ online_tlwhs.append(tlwh)+ online_ids.append(tid)+ # save results+ results.append((frame_id, online_tlwhs, online_ids))++ if is_time_record:+ track_end = time_synchronized()+ track_time += track_end - infer_end++ if cur_iter == len(self.dataloader) - 1:+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id]))+ write_results_no_score(result_filename, results)++ statistics = torch.cuda.FloatTensor([inference_time, track_time, n_samples])+ if distributed:+ data_list = gather(data_list, dst=0)+ data_list = list(itertools.chain(*data_list))+ torch.distributed.reduce(statistics, dst=0)++ eval_results = self.evaluate_prediction(data_list, statistics)+ synchronize()+ return eval_results++ def evaluate_motdt(+ self,+ model,+ distributed=False,+ half=False,+ trt_file=None,+ decoder=None,+ test_size=None,+ result_folder=None,+ model_folder=None+ ):+ """+ COCO average precision (AP) Evaluation. Iterate inference on the test dataset+ and the results are evaluated by COCO API.++ NOTE: This function will change training mode to False, please save states if needed.++ Args:+ model : model to evaluate.++ Returns:+ ap50_95 (float) : COCO AP of IoU=50:95+ ap50 (float) : COCO AP of IoU=50+ summary (sr): summary info of evaluation.+ """+ # TODO half to amp_test+ tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor+ model = model.eval()+ if half:+ model = model.half()+ ids = []+ data_list = []+ results = []+ video_names = defaultdict()+ progress_bar = tqdm if is_main_process() else iter++ inference_time = 0+ track_time = 0+ n_samples = len(self.dataloader) - 1++ if trt_file is not None:+ from torch2trt import TRTModule++ model_trt = TRTModule()+ model_trt.load_state_dict(torch.load(trt_file))++ x = torch.ones(1, 3, test_size[0], test_size[1]).cuda()+ model(x)+ model = model_trt++ tracker = OnlineTracker(model_folder, min_cls_score=self.args.track_thresh)+ for cur_iter, (imgs, _, info_imgs, ids) in enumerate(+ progress_bar(self.dataloader)+ ):+ with torch.no_grad():+ # init tracker+ frame_id = info_imgs[2].item()+ video_id = info_imgs[3].item()+ img_file_name = info_imgs[4]+ video_name = img_file_name[0].split('/')[0]++ if video_name not in video_names:+ video_names[video_id] = video_name+ if frame_id == 1:+ tracker = OnlineTracker(model_folder, min_cls_score=self.args.track_thresh)+ if len(results) != 0:+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id - 1]))+ write_results(result_filename, results)+ results = []++ imgs = imgs.type(tensor_type)++ # skip the the last iters since batchsize might be not enough for batch inference+ is_time_record = cur_iter < len(self.dataloader) - 1+ if is_time_record:+ start = time.time()++ outputs = model(imgs)+ if decoder is not None:+ outputs = decoder(outputs, dtype=outputs.type())++ outputs = postprocess(outputs, self.num_classes, self.confthre, self.nmsthre)++ if is_time_record:+ infer_end = time_synchronized()+ inference_time += infer_end - start++ output_results = self.convert_to_coco_format(outputs, info_imgs, ids)+ data_list.extend(output_results)++ # run tracking+ online_targets = tracker.update(outputs[0], info_imgs, self.img_size, img_file_name[0])+ online_tlwhs = []+ online_ids = []+ online_scores = []+ for t in online_targets:+ tlwh = t.tlwh+ tid = t.track_id+ vertical = tlwh[2] / tlwh[3] > 1.6+ if tlwh[2] * tlwh[3] > self.args.min_box_area and not vertical:+ online_tlwhs.append(tlwh)+ online_ids.append(tid)+ online_scores.append(t.score)+ # save results+ results.append((frame_id, online_tlwhs, online_ids, online_scores))++ if is_time_record:+ track_end = time_synchronized()+ track_time += track_end - infer_end++ if cur_iter == len(self.dataloader) - 1:+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id]))+ write_results(result_filename, results)++ statistics = torch.cuda.FloatTensor([inference_time, track_time, n_samples])+ if distributed:+ data_list = gather(data_list, dst=0)+ data_list = list(itertools.chain(*data_list))+ torch.distributed.reduce(statistics, dst=0)++ eval_results = self.evaluate_prediction(data_list, statistics)+ synchronize()+ return eval_results++ def convert_to_coco_format(self, outputs, info_imgs, ids):+ data_list = []+ for (output, img_h, img_w, img_id) in zip(+ outputs, info_imgs[0], info_imgs[1], ids+ ):+ if output is None:+ continue+ output = output.cpu()++ bboxes = output[:, 0:4]++ # preprocessing: resize+ scale = min(+ self.img_size[0] / float(img_h), self.img_size[1] / float(img_w)+ )+ bboxes /= scale+ bboxes = xyxy2xywh(bboxes)++ cls = output[:, 6]+ scores = output[:, 4] * output[:, 5]+ for ind in range(bboxes.shape[0]):+ label = self.dataloader.dataset.class_ids[int(cls[ind])]+ pred_data = {+ "image_id": int(img_id),+ "category_id": label,+ "bbox": bboxes[ind].numpy().tolist(),+ "score": scores[ind].numpy().item(),+ "segmentation": [],+ } # COCO json format+ data_list.append(pred_data)+ return data_list++ def evaluate_prediction(self, data_dict, statistics):+ if not is_main_process():+ return 0, 0, None++ logger.info("Evaluate in main process...")++ annType = ["segm", "bbox", "keypoints"]++ inference_time = statistics[0].item()+ track_time = statistics[1].item()+ n_samples = statistics[2].item()++ a_infer_time = 1000 * inference_time / (n_samples * self.dataloader.batch_size)+ a_track_time = 1000 * track_time / (n_samples * self.dataloader.batch_size)++ time_info = ", ".join(+ [+ "Average {} time: {:.2f} ms".format(k, v)+ for k, v in zip(+ ["forward", "track", "inference"],+ [a_infer_time, a_track_time, (a_infer_time + a_track_time)],+ )+ ]+ )++ info = time_info + "\n"++ # Evaluate the Dt (detection) json comparing with the ground truth+ if len(data_dict) > 0:+ cocoGt = self.dataloader.dataset.coco+ # TODO: since pycocotools can't process dict in py36, write data to json file.+ _, tmp = tempfile.mkstemp()+ json.dump(data_dict, open(tmp, "w"))+ cocoDt = cocoGt.loadRes(tmp)+ '''+ try:+ from yolox.layers import COCOeval_opt as COCOeval+ except ImportError:+ from pycocotools import cocoeval as COCOeval+ logger.warning("Use standard COCOeval.")+ '''+ #from pycocotools.cocoeval import COCOeval+ from yolox.layers import COCOeval_opt as COCOeval+ cocoEval = COCOeval(cocoGt, cocoDt, annType[1])+ cocoEval.evaluate()+ cocoEval.accumulate()+ redirect_string = io.StringIO()+ with contextlib.redirect_stdout(redirect_string):+ cocoEval.summarize()+ info += redirect_string.getvalue()+ return cocoEval.stats[0], cocoEval.stats[1], info+ else:+ return 0, 0, infodiff --git a/experimental/yolox/exp/__init__.py b/experimental/yolox/exp/__init__.pynew file mode 100644index 0000000..951195c--- /dev/null+++ b/experimental/yolox/exp/__init__.py@@ -0,0 +1,7 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++from .base_exp import BaseExp+from .build import get_exp+from .yolox_base import Expdiff --git a/experimental/yolox/exp/base_exp.py b/experimental/yolox/exp/base_exp.pynew file mode 100644index 0000000..c4aed89--- /dev/null+++ b/experimental/yolox/exp/base_exp.py@@ -0,0 +1,75 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import torch+from torch.nn import Module++from yolox.utils import LRScheduler++import ast+import pprint+from abc import ABCMeta, abstractmethod+from tabulate import tabulate+from typing import Dict+++class BaseExp(metaclass=ABCMeta):+ """Basic class for any experiment."""++ def __init__(self):+ self.seed = None+ self.output_dir = "./YOLOX_outputs"+ self.print_interval = 100+ self.eval_interval = 10++ @abstractmethod+ def get_model(self) -> Module:+ pass++ @abstractmethod+ def get_data_loader(+ self, batch_size: int, is_distributed: bool+ ) -> Dict[str, torch.utils.data.DataLoader]:+ pass++ @abstractmethod+ def get_optimizer(self, batch_size: int) -> torch.optim.Optimizer:+ pass++ @abstractmethod+ def get_lr_scheduler(+ self, lr: float, iters_per_epoch: int, **kwargs+ ) -> LRScheduler:+ pass++ @abstractmethod+ def get_evaluator(self):+ pass++ @abstractmethod+ def eval(self, model, evaluator, weights):+ pass++ def __repr__(self):+ table_header = ["keys", "values"]+ exp_table = [+ (str(k), pprint.pformat(v))+ for k, v in vars(self).items()+ if not k.startswith("_")+ ]+ return tabulate(exp_table, headers=table_header, tablefmt="fancy_grid")++ def merge(self, cfg_list):+ assert len(cfg_list) % 2 == 0+ for k, v in zip(cfg_list[0::2], cfg_list[1::2]):+ # only update value with same key+ if hasattr(self, k):+ src_value = getattr(self, k)+ src_type = type(src_value)+ if src_value is not None and src_type != type(v):+ try:+ v = src_type(v)+ except Exception:+ v = ast.literal_eval(v)+ setattr(self, k, v)diff --git a/experimental/yolox/exp/build.py b/experimental/yolox/exp/build.pynew file mode 100644index 0000000..411b09c--- /dev/null+++ b/experimental/yolox/exp/build.py@@ -0,0 +1,53 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import importlib+import os+import sys+++def get_exp_by_file(exp_file):+ try:+ sys.path.append(os.path.dirname(exp_file))+ current_exp = importlib.import_module(os.path.basename(exp_file).split(".")[0])+ exp = current_exp.Exp()+ except Exception:+ raise ImportError("{} doesn't contains class named 'Exp'".format(exp_file))+ return exp+++def get_exp_by_name(exp_name):+ import yolox++ yolox_path = os.path.dirname(os.path.dirname(yolox.__file__))+ filedict = {+ "yolox-s": "yolox_s.py",+ "yolox-m": "yolox_m.py",+ "yolox-l": "yolox_l.py",+ "yolox-x": "yolox_x.py",+ "yolox-tiny": "yolox_tiny.py",+ "yolox-nano": "nano.py",+ "yolov3": "yolov3.py",+ }+ filename = filedict[exp_name]+ exp_path = os.path.join(yolox_path, "exps", "default", filename)+ return get_exp_by_file(exp_path)+++def get_exp(exp_file, exp_name):+ """+ get Exp object by file or name. If exp_file and exp_name+ are both provided, get Exp by exp_file.++ Args:+ exp_file (str): file path of experiment.+ exp_name (str): name of experiment. "yolo-s",+ """+ assert (+ exp_file is not None or exp_name is not None+ ), "plz provide exp file or exp name."+ if exp_file is not None:+ return get_exp_by_file(exp_file)+ else:+ return get_exp_by_name(exp_name)diff --git a/experimental/yolox/exp/yolox_base.py b/experimental/yolox/exp/yolox_base.pynew file mode 100644index 0000000..a1b1b03--- /dev/null+++ b/experimental/yolox/exp/yolox_base.py@@ -0,0 +1,248 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import torch+import torch.distributed as dist+import torch.nn as nn++import os+import random++from .base_exp import BaseExp+++class Exp(BaseExp):+ def __init__(self):+ super().__init__()++ # ---------------- model config ---------------- #+ self.num_classes = 80+ self.depth = 1.00+ self.width = 1.00++ # ---------------- dataloader config ---------------- #+ # set worker to 4 for shorter dataloader init time+ self.data_num_workers = 4+ self.input_size = (640, 640)+ self.random_size = (14, 26)+ self.train_ann = "instances_train2017.json"+ self.val_ann = "instances_val2017.json"++ # --------------- transform config ----------------- #+ self.degrees = 10.0+ self.translate = 0.1+ self.scale = (0.1, 2)+ self.mscale = (0.8, 1.6)+ self.shear = 2.0+ self.perspective = 0.0+ self.enable_mixup = True++ # -------------- training config --------------------- #+ self.warmup_epochs = 5+ self.max_epoch = 300+ self.warmup_lr = 0+ self.basic_lr_per_img = 0.01 / 64.0+ self.scheduler = "yoloxwarmcos"+ self.no_aug_epochs = 15+ self.min_lr_ratio = 0.05+ self.ema = True++ self.weight_decay = 5e-4+ self.momentum = 0.9+ self.print_interval = 10+ self.eval_interval = 10+ self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(".")[0]++ # ----------------- testing config ------------------ #+ self.test_size = (640, 640)+ self.test_conf = 0.001+ self.nmsthre = 0.65++ def get_model(self):+ from yolox.models import YOLOPAFPN, YOLOX, YOLOXHead++ def init_yolo(M):+ for m in M.modules():+ if isinstance(m, nn.BatchNorm2d):+ m.eps = 1e-3+ m.momentum = 0.03++ if getattr(self, "model", None) is None:+ in_channels = [256, 512, 1024]+ backbone = YOLOPAFPN(self.depth, self.width, in_channels=in_channels)+ head = YOLOXHead(self.num_classes, self.width, in_channels=in_channels)+ self.model = YOLOX(backbone, head)++ self.model.apply(init_yolo)+ self.model.head.initialize_biases(1e-2)+ return self.model++ def get_data_loader(self, batch_size, is_distributed, no_aug=False):+ from yolox.data import (+ COCODataset,+ DataLoader,+ InfiniteSampler,+ MosaicDetection,+ TrainTransform,+ YoloBatchSampler+ )++ dataset = COCODataset(+ data_dir=None,+ json_file=self.train_ann,+ img_size=self.input_size,+ preproc=TrainTransform(+ rgb_means=(0.485, 0.456, 0.406),+ std=(0.229, 0.224, 0.225),+ max_labels=50,+ ),+ )++ dataset = MosaicDetection(+ dataset,+ mosaic=not no_aug,+ img_size=self.input_size,+ preproc=TrainTransform(+ rgb_means=(0.485, 0.456, 0.406),+ std=(0.229, 0.224, 0.225),+ max_labels=120,+ ),+ degrees=self.degrees,+ translate=self.translate,+ scale=self.scale,+ shear=self.shear,+ perspective=self.perspective,+ enable_mixup=self.enable_mixup,+ )++ self.dataset = dataset++ if is_distributed:+ batch_size = batch_size // dist.get_world_size()++ sampler = InfiniteSampler(len(self.dataset), seed=self.seed if self.seed else 0)++ batch_sampler = YoloBatchSampler(+ sampler=sampler,+ batch_size=batch_size,+ drop_last=False,+ input_dimension=self.input_size,+ mosaic=not no_aug,+ )++ dataloader_kwargs = {"num_workers": self.data_num_workers, "pin_memory": True}+ dataloader_kwargs["batch_sampler"] = batch_sampler+ train_loader = DataLoader(self.dataset, **dataloader_kwargs)++ return train_loader++ def random_resize(self, data_loader, epoch, rank, is_distributed):+ tensor = torch.LongTensor(2).cuda()++ if rank == 0:+ size_factor = self.input_size[1] * 1.0 / self.input_size[0]+ size = random.randint(*self.random_size)+ size = (int(32 * size), 32 * int(size * size_factor))+ tensor[0] = size[0]+ tensor[1] = size[1]++ if is_distributed:+ dist.barrier()+ dist.broadcast(tensor, 0)++ input_size = data_loader.change_input_dim(+ multiple=(tensor[0].item(), tensor[1].item()), random_range=None+ )+ return input_size++ def get_optimizer(self, batch_size):+ if "optimizer" not in self.__dict__:+ if self.warmup_epochs > 0:+ lr = self.warmup_lr+ else:+ lr = self.basic_lr_per_img * batch_size++ pg0, pg1, pg2 = [], [], [] # optimizer parameter groups++ for k, v in self.model.named_modules():+ if hasattr(v, "bias") and isinstance(v.bias, nn.Parameter):+ pg2.append(v.bias) # biases+ if isinstance(v, nn.BatchNorm2d) or "bn" in k:+ pg0.append(v.weight) # no decay+ elif hasattr(v, "weight") and isinstance(v.weight, nn.Parameter):+ pg1.append(v.weight) # apply decay++ optimizer = torch.optim.SGD(+ pg0, lr=lr, momentum=self.momentum, nesterov=True+ )+ optimizer.add_param_group(+ {"params": pg1, "weight_decay": self.weight_decay}+ ) # add pg1 with weight_decay+ optimizer.add_param_group({"params": pg2})+ self.optimizer = optimizer++ return self.optimizer++ def get_lr_scheduler(self, lr, iters_per_epoch):+ from yolox.utils import LRScheduler++ scheduler = LRScheduler(+ self.scheduler,+ lr,+ iters_per_epoch,+ self.max_epoch,+ warmup_epochs=self.warmup_epochs,+ warmup_lr_start=self.warmup_lr,+ no_aug_epochs=self.no_aug_epochs,+ min_lr_ratio=self.min_lr_ratio,+ )+ return scheduler++ def get_eval_loader(self, batch_size, is_distributed, testdev=False):+ from yolox.data import COCODataset, ValTransform++ valdataset = COCODataset(+ data_dir=None,+ json_file=self.val_ann if not testdev else "image_info_test-dev2017.json",+ name="val2017" if not testdev else "test2017",+ img_size=self.test_size,+ preproc=ValTransform(+ rgb_means=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)+ ),+ )++ if is_distributed:+ batch_size = batch_size // dist.get_world_size()+ sampler = torch.utils.data.distributed.DistributedSampler(+ valdataset, shuffle=False+ )+ else:+ sampler = torch.utils.data.SequentialSampler(valdataset)++ dataloader_kwargs = {+ "num_workers": self.data_num_workers,+ "pin_memory": True,+ "sampler": sampler,+ }+ dataloader_kwargs["batch_size"] = batch_size+ val_loader = torch.utils.data.DataLoader(valdataset, **dataloader_kwargs)++ return val_loader++ def get_evaluator(self, batch_size, is_distributed, testdev=False):+ from yolox.evaluators import COCOEvaluator++ val_loader = self.get_eval_loader(batch_size, is_distributed, testdev=testdev)+ evaluator = COCOEvaluator(+ dataloader=val_loader,+ img_size=self.test_size,+ confthre=self.test_conf,+ nmsthre=self.nmsthre,+ num_classes=self.num_classes,+ testdev=testdev,+ )+ return evaluator++ def eval(self, model, evaluator, is_distributed, half=False):+ return evaluator.evaluate(model, is_distributed, half)diff --git a/experimental/yolox/layers/__init__.py b/experimental/yolox/layers/__init__.pynew file mode 100644index 0000000..833947c--- /dev/null+++ b/experimental/yolox/layers/__init__.py@@ -0,0 +1,5 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++from .fast_coco_eval_api import COCOeval_optdiff --git a/experimental/yolox/layers/csrc/cocoeval/cocoeval.cpp b/experimental/yolox/layers/csrc/cocoeval/cocoeval.cppnew file mode 100644index 0000000..2e63bc9--- /dev/null+++ b/experimental/yolox/layers/csrc/cocoeval/cocoeval.cpp@@ -0,0 +1,502 @@+// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved+#include "cocoeval.h"+#include <time.h>+#include <algorithm>+#include <cstdint>+#include <numeric>++using namespace pybind11::literals;++namespace COCOeval {++// Sort detections from highest score to lowest, such that+// detection_instances[detection_sorted_indices[t]] >=+// detection_instances[detection_sorted_indices[t+1]]. Use stable_sort to match+// original COCO API+void SortInstancesByDetectionScore(+ const std::vector<InstanceAnnotation>& detection_instances,+ std::vector<uint64_t>* detection_sorted_indices) {+ detection_sorted_indices->resize(detection_instances.size());+ std::iota(+ detection_sorted_indices->begin(), detection_sorted_indices->end(), 0);+ std::stable_sort(+ detection_sorted_indices->begin(),+ detection_sorted_indices->end(),+ [&detection_instances](size_t j1, size_t j2) {+ return detection_instances[j1].score > detection_instances[j2].score;+ });+}++// Partition the ground truth objects based on whether or not to ignore them+// based on area+void SortInstancesByIgnore(+ const std::array<double, 2>& area_range,+ const std::vector<InstanceAnnotation>& ground_truth_instances,+ std::vector<uint64_t>* ground_truth_sorted_indices,+ std::vector<bool>* ignores) {+ ignores->clear();+ ignores->reserve(ground_truth_instances.size());+ for (auto o : ground_truth_instances) {+ ignores->push_back(+ o.ignore || o.area < area_range[0] || o.area > area_range[1]);+ }++ ground_truth_sorted_indices->resize(ground_truth_instances.size());+ std::iota(+ ground_truth_sorted_indices->begin(),+ ground_truth_sorted_indices->end(),+ 0);+ std::stable_sort(+ ground_truth_sorted_indices->begin(),+ ground_truth_sorted_indices->end(),+ [&ignores](size_t j1, size_t j2) {+ return (int)(*ignores)[j1] < (int)(*ignores)[j2];+ });+}++// For each IOU threshold, greedily match each detected instance to a ground+// truth instance (if possible) and store the results+void MatchDetectionsToGroundTruth(+ const std::vector<InstanceAnnotation>& detection_instances,+ const std::vector<uint64_t>& detection_sorted_indices,+ const std::vector<InstanceAnnotation>& ground_truth_instances,+ const std::vector<uint64_t>& ground_truth_sorted_indices,+ const std::vector<bool>& ignores,+ const std::vector<std::vector<double>>& ious,+ const std::vector<double>& iou_thresholds,+ const std::array<double, 2>& area_range,+ ImageEvaluation* results) {+ // Initialize memory to store return data matches and ignore+ const int num_iou_thresholds = iou_thresholds.size();+ const int num_ground_truth = ground_truth_sorted_indices.size();+ const int num_detections = detection_sorted_indices.size();+ std::vector<uint64_t> ground_truth_matches(+ num_iou_thresholds * num_ground_truth, 0);+ std::vector<uint64_t>& detection_matches = results->detection_matches;+ std::vector<bool>& detection_ignores = results->detection_ignores;+ std::vector<bool>& ground_truth_ignores = results->ground_truth_ignores;+ detection_matches.resize(num_iou_thresholds * num_detections, 0);+ detection_ignores.resize(num_iou_thresholds * num_detections, false);+ ground_truth_ignores.resize(num_ground_truth);+ for (auto g = 0; g < num_ground_truth; ++g) {+ ground_truth_ignores[g] = ignores[ground_truth_sorted_indices[g]];+ }++ for (auto t = 0; t < num_iou_thresholds; ++t) {+ for (auto d = 0; d < num_detections; ++d) {+ // information about best match so far (match=-1 -> unmatched)+ double best_iou = std::min(iou_thresholds[t], 1 - 1e-10);+ int match = -1;+ for (auto g = 0; g < num_ground_truth; ++g) {+ // if this ground truth instance is already matched and not a+ // crowd, it cannot be matched to another detection+ if (ground_truth_matches[t * num_ground_truth + g] > 0 &&+ !ground_truth_instances[ground_truth_sorted_indices[g]].is_crowd) {+ continue;+ }++ // if detected instance matched to a regular ground truth+ // instance, we can break on the first ground truth instance+ // tagged as ignore (because they are sorted by the ignore tag)+ if (match >= 0 && !ground_truth_ignores[match] &&+ ground_truth_ignores[g]) {+ break;+ }++ // if IOU overlap is the best so far, store the match appropriately+ if (ious[d][ground_truth_sorted_indices[g]] >= best_iou) {+ best_iou = ious[d][ground_truth_sorted_indices[g]];+ match = g;+ }+ }+ // if match was made, store id of match for both detection and+ // ground truth+ if (match >= 0) {+ detection_ignores[t * num_detections + d] = ground_truth_ignores[match];+ detection_matches[t * num_detections + d] =+ ground_truth_instances[ground_truth_sorted_indices[match]].id;+ ground_truth_matches[t * num_ground_truth + match] =+ detection_instances[detection_sorted_indices[d]].id;+ }++ // set unmatched detections outside of area range to ignore+ const InstanceAnnotation& detection =+ detection_instances[detection_sorted_indices[d]];+ detection_ignores[t * num_detections + d] =+ detection_ignores[t * num_detections + d] ||+ (detection_matches[t * num_detections + d] == 0 &&+ (detection.area < area_range[0] || detection.area > area_range[1]));+ }+ }++ // store detection score results+ results->detection_scores.resize(detection_sorted_indices.size());+ for (size_t d = 0; d < detection_sorted_indices.size(); ++d) {+ results->detection_scores[d] =+ detection_instances[detection_sorted_indices[d]].score;+ }+}++std::vector<ImageEvaluation> EvaluateImages(+ const std::vector<std::array<double, 2>>& area_ranges,+ int max_detections,+ const std::vector<double>& iou_thresholds,+ const ImageCategoryInstances<std::vector<double>>& image_category_ious,+ const ImageCategoryInstances<InstanceAnnotation>&+ image_category_ground_truth_instances,+ const ImageCategoryInstances<InstanceAnnotation>&+ image_category_detection_instances) {+ const int num_area_ranges = area_ranges.size();+ const int num_images = image_category_ground_truth_instances.size();+ const int num_categories =+ image_category_ious.size() > 0 ? image_category_ious[0].size() : 0;+ std::vector<uint64_t> detection_sorted_indices;+ std::vector<uint64_t> ground_truth_sorted_indices;+ std::vector<bool> ignores;+ std::vector<ImageEvaluation> results_all(+ num_images * num_area_ranges * num_categories);++ // Store results for each image, category, and area range combination. Results+ // for each IOU threshold are packed into the same ImageEvaluation object+ for (auto i = 0; i < num_images; ++i) {+ for (auto c = 0; c < num_categories; ++c) {+ const std::vector<InstanceAnnotation>& ground_truth_instances =+ image_category_ground_truth_instances[i][c];+ const std::vector<InstanceAnnotation>& detection_instances =+ image_category_detection_instances[i][c];++ SortInstancesByDetectionScore(+ detection_instances, &detection_sorted_indices);+ if ((int)detection_sorted_indices.size() > max_detections) {+ detection_sorted_indices.resize(max_detections);+ }++ for (size_t a = 0; a < area_ranges.size(); ++a) {+ SortInstancesByIgnore(+ area_ranges[a],+ ground_truth_instances,+ &ground_truth_sorted_indices,+ &ignores);++ MatchDetectionsToGroundTruth(+ detection_instances,+ detection_sorted_indices,+ ground_truth_instances,+ ground_truth_sorted_indices,+ ignores,+ image_category_ious[i][c],+ iou_thresholds,+ area_ranges[a],+ &results_all+ [c * num_area_ranges * num_images + a * num_images + i]);+ }+ }+ }++ return results_all;+}++// Convert a python list to a vector+template <typename T>+std::vector<T> list_to_vec(const py::list& l) {+ std::vector<T> v(py::len(l));+ for (int i = 0; i < (int)py::len(l); ++i) {+ v[i] = l[i].cast<T>();+ }+ return v;+}++// Helper function to Accumulate()+// Considers the evaluation results applicable to a particular category, area+// range, and max_detections parameter setting, which begin at+// evaluations[evaluation_index]. Extracts a sorted list of length n of all+// applicable detection instances concatenated across all images in the dataset,+// which are represented by the outputs evaluation_indices, detection_scores,+// image_detection_indices, and detection_sorted_indices--all of which are+// length n. evaluation_indices[i] stores the applicable index into+// evaluations[] for instance i, which has detection score detection_score[i],+// and is the image_detection_indices[i]'th of the list of detections+// for the image containing i. detection_sorted_indices[] defines a sorted+// permutation of the 3 other outputs+int BuildSortedDetectionList(+ const std::vector<ImageEvaluation>& evaluations,+ const int64_t evaluation_index,+ const int64_t num_images,+ const int max_detections,+ std::vector<uint64_t>* evaluation_indices,+ std::vector<double>* detection_scores,+ std::vector<uint64_t>* detection_sorted_indices,+ std::vector<uint64_t>* image_detection_indices) {+ assert(evaluations.size() >= evaluation_index + num_images);++ // Extract a list of object instances of the applicable category, area+ // range, and max detections requirements such that they can be sorted+ image_detection_indices->clear();+ evaluation_indices->clear();+ detection_scores->clear();+ image_detection_indices->reserve(num_images * max_detections);+ evaluation_indices->reserve(num_images * max_detections);+ detection_scores->reserve(num_images * max_detections);+ int num_valid_ground_truth = 0;+ for (auto i = 0; i < num_images; ++i) {+ const ImageEvaluation& evaluation = evaluations[evaluation_index + i];++ for (int d = 0;+ d < (int)evaluation.detection_scores.size() && d < max_detections;+ ++d) { // detected instances+ evaluation_indices->push_back(evaluation_index + i);+ image_detection_indices->push_back(d);+ detection_scores->push_back(evaluation.detection_scores[d]);+ }+ for (auto ground_truth_ignore : evaluation.ground_truth_ignores) {+ if (!ground_truth_ignore) {+ ++num_valid_ground_truth;+ }+ }+ }++ // Sort detections by decreasing score, using stable sort to match+ // python implementation+ detection_sorted_indices->resize(detection_scores->size());+ std::iota(+ detection_sorted_indices->begin(), detection_sorted_indices->end(), 0);+ std::stable_sort(+ detection_sorted_indices->begin(),+ detection_sorted_indices->end(),+ [&detection_scores](size_t j1, size_t j2) {+ return (*detection_scores)[j1] > (*detection_scores)[j2];+ });++ return num_valid_ground_truth;+}++// Helper function to Accumulate()+// Compute a precision recall curve given a sorted list of detected instances+// encoded in evaluations, evaluation_indices, detection_scores,+// detection_sorted_indices, image_detection_indices (see+// BuildSortedDetectionList()). Using vectors precisions and recalls+// and temporary storage, output the results into precisions_out, recalls_out,+// and scores_out, which are large buffers containing many precion/recall curves+// for all possible parameter settings, with precisions_out_index and+// recalls_out_index defining the applicable indices to store results.+void ComputePrecisionRecallCurve(+ const int64_t precisions_out_index,+ const int64_t precisions_out_stride,+ const int64_t recalls_out_index,+ const std::vector<double>& recall_thresholds,+ const int iou_threshold_index,+ const int num_iou_thresholds,+ const int num_valid_ground_truth,+ const std::vector<ImageEvaluation>& evaluations,+ const std::vector<uint64_t>& evaluation_indices,+ const std::vector<double>& detection_scores,+ const std::vector<uint64_t>& detection_sorted_indices,+ const std::vector<uint64_t>& image_detection_indices,+ std::vector<double>* precisions,+ std::vector<double>* recalls,+ std::vector<double>* precisions_out,+ std::vector<double>* scores_out,+ std::vector<double>* recalls_out) {+ assert(recalls_out->size() > recalls_out_index);++ // Compute precision/recall for each instance in the sorted list of detections+ int64_t true_positives_sum = 0, false_positives_sum = 0;+ precisions->clear();+ recalls->clear();+ precisions->reserve(detection_sorted_indices.size());+ recalls->reserve(detection_sorted_indices.size());+ assert(!evaluations.empty() || detection_sorted_indices.empty());+ for (auto detection_sorted_index : detection_sorted_indices) {+ const ImageEvaluation& evaluation =+ evaluations[evaluation_indices[detection_sorted_index]];+ const auto num_detections =+ evaluation.detection_matches.size() / num_iou_thresholds;+ const auto detection_index = iou_threshold_index * num_detections ++ image_detection_indices[detection_sorted_index];+ assert(evaluation.detection_matches.size() > detection_index);+ assert(evaluation.detection_ignores.size() > detection_index);+ const int64_t detection_match =+ evaluation.detection_matches[detection_index];+ const bool detection_ignores =+ evaluation.detection_ignores[detection_index];+ const auto true_positive = detection_match > 0 && !detection_ignores;+ const auto false_positive = detection_match == 0 && !detection_ignores;+ if (true_positive) {+ ++true_positives_sum;+ }+ if (false_positive) {+ ++false_positives_sum;+ }++ const double recall =+ static_cast<double>(true_positives_sum) / num_valid_ground_truth;+ recalls->push_back(recall);+ const int64_t num_valid_detections =+ true_positives_sum + false_positives_sum;+ const double precision = num_valid_detections > 0+ ? static_cast<double>(true_positives_sum) / num_valid_detections+ : 0.0;+ precisions->push_back(precision);+ }++ (*recalls_out)[recalls_out_index] = !recalls->empty() ? recalls->back() : 0;++ for (int64_t i = static_cast<int64_t>(precisions->size()) - 1; i > 0; --i) {+ if ((*precisions)[i] > (*precisions)[i - 1]) {+ (*precisions)[i - 1] = (*precisions)[i];+ }+ }++ // Sample the per instance precision/recall list at each recall threshold+ for (size_t r = 0; r < recall_thresholds.size(); ++r) {+ // first index in recalls >= recall_thresholds[r]+ std::vector<double>::iterator low = std::lower_bound(+ recalls->begin(), recalls->end(), recall_thresholds[r]);+ size_t precisions_index = low - recalls->begin();++ const auto results_ind = precisions_out_index + r * precisions_out_stride;+ assert(results_ind < precisions_out->size());+ assert(results_ind < scores_out->size());+ if (precisions_index < precisions->size()) {+ (*precisions_out)[results_ind] = (*precisions)[precisions_index];+ (*scores_out)[results_ind] =+ detection_scores[detection_sorted_indices[precisions_index]];+ } else {+ (*precisions_out)[results_ind] = 0;+ (*scores_out)[results_ind] = 0;+ }+ }+}+py::dict Accumulate(+ const py::object& params,+ const std::vector<ImageEvaluation>& evaluations) {+ const std::vector<double> recall_thresholds =+ list_to_vec<double>(params.attr("recThrs"));+ const std::vector<int> max_detections =+ list_to_vec<int>(params.attr("maxDets"));+ const int num_iou_thresholds = py::len(params.attr("iouThrs"));+ const int num_recall_thresholds = py::len(params.attr("recThrs"));+ const int num_categories = params.attr("useCats").cast<int>() == 1+ ? py::len(params.attr("catIds"))+ : 1;+ const int num_area_ranges = py::len(params.attr("areaRng"));+ const int num_max_detections = py::len(params.attr("maxDets"));+ const int num_images = py::len(params.attr("imgIds"));++ std::vector<double> precisions_out(+ num_iou_thresholds * num_recall_thresholds * num_categories *+ num_area_ranges * num_max_detections,+ -1);+ std::vector<double> recalls_out(+ num_iou_thresholds * num_categories * num_area_ranges *+ num_max_detections,+ -1);+ std::vector<double> scores_out(+ num_iou_thresholds * num_recall_thresholds * num_categories *+ num_area_ranges * num_max_detections,+ -1);++ // Consider the list of all detected instances in the entire dataset in one+ // large list. evaluation_indices, detection_scores,+ // image_detection_indices, and detection_sorted_indices all have the same+ // length as this list, such that each entry corresponds to one detected+ // instance+ std::vector<uint64_t> evaluation_indices; // indices into evaluations[]+ std::vector<double> detection_scores; // detection scores of each instance+ std::vector<uint64_t> detection_sorted_indices; // sorted indices of all+ // instances in the dataset+ std::vector<uint64_t>+ image_detection_indices; // indices into the list of detected instances in+ // the same image as each instance+ std::vector<double> precisions, recalls;++ for (auto c = 0; c < num_categories; ++c) {+ for (auto a = 0; a < num_area_ranges; ++a) {+ for (auto m = 0; m < num_max_detections; ++m) {+ // The COCO PythonAPI assumes evaluations[] (the return value of+ // COCOeval::EvaluateImages() is one long list storing results for each+ // combination of category, area range, and image id, with categories in+ // the outermost loop and images in the innermost loop.+ const int64_t evaluations_index =+ c * num_area_ranges * num_images + a * num_images;+ int num_valid_ground_truth = BuildSortedDetectionList(+ evaluations,+ evaluations_index,+ num_images,+ max_detections[m],+ &evaluation_indices,+ &detection_scores,+ &detection_sorted_indices,+ &image_detection_indices);++ if (num_valid_ground_truth == 0) {+ continue;+ }++ for (auto t = 0; t < num_iou_thresholds; ++t) {+ // recalls_out is a flattened vectors representing a+ // num_iou_thresholds X num_categories X num_area_ranges X+ // num_max_detections matrix+ const int64_t recalls_out_index =+ t * num_categories * num_area_ranges * num_max_detections ++ c * num_area_ranges * num_max_detections ++ a * num_max_detections + m;++ // precisions_out and scores_out are flattened vectors+ // representing a num_iou_thresholds X num_recall_thresholds X+ // num_categories X num_area_ranges X num_max_detections matrix+ const int64_t precisions_out_stride =+ num_categories * num_area_ranges * num_max_detections;+ const int64_t precisions_out_index = t * num_recall_thresholds *+ num_categories * num_area_ranges * num_max_detections ++ c * num_area_ranges * num_max_detections ++ a * num_max_detections + m;++ ComputePrecisionRecallCurve(+ precisions_out_index,+ precisions_out_stride,+ recalls_out_index,+ recall_thresholds,+ t,+ num_iou_thresholds,+ num_valid_ground_truth,+ evaluations,+ evaluation_indices,+ detection_scores,+ detection_sorted_indices,+ image_detection_indices,+ &precisions,+ &recalls,+ &precisions_out,+ &scores_out,+ &recalls_out);+ }+ }+ }+ }++ time_t rawtime;+ struct tm local_time;+ std::array<char, 200> buffer;+ time(&rawtime);+#ifdef _WIN32+ localtime_s(&local_time, &rawtime);+#else+ localtime_r(&rawtime, &local_time);+#endif+ strftime(+ buffer.data(), 200, "%Y-%m-%d %H:%num_max_detections:%S", &local_time);+ return py::dict(+ "params"_a = params,+ "counts"_a = std::vector<int64_t>({num_iou_thresholds,+ num_recall_thresholds,+ num_categories,+ num_area_ranges,+ num_max_detections}),+ "date"_a = buffer,+ "precision"_a = precisions_out,+ "recall"_a = recalls_out,+ "scores"_a = scores_out);+}++} // namespace COCOevaldiff --git a/experimental/yolox/layers/csrc/cocoeval/cocoeval.h b/experimental/yolox/layers/csrc/cocoeval/cocoeval.hnew file mode 100644index 0000000..f9def41--- /dev/null+++ b/experimental/yolox/layers/csrc/cocoeval/cocoeval.h@@ -0,0 +1,85 @@+// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved+#pragma once++#include <pybind11/numpy.h>+#include <pybind11/pybind11.h>+#include <pybind11/stl.h>+#include <pybind11/stl_bind.h>+#include <vector>++namespace py = pybind11;++namespace COCOeval {++// Annotation data for a single object instance in an image+struct InstanceAnnotation {+ InstanceAnnotation(+ uint64_t id,+ double score,+ double area,+ bool is_crowd,+ bool ignore)+ : id{id}, score{score}, area{area}, is_crowd{is_crowd}, ignore{ignore} {}+ uint64_t id;+ double score = 0.;+ double area = 0.;+ bool is_crowd = false;+ bool ignore = false;+};++// Stores intermediate results for evaluating detection results for a single+// image that has D detected instances and G ground truth instances. This stores+// matches between detected and ground truth instances+struct ImageEvaluation {+ // For each of the D detected instances, the id of the matched ground truth+ // instance, or 0 if unmatched+ std::vector<uint64_t> detection_matches;++ // The detection score of each of the D detected instances+ std::vector<double> detection_scores;++ // Marks whether or not each of G instances was ignored from evaluation (e.g.,+ // because it's outside area_range)+ std::vector<bool> ground_truth_ignores;++ // Marks whether or not each of D instances was ignored from evaluation (e.g.,+ // because it's outside aRng)+ std::vector<bool> detection_ignores;+};++template <class T>+using ImageCategoryInstances = std::vector<std::vector<std::vector<T>>>;++// C++ implementation of COCO API cocoeval.py::COCOeval.evaluateImg(). For each+// combination of image, category, area range settings, and IOU thresholds to+// evaluate, it matches detected instances to ground truth instances and stores+// the results into a vector of ImageEvaluation results, which will be+// interpreted by the COCOeval::Accumulate() function to produce precion-recall+// curves. The parameters of nested vectors have the following semantics:+// image_category_ious[i][c][d][g] is the intersection over union of the d'th+// detected instance and g'th ground truth instance of+// category category_ids[c] in image image_ids[i]+// image_category_ground_truth_instances[i][c] is a vector of ground truth+// instances in image image_ids[i] of category category_ids[c]+// image_category_detection_instances[i][c] is a vector of detected+// instances in image image_ids[i] of category category_ids[c]+std::vector<ImageEvaluation> EvaluateImages(+ const std::vector<std::array<double, 2>>& area_ranges, // vector of 2-tuples+ int max_detections,+ const std::vector<double>& iou_thresholds,+ const ImageCategoryInstances<std::vector<double>>& image_category_ious,+ const ImageCategoryInstances<InstanceAnnotation>&+ image_category_ground_truth_instances,+ const ImageCategoryInstances<InstanceAnnotation>&+ image_category_detection_instances);++// C++ implementation of COCOeval.accumulate(), which generates precision+// recall curves for each set of category, IOU threshold, detection area range,+// and max number of detections parameters. It is assumed that the parameter+// evaluations is the return value of the functon COCOeval::EvaluateImages(),+// which was called with the same parameter settings params+py::dict Accumulate(+ const py::object& params,+ const std::vector<ImageEvaluation>& evalutations);++} // namespace COCOevaldiff --git a/experimental/yolox/layers/csrc/vision.cpp b/experimental/yolox/layers/csrc/vision.cppnew file mode 100644index 0000000..7663d0f--- /dev/null+++ b/experimental/yolox/layers/csrc/vision.cpp@@ -0,0 +1,13 @@+#include "cocoeval/cocoeval.h"++PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {+ m.def("COCOevalAccumulate", &COCOeval::Accumulate, "COCOeval::Accumulate");+ m.def(+ "COCOevalEvaluateImages",+ &COCOeval::EvaluateImages,+ "COCOeval::EvaluateImages");+ pybind11::class_<COCOeval::InstanceAnnotation>(m, "InstanceAnnotation")+ .def(pybind11::init<uint64_t, double, double, bool, bool>());+ pybind11::class_<COCOeval::ImageEvaluation>(m, "ImageEvaluation")+ .def(pybind11::init<>());+}diff --git a/experimental/yolox/layers/fast_coco_eval_api.py b/experimental/yolox/layers/fast_coco_eval_api.pynew file mode 100644index 0000000..442c97e--- /dev/null+++ b/experimental/yolox/layers/fast_coco_eval_api.py@@ -0,0 +1,150 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# This file comes from+# https://github.com/facebookresearch/detectron2/blob/master/detectron2/evaluation/fast_eval_api.py+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import numpy as np+from pycocotools.cocoeval import COCOeval++# import torch first to make yolox._C work without ImportError of libc10.so+# in YOLOX, env is already set in __init__.py.+from yolox import _C++import copy+import time+++class COCOeval_opt(COCOeval):+ """+ This is a slightly modified version of the original COCO API, where the functions evaluateImg()+ and accumulate() are implemented in C++ to speedup evaluation+ """++ def evaluate(self):+ """+ Run per image evaluation on given images and store results in self.evalImgs_cpp, a+ datastructure that isn't readable from Python but is used by a c++ implementation of+ accumulate(). Unlike the original COCO PythonAPI, we don't populate the datastructure+ self.evalImgs because this datastructure is a computational bottleneck.+ :return: None+ """+ tic = time.time()++ print("Running per image evaluation...")+ p = self.params+ # add backward compatibility if useSegm is specified in params+ if p.useSegm is not None:+ p.iouType = "segm" if p.useSegm == 1 else "bbox"+ print(+ "useSegm (deprecated) is not None. Running {} evaluation".format(+ p.iouType+ )+ )+ print("Evaluate annotation type *{}*".format(p.iouType))+ p.imgIds = list(np.unique(p.imgIds))+ if p.useCats:+ p.catIds = list(np.unique(p.catIds))+ p.maxDets = sorted(p.maxDets)+ self.params = p++ self._prepare()++ # loop through images, area range, max detection number+ catIds = p.catIds if p.useCats else [-1]++ if p.iouType == "segm" or p.iouType == "bbox":+ computeIoU = self.computeIoU+ elif p.iouType == "keypoints":+ computeIoU = self.computeOks+ self.ious = {+ (imgId, catId): computeIoU(imgId, catId)+ for imgId in p.imgIds+ for catId in catIds+ }++ maxDet = p.maxDets[-1]++ # <<<< Beginning of code differences with original COCO API+ def convert_instances_to_cpp(instances, is_det=False):+ # Convert annotations for a list of instances in an image to a format that's fast+ # to access in C+++ instances_cpp = []+ for instance in instances:+ instance_cpp = _C.InstanceAnnotation(+ int(instance["id"]),+ instance["score"] if is_det else instance.get("score", 0.0),+ instance["area"],+ bool(instance.get("iscrowd", 0)),+ bool(instance.get("ignore", 0)),+ )+ instances_cpp.append(instance_cpp)+ return instances_cpp++ # Convert GT annotations, detections, and IOUs to a format that's fast to access in C+++ ground_truth_instances = [+ [convert_instances_to_cpp(self._gts[imgId, catId]) for catId in p.catIds]+ for imgId in p.imgIds+ ]+ detected_instances = [+ [+ convert_instances_to_cpp(self._dts[imgId, catId], is_det=True)+ for catId in p.catIds+ ]+ for imgId in p.imgIds+ ]+ ious = [[self.ious[imgId, catId] for catId in catIds] for imgId in p.imgIds]++ if not p.useCats:+ # For each image, flatten per-category lists into a single list+ ground_truth_instances = [+ [[o for c in i for o in c]] for i in ground_truth_instances+ ]+ detected_instances = [+ [[o for c in i for o in c]] for i in detected_instances+ ]++ # Call C++ implementation of self.evaluateImgs()+ self._evalImgs_cpp = _C.COCOevalEvaluateImages(+ p.areaRng,+ maxDet,+ p.iouThrs,+ ious,+ ground_truth_instances,+ detected_instances,+ )+ self._evalImgs = None++ self._paramsEval = copy.deepcopy(self.params)+ toc = time.time()+ print("COCOeval_opt.evaluate() finished in {:0.2f} seconds.".format(toc - tic))+ # >>>> End of code differences with original COCO API++ def accumulate(self):+ """+ Accumulate per image evaluation results and store the result in self.eval. Does not+ support changing parameter settings from those used by self.evaluate()+ """+ print("Accumulating evaluation results...")+ tic = time.time()+ if not hasattr(self, "_evalImgs_cpp"):+ print("Please run evaluate() first")++ self.eval = _C.COCOevalAccumulate(self._paramsEval, self._evalImgs_cpp)++ # recall is num_iou_thresholds X num_categories X num_area_ranges X num_max_detections+ self.eval["recall"] = np.array(self.eval["recall"]).reshape(+ self.eval["counts"][:1] + self.eval["counts"][2:]+ )++ # precision and scores are num_iou_thresholds X num_recall_thresholds X num_categories X+ # num_area_ranges X num_max_detections+ self.eval["precision"] = np.array(self.eval["precision"]).reshape(+ self.eval["counts"]+ )+ self.eval["scores"] = np.array(self.eval["scores"]).reshape(self.eval["counts"])+ toc = time.time()+ print(+ "COCOeval_opt.accumulate() finished in {:0.2f} seconds.".format(toc - tic)+ )diff --git a/experimental/yolox/models/__init__.py b/experimental/yolox/models/__init__.pynew file mode 100644index 0000000..c4641a6--- /dev/null+++ b/experimental/yolox/models/__init__.py@@ -0,0 +1,10 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++from .darknet import CSPDarknet, Darknet+from .losses import IOUloss+from .yolo_fpn import YOLOFPN+from .yolo_head import YOLOXHead+from .yolo_pafpn import YOLOPAFPN+from .yolox import YOLOXdiff --git a/experimental/yolox/models/darknet.py b/experimental/yolox/models/darknet.pynew file mode 100644index 0000000..70c79f8--- /dev/null+++ b/experimental/yolox/models/darknet.py@@ -0,0 +1,179 @@+#!/usr/bin/env python+# -*- encoding: utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++from torch import nn++from .network_blocks import BaseConv, CSPLayer, DWConv, Focus, ResLayer, SPPBottleneck+++class Darknet(nn.Module):+ # number of blocks from dark2 to dark5.+ depth2blocks = {21: [1, 2, 2, 1], 53: [2, 8, 8, 4]}++ def __init__(+ self,+ depth,+ in_channels=3,+ stem_out_channels=32,+ out_features=("dark3", "dark4", "dark5"),+ ):+ """+ Args:+ depth (int): depth of darknet used in model, usually use [21, 53] for this param.+ in_channels (int): number of input channels, for example, use 3 for RGB image.+ stem_out_channels (int): number of output chanels of darknet stem.+ It decides channels of darknet layer2 to layer5.+ out_features (Tuple[str]): desired output layer name.+ """+ super().__init__()+ assert out_features, "please provide output features of Darknet"+ self.out_features = out_features+ self.stem = nn.Sequential(+ BaseConv(in_channels, stem_out_channels, ksize=3, stride=1, act="lrelu"),+ *self.make_group_layer(stem_out_channels, num_blocks=1, stride=2),+ )+ in_channels = stem_out_channels * 2 # 64++ num_blocks = Darknet.depth2blocks[depth]+ # create darknet with `stem_out_channels` and `num_blocks` layers.+ # to make model structure more clear, we don't use `for` statement in python.+ self.dark2 = nn.Sequential(+ *self.make_group_layer(in_channels, num_blocks[0], stride=2)+ )+ in_channels *= 2 # 128+ self.dark3 = nn.Sequential(+ *self.make_group_layer(in_channels, num_blocks[1], stride=2)+ )+ in_channels *= 2 # 256+ self.dark4 = nn.Sequential(+ *self.make_group_layer(in_channels, num_blocks[2], stride=2)+ )+ in_channels *= 2 # 512++ self.dark5 = nn.Sequential(+ *self.make_group_layer(in_channels, num_blocks[3], stride=2),+ *self.make_spp_block([in_channels, in_channels * 2], in_channels * 2),+ )++ def make_group_layer(self, in_channels: int, num_blocks: int, stride: int = 1):+ "starts with conv layer then has `num_blocks` `ResLayer`"+ return [+ BaseConv(in_channels, in_channels * 2, ksize=3, stride=stride, act="lrelu"),+ *[(ResLayer(in_channels * 2)) for _ in range(num_blocks)],+ ]++ def make_spp_block(self, filters_list, in_filters):+ m = nn.Sequential(+ *[+ BaseConv(in_filters, filters_list[0], 1, stride=1, act="lrelu"),+ BaseConv(filters_list[0], filters_list[1], 3, stride=1, act="lrelu"),+ SPPBottleneck(+ in_channels=filters_list[1],+ out_channels=filters_list[0],+ activation="lrelu",+ ),+ BaseConv(filters_list[0], filters_list[1], 3, stride=1, act="lrelu"),+ BaseConv(filters_list[1], filters_list[0], 1, stride=1, act="lrelu"),+ ]+ )+ return m++ def forward(self, x):+ outputs = {}+ x = self.stem(x)+ outputs["stem"] = x+ x = self.dark2(x)+ outputs["dark2"] = x+ x = self.dark3(x)+ outputs["dark3"] = x+ x = self.dark4(x)+ outputs["dark4"] = x+ x = self.dark5(x)+ outputs["dark5"] = x+ return {k: v for k, v in outputs.items() if k in self.out_features}+++class CSPDarknet(nn.Module):+ def __init__(+ self,+ dep_mul,+ wid_mul,+ out_features=("dark3", "dark4", "dark5"),+ depthwise=False,+ act="silu",+ ):+ super().__init__()+ assert out_features, "please provide output features of Darknet"+ self.out_features = out_features+ Conv = DWConv if depthwise else BaseConv++ base_channels = int(wid_mul * 64) # 64+ base_depth = max(round(dep_mul * 3), 1) # 3++ # stem+ self.stem = Focus(3, base_channels, ksize=3, act=act)++ # dark2+ self.dark2 = nn.Sequential(+ Conv(base_channels, base_channels * 2, 3, 2, act=act),+ CSPLayer(+ base_channels * 2,+ base_channels * 2,+ n=base_depth,+ depthwise=depthwise,+ act=act,+ ),+ )++ # dark3+ self.dark3 = nn.Sequential(+ Conv(base_channels * 2, base_channels * 4, 3, 2, act=act),+ CSPLayer(+ base_channels * 4,+ base_channels * 4,+ n=base_depth * 3,+ depthwise=depthwise,+ act=act,+ ),+ )++ # dark4+ self.dark4 = nn.Sequential(+ Conv(base_channels * 4, base_channels * 8, 3, 2, act=act),+ CSPLayer(+ base_channels * 8,+ base_channels * 8,+ n=base_depth * 3,+ depthwise=depthwise,+ act=act,+ ),+ )++ # dark5+ self.dark5 = nn.Sequential(+ Conv(base_channels * 8, base_channels * 16, 3, 2, act=act),+ SPPBottleneck(base_channels * 16, base_channels * 16, activation=act),+ CSPLayer(+ base_channels * 16,+ base_channels * 16,+ n=base_depth,+ shortcut=False,+ depthwise=depthwise,+ act=act,+ ),+ )++ def forward(self, x):+ outputs = {}+ x = self.stem(x)+ outputs["stem"] = x+ x = self.dark2(x)+ outputs["dark2"] = x+ x = self.dark3(x)+ outputs["dark3"] = x+ x = self.dark4(x)+ outputs["dark4"] = x+ x = self.dark5(x)+ outputs["dark5"] = x+ return {k: v for k, v in outputs.items() if k in self.out_features}diff --git a/experimental/yolox/models/losses.py b/experimental/yolox/models/losses.pynew file mode 100644index 0000000..a789eba--- /dev/null+++ b/experimental/yolox/models/losses.py@@ -0,0 +1,81 @@+#!/usr/bin/env python+# -*- encoding: utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import torch+import torch.nn as nn+import torch.nn.functional as F+++class IOUloss(nn.Module):+ def __init__(self, reduction="none", loss_type="iou"):+ super(IOUloss, self).__init__()+ self.reduction = reduction+ self.loss_type = loss_type++ def forward(self, pred, target):+ assert pred.shape[0] == target.shape[0]++ pred = pred.view(-1, 4)+ target = target.view(-1, 4)+ tl = torch.max(+ (pred[:, :2] - pred[:, 2:] / 2), (target[:, :2] - target[:, 2:] / 2)+ )+ br = torch.min(+ (pred[:, :2] + pred[:, 2:] / 2), (target[:, :2] + target[:, 2:] / 2)+ )++ area_p = torch.prod(pred[:, 2:], 1)+ area_g = torch.prod(target[:, 2:], 1)++ en = (tl < br).type(tl.type()).prod(dim=1)+ area_i = torch.prod(br - tl, 1) * en+ iou = (area_i) / (area_p + area_g - area_i + 1e-16)++ if self.loss_type == "iou":+ loss = 1 - iou ** 2+ elif self.loss_type == "giou":+ c_tl = torch.min(+ (pred[:, :2] - pred[:, 2:] / 2), (target[:, :2] - target[:, 2:] / 2)+ )+ c_br = torch.max(+ (pred[:, :2] + pred[:, 2:] / 2), (target[:, :2] + target[:, 2:] / 2)+ )+ area_c = torch.prod(c_br - c_tl, 1)+ giou = iou - (area_c - area_i) / area_c.clamp(1e-16)+ loss = 1 - giou.clamp(min=-1.0, max=1.0)++ if self.reduction == "mean":+ loss = loss.mean()+ elif self.reduction == "sum":+ loss = loss.sum()++ return loss+++def sigmoid_focal_loss(inputs, targets, num_boxes, alpha: float = 0.25, gamma: float = 2):+ """+ Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002.+ Args:+ inputs: A float tensor of arbitrary shape.+ The predictions for each example.+ targets: A float tensor with the same shape as inputs. Stores the binary+ classification label for each element in inputs+ (0 for the negative class and 1 for the positive class).+ alpha: (optional) Weighting factor in range (0,1) to balance+ positive vs negative examples. Default = -1 (no weighting).+ gamma: Exponent of the modulating factor (1 - p_t) to+ balance easy vs hard examples.+ Returns:+ Loss tensor+ """+ prob = inputs.sigmoid()+ ce_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction="none")+ p_t = prob * targets + (1 - prob) * (1 - targets)+ loss = ce_loss * ((1 - p_t) ** gamma)++ if alpha >= 0:+ alpha_t = alpha * targets + (1 - alpha) * (1 - targets)+ loss = alpha_t * loss+ #return loss.mean(0).sum() / num_boxes+ return loss.sum() / num_boxes\ No newline at end of filediff --git a/experimental/yolox/models/network_blocks.py b/experimental/yolox/models/network_blocks.pynew file mode 100644index 0000000..4bdb2ca--- /dev/null+++ b/experimental/yolox/models/network_blocks.py@@ -0,0 +1,210 @@+#!/usr/bin/env python+# -*- encoding: utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import torch+import torch.nn as nn+++class SiLU(nn.Module):+ """export-friendly version of nn.SiLU()"""++ @staticmethod+ def forward(x):+ return x * torch.sigmoid(x)+++def get_activation(name="silu", inplace=True):+ if name == "silu":+ module = nn.SiLU(inplace=inplace)+ elif name == "relu":+ module = nn.ReLU(inplace=inplace)+ elif name == "lrelu":+ module = nn.LeakyReLU(0.1, inplace=inplace)+ else:+ raise AttributeError("Unsupported act type: {}".format(name))+ return module+++class BaseConv(nn.Module):+ """A Conv2d -> Batchnorm -> silu/leaky relu block"""++ def __init__(+ self, in_channels, out_channels, ksize, stride, groups=1, bias=False, act="silu"+ ):+ super().__init__()+ # same padding+ pad = (ksize - 1) // 2+ self.conv = nn.Conv2d(+ in_channels,+ out_channels,+ kernel_size=ksize,+ stride=stride,+ padding=pad,+ groups=groups,+ bias=bias,+ )+ self.bn = nn.BatchNorm2d(out_channels)+ self.act = get_activation(act, inplace=True)++ def forward(self, x):+ return self.act(self.bn(self.conv(x)))++ def fuseforward(self, x):+ return self.act(self.conv(x))+++class DWConv(nn.Module):+ """Depthwise Conv + Conv"""++ def __init__(self, in_channels, out_channels, ksize, stride=1, act="silu"):+ super().__init__()+ self.dconv = BaseConv(+ in_channels,+ in_channels,+ ksize=ksize,+ stride=stride,+ groups=in_channels,+ act=act,+ )+ self.pconv = BaseConv(+ in_channels, out_channels, ksize=1, stride=1, groups=1, act=act+ )++ def forward(self, x):+ x = self.dconv(x)+ return self.pconv(x)+++class Bottleneck(nn.Module):+ # Standard bottleneck+ def __init__(+ self,+ in_channels,+ out_channels,+ shortcut=True,+ expansion=0.5,+ depthwise=False,+ act="silu",+ ):+ super().__init__()+ hidden_channels = int(out_channels * expansion)+ Conv = DWConv if depthwise else BaseConv+ self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act)+ self.conv2 = Conv(hidden_channels, out_channels, 3, stride=1, act=act)+ self.use_add = shortcut and in_channels == out_channels++ def forward(self, x):+ y = self.conv2(self.conv1(x))+ if self.use_add:+ y = y + x+ return y+++class ResLayer(nn.Module):+ "Residual layer with `in_channels` inputs."++ def __init__(self, in_channels: int):+ super().__init__()+ mid_channels = in_channels // 2+ self.layer1 = BaseConv(+ in_channels, mid_channels, ksize=1, stride=1, act="lrelu"+ )+ self.layer2 = BaseConv(+ mid_channels, in_channels, ksize=3, stride=1, act="lrelu"+ )++ def forward(self, x):+ out = self.layer2(self.layer1(x))+ return x + out+++class SPPBottleneck(nn.Module):+ """Spatial pyramid pooling layer used in YOLOv3-SPP"""++ def __init__(+ self, in_channels, out_channels, kernel_sizes=(5, 9, 13), activation="silu"+ ):+ super().__init__()+ hidden_channels = in_channels // 2+ self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=activation)+ self.m = nn.ModuleList(+ [+ nn.MaxPool2d(kernel_size=ks, stride=1, padding=ks // 2)+ for ks in kernel_sizes+ ]+ )+ conv2_channels = hidden_channels * (len(kernel_sizes) + 1)+ self.conv2 = BaseConv(conv2_channels, out_channels, 1, stride=1, act=activation)++ def forward(self, x):+ x = self.conv1(x)+ x = torch.cat([x] + [m(x) for m in self.m], dim=1)+ x = self.conv2(x)+ return x+++class CSPLayer(nn.Module):+ """C3 in yolov5, CSP Bottleneck with 3 convolutions"""++ def __init__(+ self,+ in_channels,+ out_channels,+ n=1,+ shortcut=True,+ expansion=0.5,+ depthwise=False,+ act="silu",+ ):+ """+ Args:+ in_channels (int): input channels.+ out_channels (int): output channels.+ n (int): number of Bottlenecks. Default value: 1.+ """+ # ch_in, ch_out, number, shortcut, groups, expansion+ super().__init__()+ hidden_channels = int(out_channels * expansion) # hidden channels+ self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act)+ self.conv2 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act)+ self.conv3 = BaseConv(2 * hidden_channels, out_channels, 1, stride=1, act=act)+ module_list = [+ Bottleneck(+ hidden_channels, hidden_channels, shortcut, 1.0, depthwise, act=act+ )+ for _ in range(n)+ ]+ self.m = nn.Sequential(*module_list)++ def forward(self, x):+ x_1 = self.conv1(x)+ x_2 = self.conv2(x)+ x_1 = self.m(x_1)+ x = torch.cat((x_1, x_2), dim=1)+ return self.conv3(x)+++class Focus(nn.Module):+ """Focus width and height information into channel space."""++ def __init__(self, in_channels, out_channels, ksize=1, stride=1, act="silu"):+ super().__init__()+ self.conv = BaseConv(in_channels * 4, out_channels, ksize, stride, act=act)++ def forward(self, x):+ # shape of x (b,c,w,h) -> y(b,4c,w/2,h/2)+ patch_top_left = x[..., ::2, ::2]+ patch_top_right = x[..., ::2, 1::2]+ patch_bot_left = x[..., 1::2, ::2]+ patch_bot_right = x[..., 1::2, 1::2]+ x = torch.cat(+ (+ patch_top_left,+ patch_bot_left,+ patch_top_right,+ patch_bot_right,+ ),+ dim=1,+ )+ return self.conv(x)diff --git a/experimental/yolox/models/yolo_fpn.py b/experimental/yolox/models/yolo_fpn.pynew file mode 100644index 0000000..8b3ba14--- /dev/null+++ b/experimental/yolox/models/yolo_fpn.py@@ -0,0 +1,84 @@+#!/usr/bin/env python+# -*- encoding: utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import torch+import torch.nn as nn++from .darknet import Darknet+from .network_blocks import BaseConv+++class YOLOFPN(nn.Module):+ """+ YOLOFPN module. Darknet 53 is the default backbone of this model.+ """++ def __init__(+ self,+ depth=53,+ in_features=["dark3", "dark4", "dark5"],+ ):+ super().__init__()++ self.backbone = Darknet(depth)+ self.in_features = in_features++ # out 1+ self.out1_cbl = self._make_cbl(512, 256, 1)+ self.out1 = self._make_embedding([256, 512], 512 + 256)++ # out 2+ self.out2_cbl = self._make_cbl(256, 128, 1)+ self.out2 = self._make_embedding([128, 256], 256 + 128)++ # upsample+ self.upsample = nn.Upsample(scale_factor=2, mode="nearest")++ def _make_cbl(self, _in, _out, ks):+ return BaseConv(_in, _out, ks, stride=1, act="lrelu")++ def _make_embedding(self, filters_list, in_filters):+ m = nn.Sequential(+ *[+ self._make_cbl(in_filters, filters_list[0], 1),+ self._make_cbl(filters_list[0], filters_list[1], 3),+ self._make_cbl(filters_list[1], filters_list[0], 1),+ self._make_cbl(filters_list[0], filters_list[1], 3),+ self._make_cbl(filters_list[1], filters_list[0], 1),+ ]+ )+ return m++ def load_pretrained_model(self, filename="./weights/darknet53.mix.pth"):+ with open(filename, "rb") as f:+ state_dict = torch.load(f, map_location="cpu")+ print("loading pretrained weights...")+ self.backbone.load_state_dict(state_dict)++ def forward(self, inputs):+ """+ Args:+ inputs (Tensor): input image.++ Returns:+ Tuple[Tensor]: FPN output features..+ """+ # backbone+ out_features = self.backbone(inputs)+ x2, x1, x0 = [out_features[f] for f in self.in_features]++ # yolo branch 1+ x1_in = self.out1_cbl(x0)+ x1_in = self.upsample(x1_in)+ x1_in = torch.cat([x1_in, x1], 1)+ out_dark4 = self.out1(x1_in)++ # yolo branch 2+ x2_in = self.out2_cbl(out_dark4)+ x2_in = self.upsample(x2_in)+ x2_in = torch.cat([x2_in, x2], 1)+ out_dark3 = self.out2(x2_in)++ outputs = (out_dark3, out_dark4, x0)+ return outputsdiff --git a/experimental/yolox/models/yolo_head.py b/experimental/yolox/models/yolo_head.pynew file mode 100644index 0000000..ba8238f--- /dev/null+++ b/experimental/yolox/models/yolo_head.py@@ -0,0 +1,660 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++from loguru import logger++import torch+import torch.nn as nn+import torch.nn.functional as F++from yolox.utils import bboxes_iou++import math++from .losses import IOUloss+from .network_blocks import BaseConv, DWConv+++class YOLOXHead(nn.Module):+ def __init__(+ self,+ num_classes,+ width=1.0,+ strides=[8, 16, 32],+ in_channels=[256, 512, 1024],+ act="silu",+ depthwise=False,+ ):+ """+ Args:+ act (str): activation type of conv. Defalut value: "silu".+ depthwise (bool): wheather apply depthwise conv in conv branch. Defalut value: False.+ """+ super().__init__()++ self.n_anchors = 1+ self.num_classes = num_classes+ self.decode_in_inference = True # for deploy, set to False++ self.cls_convs = nn.ModuleList()+ self.reg_convs = nn.ModuleList()+ self.cls_preds = nn.ModuleList()+ self.reg_preds = nn.ModuleList()+ self.obj_preds = nn.ModuleList()+ self.stems = nn.ModuleList()+ Conv = DWConv if depthwise else BaseConv++ for i in range(len(in_channels)):+ self.stems.append(+ BaseConv(+ in_channels=int(in_channels[i] * width),+ out_channels=int(256 * width),+ ksize=1,+ stride=1,+ act=act,+ )+ )+ self.cls_convs.append(+ nn.Sequential(+ *[+ Conv(+ in_channels=int(256 * width),+ out_channels=int(256 * width),+ ksize=3,+ stride=1,+ act=act,+ ),+ Conv(+ in_channels=int(256 * width),+ out_channels=int(256 * width),+ ksize=3,+ stride=1,+ act=act,+ ),+ ]+ )+ )+ self.reg_convs.append(+ nn.Sequential(+ *[+ Conv(+ in_channels=int(256 * width),+ out_channels=int(256 * width),+ ksize=3,+ stride=1,+ act=act,+ ),+ Conv(+ in_channels=int(256 * width),+ out_channels=int(256 * width),+ ksize=3,+ stride=1,+ act=act,+ ),+ ]+ )+ )+ self.cls_preds.append(+ nn.Conv2d(+ in_channels=int(256 * width),+ out_channels=self.n_anchors * self.num_classes,+ kernel_size=1,+ stride=1,+ padding=0,+ )+ )+ self.reg_preds.append(+ nn.Conv2d(+ in_channels=int(256 * width),+ out_channels=4,+ kernel_size=1,+ stride=1,+ padding=0,+ )+ )+ self.obj_preds.append(+ nn.Conv2d(+ in_channels=int(256 * width),+ out_channels=self.n_anchors * 1,+ kernel_size=1,+ stride=1,+ padding=0,+ )+ )++ self.use_l1 = False+ self.l1_loss = nn.L1Loss(reduction="none")+ self.bcewithlog_loss = nn.BCEWithLogitsLoss(reduction="none")+ self.iou_loss = IOUloss(reduction="none")+ self.strides = strides+ self.grids = [torch.zeros(1)] * len(in_channels)+ self.expanded_strides = [None] * len(in_channels)++ def initialize_biases(self, prior_prob):+ for conv in self.cls_preds:+ b = conv.bias.view(self.n_anchors, -1)+ b.data.fill_(-math.log((1 - prior_prob) / prior_prob))+ conv.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)++ for conv in self.obj_preds:+ b = conv.bias.view(self.n_anchors, -1)+ b.data.fill_(-math.log((1 - prior_prob) / prior_prob))+ conv.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)++ def forward(self, xin, labels=None, imgs=None):+ outputs = []+ origin_preds = []+ x_shifts = []+ y_shifts = []+ expanded_strides = []++ for k, (cls_conv, reg_conv, stride_this_level, x) in enumerate(+ zip(self.cls_convs, self.reg_convs, self.strides, xin)+ ):+ x = self.stems[k](x)+ cls_x = x+ reg_x = x++ cls_feat = cls_conv(cls_x)+ cls_output = self.cls_preds[k](cls_feat)++ reg_feat = reg_conv(reg_x)+ reg_output = self.reg_preds[k](reg_feat)+ obj_output = self.obj_preds[k](reg_feat)++ if self.training:+ output = torch.cat([reg_output, obj_output, cls_output], 1)+ output, grid = self.get_output_and_grid(+ output, k, stride_this_level, xin[0].type()+ )+ x_shifts.append(grid[:, :, 0])+ y_shifts.append(grid[:, :, 1])+ expanded_strides.append(+ torch.zeros(1, grid.shape[1])+ .fill_(stride_this_level)+ .type_as(xin[0])+ )+ if self.use_l1:+ batch_size = reg_output.shape[0]+ hsize, wsize = reg_output.shape[-2:]+ reg_output = reg_output.view(+ batch_size, self.n_anchors, 4, hsize, wsize+ )+ reg_output = reg_output.permute(0, 1, 3, 4, 2).reshape(+ batch_size, -1, 4+ )+ origin_preds.append(reg_output.clone())++ else:+ output = torch.cat(+ [reg_output, obj_output.sigmoid(), cls_output.sigmoid()], 1+ )++ outputs.append(output)++ if self.training:+ return self.get_losses(+ imgs,+ x_shifts,+ y_shifts,+ expanded_strides,+ labels,+ torch.cat(outputs, 1),+ origin_preds,+ dtype=xin[0].dtype,+ )+ else:+ self.hw = [x.shape[-2:] for x in outputs]+ # [batch, n_anchors_all, 85]+ outputs = torch.cat(+ [x.flatten(start_dim=2) for x in outputs], dim=2+ ).permute(0, 2, 1)+ if self.decode_in_inference:+ return self.decode_outputs(outputs, dtype=xin[0].type())+ else:+ return outputs++ def get_output_and_grid(self, output, k, stride, dtype):+ grid = self.grids[k]++ batch_size = output.shape[0]+ n_ch = 5 + self.num_classes+ hsize, wsize = output.shape[-2:]+ if grid.shape[2:4] != output.shape[2:4]:+ yv, xv = torch.meshgrid([torch.arange(hsize), torch.arange(wsize)])+ grid = torch.stack((xv, yv), 2).view(1, 1, hsize, wsize, 2).type(dtype)+ self.grids[k] = grid++ output = output.view(batch_size, self.n_anchors, n_ch, hsize, wsize)+ output = output.permute(0, 1, 3, 4, 2).reshape(+ batch_size, self.n_anchors * hsize * wsize, -1+ )+ grid = grid.view(1, -1, 2)+ output[..., :2] = (output[..., :2] + grid) * stride+ output[..., 2:4] = torch.exp(output[..., 2:4]) * stride+ return output, grid++ def decode_outputs(self, outputs, dtype):+ grids = []+ strides = []+ for (hsize, wsize), stride in zip(self.hw, self.strides):+ yv, xv = torch.meshgrid([torch.arange(hsize), torch.arange(wsize)])+ grid = torch.stack((xv, yv), 2).view(1, -1, 2)+ grids.append(grid)+ shape = grid.shape[:2]+ strides.append(torch.full((*shape, 1), stride))++ grids = torch.cat(grids, dim=1).type(dtype)+ strides = torch.cat(strides, dim=1).type(dtype)++ outputs[..., :2] = (outputs[..., :2] + grids) * strides+ outputs[..., 2:4] = torch.exp(outputs[..., 2:4]) * strides+ return outputs++ def get_losses(+ self,+ imgs,+ x_shifts,+ y_shifts,+ expanded_strides,+ labels,+ outputs,+ origin_preds,+ dtype,+ ):+ bbox_preds = outputs[:, :, :4] # [batch, n_anchors_all, 4]+ obj_preds = outputs[:, :, 4].unsqueeze(-1) # [batch, n_anchors_all, 1]+ cls_preds = outputs[:, :, 5:] # [batch, n_anchors_all, n_cls]++ # calculate targets+ mixup = labels.shape[2] > 5+ if mixup:+ label_cut = labels[..., :5]+ else:+ label_cut = labels+ nlabel = (label_cut.sum(dim=2) > 0).sum(dim=1) # number of objects++ total_num_anchors = outputs.shape[1]+ x_shifts = torch.cat(x_shifts, 1) # [1, n_anchors_all]+ y_shifts = torch.cat(y_shifts, 1) # [1, n_anchors_all]+ expanded_strides = torch.cat(expanded_strides, 1)+ if self.use_l1:+ origin_preds = torch.cat(origin_preds, 1)++ cls_targets = []+ reg_targets = []+ l1_targets = []+ obj_targets = []+ fg_masks = []++ num_fg = 0.0+ num_gts = 0.0++ for batch_idx in range(outputs.shape[0]):+ num_gt = int(nlabel[batch_idx])+ num_gts += num_gt+ if num_gt == 0:+ cls_target = outputs.new_zeros((0, self.num_classes))+ reg_target = outputs.new_zeros((0, 4))+ l1_target = outputs.new_zeros((0, 4))+ obj_target = outputs.new_zeros((total_num_anchors, 1))+ fg_mask = outputs.new_zeros(total_num_anchors).bool()+ else:+ gt_bboxes_per_image = labels[batch_idx, :num_gt, 1:5]+ gt_classes = labels[batch_idx, :num_gt, 0]+ bboxes_preds_per_image = bbox_preds[batch_idx]++ try:+ (+ gt_matched_classes,+ fg_mask,+ pred_ious_this_matching,+ matched_gt_inds,+ num_fg_img,+ ) = self.get_assignments( # noqa+ batch_idx,+ num_gt,+ total_num_anchors,+ gt_bboxes_per_image,+ gt_classes,+ bboxes_preds_per_image,+ expanded_strides,+ x_shifts,+ y_shifts,+ cls_preds,+ bbox_preds,+ obj_preds,+ labels,+ imgs,+ )+ except RuntimeError:+ logger.info(+ "OOM RuntimeError is raised due to the huge memory cost during label assignment. \+ CPU mode is applied in this batch. If you want to avoid this issue, \+ try to reduce the batch size or image size."+ )+ print("OOM RuntimeError is raised due to the huge memory cost during label assignment. \+ CPU mode is applied in this batch. If you want to avoid this issue, \+ try to reduce the batch size or image size.")+ torch.cuda.empty_cache()+ (+ gt_matched_classes,+ fg_mask,+ pred_ious_this_matching,+ matched_gt_inds,+ num_fg_img,+ ) = self.get_assignments( # noqa+ batch_idx,+ num_gt,+ total_num_anchors,+ gt_bboxes_per_image,+ gt_classes,+ bboxes_preds_per_image,+ expanded_strides,+ x_shifts,+ y_shifts,+ cls_preds,+ bbox_preds,+ obj_preds,+ labels,+ imgs,+ "cpu",+ )+++ torch.cuda.empty_cache()+ num_fg += num_fg_img++ cls_target = F.one_hot(+ gt_matched_classes.to(torch.int64), self.num_classes+ ) * pred_ious_this_matching.unsqueeze(-1)+ obj_target = fg_mask.unsqueeze(-1)+ reg_target = gt_bboxes_per_image[matched_gt_inds]++ if self.use_l1:+ l1_target = self.get_l1_target(+ outputs.new_zeros((num_fg_img, 4)),+ gt_bboxes_per_image[matched_gt_inds],+ expanded_strides[0][fg_mask],+ x_shifts=x_shifts[0][fg_mask],+ y_shifts=y_shifts[0][fg_mask],+ )++ cls_targets.append(cls_target)+ reg_targets.append(reg_target)+ obj_targets.append(obj_target.to(dtype))+ fg_masks.append(fg_mask)+ if self.use_l1:+ l1_targets.append(l1_target)++ cls_targets = torch.cat(cls_targets, 0)+ reg_targets = torch.cat(reg_targets, 0)+ obj_targets = torch.cat(obj_targets, 0)+ fg_masks = torch.cat(fg_masks, 0)+ if self.use_l1:+ l1_targets = torch.cat(l1_targets, 0)++ num_fg = max(num_fg, 1)+ loss_iou = (+ self.iou_loss(bbox_preds.view(-1, 4)[fg_masks], reg_targets)+ ).sum() / num_fg+ loss_obj = (+ self.bcewithlog_loss(obj_preds.view(-1, 1), obj_targets)+ ).sum() / num_fg+ loss_cls = (+ self.bcewithlog_loss(+ cls_preds.view(-1, self.num_classes)[fg_masks], cls_targets+ )+ ).sum() / num_fg+ if self.use_l1:+ loss_l1 = (+ self.l1_loss(origin_preds.view(-1, 4)[fg_masks], l1_targets)+ ).sum() / num_fg+ else:+ loss_l1 = 0.0++ reg_weight = 5.0+ loss = reg_weight * loss_iou + loss_obj + loss_cls + loss_l1++ return (+ loss,+ reg_weight * loss_iou,+ loss_obj,+ loss_cls,+ loss_l1,+ num_fg / max(num_gts, 1),+ )++ def get_l1_target(self, l1_target, gt, stride, x_shifts, y_shifts, eps=1e-8):+ l1_target[:, 0] = gt[:, 0] / stride - x_shifts+ l1_target[:, 1] = gt[:, 1] / stride - y_shifts+ l1_target[:, 2] = torch.log(gt[:, 2] / stride + eps)+ l1_target[:, 3] = torch.log(gt[:, 3] / stride + eps)+ return l1_target++ @torch.no_grad()+ def get_assignments(+ self,+ batch_idx,+ num_gt,+ total_num_anchors,+ gt_bboxes_per_image,+ gt_classes,+ bboxes_preds_per_image,+ expanded_strides,+ x_shifts,+ y_shifts,+ cls_preds,+ bbox_preds,+ obj_preds,+ labels,+ imgs,+ mode="gpu",+ ):++ if mode == "cpu":+ print("------------CPU Mode for This Batch-------------")+ gt_bboxes_per_image = gt_bboxes_per_image.cpu().float()+ bboxes_preds_per_image = bboxes_preds_per_image.cpu().float()+ gt_classes = gt_classes.cpu().float()+ expanded_strides = expanded_strides.cpu().float()+ x_shifts = x_shifts.cpu()+ y_shifts = y_shifts.cpu()++ img_size = imgs.shape[2:]+ fg_mask, is_in_boxes_and_center = self.get_in_boxes_info(+ gt_bboxes_per_image,+ expanded_strides,+ x_shifts,+ y_shifts,+ total_num_anchors,+ num_gt,+ img_size+ )++ bboxes_preds_per_image = bboxes_preds_per_image[fg_mask]+ cls_preds_ = cls_preds[batch_idx][fg_mask]+ obj_preds_ = obj_preds[batch_idx][fg_mask]+ num_in_boxes_anchor = bboxes_preds_per_image.shape[0]++ if mode == "cpu":+ gt_bboxes_per_image = gt_bboxes_per_image.cpu()+ bboxes_preds_per_image = bboxes_preds_per_image.cpu()++ pair_wise_ious = bboxes_iou(gt_bboxes_per_image, bboxes_preds_per_image, False)++ gt_cls_per_image = (+ F.one_hot(gt_classes.to(torch.int64), self.num_classes)+ .float()+ .unsqueeze(1)+ .repeat(1, num_in_boxes_anchor, 1)+ )+ pair_wise_ious_loss = -torch.log(pair_wise_ious + 1e-8)++ if mode == "cpu":+ cls_preds_, obj_preds_ = cls_preds_.cpu(), obj_preds_.cpu()++ with torch.cuda.amp.autocast(enabled=False):+ cls_preds_ = (+ cls_preds_.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()+ * obj_preds_.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()+ )+ pair_wise_cls_loss = F.binary_cross_entropy(+ cls_preds_.sqrt_(), gt_cls_per_image, reduction="none"+ ).sum(-1)+ del cls_preds_++ cost = (+ pair_wise_cls_loss+ + 3.0 * pair_wise_ious_loss+ + 100000.0 * (~is_in_boxes_and_center)+ )++ (+ num_fg,+ gt_matched_classes,+ pred_ious_this_matching,+ matched_gt_inds,+ ) = self.dynamic_k_matching(cost, pair_wise_ious, gt_classes, num_gt, fg_mask)+ del pair_wise_cls_loss, cost, pair_wise_ious, pair_wise_ious_loss++ if mode == "cpu":+ gt_matched_classes = gt_matched_classes.cuda()+ fg_mask = fg_mask.cuda()+ pred_ious_this_matching = pred_ious_this_matching.cuda()+ matched_gt_inds = matched_gt_inds.cuda()++ return (+ gt_matched_classes,+ fg_mask,+ pred_ious_this_matching,+ matched_gt_inds,+ num_fg,+ )++ def get_in_boxes_info(+ self,+ gt_bboxes_per_image,+ expanded_strides,+ x_shifts,+ y_shifts,+ total_num_anchors,+ num_gt,+ img_size+ ):+ expanded_strides_per_image = expanded_strides[0]+ x_shifts_per_image = x_shifts[0] * expanded_strides_per_image+ y_shifts_per_image = y_shifts[0] * expanded_strides_per_image+ x_centers_per_image = (+ (x_shifts_per_image + 0.5 * expanded_strides_per_image)+ .unsqueeze(0)+ .repeat(num_gt, 1)+ ) # [n_anchor] -> [n_gt, n_anchor]+ y_centers_per_image = (+ (y_shifts_per_image + 0.5 * expanded_strides_per_image)+ .unsqueeze(0)+ .repeat(num_gt, 1)+ )++ gt_bboxes_per_image_l = (+ (gt_bboxes_per_image[:, 0] - 0.5 * gt_bboxes_per_image[:, 2])+ .unsqueeze(1)+ .repeat(1, total_num_anchors)+ )+ gt_bboxes_per_image_r = (+ (gt_bboxes_per_image[:, 0] + 0.5 * gt_bboxes_per_image[:, 2])+ .unsqueeze(1)+ .repeat(1, total_num_anchors)+ )+ gt_bboxes_per_image_t = (+ (gt_bboxes_per_image[:, 1] - 0.5 * gt_bboxes_per_image[:, 3])+ .unsqueeze(1)+ .repeat(1, total_num_anchors)+ )+ gt_bboxes_per_image_b = (+ (gt_bboxes_per_image[:, 1] + 0.5 * gt_bboxes_per_image[:, 3])+ .unsqueeze(1)+ .repeat(1, total_num_anchors)+ )++ b_l = x_centers_per_image - gt_bboxes_per_image_l+ b_r = gt_bboxes_per_image_r - x_centers_per_image+ b_t = y_centers_per_image - gt_bboxes_per_image_t+ b_b = gt_bboxes_per_image_b - y_centers_per_image+ bbox_deltas = torch.stack([b_l, b_t, b_r, b_b], 2)++ is_in_boxes = bbox_deltas.min(dim=-1).values > 0.0+ is_in_boxes_all = is_in_boxes.sum(dim=0) > 0+ # in fixed center++ center_radius = 2.5+ # clip center inside image+ gt_bboxes_per_image_clip = gt_bboxes_per_image[:, 0:2].clone()+ gt_bboxes_per_image_clip[:, 0] = torch.clamp(gt_bboxes_per_image_clip[:, 0], min=0, max=img_size[1])+ gt_bboxes_per_image_clip[:, 1] = torch.clamp(gt_bboxes_per_image_clip[:, 1], min=0, max=img_size[0])++ gt_bboxes_per_image_l = (gt_bboxes_per_image_clip[:, 0]).unsqueeze(1).repeat(+ 1, total_num_anchors+ ) - center_radius * expanded_strides_per_image.unsqueeze(0)+ gt_bboxes_per_image_r = (gt_bboxes_per_image_clip[:, 0]).unsqueeze(1).repeat(+ 1, total_num_anchors+ ) + center_radius * expanded_strides_per_image.unsqueeze(0)+ gt_bboxes_per_image_t = (gt_bboxes_per_image_clip[:, 1]).unsqueeze(1).repeat(+ 1, total_num_anchors+ ) - center_radius * expanded_strides_per_image.unsqueeze(0)+ gt_bboxes_per_image_b = (gt_bboxes_per_image_clip[:, 1]).unsqueeze(1).repeat(+ 1, total_num_anchors+ ) + center_radius * expanded_strides_per_image.unsqueeze(0)++ c_l = x_centers_per_image - gt_bboxes_per_image_l+ c_r = gt_bboxes_per_image_r - x_centers_per_image+ c_t = y_centers_per_image - gt_bboxes_per_image_t+ c_b = gt_bboxes_per_image_b - y_centers_per_image+ center_deltas = torch.stack([c_l, c_t, c_r, c_b], 2)+ is_in_centers = center_deltas.min(dim=-1).values > 0.0+ is_in_centers_all = is_in_centers.sum(dim=0) > 0++ # in boxes and in centers+ is_in_boxes_anchor = is_in_boxes_all | is_in_centers_all++ is_in_boxes_and_center = (+ is_in_boxes[:, is_in_boxes_anchor] & is_in_centers[:, is_in_boxes_anchor]+ )+ del gt_bboxes_per_image_clip+ return is_in_boxes_anchor, is_in_boxes_and_center++ def dynamic_k_matching(self, cost, pair_wise_ious, gt_classes, num_gt, fg_mask):+ # Dynamic K+ # ---------------------------------------------------------------+ matching_matrix = torch.zeros_like(cost)++ ious_in_boxes_matrix = pair_wise_ious+ n_candidate_k = min(10, ious_in_boxes_matrix.size(1))+ topk_ious, _ = torch.topk(ious_in_boxes_matrix, n_candidate_k, dim=1)+ dynamic_ks = torch.clamp(topk_ious.sum(1).int(), min=1)+ for gt_idx in range(num_gt):+ _, pos_idx = torch.topk(+ cost[gt_idx], k=dynamic_ks[gt_idx].item(), largest=False+ )+ matching_matrix[gt_idx][pos_idx] = 1.0++ del topk_ious, dynamic_ks, pos_idx++ anchor_matching_gt = matching_matrix.sum(0)+ if (anchor_matching_gt > 1).sum() > 0:+ cost_min, cost_argmin = torch.min(cost[:, anchor_matching_gt > 1], dim=0)+ matching_matrix[:, anchor_matching_gt > 1] *= 0.0+ matching_matrix[cost_argmin, anchor_matching_gt > 1] = 1.0+ fg_mask_inboxes = matching_matrix.sum(0) > 0.0+ num_fg = fg_mask_inboxes.sum().item()++ fg_mask[fg_mask.clone()] = fg_mask_inboxes++ matched_gt_inds = matching_matrix[:, fg_mask_inboxes].argmax(0)+ gt_matched_classes = gt_classes[matched_gt_inds]++ pred_ious_this_matching = (matching_matrix * pair_wise_ious).sum(0)[+ fg_mask_inboxes+ ]+ return num_fg, gt_matched_classes, pred_ious_this_matching, matched_gt_indsdiff --git a/experimental/yolox/models/yolo_pafpn.py b/experimental/yolox/models/yolo_pafpn.pynew file mode 100644index 0000000..c419de3--- /dev/null+++ b/experimental/yolox/models/yolo_pafpn.py@@ -0,0 +1,116 @@+#!/usr/bin/env python+# -*- encoding: utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import torch+import torch.nn as nn++from .darknet import CSPDarknet+from .network_blocks import BaseConv, CSPLayer, DWConv+++class YOLOPAFPN(nn.Module):+ """+ YOLOv3 model. Darknet 53 is the default backbone of this model.+ """++ def __init__(+ self,+ depth=1.0,+ width=1.0,+ in_features=("dark3", "dark4", "dark5"),+ in_channels=[256, 512, 1024],+ depthwise=False,+ act="silu",+ ):+ super().__init__()+ self.backbone = CSPDarknet(depth, width, depthwise=depthwise, act=act)+ self.in_features = in_features+ self.in_channels = in_channels+ Conv = DWConv if depthwise else BaseConv++ self.upsample = nn.Upsample(scale_factor=2, mode="nearest")+ self.lateral_conv0 = BaseConv(+ int(in_channels[2] * width), int(in_channels[1] * width), 1, 1, act=act+ )+ self.C3_p4 = CSPLayer(+ int(2 * in_channels[1] * width),+ int(in_channels[1] * width),+ round(3 * depth),+ False,+ depthwise=depthwise,+ act=act,+ ) # cat++ self.reduce_conv1 = BaseConv(+ int(in_channels[1] * width), int(in_channels[0] * width), 1, 1, act=act+ )+ self.C3_p3 = CSPLayer(+ int(2 * in_channels[0] * width),+ int(in_channels[0] * width),+ round(3 * depth),+ False,+ depthwise=depthwise,+ act=act,+ )++ # bottom-up conv+ self.bu_conv2 = Conv(+ int(in_channels[0] * width), int(in_channels[0] * width), 3, 2, act=act+ )+ self.C3_n3 = CSPLayer(+ int(2 * in_channels[0] * width),+ int(in_channels[1] * width),+ round(3 * depth),+ False,+ depthwise=depthwise,+ act=act,+ )++ # bottom-up conv+ self.bu_conv1 = Conv(+ int(in_channels[1] * width), int(in_channels[1] * width), 3, 2, act=act+ )+ self.C3_n4 = CSPLayer(+ int(2 * in_channels[1] * width),+ int(in_channels[2] * width),+ round(3 * depth),+ False,+ depthwise=depthwise,+ act=act,+ )++ def forward(self, input):+ """+ Args:+ inputs: input images.++ Returns:+ Tuple[Tensor]: FPN feature.+ """++ # backbone+ out_features = self.backbone(input)+ features = [out_features[f] for f in self.in_features]+ [x2, x1, x0] = features++ fpn_out0 = self.lateral_conv0(x0) # 1024->512/32+ f_out0 = self.upsample(fpn_out0) # 512/16+ f_out0 = torch.cat([f_out0, x1], 1) # 512->1024/16+ f_out0 = self.C3_p4(f_out0) # 1024->512/16++ fpn_out1 = self.reduce_conv1(f_out0) # 512->256/16+ f_out1 = self.upsample(fpn_out1) # 256/8+ f_out1 = torch.cat([f_out1, x2], 1) # 256->512/8+ pan_out2 = self.C3_p3(f_out1) # 512->256/8++ p_out1 = self.bu_conv2(pan_out2) # 256->256/16+ p_out1 = torch.cat([p_out1, fpn_out1], 1) # 256->512/16+ pan_out1 = self.C3_n3(p_out1) # 512->512/16++ p_out0 = self.bu_conv1(pan_out1) # 512->512/32+ p_out0 = torch.cat([p_out0, fpn_out0], 1) # 512->1024/32+ pan_out0 = self.C3_n4(p_out0) # 1024->1024/32++ outputs = (pan_out2, pan_out1, pan_out0)+ return outputsdiff --git a/experimental/yolox/models/yolox.py b/experimental/yolox/models/yolox.pynew file mode 100644index 0000000..2f1fa1b--- /dev/null+++ b/experimental/yolox/models/yolox.py@@ -0,0 +1,48 @@+#!/usr/bin/env python+# -*- encoding: utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import torch.nn as nn++from .yolo_head import YOLOXHead+from .yolo_pafpn import YOLOPAFPN+++class YOLOX(nn.Module):+ """+ YOLOX model module. The module list is defined by create_yolov3_modules function.+ The network returns loss values from three YOLO layers during training+ and detection results during test.+ """++ def __init__(self, backbone=None, head=None):+ super().__init__()+ if backbone is None:+ backbone = YOLOPAFPN()+ if head is None:+ head = YOLOXHead(80)++ self.backbone = backbone+ self.head = head++ def forward(self, x, targets=None):+ # fpn output content features of [dark3, dark4, dark5]+ fpn_outs = self.backbone(x)++ if self.training:+ assert targets is not None+ loss, iou_loss, conf_loss, cls_loss, l1_loss, num_fg = self.head(+ fpn_outs, targets, x+ )+ outputs = {+ "total_loss": loss,+ "iou_loss": iou_loss,+ "l1_loss": l1_loss,+ "conf_loss": conf_loss,+ "cls_loss": cls_loss,+ "num_fg": num_fg,+ }+ else:+ outputs = self.head(fpn_outs)++ return outputsdiff --git a/experimental/yolox/motdt_tracker/basetrack.py b/experimental/yolox/motdt_tracker/basetrack.pynew file mode 100644index 0000000..88b16ee--- /dev/null+++ b/experimental/yolox/motdt_tracker/basetrack.py@@ -0,0 +1,56 @@+import numpy as np+from collections import OrderedDict+++class TrackState(object):+ New = 0+ Tracked = 1+ Lost = 2+ Removed = 3+ Replaced = 4+++class BaseTrack(object):+ _count = 0++ track_id = 0+ is_activated = False+ state = TrackState.New++ history = OrderedDict()+ features = []+ curr_feature = None+ score = 0+ start_frame = 0+ frame_id = 0+ time_since_update = 0++ # multi-camera+ location = (np.inf, np.inf)++ @property+ def end_frame(self):+ return self.frame_id++ @staticmethod+ def next_id():+ BaseTrack._count += 1+ return BaseTrack._count++ def activate(self, *args):+ raise NotImplementedError++ def predict(self):+ raise NotImplementedError++ def update(self, *args, **kwargs):+ raise NotImplementedError++ def mark_lost(self):+ self.state = TrackState.Lost++ def mark_removed(self):+ self.state = TrackState.Removed++ def mark_replaced(self):+ self.state = TrackState.Replaceddiff --git a/experimental/yolox/motdt_tracker/kalman_filter.py b/experimental/yolox/motdt_tracker/kalman_filter.pynew file mode 100644index 0000000..deda8a2--- /dev/null+++ b/experimental/yolox/motdt_tracker/kalman_filter.py@@ -0,0 +1,270 @@+# vim: expandtab:ts=4:sw=4+import numpy as np+import scipy.linalg+++"""+Table for the 0.95 quantile of the chi-square distribution with N degrees of+freedom (contains values for N=1, ..., 9). Taken from MATLAB/Octave's chi2inv+function and used as Mahalanobis gating threshold.+"""+chi2inv95 = {+ 1: 3.8415,+ 2: 5.9915,+ 3: 7.8147,+ 4: 9.4877,+ 5: 11.070,+ 6: 12.592,+ 7: 14.067,+ 8: 15.507,+ 9: 16.919}+++class KalmanFilter(object):+ """+ A simple Kalman filter for tracking bounding boxes in image space.++ The 8-dimensional state space++ x, y, a, h, vx, vy, va, vh++ contains the bounding box center position (x, y), aspect ratio a, height h,+ and their respective velocities.++ Object motion follows a constant velocity model. The bounding box location+ (x, y, a, h) is taken as direct observation of the state space (linear+ observation model).++ """++ def __init__(self):+ ndim, dt = 4, 1.++ # Create Kalman filter model matrices.+ self._motion_mat = np.eye(2 * ndim, 2 * ndim)+ for i in range(ndim):+ self._motion_mat[i, ndim + i] = dt+ self._update_mat = np.eye(ndim, 2 * ndim)++ # Motion and observation uncertainty are chosen relative to the current+ # state estimate. These weights control the amount of uncertainty in+ # the model. This is a bit hacky.+ self._std_weight_position = 1. / 20+ self._std_weight_velocity = 1. / 160++ def initiate(self, measurement):+ """Create track from unassociated measurement.++ Parameters+ ----------+ measurement : ndarray+ Bounding box coordinates (x, y, a, h) with center position (x, y),+ aspect ratio a, and height h.++ Returns+ -------+ (ndarray, ndarray)+ Returns the mean vector (8 dimensional) and covariance matrix (8x8+ dimensional) of the new track. Unobserved velocities are initialized+ to 0 mean.++ """+ mean_pos = measurement+ mean_vel = np.zeros_like(mean_pos)+ mean = np.r_[mean_pos, mean_vel]++ std = [+ 2 * self._std_weight_position * measurement[3],+ 2 * self._std_weight_position * measurement[3],+ 1e-2,+ 2 * self._std_weight_position * measurement[3],+ 10 * self._std_weight_velocity * measurement[3],+ 10 * self._std_weight_velocity * measurement[3],+ 1e-5,+ 10 * self._std_weight_velocity * measurement[3]]+ covariance = np.diag(np.square(std))+ return mean, covariance++ def predict(self, mean, covariance):+ """Run Kalman filter prediction step.++ Parameters+ ----------+ mean : ndarray+ The 8 dimensional mean vector of the object state at the previous+ time step.+ covariance : ndarray+ The 8x8 dimensional covariance matrix of the object state at the+ previous time step.++ Returns+ -------+ (ndarray, ndarray)+ Returns the mean vector and covariance matrix of the predicted+ state. Unobserved velocities are initialized to 0 mean.++ """+ std_pos = [+ self._std_weight_position * mean[3],+ self._std_weight_position * mean[3],+ 1e-2,+ self._std_weight_position * mean[3]]+ std_vel = [+ self._std_weight_velocity * mean[3],+ self._std_weight_velocity * mean[3],+ 1e-5,+ self._std_weight_velocity * mean[3]]+ motion_cov = np.diag(np.square(np.r_[std_pos, std_vel]))++ #mean = np.dot(self._motion_mat, mean)+ mean = np.dot(mean, self._motion_mat.T)+ covariance = np.linalg.multi_dot((+ self._motion_mat, covariance, self._motion_mat.T)) + motion_cov++ return mean, covariance++ def project(self, mean, covariance):+ """Project state distribution to measurement space.++ Parameters+ ----------+ mean : ndarray+ The state's mean vector (8 dimensional array).+ covariance : ndarray+ The state's covariance matrix (8x8 dimensional).++ Returns+ -------+ (ndarray, ndarray)+ Returns the projected mean and covariance matrix of the given state+ estimate.++ """+ std = [+ self._std_weight_position * mean[3],+ self._std_weight_position * mean[3],+ 1e-1,+ self._std_weight_position * mean[3]]+ innovation_cov = np.diag(np.square(std))++ mean = np.dot(self._update_mat, mean)+ covariance = np.linalg.multi_dot((+ self._update_mat, covariance, self._update_mat.T))+ return mean, covariance + innovation_cov++ def multi_predict(self, mean, covariance):+ """Run Kalman filter prediction step (Vectorized version).+ Parameters+ ----------+ mean : ndarray+ The Nx8 dimensional mean matrix of the object states at the previous+ time step.+ covariance : ndarray+ The Nx8x8 dimensional covariance matrics of the object states at the+ previous time step.+ Returns+ -------+ (ndarray, ndarray)+ Returns the mean vector and covariance matrix of the predicted+ state. Unobserved velocities are initialized to 0 mean.+ """+ std_pos = [+ self._std_weight_position * mean[:, 3],+ self._std_weight_position * mean[:, 3],+ 1e-2 * np.ones_like(mean[:, 3]),+ self._std_weight_position * mean[:, 3]]+ std_vel = [+ self._std_weight_velocity * mean[:, 3],+ self._std_weight_velocity * mean[:, 3],+ 1e-5 * np.ones_like(mean[:, 3]),+ self._std_weight_velocity * mean[:, 3]]+ sqr = np.square(np.r_[std_pos, std_vel]).T++ motion_cov = []+ for i in range(len(mean)):+ motion_cov.append(np.diag(sqr[i]))+ motion_cov = np.asarray(motion_cov)++ mean = np.dot(mean, self._motion_mat.T)+ left = np.dot(self._motion_mat, covariance).transpose((1, 0, 2))+ covariance = np.dot(left, self._motion_mat.T) + motion_cov++ return mean, covariance++ def update(self, mean, covariance, measurement):+ """Run Kalman filter correction step.++ Parameters+ ----------+ mean : ndarray+ The predicted state's mean vector (8 dimensional).+ covariance : ndarray+ The state's covariance matrix (8x8 dimensional).+ measurement : ndarray+ The 4 dimensional measurement vector (x, y, a, h), where (x, y)+ is the center position, a the aspect ratio, and h the height of the+ bounding box.++ Returns+ -------+ (ndarray, ndarray)+ Returns the measurement-corrected state distribution.++ """+ projected_mean, projected_cov = self.project(mean, covariance)++ chol_factor, lower = scipy.linalg.cho_factor(+ projected_cov, lower=True, check_finite=False)+ kalman_gain = scipy.linalg.cho_solve(+ (chol_factor, lower), np.dot(covariance, self._update_mat.T).T,+ check_finite=False).T+ innovation = measurement - projected_mean++ new_mean = mean + np.dot(innovation, kalman_gain.T)+ new_covariance = covariance - np.linalg.multi_dot((+ kalman_gain, projected_cov, kalman_gain.T))+ return new_mean, new_covariance++ def gating_distance(self, mean, covariance, measurements,+ only_position=False, metric='maha'):+ """Compute gating distance between state distribution and measurements.+ A suitable distance threshold can be obtained from `chi2inv95`. If+ `only_position` is False, the chi-square distribution has 4 degrees of+ freedom, otherwise 2.+ Parameters+ ----------+ mean : ndarray+ Mean vector over the state distribution (8 dimensional).+ covariance : ndarray+ Covariance of the state distribution (8x8 dimensional).+ measurements : ndarray+ An Nx4 dimensional matrix of N measurements, each in+ format (x, y, a, h) where (x, y) is the bounding box center+ position, a the aspect ratio, and h the height.+ only_position : Optional[bool]+ If True, distance computation is done with respect to the bounding+ box center position only.+ Returns+ -------+ ndarray+ Returns an array of length N, where the i-th element contains the+ squared Mahalanobis distance between (mean, covariance) and+ `measurements[i]`.+ """+ mean, covariance = self.project(mean, covariance)+ if only_position:+ mean, covariance = mean[:2], covariance[:2, :2]+ measurements = measurements[:, :2]++ d = measurements - mean+ if metric == 'gaussian':+ return np.sum(d * d, axis=1)+ elif metric == 'maha':+ cholesky_factor = np.linalg.cholesky(covariance)+ z = scipy.linalg.solve_triangular(+ cholesky_factor, d.T, lower=True, check_finite=False,+ overwrite_b=True)+ squared_maha = np.sum(z * z, axis=0)+ return squared_maha+ else:+ raise ValueError('invalid distance metric')\ No newline at end of filediff --git a/experimental/yolox/motdt_tracker/matching.py b/experimental/yolox/motdt_tracker/matching.pynew file mode 100644index 0000000..01d07da--- /dev/null+++ b/experimental/yolox/motdt_tracker/matching.py@@ -0,0 +1,116 @@+import cv2+import numpy as np+import lap+from scipy.spatial.distance import cdist++from cython_bbox import bbox_overlaps as bbox_ious+from yolox.motdt_tracker import kalman_filter+++def _indices_to_matches(cost_matrix, indices, thresh):+ matched_cost = cost_matrix[tuple(zip(*indices))]+ matched_mask = (matched_cost <= thresh)++ matches = indices[matched_mask]+ unmatched_a = tuple(set(range(cost_matrix.shape[0])) - set(matches[:, 0]))+ unmatched_b = tuple(set(range(cost_matrix.shape[1])) - set(matches[:, 1]))++ return matches, unmatched_a, unmatched_b+++def linear_assignment(cost_matrix, thresh):+ if cost_matrix.size == 0:+ return np.empty((0, 2), dtype=int), tuple(range(cost_matrix.shape[0])), tuple(range(cost_matrix.shape[1]))+ matches, unmatched_a, unmatched_b = [], [], []+ cost, x, y = lap.lapjv(cost_matrix, extend_cost=True, cost_limit=thresh)+ for ix, mx in enumerate(x):+ if mx >= 0:+ matches.append([ix, mx])+ unmatched_a = np.where(x < 0)[0]+ unmatched_b = np.where(y < 0)[0]+ matches = np.asarray(matches)+ return matches, unmatched_a, unmatched_b+++def ious(atlbrs, btlbrs):+ """+ Compute cost based on IoU+ :type atlbrs: list[tlbr] | np.ndarray+ :type atlbrs: list[tlbr] | np.ndarray+ :rtype ious np.ndarray+ """+ ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float)+ if ious.size == 0:+ return ious++ ious = bbox_ious(+ np.ascontiguousarray(atlbrs, dtype=np.float),+ np.ascontiguousarray(btlbrs, dtype=np.float)+ )++ return ious+++def iou_distance(atracks, btracks):+ """+ Compute cost based on IoU+ :type atracks: list[STrack]+ :type btracks: list[STrack]+ :rtype cost_matrix np.ndarray+ """+ atlbrs = [track.tlbr for track in atracks]+ btlbrs = [track.tlbr for track in btracks]+ _ious = ious(atlbrs, btlbrs)+ cost_matrix = 1 - _ious++ return cost_matrix+++def nearest_reid_distance(tracks, detections, metric='cosine'):+ """+ Compute cost based on ReID features+ :type tracks: list[STrack]+ :type detections: list[BaseTrack]+ :rtype cost_matrix np.ndarray+ """+ cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float)+ if cost_matrix.size == 0:+ return cost_matrix++ det_features = np.asarray([track.curr_feature for track in detections], dtype=np.float32)+ for i, track in enumerate(tracks):+ cost_matrix[i, :] = np.maximum(0.0, cdist(track.features, det_features, metric).min(axis=0))++ return cost_matrix+++def mean_reid_distance(tracks, detections, metric='cosine'):+ """+ Compute cost based on ReID features+ :type tracks: list[STrack]+ :type detections: list[BaseTrack]+ :type metric: str+ :rtype cost_matrix np.ndarray+ """+ cost_matrix = np.empty((len(tracks), len(detections)), dtype=np.float)+ if cost_matrix.size == 0:+ return cost_matrix++ track_features = np.asarray([track.curr_feature for track in tracks], dtype=np.float32)+ det_features = np.asarray([track.curr_feature for track in detections], dtype=np.float32)+ cost_matrix = cdist(track_features, det_features, metric)++ return cost_matrix+++def gate_cost_matrix(kf, cost_matrix, tracks, detections, only_position=False):+ if cost_matrix.size == 0:+ return cost_matrix+ gating_dim = 2 if only_position else 4+ gating_threshold = kalman_filter.chi2inv95[gating_dim]+ measurements = np.asarray([det.to_xyah() for det in detections])+ for row, track in enumerate(tracks):+ gating_distance = kf.gating_distance(+ track.mean, track.covariance, measurements, only_position)+ cost_matrix[row, gating_distance > gating_threshold] = np.inf+ return cost_matrix\ No newline at end of filediff --git a/experimental/yolox/motdt_tracker/motdt_tracker.py b/experimental/yolox/motdt_tracker/motdt_tracker.pynew file mode 100644index 0000000..b3521f2--- /dev/null+++ b/experimental/yolox/motdt_tracker/motdt_tracker.py@@ -0,0 +1,358 @@+import numpy as np+#from numba import jit+from collections import OrderedDict, deque+import itertools+import os+import cv2+import torch+from torch._C import dtype+import torchvision++from yolox.motdt_tracker import matching+from .kalman_filter import KalmanFilter+from .reid_model import load_reid_model, extract_reid_features+from yolox.data.dataloading import get_yolox_datadir++from .basetrack import BaseTrack, TrackState+++class STrack(BaseTrack):++ def __init__(self, tlwh, score, max_n_features=100, from_det=True):++ # wait activate+ self._tlwh = np.asarray(tlwh, dtype=np.float)+ self.kalman_filter = None+ self.mean, self.covariance = None, None+ self.is_activated = False++ self.score = score+ self.max_n_features = max_n_features+ self.curr_feature = None+ self.last_feature = None+ self.features = deque([], maxlen=self.max_n_features)++ # classification+ self.from_det = from_det+ self.tracklet_len = 0+ self.time_by_tracking = 0++ # self-tracking+ self.tracker = None++ def set_feature(self, feature):+ if feature is None:+ return False+ self.features.append(feature)+ self.curr_feature = feature+ self.last_feature = feature+ # self._p_feature = 0+ return True++ def predict(self):+ if self.time_since_update > 0:+ self.tracklet_len = 0++ self.time_since_update += 1++ mean_state = self.mean.copy()+ if self.state != TrackState.Tracked:+ mean_state[7] = 0+ self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance)++ if self.tracker:+ self.tracker.update_roi(self.tlwh)++ def self_tracking(self, image):+ tlwh = self.tracker.predict(image) if self.tracker else self.tlwh+ return tlwh++ def activate(self, kalman_filter, frame_id, image):+ """Start a new tracklet"""+ self.kalman_filter = kalman_filter # type: KalmanFilter+ self.track_id = self.next_id()+ # cx, cy, aspect_ratio, height, dx, dy, da, dh+ self.mean, self.covariance = self.kalman_filter.initiate(self.tlwh_to_xyah(self._tlwh))++ # self.tracker = sot.SingleObjectTracker()+ # self.tracker.init(image, self.tlwh)++ del self._tlwh++ self.time_since_update = 0+ self.time_by_tracking = 0+ self.tracklet_len = 0+ self.state = TrackState.Tracked+ # self.is_activated = True+ self.frame_id = frame_id+ self.start_frame = frame_id++ def re_activate(self, new_track, frame_id, image, new_id=False):+ # self.mean, self.covariance = self.kalman_filter.initiate(self.tlwh_to_xyah(new_track.tlwh))+ self.mean, self.covariance = self.kalman_filter.update(+ self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh)+ )+ self.time_since_update = 0+ self.time_by_tracking = 0+ self.tracklet_len = 0+ self.state = TrackState.Tracked+ self.is_activated = True+ self.frame_id = frame_id+ if new_id:+ self.track_id = self.next_id()++ self.set_feature(new_track.curr_feature)++ def update(self, new_track, frame_id, image, update_feature=True):+ """+ Update a matched track+ :type new_track: STrack+ :type frame_id: int+ :type update_feature: bool+ :return:+ """+ self.frame_id = frame_id+ self.time_since_update = 0+ if new_track.from_det:+ self.time_by_tracking = 0+ else:+ self.time_by_tracking += 1+ self.tracklet_len += 1++ new_tlwh = new_track.tlwh+ self.mean, self.covariance = self.kalman_filter.update(+ self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh))+ self.state = TrackState.Tracked+ self.is_activated = True++ self.score = new_track.score++ if update_feature:+ self.set_feature(new_track.curr_feature)+ if self.tracker:+ self.tracker.update(image, self.tlwh)++ @property+ #@jit+ def tlwh(self):+ """Get current position in bounding box format `(top left x, top left y,+ width, height)`.+ """+ if self.mean is None:+ return self._tlwh.copy()+ ret = self.mean[:4].copy()+ ret[2] *= ret[3]+ ret[:2] -= ret[2:] / 2+ return ret++ @property+ #@jit+ def tlbr(self):+ """Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,+ `(top left, bottom right)`.+ """+ ret = self.tlwh.copy()+ ret[2:] += ret[:2]+ return ret++ @staticmethod+ #@jit+ def tlwh_to_xyah(tlwh):+ """Convert bounding box to format `(center x, center y, aspect ratio,+ height)`, where the aspect ratio is `width / height`.+ """+ ret = np.asarray(tlwh).copy()+ ret[:2] += ret[2:] / 2+ ret[2] /= ret[3]+ return ret++ def to_xyah(self):+ return self.tlwh_to_xyah(self.tlwh)++ def tracklet_score(self):+ # score = (1 - np.exp(-0.6 * self.hit_streak)) * np.exp(-0.03 * self.time_by_tracking)++ score = max(0, 1 - np.log(1 + 0.05 * self.time_by_tracking)) * (self.tracklet_len - self.time_by_tracking > 2)+ # score = max(0, 1 - np.log(1 + 0.05 * self.n_tracking)) * (1 - np.exp(-0.6 * self.hit_streak))+ return score++ def __repr__(self):+ return 'OT_{}_({}-{})'.format(self.track_id, self.start_frame, self.end_frame)+++class OnlineTracker(object):++ def __init__(self, model_folder, min_cls_score=0.4, min_ap_dist=0.8, max_time_lost=30, use_tracking=True, use_refind=True):++ self.min_cls_score = min_cls_score+ self.min_ap_dist = min_ap_dist+ self.max_time_lost = max_time_lost++ self.kalman_filter = KalmanFilter()++ self.tracked_stracks = [] # type: list[STrack]+ self.lost_stracks = [] # type: list[STrack]+ self.removed_stracks = [] # type: list[STrack]++ self.use_refind = use_refind+ self.use_tracking = use_tracking+ self.classifier = None+ self.reid_model = load_reid_model(model_folder)++ self.frame_id = 0++ def update(self, output_results, img_info, img_size, img_file_name):+ img_file_name = os.path.join(get_yolox_datadir(), 'mot', 'train', img_file_name)+ image = cv2.imread(img_file_name)+ # post process detections+ output_results = output_results.cpu().numpy()+ confidences = output_results[:, 4] * output_results[:, 5]++ bboxes = output_results[:, :4] # x1y1x2y2+ img_h, img_w = img_info[0], img_info[1]+ scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w))+ bboxes /= scale+ bbox_xyxy = bboxes+ tlwhs = self._xyxy_to_tlwh_array(bbox_xyxy)+ remain_inds = confidences > self.min_cls_score+ tlwhs = tlwhs[remain_inds]+ det_scores = confidences[remain_inds]++ self.frame_id += 1++ activated_starcks = []+ refind_stracks = []+ lost_stracks = []+ removed_stracks = []++ """step 1: prediction"""+ for strack in itertools.chain(self.tracked_stracks, self.lost_stracks):+ strack.predict()++ """step 2: scoring and selection"""+ if det_scores is None:+ det_scores = np.ones(len(tlwhs), dtype=float)+ detections = [STrack(tlwh, score, from_det=True) for tlwh, score in zip(tlwhs, det_scores)]+ if self.use_tracking:+ tracks = [STrack(t.self_tracking(image), 0.6 * t.tracklet_score(), from_det=False)+ for t in itertools.chain(self.tracked_stracks, self.lost_stracks) if t.is_activated]+ detections.extend(tracks)+ rois = np.asarray([d.tlbr for d in detections], dtype=np.float32)+ scores = np.asarray([d.score for d in detections], dtype=np.float32)+ # nms+ if len(detections) > 0:+ nms_out_index = torchvision.ops.batched_nms(+ torch.from_numpy(rois),+ torch.from_numpy(scores.reshape(-1)).to(torch.from_numpy(rois).dtype),+ torch.zeros_like(torch.from_numpy(scores.reshape(-1))),+ 0.7,+ )+ keep = nms_out_index.numpy()+ mask = np.zeros(len(rois), dtype=np.bool)+ mask[keep] = True+ keep = np.where(mask & (scores >= self.min_cls_score))[0]+ detections = [detections[i] for i in keep]+ scores = scores[keep]+ for d, score in zip(detections, scores):+ d.score = score+ pred_dets = [d for d in detections if not d.from_det]+ detections = [d for d in detections if d.from_det]++ # set features+ tlbrs = [det.tlbr for det in detections]+ features = extract_reid_features(self.reid_model, image, tlbrs)+ features = features.cpu().numpy()+ for i, det in enumerate(detections):+ det.set_feature(features[i])++ """step 3: association for tracked"""+ # matching for tracked targets+ unconfirmed = []+ tracked_stracks = [] # type: list[STrack]+ for track in self.tracked_stracks:+ if not track.is_activated:+ unconfirmed.append(track)+ else:+ tracked_stracks.append(track)++ dists = matching.nearest_reid_distance(tracked_stracks, detections, metric='euclidean')+ dists = matching.gate_cost_matrix(self.kalman_filter, dists, tracked_stracks, detections)+ matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.min_ap_dist)+ for itracked, idet in matches:+ tracked_stracks[itracked].update(detections[idet], self.frame_id, image)++ # matching for missing targets+ detections = [detections[i] for i in u_detection]+ dists = matching.nearest_reid_distance(self.lost_stracks, detections, metric='euclidean')+ dists = matching.gate_cost_matrix(self.kalman_filter, dists, self.lost_stracks, detections)+ matches, u_lost, u_detection = matching.linear_assignment(dists, thresh=self.min_ap_dist)+ for ilost, idet in matches:+ track = self.lost_stracks[ilost] # type: STrack+ det = detections[idet]+ track.re_activate(det, self.frame_id, image, new_id=not self.use_refind)+ refind_stracks.append(track)++ # remaining tracked+ # tracked+ len_det = len(u_detection)+ detections = [detections[i] for i in u_detection] + pred_dets+ r_tracked_stracks = [tracked_stracks[i] for i in u_track]+ dists = matching.iou_distance(r_tracked_stracks, detections)+ matches, u_track, u_detection = matching.linear_assignment(dists, thresh=0.5)+ for itracked, idet in matches:+ r_tracked_stracks[itracked].update(detections[idet], self.frame_id, image, update_feature=True)+ for it in u_track:+ track = r_tracked_stracks[it]+ track.mark_lost()+ lost_stracks.append(track)++ # unconfirmed+ detections = [detections[i] for i in u_detection if i < len_det]+ dists = matching.iou_distance(unconfirmed, detections)+ matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7)+ for itracked, idet in matches:+ unconfirmed[itracked].update(detections[idet], self.frame_id, image, update_feature=True)+ for it in u_unconfirmed:+ track = unconfirmed[it]+ track.mark_removed()+ removed_stracks.append(track)++ """step 4: init new stracks"""+ for inew in u_detection:+ track = detections[inew]+ if not track.from_det or track.score < 0.6:+ continue+ track.activate(self.kalman_filter, self.frame_id, image)+ activated_starcks.append(track)++ """step 6: update state"""+ for track in self.lost_stracks:+ if self.frame_id - track.end_frame > self.max_time_lost:+ track.mark_removed()+ removed_stracks.append(track)++ self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked]+ self.lost_stracks = [t for t in self.lost_stracks if t.state == TrackState.Lost] # type: list[STrack]+ self.tracked_stracks.extend(activated_starcks)+ self.tracked_stracks.extend(refind_stracks)+ self.lost_stracks.extend(lost_stracks)+ self.removed_stracks.extend(removed_stracks)++ # output_stracks = self.tracked_stracks + self.lost_stracks++ # get scores of lost tracks+ output_tracked_stracks = [track for track in self.tracked_stracks if track.is_activated]++ output_stracks = output_tracked_stracks++ return output_stracks++ @staticmethod+ def _xyxy_to_tlwh_array(bbox_xyxy):+ if isinstance(bbox_xyxy, np.ndarray):+ bbox_tlwh = bbox_xyxy.copy()+ elif isinstance(bbox_xyxy, torch.Tensor):+ bbox_tlwh = bbox_xyxy.clone()+ bbox_tlwh[:, 2] = bbox_xyxy[:, 2] - bbox_xyxy[:, 0]+ bbox_tlwh[:, 3] = bbox_xyxy[:, 3] - bbox_xyxy[:, 1]+ return bbox_tlwhdiff --git a/experimental/yolox/motdt_tracker/reid_model.py b/experimental/yolox/motdt_tracker/reid_model.pynew file mode 100644index 0000000..6ad49e3--- /dev/null+++ b/experimental/yolox/motdt_tracker/reid_model.py@@ -0,0 +1,270 @@+import cv2+import numpy as np+import torch+from torch.autograd import Variable+import torch.nn.functional as F+import torch.nn as nn+import pickle+import os+from torch.nn.modules import CrossMapLRN2d as SpatialCrossMapLRN+#from torch.legacy.nn import SpatialCrossMapLRN as SpatialCrossMapLRNOld+from torch.autograd import Function, Variable+from torch.nn import Module+++def clip_boxes(boxes, im_shape):+ """+ Clip boxes to image boundaries.+ """+ boxes = np.asarray(boxes)+ if boxes.shape[0] == 0:+ return boxes+ boxes = np.copy(boxes)+ # x1 >= 0+ boxes[:, 0::4] = np.maximum(np.minimum(boxes[:, 0::4], im_shape[1] - 1), 0)+ # y1 >= 0+ boxes[:, 1::4] = np.maximum(np.minimum(boxes[:, 1::4], im_shape[0] - 1), 0)+ # x2 < im_shape[1]+ boxes[:, 2::4] = np.maximum(np.minimum(boxes[:, 2::4], im_shape[1] - 1), 0)+ # y2 < im_shape[0]+ boxes[:, 3::4] = np.maximum(np.minimum(boxes[:, 3::4], im_shape[0] - 1), 0)+ return boxes+++def load_net(fname, net, prefix='', load_state_dict=False):+ import h5py+ with h5py.File(fname, mode='r') as h5f:+ h5f_is_module = True+ for k in h5f.keys():+ if not str(k).startswith('module.'):+ h5f_is_module = False+ break+ if prefix == '' and not isinstance(net, nn.DataParallel) and h5f_is_module:+ prefix = 'module.'++ for k, v in net.state_dict().items():+ k = prefix + k+ if k in h5f:+ param = torch.from_numpy(np.asarray(h5f[k]))+ if v.size() != param.size():+ print('Inconsistent shape: {}, {}'.format(v.size(), param.size()))+ else:+ v.copy_(param)+ else:+ print.warning('No layer: {}'.format(k))++ epoch = h5f.attrs['epoch'] if 'epoch' in h5f.attrs else -1++ if not load_state_dict:+ if 'learning_rates' in h5f.attrs:+ lr = h5f.attrs['learning_rates']+ else:+ lr = h5f.attrs.get('lr', -1)+ lr = np.asarray([lr] if lr > 0 else [], dtype=np.float)++ return epoch, lr++ state_file = fname + '.optimizer_state.pk'+ if os.path.isfile(state_file):+ with open(state_file, 'rb') as f:+ state_dicts = pickle.load(f)+ if not isinstance(state_dicts, list):+ state_dicts = [state_dicts]+ else:+ state_dicts = None+ return epoch, state_dicts+++# class SpatialCrossMapLRNFunc(Function):++# def __init__(self, size, alpha=1e-4, beta=0.75, k=1):+# self.size = size+# self.alpha = alpha+# self.beta = beta+# self.k = k++# def forward(self, input):+# self.save_for_backward(input)+# self.lrn = SpatialCrossMapLRNOld(self.size, self.alpha, self.beta, self.k)+# self.lrn.type(input.type())+# return self.lrn.forward(input)++# def backward(self, grad_output):+# input, = self.saved_tensors+# return self.lrn.backward(input, grad_output)+++# # use this one instead+# class SpatialCrossMapLRN(Module):+# def __init__(self, size, alpha=1e-4, beta=0.75, k=1):+# super(SpatialCrossMapLRN, self).__init__()+# self.size = size+# self.alpha = alpha+# self.beta = beta+# self.k = k++# def forward(self, input):+# return SpatialCrossMapLRNFunc(self.size, self.alpha, self.beta, self.k)(input)+++class Inception(nn.Module):+ def __init__(self, in_planes, n1x1, n3x3red, n3x3, n5x5red, n5x5, pool_planes):+ super(Inception, self).__init__()+ # 1x1 conv branch+ self.b1 = nn.Sequential(+ nn.Conv2d(in_planes, n1x1, kernel_size=1),+ nn.ReLU(True),+ )++ # 1x1 conv -> 3x3 conv branch+ self.b2 = nn.Sequential(+ nn.Conv2d(in_planes, n3x3red, kernel_size=1),+ nn.ReLU(True),+ nn.Conv2d(n3x3red, n3x3, kernel_size=3, padding=1),+ nn.ReLU(True),+ )++ # 1x1 conv -> 5x5 conv branch+ self.b3 = nn.Sequential(+ nn.Conv2d(in_planes, n5x5red, kernel_size=1),+ nn.ReLU(True),++ nn.Conv2d(n5x5red, n5x5, kernel_size=5, padding=2),+ nn.ReLU(True),+ )++ # 3x3 pool -> 1x1 conv branch+ self.b4 = nn.Sequential(+ nn.MaxPool2d(3, stride=1, padding=1),++ nn.Conv2d(in_planes, pool_planes, kernel_size=1),+ nn.ReLU(True),+ )++ def forward(self, x):+ y1 = self.b1(x)+ y2 = self.b2(x)+ y3 = self.b3(x)+ y4 = self.b4(x)+ return torch.cat([y1,y2,y3,y4], 1)+++class GoogLeNet(nn.Module):++ output_channels = 832++ def __init__(self):+ super(GoogLeNet, self).__init__()+ self.pre_layers = nn.Sequential(+ nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3),+ nn.ReLU(True),++ nn.MaxPool2d(3, stride=2, ceil_mode=True),+ SpatialCrossMapLRN(5),++ nn.Conv2d(64, 64, 1),+ nn.ReLU(True),++ nn.Conv2d(64, 192, 3, padding=1),+ nn.ReLU(True),++ SpatialCrossMapLRN(5),+ nn.MaxPool2d(3, stride=2, ceil_mode=True),+ )++ self.a3 = Inception(192, 64, 96, 128, 16, 32, 32)+ self.b3 = Inception(256, 128, 128, 192, 32, 96, 64)++ self.maxpool = nn.MaxPool2d(3, stride=2, ceil_mode=True)++ self.a4 = Inception(480, 192, 96, 208, 16, 48, 64)+ self.b4 = Inception(512, 160, 112, 224, 24, 64, 64)+ self.c4 = Inception(512, 128, 128, 256, 24, 64, 64)+ self.d4 = Inception(512, 112, 144, 288, 32, 64, 64)+ self.e4 = Inception(528, 256, 160, 320, 32, 128, 128)++ def forward(self, x):+ out = self.pre_layers(x)+ out = self.a3(out)+ out = self.b3(out)+ out = self.maxpool(out)+ out = self.a4(out)+ out = self.b4(out)+ out = self.c4(out)+ out = self.d4(out)+ out = self.e4(out)++ return out+++class Model(nn.Module):+ def __init__(self, n_parts=8):+ super(Model, self).__init__()+ self.n_parts = n_parts++ self.feat_conv = GoogLeNet()+ self.conv_input_feat = nn.Conv2d(self.feat_conv.output_channels, 512, 1)++ # part net+ self.conv_att = nn.Conv2d(512, self.n_parts, 1)++ for i in range(self.n_parts):+ setattr(self, 'linear_feature{}'.format(i+1), nn.Linear(512, 64))++ def forward(self, x):+ feature = self.feat_conv(x)+ feature = self.conv_input_feat(feature)++ att_weights = torch.sigmoid(self.conv_att(feature))++ linear_feautres = []+ for i in range(self.n_parts):+ masked_feature = feature * torch.unsqueeze(att_weights[:, i], 1)+ pooled_feature = F.avg_pool2d(masked_feature, masked_feature.size()[2:4])+ linear_feautres.append(+ getattr(self, 'linear_feature{}'.format(i+1))(pooled_feature.view(pooled_feature.size(0), -1))+ )++ concat_features = torch.cat(linear_feautres, 1)+ normed_feature = concat_features / torch.clamp(torch.norm(concat_features, 2, 1, keepdim=True), min=1e-6)++ return normed_feature+++def load_reid_model(ckpt):+ model = Model(n_parts=8)+ model.inp_size = (80, 160)+ load_net(ckpt, model)+ print('Load ReID model from {}'.format(ckpt))++ model = model.cuda()+ model.eval()+ return model+++def im_preprocess(image):+ image = np.asarray(image, np.float32)+ image -= np.array([104, 117, 123], dtype=np.float32).reshape(1, 1, -1)+ image = image.transpose((2, 0, 1))+ return image+++def extract_image_patches(image, bboxes):+ bboxes = np.round(bboxes).astype(np.int)+ bboxes = clip_boxes(bboxes, image.shape)+ patches = [image[box[1]:box[3], box[0]:box[2]] for box in bboxes]+ return patches+++def extract_reid_features(reid_model, image, tlbrs):+ if len(tlbrs) == 0:+ return torch.FloatTensor()++ patches = extract_image_patches(image, tlbrs)+ patches = np.asarray([im_preprocess(cv2.resize(p, reid_model.inp_size)) for p in patches], dtype=np.float32)++ with torch.no_grad():+ im_var = Variable(torch.from_numpy(patches))+ im_var = im_var.cuda()+ features = reid_model(im_var).data+ return features\ No newline at end of filediff --git a/experimental/yolox/sort_tracker/sort.py b/experimental/yolox/sort_tracker/sort.pynew file mode 100644index 0000000..9b708ba--- /dev/null+++ b/experimental/yolox/sort_tracker/sort.py@@ -0,0 +1,251 @@+"""+ SORT: A Simple, Online and Realtime Tracker+ Copyright (C) 2016-2020 Alex Bewley [email protected]+ This program is free software: you can redistribute it and/or modify+ it under the terms of the GNU General Public License as published by+ the Free Software Foundation, either version 3 of the License, or+ (at your option) any later version.+ This program is distributed in the hope that it will be useful,+ but WITHOUT ANY WARRANTY; without even the implied warranty of+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ GNU General Public License for more details.+ You should have received a copy of the GNU General Public License+ along with this program. If not, see <http://www.gnu.org/licenses/>.+"""+from __future__ import print_function++import os+import numpy as np++from filterpy.kalman import KalmanFilter++np.random.seed(0)+++def linear_assignment(cost_matrix):+ try:+ import lap+ _, x, y = lap.lapjv(cost_matrix, extend_cost=True)+ return np.array([[y[i],i] for i in x if i >= 0]) #+ except ImportError:+ from scipy.optimize import linear_sum_assignment+ x, y = linear_sum_assignment(cost_matrix)+ return np.array(list(zip(x, y)))+++def iou_batch(bb_test, bb_gt):+ """+ From SORT: Computes IOU between two bboxes in the form [x1,y1,x2,y2]+ """+ bb_gt = np.expand_dims(bb_gt, 0)+ bb_test = np.expand_dims(bb_test, 1)++ xx1 = np.maximum(bb_test[..., 0], bb_gt[..., 0])+ yy1 = np.maximum(bb_test[..., 1], bb_gt[..., 1])+ xx2 = np.minimum(bb_test[..., 2], bb_gt[..., 2])+ yy2 = np.minimum(bb_test[..., 3], bb_gt[..., 3])+ w = np.maximum(0., xx2 - xx1)+ h = np.maximum(0., yy2 - yy1)+ wh = w * h+ o = wh / ((bb_test[..., 2] - bb_test[..., 0]) * (bb_test[..., 3] - bb_test[..., 1])+ + (bb_gt[..., 2] - bb_gt[..., 0]) * (bb_gt[..., 3] - bb_gt[..., 1]) - wh)+ return(o)+++def convert_bbox_to_z(bbox):+ """+ Takes a bounding box in the form [x1,y1,x2,y2] and returns z in the form+ [x,y,s,r] where x,y is the centre of the box and s is the scale/area and r is+ the aspect ratio+ """+ w = bbox[2] - bbox[0]+ h = bbox[3] - bbox[1]+ x = bbox[0] + w/2.+ y = bbox[1] + h/2.+ s = w * h #scale is just area+ r = w / float(h)+ return np.array([x, y, s, r]).reshape((4, 1))+++def convert_x_to_bbox(x,score=None):+ """+ Takes a bounding box in the centre form [x,y,s,r] and returns it in the form+ [x1,y1,x2,y2] where x1,y1 is the top left and x2,y2 is the bottom right+ """+ w = np.sqrt(x[2] * x[3])+ h = x[2] / w+ if(score==None):+ return np.array([x[0]-w/2.,x[1]-h/2.,x[0]+w/2.,x[1]+h/2.]).reshape((1,4))+ else:+ return np.array([x[0]-w/2.,x[1]-h/2.,x[0]+w/2.,x[1]+h/2.,score]).reshape((1,5))+++class KalmanBoxTracker(object):+ """+ This class represents the internal state of individual tracked objects observed as bbox.+ """+ count = 0+ def __init__(self,bbox):+ """+ Initialises a tracker using initial bounding box.+ """+ #define constant velocity model+ self.kf = KalmanFilter(dim_x=7, dim_z=4)+ self.kf.F = np.array([[1,0,0,0,1,0,0],[0,1,0,0,0,1,0],[0,0,1,0,0,0,1],[0,0,0,1,0,0,0], [0,0,0,0,1,0,0],[0,0,0,0,0,1,0],[0,0,0,0,0,0,1]])+ self.kf.H = np.array([[1,0,0,0,0,0,0],[0,1,0,0,0,0,0],[0,0,1,0,0,0,0],[0,0,0,1,0,0,0]])++ self.kf.R[2:,2:] *= 10.+ self.kf.P[4:,4:] *= 1000. #give high uncertainty to the unobservable initial velocities+ self.kf.P *= 10.+ self.kf.Q[-1,-1] *= 0.01+ self.kf.Q[4:,4:] *= 0.01++ self.kf.x[:4] = convert_bbox_to_z(bbox)+ self.time_since_update = 0+ self.id = KalmanBoxTracker.count+ KalmanBoxTracker.count += 1+ self.history = []+ self.hits = 0+ self.hit_streak = 0+ self.age = 0++ def update(self,bbox):+ """+ Updates the state vector with observed bbox.+ """+ self.time_since_update = 0+ self.history = []+ self.hits += 1+ self.hit_streak += 1+ self.kf.update(convert_bbox_to_z(bbox))++ def predict(self):+ """+ Advances the state vector and returns the predicted bounding box estimate.+ """+ if((self.kf.x[6]+self.kf.x[2])<=0):+ self.kf.x[6] *= 0.0+ self.kf.predict()+ self.age += 1+ if(self.time_since_update>0):+ self.hit_streak = 0+ self.time_since_update += 1+ self.history.append(convert_x_to_bbox(self.kf.x))+ return self.history[-1]++ def get_state(self):+ """+ Returns the current bounding box estimate.+ """+ return convert_x_to_bbox(self.kf.x)+++def associate_detections_to_trackers(detections,trackers,iou_threshold = 0.3):+ """+ Assigns detections to tracked object (both represented as bounding boxes)+ Returns 3 lists of matches, unmatched_detections and unmatched_trackers+ """+ if(len(trackers)==0):+ return np.empty((0,2),dtype=int), np.arange(len(detections)), np.empty((0,5),dtype=int)++ iou_matrix = iou_batch(detections, trackers)++ if min(iou_matrix.shape) > 0:+ a = (iou_matrix > iou_threshold).astype(np.int32)+ if a.sum(1).max() == 1 and a.sum(0).max() == 1:+ matched_indices = np.stack(np.where(a), axis=1)+ else:+ matched_indices = linear_assignment(-iou_matrix)+ else:+ matched_indices = np.empty(shape=(0,2))++ unmatched_detections = []+ for d, det in enumerate(detections):+ if(d not in matched_indices[:,0]):+ unmatched_detections.append(d)+ unmatched_trackers = []+ for t, trk in enumerate(trackers):+ if(t not in matched_indices[:,1]):+ unmatched_trackers.append(t)++ #filter out matched with low IOU+ matches = []+ for m in matched_indices:+ if(iou_matrix[m[0], m[1]]<iou_threshold):+ unmatched_detections.append(m[0])+ unmatched_trackers.append(m[1])+ else:+ matches.append(m.reshape(1,2))+ if(len(matches)==0):+ matches = np.empty((0,2),dtype=int)+ else:+ matches = np.concatenate(matches,axis=0)++ return matches, np.array(unmatched_detections), np.array(unmatched_trackers)+++class Sort(object):+ def __init__(self, det_thresh, max_age=30, min_hits=3, iou_threshold=0.3):+ """+ Sets key parameters for SORT+ """+ self.max_age = max_age+ self.min_hits = min_hits+ self.iou_threshold = iou_threshold+ self.trackers = []+ self.frame_count = 0+ self.det_thresh = det_thresh++ def update(self, output_results, img_info, img_size):+ """+ Params:+ dets - a numpy array of detections in the format [[x1,y1,x2,y2,score],[x1,y1,x2,y2,score],...]+ Requires: this method must be called once for each frame even with empty detections (use np.empty((0, 5)) for frames without detections).+ Returns the a similar array, where the last column is the object ID.+ NOTE: The number of objects returned may differ from the number of detections provided.+ """+ self.frame_count += 1+ # post_process detections+ output_results = output_results.cpu().numpy()+ scores = output_results[:, 4] * output_results[:, 5]+ bboxes = output_results[:, :4] # x1y1x2y2+ img_h, img_w = img_info[0], img_info[1]+ scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w))+ bboxes /= scale+ dets = np.concatenate((bboxes, np.expand_dims(scores, axis=-1)), axis=1)+ remain_inds = scores > self.det_thresh+ dets = dets[remain_inds]+ # get predicted locations from existing trackers.+ trks = np.zeros((len(self.trackers), 5))+ to_del = []+ ret = []+ for t, trk in enumerate(trks):+ pos = self.trackers[t].predict()[0]+ trk[:] = [pos[0], pos[1], pos[2], pos[3], 0]+ if np.any(np.isnan(pos)):+ to_del.append(t)+ trks = np.ma.compress_rows(np.ma.masked_invalid(trks))+ for t in reversed(to_del):+ self.trackers.pop(t)+ matched, unmatched_dets, unmatched_trks = associate_detections_to_trackers(dets, trks, self.iou_threshold)++ # update matched trackers with assigned detections+ for m in matched:+ self.trackers[m[1]].update(dets[m[0], :])++ # create and initialise new trackers for unmatched detections+ for i in unmatched_dets:+ trk = KalmanBoxTracker(dets[i,:])+ self.trackers.append(trk)+ i = len(self.trackers)+ for trk in reversed(self.trackers):+ d = trk.get_state()[0]+ if (trk.time_since_update < 1) and (trk.hit_streak >= self.min_hits or self.frame_count <= self.min_hits):+ ret.append(np.concatenate((d,[trk.id+1])).reshape(1,-1)) # +1 as MOT benchmark requires positive+ i -= 1+ # remove dead tracklet+ if(trk.time_since_update > self.max_age):+ self.trackers.pop(i)+ if(len(ret)>0):+ return np.concatenate(ret)+ return np.empty((0,5))\ No newline at end of filediff --git a/experimental/yolox/tracker/__pycache__/basetrack.cpython-310.pyc b/experimental/yolox/tracker/__pycache__/basetrack.cpython-310.pycnew file mode 100644index 0000000..0cae8d1Binary files /dev/null and b/experimental/yolox/tracker/__pycache__/basetrack.cpython-310.pyc differdiff --git a/experimental/yolox/tracker/__pycache__/byte_tracker.cpython-310.pyc b/experimental/yolox/tracker/__pycache__/byte_tracker.cpython-310.pycnew file mode 100644index 0000000..f7b9f97Binary files /dev/null and b/experimental/yolox/tracker/__pycache__/byte_tracker.cpython-310.pyc differdiff --git a/experimental/yolox/tracker/__pycache__/kalman_filter.cpython-310.pyc b/experimental/yolox/tracker/__pycache__/kalman_filter.cpython-310.pycnew file mode 100644index 0000000..512b76dBinary files /dev/null and b/experimental/yolox/tracker/__pycache__/kalman_filter.cpython-310.pyc differdiff --git a/experimental/yolox/tracker/__pycache__/matching.cpython-310.pyc b/experimental/yolox/tracker/__pycache__/matching.cpython-310.pycnew file mode 100644index 0000000..5e8c5f8Binary files /dev/null and b/experimental/yolox/tracker/__pycache__/matching.cpython-310.pyc differdiff --git a/experimental/yolox/tracker/basetrack.py b/experimental/yolox/tracker/basetrack.pynew file mode 100644index 0000000..4fe2233--- /dev/null+++ b/experimental/yolox/tracker/basetrack.py@@ -0,0 +1,52 @@+import numpy as np+from collections import OrderedDict+++class TrackState(object):+ New = 0+ Tracked = 1+ Lost = 2+ Removed = 3+++class BaseTrack(object):+ _count = 0++ track_id = 0+ is_activated = False+ state = TrackState.New++ history = OrderedDict()+ features = []+ curr_feature = None+ score = 0+ start_frame = 0+ frame_id = 0+ time_since_update = 0++ # multi-camera+ location = (np.inf, np.inf)++ @property+ def end_frame(self):+ return self.frame_id++ @staticmethod+ def next_id():+ BaseTrack._count += 1+ return BaseTrack._count++ def activate(self, *args):+ raise NotImplementedError++ def predict(self):+ raise NotImplementedError++ def update(self, *args, **kwargs):+ raise NotImplementedError++ def mark_lost(self):+ self.state = TrackState.Lost++ def mark_removed(self):+ self.state = TrackState.Removeddiff --git a/experimental/yolox/tracker/byte_tracker.py b/experimental/yolox/tracker/byte_tracker.pynew file mode 100644index 0000000..317f2b2--- /dev/null+++ b/experimental/yolox/tracker/byte_tracker.py@@ -0,0 +1,324 @@+import numpy as np++from .kalman_filter import KalmanFilter+from yolox.tracker import matching+from .basetrack import BaseTrack, TrackState++class STrack(BaseTrack):+ shared_kalman = KalmanFilter()+ def __init__(self, tlwh, score, class_id):++ # wait activate+ self._tlwh = np.asarray(tlwh, dtype=np.float)+ self.kalman_filter = None+ self.mean, self.covariance = None, None+ self.is_activated = False++ self.score = score+ self.class_id = class_id+ self.tracklet_len = 0++ def predict(self):+ mean_state = self.mean.copy()+ if self.state != TrackState.Tracked:+ mean_state[7] = 0+ self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance)++ @staticmethod+ def multi_predict(stracks):+ if len(stracks) > 0:+ multi_mean = np.asarray([st.mean.copy() for st in stracks])+ multi_covariance = np.asarray([st.covariance for st in stracks])+ for i, st in enumerate(stracks):+ if st.state != TrackState.Tracked:+ multi_mean[i][7] = 0+ multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(multi_mean, multi_covariance)+ for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)):+ stracks[i].mean = mean+ stracks[i].covariance = cov++ def activate(self, kalman_filter, frame_id):+ """Start a new tracklet"""+ self.kalman_filter = kalman_filter+ self.track_id = self.next_id()+ self.mean, self.covariance = self.kalman_filter.initiate(self.tlwh_to_xyah(self._tlwh))++ self.tracklet_len = 0+ self.state = TrackState.Tracked+ if frame_id == 1:+ self.is_activated = True+ # self.is_activated = True+ self.frame_id = frame_id+ self.start_frame = frame_id++ def re_activate(self, new_track, frame_id, new_id=False):+ self.mean, self.covariance = self.kalman_filter.update(+ self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh)+ )+ self.tracklet_len = 0+ self.state = TrackState.Tracked+ self.is_activated = True+ self.frame_id = frame_id+ if new_id:+ self.track_id = self.next_id()+ self.score = new_track.score++ def update(self, new_track, frame_id):+ """+ Update a matched track+ :type new_track: STrack+ :type frame_id: int+ :type update_feature: bool+ :return:+ """+ self.frame_id = frame_id+ self.tracklet_len += 1++ new_tlwh = new_track.tlwh+ self.mean, self.covariance = self.kalman_filter.update(+ self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh))+ self.state = TrackState.Tracked+ self.is_activated = True++ self.score = new_track.score++ @property+ # @jit(nopython=True)+ def tlwh(self):+ """Get current position in bounding box format `(top left x, top left y,+ width, height)`.+ """+ if self.mean is None:+ return self._tlwh.copy()+ ret = self.mean[:4].copy()+ ret[2] *= ret[3]+ ret[:2] -= ret[2:] / 2+ return ret++ @property+ # @jit(nopython=True)+ def tlbr(self):+ """Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,+ `(top left, bottom right)`.+ """+ ret = self.tlwh.copy()+ ret[2:] += ret[:2]+ return ret++ @staticmethod+ # @jit(nopython=True)+ def tlwh_to_xyah(tlwh):+ """Convert bounding box to format `(center x, center y, aspect ratio,+ height)`, where the aspect ratio is `width / height`.+ """+ ret = np.asarray(tlwh).copy()+ ret[:2] += ret[2:] / 2+ ret[2] /= ret[3]+ return ret++ def to_xyah(self):+ return self.tlwh_to_xyah(self.tlwh)++ @staticmethod+ # @jit(nopython=True)+ def tlbr_to_tlwh(tlbr):+ ret = np.asarray(tlbr).copy()+ ret[2:] -= ret[:2]+ return ret++ @staticmethod+ # @jit(nopython=True)+ def tlwh_to_tlbr(tlwh):+ ret = np.asarray(tlwh).copy()+ ret[2:] += ret[:2]+ return ret++ def __repr__(self):+ return 'OT_{}_({}-{})'.format(self.track_id, self.start_frame, self.end_frame)+++class BYTETracker(object):+ def __init__(self, args, frame_rate=30):+ self.tracked_stracks = [] # type: list[STrack]+ self.lost_stracks = [] # type: list[STrack]+ self.removed_stracks = [] # type: list[STrack]++ self.frame_id = 0+ self.args = args+ #self.det_thresh = args.track_thresh+ self.det_thresh = args.track_thresh + 0.1+ self.buffer_size = int(frame_rate / 30.0 * args.track_buffer)+ self.max_time_lost = self.buffer_size+ self.kalman_filter = KalmanFilter()++ def update(self, output_results, img_info, img_size):+ self.frame_id += 1+ activated_starcks = []+ refind_stracks = []+ lost_stracks = []+ removed_stracks = []++ classes = output_results[:, 5]+ scores = output_results[:, 4]+ bboxes = output_results[:, :4]++ img_h, img_w = img_info[0], img_info[1]+ scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w))+ bboxes /= scale++ remain_inds = scores > self.args.track_thresh+ inds_low = scores > 0.1+ inds_high = scores < self.args.track_thresh++ inds_second = np.logical_and(inds_low, inds_high)+ dets_second = bboxes[inds_second]+ dets = bboxes[remain_inds]+ scores_keep = scores[remain_inds]+ scores_second = scores[inds_second]+ classes_keep = classes[remain_inds]+ classes_second = classes[inds_second]++ if len(dets) > 0:+ '''Detections'''+ detections = [STrack(STrack.tlbr_to_tlwh(tlbr), s, c) for+ (tlbr, s, c) in zip(dets, scores_keep, classes_keep)]+ else:+ detections = []++ ''' Add newly detected tracklets to tracked_stracks'''+ unconfirmed = []+ tracked_stracks = [] # type: list[STrack]+ for track in self.tracked_stracks:+ if not track.is_activated:+ unconfirmed.append(track)+ else:+ tracked_stracks.append(track)++ ''' Step 2: First association, with high score detection boxes'''+ strack_pool = joint_stracks(tracked_stracks, self.lost_stracks)+ # Predict the current location with KF+ STrack.multi_predict(strack_pool)+ dists = matching.iou_distance(strack_pool, detections)+ if not self.args.mot20:+ dists = matching.fuse_score(dists, detections)+ matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.args.match_thresh)++ for itracked, idet in matches:+ track = strack_pool[itracked]+ det = detections[idet]+ if track.state == TrackState.Tracked:+ track.update(detections[idet], self.frame_id)+ activated_starcks.append(track)+ else:+ track.re_activate(det, self.frame_id, new_id=False)+ refind_stracks.append(track)++ ''' Step 3: Second association, with low score detection boxes'''+ # association the untrack to the low score detections+ if len(dets_second) > 0:+ '''Detections'''+ detections_second = [STrack(STrack.tlbr_to_tlwh(tlbr), s, c) for+ (tlbr, s, c) in zip(dets_second, scores_second, classes_second)]+ else:+ detections_second = []+ r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked]+ dists = matching.iou_distance(r_tracked_stracks, detections_second)+ matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.5)+ for itracked, idet in matches:+ track = r_tracked_stracks[itracked]+ det = detections_second[idet]+ if track.state == TrackState.Tracked:+ track.update(det, self.frame_id)+ activated_starcks.append(track)+ else:+ track.re_activate(det, self.frame_id, new_id=False)+ refind_stracks.append(track)++ for it in u_track:+ track = r_tracked_stracks[it]+ if not track.state == TrackState.Lost:+ track.mark_lost()+ lost_stracks.append(track)++ '''Deal with unconfirmed tracks, usually tracks with only one beginning frame'''+ detections = [detections[i] for i in u_detection]+ dists = matching.iou_distance(unconfirmed, detections)+ if not self.args.mot20:+ dists = matching.fuse_score(dists, detections)+ matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7)+ for itracked, idet in matches:+ unconfirmed[itracked].update(detections[idet], self.frame_id)+ activated_starcks.append(unconfirmed[itracked])+ for it in u_unconfirmed:+ track = unconfirmed[it]+ track.mark_removed()+ removed_stracks.append(track)++ """ Step 4: Init new stracks"""+ for inew in u_detection:+ track = detections[inew]+ if track.score < self.det_thresh:+ continue+ track.activate(self.kalman_filter, self.frame_id)+ activated_starcks.append(track)+ """ Step 5: Update state"""+ for track in self.lost_stracks:+ if self.frame_id - track.end_frame > self.max_time_lost:+ track.mark_removed()+ removed_stracks.append(track)++ # print('Ramained match {} s'.format(t4-t3))++ self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked]+ self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks)+ self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks)+ self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks)+ self.lost_stracks.extend(lost_stracks)+ self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks)+ self.removed_stracks.extend(removed_stracks)+ self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks)+ # get scores of lost tracks+ output_stracks = [track for track in self.tracked_stracks if track.is_activated]++ return output_stracks+++def joint_stracks(tlista, tlistb):+ exists = {}+ res = []+ for t in tlista:+ exists[t.track_id] = 1+ res.append(t)+ for t in tlistb:+ tid = t.track_id+ if not exists.get(tid, 0):+ exists[tid] = 1+ res.append(t)+ return res+++def sub_stracks(tlista, tlistb):+ stracks = {}+ for t in tlista:+ stracks[t.track_id] = t+ for t in tlistb:+ tid = t.track_id+ if stracks.get(tid, 0):+ del stracks[tid]+ return list(stracks.values())+++def remove_duplicate_stracks(stracksa, stracksb):+ pdist = matching.iou_distance(stracksa, stracksb)+ pairs = np.where(pdist < 0.15)+ dupa, dupb = list(), list()+ for p, q in zip(*pairs):+ timep = stracksa[p].frame_id - stracksa[p].start_frame+ timeq = stracksb[q].frame_id - stracksb[q].start_frame+ if timep > timeq:+ dupb.append(q)+ else:+ dupa.append(p)+ resa = [t for i, t in enumerate(stracksa) if not i in dupa]+ resb = [t for i, t in enumerate(stracksb) if not i in dupb]+ return resa, resbdiff --git a/experimental/yolox/tracker/kalman_filter.py b/experimental/yolox/tracker/kalman_filter.pynew file mode 100644index 0000000..deda8a2--- /dev/null+++ b/experimental/yolox/tracker/kalman_filter.py@@ -0,0 +1,270 @@+# vim: expandtab:ts=4:sw=4+import numpy as np+import scipy.linalg+++"""+Table for the 0.95 quantile of the chi-square distribution with N degrees of+freedom (contains values for N=1, ..., 9). Taken from MATLAB/Octave's chi2inv+function and used as Mahalanobis gating threshold.+"""+chi2inv95 = {+ 1: 3.8415,+ 2: 5.9915,+ 3: 7.8147,+ 4: 9.4877,+ 5: 11.070,+ 6: 12.592,+ 7: 14.067,+ 8: 15.507,+ 9: 16.919}+++class KalmanFilter(object):+ """+ A simple Kalman filter for tracking bounding boxes in image space.++ The 8-dimensional state space++ x, y, a, h, vx, vy, va, vh++ contains the bounding box center position (x, y), aspect ratio a, height h,+ and their respective velocities.++ Object motion follows a constant velocity model. The bounding box location+ (x, y, a, h) is taken as direct observation of the state space (linear+ observation model).++ """++ def __init__(self):+ ndim, dt = 4, 1.++ # Create Kalman filter model matrices.+ self._motion_mat = np.eye(2 * ndim, 2 * ndim)+ for i in range(ndim):+ self._motion_mat[i, ndim + i] = dt+ self._update_mat = np.eye(ndim, 2 * ndim)++ # Motion and observation uncertainty are chosen relative to the current+ # state estimate. These weights control the amount of uncertainty in+ # the model. This is a bit hacky.+ self._std_weight_position = 1. / 20+ self._std_weight_velocity = 1. / 160++ def initiate(self, measurement):+ """Create track from unassociated measurement.++ Parameters+ ----------+ measurement : ndarray+ Bounding box coordinates (x, y, a, h) with center position (x, y),+ aspect ratio a, and height h.++ Returns+ -------+ (ndarray, ndarray)+ Returns the mean vector (8 dimensional) and covariance matrix (8x8+ dimensional) of the new track. Unobserved velocities are initialized+ to 0 mean.++ """+ mean_pos = measurement+ mean_vel = np.zeros_like(mean_pos)+ mean = np.r_[mean_pos, mean_vel]++ std = [+ 2 * self._std_weight_position * measurement[3],+ 2 * self._std_weight_position * measurement[3],+ 1e-2,+ 2 * self._std_weight_position * measurement[3],+ 10 * self._std_weight_velocity * measurement[3],+ 10 * self._std_weight_velocity * measurement[3],+ 1e-5,+ 10 * self._std_weight_velocity * measurement[3]]+ covariance = np.diag(np.square(std))+ return mean, covariance++ def predict(self, mean, covariance):+ """Run Kalman filter prediction step.++ Parameters+ ----------+ mean : ndarray+ The 8 dimensional mean vector of the object state at the previous+ time step.+ covariance : ndarray+ The 8x8 dimensional covariance matrix of the object state at the+ previous time step.++ Returns+ -------+ (ndarray, ndarray)+ Returns the mean vector and covariance matrix of the predicted+ state. Unobserved velocities are initialized to 0 mean.++ """+ std_pos = [+ self._std_weight_position * mean[3],+ self._std_weight_position * mean[3],+ 1e-2,+ self._std_weight_position * mean[3]]+ std_vel = [+ self._std_weight_velocity * mean[3],+ self._std_weight_velocity * mean[3],+ 1e-5,+ self._std_weight_velocity * mean[3]]+ motion_cov = np.diag(np.square(np.r_[std_pos, std_vel]))++ #mean = np.dot(self._motion_mat, mean)+ mean = np.dot(mean, self._motion_mat.T)+ covariance = np.linalg.multi_dot((+ self._motion_mat, covariance, self._motion_mat.T)) + motion_cov++ return mean, covariance++ def project(self, mean, covariance):+ """Project state distribution to measurement space.++ Parameters+ ----------+ mean : ndarray+ The state's mean vector (8 dimensional array).+ covariance : ndarray+ The state's covariance matrix (8x8 dimensional).++ Returns+ -------+ (ndarray, ndarray)+ Returns the projected mean and covariance matrix of the given state+ estimate.++ """+ std = [+ self._std_weight_position * mean[3],+ self._std_weight_position * mean[3],+ 1e-1,+ self._std_weight_position * mean[3]]+ innovation_cov = np.diag(np.square(std))++ mean = np.dot(self._update_mat, mean)+ covariance = np.linalg.multi_dot((+ self._update_mat, covariance, self._update_mat.T))+ return mean, covariance + innovation_cov++ def multi_predict(self, mean, covariance):+ """Run Kalman filter prediction step (Vectorized version).+ Parameters+ ----------+ mean : ndarray+ The Nx8 dimensional mean matrix of the object states at the previous+ time step.+ covariance : ndarray+ The Nx8x8 dimensional covariance matrics of the object states at the+ previous time step.+ Returns+ -------+ (ndarray, ndarray)+ Returns the mean vector and covariance matrix of the predicted+ state. Unobserved velocities are initialized to 0 mean.+ """+ std_pos = [+ self._std_weight_position * mean[:, 3],+ self._std_weight_position * mean[:, 3],+ 1e-2 * np.ones_like(mean[:, 3]),+ self._std_weight_position * mean[:, 3]]+ std_vel = [+ self._std_weight_velocity * mean[:, 3],+ self._std_weight_velocity * mean[:, 3],+ 1e-5 * np.ones_like(mean[:, 3]),+ self._std_weight_velocity * mean[:, 3]]+ sqr = np.square(np.r_[std_pos, std_vel]).T++ motion_cov = []+ for i in range(len(mean)):+ motion_cov.append(np.diag(sqr[i]))+ motion_cov = np.asarray(motion_cov)++ mean = np.dot(mean, self._motion_mat.T)+ left = np.dot(self._motion_mat, covariance).transpose((1, 0, 2))+ covariance = np.dot(left, self._motion_mat.T) + motion_cov++ return mean, covariance++ def update(self, mean, covariance, measurement):+ """Run Kalman filter correction step.++ Parameters+ ----------+ mean : ndarray+ The predicted state's mean vector (8 dimensional).+ covariance : ndarray+ The state's covariance matrix (8x8 dimensional).+ measurement : ndarray+ The 4 dimensional measurement vector (x, y, a, h), where (x, y)+ is the center position, a the aspect ratio, and h the height of the+ bounding box.++ Returns+ -------+ (ndarray, ndarray)+ Returns the measurement-corrected state distribution.++ """+ projected_mean, projected_cov = self.project(mean, covariance)++ chol_factor, lower = scipy.linalg.cho_factor(+ projected_cov, lower=True, check_finite=False)+ kalman_gain = scipy.linalg.cho_solve(+ (chol_factor, lower), np.dot(covariance, self._update_mat.T).T,+ check_finite=False).T+ innovation = measurement - projected_mean++ new_mean = mean + np.dot(innovation, kalman_gain.T)+ new_covariance = covariance - np.linalg.multi_dot((+ kalman_gain, projected_cov, kalman_gain.T))+ return new_mean, new_covariance++ def gating_distance(self, mean, covariance, measurements,+ only_position=False, metric='maha'):+ """Compute gating distance between state distribution and measurements.+ A suitable distance threshold can be obtained from `chi2inv95`. If+ `only_position` is False, the chi-square distribution has 4 degrees of+ freedom, otherwise 2.+ Parameters+ ----------+ mean : ndarray+ Mean vector over the state distribution (8 dimensional).+ covariance : ndarray+ Covariance of the state distribution (8x8 dimensional).+ measurements : ndarray+ An Nx4 dimensional matrix of N measurements, each in+ format (x, y, a, h) where (x, y) is the bounding box center+ position, a the aspect ratio, and h the height.+ only_position : Optional[bool]+ If True, distance computation is done with respect to the bounding+ box center position only.+ Returns+ -------+ ndarray+ Returns an array of length N, where the i-th element contains the+ squared Mahalanobis distance between (mean, covariance) and+ `measurements[i]`.+ """+ mean, covariance = self.project(mean, covariance)+ if only_position:+ mean, covariance = mean[:2], covariance[:2, :2]+ measurements = measurements[:, :2]++ d = measurements - mean+ if metric == 'gaussian':+ return np.sum(d * d, axis=1)+ elif metric == 'maha':+ cholesky_factor = np.linalg.cholesky(covariance)+ z = scipy.linalg.solve_triangular(+ cholesky_factor, d.T, lower=True, check_finite=False,+ overwrite_b=True)+ squared_maha = np.sum(z * z, axis=0)+ return squared_maha+ else:+ raise ValueError('invalid distance metric')\ No newline at end of filediff --git a/experimental/yolox/tracker/matching.py b/experimental/yolox/tracker/matching.pynew file mode 100644index 0000000..d36a6cf--- /dev/null+++ b/experimental/yolox/tracker/matching.py@@ -0,0 +1,181 @@+import cv2+import numpy as np+import scipy+import lap+from scipy.spatial.distance import cdist++from cython_bbox import bbox_overlaps as bbox_ious+from yolox.tracker import kalman_filter+import time++def merge_matches(m1, m2, shape):+ O,P,Q = shape+ m1 = np.asarray(m1)+ m2 = np.asarray(m2)++ M1 = scipy.sparse.coo_matrix((np.ones(len(m1)), (m1[:, 0], m1[:, 1])), shape=(O, P))+ M2 = scipy.sparse.coo_matrix((np.ones(len(m2)), (m2[:, 0], m2[:, 1])), shape=(P, Q))++ mask = M1*M2+ match = mask.nonzero()+ match = list(zip(match[0], match[1]))+ unmatched_O = tuple(set(range(O)) - set([i for i, j in match]))+ unmatched_Q = tuple(set(range(Q)) - set([j for i, j in match]))++ return match, unmatched_O, unmatched_Q+++def _indices_to_matches(cost_matrix, indices, thresh):+ matched_cost = cost_matrix[tuple(zip(*indices))]+ matched_mask = (matched_cost <= thresh)++ matches = indices[matched_mask]+ unmatched_a = tuple(set(range(cost_matrix.shape[0])) - set(matches[:, 0]))+ unmatched_b = tuple(set(range(cost_matrix.shape[1])) - set(matches[:, 1]))++ return matches, unmatched_a, unmatched_b+++def linear_assignment(cost_matrix, thresh):+ if cost_matrix.size == 0:+ return np.empty((0, 2), dtype=int), tuple(range(cost_matrix.shape[0])), tuple(range(cost_matrix.shape[1]))+ matches, unmatched_a, unmatched_b = [], [], []+ cost, x, y = lap.lapjv(cost_matrix, extend_cost=True, cost_limit=thresh)+ for ix, mx in enumerate(x):+ if mx >= 0:+ matches.append([ix, mx])+ unmatched_a = np.where(x < 0)[0]+ unmatched_b = np.where(y < 0)[0]+ matches = np.asarray(matches)+ return matches, unmatched_a, unmatched_b+++def ious(atlbrs, btlbrs):+ """+ Compute cost based on IoU+ :type atlbrs: list[tlbr] | np.ndarray+ :type atlbrs: list[tlbr] | np.ndarray++ :rtype ious np.ndarray+ """+ ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float)+ if ious.size == 0:+ return ious++ ious = bbox_ious(+ np.ascontiguousarray(atlbrs, dtype=np.float),+ np.ascontiguousarray(btlbrs, dtype=np.float)+ )++ return ious+++def iou_distance(atracks, btracks):+ """+ Compute cost based on IoU+ :type atracks: list[STrack]+ :type btracks: list[STrack]++ :rtype cost_matrix np.ndarray+ """++ if (len(atracks)>0 and isinstance(atracks[0], np.ndarray)) or (len(btracks) > 0 and isinstance(btracks[0], np.ndarray)):+ atlbrs = atracks+ btlbrs = btracks+ else:+ atlbrs = [track.tlbr for track in atracks]+ btlbrs = [track.tlbr for track in btracks]+ _ious = ious(atlbrs, btlbrs)+ cost_matrix = 1 - _ious++ return cost_matrix++def v_iou_distance(atracks, btracks):+ """+ Compute cost based on IoU+ :type atracks: list[STrack]+ :type btracks: list[STrack]++ :rtype cost_matrix np.ndarray+ """++ if (len(atracks)>0 and isinstance(atracks[0], np.ndarray)) or (len(btracks) > 0 and isinstance(btracks[0], np.ndarray)):+ atlbrs = atracks+ btlbrs = btracks+ else:+ atlbrs = [track.tlwh_to_tlbr(track.pred_bbox) for track in atracks]+ btlbrs = [track.tlwh_to_tlbr(track.pred_bbox) for track in btracks]+ _ious = ious(atlbrs, btlbrs)+ cost_matrix = 1 - _ious++ return cost_matrix++def embedding_distance(tracks, detections, metric='cosine'):+ """+ :param tracks: list[STrack]+ :param detections: list[BaseTrack]+ :param metric:+ :return: cost_matrix np.ndarray+ """++ cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float)+ if cost_matrix.size == 0:+ return cost_matrix+ det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float)+ #for i, track in enumerate(tracks):+ #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric))+ track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float)+ cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features+ return cost_matrix+++def gate_cost_matrix(kf, cost_matrix, tracks, detections, only_position=False):+ if cost_matrix.size == 0:+ return cost_matrix+ gating_dim = 2 if only_position else 4+ gating_threshold = kalman_filter.chi2inv95[gating_dim]+ measurements = np.asarray([det.to_xyah() for det in detections])+ for row, track in enumerate(tracks):+ gating_distance = kf.gating_distance(+ track.mean, track.covariance, measurements, only_position)+ cost_matrix[row, gating_distance > gating_threshold] = np.inf+ return cost_matrix+++def fuse_motion(kf, cost_matrix, tracks, detections, only_position=False, lambda_=0.98):+ if cost_matrix.size == 0:+ return cost_matrix+ gating_dim = 2 if only_position else 4+ gating_threshold = kalman_filter.chi2inv95[gating_dim]+ measurements = np.asarray([det.to_xyah() for det in detections])+ for row, track in enumerate(tracks):+ gating_distance = kf.gating_distance(+ track.mean, track.covariance, measurements, only_position, metric='maha')+ cost_matrix[row, gating_distance > gating_threshold] = np.inf+ cost_matrix[row] = lambda_ * cost_matrix[row] + (1 - lambda_) * gating_distance+ return cost_matrix+++def fuse_iou(cost_matrix, tracks, detections):+ if cost_matrix.size == 0:+ return cost_matrix+ reid_sim = 1 - cost_matrix+ iou_dist = iou_distance(tracks, detections)+ iou_sim = 1 - iou_dist+ fuse_sim = reid_sim * (1 + iou_sim) / 2+ det_scores = np.array([det.score for det in detections])+ det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0)+ #fuse_sim = fuse_sim * (1 + det_scores) / 2+ fuse_cost = 1 - fuse_sim+ return fuse_cost+++def fuse_score(cost_matrix, detections):+ if cost_matrix.size == 0:+ return cost_matrix+ iou_sim = 1 - cost_matrix+ det_scores = np.array([det.score for det in detections])+ det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0)+ fuse_sim = iou_sim * det_scores+ fuse_cost = 1 - fuse_sim+ return fuse_cost\ No newline at end of filediff --git a/experimental/yolox/tracking_utils/evaluation.py b/experimental/yolox/tracking_utils/evaluation.pynew file mode 100644index 0000000..6e2f063--- /dev/null+++ b/experimental/yolox/tracking_utils/evaluation.py@@ -0,0 +1,113 @@+import os+import numpy as np+import copy+import motmetrics as mm+mm.lap.default_solver = 'lap'++from yolox.tracking_utils.io import read_results, unzip_objs+++class Evaluator(object):++ def __init__(self, data_root, seq_name, data_type):+ self.data_root = data_root+ self.seq_name = seq_name+ self.data_type = data_type++ self.load_annotations()+ self.reset_accumulator()++ def load_annotations(self):+ assert self.data_type == 'mot'++ gt_filename = os.path.join(self.data_root, self.seq_name, 'gt', 'gt.txt')+ self.gt_frame_dict = read_results(gt_filename, self.data_type, is_gt=True)+ self.gt_ignore_frame_dict = read_results(gt_filename, self.data_type, is_ignore=True)++ def reset_accumulator(self):+ self.acc = mm.MOTAccumulator(auto_id=True)++ def eval_frame(self, frame_id, trk_tlwhs, trk_ids, rtn_events=False):+ # results+ trk_tlwhs = np.copy(trk_tlwhs)+ trk_ids = np.copy(trk_ids)++ # gts+ gt_objs = self.gt_frame_dict.get(frame_id, [])+ gt_tlwhs, gt_ids = unzip_objs(gt_objs)[:2]++ # ignore boxes+ ignore_objs = self.gt_ignore_frame_dict.get(frame_id, [])+ ignore_tlwhs = unzip_objs(ignore_objs)[0]++ # remove ignored results+ keep = np.ones(len(trk_tlwhs), dtype=bool)+ iou_distance = mm.distances.iou_matrix(ignore_tlwhs, trk_tlwhs, max_iou=0.5)+ if len(iou_distance) > 0:+ match_is, match_js = mm.lap.linear_sum_assignment(iou_distance)+ match_is, match_js = map(lambda a: np.asarray(a, dtype=int), [match_is, match_js])+ match_ious = iou_distance[match_is, match_js]++ match_js = np.asarray(match_js, dtype=int)+ match_js = match_js[np.logical_not(np.isnan(match_ious))]+ keep[match_js] = False+ trk_tlwhs = trk_tlwhs[keep]+ trk_ids = trk_ids[keep]+ #match_is, match_js = mm.lap.linear_sum_assignment(iou_distance)+ #match_is, match_js = map(lambda a: np.asarray(a, dtype=int), [match_is, match_js])+ #match_ious = iou_distance[match_is, match_js]++ #match_js = np.asarray(match_js, dtype=int)+ #match_js = match_js[np.logical_not(np.isnan(match_ious))]+ #keep[match_js] = False+ #trk_tlwhs = trk_tlwhs[keep]+ #trk_ids = trk_ids[keep]++ # get distance matrix+ iou_distance = mm.distances.iou_matrix(gt_tlwhs, trk_tlwhs, max_iou=0.5)++ # acc+ self.acc.update(gt_ids, trk_ids, iou_distance)++ if rtn_events and iou_distance.size > 0 and hasattr(self.acc, 'last_mot_events'):+ events = self.acc.last_mot_events # only supported by https://github.com/longcw/py-motmetrics+ else:+ events = None+ return events++ def eval_file(self, filename):+ self.reset_accumulator()++ result_frame_dict = read_results(filename, self.data_type, is_gt=False)+ #frames = sorted(list(set(self.gt_frame_dict.keys()) | set(result_frame_dict.keys())))+ frames = sorted(list(set(result_frame_dict.keys())))+ for frame_id in frames:+ trk_objs = result_frame_dict.get(frame_id, [])+ trk_tlwhs, trk_ids = unzip_objs(trk_objs)[:2]+ self.eval_frame(frame_id, trk_tlwhs, trk_ids, rtn_events=False)++ return self.acc++ @staticmethod+ def get_summary(accs, names, metrics=('mota', 'num_switches', 'idp', 'idr', 'idf1', 'precision', 'recall')):+ names = copy.deepcopy(names)+ if metrics is None:+ metrics = mm.metrics.motchallenge_metrics+ metrics = copy.deepcopy(metrics)++ mh = mm.metrics.create()+ summary = mh.compute_many(+ accs,+ metrics=metrics,+ names=names,+ generate_overall=True+ )++ return summary++ @staticmethod+ def save_summary(summary, filename):+ import pandas as pd+ writer = pd.ExcelWriter(filename)+ summary.to_excel(writer)+ writer.save()\ No newline at end of filediff --git a/experimental/yolox/tracking_utils/io.py b/experimental/yolox/tracking_utils/io.pynew file mode 100644index 0000000..5c8b053--- /dev/null+++ b/experimental/yolox/tracking_utils/io.py@@ -0,0 +1,116 @@+import os+from typing import Dict+import numpy as np+++def write_results(filename, results_dict: Dict, data_type: str):+ if not filename:+ return+ path = os.path.dirname(filename)+ if not os.path.exists(path):+ os.makedirs(path)++ if data_type in ('mot', 'mcmot', 'lab'):+ save_format = '{frame},{id},{x1},{y1},{w},{h},1,-1,-1,-1\n'+ elif data_type == 'kitti':+ save_format = '{frame} {id} pedestrian -1 -1 -10 {x1} {y1} {x2} {y2} -1 -1 -1 -1000 -1000 -1000 -10 {score}\n'+ else:+ raise ValueError(data_type)++ with open(filename, 'w') as f:+ for frame_id, frame_data in results_dict.items():+ if data_type == 'kitti':+ frame_id -= 1+ for tlwh, track_id in frame_data:+ if track_id < 0:+ continue+ x1, y1, w, h = tlwh+ x2, y2 = x1 + w, y1 + h+ line = save_format.format(frame=frame_id, id=track_id, x1=x1, y1=y1, x2=x2, y2=y2, w=w, h=h, score=1.0)+ f.write(line)+++def read_results(filename, data_type: str, is_gt=False, is_ignore=False):+ if data_type in ('mot', 'lab'):+ read_fun = read_mot_results+ else:+ raise ValueError('Unknown data type: {}'.format(data_type))++ return read_fun(filename, is_gt, is_ignore)+++"""+labels={'ped', ... % 1+'person_on_vhcl', ... % 2+'car', ... % 3+'bicycle', ... % 4+'mbike', ... % 5+'non_mot_vhcl', ... % 6+'static_person', ... % 7+'distractor', ... % 8+'occluder', ... % 9+'occluder_on_grnd', ... %10+'occluder_full', ... % 11+'reflection', ... % 12+'crowd' ... % 13+};+"""+++def read_mot_results(filename, is_gt, is_ignore):+ valid_labels = {1}+ ignore_labels = {2, 7, 8, 12}+ results_dict = dict()+ if os.path.isfile(filename):+ with open(filename, 'r') as f:+ for line in f.readlines():+ linelist = line.split(',')+ if len(linelist) < 7:+ continue+ fid = int(linelist[0])+ if fid < 1:+ continue+ results_dict.setdefault(fid, list())++ box_size = float(linelist[4]) * float(linelist[5])++ if is_gt:+ if 'MOT16-' in filename or 'MOT17-' in filename:+ label = int(float(linelist[7]))+ mark = int(float(linelist[6]))+ if mark == 0 or label not in valid_labels:+ continue+ score = 1+ elif is_ignore:+ if 'MOT16-' in filename or 'MOT17-' in filename:+ label = int(float(linelist[7]))+ vis_ratio = float(linelist[8])+ if label not in ignore_labels and vis_ratio >= 0:+ continue+ else:+ continue+ score = 1+ else:+ score = float(linelist[6])++ #if box_size > 7000:+ #if box_size <= 7000 or box_size >= 15000:+ #if box_size < 15000:+ #continue++ tlwh = tuple(map(float, linelist[2:6]))+ target_id = int(linelist[1])++ results_dict[fid].append((tlwh, target_id, score))++ return results_dict+++def unzip_objs(objs):+ if len(objs) > 0:+ tlwhs, ids, scores = zip(*objs)+ else:+ tlwhs, ids, scores = [], [], []+ tlwhs = np.asarray(tlwhs, dtype=float).reshape(-1, 4)++ return tlwhs, ids, scores\ No newline at end of filediff --git a/experimental/yolox/tracking_utils/timer.py b/experimental/yolox/tracking_utils/timer.pynew file mode 100644index 0000000..c9b15fb--- /dev/null+++ b/experimental/yolox/tracking_utils/timer.py@@ -0,0 +1,37 @@+import time+++class Timer(object):+ """A simple timer."""+ def __init__(self):+ self.total_time = 0.+ self.calls = 0+ self.start_time = 0.+ self.diff = 0.+ self.average_time = 0.++ self.duration = 0.++ def tic(self):+ # using time.time instead of time.clock because time time.clock+ # does not normalize for multithreading+ self.start_time = time.time()++ def toc(self, average=True):+ self.diff = time.time() - self.start_time+ self.total_time += self.diff+ self.calls += 1+ self.average_time = self.total_time / self.calls+ if average:+ self.duration = self.average_time+ else:+ self.duration = self.diff+ return self.duration++ def clear(self):+ self.total_time = 0.+ self.calls = 0+ self.start_time = 0.+ self.diff = 0.+ self.average_time = 0.+ self.duration = 0.\ No newline at end of filediff --git a/experimental/yolox/utils/__init__.py b/experimental/yolox/utils/__init__.pynew file mode 100644index 0000000..a268c1a--- /dev/null+++ b/experimental/yolox/utils/__init__.py@@ -0,0 +1,16 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++from .allreduce_norm import *+from .boxes import *+from .checkpoint import load_ckpt, save_checkpoint+from .demo_utils import *+from .dist import *+from .ema import ModelEMA+from .logger import setup_logger+from .lr_scheduler import LRScheduler+from .metric import *+from .model_utils import *+from .setup_env import *+from .visualize import *diff --git a/experimental/yolox/utils/__pycache__/__init__.cpython-310.pyc b/experimental/yolox/utils/__pycache__/__init__.cpython-310.pycnew file mode 100644index 0000000..4e40556Binary files /dev/null and b/experimental/yolox/utils/__pycache__/__init__.cpython-310.pyc differdiff --git a/experimental/yolox/utils/__pycache__/allreduce_norm.cpython-310.pyc b/experimental/yolox/utils/__pycache__/allreduce_norm.cpython-310.pycnew file mode 100644index 0000000..645044dBinary files /dev/null and b/experimental/yolox/utils/__pycache__/allreduce_norm.cpython-310.pyc differdiff --git a/experimental/yolox/utils/__pycache__/boxes.cpython-310.pyc b/experimental/yolox/utils/__pycache__/boxes.cpython-310.pycnew file mode 100644index 0000000..dafe9e8Binary files /dev/null and b/experimental/yolox/utils/__pycache__/boxes.cpython-310.pyc differdiff --git a/experimental/yolox/utils/__pycache__/checkpoint.cpython-310.pyc b/experimental/yolox/utils/__pycache__/checkpoint.cpython-310.pycnew file mode 100644index 0000000..816b801Binary files /dev/null and b/experimental/yolox/utils/__pycache__/checkpoint.cpython-310.pyc differdiff --git a/experimental/yolox/utils/__pycache__/demo_utils.cpython-310.pyc b/experimental/yolox/utils/__pycache__/demo_utils.cpython-310.pycnew file mode 100644index 0000000..b7a8e50Binary files /dev/null and b/experimental/yolox/utils/__pycache__/demo_utils.cpython-310.pyc differdiff --git a/experimental/yolox/utils/__pycache__/dist.cpython-310.pyc b/experimental/yolox/utils/__pycache__/dist.cpython-310.pycnew file mode 100644index 0000000..a6ca6adBinary files /dev/null and b/experimental/yolox/utils/__pycache__/dist.cpython-310.pyc differdiff --git a/experimental/yolox/utils/__pycache__/ema.cpython-310.pyc b/experimental/yolox/utils/__pycache__/ema.cpython-310.pycnew file mode 100644index 0000000..a64001fBinary files /dev/null and b/experimental/yolox/utils/__pycache__/ema.cpython-310.pyc differdiff --git a/experimental/yolox/utils/__pycache__/logger.cpython-310.pyc b/experimental/yolox/utils/__pycache__/logger.cpython-310.pycnew file mode 100644index 0000000..ad78568Binary files /dev/null and b/experimental/yolox/utils/__pycache__/logger.cpython-310.pyc differdiff --git a/experimental/yolox/utils/__pycache__/lr_scheduler.cpython-310.pyc b/experimental/yolox/utils/__pycache__/lr_scheduler.cpython-310.pycnew file mode 100644index 0000000..710bddbBinary files /dev/null and b/experimental/yolox/utils/__pycache__/lr_scheduler.cpython-310.pyc differdiff --git a/experimental/yolox/utils/__pycache__/metric.cpython-310.pyc b/experimental/yolox/utils/__pycache__/metric.cpython-310.pycnew file mode 100644index 0000000..38fc519Binary files /dev/null and b/experimental/yolox/utils/__pycache__/metric.cpython-310.pyc differdiff --git a/experimental/yolox/utils/__pycache__/model_utils.cpython-310.pyc b/experimental/yolox/utils/__pycache__/model_utils.cpython-310.pycnew file mode 100644index 0000000..f877c02Binary files /dev/null and b/experimental/yolox/utils/__pycache__/model_utils.cpython-310.pyc differdiff --git a/experimental/yolox/utils/__pycache__/setup_env.cpython-310.pyc b/experimental/yolox/utils/__pycache__/setup_env.cpython-310.pycnew file mode 100644index 0000000..3bc6a51Binary files /dev/null and b/experimental/yolox/utils/__pycache__/setup_env.cpython-310.pyc differdiff --git a/experimental/yolox/utils/__pycache__/visualize.cpython-310.pyc b/experimental/yolox/utils/__pycache__/visualize.cpython-310.pycnew file mode 100644index 0000000..6d019beBinary files /dev/null and b/experimental/yolox/utils/__pycache__/visualize.cpython-310.pyc differdiff --git a/experimental/yolox/utils/allreduce_norm.py b/experimental/yolox/utils/allreduce_norm.pynew file mode 100644index 0000000..d9b51e2--- /dev/null+++ b/experimental/yolox/utils/allreduce_norm.py@@ -0,0 +1,103 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import torch+from torch import distributed as dist+from torch import nn++import pickle+from collections import OrderedDict++from .dist import _get_global_gloo_group, get_world_size++ASYNC_NORM = (+ nn.BatchNorm1d,+ nn.BatchNorm2d,+ nn.BatchNorm3d,+ nn.InstanceNorm1d,+ nn.InstanceNorm2d,+ nn.InstanceNorm3d,+)++__all__ = [+ "get_async_norm_states",+ "pyobj2tensor",+ "tensor2pyobj",+ "all_reduce",+ "all_reduce_norm",+]+++def get_async_norm_states(module):+ async_norm_states = OrderedDict()+ for name, child in module.named_modules():+ if isinstance(child, ASYNC_NORM):+ for k, v in child.state_dict().items():+ async_norm_states[".".join([name, k])] = v+ return async_norm_states+++def pyobj2tensor(pyobj, device="cuda"):+ """serialize picklable python object to tensor"""+ storage = torch.ByteStorage.from_buffer(pickle.dumps(pyobj))+ return torch.ByteTensor(storage).to(device=device)+++def tensor2pyobj(tensor):+ """deserialize tensor to picklable python object"""+ return pickle.loads(tensor.cpu().numpy().tobytes())+++def _get_reduce_op(op_name):+ return {+ "sum": dist.ReduceOp.SUM,+ "mean": dist.ReduceOp.SUM,+ }[op_name.lower()]+++def all_reduce(py_dict, op="sum", group=None):+ """+ Apply all reduce function for python dict object.+ NOTE: make sure that every py_dict has the same keys and values are in the same shape.++ Args:+ py_dict (dict): dict to apply all reduce op.+ op (str): operator, could be "sum" or "mean".+ """+ world_size = get_world_size()+ if world_size == 1:+ return py_dict+ if group is None:+ group = _get_global_gloo_group()+ if dist.get_world_size(group) == 1:+ return py_dict++ # all reduce logic across different devices.+ py_key = list(py_dict.keys())+ py_key_tensor = pyobj2tensor(py_key)+ dist.broadcast(py_key_tensor, src=0)+ py_key = tensor2pyobj(py_key_tensor)++ tensor_shapes = [py_dict[k].shape for k in py_key]+ tensor_numels = [py_dict[k].numel() for k in py_key]++ flatten_tensor = torch.cat([py_dict[k].flatten() for k in py_key])+ dist.all_reduce(flatten_tensor, op=_get_reduce_op(op))+ if op == "mean":+ flatten_tensor /= world_size++ split_tensors = [+ x.reshape(shape)+ for x, shape in zip(torch.split(flatten_tensor, tensor_numels), tensor_shapes)+ ]+ return OrderedDict({k: v for k, v in zip(py_key, split_tensors)})+++def all_reduce_norm(module):+ """+ All reduce norm statistics in different devices.+ """+ states = get_async_norm_states(module)+ states = all_reduce(states, op="mean")+ module.load_state_dict(states, strict=False)diff --git a/experimental/yolox/utils/boxes.py b/experimental/yolox/utils/boxes.pynew file mode 100644index 0000000..ac262b9--- /dev/null+++ b/experimental/yolox/utils/boxes.py@@ -0,0 +1,133 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import numpy as np++import torch+import torchvision+import torch.nn.functional as F++__all__ = [+ "filter_box",+ "postprocess",+ "bboxes_iou",+ "matrix_iou",+ "adjust_box_anns",+ "xyxy2xywh",+ "xyxy2cxcywh",+]+++def filter_box(output, scale_range):+ """+ output: (N, 5+class) shape+ """+ min_scale, max_scale = scale_range+ w = output[:, 2] - output[:, 0]+ h = output[:, 3] - output[:, 1]+ keep = (w * h > min_scale * min_scale) & (w * h < max_scale * max_scale)+ return output[keep]+++def postprocess(prediction, num_classes, conf_thre=0.7, nms_thre=0.45):+ box_corner = prediction.new(prediction.shape)+ box_corner[:, :, 0] = prediction[:, :, 0] - prediction[:, :, 2] / 2+ box_corner[:, :, 1] = prediction[:, :, 1] - prediction[:, :, 3] / 2+ box_corner[:, :, 2] = prediction[:, :, 0] + prediction[:, :, 2] / 2+ box_corner[:, :, 3] = prediction[:, :, 1] + prediction[:, :, 3] / 2+ prediction[:, :, :4] = box_corner[:, :, :4]++ output = [None for _ in range(len(prediction))]+ for i, image_pred in enumerate(prediction):++ # If none are remaining => process next image+ if not image_pred.size(0):+ continue+ # Get score and class with highest confidence+ class_conf, class_pred = torch.max(+ image_pred[:, 5 : 5 + num_classes], 1, keepdim=True+ )++ conf_mask = (image_pred[:, 4] * class_conf.squeeze() >= conf_thre).squeeze()+ # _, conf_mask = torch.topk((image_pred[:, 4] * class_conf.squeeze()), 1000)+ # Detections ordered as (x1, y1, x2, y2, obj_conf, class_conf, class_pred)+ detections = torch.cat((image_pred[:, :5], class_conf, class_pred.float()), 1)+ detections = detections[conf_mask]+ if not detections.size(0):+ continue++ nms_out_index = torchvision.ops.batched_nms(+ detections[:, :4],+ detections[:, 4] * detections[:, 5],+ detections[:, 6],+ nms_thre,+ )+ detections = detections[nms_out_index]+ if output[i] is None:+ output[i] = detections+ else:+ output[i] = torch.cat((output[i], detections))++ return output+++def bboxes_iou(bboxes_a, bboxes_b, xyxy=True):+ if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4:+ raise IndexError++ if xyxy:+ tl = torch.max(bboxes_a[:, None, :2], bboxes_b[:, :2])+ br = torch.min(bboxes_a[:, None, 2:], bboxes_b[:, 2:])+ area_a = torch.prod(bboxes_a[:, 2:] - bboxes_a[:, :2], 1)+ area_b = torch.prod(bboxes_b[:, 2:] - bboxes_b[:, :2], 1)+ else:+ tl = torch.max(+ (bboxes_a[:, None, :2] - bboxes_a[:, None, 2:] / 2),+ (bboxes_b[:, :2] - bboxes_b[:, 2:] / 2),+ )+ br = torch.min(+ (bboxes_a[:, None, :2] + bboxes_a[:, None, 2:] / 2),+ (bboxes_b[:, :2] + bboxes_b[:, 2:] / 2),+ )++ area_a = torch.prod(bboxes_a[:, 2:], 1)+ area_b = torch.prod(bboxes_b[:, 2:], 1)+ en = (tl < br).type(tl.type()).prod(dim=2)+ area_i = torch.prod(br - tl, 2) * en # * ((tl < br).all())+ return area_i / (area_a[:, None] + area_b - area_i)+++def matrix_iou(a, b):+ """+ return iou of a and b, numpy version for data augenmentation+ """+ lt = np.maximum(a[:, np.newaxis, :2], b[:, :2])+ rb = np.minimum(a[:, np.newaxis, 2:], b[:, 2:])++ area_i = np.prod(rb - lt, axis=2) * (lt < rb).all(axis=2)+ area_a = np.prod(a[:, 2:] - a[:, :2], axis=1)+ area_b = np.prod(b[:, 2:] - b[:, :2], axis=1)+ return area_i / (area_a[:, np.newaxis] + area_b - area_i + 1e-12)+++def adjust_box_anns(bbox, scale_ratio, padw, padh, w_max, h_max):+ #bbox[:, 0::2] = np.clip(bbox[:, 0::2] * scale_ratio + padw, 0, w_max)+ #bbox[:, 1::2] = np.clip(bbox[:, 1::2] * scale_ratio + padh, 0, h_max)+ bbox[:, 0::2] = bbox[:, 0::2] * scale_ratio + padw+ bbox[:, 1::2] = bbox[:, 1::2] * scale_ratio + padh+ return bbox+++def xyxy2xywh(bboxes):+ bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0]+ bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1]+ return bboxes+++def xyxy2cxcywh(bboxes):+ bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0]+ bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1]+ bboxes[:, 0] = bboxes[:, 0] + bboxes[:, 2] * 0.5+ bboxes[:, 1] = bboxes[:, 1] + bboxes[:, 3] * 0.5+ return bboxesdiff --git a/experimental/yolox/utils/checkpoint.py b/experimental/yolox/utils/checkpoint.pynew file mode 100644index 0000000..55903b4--- /dev/null+++ b/experimental/yolox/utils/checkpoint.py@@ -0,0 +1,44 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.+from loguru import logger++import torch++import os+import shutil+++def load_ckpt(model, ckpt):+ model_state_dict = model.state_dict()+ load_dict = {}+ for key_model, v in model_state_dict.items():+ if key_model not in ckpt:+ logger.warning(+ "{} is not in the ckpt. Please double check and see if this is desired.".format(+ key_model+ )+ )+ continue+ v_ckpt = ckpt[key_model]+ if v.shape != v_ckpt.shape:+ logger.warning(+ "Shape of {} in checkpoint is {}, while shape of {} in model is {}.".format(+ key_model, v_ckpt.shape, key_model, v.shape+ )+ )+ continue+ load_dict[key_model] = v_ckpt++ model.load_state_dict(load_dict, strict=False)+ return model+++def save_checkpoint(state, is_best, save_dir, model_name=""):+ if not os.path.exists(save_dir):+ os.makedirs(save_dir)+ filename = os.path.join(save_dir, model_name + "_ckpt.pth.tar")+ torch.save(state, filename)+ if is_best:+ best_filename = os.path.join(save_dir, "best_ckpt.pth.tar")+ shutil.copyfile(filename, best_filename)diff --git a/experimental/yolox/utils/demo_utils.py b/experimental/yolox/utils/demo_utils.pynew file mode 100644index 0000000..093443c--- /dev/null+++ b/experimental/yolox/utils/demo_utils.py@@ -0,0 +1,96 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import numpy as np++import os++__all__ = ["mkdir", "nms", "multiclass_nms", "demo_postprocess"]+++def mkdir(path):+ if not os.path.exists(path):+ os.makedirs(path)+++def nms(boxes, scores, nms_thr):+ """Single class NMS implemented in Numpy."""+ x1 = boxes[:, 0]+ y1 = boxes[:, 1]+ x2 = boxes[:, 2]+ y2 = boxes[:, 3]++ areas = (x2 - x1 + 1) * (y2 - y1 + 1)+ order = scores.argsort()[::-1]++ keep = []+ while order.size > 0:+ i = order[0]+ keep.append(i)+ xx1 = np.maximum(x1[i], x1[order[1:]])+ yy1 = np.maximum(y1[i], y1[order[1:]])+ xx2 = np.minimum(x2[i], x2[order[1:]])+ yy2 = np.minimum(y2[i], y2[order[1:]])++ w = np.maximum(0.0, xx2 - xx1 + 1)+ h = np.maximum(0.0, yy2 - yy1 + 1)+ inter = w * h+ ovr = inter / (areas[i] + areas[order[1:]] - inter)++ inds = np.where(ovr <= nms_thr)[0]+ order = order[inds + 1]++ return keep+++def multiclass_nms(boxes, scores, nms_thr, score_thr):+ """Multiclass NMS implemented in Numpy"""+ final_dets = []+ num_classes = scores.shape[1]+ for cls_ind in range(num_classes):+ cls_scores = scores[:, cls_ind]+ valid_score_mask = cls_scores > score_thr+ if valid_score_mask.sum() == 0:+ continue+ else:+ valid_scores = cls_scores[valid_score_mask]+ valid_boxes = boxes[valid_score_mask]+ keep = nms(valid_boxes, valid_scores, nms_thr)+ if len(keep) > 0:+ cls_inds = np.ones((len(keep), 1)) * cls_ind+ dets = np.concatenate(+ [valid_boxes[keep], valid_scores[keep, None], cls_inds], 1+ )+ final_dets.append(dets)+ if len(final_dets) == 0:+ return None+ return np.concatenate(final_dets, 0)+++def demo_postprocess(outputs, img_size, p6=False):++ grids = []+ expanded_strides = []++ if not p6:+ strides = [8, 16, 32]+ else:+ strides = [8, 16, 32, 64]++ hsizes = [img_size[0] // stride for stride in strides]+ wsizes = [img_size[1] // stride for stride in strides]++ for hsize, wsize, stride in zip(hsizes, wsizes, strides):+ xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize))+ grid = np.stack((xv, yv), 2).reshape(1, -1, 2)+ grids.append(grid)+ shape = grid.shape[:2]+ expanded_strides.append(np.full((*shape, 1), stride))++ grids = np.concatenate(grids, 1)+ expanded_strides = np.concatenate(expanded_strides, 1)+ outputs[..., :2] = (outputs[..., :2] + grids) * expanded_strides+ outputs[..., 2:4] = np.exp(outputs[..., 2:4]) * expanded_strides++ return outputsdiff --git a/experimental/yolox/utils/dist.py b/experimental/yolox/utils/dist.pynew file mode 100644index 0000000..691c306--- /dev/null+++ b/experimental/yolox/utils/dist.py@@ -0,0 +1,255 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# This file mainly comes from+# https://github.com/facebookresearch/detectron2/blob/master/detectron2/utils/comm.py+# Copyright (c) Facebook, Inc. and its affiliates.+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.+"""+This file contains primitives for multi-gpu communication.+This is useful when doing distributed training.+"""++import numpy as np++import torch+from torch import distributed as dist++import functools+import logging+import pickle+import time++__all__ = [+ "is_main_process",+ "synchronize",+ "get_world_size",+ "get_rank",+ "get_local_rank",+ "get_local_size",+ "time_synchronized",+ "gather",+ "all_gather",+]++_LOCAL_PROCESS_GROUP = None+++def synchronize():+ """+ Helper function to synchronize (barrier) among all processes when using distributed training+ """+ if not dist.is_available():+ return+ if not dist.is_initialized():+ return+ world_size = dist.get_world_size()+ if world_size == 1:+ return+ dist.barrier()+++def get_world_size() -> int:+ if not dist.is_available():+ return 1+ if not dist.is_initialized():+ return 1+ return dist.get_world_size()+++def get_rank() -> int:+ if not dist.is_available():+ return 0+ if not dist.is_initialized():+ return 0+ return dist.get_rank()+++def get_local_rank() -> int:+ """+ Returns:+ The rank of the current process within the local (per-machine) process group.+ """+ if not dist.is_available():+ return 0+ if not dist.is_initialized():+ return 0+ assert _LOCAL_PROCESS_GROUP is not None+ return dist.get_rank(group=_LOCAL_PROCESS_GROUP)+++def get_local_size() -> int:+ """+ Returns:+ The size of the per-machine process group, i.e. the number of processes per machine.+ """+ if not dist.is_available():+ return 1+ if not dist.is_initialized():+ return 1+ return dist.get_world_size(group=_LOCAL_PROCESS_GROUP)+++def is_main_process() -> bool:+ return get_rank() == 0++[email protected]_cache()+def _get_global_gloo_group():+ """+ Return a process group based on gloo backend, containing all the ranks+ The result is cached.+ """+ if dist.get_backend() == "nccl":+ return dist.new_group(backend="gloo")+ else:+ return dist.group.WORLD+++def _serialize_to_tensor(data, group):+ backend = dist.get_backend(group)+ assert backend in ["gloo", "nccl"]+ device = torch.device("cpu" if backend == "gloo" else "cuda")++ buffer = pickle.dumps(data)+ if len(buffer) > 1024 ** 3:+ logger = logging.getLogger(__name__)+ logger.warning(+ "Rank {} trying to all-gather {:.2f} GB of data on device {}".format(+ get_rank(), len(buffer) / (1024 ** 3), device+ )+ )+ storage = torch.ByteStorage.from_buffer(buffer)+ tensor = torch.ByteTensor(storage).to(device=device)+ return tensor+++def _pad_to_largest_tensor(tensor, group):+ """+ Returns:+ list[int]: size of the tensor, on each rank+ Tensor: padded tensor that has the max size+ """+ world_size = dist.get_world_size(group=group)+ assert (+ world_size >= 1+ ), "comm.gather/all_gather must be called from ranks within the given group!"+ local_size = torch.tensor([tensor.numel()], dtype=torch.int64, device=tensor.device)+ size_list = [+ torch.zeros([1], dtype=torch.int64, device=tensor.device)+ for _ in range(world_size)+ ]+ dist.all_gather(size_list, local_size, group=group)+ size_list = [int(size.item()) for size in size_list]++ max_size = max(size_list)++ # we pad the tensor because torch all_gather does not support+ # gathering tensors of different shapes+ if local_size != max_size:+ padding = torch.zeros(+ (max_size - local_size,), dtype=torch.uint8, device=tensor.device+ )+ tensor = torch.cat((tensor, padding), dim=0)+ return size_list, tensor+++def all_gather(data, group=None):+ """+ Run all_gather on arbitrary picklable data (not necessarily tensors).++ Args:+ data: any picklable object+ group: a torch process group. By default, will use a group which+ contains all ranks on gloo backend.+ Returns:+ list[data]: list of data gathered from each rank+ """+ if get_world_size() == 1:+ return [data]+ if group is None:+ group = _get_global_gloo_group()+ if dist.get_world_size(group) == 1:+ return [data]++ tensor = _serialize_to_tensor(data, group)++ size_list, tensor = _pad_to_largest_tensor(tensor, group)+ max_size = max(size_list)++ # receiving Tensor from all ranks+ tensor_list = [+ torch.empty((max_size,), dtype=torch.uint8, device=tensor.device)+ for _ in size_list+ ]+ dist.all_gather(tensor_list, tensor, group=group)++ data_list = []+ for size, tensor in zip(size_list, tensor_list):+ buffer = tensor.cpu().numpy().tobytes()[:size]+ data_list.append(pickle.loads(buffer))++ return data_list+++def gather(data, dst=0, group=None):+ """+ Run gather on arbitrary picklable data (not necessarily tensors).++ Args:+ data: any picklable object+ dst (int): destination rank+ group: a torch process group. By default, will use a group which+ contains all ranks on gloo backend.++ Returns:+ list[data]: on dst, a list of data gathered from each rank. Otherwise,+ an empty list.+ """+ if get_world_size() == 1:+ return [data]+ if group is None:+ group = _get_global_gloo_group()+ if dist.get_world_size(group=group) == 1:+ return [data]+ rank = dist.get_rank(group=group)++ tensor = _serialize_to_tensor(data, group)+ size_list, tensor = _pad_to_largest_tensor(tensor, group)++ # receiving Tensor from all ranks+ if rank == dst:+ max_size = max(size_list)+ tensor_list = [+ torch.empty((max_size,), dtype=torch.uint8, device=tensor.device)+ for _ in size_list+ ]+ dist.gather(tensor, tensor_list, dst=dst, group=group)++ data_list = []+ for size, tensor in zip(size_list, tensor_list):+ buffer = tensor.cpu().numpy().tobytes()[:size]+ data_list.append(pickle.loads(buffer))+ return data_list+ else:+ dist.gather(tensor, [], dst=dst, group=group)+ return []+++def shared_random_seed():+ """+ Returns:+ int: a random number that is the same across all workers.+ If workers need a shared RNG, they can use this shared seed to+ create one.+ All workers must call this function, otherwise it will deadlock.+ """+ ints = np.random.randint(2 ** 31)+ all_ints = all_gather(ints)+ return all_ints[0]+++def time_synchronized():+ """pytorch-accurate time"""+ if torch.cuda.is_available():+ torch.cuda.synchronize()+ return time.time()diff --git a/experimental/yolox/utils/ema.py b/experimental/yolox/utils/ema.pynew file mode 100644index 0000000..e0d09ba--- /dev/null+++ b/experimental/yolox/utils/ema.py@@ -0,0 +1,72 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.+import torch+import torch.nn as nn++import math+from copy import deepcopy+++def is_parallel(model):+ """check if model is in parallel mode."""++ parallel_type = (+ nn.parallel.DataParallel,+ nn.parallel.DistributedDataParallel,+ )+ return isinstance(model, parallel_type)+++def copy_attr(a, b, include=(), exclude=()):+ # Copy attributes from b to a, options to only include [...] and to exclude [...]+ for k, v in b.__dict__.items():+ if (len(include) and k not in include) or k.startswith("_") or k in exclude:+ continue+ else:+ setattr(a, k, v)+++class ModelEMA:+ """+ Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models+ Keep a moving average of everything in the model state_dict (parameters and buffers).+ This is intended to allow functionality like+ https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage+ A smoothed version of the weights is necessary for some training schemes to perform well.+ This class is sensitive where it is initialized in the sequence of model init,+ GPU assignment and distributed training wrappers.+ """++ def __init__(self, model, decay=0.9999, updates=0):+ """+ Args:+ model (nn.Module): model to apply EMA.+ decay (float): ema decay reate.+ updates (int): counter of EMA updates.+ """+ # Create EMA(FP32)+ self.ema = deepcopy(model.module if is_parallel(model) else model).eval()+ self.updates = updates+ # decay exponential ramp (to help early epochs)+ self.decay = lambda x: decay * (1 - math.exp(-x / 2000))+ for p in self.ema.parameters():+ p.requires_grad_(False)++ def update(self, model):+ # Update EMA parameters+ with torch.no_grad():+ self.updates += 1+ d = self.decay(self.updates)++ msd = (+ model.module.state_dict() if is_parallel(model) else model.state_dict()+ ) # model state_dict+ for k, v in self.ema.state_dict().items():+ if v.dtype.is_floating_point:+ v *= d+ v += (1.0 - d) * msd[k].detach()++ def update_attr(self, model, include=(), exclude=("process_group", "reducer")):+ # Update EMA attributes+ copy_attr(self.ema, model, include, exclude)diff --git a/experimental/yolox/utils/logger.py b/experimental/yolox/utils/logger.pynew file mode 100644index 0000000..4bd51d9--- /dev/null+++ b/experimental/yolox/utils/logger.py@@ -0,0 +1,96 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++from loguru import logger++import inspect+import os+import sys+++def get_caller_name(depth=0):+ """+ Args:+ depth (int): Depth of caller conext, use 0 for caller depth. Default value: 0.++ Returns:+ str: module name of the caller+ """+ # the following logic is a little bit faster than inspect.stack() logic+ frame = inspect.currentframe().f_back+ for _ in range(depth):+ frame = frame.f_back++ return frame.f_globals["__name__"]+++class StreamToLoguru:+ """+ stream object that redirects writes to a logger instance.+ """++ def __init__(self, level="INFO", caller_names=("apex", "pycocotools")):+ """+ Args:+ level(str): log level string of loguru. Default value: "INFO".+ caller_names(tuple): caller names of redirected module.+ Default value: (apex, pycocotools).+ """+ self.level = level+ self.linebuf = ""+ self.caller_names = caller_names++ def write(self, buf):+ full_name = get_caller_name(depth=1)+ module_name = full_name.rsplit(".", maxsplit=-1)[0]+ if module_name in self.caller_names:+ for line in buf.rstrip().splitlines():+ # use caller level log+ logger.opt(depth=2).log(self.level, line.rstrip())+ else:+ sys.__stdout__.write(buf)++ def flush(self):+ pass+++def redirect_sys_output(log_level="INFO"):+ redirect_logger = StreamToLoguru(log_level)+ sys.stderr = redirect_logger+ sys.stdout = redirect_logger+++def setup_logger(save_dir, distributed_rank=0, filename="log.txt", mode="a"):+ """setup logger for training and testing.+ Args:+ save_dir(str): location to save log file+ distributed_rank(int): device rank when multi-gpu environment+ filename (string): log save name.+ mode(str): log file write mode, `append` or `override`. default is `a`.++ Return:+ logger instance.+ """+ loguru_format = (+ "<green>{time:YYYY-MM-DD HH:mm:ss}</green> | "+ "<level>{level: <8}</level> | "+ "<cyan>{name}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>"+ )++ logger.remove()+ save_file = os.path.join(save_dir, filename)+ if mode == "o" and os.path.exists(save_file):+ os.remove(save_file)+ # only keep logger in rank0 process+ if distributed_rank == 0:+ logger.add(+ sys.stderr,+ format=loguru_format,+ level="INFO",+ enqueue=True,+ )+ logger.add(save_file)++ # redirect stdout/stderr to loguru+ redirect_sys_output("INFO")diff --git a/experimental/yolox/utils/lr_scheduler.py b/experimental/yolox/utils/lr_scheduler.pynew file mode 100644index 0000000..8f85c23--- /dev/null+++ b/experimental/yolox/utils/lr_scheduler.py@@ -0,0 +1,205 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import math+from functools import partial+++class LRScheduler:+ def __init__(self, name, lr, iters_per_epoch, total_epochs, **kwargs):+ """+ Supported lr schedulers: [cos, warmcos, multistep]++ Args:+ lr (float): learning rate.+ iters_per_peoch (int): number of iterations in one epoch.+ total_epochs (int): number of epochs in training.+ kwargs (dict):+ - cos: None+ - warmcos: [warmup_epochs, warmup_lr_start (default 1e-6)]+ - multistep: [milestones (epochs), gamma (default 0.1)]+ """++ self.lr = lr+ self.iters_per_epoch = iters_per_epoch+ self.total_epochs = total_epochs+ self.total_iters = iters_per_epoch * total_epochs++ self.__dict__.update(kwargs)++ self.lr_func = self._get_lr_func(name)++ def update_lr(self, iters):+ return self.lr_func(iters)++ def _get_lr_func(self, name):+ if name == "cos": # cosine lr schedule+ lr_func = partial(cos_lr, self.lr, self.total_iters)+ elif name == "warmcos":+ warmup_total_iters = self.iters_per_epoch * self.warmup_epochs+ warmup_lr_start = getattr(self, "warmup_lr_start", 1e-6)+ lr_func = partial(+ warm_cos_lr,+ self.lr,+ self.total_iters,+ warmup_total_iters,+ warmup_lr_start,+ )+ elif name == "yoloxwarmcos":+ warmup_total_iters = self.iters_per_epoch * self.warmup_epochs+ no_aug_iters = self.iters_per_epoch * self.no_aug_epochs+ warmup_lr_start = getattr(self, "warmup_lr_start", 0)+ min_lr_ratio = getattr(self, "min_lr_ratio", 0.2)+ lr_func = partial(+ yolox_warm_cos_lr,+ self.lr,+ min_lr_ratio,+ self.total_iters,+ warmup_total_iters,+ warmup_lr_start,+ no_aug_iters,+ )+ elif name == "yoloxsemiwarmcos":+ warmup_lr_start = getattr(self, "warmup_lr_start", 0)+ min_lr_ratio = getattr(self, "min_lr_ratio", 0.2)+ warmup_total_iters = self.iters_per_epoch * self.warmup_epochs+ no_aug_iters = self.iters_per_epoch * self.no_aug_epochs+ normal_iters = self.iters_per_epoch * self.semi_epoch+ semi_iters = self.iters_per_epoch_semi * (+ self.total_epochs - self.semi_epoch - self.no_aug_epochs+ )+ lr_func = partial(+ yolox_semi_warm_cos_lr,+ self.lr,+ min_lr_ratio,+ warmup_lr_start,+ self.total_iters,+ normal_iters,+ no_aug_iters,+ warmup_total_iters,+ semi_iters,+ self.iters_per_epoch,+ self.iters_per_epoch_semi,+ )+ elif name == "multistep": # stepwise lr schedule+ milestones = [+ int(self.total_iters * milestone / self.total_epochs)+ for milestone in self.milestones+ ]+ gamma = getattr(self, "gamma", 0.1)+ lr_func = partial(multistep_lr, self.lr, milestones, gamma)+ else:+ raise ValueError("Scheduler version {} not supported.".format(name))+ return lr_func+++def cos_lr(lr, total_iters, iters):+ """Cosine learning rate"""+ lr *= 0.5 * (1.0 + math.cos(math.pi * iters / total_iters))+ return lr+++def warm_cos_lr(lr, total_iters, warmup_total_iters, warmup_lr_start, iters):+ """Cosine learning rate with warm up."""+ if iters <= warmup_total_iters:+ lr = (lr - warmup_lr_start) * iters / float(+ warmup_total_iters+ ) + warmup_lr_start+ else:+ lr *= 0.5 * (+ 1.0+ + math.cos(+ math.pi+ * (iters - warmup_total_iters)+ / (total_iters - warmup_total_iters)+ )+ )+ return lr+++def yolox_warm_cos_lr(+ lr,+ min_lr_ratio,+ total_iters,+ warmup_total_iters,+ warmup_lr_start,+ no_aug_iter,+ iters,+):+ """Cosine learning rate with warm up."""+ min_lr = lr * min_lr_ratio+ if iters <= warmup_total_iters:+ # lr = (lr - warmup_lr_start) * iters / float(warmup_total_iters) + warmup_lr_start+ lr = (lr - warmup_lr_start) * pow(+ iters / float(warmup_total_iters), 2+ ) + warmup_lr_start+ elif iters >= total_iters - no_aug_iter:+ lr = min_lr+ else:+ lr = min_lr + 0.5 * (lr - min_lr) * (+ 1.0+ + math.cos(+ math.pi+ * (iters - warmup_total_iters)+ / (total_iters - warmup_total_iters - no_aug_iter)+ )+ )+ return lr+++def yolox_semi_warm_cos_lr(+ lr,+ min_lr_ratio,+ warmup_lr_start,+ total_iters,+ normal_iters,+ no_aug_iters,+ warmup_total_iters,+ semi_iters,+ iters_per_epoch,+ iters_per_epoch_semi,+ iters,+):+ """Cosine learning rate with warm up."""+ min_lr = lr * min_lr_ratio+ if iters <= warmup_total_iters:+ # lr = (lr - warmup_lr_start) * iters / float(warmup_total_iters) + warmup_lr_start+ lr = (lr - warmup_lr_start) * pow(+ iters / float(warmup_total_iters), 2+ ) + warmup_lr_start+ elif iters >= normal_iters + semi_iters:+ lr = min_lr+ elif iters <= normal_iters:+ lr = min_lr + 0.5 * (lr - min_lr) * (+ 1.0+ + math.cos(+ math.pi+ * (iters - warmup_total_iters)+ / (total_iters - warmup_total_iters - no_aug_iters)+ )+ )+ else:+ lr = min_lr + 0.5 * (lr - min_lr) * (+ 1.0+ + math.cos(+ math.pi+ * (+ normal_iters+ - warmup_total_iters+ + (iters - normal_iters)+ * iters_per_epoch+ * 1.0+ / iters_per_epoch_semi+ )+ / (total_iters - warmup_total_iters - no_aug_iters)+ )+ )+ return lr+++def multistep_lr(lr, milestones, gamma, iters):+ """MultiStep learning rate"""+ for milestone in milestones:+ lr *= gamma if iters >= milestone else 1.0+ return lrdiff --git a/experimental/yolox/utils/metric.py b/experimental/yolox/utils/metric.pynew file mode 100644index 0000000..4840b8d--- /dev/null+++ b/experimental/yolox/utils/metric.py@@ -0,0 +1,123 @@+#!/usr/bin/env python3+# -*- coding: utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.+import numpy as np++import torch++import functools+import os+import time+from collections import defaultdict, deque++__all__ = [+ "AverageMeter",+ "MeterBuffer",+ "get_total_and_free_memory_in_Mb",+ "occupy_mem",+ "gpu_mem_usage",+]+++def get_total_and_free_memory_in_Mb(cuda_device):+ devices_info_str = os.popen(+ "nvidia-smi --query-gpu=memory.total,memory.used --format=csv,nounits,noheader"+ )+ devices_info = devices_info_str.read().strip().split("\n")+ total, used = devices_info[int(cuda_device)].split(",")+ return int(total), int(used)+++def occupy_mem(cuda_device, mem_ratio=0.95):+ """+ pre-allocate gpu memory for training to avoid memory Fragmentation.+ """+ total, used = get_total_and_free_memory_in_Mb(cuda_device)+ max_mem = int(total * mem_ratio)+ block_mem = max_mem - used+ x = torch.cuda.FloatTensor(256, 1024, block_mem)+ del x+ time.sleep(5)+++def gpu_mem_usage():+ """+ Compute the GPU memory usage for the current device (MB).+ """+ mem_usage_bytes = torch.cuda.max_memory_allocated()+ return mem_usage_bytes / (1024 * 1024)+++class AverageMeter:+ """Track a series of values and provide access to smoothed values over a+ window or the global series average.+ """++ def __init__(self, window_size=50):+ self._deque = deque(maxlen=window_size)+ self._total = 0.0+ self._count = 0++ def update(self, value):+ self._deque.append(value)+ self._count += 1+ self._total += value++ @property+ def median(self):+ d = np.array(list(self._deque))+ return np.median(d)++ @property+ def avg(self):+ # if deque is empty, nan will be returned.+ d = np.array(list(self._deque))+ return d.mean()++ @property+ def global_avg(self):+ return self._total / max(self._count, 1e-5)++ @property+ def latest(self):+ return self._deque[-1] if len(self._deque) > 0 else None++ @property+ def total(self):+ return self._total++ def reset(self):+ self._deque.clear()+ self._total = 0.0+ self._count = 0++ def clear(self):+ self._deque.clear()+++class MeterBuffer(defaultdict):+ """Computes and stores the average and current value"""++ def __init__(self, window_size=20):+ factory = functools.partial(AverageMeter, window_size=window_size)+ super().__init__(factory)++ def reset(self):+ for v in self.values():+ v.reset()++ def get_filtered_meter(self, filter_key="time"):+ return {k: v for k, v in self.items() if filter_key in k}++ def update(self, values=None, **kwargs):+ if values is None:+ values = {}+ values.update(kwargs)+ for k, v in values.items():+ if isinstance(v, torch.Tensor):+ v = v.detach()+ self[k].update(v)++ def clear_meters(self):+ for v in self.values():+ v.clear()diff --git a/experimental/yolox/utils/model_utils.py b/experimental/yolox/utils/model_utils.pynew file mode 100644index 0000000..b4e5812--- /dev/null+++ b/experimental/yolox/utils/model_utils.py@@ -0,0 +1,106 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import torch+import torch.nn as nn+from thop import profile++from copy import deepcopy++__all__ = [+ "fuse_conv_and_bn",+ "fuse_model",+ "get_model_info",+ "replace_module",+]+++def get_model_info(model, tsize):++ stride = 64+ img = torch.zeros((1, 3, stride, stride), device=next(model.parameters()).device)+ flops, params = profile(deepcopy(model), inputs=(img,), verbose=False)+ params /= 1e6+ flops /= 1e9+ flops *= tsize[0] * tsize[1] / stride / stride * 2 # Gflops+ info = "Params: {:.2f}M, Gflops: {:.2f}".format(params, flops)+ return info+++def fuse_conv_and_bn(conv, bn):+ # Fuse convolution and batchnorm layers https://tehnokv.com/posts/fusing-batchnorm-and-conv/+ fusedconv = (+ nn.Conv2d(+ conv.in_channels,+ conv.out_channels,+ kernel_size=conv.kernel_size,+ stride=conv.stride,+ padding=conv.padding,+ groups=conv.groups,+ bias=True,+ )+ .requires_grad_(False)+ .to(conv.weight.device)+ )++ # prepare filters+ w_conv = conv.weight.clone().view(conv.out_channels, -1)+ w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var)))+ fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape))++ # prepare spatial bias+ b_conv = (+ torch.zeros(conv.weight.size(0), device=conv.weight.device)+ if conv.bias is None+ else conv.bias+ )+ b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(+ torch.sqrt(bn.running_var + bn.eps)+ )+ fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn)++ return fusedconv+++def fuse_model(model):+ from yolox.models.network_blocks import BaseConv++ for m in model.modules():+ if type(m) is BaseConv and hasattr(m, "bn"):+ m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv+ delattr(m, "bn") # remove batchnorm+ m.forward = m.fuseforward # update forward+ return model+++def replace_module(module, replaced_module_type, new_module_type, replace_func=None):+ """+ Replace given type in module to a new type. mostly used in deploy.++ Args:+ module (nn.Module): model to apply replace operation.+ replaced_module_type (Type): module type to be replaced.+ new_module_type (Type)+ replace_func (function): python function to describe replace logic. Defalut value None.++ Returns:+ model (nn.Module): module that already been replaced.+ """++ def default_replace_func(replaced_module_type, new_module_type):+ return new_module_type()++ if replace_func is None:+ replace_func = default_replace_func++ model = module+ if isinstance(module, replaced_module_type):+ model = replace_func(replaced_module_type, new_module_type)+ else: # recurrsively replace+ for name, child in module.named_children():+ new_child = replace_module(child, replaced_module_type, new_module_type)+ if new_child is not child: # child is already replaced+ model.add_module(name, new_child)++ return modeldiff --git a/experimental/yolox/utils/setup_env.py b/experimental/yolox/utils/setup_env.pynew file mode 100644index 0000000..f282b1f--- /dev/null+++ b/experimental/yolox/utils/setup_env.py@@ -0,0 +1,51 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import cv2++import os+import subprocess++__all__ = ["configure_nccl", "configure_module"]+++def configure_nccl():+ """Configure multi-machine environment variables of NCCL."""+ os.environ["NCCL_LAUNCH_MODE"] = "PARALLEL"+ os.environ["NCCL_IB_HCA"] = subprocess.getoutput(+ "pushd /sys/class/infiniband/ > /dev/null; for i in mlx5_*; "+ "do cat $i/ports/1/gid_attrs/types/* 2>/dev/null "+ "| grep v >/dev/null && echo $i ; done; popd > /dev/null"+ )+ os.environ["NCCL_IB_GID_INDEX"] = "3"+ os.environ["NCCL_IB_TC"] = "106"+++def configure_module(ulimit_value=8192):+ """+ Configure pytorch module environment. setting of ulimit and cv2 will be set.++ Args:+ ulimit_value(int): default open file number on linux. Default value: 8192.+ """+ # system setting+ try:+ import resource++ rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)+ resource.setrlimit(resource.RLIMIT_NOFILE, (ulimit_value, rlimit[1]))+ except Exception:+ # Exception might be raised in Windows OS or rlimit reaches max limit number.+ # However, set rlimit value might not be necessary.+ pass++ # cv2+ # multiprocess might be harmful on performance of torch dataloader+ os.environ["OPENCV_OPENCL_RUNTIME"] = "disabled"+ try:+ cv2.setNumThreads(0)+ cv2.ocl.setUseOpenCL(False)+ except Exception:+ # cv2 version mismatch might rasie exceptions.+ passdiff --git a/experimental/yolox/utils/visualize.py b/experimental/yolox/utils/visualize.pynew file mode 100644index 0000000..1d02d47--- /dev/null+++ b/experimental/yolox/utils/visualize.py@@ -0,0 +1,166 @@+#!/usr/bin/env python3+# -*- coding:utf-8 -*-+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.++import cv2+import numpy as np++__all__ = ["vis"]+++def vis(img, boxes, scores, cls_ids, conf=0.5, class_names=None):++ for i in range(len(boxes)):+ box = boxes[i]+ cls_id = int(cls_ids[i])+ score = scores[i]+ if score < conf:+ continue+ x0 = int(box[0])+ y0 = int(box[1])+ x1 = int(box[2])+ y1 = int(box[3])++ color = (_COLORS[cls_id] * 255).astype(np.uint8).tolist()+ text = '{}:{:.1f}%'.format(class_names[cls_id], score * 100)+ txt_color = (0, 0, 0) if np.mean(_COLORS[cls_id]) > 0.5 else (255, 255, 255)+ font = cv2.FONT_HERSHEY_SIMPLEX++ txt_size = cv2.getTextSize(text, font, 0.4, 1)[0]+ cv2.rectangle(img, (x0, y0), (x1, y1), color, 2)++ txt_bk_color = (_COLORS[cls_id] * 255 * 0.7).astype(np.uint8).tolist()+ cv2.rectangle(+ img,+ (x0, y0 + 1),+ (x0 + txt_size[0] + 1, y0 + int(1.5*txt_size[1])),+ txt_bk_color,+ -1+ )+ cv2.putText(img, text, (x0, y0 + txt_size[1]), font, 0.4, txt_color, thickness=1)++ return img+++def get_color(idx):+ idx = idx * 3+ color = ((37 * idx) % 255, (17 * idx) % 255, (29 * idx) % 255)++ return color+++def plot_tracking(image, tlwhs, obj_ids, scores=None, frame_id=0, fps=0., ids2=None):+ im = np.ascontiguousarray(np.copy(image))+ im_h, im_w = im.shape[:2]++ top_view = np.zeros([im_w, im_w, 3], dtype=np.uint8) + 255++ #text_scale = max(1, image.shape[1] / 1600.)+ #text_thickness = 2+ #line_thickness = max(1, int(image.shape[1] / 500.))+ text_scale = 2+ text_thickness = 2+ line_thickness = 3++ radius = max(5, int(im_w/140.))+ cv2.putText(im, 'frame: %d fps: %.2f num: %d' % (frame_id, fps, len(tlwhs)),+ (0, int(15 * text_scale)), cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 255), thickness=2)++ for i, tlwh in enumerate(tlwhs):+ x1, y1, w, h = tlwh+ intbox = tuple(map(int, (x1, y1, x1 + w, y1 + h)))+ obj_id = int(obj_ids[i])+ id_text = '{}'.format(int(obj_id))+ if ids2 is not None:+ id_text = id_text + ', {}'.format(int(ids2[i]))+ color = get_color(abs(obj_id))+ cv2.rectangle(im, intbox[0:2], intbox[2:4], color=color, thickness=line_thickness)+ cv2.putText(im, id_text, (intbox[0], intbox[1]), cv2.FONT_HERSHEY_PLAIN, text_scale, (0, 0, 255),+ thickness=text_thickness)+ return im+++_COLORS = np.array(+ [+ 0.000, 0.447, 0.741,+ 0.850, 0.325, 0.098,+ 0.929, 0.694, 0.125,+ 0.494, 0.184, 0.556,+ 0.466, 0.674, 0.188,+ 0.301, 0.745, 0.933,+ 0.635, 0.078, 0.184,+ 0.300, 0.300, 0.300,+ 0.600, 0.600, 0.600,+ 1.000, 0.000, 0.000,+ 1.000, 0.500, 0.000,+ 0.749, 0.749, 0.000,+ 0.000, 1.000, 0.000,+ 0.000, 0.000, 1.000,+ 0.667, 0.000, 1.000,+ 0.333, 0.333, 0.000,+ 0.333, 0.667, 0.000,+ 0.333, 1.000, 0.000,+ 0.667, 0.333, 0.000,+ 0.667, 0.667, 0.000,+ 0.667, 1.000, 0.000,+ 1.000, 0.333, 0.000,+ 1.000, 0.667, 0.000,+ 1.000, 1.000, 0.000,+ 0.000, 0.333, 0.500,+ 0.000, 0.667, 0.500,+ 0.000, 1.000, 0.500,+ 0.333, 0.000, 0.500,+ 0.333, 0.333, 0.500,+ 0.333, 0.667, 0.500,+ 0.333, 1.000, 0.500,+ 0.667, 0.000, 0.500,+ 0.667, 0.333, 0.500,+ 0.667, 0.667, 0.500,+ 0.667, 1.000, 0.500,+ 1.000, 0.000, 0.500,+ 1.000, 0.333, 0.500,+ 1.000, 0.667, 0.500,+ 1.000, 1.000, 0.500,+ 0.000, 0.333, 1.000,+ 0.000, 0.667, 1.000,+ 0.000, 1.000, 1.000,+ 0.333, 0.000, 1.000,+ 0.333, 0.333, 1.000,+ 0.333, 0.667, 1.000,+ 0.333, 1.000, 1.000,+ 0.667, 0.000, 1.000,+ 0.667, 0.333, 1.000,+ 0.667, 0.667, 1.000,+ 0.667, 1.000, 1.000,+ 1.000, 0.000, 1.000,+ 1.000, 0.333, 1.000,+ 1.000, 0.667, 1.000,+ 0.333, 0.000, 0.000,+ 0.500, 0.000, 0.000,+ 0.667, 0.000, 0.000,+ 0.833, 0.000, 0.000,+ 1.000, 0.000, 0.000,+ 0.000, 0.167, 0.000,+ 0.000, 0.333, 0.000,+ 0.000, 0.500, 0.000,+ 0.000, 0.667, 0.000,+ 0.000, 0.833, 0.000,+ 0.000, 1.000, 0.000,+ 0.000, 0.000, 0.167,+ 0.000, 0.000, 0.333,+ 0.000, 0.000, 0.500,+ 0.000, 0.000, 0.667,+ 0.000, 0.000, 0.833,+ 0.000, 0.000, 1.000,+ 0.000, 0.000, 0.000,+ 0.143, 0.143, 0.143,+ 0.286, 0.286, 0.286,+ 0.429, 0.429, 0.429,+ 0.571, 0.571, 0.571,+ 0.714, 0.714, 0.714,+ 0.857, 0.857, 0.857,+ 0.000, 0.447, 0.741,+ 0.314, 0.717, 0.741,+ 0.50, 0.5, 0+ ]+).astype(np.float32).reshape(-1, 3)