Skip to content

Add object detection, tracking, mobile notifications, and search to any security camera.

add tracking stuff

roryclearf6dbfcc63a9c
diff --git a/experimental/rtsp_yolov8.py b/experimental/rtsp_yolov8.py
index ec41145..2ec043c 100644
--- a/experimental/rtsp_yolov8.py
+++ b/experimental/rtsp_yolov8.py
@@ -20,6 +20,17 @@ import threading
import shutil
from datetime import datetime, time as time_obj
+from yolox.tracker.byte_tracker import BYTETracker
+
+class Args:
+ def __init__(self):
+ self.track_thresh = 0.6
+ self.track_buffer = 60 #frames, was 30
+ self.mot20 = False
+ self.match_thresh = 0.9
+
+tracker = BYTETracker(Args())
+
#Model architecture from https://github.com/ultralytics/ultralytics/issues/189
#The upsampling class has been taken from this pull request https://github.com/tinygrad/tinygrad/pull/784 by dc-dc-dc. Now 2(?) models use upsampling. (retinet and this)
@@ -460,10 +471,15 @@ class VideoCapture:
frame = self.raw_frame.copy() if self.raw_frame is not None else None
if frame is not None:
pre = preprocess(frame)
- preds = do_inf(pre).numpy()
- preds = scale_boxes(pre.shape[:2], preds, frame.shape)
+ preds2 = do_inf(pre).numpy()
+ online_targets = tracker.update(preds2, [1280,1280], [1280,1280])
+ preds2 = []
+ for x in online_targets:
+ preds2.append(np.array([x.tlwh[0],x.tlwh[1],(x.tlwh[0]+x.tlwh[2]),(x.tlwh[1]+x.tlwh[3]),x.score,x.class_id]))
+ preds2 = np.array(preds2)
+ preds2 = scale_boxes(pre.shape[:2], preds2, frame.shape)
with self.lock:
- self.last_preds = preds
+ self.last_preds = preds2
curr_time = time.time()
fps = 1 / (curr_time - prev_time)
prev_time = curr_time
diff --git a/experimental/yolox/.DS_Store b/experimental/yolox/.DS_Store
new file mode 100644
index 0000000..ab82fe8
Binary files /dev/null and b/experimental/yolox/.DS_Store differ
diff --git a/experimental/yolox/__init__.py b/experimental/yolox/__init__.py
new file mode 100644
index 0000000..1cbc411
--- /dev/null
+++ b/experimental/yolox/__init__.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+from .utils import configure_module
+
+configure_module()
+
+__version__ = "0.1.0"
diff --git a/experimental/yolox/__pycache__/__init__.cpython-310.pyc b/experimental/yolox/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000..025cd56
Binary files /dev/null and b/experimental/yolox/__pycache__/__init__.cpython-310.pyc differ
diff --git a/experimental/yolox/core/__init__.py b/experimental/yolox/core/__init__.py
new file mode 100644
index 0000000..c2379c7
--- /dev/null
+++ b/experimental/yolox/core/__init__.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) Megvii, Inc. and its affiliates.
+
+from .launch import launch
+from .trainer import Trainer
diff --git a/experimental/yolox/core/launch.py b/experimental/yolox/core/launch.py
new file mode 100644
index 0000000..2fd5eaa
--- /dev/null
+++ b/experimental/yolox/core/launch.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Code are based on
+# https://github.com/facebookresearch/detectron2/blob/master/detectron2/engine/launch.py
+# Copyright (c) Facebook, Inc. and its affiliates.
+# Copyright (c) Megvii, Inc. and its affiliates.
+
+from loguru import logger
+
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+
+import yolox.utils.dist as comm
+from yolox.utils import configure_nccl
+
+import os
+import subprocess
+import sys
+import time
+
+__all__ = ["launch"]
+
+
+def _find_free_port():
+ """
+ Find an available port of current machine / node.
+ """
+ import socket
+
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ # Binding to port 0 will cause the OS to find an available port for us
+ sock.bind(("", 0))
+ port = sock.getsockname()[1]
+ sock.close()
+ # NOTE: there is still a chance the port could be taken by other processes.
+ return port
+
+
+def launch(
+ main_func,
+ num_gpus_per_machine,
+ num_machines=1,
+ machine_rank=0,
+ backend="nccl",
+ dist_url=None,
+ args=(),
+):
+ """
+ Args:
+ main_func: a function that will be called by `main_func(*args)`
+ num_machines (int): the total number of machines
+ machine_rank (int): the rank of this machine (one per machine)
+ dist_url (str): url to connect to for distributed training, including protocol
+ e.g. "tcp://127.0.0.1:8686".
+ Can be set to auto to automatically select a free port on localhost
+ args (tuple): arguments passed to main_func
+ """
+ world_size = num_machines * num_gpus_per_machine
+ if world_size > 1:
+ if int(os.environ.get("WORLD_SIZE", "1")) > 1:
+ dist_url = "{}:{}".format(
+ os.environ.get("MASTER_ADDR", None),
+ os.environ.get("MASTER_PORT", "None"),
+ )
+ local_rank = int(os.environ.get("LOCAL_RANK", "0"))
+ world_size = int(os.environ.get("WORLD_SIZE", "1"))
+ _distributed_worker(
+ local_rank,
+ main_func,
+ world_size,
+ num_gpus_per_machine,
+ num_machines,
+ machine_rank,
+ backend,
+ dist_url,
+ args,
+ )
+ exit()
+ launch_by_subprocess(
+ sys.argv,
+ world_size,
+ num_machines,
+ machine_rank,
+ num_gpus_per_machine,
+ dist_url,
+ args,
+ )
+ else:
+ main_func(*args)
+
+
+def launch_by_subprocess(
+ raw_argv,
+ world_size,
+ num_machines,
+ machine_rank,
+ num_gpus_per_machine,
+ dist_url,
+ args,
+):
+ assert (
+ world_size > 1
+ ), "subprocess mode doesn't support single GPU, use spawn mode instead"
+
+ if dist_url is None:
+ # ------------------------hack for multi-machine training -------------------- #
+ if num_machines > 1:
+ master_ip = subprocess.check_output(["hostname", "--fqdn"]).decode("utf-8")
+ master_ip = str(master_ip).strip()
+ dist_url = "tcp://{}".format(master_ip)
+ ip_add_file = "./" + args[1].experiment_name + "_ip_add.txt"
+ if machine_rank == 0:
+ port = _find_free_port()
+ with open(ip_add_file, "w") as ip_add:
+ ip_add.write(dist_url+'\n')
+ ip_add.write(str(port))
+ else:
+ while not os.path.exists(ip_add_file):
+ time.sleep(0.5)
+
+ with open(ip_add_file, "r") as ip_add:
+ dist_url = ip_add.readline().strip()
+ port = ip_add.readline()
+ else:
+ dist_url = "tcp://127.0.0.1"
+ port = _find_free_port()
+
+ # set PyTorch distributed related environmental variables
+ current_env = os.environ.copy()
+ current_env["MASTER_ADDR"] = dist_url
+ current_env["MASTER_PORT"] = str(port)
+ current_env["WORLD_SIZE"] = str(world_size)
+ assert num_gpus_per_machine <= torch.cuda.device_count()
+
+ if "OMP_NUM_THREADS" not in os.environ and num_gpus_per_machine > 1:
+ current_env["OMP_NUM_THREADS"] = str(1)
+ logger.info(
+ "\n*****************************************\n"
+ "Setting OMP_NUM_THREADS environment variable for each process "
+ "to be {} in default, to avoid your system being overloaded, "
+ "please further tune the variable for optimal performance in "
+ "your application as needed. \n"
+ "*****************************************".format(
+ current_env["OMP_NUM_THREADS"]
+ )
+ )
+
+ processes = []
+ for local_rank in range(0, num_gpus_per_machine):
+ # each process's rank
+ dist_rank = machine_rank * num_gpus_per_machine + local_rank
+ current_env["RANK"] = str(dist_rank)
+ current_env["LOCAL_RANK"] = str(local_rank)
+
+ # spawn the processes
+ cmd = ["python3", *raw_argv]
+
+ process = subprocess.Popen(cmd, env=current_env)
+ processes.append(process)
+
+ for process in processes:
+ process.wait()
+ if process.returncode != 0:
+ raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
+
+
+def _distributed_worker(
+ local_rank,
+ main_func,
+ world_size,
+ num_gpus_per_machine,
+ num_machines,
+ machine_rank,
+ backend,
+ dist_url,
+ args,
+):
+ assert (
+ torch.cuda.is_available()
+ ), "cuda is not available. Please check your installation."
+ configure_nccl()
+ global_rank = machine_rank * num_gpus_per_machine + local_rank
+ logger.info("Rank {} initialization finished.".format(global_rank))
+ try:
+ dist.init_process_group(
+ backend=backend,
+ init_method=dist_url,
+ world_size=world_size,
+ rank=global_rank,
+ )
+ except Exception:
+ logger.error("Process group URL: {}".format(dist_url))
+ raise
+ # synchronize is needed here to prevent a possible timeout after calling init_process_group
+ # See: https://github.com/facebookresearch/maskrcnn-benchmark/issues/172
+ comm.synchronize()
+
+ if global_rank == 0 and os.path.exists(
+ "./" + args[1].experiment_name + "_ip_add.txt"
+ ):
+ os.remove("./" + args[1].experiment_name + "_ip_add.txt")
+
+ assert num_gpus_per_machine <= torch.cuda.device_count()
+ torch.cuda.set_device(local_rank)
+
+ args[1].local_rank = local_rank
+ args[1].num_machines = num_machines
+
+ # Setup the local process group (which contains ranks within the same machine)
+ # assert comm._LOCAL_PROCESS_GROUP is None
+ # num_machines = world_size // num_gpus_per_machine
+ # for i in range(num_machines):
+ # ranks_on_i = list(range(i * num_gpus_per_machine, (i + 1) * num_gpus_per_machine))
+ # pg = dist.new_group(ranks_on_i)
+ # if i == machine_rank:
+ # comm._LOCAL_PROCESS_GROUP = pg
+
+ main_func(*args)
diff --git a/experimental/yolox/core/trainer.py b/experimental/yolox/core/trainer.py
new file mode 100644
index 0000000..5ab1006
--- /dev/null
+++ b/experimental/yolox/core/trainer.py
@@ -0,0 +1,329 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) Megvii, Inc. and its affiliates.
+
+from loguru import logger
+
+import torch
+
+from torch.nn.parallel import DistributedDataParallel as DDP
+from torch.utils.tensorboard import SummaryWriter
+
+from yolox.data import DataPrefetcher
+from yolox.utils import (
+ MeterBuffer,
+ ModelEMA,
+ all_reduce_norm,
+ get_model_info,
+ get_rank,
+ get_world_size,
+ gpu_mem_usage,
+ load_ckpt,
+ occupy_mem,
+ save_checkpoint,
+ setup_logger,
+ synchronize
+)
+
+import datetime
+import os
+import time
+
+
+class Trainer:
+ def __init__(self, exp, args):
+ # init function only defines some basic attr, other attrs like model, optimizer are built in
+ # before_train methods.
+ self.exp = exp
+ self.args = args
+
+ # training related attr
+ self.max_epoch = exp.max_epoch
+ self.amp_training = args.fp16
+ self.scaler = torch.cuda.amp.GradScaler(enabled=args.fp16)
+ self.is_distributed = get_world_size() > 1
+ self.rank = get_rank()
+ self.local_rank = args.local_rank
+ self.device = "cuda:{}".format(self.local_rank)
+ self.use_model_ema = exp.ema
+
+ # data/dataloader related attr
+ self.data_type = torch.float16 if args.fp16 else torch.float32
+ self.input_size = exp.input_size
+ self.best_ap = 0
+
+ # metric record
+ self.meter = MeterBuffer(window_size=exp.print_interval)
+ self.file_name = os.path.join(exp.output_dir, args.experiment_name)
+
+ if self.rank == 0:
+ os.makedirs(self.file_name, exist_ok=True)
+
+ setup_logger(
+ self.file_name,
+ distributed_rank=self.rank,
+ filename="train_log.txt",
+ mode="a",
+ )
+
+ def train(self):
+ self.before_train()
+ try:
+ self.train_in_epoch()
+ except Exception:
+ raise
+ finally:
+ self.after_train()
+
+ def train_in_epoch(self):
+ for self.epoch in range(self.start_epoch, self.max_epoch):
+ self.before_epoch()
+ self.train_in_iter()
+ self.after_epoch()
+
+ def train_in_iter(self):
+ for self.iter in range(self.max_iter):
+ self.before_iter()
+ self.train_one_iter()
+ self.after_iter()
+
+ def train_one_iter(self):
+ iter_start_time = time.time()
+
+ inps, targets = self.prefetcher.next()
+ track_ids = targets[:, :, 5]
+ targets = targets[:, :, :5]
+ inps = inps.to(self.data_type)
+ targets = targets.to(self.data_type)
+ targets.requires_grad = False
+ data_end_time = time.time()
+
+ with torch.cuda.amp.autocast(enabled=self.amp_training):
+ outputs = self.model(inps, targets)
+ loss = outputs["total_loss"]
+
+ self.optimizer.zero_grad()
+ self.scaler.scale(loss).backward()
+ self.scaler.step(self.optimizer)
+ self.scaler.update()
+
+ if self.use_model_ema:
+ self.ema_model.update(self.model)
+
+ lr = self.lr_scheduler.update_lr(self.progress_in_iter + 1)
+ for param_group in self.optimizer.param_groups:
+ param_group["lr"] = lr
+
+ iter_end_time = time.time()
+ self.meter.update(
+ iter_time=iter_end_time - iter_start_time,
+ data_time=data_end_time - iter_start_time,
+ lr=lr,
+ **outputs,
+ )
+
+ def before_train(self):
+ logger.info("args: {}".format(self.args))
+ logger.info("exp value:\n{}".format(self.exp))
+
+ # model related init
+ torch.cuda.set_device(self.local_rank)
+ model = self.exp.get_model()
+ logger.info(
+ "Model Summary: {}".format(get_model_info(model, self.exp.test_size))
+ )
+ model.to(self.device)
+
+ # solver related init
+ self.optimizer = self.exp.get_optimizer(self.args.batch_size)
+
+ # value of epoch will be set in `resume_train`
+ model = self.resume_train(model)
+
+ # data related init
+ self.no_aug = self.start_epoch >= self.max_epoch - self.exp.no_aug_epochs
+ self.train_loader = self.exp.get_data_loader(
+ batch_size=self.args.batch_size,
+ is_distributed=self.is_distributed,
+ no_aug=self.no_aug,
+ )
+ logger.info("init prefetcher, this might take one minute or less...")
+ self.prefetcher = DataPrefetcher(self.train_loader)
+ # max_iter means iters per epoch
+ self.max_iter = len(self.train_loader)
+
+ self.lr_scheduler = self.exp.get_lr_scheduler(
+ self.exp.basic_lr_per_img * self.args.batch_size, self.max_iter
+ )
+ if self.args.occupy:
+ occupy_mem(self.local_rank)
+
+ if self.is_distributed:
+ model = DDP(model, device_ids=[self.local_rank], broadcast_buffers=False)
+
+ if self.use_model_ema:
+ self.ema_model = ModelEMA(model, 0.9998)
+ self.ema_model.updates = self.max_iter * self.start_epoch
+
+ self.model = model
+ self.model.train()
+
+ self.evaluator = self.exp.get_evaluator(
+ batch_size=self.args.batch_size, is_distributed=self.is_distributed
+ )
+ # Tensorboard logger
+ if self.rank == 0:
+ self.tblogger = SummaryWriter(self.file_name)
+
+ logger.info("Training start...")
+ #logger.info("\n{}".format(model))
+
+ def after_train(self):
+ logger.info(
+ "Training of experiment is done and the best AP is {:.2f}".format(
+ self.best_ap * 100
+ )
+ )
+
+ def before_epoch(self):
+ logger.info("---> start train epoch{}".format(self.epoch + 1))
+
+ if self.epoch + 1 == self.max_epoch - self.exp.no_aug_epochs or self.no_aug:
+
+ logger.info("--->No mosaic aug now!")
+ self.train_loader.close_mosaic()
+ logger.info("--->Add additional L1 loss now!")
+ if self.is_distributed:
+ self.model.module.head.use_l1 = True
+ else:
+ self.model.head.use_l1 = True
+
+ self.exp.eval_interval = 1
+ if not self.no_aug:
+ self.save_ckpt(ckpt_name="last_mosaic_epoch")
+
+ def after_epoch(self):
+ if self.use_model_ema:
+ self.ema_model.update_attr(self.model)
+
+ self.save_ckpt(ckpt_name="latest")
+
+ if (self.epoch + 1) % self.exp.eval_interval == 0:
+ all_reduce_norm(self.model)
+ self.evaluate_and_save_model()
+
+ def before_iter(self):
+ pass
+
+ def after_iter(self):
+ """
+ `after_iter` contains two parts of logic:
+ * log information
+ * reset setting of resize
+ """
+ # log needed information
+ if (self.iter + 1) % self.exp.print_interval == 0:
+ # TODO check ETA logic
+ left_iters = self.max_iter * self.max_epoch - (self.progress_in_iter + 1)
+ eta_seconds = self.meter["iter_time"].global_avg * left_iters
+ eta_str = "ETA: {}".format(datetime.timedelta(seconds=int(eta_seconds)))
+
+ progress_str = "epoch: {}/{}, iter: {}/{}".format(
+ self.epoch + 1, self.max_epoch, self.iter + 1, self.max_iter
+ )
+ loss_meter = self.meter.get_filtered_meter("loss")
+ loss_str = ", ".join(
+ ["{}: {:.3f}".format(k, v.latest) for k, v in loss_meter.items()]
+ )
+
+ time_meter = self.meter.get_filtered_meter("time")
+ time_str = ", ".join(
+ ["{}: {:.3f}s".format(k, v.avg) for k, v in time_meter.items()]
+ )
+
+ logger.info(
+ "{}, mem: {:.0f}Mb, {}, {}, lr: {:.3e}".format(
+ progress_str,
+ gpu_mem_usage(),
+ time_str,
+ loss_str,
+ self.meter["lr"].latest,
+ )
+ + (", size: {:d}, {}".format(self.input_size[0], eta_str))
+ )
+ self.meter.clear_meters()
+
+ # random resizing
+ if self.exp.random_size is not None and (self.progress_in_iter + 1) % 10 == 0:
+ self.input_size = self.exp.random_resize(
+ self.train_loader, self.epoch, self.rank, self.is_distributed
+ )
+
+ @property
+ def progress_in_iter(self):
+ return self.epoch * self.max_iter + self.iter
+
+ def resume_train(self, model):
+ if self.args.resume:
+ logger.info("resume training")
+ if self.args.ckpt is None:
+ ckpt_file = os.path.join(self.file_name, "latest" + "_ckpt.pth.tar")
+ else:
+ ckpt_file = self.args.ckpt
+
+ ckpt = torch.load(ckpt_file, map_location=self.device)
+ # resume the model/optimizer state dict
+ model.load_state_dict(ckpt["model"])
+ self.optimizer.load_state_dict(ckpt["optimizer"])
+ start_epoch = (
+ self.args.start_epoch - 1
+ if self.args.start_epoch is not None
+ else ckpt["start_epoch"]
+ )
+ self.start_epoch = start_epoch
+ logger.info(
+ "loaded checkpoint '{}' (epoch {})".format(
+ self.args.resume, self.start_epoch
+ )
+ ) # noqa
+ else:
+ if self.args.ckpt is not None:
+ logger.info("loading checkpoint for fine tuning")
+ ckpt_file = self.args.ckpt
+ ckpt = torch.load(ckpt_file, map_location=self.device)["model"]
+ model = load_ckpt(model, ckpt)
+ self.start_epoch = 0
+
+ return model
+
+ def evaluate_and_save_model(self):
+ evalmodel = self.ema_model.ema if self.use_model_ema else self.model
+ ap50_95, ap50, summary = self.exp.eval(
+ evalmodel, self.evaluator, self.is_distributed
+ )
+ self.model.train()
+ if self.rank == 0:
+ self.tblogger.add_scalar("val/COCOAP50", ap50, self.epoch + 1)
+ self.tblogger.add_scalar("val/COCOAP50_95", ap50_95, self.epoch + 1)
+ logger.info("\n" + summary)
+ synchronize()
+
+ #self.best_ap = max(self.best_ap, ap50_95)
+ self.save_ckpt("last_epoch", ap50 > self.best_ap)
+ self.best_ap = max(self.best_ap, ap50)
+
+ def save_ckpt(self, ckpt_name, update_best_ckpt=False):
+ if self.rank == 0:
+ save_model = self.ema_model.ema if self.use_model_ema else self.model
+ logger.info("Save weights to {}".format(self.file_name))
+ ckpt_state = {
+ "start_epoch": self.epoch + 1,
+ "model": save_model.state_dict(),
+ "optimizer": self.optimizer.state_dict(),
+ }
+ save_checkpoint(
+ ckpt_state,
+ update_best_ckpt,
+ self.file_name,
+ ckpt_name,
+ )
diff --git a/experimental/yolox/data/__init__.py b/experimental/yolox/data/__init__.py
new file mode 100644
index 0000000..258effe
--- /dev/null
+++ b/experimental/yolox/data/__init__.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) Megvii, Inc. and its affiliates.
+
+from .data_augment import TrainTransform, ValTransform
+from .data_prefetcher import DataPrefetcher
+from .dataloading import DataLoader, get_yolox_datadir
+from .datasets import *
+from .samplers import InfiniteSampler, YoloBatchSampler
diff --git a/experimental/yolox/data/data_augment.py b/experimental/yolox/data/data_augment.py
new file mode 100644
index 0000000..99fb30a
--- /dev/null
+++ b/experimental/yolox/data/data_augment.py
@@ -0,0 +1,299 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) Megvii, Inc. and its affiliates.
+"""
+Data augmentation functionality. Passed as callable transformations to
+Dataset classes.
+
+The data augmentation procedures were interpreted from @weiliu89's SSD paper
+http://arxiv.org/abs/1512.02325
+"""
+
+import cv2
+import numpy as np
+
+import torch
+
+from yolox.utils import xyxy2cxcywh
+
+import math
+import random
+
+
+def augment_hsv(img, hgain=0.015, sgain=0.7, vgain=0.4):
+ r = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] + 1 # random gains
+ hue, sat, val = cv2.split(cv2.cvtColor(img, cv2.COLOR_BGR2HSV))
+ dtype = img.dtype # uint8
+
+ x = np.arange(0, 256, dtype=np.int16)
+ lut_hue = ((x * r[0]) % 180).astype(dtype)
+ lut_sat = np.clip(x * r[1], 0, 255).astype(dtype)
+ lut_val = np.clip(x * r[2], 0, 255).astype(dtype)
+
+ img_hsv = cv2.merge(
+ (cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val))
+ ).astype(dtype)
+ cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR, dst=img) # no return needed
+
+
+def box_candidates(box1, box2, wh_thr=2, ar_thr=20, area_thr=0.2):
+ # box1(4,n), box2(4,n)
+ # Compute candidate boxes which include follwing 5 things:
+ # box1 before augment, box2 after augment, wh_thr (pixels), aspect_ratio_thr, area_ratio
+ w1, h1 = box1[2] - box1[0], box1[3] - box1[1]
+ w2, h2 = box2[2] - box2[0], box2[3] - box2[1]
+ ar = np.maximum(w2 / (h2 + 1e-16), h2 / (w2 + 1e-16)) # aspect ratio
+ return (
+ (w2 > wh_thr)
+ & (h2 > wh_thr)
+ & (w2 * h2 / (w1 * h1 + 1e-16) > area_thr)
+ & (ar < ar_thr)
+ ) # candidates
+
+
+def random_perspective(
+ img,
+ targets=(),
+ degrees=10,
+ translate=0.1,
+ scale=0.1,
+ shear=10,
+ perspective=0.0,
+ border=(0, 0),
+):
+ # targets = [cls, xyxy]
+ height = img.shape[0] + border[0] * 2 # shape(h,w,c)
+ width = img.shape[1] + border[1] * 2
+
+ # Center
+ C = np.eye(3)
+ C[0, 2] = -img.shape[1] / 2 # x translation (pixels)
+ C[1, 2] = -img.shape[0] / 2 # y translation (pixels)
+
+ # Rotation and Scale
+ R = np.eye(3)
+ a = random.uniform(-degrees, degrees)
+ # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations
+ s = random.uniform(scale[0], scale[1])
+ # s = 2 ** random.uniform(-scale, scale)
+ R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s)
+
+ # Shear
+ S = np.eye(3)
+ S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # x shear (deg)
+ S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # y shear (deg)
+
+ # Translation
+ T = np.eye(3)
+ T[0, 2] = (
+ random.uniform(0.5 - translate, 0.5 + translate) * width
+ ) # x translation (pixels)
+ T[1, 2] = (
+ random.uniform(0.5 - translate, 0.5 + translate) * height
+ ) # y translation (pixels)
+
+ # Combined rotation matrix
+ M = T @ S @ R @ C # order of operations (right to left) is IMPORTANT
+
+ ###########################
+ # For Aug out of Mosaic
+ # s = 1.
+ # M = np.eye(3)
+ ###########################
+
+ if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed
+ if perspective:
+ img = cv2.warpPerspective(
+ img, M, dsize=(width, height), borderValue=(114, 114, 114)
+ )
+ else: # affine
+ img = cv2.warpAffine(
+ img, M[:2], dsize=(width, height), borderValue=(114, 114, 114)
+ )
+
+ # Transform label coordinates
+ n = len(targets)
+ if n:
+ # warp points
+ xy = np.ones((n * 4, 3))
+ xy[:, :2] = targets[:, [0, 1, 2, 3, 0, 3, 2, 1]].reshape(
+ n * 4, 2
+ ) # x1y1, x2y2, x1y2, x2y1
+ xy = xy @ M.T # transform
+ if perspective:
+ xy = (xy[:, :2] / xy[:, 2:3]).reshape(n, 8) # rescale
+ else: # affine
+ xy = xy[:, :2].reshape(n, 8)
+
+ # create new boxes
+ x = xy[:, [0, 2, 4, 6]]
+ y = xy[:, [1, 3, 5, 7]]
+ xy = np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T
+
+ # clip boxes
+ #xy[:, [0, 2]] = xy[:, [0, 2]].clip(0, width)
+ #xy[:, [1, 3]] = xy[:, [1, 3]].clip(0, height)
+
+ # filter candidates
+ i = box_candidates(box1=targets[:, :4].T * s, box2=xy.T)
+ targets = targets[i]
+ targets[:, :4] = xy[i]
+
+ targets = targets[targets[:, 0] < width]
+ targets = targets[targets[:, 2] > 0]
+ targets = targets[targets[:, 1] < height]
+ targets = targets[targets[:, 3] > 0]
+
+ return img, targets
+
+
+def _distort(image):
+ def _convert(image, alpha=1, beta=0):
+ tmp = image.astype(float) * alpha + beta
+ tmp[tmp < 0] = 0
+ tmp[tmp > 255] = 255
+ image[:] = tmp
+
+ image = image.copy()
+
+ if random.randrange(2):
+ _convert(image, beta=random.uniform(-32, 32))
+
+ if random.randrange(2):
+ _convert(image, alpha=random.uniform(0.5, 1.5))
+
+ image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
+
+ if random.randrange(2):
+ tmp = image[:, :, 0].astype(int) + random.randint(-18, 18)
+ tmp %= 180
+ image[:, :, 0] = tmp
+
+ if random.randrange(2):
+ _convert(image[:, :, 1], alpha=random.uniform(0.5, 1.5))
+
+ image = cv2.cvtColor(image, cv2.COLOR_HSV2BGR)
+
+ return image
+
+
+def _mirror(image, boxes):
+ _, width, _ = image.shape
+ if random.randrange(2):
+ image = image[:, ::-1]
+ boxes = boxes.copy()
+ boxes[:, 0::2] = width - boxes[:, 2::-2]
+ return image, boxes
+
+
+def preproc(image, input_size, mean, std, swap=(2, 0, 1)):
+ if len(image.shape) == 3:
+ padded_img = np.ones((input_size[0], input_size[1], 3)) * 114.0
+ else:
+ padded_img = np.ones(input_size) * 114.0
+ img = np.array(image)
+ r = min(input_size[0] / img.shape[0], input_size[1] / img.shape[1])
+ resized_img = cv2.resize(
+ img,
+ (int(img.shape[1] * r), int(img.shape[0] * r)),
+ interpolation=cv2.INTER_LINEAR,
+ ).astype(np.float32)
+ padded_img[: int(img.shape[0] * r), : int(img.shape[1] * r)] = resized_img
+
+ padded_img = padded_img[:, :, ::-1]
+ padded_img /= 255.0
+ if mean is not None:
+ padded_img -= mean
+ if std is not None:
+ padded_img /= std
+ padded_img = padded_img.transpose(swap)
+ padded_img = np.ascontiguousarray(padded_img, dtype=np.float32)
+ return padded_img, r
+
+
+class TrainTransform:
+ def __init__(self, p=0.5, rgb_means=None, std=None, max_labels=100):
+ self.means = rgb_means
+ self.std = std
+ self.p = p
+ self.max_labels = max_labels
+
+ def __call__(self, image, targets, input_dim):
+ boxes = targets[:, :4].copy()
+ labels = targets[:, 4].copy()
+ ids = targets[:, 5].copy()
+ if len(boxes) == 0:
+ targets = np.zeros((self.max_labels, 6), dtype=np.float32)
+ image, r_o = preproc(image, input_dim, self.means, self.std)
+ image = np.ascontiguousarray(image, dtype=np.float32)
+ return image, targets
+
+ image_o = image.copy()
+ targets_o = targets.copy()
+ height_o, width_o, _ = image_o.shape
+ boxes_o = targets_o[:, :4]
+ labels_o = targets_o[:, 4]
+ ids_o = targets_o[:, 5]
+ # bbox_o: [xyxy] to [c_x,c_y,w,h]
+ boxes_o = xyxy2cxcywh(boxes_o)
+
+ image_t = _distort(image)
+ image_t, boxes = _mirror(image_t, boxes)
+ height, width, _ = image_t.shape
+ image_t, r_ = preproc(image_t, input_dim, self.means, self.std)
+ # boxes [xyxy] 2 [cx,cy,w,h]
+ boxes = xyxy2cxcywh(boxes)
+ boxes *= r_
+
+ mask_b = np.minimum(boxes[:, 2], boxes[:, 3]) > 1
+ boxes_t = boxes[mask_b]
+ labels_t = labels[mask_b]
+ ids_t = ids[mask_b]
+
+ if len(boxes_t) == 0:
+ image_t, r_o = preproc(image_o, input_dim, self.means, self.std)
+ boxes_o *= r_o
+ boxes_t = boxes_o
+ labels_t = labels_o
+ ids_t = ids_o
+
+ labels_t = np.expand_dims(labels_t, 1)
+ ids_t = np.expand_dims(ids_t, 1)
+
+ targets_t = np.hstack((labels_t, boxes_t, ids_t))
+ padded_labels = np.zeros((self.max_labels, 6))
+ padded_labels[range(len(targets_t))[: self.max_labels]] = targets_t[
+ : self.max_labels
+ ]
+ padded_labels = np.ascontiguousarray(padded_labels, dtype=np.float32)
+ image_t = np.ascontiguousarray(image_t, dtype=np.float32)
+ return image_t, padded_labels
+
+
+class ValTransform:
+ """
+ Defines the transformations that should be applied to test PIL image
+ for input into the network
+
+ dimension -> tensorize -> color adj
+
+ Arguments:
+ resize (int): input dimension to SSD
+ rgb_means ((int,int,int)): average RGB of the dataset
+ (104,117,123)
+ swap ((int,int,int)): final order of channels
+
+ Returns:
+ transform (transform) : callable transform to be applied to test/val
+ data
+ """
+
+ def __init__(self, rgb_means=None, std=None, swap=(2, 0, 1)):
+ self.means = rgb_means
+ self.swap = swap
+ self.std = std
+
+ # assume input is cv2 img for now
+ def __call__(self, img, res, input_size):
+ img, _ = preproc(img, input_size, self.means, self.std, self.swap)
+ return img, np.zeros((1, 5))
diff --git a/experimental/yolox/data/data_prefetcher.py b/experimental/yolox/data/data_prefetcher.py
new file mode 100644
index 0000000..0f5d2b5
--- /dev/null
+++ b/experimental/yolox/data/data_prefetcher.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) Megvii, Inc. and its affiliates.
+
+import torch
+import torch.distributed as dist
+
+from yolox.utils import synchronize
+
+import random
+
+
+class DataPrefetcher:
+ """
+ DataPrefetcher is inspired by code of following file:
+ https://github.com/NVIDIA/apex/blob/master/examples/imagenet/main_amp.py
+ It could speedup your pytorch dataloader. For more information, please check
+ https://github.com/NVIDIA/apex/issues/304#issuecomment-493562789.
+ """
+
+ def __init__(self, loader):
+ self.loader = iter(loader)
+ self.stream = torch.cuda.Stream()
+ self.input_cuda = self._input_cuda_for_image
+ self.record_stream = DataPrefetcher._record_stream_for_image
+ self.preload()
+
+ def preload(self):
+ try:
+ self.next_input, self.next_target, _, _ = next(self.loader)
+ except StopIteration:
+ self.next_input = None
+ self.next_target = None
+ return
+
+ with torch.cuda.stream(self.stream):
+ self.input_cuda()
+ self.next_target = self.next_target.cuda(non_blocking=True)
+
+ def next(self):
+ torch.cuda.current_stream().wait_stream(self.stream)
+ input = self.next_input
+ target = self.next_target
+ if input is not None:
+ self.record_stream(input)
+ if target is not None:
+ target.record_stream(torch.cuda.current_stream())
+ self.preload()
+ return input, target
+
+ def _input_cuda_for_image(self):
+ self.next_input = self.next_input.cuda(non_blocking=True)
+
+ @staticmethod
+ def _record_stream_for_image(input):
+ input.record_stream(torch.cuda.current_stream())
+
+
+def random_resize(data_loader, exp, epoch, rank, is_distributed):
+ tensor = torch.LongTensor(1).cuda()
+ if is_distributed:
+ synchronize()
+
+ if rank == 0:
+ if epoch > exp.max_epoch - 10:
+ size = exp.input_size
+ else:
+ size = random.randint(*exp.random_size)
+ size = int(32 * size)
+ tensor.fill_(size)
+
+ if is_distributed:
+ synchronize()
+ dist.broadcast(tensor, 0)
+
+ input_size = data_loader.change_input_dim(multiple=tensor.item(), random_range=None)
+ return input_size
diff --git a/experimental/yolox/data/dataloading.py b/experimental/yolox/data/dataloading.py
new file mode 100644
index 0000000..2756f2f
--- /dev/null
+++ b/experimental/yolox/data/dataloading.py
@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) Megvii, Inc. and its affiliates.
+
+import torch
+from torch.utils.data.dataloader import DataLoader as torchDataLoader
+from torch.utils.data.dataloader import default_collate
+
+import os
+import random
+
+from .samplers import YoloBatchSampler
+
+
+def get_yolox_datadir():
+ """
+ get dataset dir of YOLOX. If environment variable named `YOLOX_DATADIR` is set,
+ this function will return value of the environment variable. Otherwise, use data
+ """
+ yolox_datadir = os.getenv("YOLOX_DATADIR", None)
+ if yolox_datadir is None:
+ import yolox
+
+ yolox_path = os.path.dirname(os.path.dirname(yolox.__file__))
+ yolox_datadir = os.path.join(yolox_path, "datasets")
+ return yolox_datadir
+
+
+class DataLoader(torchDataLoader):
+ """
+ Lightnet dataloader that enables on the fly resizing of the images.
+ See :class:`torch.utils.data.DataLoader` for more information on the arguments.
+ Check more on the following website:
+ https://gitlab.com/EAVISE/lightnet/-/blob/master/lightnet/data/_dataloading.py
+
+ Note:
+ This dataloader only works with :class:`lightnet.data.Dataset` based datasets.
+
+ Example:
+ >>> class CustomSet(ln.data.Dataset):
+ ... def __len__(self):
+ ... return 4
+ ... @ln.data.Dataset.resize_getitem
+ ... def __getitem__(self, index):
+ ... # Should return (image, anno) but here we return (input_dim,)
+ ... return (self.input_dim,)
+ >>> dl = ln.data.DataLoader(
+ ... CustomSet((200,200)),
+ ... batch_size = 2,
+ ... collate_fn = ln.data.list_collate # We want the data to be grouped as a list
+ ... )
+ >>> dl.dataset.input_dim # Default input_dim
+ (200, 200)
+ >>> for d in dl:
+ ... d
+ [[(200, 200), (200, 200)]]
+ [[(200, 200), (200, 200)]]
+ >>> dl.change_input_dim(320, random_range=None)
+ (320, 320)
+ >>> for d in dl:
+ ... d
+ [[(320, 320), (320, 320)]]
+ [[(320, 320), (320, 320)]]
+ >>> dl.change_input_dim((480, 320), random_range=None)
+ (480, 320)
+ >>> for d in dl:
+ ... d
+ [[(480, 320), (480, 320)]]
+ [[(480, 320), (480, 320)]]
+ """
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.__initialized = False
+ shuffle = False
+ batch_sampler = None
+ if len(args) > 5:
+ shuffle = args[2]
+ sampler = args[3]
+ batch_sampler = args[4]
+ elif len(args) > 4:
+ shuffle = args[2]
+ sampler = args[3]
+ if "batch_sampler" in kwargs:
+ batch_sampler = kwargs["batch_sampler"]
+ elif len(args) > 3:
+ shuffle = args[2]
+ if "sampler" in kwargs:
+ sampler = kwargs["sampler"]
+ if "batch_sampler" in kwargs:
+ batch_sampler = kwargs["batch_sampler"]
+ else:
+ if "shuffle" in kwargs:
+ shuffle = kwargs["shuffle"]
+ if "sampler" in kwargs:
+ sampler = kwargs["sampler"]
+ if "batch_sampler" in kwargs:
+ batch_sampler = kwargs["batch_sampler"]
+
+ # Use custom BatchSampler
+ if batch_sampler is None:
+ if sampler is None:
+ if shuffle:
+ sampler = torch.utils.data.sampler.RandomSampler(self.dataset)
+ # sampler = torch.utils.data.DistributedSampler(self.dataset)
+ else:
+ sampler = torch.utils.data.sampler.SequentialSampler(self.dataset)
+ batch_sampler = YoloBatchSampler(
+ sampler,
+ self.batch_size,
+ self.drop_last,
+ input_dimension=self.dataset.input_dim,
+ )
+ # batch_sampler = IterationBasedBatchSampler(batch_sampler, num_iterations =
+
+ self.batch_sampler = batch_sampler
+
+ self.__initialized = True
+
+ def close_mosaic(self):
+ self.batch_sampler.mosaic = False
+
+ def change_input_dim(self, multiple=32, random_range=(10, 19)):
+ """This function will compute a new size and update it on the next mini_batch.
+
+ Args:
+ multiple (int or tuple, optional): values to multiply the randomly generated range by.
+ Default **32**
+ random_range (tuple, optional): This (min, max) tuple sets the range
+ for the randomisation; Default **(10, 19)**
+
+ Return:
+ tuple: width, height tuple with new dimension
+
+ Note:
+ The new size is generated as follows: |br|
+ First we compute a random integer inside ``[random_range]``.
+ We then multiply that number with the ``multiple`` argument,
+ which gives our final new input size. |br|
+ If ``multiple`` is an integer we generate a square size. If you give a tuple
+ of **(width, height)**, the size is computed
+ as :math:`rng * multiple[0], rng * multiple[1]`.
+
+ Note:
+ You can set the ``random_range`` argument to **None** to set
+ an exact size of multiply. |br|
+ See the example above for how this works.
+ """
+ if random_range is None:
+ size = 1
+ else:
+ size = random.randint(*random_range)
+
+ if isinstance(multiple, int):
+ size = (size * multiple, size * multiple)
+ else:
+ size = (size * multiple[0], size * multiple[1])
+
+ self.batch_sampler.new_input_dim = size
+
+ return size
+
+
+def list_collate(batch):
+ """
+ Function that collates lists or tuples together into one list (of lists/tuples).
+ Use this as the collate function in a Dataloader, if you want to have a list of
+ items as an output, as opposed to tensors (eg. Brambox.boxes).
+ """
+ items = list(zip(*batch))
+
+ for i in range(len(items)):
+ if isinstance(items[i][0], (list, tuple)):
+ items[i] = list(items[i])
+ else:
+ items[i] = default_collate(items[i])
+
+ return items
diff --git a/experimental/yolox/data/datasets/__init__.py b/experimental/yolox/data/datasets/__init__.py
new file mode 100644
index 0000000..61065a8
--- /dev/null
+++ b/experimental/yolox/data/datasets/__init__.py
@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) Megvii, Inc. and its affiliates.
+
+from .datasets_wrapper import ConcatDataset, Dataset, MixConcatDataset
+from .mosaicdetection import MosaicDetection
+from .mot import MOTDataset
diff --git a/experimental/yolox/data/datasets/datasets_wrapper.py b/experimental/yolox/data/datasets/datasets_wrapper.py
new file mode 100644
index 0000000..a262e6a
--- /dev/null
+++ b/experimental/yolox/data/datasets/datasets_wrapper.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) Megvii, Inc. and its affiliates.
+
+from torch.utils.data.dataset import ConcatDataset as torchConcatDataset
+from torch.utils.data.dataset import Dataset as torchDataset
+
+import bisect
+from functools import wraps
+
+
+class ConcatDataset(torchConcatDataset):
+ def __init__(self, datasets):
+ super(ConcatDataset, self).__init__(datasets)
+ if hasattr(self.datasets[0], "input_dim"):
+ self._input_dim = self.datasets[0].input_dim
+ self.input_dim = self.datasets[0].input_dim
+
+ def pull_item(self, idx):
+ if idx < 0:
+ if -idx > len(self):
+ raise ValueError(
+ "absolute value of index should not exceed dataset length"
+ )
+ idx = len(self) + idx
+ dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx)
+ if dataset_idx == 0:
+ sample_idx = idx
+ else:
+ sample_idx = idx - self.cumulative_sizes[dataset_idx - 1]
+ return self.datasets[dataset_idx].pull_item(sample_idx)
+
+
+class MixConcatDataset(torchConcatDataset):
+ def __init__(self, datasets):
+ super(MixConcatDataset, self).__init__(datasets)
+ if hasattr(self.datasets[0], "input_dim"):
+ self._input_dim = self.datasets[0].input_dim
+ self.input_dim = self.datasets[0].input_dim
+
+ def __getitem__(self, index):
+
+ if not isinstance(index, int):
+ idx = index[1]
+ if idx < 0:
+ if -idx > len(self):
+ raise ValueError(
+ "absolute value of index should not exceed dataset length"
+ )
+ idx = len(self) + idx
+ dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx)
+ if dataset_idx == 0:
+ sample_idx = idx
+ else:
+ sample_idx = idx - self.cumulative_sizes[dataset_idx - 1]
+ if not isinstance(index, int):
+ index = (index[0], sample_idx, index[2])
+
+ return self.datasets[dataset_idx][index]
+
+
+class Dataset(torchDataset):
+ """ This class is a subclass of the base :class:`torch.utils.data.Dataset`,
+ that enables on the fly resizing of the ``input_dim``.
+
+ Args:
+ input_dimension (tuple): (width,height) tuple with default dimensions of the network
+ """
+
+ def __init__(self, input_dimension, mosaic=True):
+ super().__init__()
+ self.__input_dim = input_dimension[:2]
+ self.enable_mosaic = mosaic
+
+ @property
+ def input_dim(self):
+ """
+ Dimension that can be used by transforms to set the correct image size, etc.
+ This allows transforms to have a single source of truth
+ for the input dimension of the network.
+
+ Return:
+ list: Tuple containing the current width,height
+ """
+ if hasattr(self, "_input_dim"):
+ return self._input_dim
+ return self.__input_dim
+
+ @staticmethod
+ def resize_getitem(getitem_fn):
+ """
+ Decorator method that needs to be used around the ``__getitem__`` method. |br|
+ This decorator enables the on the fly resizing of
+ the ``input_dim`` with our :class:`~lightnet.data.DataLoader` class.
+
+ Example:
+ >>> class CustomSet(ln.data.Dataset):
+ ... def __len__(self):
+ ... return 10
+ ... @ln.data.Dataset.resize_getitem
+ ... def __getitem__(self, index):
+ ... # Should return (image, anno) but here we return input_dim
+ ... return self.input_dim
+ >>> data = CustomSet((200,200))
+ >>> data[0]
+ (200, 200)
+ >>> data[(480,320), 0]
+ (480, 320)
+ """
+
+ @wraps(getitem_fn)
+ def wrapper(self, index):
+ if not isinstance(index, int):
+ has_dim = True
+ self._input_dim = index[0]
+ self.enable_mosaic = index[2]
+ index = index[1]
+ else:
+ has_dim = False
+
+ ret_val = getitem_fn(self, index)
+
+ if has_dim:
+ del self._input_dim
+
+ return ret_val
+
+ return wrapper
diff --git a/experimental/yolox/data/datasets/mosaicdetection.py b/experimental/yolox/data/datasets/mosaicdetection.py
new file mode 100644
index 0000000..d2bf39f
--- /dev/null
+++ b/experimental/yolox/data/datasets/mosaicdetection.py
@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) Megvii, Inc. and its affiliates.
+
+import cv2
+import numpy as np
+
+from yolox.utils import adjust_box_anns
+
+import random
+
+from ..data_augment import box_candidates, random_perspective, augment_hsv
+from .datasets_wrapper import Dataset
+
+
+def get_mosaic_coordinate(mosaic_image, mosaic_index, xc, yc, w, h, input_h, input_w):
+ # TODO update doc
+ # index0 to top left part of image
+ if mosaic_index == 0:
+ x1, y1, x2, y2 = max(xc - w, 0), max(yc - h, 0), xc, yc
+ small_coord = w - (x2 - x1), h - (y2 - y1), w, h
+ # index1 to top right part of image
+ elif mosaic_index == 1:
+ x1, y1, x2, y2 = xc, max(yc - h, 0), min(xc + w, input_w * 2), yc
+ small_coord = 0, h - (y2 - y1), min(w, x2 - x1), h
+ # index2 to bottom left part of image
+ elif mosaic_index == 2:
+ x1, y1, x2, y2 = max(xc - w, 0), yc, xc, min(input_h * 2, yc + h)
+ small_coord = w - (x2 - x1), 0, w, min(y2 - y1, h)
+ # index2 to bottom right part of image
+ elif mosaic_index == 3:
+ x1, y1, x2, y2 = xc, yc, min(xc + w, input_w * 2), min(input_h * 2, yc + h) # noqa
+ small_coord = 0, 0, min(w, x2 - x1), min(y2 - y1, h)
+ return (x1, y1, x2, y2), small_coord
+
+
+class MosaicDetection(Dataset):
+ """Detection dataset wrapper that performs mixup for normal dataset."""
+
+ def __init__(
+ self, dataset, img_size, mosaic=True, preproc=None,
+ degrees=10.0, translate=0.1, scale=(0.5, 1.5), mscale=(0.5, 1.5),
+ shear=2.0, perspective=0.0, enable_mixup=True, *args
+ ):
+ """
+
+ Args:
+ dataset(Dataset) : Pytorch dataset object.
+ img_size (tuple):
+ mosaic (bool): enable mosaic augmentation or not.
+ preproc (func):
+ degrees (float):
+ translate (float):
+ scale (tuple):
+ mscale (tuple):
+ shear (float):
+ perspective (float):
+ enable_mixup (bool):
+ *args(tuple) : Additional arguments for mixup random sampler.
+ """
+ super().__init__(img_size, mosaic=mosaic)
+ self._dataset = dataset
+ self.preproc = preproc
+ self.degrees = degrees
+ self.translate = translate
+ self.scale = scale
+ self.shear = shear
+ self.perspective = perspective
+ self.mixup_scale = mscale
+ self.enable_mosaic = mosaic
+ self.enable_mixup = enable_mixup
+
+ def __len__(self):
+ return len(self._dataset)
+
+ @Dataset.resize_getitem
+ def __getitem__(self, idx):
+ if self.enable_mosaic:
+ mosaic_labels = []
+ input_dim = self._dataset.input_dim
+ input_h, input_w = input_dim[0], input_dim[1]
+
+ # yc, xc = s, s # mosaic center x, y
+ yc = int(random.uniform(0.5 * input_h, 1.5 * input_h))
+ xc = int(random.uniform(0.5 * input_w, 1.5 * input_w))
+
+ # 3 additional image indices
+ indices = [idx] + [random.randint(0, len(self._dataset) - 1) for _ in range(3)]
+
+ for i_mosaic, index in enumerate(indices):
+ img, _labels, _, _ = self._dataset.pull_item(index)
+ h0, w0 = img.shape[:2] # orig hw
+ scale = min(1. * input_h / h0, 1. * input_w / w0)
+ img = cv2.resize(
+ img, (int(w0 * scale), int(h0 * scale)), interpolation=cv2.INTER_LINEAR
+ )
+ # generate output mosaic image
+ (h, w, c) = img.shape[:3]
+ if i_mosaic == 0:
+ mosaic_img = np.full((input_h * 2, input_w * 2, c), 114, dtype=np.uint8)
+
+ # suffix l means large image, while s means small image in mosaic aug.
+ (l_x1, l_y1, l_x2, l_y2), (s_x1, s_y1, s_x2, s_y2) = get_mosaic_coordinate(
+ mosaic_img, i_mosaic, xc, yc, w, h, input_h, input_w
+ )
+
+ mosaic_img[l_y1:l_y2, l_x1:l_x2] = img[s_y1:s_y2, s_x1:s_x2]
+ padw, padh = l_x1 - s_x1, l_y1 - s_y1
+
+ labels = _labels.copy()
+ # Normalized xywh to pixel xyxy format
+ if _labels.size > 0:
+ labels[:, 0] = scale * _labels[:, 0] + padw
+ labels[:, 1] = scale * _labels[:, 1] + padh
+ labels[:, 2] = scale * _labels[:, 2] + padw
+ labels[:, 3] = scale * _labels[:, 3] + padh
+ mosaic_labels.append(labels)
+
+ if len(mosaic_labels):
+ mosaic_labels = np.concatenate(mosaic_labels, 0)
+ '''
+ np.clip(mosaic_labels[:, 0], 0, 2 * input_w, out=mosaic_labels[:, 0])
+ np.clip(mosaic_labels[:, 1], 0, 2 * input_h, out=mosaic_labels[:, 1])
+ np.clip(mosaic_labels[:, 2], 0, 2 * input_w, out=mosaic_labels[:, 2])
+ np.clip(mosaic_labels[:, 3], 0, 2 * input_h, out=mosaic_labels[:, 3])
+ '''
+
+ mosaic_labels = mosaic_labels[mosaic_labels[:, 0] < 2 * input_w]
+ mosaic_labels = mosaic_labels[mosaic_labels[:, 2] > 0]
+ mosaic_labels = mosaic_labels[mosaic_labels[:, 1] < 2 * input_h]
+ mosaic_labels = mosaic_labels[mosaic_labels[:, 3] > 0]
+
+ #augment_hsv(mosaic_img)
+ mosaic_img, mosaic_labels = random_perspective(
+ mosaic_img,
+ mosaic_labels,
+ degrees=self.degrees,
+ translate=self.translate,
+ scale=self.scale,
+ shear=self.shear,
+ perspective=self.perspective,
+ border=[-input_h // 2, -input_w // 2],
+ ) # border to remove
+
+ # -----------------------------------------------------------------
+ # CopyPaste: https://arxiv.org/abs/2012.07177
+ # -----------------------------------------------------------------
+ if self.enable_mixup and not len(mosaic_labels) == 0:
+ mosaic_img, mosaic_labels = self.mixup(mosaic_img, mosaic_labels, self.input_dim)
+
+ mix_img, padded_labels = self.preproc(mosaic_img, mosaic_labels, self.input_dim)
+ img_info = (mix_img.shape[1], mix_img.shape[0])
+
+ return mix_img, padded_labels, img_info, np.array([idx])
+
+ else:
+ self._dataset._input_dim = self.input_dim
+ img, label, img_info, id_ = self._dataset.pull_item(idx)
+ img, label = self.preproc(img, label, self.input_dim)
+ return img, label, img_info, id_
+
+ def mixup(self, origin_img, origin_labels, input_dim):
+ jit_factor = random.uniform(*self.mixup_scale)
+ FLIP = random.uniform(0, 1) > 0.5
+ cp_labels = []
+ while len(cp_labels) == 0:
+ cp_index = random.randint(0, self.__len__() - 1)
+ cp_labels = self._dataset.load_anno(cp_index)
+ img, cp_labels, _, _ = self._dataset.pull_item(cp_index)
+
+ if len(img.shape) == 3:
+ cp_img = np.ones((input_dim[0], input_dim[1], 3)) * 114.0
+ else:
+ cp_img = np.ones(input_dim) * 114.0
+ cp_scale_ratio = min(input_dim[0] / img.shape[0], input_dim[1] / img.shape[1])
+ resized_img = cv2.resize(
+ img,
+ (int(img.shape[1] * cp_scale_ratio), int(img.shape[0] * cp_scale_ratio)),
+ interpolation=cv2.INTER_LINEAR,
+ ).astype(np.float32)
+ cp_img[
+ : int(img.shape[0] * cp_scale_ratio), : int(img.shape[1] * cp_scale_ratio)
+ ] = resized_img
+ cp_img = cv2.resize(
+ cp_img,
+ (int(cp_img.shape[1] * jit_factor), int(cp_img.shape[0] * jit_factor)),
+ )
+ cp_scale_ratio *= jit_factor
+ if FLIP:
+ cp_img = cp_img[:, ::-1, :]
+
+ origin_h, origin_w = cp_img.shape[:2]
+ target_h, target_w = origin_img.shape[:2]
+ padded_img = np.zeros(
+ (max(origin_h, target_h), max(origin_w, target_w), 3)
+ ).astype(np.uint8)
+ padded_img[:origin_h, :origin_w] = cp_img
+
+ x_offset, y_offset = 0, 0
+ if padded_img.shape[0] > target_h:
+ y_offset = random.randint(0, padded_img.shape[0] - target_h - 1)
+ if padded_img.shape[1] > target_w:
+ x_offset = random.randint(0, padded_img.shape[1] - target_w - 1)
+ padded_cropped_img = padded_img[
+ y_offset: y_offset + target_h, x_offset: x_offset + target_w
+ ]
+
+ cp_bboxes_origin_np = adjust_box_anns(
+ cp_labels[:, :4].copy(), cp_scale_ratio, 0, 0, origin_w, origin_h
+ )
+ if FLIP:
+ cp_bboxes_origin_np[:, 0::2] = (
+ origin_w - cp_bboxes_origin_np[:, 0::2][:, ::-1]
+ )
+ cp_bboxes_transformed_np = cp_bboxes_origin_np.copy()
+ '''
+ cp_bboxes_transformed_np[:, 0::2] = np.clip(
+ cp_bboxes_transformed_np[:, 0::2] - x_offset, 0, target_w
+ )
+ cp_bboxes_transformed_np[:, 1::2] = np.clip(
+ cp_bboxes_transformed_np[:, 1::2] - y_offset, 0, target_h
+ )
+ '''
+ cp_bboxes_transformed_np[:, 0::2] = cp_bboxes_transformed_np[:, 0::2] - x_offset
+ cp_bboxes_transformed_np[:, 1::2] = cp_bboxes_transformed_np[:, 1::2] - y_offset
+ keep_list = box_candidates(cp_bboxes_origin_np.T, cp_bboxes_transformed_np.T, 5)
+
+ if keep_list.sum() >= 1.0:
+ cls_labels = cp_labels[keep_list, 4:5].copy()
+ id_labels = cp_labels[keep_list, 5:6].copy()
+ box_labels = cp_bboxes_transformed_np[keep_list]
+ labels = np.hstack((box_labels, cls_labels, id_labels))
+ # remove outside bbox
+ labels = labels[labels[:, 0] < target_w]
+ labels = labels[labels[:, 2] > 0]
+ labels = labels[labels[:, 1] < target_h]
+ labels = labels[labels[:, 3] > 0]
+ origin_labels = np.vstack((origin_labels, labels))
+ origin_img = origin_img.astype(np.float32)
+ origin_img = 0.5 * origin_img + 0.5 * padded_cropped_img.astype(np.float32)
+
+ return origin_img, origin_labels
diff --git a/experimental/yolox/data/datasets/mot.py b/experimental/yolox/data/datasets/mot.py
new file mode 100644
index 0000000..d52febc
--- /dev/null
+++ b/experimental/yolox/data/datasets/mot.py
@@ -0,0 +1,132 @@
+import cv2
+import numpy as np
+from pycocotools.coco import COCO
+
+import os
+
+from ..dataloading import get_yolox_datadir
+from .datasets_wrapper import Dataset
+
+
+class MOTDataset(Dataset):
+ """
+ COCO dataset class.
+ """
+
+ def __init__(
+ self,
+ data_dir=None,
+ json_file="train_half.json",
+ name="train",
+ img_size=(608, 1088),
+ preproc=None,
+ ):
+ """
+ COCO dataset initialization. Annotation data are read into memory by COCO API.
+ Args:
+ data_dir (str): dataset root directory
+ json_file (str): COCO json file name
+ name (str): COCO data name (e.g. 'train2017' or 'val2017')
+ img_size (int): target image size after pre-processing
+ preproc: data augmentation strategy
+ """
+ super().__init__(img_size)
+ if data_dir is None:
+ data_dir = os.path.join(get_yolox_datadir(), "mot")
+ self.data_dir = data_dir
+ self.json_file = json_file
+
+ self.coco = COCO(os.path.join(self.data_dir, "annotations", self.json_file))
+ self.ids = self.coco.getImgIds()
+ self.class_ids = sorted(self.coco.getCatIds())
+ cats = self.coco.loadCats(self.coco.getCatIds())
+ self._classes = tuple([c["name"] for c in cats])
+ self.annotations = self._load_coco_annotations()
+ self.name = name
+ self.img_size = img_size
+ self.preproc = preproc
+
+ def __len__(self):
+ return len(self.ids)
+
+ def _load_coco_annotations(self):
+ return [self.load_anno_from_ids(_ids) for _ids in self.ids]
+
+ def load_anno_from_ids(self, id_):
+ im_ann = self.coco.loadImgs(id_)[0]
+ width = im_ann["width"]
+ height = im_ann["height"]
+ frame_id = im_ann["frame_id"]
+ video_id = im_ann["video_id"]
+ anno_ids = self.coco.getAnnIds(imgIds=[int(id_)], iscrowd=False)
+ annotations = self.coco.loadAnns(anno_ids)
+ objs = []
+ for obj in annotations:
+ x1 = obj["bbox"][0]
+ y1 = obj["bbox"][1]
+ x2 = x1 + obj["bbox"][2]
+ y2 = y1 + obj["bbox"][3]
+ if obj["area"] > 0 and x2 >= x1 and y2 >= y1:
+ obj["clean_bbox"] = [x1, y1, x2, y2]
+ objs.append(obj)
+
+ num_objs = len(objs)
+
+ res = np.zeros((num_objs, 6))
+
+ for ix, obj in enumerate(objs):
+ cls = self.class_ids.index(obj["category_id"])
+ res[ix, 0:4] = obj["clean_bbox"]
+ res[ix, 4] = cls
+ res[ix, 5] = obj["track_id"]
+
+ file_name = im_ann["file_name"] if "file_name" in im_ann else "{:012}".format(id_) + ".jpg"
+ img_info = (height, width, frame_id, video_id, file_name)
+
+ del im_ann, annotations
+
+ return (res, img_info, file_name)
+
+ def load_anno(self, index):
+ return self.annotations[index][0]
+
+ def pull_item(self, index):
+ id_ = self.ids[index]
+
+ res, img_info, file_name = self.annotations[index]
+ # load image and preprocess
+ img_file = os.path.join(
+ self.data_dir, self.name, file_name
+ )
+ img = cv2.imread(img_file)
+ assert img is not None
+
+ return img, res.copy(), img_info, np.array([id_])
+
+ @Dataset.resize_getitem
+ def __getitem__(self, index):
+ """
+ One image / label pair for the given index is picked up and pre-processed.
+
+ Args:
+ index (int): data index
+
+ Returns:
+ img (numpy.ndarray): pre-processed image
+ padded_labels (torch.Tensor): pre-processed label data.
+ The shape is :math:`[max_labels, 5]`.
+ each label consists of [class, xc, yc, w, h]:
+ class (float): class index.
+ xc, yc (float) : center of bbox whose values range from 0 to 1.
+ w, h (float) : size of bbox whose values range from 0 to 1.
+ info_img : tuple of h, w, nh, nw, dx, dy.
+ h, w (int): original shape of the image
+ nh, nw (int): shape of the resized image without padding
+ dx, dy (int): pad size
+ img_id (int): same as the input index. Used for evaluation.
+ """
+ img, target, img_info, img_id = self.pull_item(index)
+
+ if self.preproc is not None:
+ img, target = self.preproc(img, target, self.input_dim)
+ return img, target, img_info, img_id
diff --git a/experimental/yolox/data/samplers.py b/experimental/yolox/data/samplers.py
new file mode 100644
index 0000000..064b133
--- /dev/null
+++ b/experimental/yolox/data/samplers.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) Megvii, Inc. and its affiliates.
+
+import torch
+import torch.distributed as dist
+from torch.utils.data.sampler import BatchSampler as torchBatchSampler
+from torch.utils.data.sampler import Sampler
+
+import itertools
+from typing import Optional
+
+
+class YoloBatchSampler(torchBatchSampler):
+ """
+ This batch sampler will generate mini-batches of (dim, index) tuples from another sampler.
+ It works just like the :class:`torch.utils.data.sampler.BatchSampler`,
+ but it will prepend a dimension, whilst ensuring it stays the same across one mini-batch.
+ """
+
+ def __init__(self, *args, input_dimension=None, mosaic=True, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.input_dim = input_dimension
+ self.new_input_dim = None
+ self.mosaic = mosaic
+
+ def __iter__(self):
+ self.__set_input_dim()
+ for batch in super().__iter__():
+ yield [(self.input_dim, idx, self.mosaic) for idx in batch]
+ self.__set_input_dim()
+
+ def __set_input_dim(self):
+ """ This function randomly changes the the input dimension of the dataset. """
+ if self.new_input_dim is not None:
+ self.input_dim = (self.new_input_dim[0], self.new_input_dim[1])
+ self.new_input_dim = None
+
+
+class InfiniteSampler(Sampler):
+ """
+ In training, we only care about the "infinite stream" of training data.
+ So this sampler produces an infinite stream of indices and
+ all workers cooperate to correctly shuffle the indices and sample different indices.
+ The samplers in each worker effectively produces `indices[worker_id::num_workers]`
+ where `indices` is an infinite stream of indices consisting of
+ `shuffle(range(size)) + shuffle(range(size)) + ...` (if shuffle is True)
+ or `range(size) + range(size) + ...` (if shuffle is False)
+ """
+
+ def __init__(
+ self,
+ size: int,
+ shuffle: bool = True,
+ seed: Optional[int] = 0,
+ rank=0,
+ world_size=1,
+ ):
+ """
+ Args:
+ size (int): the total number of data of the underlying dataset to sample from
+ shuffle (bool): whether to shuffle the indices or not
+ seed (int): the initial seed of the shuffle. Must be the same
+ across all workers. If None, will use a random seed shared
+ among workers (require synchronization among all workers).
+ """
+ self._size = size
+ assert size > 0
+ self._shuffle = shuffle
+ self._seed = int(seed)
+
+ if dist.is_available() and dist.is_initialized():
+ self._rank = dist.get_rank()
+ self._world_size = dist.get_world_size()
+ else:
+ self._rank = rank
+ self._world_size = world_size
+
+ def __iter__(self):
+ start = self._rank
+ yield from itertools.islice(
+ self._infinite_indices(), start, None, self._world_size
+ )
+
+ def _infinite_indices(self):
+ g = torch.Generator()
+ g.manual_seed(self._seed)
+ while True:
+ if self._shuffle:
+ yield from torch.randperm(self._size, generator=g)
+ else:
+ yield from torch.arange(self._size)
+
+ def __len__(self):
+ return self._size // self._world_size
diff --git a/experimental/yolox/deepsort_tracker/deepsort.py b/experimental/yolox/deepsort_tracker/deepsort.py
new file mode 100644
index 0000000..198f3b0
--- /dev/null
+++ b/experimental/yolox/deepsort_tracker/deepsort.py
@@ -0,0 +1,295 @@
+import numpy as np
+import torch
+import cv2
+import os
+
+from .reid_model import Extractor
+from yolox.deepsort_tracker import kalman_filter, linear_assignment, iou_matching
+from yolox.data.dataloading import get_yolox_datadir
+from .detection import Detection
+from .track import Track
+
+
+def _cosine_distance(a, b, data_is_normalized=False):
+ if not data_is_normalized:
+ a = np.asarray(a) / np.linalg.norm(a, axis=1, keepdims=True)
+ b = np.asarray(b) / np.linalg.norm(b, axis=1, keepdims=True)
+ return 1. - np.dot(a, b.T)
+
+
+def _nn_cosine_distance(x, y):
+ distances = _cosine_distance(x, y)
+ return distances.min(axis=0)
+
+
+class Tracker:
+ def __init__(self, metric, max_iou_distance=0.7, max_age=70, n_init=3):
+ self.metric = metric
+ self.max_iou_distance = max_iou_distance
+ self.max_age = max_age
+ self.n_init = n_init
+
+ self.kf = kalman_filter.KalmanFilter()
+ self.tracks = []
+ self._next_id = 1
+
+ def predict(self):
+ """Propagate track state distributions one time step forward.
+ This function should be called once every time step, before `update`.
+ """
+ for track in self.tracks:
+ track.predict(self.kf)
+
+ def increment_ages(self):
+ for track in self.tracks:
+ track.increment_age()
+ track.mark_missed()
+
+ def update(self, detections, classes):
+ """Perform measurement update and track management.
+ Parameters
+ ----------
+ detections : List[deep_sort.detection.Detection]
+ A list of detections at the current time step.
+ """
+ # Run matching cascade.
+ matches, unmatched_tracks, unmatched_detections = \
+ self._match(detections)
+
+ # Update track set.
+ for track_idx, detection_idx in matches:
+ self.tracks[track_idx].update(
+ self.kf, detections[detection_idx])
+ for track_idx in unmatched_tracks:
+ self.tracks[track_idx].mark_missed()
+ for detection_idx in unmatched_detections:
+ self._initiate_track(detections[detection_idx], classes[detection_idx].item())
+ self.tracks = [t for t in self.tracks if not t.is_deleted()]
+
+ # Update distance metric.
+ active_targets = [t.track_id for t in self.tracks if t.is_confirmed()]
+ features, targets = [], []
+ for track in self.tracks:
+ if not track.is_confirmed():
+ continue
+ features += track.features
+ targets += [track.track_id for _ in track.features]
+ track.features = []
+ self.metric.partial_fit(
+ np.asarray(features), np.asarray(targets), active_targets)
+
+ def _match(self, detections):
+
+ def gated_metric(tracks, dets, track_indices, detection_indices):
+ features = np.array([dets[i].feature for i in detection_indices])
+ targets = np.array([tracks[i].track_id for i in track_indices])
+ cost_matrix = self.metric.distance(features, targets)
+ cost_matrix = linear_assignment.gate_cost_matrix(
+ self.kf, cost_matrix, tracks, dets, track_indices,
+ detection_indices)
+
+ return cost_matrix
+
+ # Split track set into confirmed and unconfirmed tracks.
+ confirmed_tracks = [
+ i for i, t in enumerate(self.tracks) if t.is_confirmed()]
+ unconfirmed_tracks = [
+ i for i, t in enumerate(self.tracks) if not t.is_confirmed()]
+
+ # Associate confirmed tracks using appearance features.
+ matches_a, unmatched_tracks_a, unmatched_detections = \
+ linear_assignment.matching_cascade(
+ gated_metric, self.metric.matching_threshold, self.max_age,
+ self.tracks, detections, confirmed_tracks)
+
+ # Associate remaining tracks together with unconfirmed tracks using IOU.
+ iou_track_candidates = unconfirmed_tracks + [
+ k for k in unmatched_tracks_a if
+ self.tracks[k].time_since_update == 1]
+ unmatched_tracks_a = [
+ k for k in unmatched_tracks_a if
+ self.tracks[k].time_since_update != 1]
+ matches_b, unmatched_tracks_b, unmatched_detections = \
+ linear_assignment.min_cost_matching(
+ iou_matching.iou_cost, self.max_iou_distance, self.tracks,
+ detections, iou_track_candidates, unmatched_detections)
+
+ matches = matches_a + matches_b
+ unmatched_tracks = list(set(unmatched_tracks_a + unmatched_tracks_b))
+ return matches, unmatched_tracks, unmatched_detections
+
+ def _initiate_track(self, detection, class_id):
+ mean, covariance = self.kf.initiate(detection.to_xyah())
+ self.tracks.append(Track(
+ mean, covariance, self._next_id, class_id, self.n_init, self.max_age,
+ detection.feature))
+ self._next_id += 1
+
+
+class NearestNeighborDistanceMetric(object):
+ def __init__(self, metric, matching_threshold, budget=None):
+
+ if metric == "cosine":
+ self._metric = _nn_cosine_distance
+ else:
+ raise ValueError(
+ "Invalid metric; must be either 'euclidean' or 'cosine'")
+ self.matching_threshold = matching_threshold
+ self.budget = budget
+ self.samples = {}
+
+ def partial_fit(self, features, targets, active_targets):
+ for feature, target in zip(features, targets):
+ self.samples.setdefault(target, []).append(feature)
+ if self.budget is not None:
+ self.samples[target] = self.samples[target][-self.budget:]
+ self.samples = {k: self.samples[k] for k in active_targets}
+
+ def distance(self, features, targets):
+ cost_matrix = np.zeros((len(targets), len(features)))
+ for i, target in enumerate(targets):
+ cost_matrix[i, :] = self._metric(self.samples[target], features)
+ return cost_matrix
+
+
+class DeepSort(object):
+ def __init__(self, model_path, max_dist=0.1, min_confidence=0.3, nms_max_overlap=1.0, max_iou_distance=0.7, max_age=30, n_init=3, nn_budget=100, use_cuda=True):
+ self.min_confidence = min_confidence
+ self.nms_max_overlap = nms_max_overlap
+
+ self.extractor = Extractor(model_path, use_cuda=use_cuda)
+
+ max_cosine_distance = max_dist
+ metric = NearestNeighborDistanceMetric(
+ "cosine", max_cosine_distance, nn_budget)
+ self.tracker = Tracker(
+ metric, max_iou_distance=max_iou_distance, max_age=max_age, n_init=n_init)
+
+ def update(self, output_results, img_info, img_size, img_file_name):
+ img_file_name = os.path.join(get_yolox_datadir(), 'mot', 'train', img_file_name)
+ ori_img = cv2.imread(img_file_name)
+ self.height, self.width = ori_img.shape[:2]
+ # post process detections
+ output_results = output_results.cpu().numpy()
+ confidences = output_results[:, 4] * output_results[:, 5]
+
+ bboxes = output_results[:, :4] # x1y1x2y2
+ img_h, img_w = img_info[0], img_info[1]
+ scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w))
+ bboxes /= scale
+ bbox_xyxy = bboxes
+ bbox_tlwh = self._xyxy_to_tlwh_array(bbox_xyxy)
+ remain_inds = confidences > self.min_confidence
+ bbox_tlwh = bbox_tlwh[remain_inds]
+ confidences = confidences[remain_inds]
+
+ # generate detections
+ features = self._get_features(bbox_tlwh, ori_img)
+ detections = [Detection(bbox_tlwh[i], conf, features[i]) for i, conf in enumerate(
+ confidences) if conf > self.min_confidence]
+ classes = np.zeros((len(detections), ))
+
+ # run on non-maximum supression
+ boxes = np.array([d.tlwh for d in detections])
+ scores = np.array([d.confidence for d in detections])
+
+ # update tracker
+ self.tracker.predict()
+ self.tracker.update(detections, classes)
+
+ # output bbox identities
+ outputs = []
+ for track in self.tracker.tracks:
+ if not track.is_confirmed() or track.time_since_update > 1:
+ continue
+ box = track.to_tlwh()
+ x1, y1, x2, y2 = self._tlwh_to_xyxy_noclip(box)
+ track_id = track.track_id
+ class_id = track.class_id
+ outputs.append(np.array([x1, y1, x2, y2, track_id, class_id], dtype=np.int))
+ if len(outputs) > 0:
+ outputs = np.stack(outputs, axis=0)
+ return outputs
+
+ """
+ TODO:
+ Convert bbox from xc_yc_w_h to xtl_ytl_w_h
+ Thanks [email protected] for reporting this bug!
+ """
+ @staticmethod
+ def _xywh_to_tlwh(bbox_xywh):
+ if isinstance(bbox_xywh, np.ndarray):
+ bbox_tlwh = bbox_xywh.copy()
+ elif isinstance(bbox_xywh, torch.Tensor):
+ bbox_tlwh = bbox_xywh.clone()
+ bbox_tlwh[:, 0] = bbox_xywh[:, 0] - bbox_xywh[:, 2] / 2.
+ bbox_tlwh[:, 1] = bbox_xywh[:, 1] - bbox_xywh[:, 3] / 2.
+ return bbox_tlwh
+
+ @staticmethod
+ def _xyxy_to_tlwh_array(bbox_xyxy):
+ if isinstance(bbox_xyxy, np.ndarray):
+ bbox_tlwh = bbox_xyxy.copy()
+ elif isinstance(bbox_xyxy, torch.Tensor):
+ bbox_tlwh = bbox_xyxy.clone()
+ bbox_tlwh[:, 2] = bbox_xyxy[:, 2] - bbox_xyxy[:, 0]
+ bbox_tlwh[:, 3] = bbox_xyxy[:, 3] - bbox_xyxy[:, 1]
+ return bbox_tlwh
+
+ def _xywh_to_xyxy(self, bbox_xywh):
+ x, y, w, h = bbox_xywh
+ x1 = max(int(x - w / 2), 0)
+ x2 = min(int(x + w / 2), self.width - 1)
+ y1 = max(int(y - h / 2), 0)
+ y2 = min(int(y + h / 2), self.height - 1)
+ return x1, y1, x2, y2
+
+ def _tlwh_to_xyxy(self, bbox_tlwh):
+ """
+ TODO:
+ Convert bbox from xtl_ytl_w_h to xc_yc_w_h
+ Thanks [email protected] for reporting this bug!
+ """
+ x, y, w, h = bbox_tlwh
+ x1 = max(int(x), 0)
+ x2 = min(int(x+w), self.width - 1)
+ y1 = max(int(y), 0)
+ y2 = min(int(y+h), self.height - 1)
+ return x1, y1, x2, y2
+
+ def _tlwh_to_xyxy_noclip(self, bbox_tlwh):
+ """
+ TODO:
+ Convert bbox from xtl_ytl_w_h to xc_yc_w_h
+ Thanks [email protected] for reporting this bug!
+ """
+ x, y, w, h = bbox_tlwh
+ x1 = x
+ x2 = x + w
+ y1 = y
+ y2 = y + h
+ return x1, y1, x2, y2
+
+ def increment_ages(self):
+ self.tracker.increment_ages()
+
+ def _xyxy_to_tlwh(self, bbox_xyxy):
+ x1, y1, x2, y2 = bbox_xyxy
+
+ t = x1
+ l = y1
+ w = int(x2 - x1)
+ h = int(y2 - y1)
+ return t, l, w, h
+
+ def _get_features(self, bbox_xywh, ori_img):
+ im_crops = []
+ for box in bbox_xywh:
+ x1, y1, x2, y2 = self._tlwh_to_xyxy(box)
+ im = ori_img[y1:y2, x1:x2]
+ im_crops.append(im)
+ if im_crops:
+ features = self.extractor(im_crops)
+ else:
+ features = np.array([])
+ return features
diff --git a/experimental/yolox/deepsort_tracker/detection.py b/experimental/yolox/deepsort_tracker/detection.py
new file mode 100644
index 0000000..9bd8497
--- /dev/null
+++ b/experimental/yolox/deepsort_tracker/detection.py
@@ -0,0 +1,46 @@
+# vim: expandtab:ts=4:sw=4
+import numpy as np
+
+
+class Detection(object):
+ """
+ This class represents a bounding box detection in a single image.
+ Parameters
+ ----------
+ tlwh : array_like
+ Bounding box in format `(x, y, w, h)`.
+ confidence : float
+ Detector confidence score.
+ feature : array_like
+ A feature vector that describes the object contained in this image.
+ Attributes
+ ----------
+ tlwh : ndarray
+ Bounding box in format `(top left x, top left y, width, height)`.
+ confidence : ndarray
+ Detector confidence score.
+ feature : ndarray | NoneType
+ A feature vector that describes the object contained in this image.
+ """
+
+ def __init__(self, tlwh, confidence, feature):
+ self.tlwh = np.asarray(tlwh, dtype=np.float)
+ self.confidence = float(confidence)
+ self.feature = np.asarray(feature, dtype=np.float32)
+
+ def to_tlbr(self):
+ """Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,
+ `(top left, bottom right)`.
+ """
+ ret = self.tlwh.copy()
+ ret[2:] += ret[:2]
+ return ret
+
+ def to_xyah(self):
+ """Convert bounding box to format `(center x, center y, aspect ratio,
+ height)`, where the aspect ratio is `width / height`.
+ """
+ ret = self.tlwh.copy()
+ ret[:2] += ret[2:] / 2
+ ret[2] /= ret[3]
+ return ret
\ No newline at end of file
diff --git a/experimental/yolox/deepsort_tracker/iou_matching.py b/experimental/yolox/deepsort_tracker/iou_matching.py
new file mode 100644
index 0000000..7a27a4d
--- /dev/null
+++ b/experimental/yolox/deepsort_tracker/iou_matching.py
@@ -0,0 +1,76 @@
+# vim: expandtab:ts=4:sw=4
+from __future__ import absolute_import
+import numpy as np
+from yolox.deepsort_tracker import linear_assignment
+
+
+def iou(bbox, candidates):
+ """Computer intersection over union.
+ Parameters
+ ----------
+ bbox : ndarray
+ A bounding box in format `(top left x, top left y, width, height)`.
+ candidates : ndarray
+ A matrix of candidate bounding boxes (one per row) in the same format
+ as `bbox`.
+ Returns
+ -------
+ ndarray
+ The intersection over union in [0, 1] between the `bbox` and each
+ candidate. A higher score means a larger fraction of the `bbox` is
+ occluded by the candidate.
+ """
+ bbox_tl, bbox_br = bbox[:2], bbox[:2] + bbox[2:]
+ candidates_tl = candidates[:, :2]
+ candidates_br = candidates[:, :2] + candidates[:, 2:]
+
+ tl = np.c_[np.maximum(bbox_tl[0], candidates_tl[:, 0])[:, np.newaxis],
+ np.maximum(bbox_tl[1], candidates_tl[:, 1])[:, np.newaxis]]
+ br = np.c_[np.minimum(bbox_br[0], candidates_br[:, 0])[:, np.newaxis],
+ np.minimum(bbox_br[1], candidates_br[:, 1])[:, np.newaxis]]
+ wh = np.maximum(0., br - tl)
+
+ area_intersection = wh.prod(axis=1)
+ area_bbox = bbox[2:].prod()
+ area_candidates = candidates[:, 2:].prod(axis=1)
+ return area_intersection / (area_bbox + area_candidates - area_intersection)
+
+
+def iou_cost(tracks, detections, track_indices=None,
+ detection_indices=None):
+ """An intersection over union distance metric.
+ Parameters
+ ----------
+ tracks : List[deep_sort.track.Track]
+ A list of tracks.
+ detections : List[deep_sort.detection.Detection]
+ A list of detections.
+ track_indices : Optional[List[int]]
+ A list of indices to tracks that should be matched. Defaults to
+ all `tracks`.
+ detection_indices : Optional[List[int]]
+ A list of indices to detections that should be matched. Defaults
+ to all `detections`.
+ Returns
+ -------
+ ndarray
+ Returns a cost matrix of shape
+ len(track_indices), len(detection_indices) where entry (i, j) is
+ `1 - iou(tracks[track_indices[i]], detections[detection_indices[j]])`.
+ """
+ if track_indices is None:
+ track_indices = np.arange(len(tracks))
+ if detection_indices is None:
+ detection_indices = np.arange(len(detections))
+
+ cost_matrix = np.zeros((len(track_indices), len(detection_indices)))
+ for row, track_idx in enumerate(track_indices):
+ if tracks[track_idx].time_since_update > 1:
+ cost_matrix[row, :] = linear_assignment.INFTY_COST
+ continue
+
+ bbox = tracks[track_idx].to_tlwh()
+ candidates = np.asarray(
+ [detections[i].tlwh for i in detection_indices])
+ cost_matrix[row, :] = 1. - iou(bbox, candidates)
+ return cost_matrix
\ No newline at end of file
diff --git a/experimental/yolox/deepsort_tracker/kalman_filter.py b/experimental/yolox/deepsort_tracker/kalman_filter.py
new file mode 100644
index 0000000..1d4bcc3
--- /dev/null
+++ b/experimental/yolox/deepsort_tracker/kalman_filter.py
@@ -0,0 +1,208 @@
+# vim: expandtab:ts=4:sw=4
+import numpy as np
+import scipy.linalg
+
+
+"""
+Table for the 0.95 quantile of the chi-square distribution with N degrees of
+freedom (contains values for N=1, ..., 9). Taken from MATLAB/Octave's chi2inv
+function and used as Mahalanobis gating threshold.
+"""
+chi2inv95 = {
+ 1: 3.8415,
+ 2: 5.9915,
+ 3: 7.8147,
+ 4: 9.4877,
+ 5: 11.070,
+ 6: 12.592,
+ 7: 14.067,
+ 8: 15.507,
+ 9: 16.919}
+
+
+class KalmanFilter(object):
+ """
+ A simple Kalman filter for tracking bounding boxes in image space.
+ The 8-dimensional state space
+ x, y, a, h, vx, vy, va, vh
+ contains the bounding box center position (x, y), aspect ratio a, height h,
+ and their respective velocities.
+ Object motion follows a constant velocity model. The bounding box location
+ (x, y, a, h) is taken as direct observation of the state space (linear
+ observation model).
+ """
+
+ def __init__(self):
+ ndim, dt = 4, 1.
+
+ # Create Kalman filter model matrices.
+ self._motion_mat = np.eye(2 * ndim, 2 * ndim)
+ for i in range(ndim):
+ self._motion_mat[i, ndim + i] = dt
+ self._update_mat = np.eye(ndim, 2 * ndim)
+
+ # Motion and observation uncertainty are chosen relative to the current
+ # state estimate. These weights control the amount of uncertainty in
+ # the model. This is a bit hacky.
+ self._std_weight_position = 1. / 20
+ self._std_weight_velocity = 1. / 160
+
+ def initiate(self, measurement):
+ """Create track from unassociated measurement.
+ Parameters
+ ----------
+ measurement : ndarray
+ Bounding box coordinates (x, y, a, h) with center position (x, y),
+ aspect ratio a, and height h.
+ Returns
+ -------
+ (ndarray, ndarray)
+ Returns the mean vector (8 dimensional) and covariance matrix (8x8
+ dimensional) of the new track. Unobserved velocities are initialized
+ to 0 mean.
+ """
+ mean_pos = measurement
+ mean_vel = np.zeros_like(mean_pos)
+ mean = np.r_[mean_pos, mean_vel]
+
+ std = [
+ 2 * self._std_weight_position * measurement[3],
+ 2 * self._std_weight_position * measurement[3],
+ 1e-2,
+ 2 * self._std_weight_position * measurement[3],
+ 10 * self._std_weight_velocity * measurement[3],
+ 10 * self._std_weight_velocity * measurement[3],
+ 1e-5,
+ 10 * self._std_weight_velocity * measurement[3]]
+ covariance = np.diag(np.square(std))
+ return mean, covariance
+
+ def predict(self, mean, covariance):
+ """Run Kalman filter prediction step.
+ Parameters
+ ----------
+ mean : ndarray
+ The 8 dimensional mean vector of the object state at the previous
+ time step.
+ covariance : ndarray
+ The 8x8 dimensional covariance matrix of the object state at the
+ previous time step.
+ Returns
+ -------
+ (ndarray, ndarray)
+ Returns the mean vector and covariance matrix of the predicted
+ state. Unobserved velocities are initialized to 0 mean.
+ """
+ std_pos = [
+ self._std_weight_position * mean[3],
+ self._std_weight_position * mean[3],
+ 1e-2,
+ self._std_weight_position * mean[3]]
+ std_vel = [
+ self._std_weight_velocity * mean[3],
+ self._std_weight_velocity * mean[3],
+ 1e-5,
+ self._std_weight_velocity * mean[3]]
+ motion_cov = np.diag(np.square(np.r_[std_pos, std_vel]))
+
+ mean = np.dot(self._motion_mat, mean)
+ covariance = np.linalg.multi_dot((
+ self._motion_mat, covariance, self._motion_mat.T)) + motion_cov
+
+ return mean, covariance
+
+ def project(self, mean, covariance):
+ """Project state distribution to measurement space.
+ Parameters
+ ----------
+ mean : ndarray
+ The state's mean vector (8 dimensional array).
+ covariance : ndarray
+ The state's covariance matrix (8x8 dimensional).
+ Returns
+ -------
+ (ndarray, ndarray)
+ Returns the projected mean and covariance matrix of the given state
+ estimate.
+ """
+ std = [
+ self._std_weight_position * mean[3],
+ self._std_weight_position * mean[3],
+ 1e-1,
+ self._std_weight_position * mean[3]]
+ innovation_cov = np.diag(np.square(std))
+
+ mean = np.dot(self._update_mat, mean)
+ covariance = np.linalg.multi_dot((
+ self._update_mat, covariance, self._update_mat.T))
+ return mean, covariance + innovation_cov
+
+ def update(self, mean, covariance, measurement):
+ """Run Kalman filter correction step.
+ Parameters
+ ----------
+ mean : ndarray
+ The predicted state's mean vector (8 dimensional).
+ covariance : ndarray
+ The state's covariance matrix (8x8 dimensional).
+ measurement : ndarray
+ The 4 dimensional measurement vector (x, y, a, h), where (x, y)
+ is the center position, a the aspect ratio, and h the height of the
+ bounding box.
+ Returns
+ -------
+ (ndarray, ndarray)
+ Returns the measurement-corrected state distribution.
+ """
+ projected_mean, projected_cov = self.project(mean, covariance)
+
+ chol_factor, lower = scipy.linalg.cho_factor(
+ projected_cov, lower=True, check_finite=False)
+ kalman_gain = scipy.linalg.cho_solve(
+ (chol_factor, lower), np.dot(covariance, self._update_mat.T).T,
+ check_finite=False).T
+ innovation = measurement - projected_mean
+
+ new_mean = mean + np.dot(innovation, kalman_gain.T)
+ new_covariance = covariance - np.linalg.multi_dot((
+ kalman_gain, projected_cov, kalman_gain.T))
+ return new_mean, new_covariance
+
+ def gating_distance(self, mean, covariance, measurements,
+ only_position=False):
+ """Compute gating distance between state distribution and measurements.
+ A suitable distance threshold can be obtained from `chi2inv95`. If
+ `only_position` is False, the chi-square distribution has 4 degrees of
+ freedom, otherwise 2.
+ Parameters
+ ----------
+ mean : ndarray
+ Mean vector over the state distribution (8 dimensional).
+ covariance : ndarray
+ Covariance of the state distribution (8x8 dimensional).
+ measurements : ndarray
+ An Nx4 dimensional matrix of N measurements, each in
+ format (x, y, a, h) where (x, y) is the bounding box center
+ position, a the aspect ratio, and h the height.
+ only_position : Optional[bool]
+ If True, distance computation is done with respect to the bounding
+ box center position only.
+ Returns
+ -------
+ ndarray
+ Returns an array of length N, where the i-th element contains the
+ squared Mahalanobis distance between (mean, covariance) and
+ `measurements[i]`.
+ """
+ mean, covariance = self.project(mean, covariance)
+ if only_position:
+ mean, covariance = mean[:2], covariance[:2, :2]
+ measurements = measurements[:, :2]
+
+ cholesky_factor = np.linalg.cholesky(covariance)
+ d = measurements - mean
+ z = scipy.linalg.solve_triangular(
+ cholesky_factor, d.T, lower=True, check_finite=False,
+ overwrite_b=True)
+ squared_maha = np.sum(z * z, axis=0)
+ return squared_maha
\ No newline at end of file
diff --git a/experimental/yolox/deepsort_tracker/linear_assignment.py b/experimental/yolox/deepsort_tracker/linear_assignment.py
new file mode 100644
index 0000000..5651893
--- /dev/null
+++ b/experimental/yolox/deepsort_tracker/linear_assignment.py
@@ -0,0 +1,182 @@
+from __future__ import absolute_import
+import numpy as np
+# from sklearn.utils.linear_assignment_ import linear_assignment
+from scipy.optimize import linear_sum_assignment as linear_assignment
+from yolox.deepsort_tracker import kalman_filter
+
+
+INFTY_COST = 1e+5
+
+
+def min_cost_matching(
+ distance_metric, max_distance, tracks, detections, track_indices=None,
+ detection_indices=None):
+ """Solve linear assignment problem.
+ Parameters
+ ----------
+ distance_metric : Callable[List[Track], List[Detection], List[int], List[int]) -> ndarray
+ The distance metric is given a list of tracks and detections as well as
+ a list of N track indices and M detection indices. The metric should
+ return the NxM dimensional cost matrix, where element (i, j) is the
+ association cost between the i-th track in the given track indices and
+ the j-th detection in the given detection_indices.
+ max_distance : float
+ Gating threshold. Associations with cost larger than this value are
+ disregarded.
+ tracks : List[track.Track]
+ A list of predicted tracks at the current time step.
+ detections : List[detection.Detection]
+ A list of detections at the current time step.
+ track_indices : List[int]
+ List of track indices that maps rows in `cost_matrix` to tracks in
+ `tracks` (see description above).
+ detection_indices : List[int]
+ List of detection indices that maps columns in `cost_matrix` to
+ detections in `detections` (see description above).
+ Returns
+ -------
+ (List[(int, int)], List[int], List[int])
+ Returns a tuple with the following three entries:
+ * A list of matched track and detection indices.
+ * A list of unmatched track indices.
+ * A list of unmatched detection indices.
+ """
+ if track_indices is None:
+ track_indices = np.arange(len(tracks))
+ if detection_indices is None:
+ detection_indices = np.arange(len(detections))
+
+ if len(detection_indices) == 0 or len(track_indices) == 0:
+ return [], track_indices, detection_indices # Nothing to match.
+
+ cost_matrix = distance_metric(
+ tracks, detections, track_indices, detection_indices)
+ cost_matrix[cost_matrix > max_distance] = max_distance + 1e-5
+
+ row_indices, col_indices = linear_assignment(cost_matrix)
+
+ matches, unmatched_tracks, unmatched_detections = [], [], []
+ for col, detection_idx in enumerate(detection_indices):
+ if col not in col_indices:
+ unmatched_detections.append(detection_idx)
+ for row, track_idx in enumerate(track_indices):
+ if row not in row_indices:
+ unmatched_tracks.append(track_idx)
+ for row, col in zip(row_indices, col_indices):
+ track_idx = track_indices[row]
+ detection_idx = detection_indices[col]
+ if cost_matrix[row, col] > max_distance:
+ unmatched_tracks.append(track_idx)
+ unmatched_detections.append(detection_idx)
+ else:
+ matches.append((track_idx, detection_idx))
+ return matches, unmatched_tracks, unmatched_detections
+
+
+def matching_cascade(
+ distance_metric, max_distance, cascade_depth, tracks, detections,
+ track_indices=None, detection_indices=None):
+ """Run matching cascade.
+ Parameters
+ ----------
+ distance_metric : Callable[List[Track], List[Detection], List[int], List[int]) -> ndarray
+ The distance metric is given a list of tracks and detections as well as
+ a list of N track indices and M detection indices. The metric should
+ return the NxM dimensional cost matrix, where element (i, j) is the
+ association cost between the i-th track in the given track indices and
+ the j-th detection in the given detection indices.
+ max_distance : float
+ Gating threshold. Associations with cost larger than this value are
+ disregarded.
+ cascade_depth: int
+ The cascade depth, should be se to the maximum track age.
+ tracks : List[track.Track]
+ A list of predicted tracks at the current time step.
+ detections : List[detection.Detection]
+ A list of detections at the current time step.
+ track_indices : Optional[List[int]]
+ List of track indices that maps rows in `cost_matrix` to tracks in
+ `tracks` (see description above). Defaults to all tracks.
+ detection_indices : Optional[List[int]]
+ List of detection indices that maps columns in `cost_matrix` to
+ detections in `detections` (see description above). Defaults to all
+ detections.
+ Returns
+ -------
+ (List[(int, int)], List[int], List[int])
+ Returns a tuple with the following three entries:
+ * A list of matched track and detection indices.
+ * A list of unmatched track indices.
+ * A list of unmatched detection indices.
+ """
+ if track_indices is None:
+ track_indices = list(range(len(tracks)))
+ if detection_indices is None:
+ detection_indices = list(range(len(detections)))
+
+ unmatched_detections = detection_indices
+ matches = []
+ for level in range(cascade_depth):
+ if len(unmatched_detections) == 0: # No detections left
+ break
+
+ track_indices_l = [
+ k for k in track_indices
+ if tracks[k].time_since_update == 1 + level
+ ]
+ if len(track_indices_l) == 0: # Nothing to match at this level
+ continue
+
+ matches_l, _, unmatched_detections = \
+ min_cost_matching(
+ distance_metric, max_distance, tracks, detections,
+ track_indices_l, unmatched_detections)
+ matches += matches_l
+ unmatched_tracks = list(set(track_indices) - set(k for k, _ in matches))
+ return matches, unmatched_tracks, unmatched_detections
+
+
+def gate_cost_matrix(
+ kf, cost_matrix, tracks, detections, track_indices, detection_indices,
+ gated_cost=INFTY_COST, only_position=False):
+ """Invalidate infeasible entries in cost matrix based on the state
+ distributions obtained by Kalman filtering.
+ Parameters
+ ----------
+ kf : The Kalman filter.
+ cost_matrix : ndarray
+ The NxM dimensional cost matrix, where N is the number of track indices
+ and M is the number of detection indices, such that entry (i, j) is the
+ association cost between `tracks[track_indices[i]]` and
+ `detections[detection_indices[j]]`.
+ tracks : List[track.Track]
+ A list of predicted tracks at the current time step.
+ detections : List[detection.Detection]
+ A list of detections at the current time step.
+ track_indices : List[int]
+ List of track indices that maps rows in `cost_matrix` to tracks in
+ `tracks` (see description above).
+ detection_indices : List[int]
+ List of detection indices that maps columns in `cost_matrix` to
+ detections in `detections` (see description above).
+ gated_cost : Optional[float]
+ Entries in the cost matrix corresponding to infeasible associations are
+ set this value. Defaults to a very large value.
+ only_position : Optional[bool]
+ If True, only the x, y position of the state distribution is considered
+ during gating. Defaults to False.
+ Returns
+ -------
+ ndarray
+ Returns the modified cost matrix.
+ """
+ gating_dim = 2 if only_position else 4
+ gating_threshold = kalman_filter.chi2inv95[gating_dim]
+ measurements = np.asarray(
+ [detections[i].to_xyah() for i in detection_indices])
+ for row, track_idx in enumerate(track_indices):
+ track = tracks[track_idx]
+ gating_distance = kf.gating_distance(
+ track.mean, track.covariance, measurements, only_position)
+ cost_matrix[row, gating_distance > gating_threshold] = gated_cost
+ return cost_matrix
\ No newline at end of file
diff --git a/experimental/yolox/deepsort_tracker/reid_model.py b/experimental/yolox/deepsort_tracker/reid_model.py
new file mode 100644
index 0000000..6aff885
--- /dev/null
+++ b/experimental/yolox/deepsort_tracker/reid_model.py
@@ -0,0 +1,145 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+import cv2
+import logging
+import torchvision.transforms as transforms
+
+
+class BasicBlock(nn.Module):
+ def __init__(self, c_in, c_out, is_downsample=False):
+ super(BasicBlock, self).__init__()
+ self.is_downsample = is_downsample
+ if is_downsample:
+ self.conv1 = nn.Conv2d(
+ c_in, c_out, 3, stride=2, padding=1, bias=False)
+ else:
+ self.conv1 = nn.Conv2d(
+ c_in, c_out, 3, stride=1, padding=1, bias=False)
+ self.bn1 = nn.BatchNorm2d(c_out)
+ self.relu = nn.ReLU(True)
+ self.conv2 = nn.Conv2d(c_out, c_out, 3, stride=1,
+ padding=1, bias=False)
+ self.bn2 = nn.BatchNorm2d(c_out)
+ if is_downsample:
+ self.downsample = nn.Sequential(
+ nn.Conv2d(c_in, c_out, 1, stride=2, bias=False),
+ nn.BatchNorm2d(c_out)
+ )
+ elif c_in != c_out:
+ self.downsample = nn.Sequential(
+ nn.Conv2d(c_in, c_out, 1, stride=1, bias=False),
+ nn.BatchNorm2d(c_out)
+ )
+ self.is_downsample = True
+
+ def forward(self, x):
+ y = self.conv1(x)
+ y = self.bn1(y)
+ y = self.relu(y)
+ y = self.conv2(y)
+ y = self.bn2(y)
+ if self.is_downsample:
+ x = self.downsample(x)
+ return F.relu(x.add(y), True)
+
+
+def make_layers(c_in, c_out, repeat_times, is_downsample=False):
+ blocks = []
+ for i in range(repeat_times):
+ if i == 0:
+ blocks += [BasicBlock(c_in, c_out, is_downsample=is_downsample), ]
+ else:
+ blocks += [BasicBlock(c_out, c_out), ]
+ return nn.Sequential(*blocks)
+
+
+class Net(nn.Module):
+ def __init__(self, num_classes=751, reid=False):
+ super(Net, self).__init__()
+ # 3 128 64
+ self.conv = nn.Sequential(
+ nn.Conv2d(3, 64, 3, stride=1, padding=1),
+ nn.BatchNorm2d(64),
+ nn.ReLU(inplace=True),
+ # nn.Conv2d(32,32,3,stride=1,padding=1),
+ # nn.BatchNorm2d(32),
+ # nn.ReLU(inplace=True),
+ nn.MaxPool2d(3, 2, padding=1),
+ )
+ # 32 64 32
+ self.layer1 = make_layers(64, 64, 2, False)
+ # 32 64 32
+ self.layer2 = make_layers(64, 128, 2, True)
+ # 64 32 16
+ self.layer3 = make_layers(128, 256, 2, True)
+ # 128 16 8
+ self.layer4 = make_layers(256, 512, 2, True)
+ # 256 8 4
+ self.avgpool = nn.AvgPool2d((8, 4), 1)
+ # 256 1 1
+ self.reid = reid
+ self.classifier = nn.Sequential(
+ nn.Linear(512, 256),
+ nn.BatchNorm1d(256),
+ nn.ReLU(inplace=True),
+ nn.Dropout(),
+ nn.Linear(256, num_classes),
+ )
+
+ def forward(self, x):
+ x = self.conv(x)
+ x = self.layer1(x)
+ x = self.layer2(x)
+ x = self.layer3(x)
+ x = self.layer4(x)
+ x = self.avgpool(x)
+ x = x.view(x.size(0), -1)
+ # B x 128
+ if self.reid:
+ x = x.div(x.norm(p=2, dim=1, keepdim=True))
+ return x
+ # classifier
+ x = self.classifier(x)
+ return x
+
+
+class Extractor(object):
+ def __init__(self, model_path, use_cuda=True):
+ self.net = Net(reid=True)
+ self.device = "cuda" if torch.cuda.is_available() and use_cuda else "cpu"
+ state_dict = torch.load(model_path, map_location=torch.device(self.device))[
+ 'net_dict']
+ self.net.load_state_dict(state_dict)
+ logger = logging.getLogger("root.tracker")
+ logger.info("Loading weights from {}... Done!".format(model_path))
+ self.net.to(self.device)
+ self.size = (64, 128)
+ self.norm = transforms.Compose([
+ transforms.ToTensor(),
+ transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
+ ])
+
+ def _preprocess(self, im_crops):
+ """
+ TODO:
+ 1. to float with scale from 0 to 1
+ 2. resize to (64, 128) as Market1501 dataset did
+ 3. concatenate to a numpy array
+ 3. to torch Tensor
+ 4. normalize
+ """
+ def _resize(im, size):
+ return cv2.resize(im.astype(np.float32)/255., size)
+
+ im_batch = torch.cat([self.norm(_resize(im, self.size)).unsqueeze(
+ 0) for im in im_crops], dim=0).float()
+ return im_batch
+
+ def __call__(self, im_crops):
+ im_batch = self._preprocess(im_crops)
+ with torch.no_grad():
+ im_batch = im_batch.to(self.device)
+ features = self.net(im_batch)
+ return features.cpu().numpy()
\ No newline at end of file
diff --git a/experimental/yolox/deepsort_tracker/track.py b/experimental/yolox/deepsort_tracker/track.py
new file mode 100644
index 0000000..6867441
--- /dev/null
+++ b/experimental/yolox/deepsort_tracker/track.py
@@ -0,0 +1,158 @@
+# vim: expandtab:ts=4:sw=4
+
+
+class TrackState:
+ """
+ Enumeration type for the single target track state. Newly created tracks are
+ classified as `tentative` until enough evidence has been collected. Then,
+ the track state is changed to `confirmed`. Tracks that are no longer alive
+ are classified as `deleted` to mark them for removal from the set of active
+ tracks.
+ """
+
+ Tentative = 1
+ Confirmed = 2
+ Deleted = 3
+
+
+class Track:
+ """
+ A single target track with state space `(x, y, a, h)` and associated
+ velocities, where `(x, y)` is the center of the bounding box, `a` is the
+ aspect ratio and `h` is the height.
+ Parameters
+ ----------
+ mean : ndarray
+ Mean vector of the initial state distribution.
+ covariance : ndarray
+ Covariance matrix of the initial state distribution.
+ track_id : int
+ A unique track identifier.
+ n_init : int
+ Number of consecutive detections before the track is confirmed. The
+ track state is set to `Deleted` if a miss occurs within the first
+ `n_init` frames.
+ max_age : int
+ The maximum number of consecutive misses before the track state is
+ set to `Deleted`.
+ feature : Optional[ndarray]
+ Feature vector of the detection this track originates from. If not None,
+ this feature is added to the `features` cache.
+ Attributes
+ ----------
+ mean : ndarray
+ Mean vector of the initial state distribution.
+ covariance : ndarray
+ Covariance matrix of the initial state distribution.
+ track_id : int
+ A unique track identifier.
+ hits : int
+ Total number of measurement updates.
+ age : int
+ Total number of frames since first occurance.
+ time_since_update : int
+ Total number of frames since last measurement update.
+ state : TrackState
+ The current track state.
+ features : List[ndarray]
+ A cache of features. On each measurement update, the associated feature
+ vector is added to this list.
+ """
+
+ def __init__(self, mean, covariance, track_id, class_id, n_init, max_age,
+ feature=None):
+ self.mean = mean
+ self.covariance = covariance
+ self.track_id = track_id
+ self.class_id = class_id
+ self.hits = 1
+ self.age = 1
+ self.time_since_update = 0
+
+ self.state = TrackState.Tentative
+ self.features = []
+ if feature is not None:
+ self.features.append(feature)
+
+ self._n_init = n_init
+ self._max_age = max_age
+
+ def to_tlwh(self):
+ """Get current position in bounding box format `(top left x, top left y,
+ width, height)`.
+ Returns
+ -------
+ ndarray
+ The bounding box.
+ """
+ ret = self.mean[:4].copy()
+ ret[2] *= ret[3]
+ ret[:2] -= ret[2:] / 2
+ return ret
+
+ def to_tlbr(self):
+ """Get current position in bounding box format `(min x, miny, max x,
+ max y)`.
+ Returns
+ -------
+ ndarray
+ The bounding box.
+ """
+ ret = self.to_tlwh()
+ ret[2:] = ret[:2] + ret[2:]
+ return ret
+
+ def increment_age(self):
+ self.age += 1
+ self.time_since_update += 1
+
+ def predict(self, kf):
+ """Propagate the state distribution to the current time step using a
+ Kalman filter prediction step.
+ Parameters
+ ----------
+ kf : kalman_filter.KalmanFilter
+ The Kalman filter.
+ """
+ self.mean, self.covariance = kf.predict(self.mean, self.covariance)
+ self.increment_age()
+
+ def update(self, kf, detection):
+ """Perform Kalman filter measurement update step and update the feature
+ cache.
+ Parameters
+ ----------
+ kf : kalman_filter.KalmanFilter
+ The Kalman filter.
+ detection : Detection
+ The associated detection.
+ """
+ self.mean, self.covariance = kf.update(
+ self.mean, self.covariance, detection.to_xyah())
+ self.features.append(detection.feature)
+
+ self.hits += 1
+ self.time_since_update = 0
+ if self.state == TrackState.Tentative and self.hits >= self._n_init:
+ self.state = TrackState.Confirmed
+
+ def mark_missed(self):
+ """Mark this track as missed (no association at the current time step).
+ """
+ if self.state == TrackState.Tentative:
+ self.state = TrackState.Deleted
+ elif self.time_since_update > self._max_age:
+ self.state = TrackState.Deleted
+
+ def is_tentative(self):
+ """Returns True if this track is tentative (unconfirmed).
+ """
+ return self.state == TrackState.Tentative
+
+ def is_confirmed(self):
+ """Returns True if this track is confirmed."""
+ return self.state == TrackState.Confirmed
+
+ def is_deleted(self):
+ """Returns True if this track is dead and should be deleted."""
+ return self.state == TrackState.Deleted
\ No newline at end of file
diff --git a/experimental/yolox/evaluators/__init__.py b/experimental/yolox/evaluators/__init__.py
new file mode 100644
index 0000000..5d704e0
--- /dev/null
+++ b/experimental/yolox/evaluators/__init__.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) Megvii, Inc. and its affiliates.
+
+from .coco_evaluator import COCOEvaluator
+from .mot_evaluator import MOTEvaluator
diff --git a/experimental/yolox/evaluators/coco_evaluator.py b/experimental/yolox/evaluators/coco_evaluator.py
new file mode 100644
index 0000000..24dce23
--- /dev/null
+++ b/experimental/yolox/evaluators/coco_evaluator.py
@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) Megvii, Inc. and its affiliates.
+
+from loguru import logger
+from tqdm import tqdm
+
+import torch
+
+from yolox.utils import (
+ gather,
+ is_main_process,
+ postprocess,
+ synchronize,
+ time_synchronized,
+ xyxy2xywh
+)
+
+import contextlib
+import io
+import itertools
+import json
+import tempfile
+import time
+
+
+class COCOEvaluator:
+ """
+ COCO AP Evaluation class. All the data in the val2017 dataset are processed
+ and evaluated by COCO API.
+ """
+
+ def __init__(
+ self, dataloader, img_size, confthre, nmsthre, num_classes, testdev=False
+ ):
+ """
+ Args:
+ dataloader (Dataloader): evaluate dataloader.
+ img_size (int): image size after preprocess. images are resized
+ to squares whose shape is (img_size, img_size).
+ confthre (float): confidence threshold ranging from 0 to 1, which
+ is defined in the config file.
+ nmsthre (float): IoU threshold of non-max supression ranging from 0 to 1.
+ """
+ self.dataloader = dataloader
+ self.img_size = img_size
+ self.confthre = confthre
+ self.nmsthre = nmsthre
+ self.num_classes = num_classes
+ self.testdev = testdev
+
+ def evaluate(
+ self,
+ model,
+ distributed=False,
+ half=False,
+ trt_file=None,
+ decoder=None,
+ test_size=None,
+ ):
+ """
+ COCO average precision (AP) Evaluation. Iterate inference on the test dataset
+ and the results are evaluated by COCO API.
+
+ NOTE: This function will change training mode to False, please save states if needed.
+
+ Args:
+ model : model to evaluate.
+
+ Returns:
+ ap50_95 (float) : COCO AP of IoU=50:95
+ ap50 (float) : COCO AP of IoU=50
+ summary (sr): summary info of evaluation.
+ """
+ # TODO half to amp_test
+ tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor
+ model = model.eval()
+ if half:
+ model = model.half()
+ ids = []
+ data_list = []
+ progress_bar = tqdm if is_main_process() else iter
+
+ inference_time = 0
+ nms_time = 0
+ n_samples = len(self.dataloader) - 1
+
+ if trt_file is not None:
+ from torch2trt import TRTModule
+
+ model_trt = TRTModule()
+ model_trt.load_state_dict(torch.load(trt_file))
+
+ x = torch.ones(1, 3, test_size[0], test_size[1]).cuda()
+ model(x)
+ model = model_trt
+
+ for cur_iter, (imgs, _, info_imgs, ids) in enumerate(
+ progress_bar(self.dataloader)
+ ):
+ with torch.no_grad():
+ imgs = imgs.type(tensor_type)
+
+ # skip the the last iters since batchsize might be not enough for batch inference
+ is_time_record = cur_iter < len(self.dataloader) - 1
+ if is_time_record:
+ start = time.time()
+
+ outputs = model(imgs)
+ if decoder is not None:
+ outputs = decoder(outputs, dtype=outputs.type())
+
+ if is_time_record:
+ infer_end = time_synchronized()
+ inference_time += infer_end - start
+
+ outputs = postprocess(
+ outputs, self.num_classes, self.confthre, self.nmsthre
+ )
+ if is_time_record:
+ nms_end = time_synchronized()
+ nms_time += nms_end - infer_end
+
+ data_list.extend(self.convert_to_coco_format(outputs, info_imgs, ids))
+
+ statistics = torch.cuda.FloatTensor([inference_time, nms_time, n_samples])
+ if distributed:
+ data_list = gather(data_list, dst=0)
+ data_list = list(itertools.chain(*data_list))
+ torch.distributed.reduce(statistics, dst=0)
+
+ eval_results = self.evaluate_prediction(data_list, statistics)
+ synchronize()
+ return eval_results
+
+ def convert_to_coco_format(self, outputs, info_imgs, ids):
+ data_list = []
+ for (output, img_h, img_w, img_id) in zip(
+ outputs, info_imgs[0], info_imgs[1], ids
+ ):
+ if output is None:
+ continue
+ output = output.cpu()
+
+ bboxes = output[:, 0:4]
+
+ # preprocessing: resize
+ scale = min(
+ self.img_size[0] / float(img_h), self.img_size[1] / float(img_w)
+ )
+ bboxes /= scale
+ bboxes = xyxy2xywh(bboxes)
+
+ cls = output[:, 6]
+ scores = output[:, 4] * output[:, 5]
+ for ind in range(bboxes.shape[0]):
+ label = self.dataloader.dataset.class_ids[int(cls[ind])]
+ pred_data = {
+ "image_id": int(img_id),
+ "category_id": label,
+ "bbox": bboxes[ind].numpy().tolist(),
+ "score": scores[ind].numpy().item(),
+ "segmentation": [],
+ } # COCO json format
+ data_list.append(pred_data)
+ return data_list
+
+ def evaluate_prediction(self, data_dict, statistics):
+ if not is_main_process():
+ return 0, 0, None
+
+ logger.info("Evaluate in main process...")
+
+ annType = ["segm", "bbox", "keypoints"]
+
+ inference_time = statistics[0].item()
+ nms_time = statistics[1].item()
+ n_samples = statistics[2].item()
+
+ a_infer_time = 1000 * inference_time / (n_samples * self.dataloader.batch_size)
+ a_nms_time = 1000 * nms_time / (n_samples * self.dataloader.batch_size)
+
+ time_info = ", ".join(
+ [
+ "Average {} time: {:.2f} ms".format(k, v)
+ for k, v in zip(
+ ["forward", "NMS", "inference"],
+ [a_infer_time, a_nms_time, (a_infer_time + a_nms_time)],
+ )
+ ]
+ )
+
+ info = time_info + "\n"
+
+ # Evaluate the Dt (detection) json comparing with the ground truth
+ if len(data_dict) > 0:
+ cocoGt = self.dataloader.dataset.coco
+ # TODO: since pycocotools can't process dict in py36, write data to json file.
+ if self.testdev:
+ json.dump(data_dict, open("./yolox_testdev_2017.json", "w"))
+ cocoDt = cocoGt.loadRes("./yolox_testdev_2017.json")
+ else:
+ _, tmp = tempfile.mkstemp()
+ json.dump(data_dict, open(tmp, "w"))
+ cocoDt = cocoGt.loadRes(tmp)
+ '''
+ try:
+ from yolox.layers import COCOeval_opt as COCOeval
+ except ImportError:
+ from pycocotools import cocoeval as COCOeval
+ logger.warning("Use standard COCOeval.")
+ '''
+ #from pycocotools.cocoeval import COCOeval
+ from yolox.layers import COCOeval_opt as COCOeval
+ cocoEval = COCOeval(cocoGt, cocoDt, annType[1])
+ cocoEval.evaluate()
+ cocoEval.accumulate()
+ redirect_string = io.StringIO()
+ with contextlib.redirect_stdout(redirect_string):
+ cocoEval.summarize()
+ info += redirect_string.getvalue()
+ return cocoEval.stats[0], cocoEval.stats[1], info
+ else:
+ return 0, 0, info
diff --git a/experimental/yolox/evaluators/evaluation.py b/experimental/yolox/evaluators/evaluation.py
new file mode 100644
index 0000000..fd72f82
--- /dev/null
+++ b/experimental/yolox/evaluators/evaluation.py
@@ -0,0 +1,200 @@
+import os
+import numpy as np
+import copy
+import motmetrics as mm
+mm.lap.default_solver = 'lap'
+
+
+class Evaluator(object):
+
+ def __init__(self, data_root, seq_name, data_type):
+ self.data_root = data_root
+ self.seq_name = seq_name
+ self.data_type = data_type
+
+ self.load_annotations()
+ self.reset_accumulator()
+
+ def load_annotations(self):
+ assert self.data_type == 'mot'
+
+ gt_filename = os.path.join(self.data_root, self.seq_name, 'gt', 'gt.txt')
+ self.gt_frame_dict = read_results(gt_filename, self.data_type, is_gt=True)
+ self.gt_ignore_frame_dict = read_results(gt_filename, self.data_type, is_ignore=True)
+
+ def reset_accumulator(self):
+ self.acc = mm.MOTAccumulator(auto_id=True)
+
+ def eval_frame(self, frame_id, trk_tlwhs, trk_ids, rtn_events=False):
+ # results
+ trk_tlwhs = np.copy(trk_tlwhs)
+ trk_ids = np.copy(trk_ids)
+
+ # gts
+ gt_objs = self.gt_frame_dict.get(frame_id, [])
+ gt_tlwhs, gt_ids = unzip_objs(gt_objs)[:2]
+
+ # ignore boxes
+ ignore_objs = self.gt_ignore_frame_dict.get(frame_id, [])
+ ignore_tlwhs = unzip_objs(ignore_objs)[0]
+
+ # remove ignored results
+ keep = np.ones(len(trk_tlwhs), dtype=bool)
+ iou_distance = mm.distances.iou_matrix(ignore_tlwhs, trk_tlwhs, max_iou=0.5)
+ if len(iou_distance) > 0:
+ match_is, match_js = mm.lap.linear_sum_assignment(iou_distance)
+ match_is, match_js = map(lambda a: np.asarray(a, dtype=int), [match_is, match_js])
+ match_ious = iou_distance[match_is, match_js]
+
+ match_js = np.asarray(match_js, dtype=int)
+ match_js = match_js[np.logical_not(np.isnan(match_ious))]
+ keep[match_js] = False
+ trk_tlwhs = trk_tlwhs[keep]
+ trk_ids = trk_ids[keep]
+ #match_is, match_js = mm.lap.linear_sum_assignment(iou_distance)
+ #match_is, match_js = map(lambda a: np.asarray(a, dtype=int), [match_is, match_js])
+ #match_ious = iou_distance[match_is, match_js]
+
+ #match_js = np.asarray(match_js, dtype=int)
+ #match_js = match_js[np.logical_not(np.isnan(match_ious))]
+ #keep[match_js] = False
+ #trk_tlwhs = trk_tlwhs[keep]
+ #trk_ids = trk_ids[keep]
+
+ # get distance matrix
+ iou_distance = mm.distances.iou_matrix(gt_tlwhs, trk_tlwhs, max_iou=0.5)
+
+ # acc
+ self.acc.update(gt_ids, trk_ids, iou_distance)
+
+ if rtn_events and iou_distance.size > 0 and hasattr(self.acc, 'last_mot_events'):
+ events = self.acc.last_mot_events # only supported by https://github.com/longcw/py-motmetrics
+ else:
+ events = None
+ return events
+
+ def eval_file(self, filename):
+ self.reset_accumulator()
+
+ result_frame_dict = read_results(filename, self.data_type, is_gt=False)
+ #frames = sorted(list(set(self.gt_frame_dict.keys()) | set(result_frame_dict.keys())))
+ frames = sorted(list(set(result_frame_dict.keys())))
+ for frame_id in frames:
+ trk_objs = result_frame_dict.get(frame_id, [])
+ trk_tlwhs, trk_ids = unzip_objs(trk_objs)[:2]
+ self.eval_frame(frame_id, trk_tlwhs, trk_ids, rtn_events=False)
+
+ return self.acc
+
+ @staticmethod
+ def get_summary(accs, names, metrics=('mota', 'num_switches', 'idp', 'idr', 'idf1', 'precision', 'recall')):
+ names = copy.deepcopy(names)
+ if metrics is None:
+ metrics = mm.metrics.motchallenge_metrics
+ metrics = copy.deepcopy(metrics)
+
+ mh = mm.metrics.create()
+ summary = mh.compute_many(
+ accs,
+ metrics=metrics,
+ names=names,
+ generate_overall=True
+ )
+
+ return summary
+
+ @staticmethod
+ def save_summary(summary, filename):
+ import pandas as pd
+ writer = pd.ExcelWriter(filename)
+ summary.to_excel(writer)
+ writer.save()
+
+
+
+
+
+def read_results(filename, data_type: str, is_gt=False, is_ignore=False):
+ if data_type in ('mot', 'lab'):
+ read_fun = read_mot_results
+ else:
+ raise ValueError('Unknown data type: {}'.format(data_type))
+
+ return read_fun(filename, is_gt, is_ignore)
+
+
+"""
+labels={'ped', ... % 1
+'person_on_vhcl', ... % 2
+'car', ... % 3
+'bicycle', ... % 4
+'mbike', ... % 5
+'non_mot_vhcl', ... % 6
+'static_person', ... % 7
+'distractor', ... % 8
+'occluder', ... % 9
+'occluder_on_grnd', ... %10
+'occluder_full', ... % 11
+'reflection', ... % 12
+'crowd' ... % 13
+};
+"""
+
+
+def read_mot_results(filename, is_gt, is_ignore):
+ valid_labels = {1}
+ ignore_labels = {2, 7, 8, 12}
+ results_dict = dict()
+ if os.path.isfile(filename):
+ with open(filename, 'r') as f:
+ for line in f.readlines():
+ linelist = line.split(',')
+ if len(linelist) < 7:
+ continue
+ fid = int(linelist[0])
+ if fid < 1:
+ continue
+ results_dict.setdefault(fid, list())
+
+ box_size = float(linelist[4]) * float(linelist[5])
+
+ if is_gt:
+ if 'MOT16-' in filename or 'MOT17-' in filename:
+ label = int(float(linelist[7]))
+ mark = int(float(linelist[6]))
+ if mark == 0 or label not in valid_labels:
+ continue
+ score = 1
+ elif is_ignore:
+ if 'MOT16-' in filename or 'MOT17-' in filename:
+ label = int(float(linelist[7]))
+ vis_ratio = float(linelist[8])
+ if label not in ignore_labels and vis_ratio >= 0:
+ continue
+ else:
+ continue
+ score = 1
+ else:
+ score = float(linelist[6])
+
+ #if box_size > 7000:
+ #if box_size <= 7000 or box_size >= 15000:
+ #if box_size < 15000:
+ #continue
+
+ tlwh = tuple(map(float, linelist[2:6]))
+ target_id = int(linelist[1])
+
+ results_dict[fid].append((tlwh, target_id, score))
+
+ return results_dict
+
+
+def unzip_objs(objs):
+ if len(objs) > 0:
+ tlwhs, ids, scores = zip(*objs)
+ else:
+ tlwhs, ids, scores = [], [], []
+ tlwhs = np.asarray(tlwhs, dtype=float).reshape(-1, 4)
+
+ return tlwhs, ids, scores
\ No newline at end of file
diff --git a/experimental/yolox/evaluators/mot_evaluator.py b/experimental/yolox/evaluators/mot_evaluator.py
new file mode 100644
index 0000000..f8c2d83
--- /dev/null
+++ b/experimental/yolox/evaluators/mot_evaluator.py
@@ -0,0 +1,676 @@
+from collections import defaultdict
+from loguru import logger
+from tqdm import tqdm
+
+import torch
+
+from yolox.utils import (
+ gather,
+ is_main_process,
+ postprocess,
+ synchronize,
+ time_synchronized,
+ xyxy2xywh
+)
+from yolox.tracker.byte_tracker import BYTETracker
+from yolox.sort_tracker.sort import Sort
+from yolox.deepsort_tracker.deepsort import DeepSort
+from yolox.motdt_tracker.motdt_tracker import OnlineTracker
+
+import contextlib
+import io
+import os
+import itertools
+import json
+import tempfile
+import time
+
+
+def write_results(filename, results):
+ save_format = '{frame},{id},{x1},{y1},{w},{h},{s},-1,-1,-1\n'
+ with open(filename, 'w') as f:
+ for frame_id, tlwhs, track_ids, scores in results:
+ for tlwh, track_id, score in zip(tlwhs, track_ids, scores):
+ if track_id < 0:
+ continue
+ x1, y1, w, h = tlwh
+ line = save_format.format(frame=frame_id, id=track_id, x1=round(x1, 1), y1=round(y1, 1), w=round(w, 1), h=round(h, 1), s=round(score, 2))
+ f.write(line)
+ logger.info('save results to {}'.format(filename))
+
+
+def write_results_no_score(filename, results):
+ save_format = '{frame},{id},{x1},{y1},{w},{h},-1,-1,-1,-1\n'
+ with open(filename, 'w') as f:
+ for frame_id, tlwhs, track_ids in results:
+ for tlwh, track_id in zip(tlwhs, track_ids):
+ if track_id < 0:
+ continue
+ x1, y1, w, h = tlwh
+ line = save_format.format(frame=frame_id, id=track_id, x1=round(x1, 1), y1=round(y1, 1), w=round(w, 1), h=round(h, 1))
+ f.write(line)
+ logger.info('save results to {}'.format(filename))
+
+
+class MOTEvaluator:
+ """
+ COCO AP Evaluation class. All the data in the val2017 dataset are processed
+ and evaluated by COCO API.
+ """
+
+ def __init__(
+ self, args, dataloader, img_size, confthre, nmsthre, num_classes):
+ """
+ Args:
+ dataloader (Dataloader): evaluate dataloader.
+ img_size (int): image size after preprocess. images are resized
+ to squares whose shape is (img_size, img_size).
+ confthre (float): confidence threshold ranging from 0 to 1, which
+ is defined in the config file.
+ nmsthre (float): IoU threshold of non-max supression ranging from 0 to 1.
+ """
+ self.dataloader = dataloader
+ self.img_size = img_size
+ self.confthre = confthre
+ self.nmsthre = nmsthre
+ self.num_classes = num_classes
+ self.args = args
+
+ def evaluate(
+ self,
+ model,
+ distributed=False,
+ half=False,
+ trt_file=None,
+ decoder=None,
+ test_size=None,
+ result_folder=None
+ ):
+ """
+ COCO average precision (AP) Evaluation. Iterate inference on the test dataset
+ and the results are evaluated by COCO API.
+
+ NOTE: This function will change training mode to False, please save states if needed.
+
+ Args:
+ model : model to evaluate.
+
+ Returns:
+ ap50_95 (float) : COCO AP of IoU=50:95
+ ap50 (float) : COCO AP of IoU=50
+ summary (sr): summary info of evaluation.
+ """
+ # TODO half to amp_test
+ tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor
+ model = model.eval()
+ if half:
+ model = model.half()
+ ids = []
+ data_list = []
+ results = []
+ video_names = defaultdict()
+ progress_bar = tqdm if is_main_process() else iter
+
+ inference_time = 0
+ track_time = 0
+ n_samples = len(self.dataloader) - 1
+
+ if trt_file is not None:
+ from torch2trt import TRTModule
+
+ model_trt = TRTModule()
+ model_trt.load_state_dict(torch.load(trt_file))
+
+ x = torch.ones(1, 3, test_size[0], test_size[1]).cuda()
+ model(x)
+ model = model_trt
+
+ tracker = BYTETracker(self.args)
+ ori_thresh = self.args.track_thresh
+ for cur_iter, (imgs, _, info_imgs, ids) in enumerate(
+ progress_bar(self.dataloader)
+ ):
+ with torch.no_grad():
+ # init tracker
+ frame_id = info_imgs[2].item()
+ video_id = info_imgs[3].item()
+ img_file_name = info_imgs[4]
+ video_name = img_file_name[0].split('/')[0]
+ if video_name == 'MOT17-05-FRCNN' or video_name == 'MOT17-06-FRCNN':
+ self.args.track_buffer = 14
+ elif video_name == 'MOT17-13-FRCNN' or video_name == 'MOT17-14-FRCNN':
+ self.args.track_buffer = 25
+ else:
+ self.args.track_buffer = 30
+
+ if video_name == 'MOT17-01-FRCNN':
+ self.args.track_thresh = 0.65
+ elif video_name == 'MOT17-06-FRCNN':
+ self.args.track_thresh = 0.65
+ elif video_name == 'MOT17-12-FRCNN':
+ self.args.track_thresh = 0.7
+ elif video_name == 'MOT17-14-FRCNN':
+ self.args.track_thresh = 0.67
+ elif video_name in ['MOT20-06', 'MOT20-08']:
+ self.args.track_thresh = 0.3
+ else:
+ self.args.track_thresh = ori_thresh
+
+ if video_name not in video_names:
+ video_names[video_id] = video_name
+ if frame_id == 1:
+ tracker = BYTETracker(self.args)
+ if len(results) != 0:
+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id - 1]))
+ write_results(result_filename, results)
+ results = []
+
+ imgs = imgs.type(tensor_type)
+
+ # skip the the last iters since batchsize might be not enough for batch inference
+ is_time_record = cur_iter < len(self.dataloader) - 1
+ if is_time_record:
+ start = time.time()
+
+ outputs = model(imgs)
+ if decoder is not None:
+ outputs = decoder(outputs, dtype=outputs.type())
+
+ outputs = postprocess(outputs, self.num_classes, self.confthre, self.nmsthre)
+
+ if is_time_record:
+ infer_end = time_synchronized()
+ inference_time += infer_end - start
+
+ output_results = self.convert_to_coco_format(outputs, info_imgs, ids)
+ data_list.extend(output_results)
+
+ # run tracking
+ if outputs[0] is not None:
+ online_targets = tracker.update(outputs[0], info_imgs, self.img_size)
+ online_tlwhs = []
+ online_ids = []
+ online_scores = []
+ for t in online_targets:
+ tlwh = t.tlwh
+ tid = t.track_id
+ vertical = tlwh[2] / tlwh[3] > 1.6
+ if tlwh[2] * tlwh[3] > self.args.min_box_area and not vertical:
+ online_tlwhs.append(tlwh)
+ online_ids.append(tid)
+ online_scores.append(t.score)
+ # save results
+ results.append((frame_id, online_tlwhs, online_ids, online_scores))
+
+ if is_time_record:
+ track_end = time_synchronized()
+ track_time += track_end - infer_end
+
+ if cur_iter == len(self.dataloader) - 1:
+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id]))
+ write_results(result_filename, results)
+
+ statistics = torch.cuda.FloatTensor([inference_time, track_time, n_samples])
+ if distributed:
+ data_list = gather(data_list, dst=0)
+ data_list = list(itertools.chain(*data_list))
+ torch.distributed.reduce(statistics, dst=0)
+
+ eval_results = self.evaluate_prediction(data_list, statistics)
+ synchronize()
+ return eval_results
+
+ def evaluate_sort(
+ self,
+ model,
+ distributed=False,
+ half=False,
+ trt_file=None,
+ decoder=None,
+ test_size=None,
+ result_folder=None
+ ):
+ """
+ COCO average precision (AP) Evaluation. Iterate inference on the test dataset
+ and the results are evaluated by COCO API.
+
+ NOTE: This function will change training mode to False, please save states if needed.
+
+ Args:
+ model : model to evaluate.
+
+ Returns:
+ ap50_95 (float) : COCO AP of IoU=50:95
+ ap50 (float) : COCO AP of IoU=50
+ summary (sr): summary info of evaluation.
+ """
+ # TODO half to amp_test
+ tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor
+ model = model.eval()
+ if half:
+ model = model.half()
+ ids = []
+ data_list = []
+ results = []
+ video_names = defaultdict()
+ progress_bar = tqdm if is_main_process() else iter
+
+ inference_time = 0
+ track_time = 0
+ n_samples = len(self.dataloader) - 1
+
+ if trt_file is not None:
+ from torch2trt import TRTModule
+
+ model_trt = TRTModule()
+ model_trt.load_state_dict(torch.load(trt_file))
+
+ x = torch.ones(1, 3, test_size[0], test_size[1]).cuda()
+ model(x)
+ model = model_trt
+
+ tracker = Sort(self.args.track_thresh)
+
+ for cur_iter, (imgs, _, info_imgs, ids) in enumerate(
+ progress_bar(self.dataloader)
+ ):
+ with torch.no_grad():
+ # init tracker
+ frame_id = info_imgs[2].item()
+ video_id = info_imgs[3].item()
+ img_file_name = info_imgs[4]
+ video_name = img_file_name[0].split('/')[0]
+
+ if video_name not in video_names:
+ video_names[video_id] = video_name
+ if frame_id == 1:
+ tracker = Sort(self.args.track_thresh)
+ if len(results) != 0:
+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id - 1]))
+ write_results_no_score(result_filename, results)
+ results = []
+
+ imgs = imgs.type(tensor_type)
+
+ # skip the the last iters since batchsize might be not enough for batch inference
+ is_time_record = cur_iter < len(self.dataloader) - 1
+ if is_time_record:
+ start = time.time()
+
+ outputs = model(imgs)
+ if decoder is not None:
+ outputs = decoder(outputs, dtype=outputs.type())
+
+ outputs = postprocess(outputs, self.num_classes, self.confthre, self.nmsthre)
+
+ if is_time_record:
+ infer_end = time_synchronized()
+ inference_time += infer_end - start
+
+ output_results = self.convert_to_coco_format(outputs, info_imgs, ids)
+ data_list.extend(output_results)
+
+ # run tracking
+ online_targets = tracker.update(outputs[0], info_imgs, self.img_size)
+ online_tlwhs = []
+ online_ids = []
+ for t in online_targets:
+ tlwh = [t[0], t[1], t[2] - t[0], t[3] - t[1]]
+ tid = t[4]
+ vertical = tlwh[2] / tlwh[3] > 1.6
+ if tlwh[2] * tlwh[3] > self.args.min_box_area and not vertical:
+ online_tlwhs.append(tlwh)
+ online_ids.append(tid)
+ # save results
+ results.append((frame_id, online_tlwhs, online_ids))
+
+ if is_time_record:
+ track_end = time_synchronized()
+ track_time += track_end - infer_end
+
+ if cur_iter == len(self.dataloader) - 1:
+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id]))
+ write_results_no_score(result_filename, results)
+
+ statistics = torch.cuda.FloatTensor([inference_time, track_time, n_samples])
+ if distributed:
+ data_list = gather(data_list, dst=0)
+ data_list = list(itertools.chain(*data_list))
+ torch.distributed.reduce(statistics, dst=0)
+
+ eval_results = self.evaluate_prediction(data_list, statistics)
+ synchronize()
+ return eval_results
+
+ def evaluate_deepsort(
+ self,
+ model,
+ distributed=False,
+ half=False,
+ trt_file=None,
+ decoder=None,
+ test_size=None,
+ result_folder=None,
+ model_folder=None
+ ):
+ """
+ COCO average precision (AP) Evaluation. Iterate inference on the test dataset
+ and the results are evaluated by COCO API.
+
+ NOTE: This function will change training mode to False, please save states if needed.
+
+ Args:
+ model : model to evaluate.
+
+ Returns:
+ ap50_95 (float) : COCO AP of IoU=50:95
+ ap50 (float) : COCO AP of IoU=50
+ summary (sr): summary info of evaluation.
+ """
+ # TODO half to amp_test
+ tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor
+ model = model.eval()
+ if half:
+ model = model.half()
+ ids = []
+ data_list = []
+ results = []
+ video_names = defaultdict()
+ progress_bar = tqdm if is_main_process() else iter
+
+ inference_time = 0
+ track_time = 0
+ n_samples = len(self.dataloader) - 1
+
+ if trt_file is not None:
+ from torch2trt import TRTModule
+
+ model_trt = TRTModule()
+ model_trt.load_state_dict(torch.load(trt_file))
+
+ x = torch.ones(1, 3, test_size[0], test_size[1]).cuda()
+ model(x)
+ model = model_trt
+
+ tracker = DeepSort(model_folder, min_confidence=self.args.track_thresh)
+
+ for cur_iter, (imgs, _, info_imgs, ids) in enumerate(
+ progress_bar(self.dataloader)
+ ):
+ with torch.no_grad():
+ # init tracker
+ frame_id = info_imgs[2].item()
+ video_id = info_imgs[3].item()
+ img_file_name = info_imgs[4]
+ video_name = img_file_name[0].split('/')[0]
+
+ if video_name not in video_names:
+ video_names[video_id] = video_name
+ if frame_id == 1:
+ tracker = DeepSort(model_folder, min_confidence=self.args.track_thresh)
+ if len(results) != 0:
+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id - 1]))
+ write_results_no_score(result_filename, results)
+ results = []
+
+ imgs = imgs.type(tensor_type)
+
+ # skip the the last iters since batchsize might be not enough for batch inference
+ is_time_record = cur_iter < len(self.dataloader) - 1
+ if is_time_record:
+ start = time.time()
+
+ outputs = model(imgs)
+ if decoder is not None:
+ outputs = decoder(outputs, dtype=outputs.type())
+
+ outputs = postprocess(outputs, self.num_classes, self.confthre, self.nmsthre)
+
+ if is_time_record:
+ infer_end = time_synchronized()
+ inference_time += infer_end - start
+
+ output_results = self.convert_to_coco_format(outputs, info_imgs, ids)
+ data_list.extend(output_results)
+
+ # run tracking
+ online_targets = tracker.update(outputs[0], info_imgs, self.img_size, img_file_name[0])
+ online_tlwhs = []
+ online_ids = []
+ for t in online_targets:
+ tlwh = [t[0], t[1], t[2] - t[0], t[3] - t[1]]
+ tid = t[4]
+ vertical = tlwh[2] / tlwh[3] > 1.6
+ if tlwh[2] * tlwh[3] > self.args.min_box_area and not vertical:
+ online_tlwhs.append(tlwh)
+ online_ids.append(tid)
+ # save results
+ results.append((frame_id, online_tlwhs, online_ids))
+
+ if is_time_record:
+ track_end = time_synchronized()
+ track_time += track_end - infer_end
+
+ if cur_iter == len(self.dataloader) - 1:
+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id]))
+ write_results_no_score(result_filename, results)
+
+ statistics = torch.cuda.FloatTensor([inference_time, track_time, n_samples])
+ if distributed:
+ data_list = gather(data_list, dst=0)
+ data_list = list(itertools.chain(*data_list))
+ torch.distributed.reduce(statistics, dst=0)
+
+ eval_results = self.evaluate_prediction(data_list, statistics)
+ synchronize()
+ return eval_results
+
+ def evaluate_motdt(
+ self,
+ model,
+ distributed=False,
+ half=False,
+ trt_file=None,
+ decoder=None,
+ test_size=None,
+ result_folder=None,
+ model_folder=None
+ ):
+ """
+ COCO average precision (AP) Evaluation. Iterate inference on the test dataset
+ and the results are evaluated by COCO API.
+
+ NOTE: This function will change training mode to False, please save states if needed.
+
+ Args:
+ model : model to evaluate.
+
+ Returns:
+ ap50_95 (float) : COCO AP of IoU=50:95
+ ap50 (float) : COCO AP of IoU=50
+ summary (sr): summary info of evaluation.
+ """
+ # TODO half to amp_test
+ tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor
+ model = model.eval()
+ if half:
+ model = model.half()
+ ids = []
+ data_list = []
+ results = []
+ video_names = defaultdict()
+ progress_bar = tqdm if is_main_process() else iter
+
+ inference_time = 0
+ track_time = 0
+ n_samples = len(self.dataloader) - 1
+
+ if trt_file is not None:
+ from torch2trt import TRTModule
+
+ model_trt = TRTModule()
+ model_trt.load_state_dict(torch.load(trt_file))
+
+ x = torch.ones(1, 3, test_size[0], test_size[1]).cuda()
+ model(x)
+ model = model_trt
+
+ tracker = OnlineTracker(model_folder, min_cls_score=self.args.track_thresh)
+ for cur_iter, (imgs, _, info_imgs, ids) in enumerate(
+ progress_bar(self.dataloader)
+ ):
+ with torch.no_grad():
+ # init tracker
+ frame_id = info_imgs[2].item()
+ video_id = info_imgs[3].item()
+ img_file_name = info_imgs[4]
+ video_name = img_file_name[0].split('/')[0]
+
+ if video_name not in video_names:
+ video_names[video_id] = video_name
+ if frame_id == 1:
+ tracker = OnlineTracker(model_folder, min_cls_score=self.args.track_thresh)
+ if len(results) != 0:
+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id - 1]))
+ write_results(result_filename, results)
+ results = []
+
+ imgs = imgs.type(tensor_type)
+
+ # skip the the last iters since batchsize might be not enough for batch inference
+ is_time_record = cur_iter < len(self.dataloader) - 1
+ if is_time_record:
+ start = time.time()
+
+ outputs = model(imgs)
+ if decoder is not None:
+ outputs = decoder(outputs, dtype=outputs.type())
+
+ outputs = postprocess(outputs, self.num_classes, self.confthre, self.nmsthre)
+
+ if is_time_record:
+ infer_end = time_synchronized()
+ inference_time += infer_end - start
+
+ output_results = self.convert_to_coco_format(outputs, info_imgs, ids)
+ data_list.extend(output_results)
+
+ # run tracking
+ online_targets = tracker.update(outputs[0], info_imgs, self.img_size, img_file_name[0])
+ online_tlwhs = []
+ online_ids = []
+ online_scores = []
+ for t in online_targets:
+ tlwh = t.tlwh
+ tid = t.track_id
+ vertical = tlwh[2] / tlwh[3] > 1.6
+ if tlwh[2] * tlwh[3] > self.args.min_box_area and not vertical:
+ online_tlwhs.append(tlwh)
+ online_ids.append(tid)
+ online_scores.append(t.score)
+ # save results
+ results.append((frame_id, online_tlwhs, online_ids, online_scores))
+
+ if is_time_record:
+ track_end = time_synchronized()
+ track_time += track_end - infer_end
+
+ if cur_iter == len(self.dataloader) - 1:
+ result_filename = os.path.join(result_folder, '{}.txt'.format(video_names[video_id]))
+ write_results(result_filename, results)
+
+ statistics = torch.cuda.FloatTensor([inference_time, track_time, n_samples])
+ if distributed:
+ data_list = gather(data_list, dst=0)
+ data_list = list(itertools.chain(*data_list))
+ torch.distributed.reduce(statistics, dst=0)
+
+ eval_results = self.evaluate_prediction(data_list, statistics)
+ synchronize()
+ return eval_results
+
+ def convert_to_coco_format(self, outputs, info_imgs, ids):
+ data_list = []
+ for (output, img_h, img_w, img_id) in zip(
+ outputs, info_imgs[0], info_imgs[1], ids
+ ):
+ if output is None:
+ continue
+ output = output.cpu()
+
+ bboxes = output[:, 0:4]
+
+ # preprocessing: resize
+ scale = min(
+ self.img_size[0] / float(img_h), self.img_size[1] / float(img_w)
+ )
+ bboxes /= scale
+ bboxes = xyxy2xywh(bboxes)
+
+ cls = output[:, 6]
+ scores = output[:, 4] * output[:, 5]
+ for ind in range(bboxes.shape[0]):
+ label = self.dataloader.dataset.class_ids[int(cls[ind])]
+ pred_data = {
+ "image_id": int(img_id),
+ "category_id": label,
+ "bbox": bboxes[ind].numpy().tolist(),
+ "score": scores[ind].numpy().item(),
+ "segmentation": [],
+ } # COCO json format
+ data_list.append(pred_data)
+ return data_list
+
+ def evaluate_prediction(self, data_dict, statistics):
+ if not is_main_process():
+ return 0, 0, None
+
+ logger.info("Evaluate in main process...")
+
+ annType = ["segm", "bbox", "keypoints"]
+
+ inference_time = statistics[0].item()
+ track_time = statistics[1].item()
+ n_samples = statistics[2].item()
+
+ a_infer_time = 1000 * inference_time / (n_samples * self.dataloader.batch_size)
+ a_track_time = 1000 * track_time / (n_samples * self.dataloader.batch_size)
+
+ time_info = ", ".join(
+ [
+ "Average {} time: {:.2f} ms".format(k, v)
+ for k, v in zip(
+ ["forward", "track", "inference"],
+ [a_infer_time, a_track_time, (a_infer_time + a_track_time)],
+ )
+ ]
+ )
+
+ info = time_info + "\n"
+
+ # Evaluate the Dt (detection) json comparing with the ground truth
+ if len(data_dict) > 0:
+ cocoGt = self.dataloader.dataset.coco
+ # TODO: since pycocotools can't process dict in py36, write data to json file.
+ _, tmp = tempfile.mkstemp()
+ json.dump(data_dict, open(tmp, "w"))
+ cocoDt = cocoGt.loadRes(tmp)
+ '''
+ try:
+ from yolox.layers import COCOeval_opt as COCOeval
+ except ImportError:
+ from pycocotools import cocoeval as COCOeval
+ logger.warning("Use standard COCOeval.")
+ '''
+ #from pycocotools.cocoeval import COCOeval
+ from yolox.layers import COCOeval_opt as COCOeval
+ cocoEval = COCOeval(cocoGt, cocoDt, annType[1])
+ cocoEval.evaluate()
+ cocoEval.accumulate()
+ redirect_string = io.StringIO()
+ with contextlib.redirect_stdout(redirect_string):
+ cocoEval.summarize()
+ info += redirect_string.getvalue()
+ return cocoEval.stats[0], cocoEval.stats[1], info
+ else:
+ return 0, 0, info
diff --git a/experimental/yolox/exp/__init__.py b/experimental/yolox/exp/__init__.py
new file mode 100644
index 0000000..951195c
--- /dev/null
+++ b/experimental/yolox/exp/__init__.py
@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+from .base_exp import BaseExp
+from .build import get_exp
+from .yolox_base import Exp
diff --git a/experimental/yolox/exp/base_exp.py b/experimental/yolox/exp/base_exp.py
new file mode 100644
index 0000000..c4aed89
--- /dev/null
+++ b/experimental/yolox/exp/base_exp.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import torch
+from torch.nn import Module
+
+from yolox.utils import LRScheduler
+
+import ast
+import pprint
+from abc import ABCMeta, abstractmethod
+from tabulate import tabulate
+from typing import Dict
+
+
+class BaseExp(metaclass=ABCMeta):
+ """Basic class for any experiment."""
+
+ def __init__(self):
+ self.seed = None
+ self.output_dir = "./YOLOX_outputs"
+ self.print_interval = 100
+ self.eval_interval = 10
+
+ @abstractmethod
+ def get_model(self) -> Module:
+ pass
+
+ @abstractmethod
+ def get_data_loader(
+ self, batch_size: int, is_distributed: bool
+ ) -> Dict[str, torch.utils.data.DataLoader]:
+ pass
+
+ @abstractmethod
+ def get_optimizer(self, batch_size: int) -> torch.optim.Optimizer:
+ pass
+
+ @abstractmethod
+ def get_lr_scheduler(
+ self, lr: float, iters_per_epoch: int, **kwargs
+ ) -> LRScheduler:
+ pass
+
+ @abstractmethod
+ def get_evaluator(self):
+ pass
+
+ @abstractmethod
+ def eval(self, model, evaluator, weights):
+ pass
+
+ def __repr__(self):
+ table_header = ["keys", "values"]
+ exp_table = [
+ (str(k), pprint.pformat(v))
+ for k, v in vars(self).items()
+ if not k.startswith("_")
+ ]
+ return tabulate(exp_table, headers=table_header, tablefmt="fancy_grid")
+
+ def merge(self, cfg_list):
+ assert len(cfg_list) % 2 == 0
+ for k, v in zip(cfg_list[0::2], cfg_list[1::2]):
+ # only update value with same key
+ if hasattr(self, k):
+ src_value = getattr(self, k)
+ src_type = type(src_value)
+ if src_value is not None and src_type != type(v):
+ try:
+ v = src_type(v)
+ except Exception:
+ v = ast.literal_eval(v)
+ setattr(self, k, v)
diff --git a/experimental/yolox/exp/build.py b/experimental/yolox/exp/build.py
new file mode 100644
index 0000000..411b09c
--- /dev/null
+++ b/experimental/yolox/exp/build.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import importlib
+import os
+import sys
+
+
+def get_exp_by_file(exp_file):
+ try:
+ sys.path.append(os.path.dirname(exp_file))
+ current_exp = importlib.import_module(os.path.basename(exp_file).split(".")[0])
+ exp = current_exp.Exp()
+ except Exception:
+ raise ImportError("{} doesn't contains class named 'Exp'".format(exp_file))
+ return exp
+
+
+def get_exp_by_name(exp_name):
+ import yolox
+
+ yolox_path = os.path.dirname(os.path.dirname(yolox.__file__))
+ filedict = {
+ "yolox-s": "yolox_s.py",
+ "yolox-m": "yolox_m.py",
+ "yolox-l": "yolox_l.py",
+ "yolox-x": "yolox_x.py",
+ "yolox-tiny": "yolox_tiny.py",
+ "yolox-nano": "nano.py",
+ "yolov3": "yolov3.py",
+ }
+ filename = filedict[exp_name]
+ exp_path = os.path.join(yolox_path, "exps", "default", filename)
+ return get_exp_by_file(exp_path)
+
+
+def get_exp(exp_file, exp_name):
+ """
+ get Exp object by file or name. If exp_file and exp_name
+ are both provided, get Exp by exp_file.
+
+ Args:
+ exp_file (str): file path of experiment.
+ exp_name (str): name of experiment. "yolo-s",
+ """
+ assert (
+ exp_file is not None or exp_name is not None
+ ), "plz provide exp file or exp name."
+ if exp_file is not None:
+ return get_exp_by_file(exp_file)
+ else:
+ return get_exp_by_name(exp_name)
diff --git a/experimental/yolox/exp/yolox_base.py b/experimental/yolox/exp/yolox_base.py
new file mode 100644
index 0000000..a1b1b03
--- /dev/null
+++ b/experimental/yolox/exp/yolox_base.py
@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+
+import os
+import random
+
+from .base_exp import BaseExp
+
+
+class Exp(BaseExp):
+ def __init__(self):
+ super().__init__()
+
+ # ---------------- model config ---------------- #
+ self.num_classes = 80
+ self.depth = 1.00
+ self.width = 1.00
+
+ # ---------------- dataloader config ---------------- #
+ # set worker to 4 for shorter dataloader init time
+ self.data_num_workers = 4
+ self.input_size = (640, 640)
+ self.random_size = (14, 26)
+ self.train_ann = "instances_train2017.json"
+ self.val_ann = "instances_val2017.json"
+
+ # --------------- transform config ----------------- #
+ self.degrees = 10.0
+ self.translate = 0.1
+ self.scale = (0.1, 2)
+ self.mscale = (0.8, 1.6)
+ self.shear = 2.0
+ self.perspective = 0.0
+ self.enable_mixup = True
+
+ # -------------- training config --------------------- #
+ self.warmup_epochs = 5
+ self.max_epoch = 300
+ self.warmup_lr = 0
+ self.basic_lr_per_img = 0.01 / 64.0
+ self.scheduler = "yoloxwarmcos"
+ self.no_aug_epochs = 15
+ self.min_lr_ratio = 0.05
+ self.ema = True
+
+ self.weight_decay = 5e-4
+ self.momentum = 0.9
+ self.print_interval = 10
+ self.eval_interval = 10
+ self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(".")[0]
+
+ # ----------------- testing config ------------------ #
+ self.test_size = (640, 640)
+ self.test_conf = 0.001
+ self.nmsthre = 0.65
+
+ def get_model(self):
+ from yolox.models import YOLOPAFPN, YOLOX, YOLOXHead
+
+ def init_yolo(M):
+ for m in M.modules():
+ if isinstance(m, nn.BatchNorm2d):
+ m.eps = 1e-3
+ m.momentum = 0.03
+
+ if getattr(self, "model", None) is None:
+ in_channels = [256, 512, 1024]
+ backbone = YOLOPAFPN(self.depth, self.width, in_channels=in_channels)
+ head = YOLOXHead(self.num_classes, self.width, in_channels=in_channels)
+ self.model = YOLOX(backbone, head)
+
+ self.model.apply(init_yolo)
+ self.model.head.initialize_biases(1e-2)
+ return self.model
+
+ def get_data_loader(self, batch_size, is_distributed, no_aug=False):
+ from yolox.data import (
+ COCODataset,
+ DataLoader,
+ InfiniteSampler,
+ MosaicDetection,
+ TrainTransform,
+ YoloBatchSampler
+ )
+
+ dataset = COCODataset(
+ data_dir=None,
+ json_file=self.train_ann,
+ img_size=self.input_size,
+ preproc=TrainTransform(
+ rgb_means=(0.485, 0.456, 0.406),
+ std=(0.229, 0.224, 0.225),
+ max_labels=50,
+ ),
+ )
+
+ dataset = MosaicDetection(
+ dataset,
+ mosaic=not no_aug,
+ img_size=self.input_size,
+ preproc=TrainTransform(
+ rgb_means=(0.485, 0.456, 0.406),
+ std=(0.229, 0.224, 0.225),
+ max_labels=120,
+ ),
+ degrees=self.degrees,
+ translate=self.translate,
+ scale=self.scale,
+ shear=self.shear,
+ perspective=self.perspective,
+ enable_mixup=self.enable_mixup,
+ )
+
+ self.dataset = dataset
+
+ if is_distributed:
+ batch_size = batch_size // dist.get_world_size()
+
+ sampler = InfiniteSampler(len(self.dataset), seed=self.seed if self.seed else 0)
+
+ batch_sampler = YoloBatchSampler(
+ sampler=sampler,
+ batch_size=batch_size,
+ drop_last=False,
+ input_dimension=self.input_size,
+ mosaic=not no_aug,
+ )
+
+ dataloader_kwargs = {"num_workers": self.data_num_workers, "pin_memory": True}
+ dataloader_kwargs["batch_sampler"] = batch_sampler
+ train_loader = DataLoader(self.dataset, **dataloader_kwargs)
+
+ return train_loader
+
+ def random_resize(self, data_loader, epoch, rank, is_distributed):
+ tensor = torch.LongTensor(2).cuda()
+
+ if rank == 0:
+ size_factor = self.input_size[1] * 1.0 / self.input_size[0]
+ size = random.randint(*self.random_size)
+ size = (int(32 * size), 32 * int(size * size_factor))
+ tensor[0] = size[0]
+ tensor[1] = size[1]
+
+ if is_distributed:
+ dist.barrier()
+ dist.broadcast(tensor, 0)
+
+ input_size = data_loader.change_input_dim(
+ multiple=(tensor[0].item(), tensor[1].item()), random_range=None
+ )
+ return input_size
+
+ def get_optimizer(self, batch_size):
+ if "optimizer" not in self.__dict__:
+ if self.warmup_epochs > 0:
+ lr = self.warmup_lr
+ else:
+ lr = self.basic_lr_per_img * batch_size
+
+ pg0, pg1, pg2 = [], [], [] # optimizer parameter groups
+
+ for k, v in self.model.named_modules():
+ if hasattr(v, "bias") and isinstance(v.bias, nn.Parameter):
+ pg2.append(v.bias) # biases
+ if isinstance(v, nn.BatchNorm2d) or "bn" in k:
+ pg0.append(v.weight) # no decay
+ elif hasattr(v, "weight") and isinstance(v.weight, nn.Parameter):
+ pg1.append(v.weight) # apply decay
+
+ optimizer = torch.optim.SGD(
+ pg0, lr=lr, momentum=self.momentum, nesterov=True
+ )
+ optimizer.add_param_group(
+ {"params": pg1, "weight_decay": self.weight_decay}
+ ) # add pg1 with weight_decay
+ optimizer.add_param_group({"params": pg2})
+ self.optimizer = optimizer
+
+ return self.optimizer
+
+ def get_lr_scheduler(self, lr, iters_per_epoch):
+ from yolox.utils import LRScheduler
+
+ scheduler = LRScheduler(
+ self.scheduler,
+ lr,
+ iters_per_epoch,
+ self.max_epoch,
+ warmup_epochs=self.warmup_epochs,
+ warmup_lr_start=self.warmup_lr,
+ no_aug_epochs=self.no_aug_epochs,
+ min_lr_ratio=self.min_lr_ratio,
+ )
+ return scheduler
+
+ def get_eval_loader(self, batch_size, is_distributed, testdev=False):
+ from yolox.data import COCODataset, ValTransform
+
+ valdataset = COCODataset(
+ data_dir=None,
+ json_file=self.val_ann if not testdev else "image_info_test-dev2017.json",
+ name="val2017" if not testdev else "test2017",
+ img_size=self.test_size,
+ preproc=ValTransform(
+ rgb_means=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)
+ ),
+ )
+
+ if is_distributed:
+ batch_size = batch_size // dist.get_world_size()
+ sampler = torch.utils.data.distributed.DistributedSampler(
+ valdataset, shuffle=False
+ )
+ else:
+ sampler = torch.utils.data.SequentialSampler(valdataset)
+
+ dataloader_kwargs = {
+ "num_workers": self.data_num_workers,
+ "pin_memory": True,
+ "sampler": sampler,
+ }
+ dataloader_kwargs["batch_size"] = batch_size
+ val_loader = torch.utils.data.DataLoader(valdataset, **dataloader_kwargs)
+
+ return val_loader
+
+ def get_evaluator(self, batch_size, is_distributed, testdev=False):
+ from yolox.evaluators import COCOEvaluator
+
+ val_loader = self.get_eval_loader(batch_size, is_distributed, testdev=testdev)
+ evaluator = COCOEvaluator(
+ dataloader=val_loader,
+ img_size=self.test_size,
+ confthre=self.test_conf,
+ nmsthre=self.nmsthre,
+ num_classes=self.num_classes,
+ testdev=testdev,
+ )
+ return evaluator
+
+ def eval(self, model, evaluator, is_distributed, half=False):
+ return evaluator.evaluate(model, is_distributed, half)
diff --git a/experimental/yolox/layers/__init__.py b/experimental/yolox/layers/__init__.py
new file mode 100644
index 0000000..833947c
--- /dev/null
+++ b/experimental/yolox/layers/__init__.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+from .fast_coco_eval_api import COCOeval_opt
diff --git a/experimental/yolox/layers/csrc/cocoeval/cocoeval.cpp b/experimental/yolox/layers/csrc/cocoeval/cocoeval.cpp
new file mode 100644
index 0000000..2e63bc9
--- /dev/null
+++ b/experimental/yolox/layers/csrc/cocoeval/cocoeval.cpp
@@ -0,0 +1,502 @@
+// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+#include "cocoeval.h"
+#include <time.h>
+#include <algorithm>
+#include <cstdint>
+#include <numeric>
+
+using namespace pybind11::literals;
+
+namespace COCOeval {
+
+// Sort detections from highest score to lowest, such that
+// detection_instances[detection_sorted_indices[t]] >=
+// detection_instances[detection_sorted_indices[t+1]]. Use stable_sort to match
+// original COCO API
+void SortInstancesByDetectionScore(
+ const std::vector<InstanceAnnotation>& detection_instances,
+ std::vector<uint64_t>* detection_sorted_indices) {
+ detection_sorted_indices->resize(detection_instances.size());
+ std::iota(
+ detection_sorted_indices->begin(), detection_sorted_indices->end(), 0);
+ std::stable_sort(
+ detection_sorted_indices->begin(),
+ detection_sorted_indices->end(),
+ [&detection_instances](size_t j1, size_t j2) {
+ return detection_instances[j1].score > detection_instances[j2].score;
+ });
+}
+
+// Partition the ground truth objects based on whether or not to ignore them
+// based on area
+void SortInstancesByIgnore(
+ const std::array<double, 2>& area_range,
+ const std::vector<InstanceAnnotation>& ground_truth_instances,
+ std::vector<uint64_t>* ground_truth_sorted_indices,
+ std::vector<bool>* ignores) {
+ ignores->clear();
+ ignores->reserve(ground_truth_instances.size());
+ for (auto o : ground_truth_instances) {
+ ignores->push_back(
+ o.ignore || o.area < area_range[0] || o.area > area_range[1]);
+ }
+
+ ground_truth_sorted_indices->resize(ground_truth_instances.size());
+ std::iota(
+ ground_truth_sorted_indices->begin(),
+ ground_truth_sorted_indices->end(),
+ 0);
+ std::stable_sort(
+ ground_truth_sorted_indices->begin(),
+ ground_truth_sorted_indices->end(),
+ [&ignores](size_t j1, size_t j2) {
+ return (int)(*ignores)[j1] < (int)(*ignores)[j2];
+ });
+}
+
+// For each IOU threshold, greedily match each detected instance to a ground
+// truth instance (if possible) and store the results
+void MatchDetectionsToGroundTruth(
+ const std::vector<InstanceAnnotation>& detection_instances,
+ const std::vector<uint64_t>& detection_sorted_indices,
+ const std::vector<InstanceAnnotation>& ground_truth_instances,
+ const std::vector<uint64_t>& ground_truth_sorted_indices,
+ const std::vector<bool>& ignores,
+ const std::vector<std::vector<double>>& ious,
+ const std::vector<double>& iou_thresholds,
+ const std::array<double, 2>& area_range,
+ ImageEvaluation* results) {
+ // Initialize memory to store return data matches and ignore
+ const int num_iou_thresholds = iou_thresholds.size();
+ const int num_ground_truth = ground_truth_sorted_indices.size();
+ const int num_detections = detection_sorted_indices.size();
+ std::vector<uint64_t> ground_truth_matches(
+ num_iou_thresholds * num_ground_truth, 0);
+ std::vector<uint64_t>& detection_matches = results->detection_matches;
+ std::vector<bool>& detection_ignores = results->detection_ignores;
+ std::vector<bool>& ground_truth_ignores = results->ground_truth_ignores;
+ detection_matches.resize(num_iou_thresholds * num_detections, 0);
+ detection_ignores.resize(num_iou_thresholds * num_detections, false);
+ ground_truth_ignores.resize(num_ground_truth);
+ for (auto g = 0; g < num_ground_truth; ++g) {
+ ground_truth_ignores[g] = ignores[ground_truth_sorted_indices[g]];
+ }
+
+ for (auto t = 0; t < num_iou_thresholds; ++t) {
+ for (auto d = 0; d < num_detections; ++d) {
+ // information about best match so far (match=-1 -> unmatched)
+ double best_iou = std::min(iou_thresholds[t], 1 - 1e-10);
+ int match = -1;
+ for (auto g = 0; g < num_ground_truth; ++g) {
+ // if this ground truth instance is already matched and not a
+ // crowd, it cannot be matched to another detection
+ if (ground_truth_matches[t * num_ground_truth + g] > 0 &&
+ !ground_truth_instances[ground_truth_sorted_indices[g]].is_crowd) {
+ continue;
+ }
+
+ // if detected instance matched to a regular ground truth
+ // instance, we can break on the first ground truth instance
+ // tagged as ignore (because they are sorted by the ignore tag)
+ if (match >= 0 && !ground_truth_ignores[match] &&
+ ground_truth_ignores[g]) {
+ break;
+ }
+
+ // if IOU overlap is the best so far, store the match appropriately
+ if (ious[d][ground_truth_sorted_indices[g]] >= best_iou) {
+ best_iou = ious[d][ground_truth_sorted_indices[g]];
+ match = g;
+ }
+ }
+ // if match was made, store id of match for both detection and
+ // ground truth
+ if (match >= 0) {
+ detection_ignores[t * num_detections + d] = ground_truth_ignores[match];
+ detection_matches[t * num_detections + d] =
+ ground_truth_instances[ground_truth_sorted_indices[match]].id;
+ ground_truth_matches[t * num_ground_truth + match] =
+ detection_instances[detection_sorted_indices[d]].id;
+ }
+
+ // set unmatched detections outside of area range to ignore
+ const InstanceAnnotation& detection =
+ detection_instances[detection_sorted_indices[d]];
+ detection_ignores[t * num_detections + d] =
+ detection_ignores[t * num_detections + d] ||
+ (detection_matches[t * num_detections + d] == 0 &&
+ (detection.area < area_range[0] || detection.area > area_range[1]));
+ }
+ }
+
+ // store detection score results
+ results->detection_scores.resize(detection_sorted_indices.size());
+ for (size_t d = 0; d < detection_sorted_indices.size(); ++d) {
+ results->detection_scores[d] =
+ detection_instances[detection_sorted_indices[d]].score;
+ }
+}
+
+std::vector<ImageEvaluation> EvaluateImages(
+ const std::vector<std::array<double, 2>>& area_ranges,
+ int max_detections,
+ const std::vector<double>& iou_thresholds,
+ const ImageCategoryInstances<std::vector<double>>& image_category_ious,
+ const ImageCategoryInstances<InstanceAnnotation>&
+ image_category_ground_truth_instances,
+ const ImageCategoryInstances<InstanceAnnotation>&
+ image_category_detection_instances) {
+ const int num_area_ranges = area_ranges.size();
+ const int num_images = image_category_ground_truth_instances.size();
+ const int num_categories =
+ image_category_ious.size() > 0 ? image_category_ious[0].size() : 0;
+ std::vector<uint64_t> detection_sorted_indices;
+ std::vector<uint64_t> ground_truth_sorted_indices;
+ std::vector<bool> ignores;
+ std::vector<ImageEvaluation> results_all(
+ num_images * num_area_ranges * num_categories);
+
+ // Store results for each image, category, and area range combination. Results
+ // for each IOU threshold are packed into the same ImageEvaluation object
+ for (auto i = 0; i < num_images; ++i) {
+ for (auto c = 0; c < num_categories; ++c) {
+ const std::vector<InstanceAnnotation>& ground_truth_instances =
+ image_category_ground_truth_instances[i][c];
+ const std::vector<InstanceAnnotation>& detection_instances =
+ image_category_detection_instances[i][c];
+
+ SortInstancesByDetectionScore(
+ detection_instances, &detection_sorted_indices);
+ if ((int)detection_sorted_indices.size() > max_detections) {
+ detection_sorted_indices.resize(max_detections);
+ }
+
+ for (size_t a = 0; a < area_ranges.size(); ++a) {
+ SortInstancesByIgnore(
+ area_ranges[a],
+ ground_truth_instances,
+ &ground_truth_sorted_indices,
+ &ignores);
+
+ MatchDetectionsToGroundTruth(
+ detection_instances,
+ detection_sorted_indices,
+ ground_truth_instances,
+ ground_truth_sorted_indices,
+ ignores,
+ image_category_ious[i][c],
+ iou_thresholds,
+ area_ranges[a],
+ &results_all
+ [c * num_area_ranges * num_images + a * num_images + i]);
+ }
+ }
+ }
+
+ return results_all;
+}
+
+// Convert a python list to a vector
+template <typename T>
+std::vector<T> list_to_vec(const py::list& l) {
+ std::vector<T> v(py::len(l));
+ for (int i = 0; i < (int)py::len(l); ++i) {
+ v[i] = l[i].cast<T>();
+ }
+ return v;
+}
+
+// Helper function to Accumulate()
+// Considers the evaluation results applicable to a particular category, area
+// range, and max_detections parameter setting, which begin at
+// evaluations[evaluation_index]. Extracts a sorted list of length n of all
+// applicable detection instances concatenated across all images in the dataset,
+// which are represented by the outputs evaluation_indices, detection_scores,
+// image_detection_indices, and detection_sorted_indices--all of which are
+// length n. evaluation_indices[i] stores the applicable index into
+// evaluations[] for instance i, which has detection score detection_score[i],
+// and is the image_detection_indices[i]'th of the list of detections
+// for the image containing i. detection_sorted_indices[] defines a sorted
+// permutation of the 3 other outputs
+int BuildSortedDetectionList(
+ const std::vector<ImageEvaluation>& evaluations,
+ const int64_t evaluation_index,
+ const int64_t num_images,
+ const int max_detections,
+ std::vector<uint64_t>* evaluation_indices,
+ std::vector<double>* detection_scores,
+ std::vector<uint64_t>* detection_sorted_indices,
+ std::vector<uint64_t>* image_detection_indices) {
+ assert(evaluations.size() >= evaluation_index + num_images);
+
+ // Extract a list of object instances of the applicable category, area
+ // range, and max detections requirements such that they can be sorted
+ image_detection_indices->clear();
+ evaluation_indices->clear();
+ detection_scores->clear();
+ image_detection_indices->reserve(num_images * max_detections);
+ evaluation_indices->reserve(num_images * max_detections);
+ detection_scores->reserve(num_images * max_detections);
+ int num_valid_ground_truth = 0;
+ for (auto i = 0; i < num_images; ++i) {
+ const ImageEvaluation& evaluation = evaluations[evaluation_index + i];
+
+ for (int d = 0;
+ d < (int)evaluation.detection_scores.size() && d < max_detections;
+ ++d) { // detected instances
+ evaluation_indices->push_back(evaluation_index + i);
+ image_detection_indices->push_back(d);
+ detection_scores->push_back(evaluation.detection_scores[d]);
+ }
+ for (auto ground_truth_ignore : evaluation.ground_truth_ignores) {
+ if (!ground_truth_ignore) {
+ ++num_valid_ground_truth;
+ }
+ }
+ }
+
+ // Sort detections by decreasing score, using stable sort to match
+ // python implementation
+ detection_sorted_indices->resize(detection_scores->size());
+ std::iota(
+ detection_sorted_indices->begin(), detection_sorted_indices->end(), 0);
+ std::stable_sort(
+ detection_sorted_indices->begin(),
+ detection_sorted_indices->end(),
+ [&detection_scores](size_t j1, size_t j2) {
+ return (*detection_scores)[j1] > (*detection_scores)[j2];
+ });
+
+ return num_valid_ground_truth;
+}
+
+// Helper function to Accumulate()
+// Compute a precision recall curve given a sorted list of detected instances
+// encoded in evaluations, evaluation_indices, detection_scores,
+// detection_sorted_indices, image_detection_indices (see
+// BuildSortedDetectionList()). Using vectors precisions and recalls
+// and temporary storage, output the results into precisions_out, recalls_out,
+// and scores_out, which are large buffers containing many precion/recall curves
+// for all possible parameter settings, with precisions_out_index and
+// recalls_out_index defining the applicable indices to store results.
+void ComputePrecisionRecallCurve(
+ const int64_t precisions_out_index,
+ const int64_t precisions_out_stride,
+ const int64_t recalls_out_index,
+ const std::vector<double>& recall_thresholds,
+ const int iou_threshold_index,
+ const int num_iou_thresholds,
+ const int num_valid_ground_truth,
+ const std::vector<ImageEvaluation>& evaluations,
+ const std::vector<uint64_t>& evaluation_indices,
+ const std::vector<double>& detection_scores,
+ const std::vector<uint64_t>& detection_sorted_indices,
+ const std::vector<uint64_t>& image_detection_indices,
+ std::vector<double>* precisions,
+ std::vector<double>* recalls,
+ std::vector<double>* precisions_out,
+ std::vector<double>* scores_out,
+ std::vector<double>* recalls_out) {
+ assert(recalls_out->size() > recalls_out_index);
+
+ // Compute precision/recall for each instance in the sorted list of detections
+ int64_t true_positives_sum = 0, false_positives_sum = 0;
+ precisions->clear();
+ recalls->clear();
+ precisions->reserve(detection_sorted_indices.size());
+ recalls->reserve(detection_sorted_indices.size());
+ assert(!evaluations.empty() || detection_sorted_indices.empty());
+ for (auto detection_sorted_index : detection_sorted_indices) {
+ const ImageEvaluation& evaluation =
+ evaluations[evaluation_indices[detection_sorted_index]];
+ const auto num_detections =
+ evaluation.detection_matches.size() / num_iou_thresholds;
+ const auto detection_index = iou_threshold_index * num_detections +
+ image_detection_indices[detection_sorted_index];
+ assert(evaluation.detection_matches.size() > detection_index);
+ assert(evaluation.detection_ignores.size() > detection_index);
+ const int64_t detection_match =
+ evaluation.detection_matches[detection_index];
+ const bool detection_ignores =
+ evaluation.detection_ignores[detection_index];
+ const auto true_positive = detection_match > 0 && !detection_ignores;
+ const auto false_positive = detection_match == 0 && !detection_ignores;
+ if (true_positive) {
+ ++true_positives_sum;
+ }
+ if (false_positive) {
+ ++false_positives_sum;
+ }
+
+ const double recall =
+ static_cast<double>(true_positives_sum) / num_valid_ground_truth;
+ recalls->push_back(recall);
+ const int64_t num_valid_detections =
+ true_positives_sum + false_positives_sum;
+ const double precision = num_valid_detections > 0
+ ? static_cast<double>(true_positives_sum) / num_valid_detections
+ : 0.0;
+ precisions->push_back(precision);
+ }
+
+ (*recalls_out)[recalls_out_index] = !recalls->empty() ? recalls->back() : 0;
+
+ for (int64_t i = static_cast<int64_t>(precisions->size()) - 1; i > 0; --i) {
+ if ((*precisions)[i] > (*precisions)[i - 1]) {
+ (*precisions)[i - 1] = (*precisions)[i];
+ }
+ }
+
+ // Sample the per instance precision/recall list at each recall threshold
+ for (size_t r = 0; r < recall_thresholds.size(); ++r) {
+ // first index in recalls >= recall_thresholds[r]
+ std::vector<double>::iterator low = std::lower_bound(
+ recalls->begin(), recalls->end(), recall_thresholds[r]);
+ size_t precisions_index = low - recalls->begin();
+
+ const auto results_ind = precisions_out_index + r * precisions_out_stride;
+ assert(results_ind < precisions_out->size());
+ assert(results_ind < scores_out->size());
+ if (precisions_index < precisions->size()) {
+ (*precisions_out)[results_ind] = (*precisions)[precisions_index];
+ (*scores_out)[results_ind] =
+ detection_scores[detection_sorted_indices[precisions_index]];
+ } else {
+ (*precisions_out)[results_ind] = 0;
+ (*scores_out)[results_ind] = 0;
+ }
+ }
+}
+py::dict Accumulate(
+ const py::object& params,
+ const std::vector<ImageEvaluation>& evaluations) {
+ const std::vector<double> recall_thresholds =
+ list_to_vec<double>(params.attr("recThrs"));
+ const std::vector<int> max_detections =
+ list_to_vec<int>(params.attr("maxDets"));
+ const int num_iou_thresholds = py::len(params.attr("iouThrs"));
+ const int num_recall_thresholds = py::len(params.attr("recThrs"));
+ const int num_categories = params.attr("useCats").cast<int>() == 1
+ ? py::len(params.attr("catIds"))
+ : 1;
+ const int num_area_ranges = py::len(params.attr("areaRng"));
+ const int num_max_detections = py::len(params.attr("maxDets"));
+ const int num_images = py::len(params.attr("imgIds"));
+
+ std::vector<double> precisions_out(
+ num_iou_thresholds * num_recall_thresholds * num_categories *
+ num_area_ranges * num_max_detections,
+ -1);
+ std::vector<double> recalls_out(
+ num_iou_thresholds * num_categories * num_area_ranges *
+ num_max_detections,
+ -1);
+ std::vector<double> scores_out(
+ num_iou_thresholds * num_recall_thresholds * num_categories *
+ num_area_ranges * num_max_detections,
+ -1);
+
+ // Consider the list of all detected instances in the entire dataset in one
+ // large list. evaluation_indices, detection_scores,
+ // image_detection_indices, and detection_sorted_indices all have the same
+ // length as this list, such that each entry corresponds to one detected
+ // instance
+ std::vector<uint64_t> evaluation_indices; // indices into evaluations[]
+ std::vector<double> detection_scores; // detection scores of each instance
+ std::vector<uint64_t> detection_sorted_indices; // sorted indices of all
+ // instances in the dataset
+ std::vector<uint64_t>
+ image_detection_indices; // indices into the list of detected instances in
+ // the same image as each instance
+ std::vector<double> precisions, recalls;
+
+ for (auto c = 0; c < num_categories; ++c) {
+ for (auto a = 0; a < num_area_ranges; ++a) {
+ for (auto m = 0; m < num_max_detections; ++m) {
+ // The COCO PythonAPI assumes evaluations[] (the return value of
+ // COCOeval::EvaluateImages() is one long list storing results for each
+ // combination of category, area range, and image id, with categories in
+ // the outermost loop and images in the innermost loop.
+ const int64_t evaluations_index =
+ c * num_area_ranges * num_images + a * num_images;
+ int num_valid_ground_truth = BuildSortedDetectionList(
+ evaluations,
+ evaluations_index,
+ num_images,
+ max_detections[m],
+ &evaluation_indices,
+ &detection_scores,
+ &detection_sorted_indices,
+ &image_detection_indices);
+
+ if (num_valid_ground_truth == 0) {
+ continue;
+ }
+
+ for (auto t = 0; t < num_iou_thresholds; ++t) {
+ // recalls_out is a flattened vectors representing a
+ // num_iou_thresholds X num_categories X num_area_ranges X
+ // num_max_detections matrix
+ const int64_t recalls_out_index =
+ t * num_categories * num_area_ranges * num_max_detections +
+ c * num_area_ranges * num_max_detections +
+ a * num_max_detections + m;
+
+ // precisions_out and scores_out are flattened vectors
+ // representing a num_iou_thresholds X num_recall_thresholds X
+ // num_categories X num_area_ranges X num_max_detections matrix
+ const int64_t precisions_out_stride =
+ num_categories * num_area_ranges * num_max_detections;
+ const int64_t precisions_out_index = t * num_recall_thresholds *
+ num_categories * num_area_ranges * num_max_detections +
+ c * num_area_ranges * num_max_detections +
+ a * num_max_detections + m;
+
+ ComputePrecisionRecallCurve(
+ precisions_out_index,
+ precisions_out_stride,
+ recalls_out_index,
+ recall_thresholds,
+ t,
+ num_iou_thresholds,
+ num_valid_ground_truth,
+ evaluations,
+ evaluation_indices,
+ detection_scores,
+ detection_sorted_indices,
+ image_detection_indices,
+ &precisions,
+ &recalls,
+ &precisions_out,
+ &scores_out,
+ &recalls_out);
+ }
+ }
+ }
+ }
+
+ time_t rawtime;
+ struct tm local_time;
+ std::array<char, 200> buffer;
+ time(&rawtime);
+#ifdef _WIN32
+ localtime_s(&local_time, &rawtime);
+#else
+ localtime_r(&rawtime, &local_time);
+#endif
+ strftime(
+ buffer.data(), 200, "%Y-%m-%d %H:%num_max_detections:%S", &local_time);
+ return py::dict(
+ "params"_a = params,
+ "counts"_a = std::vector<int64_t>({num_iou_thresholds,
+ num_recall_thresholds,
+ num_categories,
+ num_area_ranges,
+ num_max_detections}),
+ "date"_a = buffer,
+ "precision"_a = precisions_out,
+ "recall"_a = recalls_out,
+ "scores"_a = scores_out);
+}
+
+} // namespace COCOeval
diff --git a/experimental/yolox/layers/csrc/cocoeval/cocoeval.h b/experimental/yolox/layers/csrc/cocoeval/cocoeval.h
new file mode 100644
index 0000000..f9def41
--- /dev/null
+++ b/experimental/yolox/layers/csrc/cocoeval/cocoeval.h
@@ -0,0 +1,85 @@
+// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+#pragma once
+
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <pybind11/stl_bind.h>
+#include <vector>
+
+namespace py = pybind11;
+
+namespace COCOeval {
+
+// Annotation data for a single object instance in an image
+struct InstanceAnnotation {
+ InstanceAnnotation(
+ uint64_t id,
+ double score,
+ double area,
+ bool is_crowd,
+ bool ignore)
+ : id{id}, score{score}, area{area}, is_crowd{is_crowd}, ignore{ignore} {}
+ uint64_t id;
+ double score = 0.;
+ double area = 0.;
+ bool is_crowd = false;
+ bool ignore = false;
+};
+
+// Stores intermediate results for evaluating detection results for a single
+// image that has D detected instances and G ground truth instances. This stores
+// matches between detected and ground truth instances
+struct ImageEvaluation {
+ // For each of the D detected instances, the id of the matched ground truth
+ // instance, or 0 if unmatched
+ std::vector<uint64_t> detection_matches;
+
+ // The detection score of each of the D detected instances
+ std::vector<double> detection_scores;
+
+ // Marks whether or not each of G instances was ignored from evaluation (e.g.,
+ // because it's outside area_range)
+ std::vector<bool> ground_truth_ignores;
+
+ // Marks whether or not each of D instances was ignored from evaluation (e.g.,
+ // because it's outside aRng)
+ std::vector<bool> detection_ignores;
+};
+
+template <class T>
+using ImageCategoryInstances = std::vector<std::vector<std::vector<T>>>;
+
+// C++ implementation of COCO API cocoeval.py::COCOeval.evaluateImg(). For each
+// combination of image, category, area range settings, and IOU thresholds to
+// evaluate, it matches detected instances to ground truth instances and stores
+// the results into a vector of ImageEvaluation results, which will be
+// interpreted by the COCOeval::Accumulate() function to produce precion-recall
+// curves. The parameters of nested vectors have the following semantics:
+// image_category_ious[i][c][d][g] is the intersection over union of the d'th
+// detected instance and g'th ground truth instance of
+// category category_ids[c] in image image_ids[i]
+// image_category_ground_truth_instances[i][c] is a vector of ground truth
+// instances in image image_ids[i] of category category_ids[c]
+// image_category_detection_instances[i][c] is a vector of detected
+// instances in image image_ids[i] of category category_ids[c]
+std::vector<ImageEvaluation> EvaluateImages(
+ const std::vector<std::array<double, 2>>& area_ranges, // vector of 2-tuples
+ int max_detections,
+ const std::vector<double>& iou_thresholds,
+ const ImageCategoryInstances<std::vector<double>>& image_category_ious,
+ const ImageCategoryInstances<InstanceAnnotation>&
+ image_category_ground_truth_instances,
+ const ImageCategoryInstances<InstanceAnnotation>&
+ image_category_detection_instances);
+
+// C++ implementation of COCOeval.accumulate(), which generates precision
+// recall curves for each set of category, IOU threshold, detection area range,
+// and max number of detections parameters. It is assumed that the parameter
+// evaluations is the return value of the functon COCOeval::EvaluateImages(),
+// which was called with the same parameter settings params
+py::dict Accumulate(
+ const py::object& params,
+ const std::vector<ImageEvaluation>& evalutations);
+
+} // namespace COCOeval
diff --git a/experimental/yolox/layers/csrc/vision.cpp b/experimental/yolox/layers/csrc/vision.cpp
new file mode 100644
index 0000000..7663d0f
--- /dev/null
+++ b/experimental/yolox/layers/csrc/vision.cpp
@@ -0,0 +1,13 @@
+#include "cocoeval/cocoeval.h"
+
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
+ m.def("COCOevalAccumulate", &COCOeval::Accumulate, "COCOeval::Accumulate");
+ m.def(
+ "COCOevalEvaluateImages",
+ &COCOeval::EvaluateImages,
+ "COCOeval::EvaluateImages");
+ pybind11::class_<COCOeval::InstanceAnnotation>(m, "InstanceAnnotation")
+ .def(pybind11::init<uint64_t, double, double, bool, bool>());
+ pybind11::class_<COCOeval::ImageEvaluation>(m, "ImageEvaluation")
+ .def(pybind11::init<>());
+}
diff --git a/experimental/yolox/layers/fast_coco_eval_api.py b/experimental/yolox/layers/fast_coco_eval_api.py
new file mode 100644
index 0000000..442c97e
--- /dev/null
+++ b/experimental/yolox/layers/fast_coco_eval_api.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# This file comes from
+# https://github.com/facebookresearch/detectron2/blob/master/detectron2/evaluation/fast_eval_api.py
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import numpy as np
+from pycocotools.cocoeval import COCOeval
+
+# import torch first to make yolox._C work without ImportError of libc10.so
+# in YOLOX, env is already set in __init__.py.
+from yolox import _C
+
+import copy
+import time
+
+
+class COCOeval_opt(COCOeval):
+ """
+ This is a slightly modified version of the original COCO API, where the functions evaluateImg()
+ and accumulate() are implemented in C++ to speedup evaluation
+ """
+
+ def evaluate(self):
+ """
+ Run per image evaluation on given images and store results in self.evalImgs_cpp, a
+ datastructure that isn't readable from Python but is used by a c++ implementation of
+ accumulate(). Unlike the original COCO PythonAPI, we don't populate the datastructure
+ self.evalImgs because this datastructure is a computational bottleneck.
+ :return: None
+ """
+ tic = time.time()
+
+ print("Running per image evaluation...")
+ p = self.params
+ # add backward compatibility if useSegm is specified in params
+ if p.useSegm is not None:
+ p.iouType = "segm" if p.useSegm == 1 else "bbox"
+ print(
+ "useSegm (deprecated) is not None. Running {} evaluation".format(
+ p.iouType
+ )
+ )
+ print("Evaluate annotation type *{}*".format(p.iouType))
+ p.imgIds = list(np.unique(p.imgIds))
+ if p.useCats:
+ p.catIds = list(np.unique(p.catIds))
+ p.maxDets = sorted(p.maxDets)
+ self.params = p
+
+ self._prepare()
+
+ # loop through images, area range, max detection number
+ catIds = p.catIds if p.useCats else [-1]
+
+ if p.iouType == "segm" or p.iouType == "bbox":
+ computeIoU = self.computeIoU
+ elif p.iouType == "keypoints":
+ computeIoU = self.computeOks
+ self.ious = {
+ (imgId, catId): computeIoU(imgId, catId)
+ for imgId in p.imgIds
+ for catId in catIds
+ }
+
+ maxDet = p.maxDets[-1]
+
+ # <<<< Beginning of code differences with original COCO API
+ def convert_instances_to_cpp(instances, is_det=False):
+ # Convert annotations for a list of instances in an image to a format that's fast
+ # to access in C++
+ instances_cpp = []
+ for instance in instances:
+ instance_cpp = _C.InstanceAnnotation(
+ int(instance["id"]),
+ instance["score"] if is_det else instance.get("score", 0.0),
+ instance["area"],
+ bool(instance.get("iscrowd", 0)),
+ bool(instance.get("ignore", 0)),
+ )
+ instances_cpp.append(instance_cpp)
+ return instances_cpp
+
+ # Convert GT annotations, detections, and IOUs to a format that's fast to access in C++
+ ground_truth_instances = [
+ [convert_instances_to_cpp(self._gts[imgId, catId]) for catId in p.catIds]
+ for imgId in p.imgIds
+ ]
+ detected_instances = [
+ [
+ convert_instances_to_cpp(self._dts[imgId, catId], is_det=True)
+ for catId in p.catIds
+ ]
+ for imgId in p.imgIds
+ ]
+ ious = [[self.ious[imgId, catId] for catId in catIds] for imgId in p.imgIds]
+
+ if not p.useCats:
+ # For each image, flatten per-category lists into a single list
+ ground_truth_instances = [
+ [[o for c in i for o in c]] for i in ground_truth_instances
+ ]
+ detected_instances = [
+ [[o for c in i for o in c]] for i in detected_instances
+ ]
+
+ # Call C++ implementation of self.evaluateImgs()
+ self._evalImgs_cpp = _C.COCOevalEvaluateImages(
+ p.areaRng,
+ maxDet,
+ p.iouThrs,
+ ious,
+ ground_truth_instances,
+ detected_instances,
+ )
+ self._evalImgs = None
+
+ self._paramsEval = copy.deepcopy(self.params)
+ toc = time.time()
+ print("COCOeval_opt.evaluate() finished in {:0.2f} seconds.".format(toc - tic))
+ # >>>> End of code differences with original COCO API
+
+ def accumulate(self):
+ """
+ Accumulate per image evaluation results and store the result in self.eval. Does not
+ support changing parameter settings from those used by self.evaluate()
+ """
+ print("Accumulating evaluation results...")
+ tic = time.time()
+ if not hasattr(self, "_evalImgs_cpp"):
+ print("Please run evaluate() first")
+
+ self.eval = _C.COCOevalAccumulate(self._paramsEval, self._evalImgs_cpp)
+
+ # recall is num_iou_thresholds X num_categories X num_area_ranges X num_max_detections
+ self.eval["recall"] = np.array(self.eval["recall"]).reshape(
+ self.eval["counts"][:1] + self.eval["counts"][2:]
+ )
+
+ # precision and scores are num_iou_thresholds X num_recall_thresholds X num_categories X
+ # num_area_ranges X num_max_detections
+ self.eval["precision"] = np.array(self.eval["precision"]).reshape(
+ self.eval["counts"]
+ )
+ self.eval["scores"] = np.array(self.eval["scores"]).reshape(self.eval["counts"])
+ toc = time.time()
+ print(
+ "COCOeval_opt.accumulate() finished in {:0.2f} seconds.".format(toc - tic)
+ )
diff --git a/experimental/yolox/models/__init__.py b/experimental/yolox/models/__init__.py
new file mode 100644
index 0000000..c4641a6
--- /dev/null
+++ b/experimental/yolox/models/__init__.py
@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+from .darknet import CSPDarknet, Darknet
+from .losses import IOUloss
+from .yolo_fpn import YOLOFPN
+from .yolo_head import YOLOXHead
+from .yolo_pafpn import YOLOPAFPN
+from .yolox import YOLOX
diff --git a/experimental/yolox/models/darknet.py b/experimental/yolox/models/darknet.py
new file mode 100644
index 0000000..70c79f8
--- /dev/null
+++ b/experimental/yolox/models/darknet.py
@@ -0,0 +1,179 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+from torch import nn
+
+from .network_blocks import BaseConv, CSPLayer, DWConv, Focus, ResLayer, SPPBottleneck
+
+
+class Darknet(nn.Module):
+ # number of blocks from dark2 to dark5.
+ depth2blocks = {21: [1, 2, 2, 1], 53: [2, 8, 8, 4]}
+
+ def __init__(
+ self,
+ depth,
+ in_channels=3,
+ stem_out_channels=32,
+ out_features=("dark3", "dark4", "dark5"),
+ ):
+ """
+ Args:
+ depth (int): depth of darknet used in model, usually use [21, 53] for this param.
+ in_channels (int): number of input channels, for example, use 3 for RGB image.
+ stem_out_channels (int): number of output chanels of darknet stem.
+ It decides channels of darknet layer2 to layer5.
+ out_features (Tuple[str]): desired output layer name.
+ """
+ super().__init__()
+ assert out_features, "please provide output features of Darknet"
+ self.out_features = out_features
+ self.stem = nn.Sequential(
+ BaseConv(in_channels, stem_out_channels, ksize=3, stride=1, act="lrelu"),
+ *self.make_group_layer(stem_out_channels, num_blocks=1, stride=2),
+ )
+ in_channels = stem_out_channels * 2 # 64
+
+ num_blocks = Darknet.depth2blocks[depth]
+ # create darknet with `stem_out_channels` and `num_blocks` layers.
+ # to make model structure more clear, we don't use `for` statement in python.
+ self.dark2 = nn.Sequential(
+ *self.make_group_layer(in_channels, num_blocks[0], stride=2)
+ )
+ in_channels *= 2 # 128
+ self.dark3 = nn.Sequential(
+ *self.make_group_layer(in_channels, num_blocks[1], stride=2)
+ )
+ in_channels *= 2 # 256
+ self.dark4 = nn.Sequential(
+ *self.make_group_layer(in_channels, num_blocks[2], stride=2)
+ )
+ in_channels *= 2 # 512
+
+ self.dark5 = nn.Sequential(
+ *self.make_group_layer(in_channels, num_blocks[3], stride=2),
+ *self.make_spp_block([in_channels, in_channels * 2], in_channels * 2),
+ )
+
+ def make_group_layer(self, in_channels: int, num_blocks: int, stride: int = 1):
+ "starts with conv layer then has `num_blocks` `ResLayer`"
+ return [
+ BaseConv(in_channels, in_channels * 2, ksize=3, stride=stride, act="lrelu"),
+ *[(ResLayer(in_channels * 2)) for _ in range(num_blocks)],
+ ]
+
+ def make_spp_block(self, filters_list, in_filters):
+ m = nn.Sequential(
+ *[
+ BaseConv(in_filters, filters_list[0], 1, stride=1, act="lrelu"),
+ BaseConv(filters_list[0], filters_list[1], 3, stride=1, act="lrelu"),
+ SPPBottleneck(
+ in_channels=filters_list[1],
+ out_channels=filters_list[0],
+ activation="lrelu",
+ ),
+ BaseConv(filters_list[0], filters_list[1], 3, stride=1, act="lrelu"),
+ BaseConv(filters_list[1], filters_list[0], 1, stride=1, act="lrelu"),
+ ]
+ )
+ return m
+
+ def forward(self, x):
+ outputs = {}
+ x = self.stem(x)
+ outputs["stem"] = x
+ x = self.dark2(x)
+ outputs["dark2"] = x
+ x = self.dark3(x)
+ outputs["dark3"] = x
+ x = self.dark4(x)
+ outputs["dark4"] = x
+ x = self.dark5(x)
+ outputs["dark5"] = x
+ return {k: v for k, v in outputs.items() if k in self.out_features}
+
+
+class CSPDarknet(nn.Module):
+ def __init__(
+ self,
+ dep_mul,
+ wid_mul,
+ out_features=("dark3", "dark4", "dark5"),
+ depthwise=False,
+ act="silu",
+ ):
+ super().__init__()
+ assert out_features, "please provide output features of Darknet"
+ self.out_features = out_features
+ Conv = DWConv if depthwise else BaseConv
+
+ base_channels = int(wid_mul * 64) # 64
+ base_depth = max(round(dep_mul * 3), 1) # 3
+
+ # stem
+ self.stem = Focus(3, base_channels, ksize=3, act=act)
+
+ # dark2
+ self.dark2 = nn.Sequential(
+ Conv(base_channels, base_channels * 2, 3, 2, act=act),
+ CSPLayer(
+ base_channels * 2,
+ base_channels * 2,
+ n=base_depth,
+ depthwise=depthwise,
+ act=act,
+ ),
+ )
+
+ # dark3
+ self.dark3 = nn.Sequential(
+ Conv(base_channels * 2, base_channels * 4, 3, 2, act=act),
+ CSPLayer(
+ base_channels * 4,
+ base_channels * 4,
+ n=base_depth * 3,
+ depthwise=depthwise,
+ act=act,
+ ),
+ )
+
+ # dark4
+ self.dark4 = nn.Sequential(
+ Conv(base_channels * 4, base_channels * 8, 3, 2, act=act),
+ CSPLayer(
+ base_channels * 8,
+ base_channels * 8,
+ n=base_depth * 3,
+ depthwise=depthwise,
+ act=act,
+ ),
+ )
+
+ # dark5
+ self.dark5 = nn.Sequential(
+ Conv(base_channels * 8, base_channels * 16, 3, 2, act=act),
+ SPPBottleneck(base_channels * 16, base_channels * 16, activation=act),
+ CSPLayer(
+ base_channels * 16,
+ base_channels * 16,
+ n=base_depth,
+ shortcut=False,
+ depthwise=depthwise,
+ act=act,
+ ),
+ )
+
+ def forward(self, x):
+ outputs = {}
+ x = self.stem(x)
+ outputs["stem"] = x
+ x = self.dark2(x)
+ outputs["dark2"] = x
+ x = self.dark3(x)
+ outputs["dark3"] = x
+ x = self.dark4(x)
+ outputs["dark4"] = x
+ x = self.dark5(x)
+ outputs["dark5"] = x
+ return {k: v for k, v in outputs.items() if k in self.out_features}
diff --git a/experimental/yolox/models/losses.py b/experimental/yolox/models/losses.py
new file mode 100644
index 0000000..a789eba
--- /dev/null
+++ b/experimental/yolox/models/losses.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class IOUloss(nn.Module):
+ def __init__(self, reduction="none", loss_type="iou"):
+ super(IOUloss, self).__init__()
+ self.reduction = reduction
+ self.loss_type = loss_type
+
+ def forward(self, pred, target):
+ assert pred.shape[0] == target.shape[0]
+
+ pred = pred.view(-1, 4)
+ target = target.view(-1, 4)
+ tl = torch.max(
+ (pred[:, :2] - pred[:, 2:] / 2), (target[:, :2] - target[:, 2:] / 2)
+ )
+ br = torch.min(
+ (pred[:, :2] + pred[:, 2:] / 2), (target[:, :2] + target[:, 2:] / 2)
+ )
+
+ area_p = torch.prod(pred[:, 2:], 1)
+ area_g = torch.prod(target[:, 2:], 1)
+
+ en = (tl < br).type(tl.type()).prod(dim=1)
+ area_i = torch.prod(br - tl, 1) * en
+ iou = (area_i) / (area_p + area_g - area_i + 1e-16)
+
+ if self.loss_type == "iou":
+ loss = 1 - iou ** 2
+ elif self.loss_type == "giou":
+ c_tl = torch.min(
+ (pred[:, :2] - pred[:, 2:] / 2), (target[:, :2] - target[:, 2:] / 2)
+ )
+ c_br = torch.max(
+ (pred[:, :2] + pred[:, 2:] / 2), (target[:, :2] + target[:, 2:] / 2)
+ )
+ area_c = torch.prod(c_br - c_tl, 1)
+ giou = iou - (area_c - area_i) / area_c.clamp(1e-16)
+ loss = 1 - giou.clamp(min=-1.0, max=1.0)
+
+ if self.reduction == "mean":
+ loss = loss.mean()
+ elif self.reduction == "sum":
+ loss = loss.sum()
+
+ return loss
+
+
+def sigmoid_focal_loss(inputs, targets, num_boxes, alpha: float = 0.25, gamma: float = 2):
+ """
+ Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002.
+ Args:
+ inputs: A float tensor of arbitrary shape.
+ The predictions for each example.
+ targets: A float tensor with the same shape as inputs. Stores the binary
+ classification label for each element in inputs
+ (0 for the negative class and 1 for the positive class).
+ alpha: (optional) Weighting factor in range (0,1) to balance
+ positive vs negative examples. Default = -1 (no weighting).
+ gamma: Exponent of the modulating factor (1 - p_t) to
+ balance easy vs hard examples.
+ Returns:
+ Loss tensor
+ """
+ prob = inputs.sigmoid()
+ ce_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction="none")
+ p_t = prob * targets + (1 - prob) * (1 - targets)
+ loss = ce_loss * ((1 - p_t) ** gamma)
+
+ if alpha >= 0:
+ alpha_t = alpha * targets + (1 - alpha) * (1 - targets)
+ loss = alpha_t * loss
+ #return loss.mean(0).sum() / num_boxes
+ return loss.sum() / num_boxes
\ No newline at end of file
diff --git a/experimental/yolox/models/network_blocks.py b/experimental/yolox/models/network_blocks.py
new file mode 100644
index 0000000..4bdb2ca
--- /dev/null
+++ b/experimental/yolox/models/network_blocks.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import torch
+import torch.nn as nn
+
+
+class SiLU(nn.Module):
+ """export-friendly version of nn.SiLU()"""
+
+ @staticmethod
+ def forward(x):
+ return x * torch.sigmoid(x)
+
+
+def get_activation(name="silu", inplace=True):
+ if name == "silu":
+ module = nn.SiLU(inplace=inplace)
+ elif name == "relu":
+ module = nn.ReLU(inplace=inplace)
+ elif name == "lrelu":
+ module = nn.LeakyReLU(0.1, inplace=inplace)
+ else:
+ raise AttributeError("Unsupported act type: {}".format(name))
+ return module
+
+
+class BaseConv(nn.Module):
+ """A Conv2d -> Batchnorm -> silu/leaky relu block"""
+
+ def __init__(
+ self, in_channels, out_channels, ksize, stride, groups=1, bias=False, act="silu"
+ ):
+ super().__init__()
+ # same padding
+ pad = (ksize - 1) // 2
+ self.conv = nn.Conv2d(
+ in_channels,
+ out_channels,
+ kernel_size=ksize,
+ stride=stride,
+ padding=pad,
+ groups=groups,
+ bias=bias,
+ )
+ self.bn = nn.BatchNorm2d(out_channels)
+ self.act = get_activation(act, inplace=True)
+
+ def forward(self, x):
+ return self.act(self.bn(self.conv(x)))
+
+ def fuseforward(self, x):
+ return self.act(self.conv(x))
+
+
+class DWConv(nn.Module):
+ """Depthwise Conv + Conv"""
+
+ def __init__(self, in_channels, out_channels, ksize, stride=1, act="silu"):
+ super().__init__()
+ self.dconv = BaseConv(
+ in_channels,
+ in_channels,
+ ksize=ksize,
+ stride=stride,
+ groups=in_channels,
+ act=act,
+ )
+ self.pconv = BaseConv(
+ in_channels, out_channels, ksize=1, stride=1, groups=1, act=act
+ )
+
+ def forward(self, x):
+ x = self.dconv(x)
+ return self.pconv(x)
+
+
+class Bottleneck(nn.Module):
+ # Standard bottleneck
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ shortcut=True,
+ expansion=0.5,
+ depthwise=False,
+ act="silu",
+ ):
+ super().__init__()
+ hidden_channels = int(out_channels * expansion)
+ Conv = DWConv if depthwise else BaseConv
+ self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act)
+ self.conv2 = Conv(hidden_channels, out_channels, 3, stride=1, act=act)
+ self.use_add = shortcut and in_channels == out_channels
+
+ def forward(self, x):
+ y = self.conv2(self.conv1(x))
+ if self.use_add:
+ y = y + x
+ return y
+
+
+class ResLayer(nn.Module):
+ "Residual layer with `in_channels` inputs."
+
+ def __init__(self, in_channels: int):
+ super().__init__()
+ mid_channels = in_channels // 2
+ self.layer1 = BaseConv(
+ in_channels, mid_channels, ksize=1, stride=1, act="lrelu"
+ )
+ self.layer2 = BaseConv(
+ mid_channels, in_channels, ksize=3, stride=1, act="lrelu"
+ )
+
+ def forward(self, x):
+ out = self.layer2(self.layer1(x))
+ return x + out
+
+
+class SPPBottleneck(nn.Module):
+ """Spatial pyramid pooling layer used in YOLOv3-SPP"""
+
+ def __init__(
+ self, in_channels, out_channels, kernel_sizes=(5, 9, 13), activation="silu"
+ ):
+ super().__init__()
+ hidden_channels = in_channels // 2
+ self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=activation)
+ self.m = nn.ModuleList(
+ [
+ nn.MaxPool2d(kernel_size=ks, stride=1, padding=ks // 2)
+ for ks in kernel_sizes
+ ]
+ )
+ conv2_channels = hidden_channels * (len(kernel_sizes) + 1)
+ self.conv2 = BaseConv(conv2_channels, out_channels, 1, stride=1, act=activation)
+
+ def forward(self, x):
+ x = self.conv1(x)
+ x = torch.cat([x] + [m(x) for m in self.m], dim=1)
+ x = self.conv2(x)
+ return x
+
+
+class CSPLayer(nn.Module):
+ """C3 in yolov5, CSP Bottleneck with 3 convolutions"""
+
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ n=1,
+ shortcut=True,
+ expansion=0.5,
+ depthwise=False,
+ act="silu",
+ ):
+ """
+ Args:
+ in_channels (int): input channels.
+ out_channels (int): output channels.
+ n (int): number of Bottlenecks. Default value: 1.
+ """
+ # ch_in, ch_out, number, shortcut, groups, expansion
+ super().__init__()
+ hidden_channels = int(out_channels * expansion) # hidden channels
+ self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act)
+ self.conv2 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act)
+ self.conv3 = BaseConv(2 * hidden_channels, out_channels, 1, stride=1, act=act)
+ module_list = [
+ Bottleneck(
+ hidden_channels, hidden_channels, shortcut, 1.0, depthwise, act=act
+ )
+ for _ in range(n)
+ ]
+ self.m = nn.Sequential(*module_list)
+
+ def forward(self, x):
+ x_1 = self.conv1(x)
+ x_2 = self.conv2(x)
+ x_1 = self.m(x_1)
+ x = torch.cat((x_1, x_2), dim=1)
+ return self.conv3(x)
+
+
+class Focus(nn.Module):
+ """Focus width and height information into channel space."""
+
+ def __init__(self, in_channels, out_channels, ksize=1, stride=1, act="silu"):
+ super().__init__()
+ self.conv = BaseConv(in_channels * 4, out_channels, ksize, stride, act=act)
+
+ def forward(self, x):
+ # shape of x (b,c,w,h) -> y(b,4c,w/2,h/2)
+ patch_top_left = x[..., ::2, ::2]
+ patch_top_right = x[..., ::2, 1::2]
+ patch_bot_left = x[..., 1::2, ::2]
+ patch_bot_right = x[..., 1::2, 1::2]
+ x = torch.cat(
+ (
+ patch_top_left,
+ patch_bot_left,
+ patch_top_right,
+ patch_bot_right,
+ ),
+ dim=1,
+ )
+ return self.conv(x)
diff --git a/experimental/yolox/models/yolo_fpn.py b/experimental/yolox/models/yolo_fpn.py
new file mode 100644
index 0000000..8b3ba14
--- /dev/null
+++ b/experimental/yolox/models/yolo_fpn.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import torch
+import torch.nn as nn
+
+from .darknet import Darknet
+from .network_blocks import BaseConv
+
+
+class YOLOFPN(nn.Module):
+ """
+ YOLOFPN module. Darknet 53 is the default backbone of this model.
+ """
+
+ def __init__(
+ self,
+ depth=53,
+ in_features=["dark3", "dark4", "dark5"],
+ ):
+ super().__init__()
+
+ self.backbone = Darknet(depth)
+ self.in_features = in_features
+
+ # out 1
+ self.out1_cbl = self._make_cbl(512, 256, 1)
+ self.out1 = self._make_embedding([256, 512], 512 + 256)
+
+ # out 2
+ self.out2_cbl = self._make_cbl(256, 128, 1)
+ self.out2 = self._make_embedding([128, 256], 256 + 128)
+
+ # upsample
+ self.upsample = nn.Upsample(scale_factor=2, mode="nearest")
+
+ def _make_cbl(self, _in, _out, ks):
+ return BaseConv(_in, _out, ks, stride=1, act="lrelu")
+
+ def _make_embedding(self, filters_list, in_filters):
+ m = nn.Sequential(
+ *[
+ self._make_cbl(in_filters, filters_list[0], 1),
+ self._make_cbl(filters_list[0], filters_list[1], 3),
+ self._make_cbl(filters_list[1], filters_list[0], 1),
+ self._make_cbl(filters_list[0], filters_list[1], 3),
+ self._make_cbl(filters_list[1], filters_list[0], 1),
+ ]
+ )
+ return m
+
+ def load_pretrained_model(self, filename="./weights/darknet53.mix.pth"):
+ with open(filename, "rb") as f:
+ state_dict = torch.load(f, map_location="cpu")
+ print("loading pretrained weights...")
+ self.backbone.load_state_dict(state_dict)
+
+ def forward(self, inputs):
+ """
+ Args:
+ inputs (Tensor): input image.
+
+ Returns:
+ Tuple[Tensor]: FPN output features..
+ """
+ # backbone
+ out_features = self.backbone(inputs)
+ x2, x1, x0 = [out_features[f] for f in self.in_features]
+
+ # yolo branch 1
+ x1_in = self.out1_cbl(x0)
+ x1_in = self.upsample(x1_in)
+ x1_in = torch.cat([x1_in, x1], 1)
+ out_dark4 = self.out1(x1_in)
+
+ # yolo branch 2
+ x2_in = self.out2_cbl(out_dark4)
+ x2_in = self.upsample(x2_in)
+ x2_in = torch.cat([x2_in, x2], 1)
+ out_dark3 = self.out2(x2_in)
+
+ outputs = (out_dark3, out_dark4, x0)
+ return outputs
diff --git a/experimental/yolox/models/yolo_head.py b/experimental/yolox/models/yolo_head.py
new file mode 100644
index 0000000..ba8238f
--- /dev/null
+++ b/experimental/yolox/models/yolo_head.py
@@ -0,0 +1,660 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+from loguru import logger
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from yolox.utils import bboxes_iou
+
+import math
+
+from .losses import IOUloss
+from .network_blocks import BaseConv, DWConv
+
+
+class YOLOXHead(nn.Module):
+ def __init__(
+ self,
+ num_classes,
+ width=1.0,
+ strides=[8, 16, 32],
+ in_channels=[256, 512, 1024],
+ act="silu",
+ depthwise=False,
+ ):
+ """
+ Args:
+ act (str): activation type of conv. Defalut value: "silu".
+ depthwise (bool): wheather apply depthwise conv in conv branch. Defalut value: False.
+ """
+ super().__init__()
+
+ self.n_anchors = 1
+ self.num_classes = num_classes
+ self.decode_in_inference = True # for deploy, set to False
+
+ self.cls_convs = nn.ModuleList()
+ self.reg_convs = nn.ModuleList()
+ self.cls_preds = nn.ModuleList()
+ self.reg_preds = nn.ModuleList()
+ self.obj_preds = nn.ModuleList()
+ self.stems = nn.ModuleList()
+ Conv = DWConv if depthwise else BaseConv
+
+ for i in range(len(in_channels)):
+ self.stems.append(
+ BaseConv(
+ in_channels=int(in_channels[i] * width),
+ out_channels=int(256 * width),
+ ksize=1,
+ stride=1,
+ act=act,
+ )
+ )
+ self.cls_convs.append(
+ nn.Sequential(
+ *[
+ Conv(
+ in_channels=int(256 * width),
+ out_channels=int(256 * width),
+ ksize=3,
+ stride=1,
+ act=act,
+ ),
+ Conv(
+ in_channels=int(256 * width),
+ out_channels=int(256 * width),
+ ksize=3,
+ stride=1,
+ act=act,
+ ),
+ ]
+ )
+ )
+ self.reg_convs.append(
+ nn.Sequential(
+ *[
+ Conv(
+ in_channels=int(256 * width),
+ out_channels=int(256 * width),
+ ksize=3,
+ stride=1,
+ act=act,
+ ),
+ Conv(
+ in_channels=int(256 * width),
+ out_channels=int(256 * width),
+ ksize=3,
+ stride=1,
+ act=act,
+ ),
+ ]
+ )
+ )
+ self.cls_preds.append(
+ nn.Conv2d(
+ in_channels=int(256 * width),
+ out_channels=self.n_anchors * self.num_classes,
+ kernel_size=1,
+ stride=1,
+ padding=0,
+ )
+ )
+ self.reg_preds.append(
+ nn.Conv2d(
+ in_channels=int(256 * width),
+ out_channels=4,
+ kernel_size=1,
+ stride=1,
+ padding=0,
+ )
+ )
+ self.obj_preds.append(
+ nn.Conv2d(
+ in_channels=int(256 * width),
+ out_channels=self.n_anchors * 1,
+ kernel_size=1,
+ stride=1,
+ padding=0,
+ )
+ )
+
+ self.use_l1 = False
+ self.l1_loss = nn.L1Loss(reduction="none")
+ self.bcewithlog_loss = nn.BCEWithLogitsLoss(reduction="none")
+ self.iou_loss = IOUloss(reduction="none")
+ self.strides = strides
+ self.grids = [torch.zeros(1)] * len(in_channels)
+ self.expanded_strides = [None] * len(in_channels)
+
+ def initialize_biases(self, prior_prob):
+ for conv in self.cls_preds:
+ b = conv.bias.view(self.n_anchors, -1)
+ b.data.fill_(-math.log((1 - prior_prob) / prior_prob))
+ conv.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)
+
+ for conv in self.obj_preds:
+ b = conv.bias.view(self.n_anchors, -1)
+ b.data.fill_(-math.log((1 - prior_prob) / prior_prob))
+ conv.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)
+
+ def forward(self, xin, labels=None, imgs=None):
+ outputs = []
+ origin_preds = []
+ x_shifts = []
+ y_shifts = []
+ expanded_strides = []
+
+ for k, (cls_conv, reg_conv, stride_this_level, x) in enumerate(
+ zip(self.cls_convs, self.reg_convs, self.strides, xin)
+ ):
+ x = self.stems[k](x)
+ cls_x = x
+ reg_x = x
+
+ cls_feat = cls_conv(cls_x)
+ cls_output = self.cls_preds[k](cls_feat)
+
+ reg_feat = reg_conv(reg_x)
+ reg_output = self.reg_preds[k](reg_feat)
+ obj_output = self.obj_preds[k](reg_feat)
+
+ if self.training:
+ output = torch.cat([reg_output, obj_output, cls_output], 1)
+ output, grid = self.get_output_and_grid(
+ output, k, stride_this_level, xin[0].type()
+ )
+ x_shifts.append(grid[:, :, 0])
+ y_shifts.append(grid[:, :, 1])
+ expanded_strides.append(
+ torch.zeros(1, grid.shape[1])
+ .fill_(stride_this_level)
+ .type_as(xin[0])
+ )
+ if self.use_l1:
+ batch_size = reg_output.shape[0]
+ hsize, wsize = reg_output.shape[-2:]
+ reg_output = reg_output.view(
+ batch_size, self.n_anchors, 4, hsize, wsize
+ )
+ reg_output = reg_output.permute(0, 1, 3, 4, 2).reshape(
+ batch_size, -1, 4
+ )
+ origin_preds.append(reg_output.clone())
+
+ else:
+ output = torch.cat(
+ [reg_output, obj_output.sigmoid(), cls_output.sigmoid()], 1
+ )
+
+ outputs.append(output)
+
+ if self.training:
+ return self.get_losses(
+ imgs,
+ x_shifts,
+ y_shifts,
+ expanded_strides,
+ labels,
+ torch.cat(outputs, 1),
+ origin_preds,
+ dtype=xin[0].dtype,
+ )
+ else:
+ self.hw = [x.shape[-2:] for x in outputs]
+ # [batch, n_anchors_all, 85]
+ outputs = torch.cat(
+ [x.flatten(start_dim=2) for x in outputs], dim=2
+ ).permute(0, 2, 1)
+ if self.decode_in_inference:
+ return self.decode_outputs(outputs, dtype=xin[0].type())
+ else:
+ return outputs
+
+ def get_output_and_grid(self, output, k, stride, dtype):
+ grid = self.grids[k]
+
+ batch_size = output.shape[0]
+ n_ch = 5 + self.num_classes
+ hsize, wsize = output.shape[-2:]
+ if grid.shape[2:4] != output.shape[2:4]:
+ yv, xv = torch.meshgrid([torch.arange(hsize), torch.arange(wsize)])
+ grid = torch.stack((xv, yv), 2).view(1, 1, hsize, wsize, 2).type(dtype)
+ self.grids[k] = grid
+
+ output = output.view(batch_size, self.n_anchors, n_ch, hsize, wsize)
+ output = output.permute(0, 1, 3, 4, 2).reshape(
+ batch_size, self.n_anchors * hsize * wsize, -1
+ )
+ grid = grid.view(1, -1, 2)
+ output[..., :2] = (output[..., :2] + grid) * stride
+ output[..., 2:4] = torch.exp(output[..., 2:4]) * stride
+ return output, grid
+
+ def decode_outputs(self, outputs, dtype):
+ grids = []
+ strides = []
+ for (hsize, wsize), stride in zip(self.hw, self.strides):
+ yv, xv = torch.meshgrid([torch.arange(hsize), torch.arange(wsize)])
+ grid = torch.stack((xv, yv), 2).view(1, -1, 2)
+ grids.append(grid)
+ shape = grid.shape[:2]
+ strides.append(torch.full((*shape, 1), stride))
+
+ grids = torch.cat(grids, dim=1).type(dtype)
+ strides = torch.cat(strides, dim=1).type(dtype)
+
+ outputs[..., :2] = (outputs[..., :2] + grids) * strides
+ outputs[..., 2:4] = torch.exp(outputs[..., 2:4]) * strides
+ return outputs
+
+ def get_losses(
+ self,
+ imgs,
+ x_shifts,
+ y_shifts,
+ expanded_strides,
+ labels,
+ outputs,
+ origin_preds,
+ dtype,
+ ):
+ bbox_preds = outputs[:, :, :4] # [batch, n_anchors_all, 4]
+ obj_preds = outputs[:, :, 4].unsqueeze(-1) # [batch, n_anchors_all, 1]
+ cls_preds = outputs[:, :, 5:] # [batch, n_anchors_all, n_cls]
+
+ # calculate targets
+ mixup = labels.shape[2] > 5
+ if mixup:
+ label_cut = labels[..., :5]
+ else:
+ label_cut = labels
+ nlabel = (label_cut.sum(dim=2) > 0).sum(dim=1) # number of objects
+
+ total_num_anchors = outputs.shape[1]
+ x_shifts = torch.cat(x_shifts, 1) # [1, n_anchors_all]
+ y_shifts = torch.cat(y_shifts, 1) # [1, n_anchors_all]
+ expanded_strides = torch.cat(expanded_strides, 1)
+ if self.use_l1:
+ origin_preds = torch.cat(origin_preds, 1)
+
+ cls_targets = []
+ reg_targets = []
+ l1_targets = []
+ obj_targets = []
+ fg_masks = []
+
+ num_fg = 0.0
+ num_gts = 0.0
+
+ for batch_idx in range(outputs.shape[0]):
+ num_gt = int(nlabel[batch_idx])
+ num_gts += num_gt
+ if num_gt == 0:
+ cls_target = outputs.new_zeros((0, self.num_classes))
+ reg_target = outputs.new_zeros((0, 4))
+ l1_target = outputs.new_zeros((0, 4))
+ obj_target = outputs.new_zeros((total_num_anchors, 1))
+ fg_mask = outputs.new_zeros(total_num_anchors).bool()
+ else:
+ gt_bboxes_per_image = labels[batch_idx, :num_gt, 1:5]
+ gt_classes = labels[batch_idx, :num_gt, 0]
+ bboxes_preds_per_image = bbox_preds[batch_idx]
+
+ try:
+ (
+ gt_matched_classes,
+ fg_mask,
+ pred_ious_this_matching,
+ matched_gt_inds,
+ num_fg_img,
+ ) = self.get_assignments( # noqa
+ batch_idx,
+ num_gt,
+ total_num_anchors,
+ gt_bboxes_per_image,
+ gt_classes,
+ bboxes_preds_per_image,
+ expanded_strides,
+ x_shifts,
+ y_shifts,
+ cls_preds,
+ bbox_preds,
+ obj_preds,
+ labels,
+ imgs,
+ )
+ except RuntimeError:
+ logger.info(
+ "OOM RuntimeError is raised due to the huge memory cost during label assignment. \
+ CPU mode is applied in this batch. If you want to avoid this issue, \
+ try to reduce the batch size or image size."
+ )
+ print("OOM RuntimeError is raised due to the huge memory cost during label assignment. \
+ CPU mode is applied in this batch. If you want to avoid this issue, \
+ try to reduce the batch size or image size.")
+ torch.cuda.empty_cache()
+ (
+ gt_matched_classes,
+ fg_mask,
+ pred_ious_this_matching,
+ matched_gt_inds,
+ num_fg_img,
+ ) = self.get_assignments( # noqa
+ batch_idx,
+ num_gt,
+ total_num_anchors,
+ gt_bboxes_per_image,
+ gt_classes,
+ bboxes_preds_per_image,
+ expanded_strides,
+ x_shifts,
+ y_shifts,
+ cls_preds,
+ bbox_preds,
+ obj_preds,
+ labels,
+ imgs,
+ "cpu",
+ )
+
+
+ torch.cuda.empty_cache()
+ num_fg += num_fg_img
+
+ cls_target = F.one_hot(
+ gt_matched_classes.to(torch.int64), self.num_classes
+ ) * pred_ious_this_matching.unsqueeze(-1)
+ obj_target = fg_mask.unsqueeze(-1)
+ reg_target = gt_bboxes_per_image[matched_gt_inds]
+
+ if self.use_l1:
+ l1_target = self.get_l1_target(
+ outputs.new_zeros((num_fg_img, 4)),
+ gt_bboxes_per_image[matched_gt_inds],
+ expanded_strides[0][fg_mask],
+ x_shifts=x_shifts[0][fg_mask],
+ y_shifts=y_shifts[0][fg_mask],
+ )
+
+ cls_targets.append(cls_target)
+ reg_targets.append(reg_target)
+ obj_targets.append(obj_target.to(dtype))
+ fg_masks.append(fg_mask)
+ if self.use_l1:
+ l1_targets.append(l1_target)
+
+ cls_targets = torch.cat(cls_targets, 0)
+ reg_targets = torch.cat(reg_targets, 0)
+ obj_targets = torch.cat(obj_targets, 0)
+ fg_masks = torch.cat(fg_masks, 0)
+ if self.use_l1:
+ l1_targets = torch.cat(l1_targets, 0)
+
+ num_fg = max(num_fg, 1)
+ loss_iou = (
+ self.iou_loss(bbox_preds.view(-1, 4)[fg_masks], reg_targets)
+ ).sum() / num_fg
+ loss_obj = (
+ self.bcewithlog_loss(obj_preds.view(-1, 1), obj_targets)
+ ).sum() / num_fg
+ loss_cls = (
+ self.bcewithlog_loss(
+ cls_preds.view(-1, self.num_classes)[fg_masks], cls_targets
+ )
+ ).sum() / num_fg
+ if self.use_l1:
+ loss_l1 = (
+ self.l1_loss(origin_preds.view(-1, 4)[fg_masks], l1_targets)
+ ).sum() / num_fg
+ else:
+ loss_l1 = 0.0
+
+ reg_weight = 5.0
+ loss = reg_weight * loss_iou + loss_obj + loss_cls + loss_l1
+
+ return (
+ loss,
+ reg_weight * loss_iou,
+ loss_obj,
+ loss_cls,
+ loss_l1,
+ num_fg / max(num_gts, 1),
+ )
+
+ def get_l1_target(self, l1_target, gt, stride, x_shifts, y_shifts, eps=1e-8):
+ l1_target[:, 0] = gt[:, 0] / stride - x_shifts
+ l1_target[:, 1] = gt[:, 1] / stride - y_shifts
+ l1_target[:, 2] = torch.log(gt[:, 2] / stride + eps)
+ l1_target[:, 3] = torch.log(gt[:, 3] / stride + eps)
+ return l1_target
+
+ @torch.no_grad()
+ def get_assignments(
+ self,
+ batch_idx,
+ num_gt,
+ total_num_anchors,
+ gt_bboxes_per_image,
+ gt_classes,
+ bboxes_preds_per_image,
+ expanded_strides,
+ x_shifts,
+ y_shifts,
+ cls_preds,
+ bbox_preds,
+ obj_preds,
+ labels,
+ imgs,
+ mode="gpu",
+ ):
+
+ if mode == "cpu":
+ print("------------CPU Mode for This Batch-------------")
+ gt_bboxes_per_image = gt_bboxes_per_image.cpu().float()
+ bboxes_preds_per_image = bboxes_preds_per_image.cpu().float()
+ gt_classes = gt_classes.cpu().float()
+ expanded_strides = expanded_strides.cpu().float()
+ x_shifts = x_shifts.cpu()
+ y_shifts = y_shifts.cpu()
+
+ img_size = imgs.shape[2:]
+ fg_mask, is_in_boxes_and_center = self.get_in_boxes_info(
+ gt_bboxes_per_image,
+ expanded_strides,
+ x_shifts,
+ y_shifts,
+ total_num_anchors,
+ num_gt,
+ img_size
+ )
+
+ bboxes_preds_per_image = bboxes_preds_per_image[fg_mask]
+ cls_preds_ = cls_preds[batch_idx][fg_mask]
+ obj_preds_ = obj_preds[batch_idx][fg_mask]
+ num_in_boxes_anchor = bboxes_preds_per_image.shape[0]
+
+ if mode == "cpu":
+ gt_bboxes_per_image = gt_bboxes_per_image.cpu()
+ bboxes_preds_per_image = bboxes_preds_per_image.cpu()
+
+ pair_wise_ious = bboxes_iou(gt_bboxes_per_image, bboxes_preds_per_image, False)
+
+ gt_cls_per_image = (
+ F.one_hot(gt_classes.to(torch.int64), self.num_classes)
+ .float()
+ .unsqueeze(1)
+ .repeat(1, num_in_boxes_anchor, 1)
+ )
+ pair_wise_ious_loss = -torch.log(pair_wise_ious + 1e-8)
+
+ if mode == "cpu":
+ cls_preds_, obj_preds_ = cls_preds_.cpu(), obj_preds_.cpu()
+
+ with torch.cuda.amp.autocast(enabled=False):
+ cls_preds_ = (
+ cls_preds_.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()
+ * obj_preds_.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()
+ )
+ pair_wise_cls_loss = F.binary_cross_entropy(
+ cls_preds_.sqrt_(), gt_cls_per_image, reduction="none"
+ ).sum(-1)
+ del cls_preds_
+
+ cost = (
+ pair_wise_cls_loss
+ + 3.0 * pair_wise_ious_loss
+ + 100000.0 * (~is_in_boxes_and_center)
+ )
+
+ (
+ num_fg,
+ gt_matched_classes,
+ pred_ious_this_matching,
+ matched_gt_inds,
+ ) = self.dynamic_k_matching(cost, pair_wise_ious, gt_classes, num_gt, fg_mask)
+ del pair_wise_cls_loss, cost, pair_wise_ious, pair_wise_ious_loss
+
+ if mode == "cpu":
+ gt_matched_classes = gt_matched_classes.cuda()
+ fg_mask = fg_mask.cuda()
+ pred_ious_this_matching = pred_ious_this_matching.cuda()
+ matched_gt_inds = matched_gt_inds.cuda()
+
+ return (
+ gt_matched_classes,
+ fg_mask,
+ pred_ious_this_matching,
+ matched_gt_inds,
+ num_fg,
+ )
+
+ def get_in_boxes_info(
+ self,
+ gt_bboxes_per_image,
+ expanded_strides,
+ x_shifts,
+ y_shifts,
+ total_num_anchors,
+ num_gt,
+ img_size
+ ):
+ expanded_strides_per_image = expanded_strides[0]
+ x_shifts_per_image = x_shifts[0] * expanded_strides_per_image
+ y_shifts_per_image = y_shifts[0] * expanded_strides_per_image
+ x_centers_per_image = (
+ (x_shifts_per_image + 0.5 * expanded_strides_per_image)
+ .unsqueeze(0)
+ .repeat(num_gt, 1)
+ ) # [n_anchor] -> [n_gt, n_anchor]
+ y_centers_per_image = (
+ (y_shifts_per_image + 0.5 * expanded_strides_per_image)
+ .unsqueeze(0)
+ .repeat(num_gt, 1)
+ )
+
+ gt_bboxes_per_image_l = (
+ (gt_bboxes_per_image[:, 0] - 0.5 * gt_bboxes_per_image[:, 2])
+ .unsqueeze(1)
+ .repeat(1, total_num_anchors)
+ )
+ gt_bboxes_per_image_r = (
+ (gt_bboxes_per_image[:, 0] + 0.5 * gt_bboxes_per_image[:, 2])
+ .unsqueeze(1)
+ .repeat(1, total_num_anchors)
+ )
+ gt_bboxes_per_image_t = (
+ (gt_bboxes_per_image[:, 1] - 0.5 * gt_bboxes_per_image[:, 3])
+ .unsqueeze(1)
+ .repeat(1, total_num_anchors)
+ )
+ gt_bboxes_per_image_b = (
+ (gt_bboxes_per_image[:, 1] + 0.5 * gt_bboxes_per_image[:, 3])
+ .unsqueeze(1)
+ .repeat(1, total_num_anchors)
+ )
+
+ b_l = x_centers_per_image - gt_bboxes_per_image_l
+ b_r = gt_bboxes_per_image_r - x_centers_per_image
+ b_t = y_centers_per_image - gt_bboxes_per_image_t
+ b_b = gt_bboxes_per_image_b - y_centers_per_image
+ bbox_deltas = torch.stack([b_l, b_t, b_r, b_b], 2)
+
+ is_in_boxes = bbox_deltas.min(dim=-1).values > 0.0
+ is_in_boxes_all = is_in_boxes.sum(dim=0) > 0
+ # in fixed center
+
+ center_radius = 2.5
+ # clip center inside image
+ gt_bboxes_per_image_clip = gt_bboxes_per_image[:, 0:2].clone()
+ gt_bboxes_per_image_clip[:, 0] = torch.clamp(gt_bboxes_per_image_clip[:, 0], min=0, max=img_size[1])
+ gt_bboxes_per_image_clip[:, 1] = torch.clamp(gt_bboxes_per_image_clip[:, 1], min=0, max=img_size[0])
+
+ gt_bboxes_per_image_l = (gt_bboxes_per_image_clip[:, 0]).unsqueeze(1).repeat(
+ 1, total_num_anchors
+ ) - center_radius * expanded_strides_per_image.unsqueeze(0)
+ gt_bboxes_per_image_r = (gt_bboxes_per_image_clip[:, 0]).unsqueeze(1).repeat(
+ 1, total_num_anchors
+ ) + center_radius * expanded_strides_per_image.unsqueeze(0)
+ gt_bboxes_per_image_t = (gt_bboxes_per_image_clip[:, 1]).unsqueeze(1).repeat(
+ 1, total_num_anchors
+ ) - center_radius * expanded_strides_per_image.unsqueeze(0)
+ gt_bboxes_per_image_b = (gt_bboxes_per_image_clip[:, 1]).unsqueeze(1).repeat(
+ 1, total_num_anchors
+ ) + center_radius * expanded_strides_per_image.unsqueeze(0)
+
+ c_l = x_centers_per_image - gt_bboxes_per_image_l
+ c_r = gt_bboxes_per_image_r - x_centers_per_image
+ c_t = y_centers_per_image - gt_bboxes_per_image_t
+ c_b = gt_bboxes_per_image_b - y_centers_per_image
+ center_deltas = torch.stack([c_l, c_t, c_r, c_b], 2)
+ is_in_centers = center_deltas.min(dim=-1).values > 0.0
+ is_in_centers_all = is_in_centers.sum(dim=0) > 0
+
+ # in boxes and in centers
+ is_in_boxes_anchor = is_in_boxes_all | is_in_centers_all
+
+ is_in_boxes_and_center = (
+ is_in_boxes[:, is_in_boxes_anchor] & is_in_centers[:, is_in_boxes_anchor]
+ )
+ del gt_bboxes_per_image_clip
+ return is_in_boxes_anchor, is_in_boxes_and_center
+
+ def dynamic_k_matching(self, cost, pair_wise_ious, gt_classes, num_gt, fg_mask):
+ # Dynamic K
+ # ---------------------------------------------------------------
+ matching_matrix = torch.zeros_like(cost)
+
+ ious_in_boxes_matrix = pair_wise_ious
+ n_candidate_k = min(10, ious_in_boxes_matrix.size(1))
+ topk_ious, _ = torch.topk(ious_in_boxes_matrix, n_candidate_k, dim=1)
+ dynamic_ks = torch.clamp(topk_ious.sum(1).int(), min=1)
+ for gt_idx in range(num_gt):
+ _, pos_idx = torch.topk(
+ cost[gt_idx], k=dynamic_ks[gt_idx].item(), largest=False
+ )
+ matching_matrix[gt_idx][pos_idx] = 1.0
+
+ del topk_ious, dynamic_ks, pos_idx
+
+ anchor_matching_gt = matching_matrix.sum(0)
+ if (anchor_matching_gt > 1).sum() > 0:
+ cost_min, cost_argmin = torch.min(cost[:, anchor_matching_gt > 1], dim=0)
+ matching_matrix[:, anchor_matching_gt > 1] *= 0.0
+ matching_matrix[cost_argmin, anchor_matching_gt > 1] = 1.0
+ fg_mask_inboxes = matching_matrix.sum(0) > 0.0
+ num_fg = fg_mask_inboxes.sum().item()
+
+ fg_mask[fg_mask.clone()] = fg_mask_inboxes
+
+ matched_gt_inds = matching_matrix[:, fg_mask_inboxes].argmax(0)
+ gt_matched_classes = gt_classes[matched_gt_inds]
+
+ pred_ious_this_matching = (matching_matrix * pair_wise_ious).sum(0)[
+ fg_mask_inboxes
+ ]
+ return num_fg, gt_matched_classes, pred_ious_this_matching, matched_gt_inds
diff --git a/experimental/yolox/models/yolo_pafpn.py b/experimental/yolox/models/yolo_pafpn.py
new file mode 100644
index 0000000..c419de3
--- /dev/null
+++ b/experimental/yolox/models/yolo_pafpn.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import torch
+import torch.nn as nn
+
+from .darknet import CSPDarknet
+from .network_blocks import BaseConv, CSPLayer, DWConv
+
+
+class YOLOPAFPN(nn.Module):
+ """
+ YOLOv3 model. Darknet 53 is the default backbone of this model.
+ """
+
+ def __init__(
+ self,
+ depth=1.0,
+ width=1.0,
+ in_features=("dark3", "dark4", "dark5"),
+ in_channels=[256, 512, 1024],
+ depthwise=False,
+ act="silu",
+ ):
+ super().__init__()
+ self.backbone = CSPDarknet(depth, width, depthwise=depthwise, act=act)
+ self.in_features = in_features
+ self.in_channels = in_channels
+ Conv = DWConv if depthwise else BaseConv
+
+ self.upsample = nn.Upsample(scale_factor=2, mode="nearest")
+ self.lateral_conv0 = BaseConv(
+ int(in_channels[2] * width), int(in_channels[1] * width), 1, 1, act=act
+ )
+ self.C3_p4 = CSPLayer(
+ int(2 * in_channels[1] * width),
+ int(in_channels[1] * width),
+ round(3 * depth),
+ False,
+ depthwise=depthwise,
+ act=act,
+ ) # cat
+
+ self.reduce_conv1 = BaseConv(
+ int(in_channels[1] * width), int(in_channels[0] * width), 1, 1, act=act
+ )
+ self.C3_p3 = CSPLayer(
+ int(2 * in_channels[0] * width),
+ int(in_channels[0] * width),
+ round(3 * depth),
+ False,
+ depthwise=depthwise,
+ act=act,
+ )
+
+ # bottom-up conv
+ self.bu_conv2 = Conv(
+ int(in_channels[0] * width), int(in_channels[0] * width), 3, 2, act=act
+ )
+ self.C3_n3 = CSPLayer(
+ int(2 * in_channels[0] * width),
+ int(in_channels[1] * width),
+ round(3 * depth),
+ False,
+ depthwise=depthwise,
+ act=act,
+ )
+
+ # bottom-up conv
+ self.bu_conv1 = Conv(
+ int(in_channels[1] * width), int(in_channels[1] * width), 3, 2, act=act
+ )
+ self.C3_n4 = CSPLayer(
+ int(2 * in_channels[1] * width),
+ int(in_channels[2] * width),
+ round(3 * depth),
+ False,
+ depthwise=depthwise,
+ act=act,
+ )
+
+ def forward(self, input):
+ """
+ Args:
+ inputs: input images.
+
+ Returns:
+ Tuple[Tensor]: FPN feature.
+ """
+
+ # backbone
+ out_features = self.backbone(input)
+ features = [out_features[f] for f in self.in_features]
+ [x2, x1, x0] = features
+
+ fpn_out0 = self.lateral_conv0(x0) # 1024->512/32
+ f_out0 = self.upsample(fpn_out0) # 512/16
+ f_out0 = torch.cat([f_out0, x1], 1) # 512->1024/16
+ f_out0 = self.C3_p4(f_out0) # 1024->512/16
+
+ fpn_out1 = self.reduce_conv1(f_out0) # 512->256/16
+ f_out1 = self.upsample(fpn_out1) # 256/8
+ f_out1 = torch.cat([f_out1, x2], 1) # 256->512/8
+ pan_out2 = self.C3_p3(f_out1) # 512->256/8
+
+ p_out1 = self.bu_conv2(pan_out2) # 256->256/16
+ p_out1 = torch.cat([p_out1, fpn_out1], 1) # 256->512/16
+ pan_out1 = self.C3_n3(p_out1) # 512->512/16
+
+ p_out0 = self.bu_conv1(pan_out1) # 512->512/32
+ p_out0 = torch.cat([p_out0, fpn_out0], 1) # 512->1024/32
+ pan_out0 = self.C3_n4(p_out0) # 1024->1024/32
+
+ outputs = (pan_out2, pan_out1, pan_out0)
+ return outputs
diff --git a/experimental/yolox/models/yolox.py b/experimental/yolox/models/yolox.py
new file mode 100644
index 0000000..2f1fa1b
--- /dev/null
+++ b/experimental/yolox/models/yolox.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import torch.nn as nn
+
+from .yolo_head import YOLOXHead
+from .yolo_pafpn import YOLOPAFPN
+
+
+class YOLOX(nn.Module):
+ """
+ YOLOX model module. The module list is defined by create_yolov3_modules function.
+ The network returns loss values from three YOLO layers during training
+ and detection results during test.
+ """
+
+ def __init__(self, backbone=None, head=None):
+ super().__init__()
+ if backbone is None:
+ backbone = YOLOPAFPN()
+ if head is None:
+ head = YOLOXHead(80)
+
+ self.backbone = backbone
+ self.head = head
+
+ def forward(self, x, targets=None):
+ # fpn output content features of [dark3, dark4, dark5]
+ fpn_outs = self.backbone(x)
+
+ if self.training:
+ assert targets is not None
+ loss, iou_loss, conf_loss, cls_loss, l1_loss, num_fg = self.head(
+ fpn_outs, targets, x
+ )
+ outputs = {
+ "total_loss": loss,
+ "iou_loss": iou_loss,
+ "l1_loss": l1_loss,
+ "conf_loss": conf_loss,
+ "cls_loss": cls_loss,
+ "num_fg": num_fg,
+ }
+ else:
+ outputs = self.head(fpn_outs)
+
+ return outputs
diff --git a/experimental/yolox/motdt_tracker/basetrack.py b/experimental/yolox/motdt_tracker/basetrack.py
new file mode 100644
index 0000000..88b16ee
--- /dev/null
+++ b/experimental/yolox/motdt_tracker/basetrack.py
@@ -0,0 +1,56 @@
+import numpy as np
+from collections import OrderedDict
+
+
+class TrackState(object):
+ New = 0
+ Tracked = 1
+ Lost = 2
+ Removed = 3
+ Replaced = 4
+
+
+class BaseTrack(object):
+ _count = 0
+
+ track_id = 0
+ is_activated = False
+ state = TrackState.New
+
+ history = OrderedDict()
+ features = []
+ curr_feature = None
+ score = 0
+ start_frame = 0
+ frame_id = 0
+ time_since_update = 0
+
+ # multi-camera
+ location = (np.inf, np.inf)
+
+ @property
+ def end_frame(self):
+ return self.frame_id
+
+ @staticmethod
+ def next_id():
+ BaseTrack._count += 1
+ return BaseTrack._count
+
+ def activate(self, *args):
+ raise NotImplementedError
+
+ def predict(self):
+ raise NotImplementedError
+
+ def update(self, *args, **kwargs):
+ raise NotImplementedError
+
+ def mark_lost(self):
+ self.state = TrackState.Lost
+
+ def mark_removed(self):
+ self.state = TrackState.Removed
+
+ def mark_replaced(self):
+ self.state = TrackState.Replaced
diff --git a/experimental/yolox/motdt_tracker/kalman_filter.py b/experimental/yolox/motdt_tracker/kalman_filter.py
new file mode 100644
index 0000000..deda8a2
--- /dev/null
+++ b/experimental/yolox/motdt_tracker/kalman_filter.py
@@ -0,0 +1,270 @@
+# vim: expandtab:ts=4:sw=4
+import numpy as np
+import scipy.linalg
+
+
+"""
+Table for the 0.95 quantile of the chi-square distribution with N degrees of
+freedom (contains values for N=1, ..., 9). Taken from MATLAB/Octave's chi2inv
+function and used as Mahalanobis gating threshold.
+"""
+chi2inv95 = {
+ 1: 3.8415,
+ 2: 5.9915,
+ 3: 7.8147,
+ 4: 9.4877,
+ 5: 11.070,
+ 6: 12.592,
+ 7: 14.067,
+ 8: 15.507,
+ 9: 16.919}
+
+
+class KalmanFilter(object):
+ """
+ A simple Kalman filter for tracking bounding boxes in image space.
+
+ The 8-dimensional state space
+
+ x, y, a, h, vx, vy, va, vh
+
+ contains the bounding box center position (x, y), aspect ratio a, height h,
+ and their respective velocities.
+
+ Object motion follows a constant velocity model. The bounding box location
+ (x, y, a, h) is taken as direct observation of the state space (linear
+ observation model).
+
+ """
+
+ def __init__(self):
+ ndim, dt = 4, 1.
+
+ # Create Kalman filter model matrices.
+ self._motion_mat = np.eye(2 * ndim, 2 * ndim)
+ for i in range(ndim):
+ self._motion_mat[i, ndim + i] = dt
+ self._update_mat = np.eye(ndim, 2 * ndim)
+
+ # Motion and observation uncertainty are chosen relative to the current
+ # state estimate. These weights control the amount of uncertainty in
+ # the model. This is a bit hacky.
+ self._std_weight_position = 1. / 20
+ self._std_weight_velocity = 1. / 160
+
+ def initiate(self, measurement):
+ """Create track from unassociated measurement.
+
+ Parameters
+ ----------
+ measurement : ndarray
+ Bounding box coordinates (x, y, a, h) with center position (x, y),
+ aspect ratio a, and height h.
+
+ Returns
+ -------
+ (ndarray, ndarray)
+ Returns the mean vector (8 dimensional) and covariance matrix (8x8
+ dimensional) of the new track. Unobserved velocities are initialized
+ to 0 mean.
+
+ """
+ mean_pos = measurement
+ mean_vel = np.zeros_like(mean_pos)
+ mean = np.r_[mean_pos, mean_vel]
+
+ std = [
+ 2 * self._std_weight_position * measurement[3],
+ 2 * self._std_weight_position * measurement[3],
+ 1e-2,
+ 2 * self._std_weight_position * measurement[3],
+ 10 * self._std_weight_velocity * measurement[3],
+ 10 * self._std_weight_velocity * measurement[3],
+ 1e-5,
+ 10 * self._std_weight_velocity * measurement[3]]
+ covariance = np.diag(np.square(std))
+ return mean, covariance
+
+ def predict(self, mean, covariance):
+ """Run Kalman filter prediction step.
+
+ Parameters
+ ----------
+ mean : ndarray
+ The 8 dimensional mean vector of the object state at the previous
+ time step.
+ covariance : ndarray
+ The 8x8 dimensional covariance matrix of the object state at the
+ previous time step.
+
+ Returns
+ -------
+ (ndarray, ndarray)
+ Returns the mean vector and covariance matrix of the predicted
+ state. Unobserved velocities are initialized to 0 mean.
+
+ """
+ std_pos = [
+ self._std_weight_position * mean[3],
+ self._std_weight_position * mean[3],
+ 1e-2,
+ self._std_weight_position * mean[3]]
+ std_vel = [
+ self._std_weight_velocity * mean[3],
+ self._std_weight_velocity * mean[3],
+ 1e-5,
+ self._std_weight_velocity * mean[3]]
+ motion_cov = np.diag(np.square(np.r_[std_pos, std_vel]))
+
+ #mean = np.dot(self._motion_mat, mean)
+ mean = np.dot(mean, self._motion_mat.T)
+ covariance = np.linalg.multi_dot((
+ self._motion_mat, covariance, self._motion_mat.T)) + motion_cov
+
+ return mean, covariance
+
+ def project(self, mean, covariance):
+ """Project state distribution to measurement space.
+
+ Parameters
+ ----------
+ mean : ndarray
+ The state's mean vector (8 dimensional array).
+ covariance : ndarray
+ The state's covariance matrix (8x8 dimensional).
+
+ Returns
+ -------
+ (ndarray, ndarray)
+ Returns the projected mean and covariance matrix of the given state
+ estimate.
+
+ """
+ std = [
+ self._std_weight_position * mean[3],
+ self._std_weight_position * mean[3],
+ 1e-1,
+ self._std_weight_position * mean[3]]
+ innovation_cov = np.diag(np.square(std))
+
+ mean = np.dot(self._update_mat, mean)
+ covariance = np.linalg.multi_dot((
+ self._update_mat, covariance, self._update_mat.T))
+ return mean, covariance + innovation_cov
+
+ def multi_predict(self, mean, covariance):
+ """Run Kalman filter prediction step (Vectorized version).
+ Parameters
+ ----------
+ mean : ndarray
+ The Nx8 dimensional mean matrix of the object states at the previous
+ time step.
+ covariance : ndarray
+ The Nx8x8 dimensional covariance matrics of the object states at the
+ previous time step.
+ Returns
+ -------
+ (ndarray, ndarray)
+ Returns the mean vector and covariance matrix of the predicted
+ state. Unobserved velocities are initialized to 0 mean.
+ """
+ std_pos = [
+ self._std_weight_position * mean[:, 3],
+ self._std_weight_position * mean[:, 3],
+ 1e-2 * np.ones_like(mean[:, 3]),
+ self._std_weight_position * mean[:, 3]]
+ std_vel = [
+ self._std_weight_velocity * mean[:, 3],
+ self._std_weight_velocity * mean[:, 3],
+ 1e-5 * np.ones_like(mean[:, 3]),
+ self._std_weight_velocity * mean[:, 3]]
+ sqr = np.square(np.r_[std_pos, std_vel]).T
+
+ motion_cov = []
+ for i in range(len(mean)):
+ motion_cov.append(np.diag(sqr[i]))
+ motion_cov = np.asarray(motion_cov)
+
+ mean = np.dot(mean, self._motion_mat.T)
+ left = np.dot(self._motion_mat, covariance).transpose((1, 0, 2))
+ covariance = np.dot(left, self._motion_mat.T) + motion_cov
+
+ return mean, covariance
+
+ def update(self, mean, covariance, measurement):
+ """Run Kalman filter correction step.
+
+ Parameters
+ ----------
+ mean : ndarray
+ The predicted state's mean vector (8 dimensional).
+ covariance : ndarray
+ The state's covariance matrix (8x8 dimensional).
+ measurement : ndarray
+ The 4 dimensional measurement vector (x, y, a, h), where (x, y)
+ is the center position, a the aspect ratio, and h the height of the
+ bounding box.
+
+ Returns
+ -------
+ (ndarray, ndarray)
+ Returns the measurement-corrected state distribution.
+
+ """
+ projected_mean, projected_cov = self.project(mean, covariance)
+
+ chol_factor, lower = scipy.linalg.cho_factor(
+ projected_cov, lower=True, check_finite=False)
+ kalman_gain = scipy.linalg.cho_solve(
+ (chol_factor, lower), np.dot(covariance, self._update_mat.T).T,
+ check_finite=False).T
+ innovation = measurement - projected_mean
+
+ new_mean = mean + np.dot(innovation, kalman_gain.T)
+ new_covariance = covariance - np.linalg.multi_dot((
+ kalman_gain, projected_cov, kalman_gain.T))
+ return new_mean, new_covariance
+
+ def gating_distance(self, mean, covariance, measurements,
+ only_position=False, metric='maha'):
+ """Compute gating distance between state distribution and measurements.
+ A suitable distance threshold can be obtained from `chi2inv95`. If
+ `only_position` is False, the chi-square distribution has 4 degrees of
+ freedom, otherwise 2.
+ Parameters
+ ----------
+ mean : ndarray
+ Mean vector over the state distribution (8 dimensional).
+ covariance : ndarray
+ Covariance of the state distribution (8x8 dimensional).
+ measurements : ndarray
+ An Nx4 dimensional matrix of N measurements, each in
+ format (x, y, a, h) where (x, y) is the bounding box center
+ position, a the aspect ratio, and h the height.
+ only_position : Optional[bool]
+ If True, distance computation is done with respect to the bounding
+ box center position only.
+ Returns
+ -------
+ ndarray
+ Returns an array of length N, where the i-th element contains the
+ squared Mahalanobis distance between (mean, covariance) and
+ `measurements[i]`.
+ """
+ mean, covariance = self.project(mean, covariance)
+ if only_position:
+ mean, covariance = mean[:2], covariance[:2, :2]
+ measurements = measurements[:, :2]
+
+ d = measurements - mean
+ if metric == 'gaussian':
+ return np.sum(d * d, axis=1)
+ elif metric == 'maha':
+ cholesky_factor = np.linalg.cholesky(covariance)
+ z = scipy.linalg.solve_triangular(
+ cholesky_factor, d.T, lower=True, check_finite=False,
+ overwrite_b=True)
+ squared_maha = np.sum(z * z, axis=0)
+ return squared_maha
+ else:
+ raise ValueError('invalid distance metric')
\ No newline at end of file
diff --git a/experimental/yolox/motdt_tracker/matching.py b/experimental/yolox/motdt_tracker/matching.py
new file mode 100644
index 0000000..01d07da
--- /dev/null
+++ b/experimental/yolox/motdt_tracker/matching.py
@@ -0,0 +1,116 @@
+import cv2
+import numpy as np
+import lap
+from scipy.spatial.distance import cdist
+
+from cython_bbox import bbox_overlaps as bbox_ious
+from yolox.motdt_tracker import kalman_filter
+
+
+def _indices_to_matches(cost_matrix, indices, thresh):
+ matched_cost = cost_matrix[tuple(zip(*indices))]
+ matched_mask = (matched_cost <= thresh)
+
+ matches = indices[matched_mask]
+ unmatched_a = tuple(set(range(cost_matrix.shape[0])) - set(matches[:, 0]))
+ unmatched_b = tuple(set(range(cost_matrix.shape[1])) - set(matches[:, 1]))
+
+ return matches, unmatched_a, unmatched_b
+
+
+def linear_assignment(cost_matrix, thresh):
+ if cost_matrix.size == 0:
+ return np.empty((0, 2), dtype=int), tuple(range(cost_matrix.shape[0])), tuple(range(cost_matrix.shape[1]))
+ matches, unmatched_a, unmatched_b = [], [], []
+ cost, x, y = lap.lapjv(cost_matrix, extend_cost=True, cost_limit=thresh)
+ for ix, mx in enumerate(x):
+ if mx >= 0:
+ matches.append([ix, mx])
+ unmatched_a = np.where(x < 0)[0]
+ unmatched_b = np.where(y < 0)[0]
+ matches = np.asarray(matches)
+ return matches, unmatched_a, unmatched_b
+
+
+def ious(atlbrs, btlbrs):
+ """
+ Compute cost based on IoU
+ :type atlbrs: list[tlbr] | np.ndarray
+ :type atlbrs: list[tlbr] | np.ndarray
+ :rtype ious np.ndarray
+ """
+ ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float)
+ if ious.size == 0:
+ return ious
+
+ ious = bbox_ious(
+ np.ascontiguousarray(atlbrs, dtype=np.float),
+ np.ascontiguousarray(btlbrs, dtype=np.float)
+ )
+
+ return ious
+
+
+def iou_distance(atracks, btracks):
+ """
+ Compute cost based on IoU
+ :type atracks: list[STrack]
+ :type btracks: list[STrack]
+ :rtype cost_matrix np.ndarray
+ """
+ atlbrs = [track.tlbr for track in atracks]
+ btlbrs = [track.tlbr for track in btracks]
+ _ious = ious(atlbrs, btlbrs)
+ cost_matrix = 1 - _ious
+
+ return cost_matrix
+
+
+def nearest_reid_distance(tracks, detections, metric='cosine'):
+ """
+ Compute cost based on ReID features
+ :type tracks: list[STrack]
+ :type detections: list[BaseTrack]
+ :rtype cost_matrix np.ndarray
+ """
+ cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float)
+ if cost_matrix.size == 0:
+ return cost_matrix
+
+ det_features = np.asarray([track.curr_feature for track in detections], dtype=np.float32)
+ for i, track in enumerate(tracks):
+ cost_matrix[i, :] = np.maximum(0.0, cdist(track.features, det_features, metric).min(axis=0))
+
+ return cost_matrix
+
+
+def mean_reid_distance(tracks, detections, metric='cosine'):
+ """
+ Compute cost based on ReID features
+ :type tracks: list[STrack]
+ :type detections: list[BaseTrack]
+ :type metric: str
+ :rtype cost_matrix np.ndarray
+ """
+ cost_matrix = np.empty((len(tracks), len(detections)), dtype=np.float)
+ if cost_matrix.size == 0:
+ return cost_matrix
+
+ track_features = np.asarray([track.curr_feature for track in tracks], dtype=np.float32)
+ det_features = np.asarray([track.curr_feature for track in detections], dtype=np.float32)
+ cost_matrix = cdist(track_features, det_features, metric)
+
+ return cost_matrix
+
+
+def gate_cost_matrix(kf, cost_matrix, tracks, detections, only_position=False):
+ if cost_matrix.size == 0:
+ return cost_matrix
+ gating_dim = 2 if only_position else 4
+ gating_threshold = kalman_filter.chi2inv95[gating_dim]
+ measurements = np.asarray([det.to_xyah() for det in detections])
+ for row, track in enumerate(tracks):
+ gating_distance = kf.gating_distance(
+ track.mean, track.covariance, measurements, only_position)
+ cost_matrix[row, gating_distance > gating_threshold] = np.inf
+ return cost_matrix
\ No newline at end of file
diff --git a/experimental/yolox/motdt_tracker/motdt_tracker.py b/experimental/yolox/motdt_tracker/motdt_tracker.py
new file mode 100644
index 0000000..b3521f2
--- /dev/null
+++ b/experimental/yolox/motdt_tracker/motdt_tracker.py
@@ -0,0 +1,358 @@
+import numpy as np
+#from numba import jit
+from collections import OrderedDict, deque
+import itertools
+import os
+import cv2
+import torch
+from torch._C import dtype
+import torchvision
+
+from yolox.motdt_tracker import matching
+from .kalman_filter import KalmanFilter
+from .reid_model import load_reid_model, extract_reid_features
+from yolox.data.dataloading import get_yolox_datadir
+
+from .basetrack import BaseTrack, TrackState
+
+
+class STrack(BaseTrack):
+
+ def __init__(self, tlwh, score, max_n_features=100, from_det=True):
+
+ # wait activate
+ self._tlwh = np.asarray(tlwh, dtype=np.float)
+ self.kalman_filter = None
+ self.mean, self.covariance = None, None
+ self.is_activated = False
+
+ self.score = score
+ self.max_n_features = max_n_features
+ self.curr_feature = None
+ self.last_feature = None
+ self.features = deque([], maxlen=self.max_n_features)
+
+ # classification
+ self.from_det = from_det
+ self.tracklet_len = 0
+ self.time_by_tracking = 0
+
+ # self-tracking
+ self.tracker = None
+
+ def set_feature(self, feature):
+ if feature is None:
+ return False
+ self.features.append(feature)
+ self.curr_feature = feature
+ self.last_feature = feature
+ # self._p_feature = 0
+ return True
+
+ def predict(self):
+ if self.time_since_update > 0:
+ self.tracklet_len = 0
+
+ self.time_since_update += 1
+
+ mean_state = self.mean.copy()
+ if self.state != TrackState.Tracked:
+ mean_state[7] = 0
+ self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance)
+
+ if self.tracker:
+ self.tracker.update_roi(self.tlwh)
+
+ def self_tracking(self, image):
+ tlwh = self.tracker.predict(image) if self.tracker else self.tlwh
+ return tlwh
+
+ def activate(self, kalman_filter, frame_id, image):
+ """Start a new tracklet"""
+ self.kalman_filter = kalman_filter # type: KalmanFilter
+ self.track_id = self.next_id()
+ # cx, cy, aspect_ratio, height, dx, dy, da, dh
+ self.mean, self.covariance = self.kalman_filter.initiate(self.tlwh_to_xyah(self._tlwh))
+
+ # self.tracker = sot.SingleObjectTracker()
+ # self.tracker.init(image, self.tlwh)
+
+ del self._tlwh
+
+ self.time_since_update = 0
+ self.time_by_tracking = 0
+ self.tracklet_len = 0
+ self.state = TrackState.Tracked
+ # self.is_activated = True
+ self.frame_id = frame_id
+ self.start_frame = frame_id
+
+ def re_activate(self, new_track, frame_id, image, new_id=False):
+ # self.mean, self.covariance = self.kalman_filter.initiate(self.tlwh_to_xyah(new_track.tlwh))
+ self.mean, self.covariance = self.kalman_filter.update(
+ self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh)
+ )
+ self.time_since_update = 0
+ self.time_by_tracking = 0
+ self.tracklet_len = 0
+ self.state = TrackState.Tracked
+ self.is_activated = True
+ self.frame_id = frame_id
+ if new_id:
+ self.track_id = self.next_id()
+
+ self.set_feature(new_track.curr_feature)
+
+ def update(self, new_track, frame_id, image, update_feature=True):
+ """
+ Update a matched track
+ :type new_track: STrack
+ :type frame_id: int
+ :type update_feature: bool
+ :return:
+ """
+ self.frame_id = frame_id
+ self.time_since_update = 0
+ if new_track.from_det:
+ self.time_by_tracking = 0
+ else:
+ self.time_by_tracking += 1
+ self.tracklet_len += 1
+
+ new_tlwh = new_track.tlwh
+ self.mean, self.covariance = self.kalman_filter.update(
+ self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh))
+ self.state = TrackState.Tracked
+ self.is_activated = True
+
+ self.score = new_track.score
+
+ if update_feature:
+ self.set_feature(new_track.curr_feature)
+ if self.tracker:
+ self.tracker.update(image, self.tlwh)
+
+ @property
+ #@jit
+ def tlwh(self):
+ """Get current position in bounding box format `(top left x, top left y,
+ width, height)`.
+ """
+ if self.mean is None:
+ return self._tlwh.copy()
+ ret = self.mean[:4].copy()
+ ret[2] *= ret[3]
+ ret[:2] -= ret[2:] / 2
+ return ret
+
+ @property
+ #@jit
+ def tlbr(self):
+ """Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,
+ `(top left, bottom right)`.
+ """
+ ret = self.tlwh.copy()
+ ret[2:] += ret[:2]
+ return ret
+
+ @staticmethod
+ #@jit
+ def tlwh_to_xyah(tlwh):
+ """Convert bounding box to format `(center x, center y, aspect ratio,
+ height)`, where the aspect ratio is `width / height`.
+ """
+ ret = np.asarray(tlwh).copy()
+ ret[:2] += ret[2:] / 2
+ ret[2] /= ret[3]
+ return ret
+
+ def to_xyah(self):
+ return self.tlwh_to_xyah(self.tlwh)
+
+ def tracklet_score(self):
+ # score = (1 - np.exp(-0.6 * self.hit_streak)) * np.exp(-0.03 * self.time_by_tracking)
+
+ score = max(0, 1 - np.log(1 + 0.05 * self.time_by_tracking)) * (self.tracklet_len - self.time_by_tracking > 2)
+ # score = max(0, 1 - np.log(1 + 0.05 * self.n_tracking)) * (1 - np.exp(-0.6 * self.hit_streak))
+ return score
+
+ def __repr__(self):
+ return 'OT_{}_({}-{})'.format(self.track_id, self.start_frame, self.end_frame)
+
+
+class OnlineTracker(object):
+
+ def __init__(self, model_folder, min_cls_score=0.4, min_ap_dist=0.8, max_time_lost=30, use_tracking=True, use_refind=True):
+
+ self.min_cls_score = min_cls_score
+ self.min_ap_dist = min_ap_dist
+ self.max_time_lost = max_time_lost
+
+ self.kalman_filter = KalmanFilter()
+
+ self.tracked_stracks = [] # type: list[STrack]
+ self.lost_stracks = [] # type: list[STrack]
+ self.removed_stracks = [] # type: list[STrack]
+
+ self.use_refind = use_refind
+ self.use_tracking = use_tracking
+ self.classifier = None
+ self.reid_model = load_reid_model(model_folder)
+
+ self.frame_id = 0
+
+ def update(self, output_results, img_info, img_size, img_file_name):
+ img_file_name = os.path.join(get_yolox_datadir(), 'mot', 'train', img_file_name)
+ image = cv2.imread(img_file_name)
+ # post process detections
+ output_results = output_results.cpu().numpy()
+ confidences = output_results[:, 4] * output_results[:, 5]
+
+ bboxes = output_results[:, :4] # x1y1x2y2
+ img_h, img_w = img_info[0], img_info[1]
+ scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w))
+ bboxes /= scale
+ bbox_xyxy = bboxes
+ tlwhs = self._xyxy_to_tlwh_array(bbox_xyxy)
+ remain_inds = confidences > self.min_cls_score
+ tlwhs = tlwhs[remain_inds]
+ det_scores = confidences[remain_inds]
+
+ self.frame_id += 1
+
+ activated_starcks = []
+ refind_stracks = []
+ lost_stracks = []
+ removed_stracks = []
+
+ """step 1: prediction"""
+ for strack in itertools.chain(self.tracked_stracks, self.lost_stracks):
+ strack.predict()
+
+ """step 2: scoring and selection"""
+ if det_scores is None:
+ det_scores = np.ones(len(tlwhs), dtype=float)
+ detections = [STrack(tlwh, score, from_det=True) for tlwh, score in zip(tlwhs, det_scores)]
+ if self.use_tracking:
+ tracks = [STrack(t.self_tracking(image), 0.6 * t.tracklet_score(), from_det=False)
+ for t in itertools.chain(self.tracked_stracks, self.lost_stracks) if t.is_activated]
+ detections.extend(tracks)
+ rois = np.asarray([d.tlbr for d in detections], dtype=np.float32)
+ scores = np.asarray([d.score for d in detections], dtype=np.float32)
+ # nms
+ if len(detections) > 0:
+ nms_out_index = torchvision.ops.batched_nms(
+ torch.from_numpy(rois),
+ torch.from_numpy(scores.reshape(-1)).to(torch.from_numpy(rois).dtype),
+ torch.zeros_like(torch.from_numpy(scores.reshape(-1))),
+ 0.7,
+ )
+ keep = nms_out_index.numpy()
+ mask = np.zeros(len(rois), dtype=np.bool)
+ mask[keep] = True
+ keep = np.where(mask & (scores >= self.min_cls_score))[0]
+ detections = [detections[i] for i in keep]
+ scores = scores[keep]
+ for d, score in zip(detections, scores):
+ d.score = score
+ pred_dets = [d for d in detections if not d.from_det]
+ detections = [d for d in detections if d.from_det]
+
+ # set features
+ tlbrs = [det.tlbr for det in detections]
+ features = extract_reid_features(self.reid_model, image, tlbrs)
+ features = features.cpu().numpy()
+ for i, det in enumerate(detections):
+ det.set_feature(features[i])
+
+ """step 3: association for tracked"""
+ # matching for tracked targets
+ unconfirmed = []
+ tracked_stracks = [] # type: list[STrack]
+ for track in self.tracked_stracks:
+ if not track.is_activated:
+ unconfirmed.append(track)
+ else:
+ tracked_stracks.append(track)
+
+ dists = matching.nearest_reid_distance(tracked_stracks, detections, metric='euclidean')
+ dists = matching.gate_cost_matrix(self.kalman_filter, dists, tracked_stracks, detections)
+ matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.min_ap_dist)
+ for itracked, idet in matches:
+ tracked_stracks[itracked].update(detections[idet], self.frame_id, image)
+
+ # matching for missing targets
+ detections = [detections[i] for i in u_detection]
+ dists = matching.nearest_reid_distance(self.lost_stracks, detections, metric='euclidean')
+ dists = matching.gate_cost_matrix(self.kalman_filter, dists, self.lost_stracks, detections)
+ matches, u_lost, u_detection = matching.linear_assignment(dists, thresh=self.min_ap_dist)
+ for ilost, idet in matches:
+ track = self.lost_stracks[ilost] # type: STrack
+ det = detections[idet]
+ track.re_activate(det, self.frame_id, image, new_id=not self.use_refind)
+ refind_stracks.append(track)
+
+ # remaining tracked
+ # tracked
+ len_det = len(u_detection)
+ detections = [detections[i] for i in u_detection] + pred_dets
+ r_tracked_stracks = [tracked_stracks[i] for i in u_track]
+ dists = matching.iou_distance(r_tracked_stracks, detections)
+ matches, u_track, u_detection = matching.linear_assignment(dists, thresh=0.5)
+ for itracked, idet in matches:
+ r_tracked_stracks[itracked].update(detections[idet], self.frame_id, image, update_feature=True)
+ for it in u_track:
+ track = r_tracked_stracks[it]
+ track.mark_lost()
+ lost_stracks.append(track)
+
+ # unconfirmed
+ detections = [detections[i] for i in u_detection if i < len_det]
+ dists = matching.iou_distance(unconfirmed, detections)
+ matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7)
+ for itracked, idet in matches:
+ unconfirmed[itracked].update(detections[idet], self.frame_id, image, update_feature=True)
+ for it in u_unconfirmed:
+ track = unconfirmed[it]
+ track.mark_removed()
+ removed_stracks.append(track)
+
+ """step 4: init new stracks"""
+ for inew in u_detection:
+ track = detections[inew]
+ if not track.from_det or track.score < 0.6:
+ continue
+ track.activate(self.kalman_filter, self.frame_id, image)
+ activated_starcks.append(track)
+
+ """step 6: update state"""
+ for track in self.lost_stracks:
+ if self.frame_id - track.end_frame > self.max_time_lost:
+ track.mark_removed()
+ removed_stracks.append(track)
+
+ self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked]
+ self.lost_stracks = [t for t in self.lost_stracks if t.state == TrackState.Lost] # type: list[STrack]
+ self.tracked_stracks.extend(activated_starcks)
+ self.tracked_stracks.extend(refind_stracks)
+ self.lost_stracks.extend(lost_stracks)
+ self.removed_stracks.extend(removed_stracks)
+
+ # output_stracks = self.tracked_stracks + self.lost_stracks
+
+ # get scores of lost tracks
+ output_tracked_stracks = [track for track in self.tracked_stracks if track.is_activated]
+
+ output_stracks = output_tracked_stracks
+
+ return output_stracks
+
+ @staticmethod
+ def _xyxy_to_tlwh_array(bbox_xyxy):
+ if isinstance(bbox_xyxy, np.ndarray):
+ bbox_tlwh = bbox_xyxy.copy()
+ elif isinstance(bbox_xyxy, torch.Tensor):
+ bbox_tlwh = bbox_xyxy.clone()
+ bbox_tlwh[:, 2] = bbox_xyxy[:, 2] - bbox_xyxy[:, 0]
+ bbox_tlwh[:, 3] = bbox_xyxy[:, 3] - bbox_xyxy[:, 1]
+ return bbox_tlwh
diff --git a/experimental/yolox/motdt_tracker/reid_model.py b/experimental/yolox/motdt_tracker/reid_model.py
new file mode 100644
index 0000000..6ad49e3
--- /dev/null
+++ b/experimental/yolox/motdt_tracker/reid_model.py
@@ -0,0 +1,270 @@
+import cv2
+import numpy as np
+import torch
+from torch.autograd import Variable
+import torch.nn.functional as F
+import torch.nn as nn
+import pickle
+import os
+from torch.nn.modules import CrossMapLRN2d as SpatialCrossMapLRN
+#from torch.legacy.nn import SpatialCrossMapLRN as SpatialCrossMapLRNOld
+from torch.autograd import Function, Variable
+from torch.nn import Module
+
+
+def clip_boxes(boxes, im_shape):
+ """
+ Clip boxes to image boundaries.
+ """
+ boxes = np.asarray(boxes)
+ if boxes.shape[0] == 0:
+ return boxes
+ boxes = np.copy(boxes)
+ # x1 >= 0
+ boxes[:, 0::4] = np.maximum(np.minimum(boxes[:, 0::4], im_shape[1] - 1), 0)
+ # y1 >= 0
+ boxes[:, 1::4] = np.maximum(np.minimum(boxes[:, 1::4], im_shape[0] - 1), 0)
+ # x2 < im_shape[1]
+ boxes[:, 2::4] = np.maximum(np.minimum(boxes[:, 2::4], im_shape[1] - 1), 0)
+ # y2 < im_shape[0]
+ boxes[:, 3::4] = np.maximum(np.minimum(boxes[:, 3::4], im_shape[0] - 1), 0)
+ return boxes
+
+
+def load_net(fname, net, prefix='', load_state_dict=False):
+ import h5py
+ with h5py.File(fname, mode='r') as h5f:
+ h5f_is_module = True
+ for k in h5f.keys():
+ if not str(k).startswith('module.'):
+ h5f_is_module = False
+ break
+ if prefix == '' and not isinstance(net, nn.DataParallel) and h5f_is_module:
+ prefix = 'module.'
+
+ for k, v in net.state_dict().items():
+ k = prefix + k
+ if k in h5f:
+ param = torch.from_numpy(np.asarray(h5f[k]))
+ if v.size() != param.size():
+ print('Inconsistent shape: {}, {}'.format(v.size(), param.size()))
+ else:
+ v.copy_(param)
+ else:
+ print.warning('No layer: {}'.format(k))
+
+ epoch = h5f.attrs['epoch'] if 'epoch' in h5f.attrs else -1
+
+ if not load_state_dict:
+ if 'learning_rates' in h5f.attrs:
+ lr = h5f.attrs['learning_rates']
+ else:
+ lr = h5f.attrs.get('lr', -1)
+ lr = np.asarray([lr] if lr > 0 else [], dtype=np.float)
+
+ return epoch, lr
+
+ state_file = fname + '.optimizer_state.pk'
+ if os.path.isfile(state_file):
+ with open(state_file, 'rb') as f:
+ state_dicts = pickle.load(f)
+ if not isinstance(state_dicts, list):
+ state_dicts = [state_dicts]
+ else:
+ state_dicts = None
+ return epoch, state_dicts
+
+
+# class SpatialCrossMapLRNFunc(Function):
+
+# def __init__(self, size, alpha=1e-4, beta=0.75, k=1):
+# self.size = size
+# self.alpha = alpha
+# self.beta = beta
+# self.k = k
+
+# def forward(self, input):
+# self.save_for_backward(input)
+# self.lrn = SpatialCrossMapLRNOld(self.size, self.alpha, self.beta, self.k)
+# self.lrn.type(input.type())
+# return self.lrn.forward(input)
+
+# def backward(self, grad_output):
+# input, = self.saved_tensors
+# return self.lrn.backward(input, grad_output)
+
+
+# # use this one instead
+# class SpatialCrossMapLRN(Module):
+# def __init__(self, size, alpha=1e-4, beta=0.75, k=1):
+# super(SpatialCrossMapLRN, self).__init__()
+# self.size = size
+# self.alpha = alpha
+# self.beta = beta
+# self.k = k
+
+# def forward(self, input):
+# return SpatialCrossMapLRNFunc(self.size, self.alpha, self.beta, self.k)(input)
+
+
+class Inception(nn.Module):
+ def __init__(self, in_planes, n1x1, n3x3red, n3x3, n5x5red, n5x5, pool_planes):
+ super(Inception, self).__init__()
+ # 1x1 conv branch
+ self.b1 = nn.Sequential(
+ nn.Conv2d(in_planes, n1x1, kernel_size=1),
+ nn.ReLU(True),
+ )
+
+ # 1x1 conv -> 3x3 conv branch
+ self.b2 = nn.Sequential(
+ nn.Conv2d(in_planes, n3x3red, kernel_size=1),
+ nn.ReLU(True),
+ nn.Conv2d(n3x3red, n3x3, kernel_size=3, padding=1),
+ nn.ReLU(True),
+ )
+
+ # 1x1 conv -> 5x5 conv branch
+ self.b3 = nn.Sequential(
+ nn.Conv2d(in_planes, n5x5red, kernel_size=1),
+ nn.ReLU(True),
+
+ nn.Conv2d(n5x5red, n5x5, kernel_size=5, padding=2),
+ nn.ReLU(True),
+ )
+
+ # 3x3 pool -> 1x1 conv branch
+ self.b4 = nn.Sequential(
+ nn.MaxPool2d(3, stride=1, padding=1),
+
+ nn.Conv2d(in_planes, pool_planes, kernel_size=1),
+ nn.ReLU(True),
+ )
+
+ def forward(self, x):
+ y1 = self.b1(x)
+ y2 = self.b2(x)
+ y3 = self.b3(x)
+ y4 = self.b4(x)
+ return torch.cat([y1,y2,y3,y4], 1)
+
+
+class GoogLeNet(nn.Module):
+
+ output_channels = 832
+
+ def __init__(self):
+ super(GoogLeNet, self).__init__()
+ self.pre_layers = nn.Sequential(
+ nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3),
+ nn.ReLU(True),
+
+ nn.MaxPool2d(3, stride=2, ceil_mode=True),
+ SpatialCrossMapLRN(5),
+
+ nn.Conv2d(64, 64, 1),
+ nn.ReLU(True),
+
+ nn.Conv2d(64, 192, 3, padding=1),
+ nn.ReLU(True),
+
+ SpatialCrossMapLRN(5),
+ nn.MaxPool2d(3, stride=2, ceil_mode=True),
+ )
+
+ self.a3 = Inception(192, 64, 96, 128, 16, 32, 32)
+ self.b3 = Inception(256, 128, 128, 192, 32, 96, 64)
+
+ self.maxpool = nn.MaxPool2d(3, stride=2, ceil_mode=True)
+
+ self.a4 = Inception(480, 192, 96, 208, 16, 48, 64)
+ self.b4 = Inception(512, 160, 112, 224, 24, 64, 64)
+ self.c4 = Inception(512, 128, 128, 256, 24, 64, 64)
+ self.d4 = Inception(512, 112, 144, 288, 32, 64, 64)
+ self.e4 = Inception(528, 256, 160, 320, 32, 128, 128)
+
+ def forward(self, x):
+ out = self.pre_layers(x)
+ out = self.a3(out)
+ out = self.b3(out)
+ out = self.maxpool(out)
+ out = self.a4(out)
+ out = self.b4(out)
+ out = self.c4(out)
+ out = self.d4(out)
+ out = self.e4(out)
+
+ return out
+
+
+class Model(nn.Module):
+ def __init__(self, n_parts=8):
+ super(Model, self).__init__()
+ self.n_parts = n_parts
+
+ self.feat_conv = GoogLeNet()
+ self.conv_input_feat = nn.Conv2d(self.feat_conv.output_channels, 512, 1)
+
+ # part net
+ self.conv_att = nn.Conv2d(512, self.n_parts, 1)
+
+ for i in range(self.n_parts):
+ setattr(self, 'linear_feature{}'.format(i+1), nn.Linear(512, 64))
+
+ def forward(self, x):
+ feature = self.feat_conv(x)
+ feature = self.conv_input_feat(feature)
+
+ att_weights = torch.sigmoid(self.conv_att(feature))
+
+ linear_feautres = []
+ for i in range(self.n_parts):
+ masked_feature = feature * torch.unsqueeze(att_weights[:, i], 1)
+ pooled_feature = F.avg_pool2d(masked_feature, masked_feature.size()[2:4])
+ linear_feautres.append(
+ getattr(self, 'linear_feature{}'.format(i+1))(pooled_feature.view(pooled_feature.size(0), -1))
+ )
+
+ concat_features = torch.cat(linear_feautres, 1)
+ normed_feature = concat_features / torch.clamp(torch.norm(concat_features, 2, 1, keepdim=True), min=1e-6)
+
+ return normed_feature
+
+
+def load_reid_model(ckpt):
+ model = Model(n_parts=8)
+ model.inp_size = (80, 160)
+ load_net(ckpt, model)
+ print('Load ReID model from {}'.format(ckpt))
+
+ model = model.cuda()
+ model.eval()
+ return model
+
+
+def im_preprocess(image):
+ image = np.asarray(image, np.float32)
+ image -= np.array([104, 117, 123], dtype=np.float32).reshape(1, 1, -1)
+ image = image.transpose((2, 0, 1))
+ return image
+
+
+def extract_image_patches(image, bboxes):
+ bboxes = np.round(bboxes).astype(np.int)
+ bboxes = clip_boxes(bboxes, image.shape)
+ patches = [image[box[1]:box[3], box[0]:box[2]] for box in bboxes]
+ return patches
+
+
+def extract_reid_features(reid_model, image, tlbrs):
+ if len(tlbrs) == 0:
+ return torch.FloatTensor()
+
+ patches = extract_image_patches(image, tlbrs)
+ patches = np.asarray([im_preprocess(cv2.resize(p, reid_model.inp_size)) for p in patches], dtype=np.float32)
+
+ with torch.no_grad():
+ im_var = Variable(torch.from_numpy(patches))
+ im_var = im_var.cuda()
+ features = reid_model(im_var).data
+ return features
\ No newline at end of file
diff --git a/experimental/yolox/sort_tracker/sort.py b/experimental/yolox/sort_tracker/sort.py
new file mode 100644
index 0000000..9b708ba
--- /dev/null
+++ b/experimental/yolox/sort_tracker/sort.py
@@ -0,0 +1,251 @@
+"""
+ SORT: A Simple, Online and Realtime Tracker
+ Copyright (C) 2016-2020 Alex Bewley [email protected]
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+from __future__ import print_function
+
+import os
+import numpy as np
+
+from filterpy.kalman import KalmanFilter
+
+np.random.seed(0)
+
+
+def linear_assignment(cost_matrix):
+ try:
+ import lap
+ _, x, y = lap.lapjv(cost_matrix, extend_cost=True)
+ return np.array([[y[i],i] for i in x if i >= 0]) #
+ except ImportError:
+ from scipy.optimize import linear_sum_assignment
+ x, y = linear_sum_assignment(cost_matrix)
+ return np.array(list(zip(x, y)))
+
+
+def iou_batch(bb_test, bb_gt):
+ """
+ From SORT: Computes IOU between two bboxes in the form [x1,y1,x2,y2]
+ """
+ bb_gt = np.expand_dims(bb_gt, 0)
+ bb_test = np.expand_dims(bb_test, 1)
+
+ xx1 = np.maximum(bb_test[..., 0], bb_gt[..., 0])
+ yy1 = np.maximum(bb_test[..., 1], bb_gt[..., 1])
+ xx2 = np.minimum(bb_test[..., 2], bb_gt[..., 2])
+ yy2 = np.minimum(bb_test[..., 3], bb_gt[..., 3])
+ w = np.maximum(0., xx2 - xx1)
+ h = np.maximum(0., yy2 - yy1)
+ wh = w * h
+ o = wh / ((bb_test[..., 2] - bb_test[..., 0]) * (bb_test[..., 3] - bb_test[..., 1])
+ + (bb_gt[..., 2] - bb_gt[..., 0]) * (bb_gt[..., 3] - bb_gt[..., 1]) - wh)
+ return(o)
+
+
+def convert_bbox_to_z(bbox):
+ """
+ Takes a bounding box in the form [x1,y1,x2,y2] and returns z in the form
+ [x,y,s,r] where x,y is the centre of the box and s is the scale/area and r is
+ the aspect ratio
+ """
+ w = bbox[2] - bbox[0]
+ h = bbox[3] - bbox[1]
+ x = bbox[0] + w/2.
+ y = bbox[1] + h/2.
+ s = w * h #scale is just area
+ r = w / float(h)
+ return np.array([x, y, s, r]).reshape((4, 1))
+
+
+def convert_x_to_bbox(x,score=None):
+ """
+ Takes a bounding box in the centre form [x,y,s,r] and returns it in the form
+ [x1,y1,x2,y2] where x1,y1 is the top left and x2,y2 is the bottom right
+ """
+ w = np.sqrt(x[2] * x[3])
+ h = x[2] / w
+ if(score==None):
+ return np.array([x[0]-w/2.,x[1]-h/2.,x[0]+w/2.,x[1]+h/2.]).reshape((1,4))
+ else:
+ return np.array([x[0]-w/2.,x[1]-h/2.,x[0]+w/2.,x[1]+h/2.,score]).reshape((1,5))
+
+
+class KalmanBoxTracker(object):
+ """
+ This class represents the internal state of individual tracked objects observed as bbox.
+ """
+ count = 0
+ def __init__(self,bbox):
+ """
+ Initialises a tracker using initial bounding box.
+ """
+ #define constant velocity model
+ self.kf = KalmanFilter(dim_x=7, dim_z=4)
+ self.kf.F = np.array([[1,0,0,0,1,0,0],[0,1,0,0,0,1,0],[0,0,1,0,0,0,1],[0,0,0,1,0,0,0], [0,0,0,0,1,0,0],[0,0,0,0,0,1,0],[0,0,0,0,0,0,1]])
+ self.kf.H = np.array([[1,0,0,0,0,0,0],[0,1,0,0,0,0,0],[0,0,1,0,0,0,0],[0,0,0,1,0,0,0]])
+
+ self.kf.R[2:,2:] *= 10.
+ self.kf.P[4:,4:] *= 1000. #give high uncertainty to the unobservable initial velocities
+ self.kf.P *= 10.
+ self.kf.Q[-1,-1] *= 0.01
+ self.kf.Q[4:,4:] *= 0.01
+
+ self.kf.x[:4] = convert_bbox_to_z(bbox)
+ self.time_since_update = 0
+ self.id = KalmanBoxTracker.count
+ KalmanBoxTracker.count += 1
+ self.history = []
+ self.hits = 0
+ self.hit_streak = 0
+ self.age = 0
+
+ def update(self,bbox):
+ """
+ Updates the state vector with observed bbox.
+ """
+ self.time_since_update = 0
+ self.history = []
+ self.hits += 1
+ self.hit_streak += 1
+ self.kf.update(convert_bbox_to_z(bbox))
+
+ def predict(self):
+ """
+ Advances the state vector and returns the predicted bounding box estimate.
+ """
+ if((self.kf.x[6]+self.kf.x[2])<=0):
+ self.kf.x[6] *= 0.0
+ self.kf.predict()
+ self.age += 1
+ if(self.time_since_update>0):
+ self.hit_streak = 0
+ self.time_since_update += 1
+ self.history.append(convert_x_to_bbox(self.kf.x))
+ return self.history[-1]
+
+ def get_state(self):
+ """
+ Returns the current bounding box estimate.
+ """
+ return convert_x_to_bbox(self.kf.x)
+
+
+def associate_detections_to_trackers(detections,trackers,iou_threshold = 0.3):
+ """
+ Assigns detections to tracked object (both represented as bounding boxes)
+ Returns 3 lists of matches, unmatched_detections and unmatched_trackers
+ """
+ if(len(trackers)==0):
+ return np.empty((0,2),dtype=int), np.arange(len(detections)), np.empty((0,5),dtype=int)
+
+ iou_matrix = iou_batch(detections, trackers)
+
+ if min(iou_matrix.shape) > 0:
+ a = (iou_matrix > iou_threshold).astype(np.int32)
+ if a.sum(1).max() == 1 and a.sum(0).max() == 1:
+ matched_indices = np.stack(np.where(a), axis=1)
+ else:
+ matched_indices = linear_assignment(-iou_matrix)
+ else:
+ matched_indices = np.empty(shape=(0,2))
+
+ unmatched_detections = []
+ for d, det in enumerate(detections):
+ if(d not in matched_indices[:,0]):
+ unmatched_detections.append(d)
+ unmatched_trackers = []
+ for t, trk in enumerate(trackers):
+ if(t not in matched_indices[:,1]):
+ unmatched_trackers.append(t)
+
+ #filter out matched with low IOU
+ matches = []
+ for m in matched_indices:
+ if(iou_matrix[m[0], m[1]]<iou_threshold):
+ unmatched_detections.append(m[0])
+ unmatched_trackers.append(m[1])
+ else:
+ matches.append(m.reshape(1,2))
+ if(len(matches)==0):
+ matches = np.empty((0,2),dtype=int)
+ else:
+ matches = np.concatenate(matches,axis=0)
+
+ return matches, np.array(unmatched_detections), np.array(unmatched_trackers)
+
+
+class Sort(object):
+ def __init__(self, det_thresh, max_age=30, min_hits=3, iou_threshold=0.3):
+ """
+ Sets key parameters for SORT
+ """
+ self.max_age = max_age
+ self.min_hits = min_hits
+ self.iou_threshold = iou_threshold
+ self.trackers = []
+ self.frame_count = 0
+ self.det_thresh = det_thresh
+
+ def update(self, output_results, img_info, img_size):
+ """
+ Params:
+ dets - a numpy array of detections in the format [[x1,y1,x2,y2,score],[x1,y1,x2,y2,score],...]
+ Requires: this method must be called once for each frame even with empty detections (use np.empty((0, 5)) for frames without detections).
+ Returns the a similar array, where the last column is the object ID.
+ NOTE: The number of objects returned may differ from the number of detections provided.
+ """
+ self.frame_count += 1
+ # post_process detections
+ output_results = output_results.cpu().numpy()
+ scores = output_results[:, 4] * output_results[:, 5]
+ bboxes = output_results[:, :4] # x1y1x2y2
+ img_h, img_w = img_info[0], img_info[1]
+ scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w))
+ bboxes /= scale
+ dets = np.concatenate((bboxes, np.expand_dims(scores, axis=-1)), axis=1)
+ remain_inds = scores > self.det_thresh
+ dets = dets[remain_inds]
+ # get predicted locations from existing trackers.
+ trks = np.zeros((len(self.trackers), 5))
+ to_del = []
+ ret = []
+ for t, trk in enumerate(trks):
+ pos = self.trackers[t].predict()[0]
+ trk[:] = [pos[0], pos[1], pos[2], pos[3], 0]
+ if np.any(np.isnan(pos)):
+ to_del.append(t)
+ trks = np.ma.compress_rows(np.ma.masked_invalid(trks))
+ for t in reversed(to_del):
+ self.trackers.pop(t)
+ matched, unmatched_dets, unmatched_trks = associate_detections_to_trackers(dets, trks, self.iou_threshold)
+
+ # update matched trackers with assigned detections
+ for m in matched:
+ self.trackers[m[1]].update(dets[m[0], :])
+
+ # create and initialise new trackers for unmatched detections
+ for i in unmatched_dets:
+ trk = KalmanBoxTracker(dets[i,:])
+ self.trackers.append(trk)
+ i = len(self.trackers)
+ for trk in reversed(self.trackers):
+ d = trk.get_state()[0]
+ if (trk.time_since_update < 1) and (trk.hit_streak >= self.min_hits or self.frame_count <= self.min_hits):
+ ret.append(np.concatenate((d,[trk.id+1])).reshape(1,-1)) # +1 as MOT benchmark requires positive
+ i -= 1
+ # remove dead tracklet
+ if(trk.time_since_update > self.max_age):
+ self.trackers.pop(i)
+ if(len(ret)>0):
+ return np.concatenate(ret)
+ return np.empty((0,5))
\ No newline at end of file
diff --git a/experimental/yolox/tracker/__pycache__/basetrack.cpython-310.pyc b/experimental/yolox/tracker/__pycache__/basetrack.cpython-310.pyc
new file mode 100644
index 0000000..0cae8d1
Binary files /dev/null and b/experimental/yolox/tracker/__pycache__/basetrack.cpython-310.pyc differ
diff --git a/experimental/yolox/tracker/__pycache__/byte_tracker.cpython-310.pyc b/experimental/yolox/tracker/__pycache__/byte_tracker.cpython-310.pyc
new file mode 100644
index 0000000..f7b9f97
Binary files /dev/null and b/experimental/yolox/tracker/__pycache__/byte_tracker.cpython-310.pyc differ
diff --git a/experimental/yolox/tracker/__pycache__/kalman_filter.cpython-310.pyc b/experimental/yolox/tracker/__pycache__/kalman_filter.cpython-310.pyc
new file mode 100644
index 0000000..512b76d
Binary files /dev/null and b/experimental/yolox/tracker/__pycache__/kalman_filter.cpython-310.pyc differ
diff --git a/experimental/yolox/tracker/__pycache__/matching.cpython-310.pyc b/experimental/yolox/tracker/__pycache__/matching.cpython-310.pyc
new file mode 100644
index 0000000..5e8c5f8
Binary files /dev/null and b/experimental/yolox/tracker/__pycache__/matching.cpython-310.pyc differ
diff --git a/experimental/yolox/tracker/basetrack.py b/experimental/yolox/tracker/basetrack.py
new file mode 100644
index 0000000..4fe2233
--- /dev/null
+++ b/experimental/yolox/tracker/basetrack.py
@@ -0,0 +1,52 @@
+import numpy as np
+from collections import OrderedDict
+
+
+class TrackState(object):
+ New = 0
+ Tracked = 1
+ Lost = 2
+ Removed = 3
+
+
+class BaseTrack(object):
+ _count = 0
+
+ track_id = 0
+ is_activated = False
+ state = TrackState.New
+
+ history = OrderedDict()
+ features = []
+ curr_feature = None
+ score = 0
+ start_frame = 0
+ frame_id = 0
+ time_since_update = 0
+
+ # multi-camera
+ location = (np.inf, np.inf)
+
+ @property
+ def end_frame(self):
+ return self.frame_id
+
+ @staticmethod
+ def next_id():
+ BaseTrack._count += 1
+ return BaseTrack._count
+
+ def activate(self, *args):
+ raise NotImplementedError
+
+ def predict(self):
+ raise NotImplementedError
+
+ def update(self, *args, **kwargs):
+ raise NotImplementedError
+
+ def mark_lost(self):
+ self.state = TrackState.Lost
+
+ def mark_removed(self):
+ self.state = TrackState.Removed
diff --git a/experimental/yolox/tracker/byte_tracker.py b/experimental/yolox/tracker/byte_tracker.py
new file mode 100644
index 0000000..317f2b2
--- /dev/null
+++ b/experimental/yolox/tracker/byte_tracker.py
@@ -0,0 +1,324 @@
+import numpy as np
+
+from .kalman_filter import KalmanFilter
+from yolox.tracker import matching
+from .basetrack import BaseTrack, TrackState
+
+class STrack(BaseTrack):
+ shared_kalman = KalmanFilter()
+ def __init__(self, tlwh, score, class_id):
+
+ # wait activate
+ self._tlwh = np.asarray(tlwh, dtype=np.float)
+ self.kalman_filter = None
+ self.mean, self.covariance = None, None
+ self.is_activated = False
+
+ self.score = score
+ self.class_id = class_id
+ self.tracklet_len = 0
+
+ def predict(self):
+ mean_state = self.mean.copy()
+ if self.state != TrackState.Tracked:
+ mean_state[7] = 0
+ self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance)
+
+ @staticmethod
+ def multi_predict(stracks):
+ if len(stracks) > 0:
+ multi_mean = np.asarray([st.mean.copy() for st in stracks])
+ multi_covariance = np.asarray([st.covariance for st in stracks])
+ for i, st in enumerate(stracks):
+ if st.state != TrackState.Tracked:
+ multi_mean[i][7] = 0
+ multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(multi_mean, multi_covariance)
+ for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)):
+ stracks[i].mean = mean
+ stracks[i].covariance = cov
+
+ def activate(self, kalman_filter, frame_id):
+ """Start a new tracklet"""
+ self.kalman_filter = kalman_filter
+ self.track_id = self.next_id()
+ self.mean, self.covariance = self.kalman_filter.initiate(self.tlwh_to_xyah(self._tlwh))
+
+ self.tracklet_len = 0
+ self.state = TrackState.Tracked
+ if frame_id == 1:
+ self.is_activated = True
+ # self.is_activated = True
+ self.frame_id = frame_id
+ self.start_frame = frame_id
+
+ def re_activate(self, new_track, frame_id, new_id=False):
+ self.mean, self.covariance = self.kalman_filter.update(
+ self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh)
+ )
+ self.tracklet_len = 0
+ self.state = TrackState.Tracked
+ self.is_activated = True
+ self.frame_id = frame_id
+ if new_id:
+ self.track_id = self.next_id()
+ self.score = new_track.score
+
+ def update(self, new_track, frame_id):
+ """
+ Update a matched track
+ :type new_track: STrack
+ :type frame_id: int
+ :type update_feature: bool
+ :return:
+ """
+ self.frame_id = frame_id
+ self.tracklet_len += 1
+
+ new_tlwh = new_track.tlwh
+ self.mean, self.covariance = self.kalman_filter.update(
+ self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh))
+ self.state = TrackState.Tracked
+ self.is_activated = True
+
+ self.score = new_track.score
+
+ @property
+ # @jit(nopython=True)
+ def tlwh(self):
+ """Get current position in bounding box format `(top left x, top left y,
+ width, height)`.
+ """
+ if self.mean is None:
+ return self._tlwh.copy()
+ ret = self.mean[:4].copy()
+ ret[2] *= ret[3]
+ ret[:2] -= ret[2:] / 2
+ return ret
+
+ @property
+ # @jit(nopython=True)
+ def tlbr(self):
+ """Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,
+ `(top left, bottom right)`.
+ """
+ ret = self.tlwh.copy()
+ ret[2:] += ret[:2]
+ return ret
+
+ @staticmethod
+ # @jit(nopython=True)
+ def tlwh_to_xyah(tlwh):
+ """Convert bounding box to format `(center x, center y, aspect ratio,
+ height)`, where the aspect ratio is `width / height`.
+ """
+ ret = np.asarray(tlwh).copy()
+ ret[:2] += ret[2:] / 2
+ ret[2] /= ret[3]
+ return ret
+
+ def to_xyah(self):
+ return self.tlwh_to_xyah(self.tlwh)
+
+ @staticmethod
+ # @jit(nopython=True)
+ def tlbr_to_tlwh(tlbr):
+ ret = np.asarray(tlbr).copy()
+ ret[2:] -= ret[:2]
+ return ret
+
+ @staticmethod
+ # @jit(nopython=True)
+ def tlwh_to_tlbr(tlwh):
+ ret = np.asarray(tlwh).copy()
+ ret[2:] += ret[:2]
+ return ret
+
+ def __repr__(self):
+ return 'OT_{}_({}-{})'.format(self.track_id, self.start_frame, self.end_frame)
+
+
+class BYTETracker(object):
+ def __init__(self, args, frame_rate=30):
+ self.tracked_stracks = [] # type: list[STrack]
+ self.lost_stracks = [] # type: list[STrack]
+ self.removed_stracks = [] # type: list[STrack]
+
+ self.frame_id = 0
+ self.args = args
+ #self.det_thresh = args.track_thresh
+ self.det_thresh = args.track_thresh + 0.1
+ self.buffer_size = int(frame_rate / 30.0 * args.track_buffer)
+ self.max_time_lost = self.buffer_size
+ self.kalman_filter = KalmanFilter()
+
+ def update(self, output_results, img_info, img_size):
+ self.frame_id += 1
+ activated_starcks = []
+ refind_stracks = []
+ lost_stracks = []
+ removed_stracks = []
+
+ classes = output_results[:, 5]
+ scores = output_results[:, 4]
+ bboxes = output_results[:, :4]
+
+ img_h, img_w = img_info[0], img_info[1]
+ scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w))
+ bboxes /= scale
+
+ remain_inds = scores > self.args.track_thresh
+ inds_low = scores > 0.1
+ inds_high = scores < self.args.track_thresh
+
+ inds_second = np.logical_and(inds_low, inds_high)
+ dets_second = bboxes[inds_second]
+ dets = bboxes[remain_inds]
+ scores_keep = scores[remain_inds]
+ scores_second = scores[inds_second]
+ classes_keep = classes[remain_inds]
+ classes_second = classes[inds_second]
+
+ if len(dets) > 0:
+ '''Detections'''
+ detections = [STrack(STrack.tlbr_to_tlwh(tlbr), s, c) for
+ (tlbr, s, c) in zip(dets, scores_keep, classes_keep)]
+ else:
+ detections = []
+
+ ''' Add newly detected tracklets to tracked_stracks'''
+ unconfirmed = []
+ tracked_stracks = [] # type: list[STrack]
+ for track in self.tracked_stracks:
+ if not track.is_activated:
+ unconfirmed.append(track)
+ else:
+ tracked_stracks.append(track)
+
+ ''' Step 2: First association, with high score detection boxes'''
+ strack_pool = joint_stracks(tracked_stracks, self.lost_stracks)
+ # Predict the current location with KF
+ STrack.multi_predict(strack_pool)
+ dists = matching.iou_distance(strack_pool, detections)
+ if not self.args.mot20:
+ dists = matching.fuse_score(dists, detections)
+ matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.args.match_thresh)
+
+ for itracked, idet in matches:
+ track = strack_pool[itracked]
+ det = detections[idet]
+ if track.state == TrackState.Tracked:
+ track.update(detections[idet], self.frame_id)
+ activated_starcks.append(track)
+ else:
+ track.re_activate(det, self.frame_id, new_id=False)
+ refind_stracks.append(track)
+
+ ''' Step 3: Second association, with low score detection boxes'''
+ # association the untrack to the low score detections
+ if len(dets_second) > 0:
+ '''Detections'''
+ detections_second = [STrack(STrack.tlbr_to_tlwh(tlbr), s, c) for
+ (tlbr, s, c) in zip(dets_second, scores_second, classes_second)]
+ else:
+ detections_second = []
+ r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked]
+ dists = matching.iou_distance(r_tracked_stracks, detections_second)
+ matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.5)
+ for itracked, idet in matches:
+ track = r_tracked_stracks[itracked]
+ det = detections_second[idet]
+ if track.state == TrackState.Tracked:
+ track.update(det, self.frame_id)
+ activated_starcks.append(track)
+ else:
+ track.re_activate(det, self.frame_id, new_id=False)
+ refind_stracks.append(track)
+
+ for it in u_track:
+ track = r_tracked_stracks[it]
+ if not track.state == TrackState.Lost:
+ track.mark_lost()
+ lost_stracks.append(track)
+
+ '''Deal with unconfirmed tracks, usually tracks with only one beginning frame'''
+ detections = [detections[i] for i in u_detection]
+ dists = matching.iou_distance(unconfirmed, detections)
+ if not self.args.mot20:
+ dists = matching.fuse_score(dists, detections)
+ matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7)
+ for itracked, idet in matches:
+ unconfirmed[itracked].update(detections[idet], self.frame_id)
+ activated_starcks.append(unconfirmed[itracked])
+ for it in u_unconfirmed:
+ track = unconfirmed[it]
+ track.mark_removed()
+ removed_stracks.append(track)
+
+ """ Step 4: Init new stracks"""
+ for inew in u_detection:
+ track = detections[inew]
+ if track.score < self.det_thresh:
+ continue
+ track.activate(self.kalman_filter, self.frame_id)
+ activated_starcks.append(track)
+ """ Step 5: Update state"""
+ for track in self.lost_stracks:
+ if self.frame_id - track.end_frame > self.max_time_lost:
+ track.mark_removed()
+ removed_stracks.append(track)
+
+ # print('Ramained match {} s'.format(t4-t3))
+
+ self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked]
+ self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks)
+ self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks)
+ self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks)
+ self.lost_stracks.extend(lost_stracks)
+ self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks)
+ self.removed_stracks.extend(removed_stracks)
+ self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks)
+ # get scores of lost tracks
+ output_stracks = [track for track in self.tracked_stracks if track.is_activated]
+
+ return output_stracks
+
+
+def joint_stracks(tlista, tlistb):
+ exists = {}
+ res = []
+ for t in tlista:
+ exists[t.track_id] = 1
+ res.append(t)
+ for t in tlistb:
+ tid = t.track_id
+ if not exists.get(tid, 0):
+ exists[tid] = 1
+ res.append(t)
+ return res
+
+
+def sub_stracks(tlista, tlistb):
+ stracks = {}
+ for t in tlista:
+ stracks[t.track_id] = t
+ for t in tlistb:
+ tid = t.track_id
+ if stracks.get(tid, 0):
+ del stracks[tid]
+ return list(stracks.values())
+
+
+def remove_duplicate_stracks(stracksa, stracksb):
+ pdist = matching.iou_distance(stracksa, stracksb)
+ pairs = np.where(pdist < 0.15)
+ dupa, dupb = list(), list()
+ for p, q in zip(*pairs):
+ timep = stracksa[p].frame_id - stracksa[p].start_frame
+ timeq = stracksb[q].frame_id - stracksb[q].start_frame
+ if timep > timeq:
+ dupb.append(q)
+ else:
+ dupa.append(p)
+ resa = [t for i, t in enumerate(stracksa) if not i in dupa]
+ resb = [t for i, t in enumerate(stracksb) if not i in dupb]
+ return resa, resb
diff --git a/experimental/yolox/tracker/kalman_filter.py b/experimental/yolox/tracker/kalman_filter.py
new file mode 100644
index 0000000..deda8a2
--- /dev/null
+++ b/experimental/yolox/tracker/kalman_filter.py
@@ -0,0 +1,270 @@
+# vim: expandtab:ts=4:sw=4
+import numpy as np
+import scipy.linalg
+
+
+"""
+Table for the 0.95 quantile of the chi-square distribution with N degrees of
+freedom (contains values for N=1, ..., 9). Taken from MATLAB/Octave's chi2inv
+function and used as Mahalanobis gating threshold.
+"""
+chi2inv95 = {
+ 1: 3.8415,
+ 2: 5.9915,
+ 3: 7.8147,
+ 4: 9.4877,
+ 5: 11.070,
+ 6: 12.592,
+ 7: 14.067,
+ 8: 15.507,
+ 9: 16.919}
+
+
+class KalmanFilter(object):
+ """
+ A simple Kalman filter for tracking bounding boxes in image space.
+
+ The 8-dimensional state space
+
+ x, y, a, h, vx, vy, va, vh
+
+ contains the bounding box center position (x, y), aspect ratio a, height h,
+ and their respective velocities.
+
+ Object motion follows a constant velocity model. The bounding box location
+ (x, y, a, h) is taken as direct observation of the state space (linear
+ observation model).
+
+ """
+
+ def __init__(self):
+ ndim, dt = 4, 1.
+
+ # Create Kalman filter model matrices.
+ self._motion_mat = np.eye(2 * ndim, 2 * ndim)
+ for i in range(ndim):
+ self._motion_mat[i, ndim + i] = dt
+ self._update_mat = np.eye(ndim, 2 * ndim)
+
+ # Motion and observation uncertainty are chosen relative to the current
+ # state estimate. These weights control the amount of uncertainty in
+ # the model. This is a bit hacky.
+ self._std_weight_position = 1. / 20
+ self._std_weight_velocity = 1. / 160
+
+ def initiate(self, measurement):
+ """Create track from unassociated measurement.
+
+ Parameters
+ ----------
+ measurement : ndarray
+ Bounding box coordinates (x, y, a, h) with center position (x, y),
+ aspect ratio a, and height h.
+
+ Returns
+ -------
+ (ndarray, ndarray)
+ Returns the mean vector (8 dimensional) and covariance matrix (8x8
+ dimensional) of the new track. Unobserved velocities are initialized
+ to 0 mean.
+
+ """
+ mean_pos = measurement
+ mean_vel = np.zeros_like(mean_pos)
+ mean = np.r_[mean_pos, mean_vel]
+
+ std = [
+ 2 * self._std_weight_position * measurement[3],
+ 2 * self._std_weight_position * measurement[3],
+ 1e-2,
+ 2 * self._std_weight_position * measurement[3],
+ 10 * self._std_weight_velocity * measurement[3],
+ 10 * self._std_weight_velocity * measurement[3],
+ 1e-5,
+ 10 * self._std_weight_velocity * measurement[3]]
+ covariance = np.diag(np.square(std))
+ return mean, covariance
+
+ def predict(self, mean, covariance):
+ """Run Kalman filter prediction step.
+
+ Parameters
+ ----------
+ mean : ndarray
+ The 8 dimensional mean vector of the object state at the previous
+ time step.
+ covariance : ndarray
+ The 8x8 dimensional covariance matrix of the object state at the
+ previous time step.
+
+ Returns
+ -------
+ (ndarray, ndarray)
+ Returns the mean vector and covariance matrix of the predicted
+ state. Unobserved velocities are initialized to 0 mean.
+
+ """
+ std_pos = [
+ self._std_weight_position * mean[3],
+ self._std_weight_position * mean[3],
+ 1e-2,
+ self._std_weight_position * mean[3]]
+ std_vel = [
+ self._std_weight_velocity * mean[3],
+ self._std_weight_velocity * mean[3],
+ 1e-5,
+ self._std_weight_velocity * mean[3]]
+ motion_cov = np.diag(np.square(np.r_[std_pos, std_vel]))
+
+ #mean = np.dot(self._motion_mat, mean)
+ mean = np.dot(mean, self._motion_mat.T)
+ covariance = np.linalg.multi_dot((
+ self._motion_mat, covariance, self._motion_mat.T)) + motion_cov
+
+ return mean, covariance
+
+ def project(self, mean, covariance):
+ """Project state distribution to measurement space.
+
+ Parameters
+ ----------
+ mean : ndarray
+ The state's mean vector (8 dimensional array).
+ covariance : ndarray
+ The state's covariance matrix (8x8 dimensional).
+
+ Returns
+ -------
+ (ndarray, ndarray)
+ Returns the projected mean and covariance matrix of the given state
+ estimate.
+
+ """
+ std = [
+ self._std_weight_position * mean[3],
+ self._std_weight_position * mean[3],
+ 1e-1,
+ self._std_weight_position * mean[3]]
+ innovation_cov = np.diag(np.square(std))
+
+ mean = np.dot(self._update_mat, mean)
+ covariance = np.linalg.multi_dot((
+ self._update_mat, covariance, self._update_mat.T))
+ return mean, covariance + innovation_cov
+
+ def multi_predict(self, mean, covariance):
+ """Run Kalman filter prediction step (Vectorized version).
+ Parameters
+ ----------
+ mean : ndarray
+ The Nx8 dimensional mean matrix of the object states at the previous
+ time step.
+ covariance : ndarray
+ The Nx8x8 dimensional covariance matrics of the object states at the
+ previous time step.
+ Returns
+ -------
+ (ndarray, ndarray)
+ Returns the mean vector and covariance matrix of the predicted
+ state. Unobserved velocities are initialized to 0 mean.
+ """
+ std_pos = [
+ self._std_weight_position * mean[:, 3],
+ self._std_weight_position * mean[:, 3],
+ 1e-2 * np.ones_like(mean[:, 3]),
+ self._std_weight_position * mean[:, 3]]
+ std_vel = [
+ self._std_weight_velocity * mean[:, 3],
+ self._std_weight_velocity * mean[:, 3],
+ 1e-5 * np.ones_like(mean[:, 3]),
+ self._std_weight_velocity * mean[:, 3]]
+ sqr = np.square(np.r_[std_pos, std_vel]).T
+
+ motion_cov = []
+ for i in range(len(mean)):
+ motion_cov.append(np.diag(sqr[i]))
+ motion_cov = np.asarray(motion_cov)
+
+ mean = np.dot(mean, self._motion_mat.T)
+ left = np.dot(self._motion_mat, covariance).transpose((1, 0, 2))
+ covariance = np.dot(left, self._motion_mat.T) + motion_cov
+
+ return mean, covariance
+
+ def update(self, mean, covariance, measurement):
+ """Run Kalman filter correction step.
+
+ Parameters
+ ----------
+ mean : ndarray
+ The predicted state's mean vector (8 dimensional).
+ covariance : ndarray
+ The state's covariance matrix (8x8 dimensional).
+ measurement : ndarray
+ The 4 dimensional measurement vector (x, y, a, h), where (x, y)
+ is the center position, a the aspect ratio, and h the height of the
+ bounding box.
+
+ Returns
+ -------
+ (ndarray, ndarray)
+ Returns the measurement-corrected state distribution.
+
+ """
+ projected_mean, projected_cov = self.project(mean, covariance)
+
+ chol_factor, lower = scipy.linalg.cho_factor(
+ projected_cov, lower=True, check_finite=False)
+ kalman_gain = scipy.linalg.cho_solve(
+ (chol_factor, lower), np.dot(covariance, self._update_mat.T).T,
+ check_finite=False).T
+ innovation = measurement - projected_mean
+
+ new_mean = mean + np.dot(innovation, kalman_gain.T)
+ new_covariance = covariance - np.linalg.multi_dot((
+ kalman_gain, projected_cov, kalman_gain.T))
+ return new_mean, new_covariance
+
+ def gating_distance(self, mean, covariance, measurements,
+ only_position=False, metric='maha'):
+ """Compute gating distance between state distribution and measurements.
+ A suitable distance threshold can be obtained from `chi2inv95`. If
+ `only_position` is False, the chi-square distribution has 4 degrees of
+ freedom, otherwise 2.
+ Parameters
+ ----------
+ mean : ndarray
+ Mean vector over the state distribution (8 dimensional).
+ covariance : ndarray
+ Covariance of the state distribution (8x8 dimensional).
+ measurements : ndarray
+ An Nx4 dimensional matrix of N measurements, each in
+ format (x, y, a, h) where (x, y) is the bounding box center
+ position, a the aspect ratio, and h the height.
+ only_position : Optional[bool]
+ If True, distance computation is done with respect to the bounding
+ box center position only.
+ Returns
+ -------
+ ndarray
+ Returns an array of length N, where the i-th element contains the
+ squared Mahalanobis distance between (mean, covariance) and
+ `measurements[i]`.
+ """
+ mean, covariance = self.project(mean, covariance)
+ if only_position:
+ mean, covariance = mean[:2], covariance[:2, :2]
+ measurements = measurements[:, :2]
+
+ d = measurements - mean
+ if metric == 'gaussian':
+ return np.sum(d * d, axis=1)
+ elif metric == 'maha':
+ cholesky_factor = np.linalg.cholesky(covariance)
+ z = scipy.linalg.solve_triangular(
+ cholesky_factor, d.T, lower=True, check_finite=False,
+ overwrite_b=True)
+ squared_maha = np.sum(z * z, axis=0)
+ return squared_maha
+ else:
+ raise ValueError('invalid distance metric')
\ No newline at end of file
diff --git a/experimental/yolox/tracker/matching.py b/experimental/yolox/tracker/matching.py
new file mode 100644
index 0000000..d36a6cf
--- /dev/null
+++ b/experimental/yolox/tracker/matching.py
@@ -0,0 +1,181 @@
+import cv2
+import numpy as np
+import scipy
+import lap
+from scipy.spatial.distance import cdist
+
+from cython_bbox import bbox_overlaps as bbox_ious
+from yolox.tracker import kalman_filter
+import time
+
+def merge_matches(m1, m2, shape):
+ O,P,Q = shape
+ m1 = np.asarray(m1)
+ m2 = np.asarray(m2)
+
+ M1 = scipy.sparse.coo_matrix((np.ones(len(m1)), (m1[:, 0], m1[:, 1])), shape=(O, P))
+ M2 = scipy.sparse.coo_matrix((np.ones(len(m2)), (m2[:, 0], m2[:, 1])), shape=(P, Q))
+
+ mask = M1*M2
+ match = mask.nonzero()
+ match = list(zip(match[0], match[1]))
+ unmatched_O = tuple(set(range(O)) - set([i for i, j in match]))
+ unmatched_Q = tuple(set(range(Q)) - set([j for i, j in match]))
+
+ return match, unmatched_O, unmatched_Q
+
+
+def _indices_to_matches(cost_matrix, indices, thresh):
+ matched_cost = cost_matrix[tuple(zip(*indices))]
+ matched_mask = (matched_cost <= thresh)
+
+ matches = indices[matched_mask]
+ unmatched_a = tuple(set(range(cost_matrix.shape[0])) - set(matches[:, 0]))
+ unmatched_b = tuple(set(range(cost_matrix.shape[1])) - set(matches[:, 1]))
+
+ return matches, unmatched_a, unmatched_b
+
+
+def linear_assignment(cost_matrix, thresh):
+ if cost_matrix.size == 0:
+ return np.empty((0, 2), dtype=int), tuple(range(cost_matrix.shape[0])), tuple(range(cost_matrix.shape[1]))
+ matches, unmatched_a, unmatched_b = [], [], []
+ cost, x, y = lap.lapjv(cost_matrix, extend_cost=True, cost_limit=thresh)
+ for ix, mx in enumerate(x):
+ if mx >= 0:
+ matches.append([ix, mx])
+ unmatched_a = np.where(x < 0)[0]
+ unmatched_b = np.where(y < 0)[0]
+ matches = np.asarray(matches)
+ return matches, unmatched_a, unmatched_b
+
+
+def ious(atlbrs, btlbrs):
+ """
+ Compute cost based on IoU
+ :type atlbrs: list[tlbr] | np.ndarray
+ :type atlbrs: list[tlbr] | np.ndarray
+
+ :rtype ious np.ndarray
+ """
+ ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float)
+ if ious.size == 0:
+ return ious
+
+ ious = bbox_ious(
+ np.ascontiguousarray(atlbrs, dtype=np.float),
+ np.ascontiguousarray(btlbrs, dtype=np.float)
+ )
+
+ return ious
+
+
+def iou_distance(atracks, btracks):
+ """
+ Compute cost based on IoU
+ :type atracks: list[STrack]
+ :type btracks: list[STrack]
+
+ :rtype cost_matrix np.ndarray
+ """
+
+ if (len(atracks)>0 and isinstance(atracks[0], np.ndarray)) or (len(btracks) > 0 and isinstance(btracks[0], np.ndarray)):
+ atlbrs = atracks
+ btlbrs = btracks
+ else:
+ atlbrs = [track.tlbr for track in atracks]
+ btlbrs = [track.tlbr for track in btracks]
+ _ious = ious(atlbrs, btlbrs)
+ cost_matrix = 1 - _ious
+
+ return cost_matrix
+
+def v_iou_distance(atracks, btracks):
+ """
+ Compute cost based on IoU
+ :type atracks: list[STrack]
+ :type btracks: list[STrack]
+
+ :rtype cost_matrix np.ndarray
+ """
+
+ if (len(atracks)>0 and isinstance(atracks[0], np.ndarray)) or (len(btracks) > 0 and isinstance(btracks[0], np.ndarray)):
+ atlbrs = atracks
+ btlbrs = btracks
+ else:
+ atlbrs = [track.tlwh_to_tlbr(track.pred_bbox) for track in atracks]
+ btlbrs = [track.tlwh_to_tlbr(track.pred_bbox) for track in btracks]
+ _ious = ious(atlbrs, btlbrs)
+ cost_matrix = 1 - _ious
+
+ return cost_matrix
+
+def embedding_distance(tracks, detections, metric='cosine'):
+ """
+ :param tracks: list[STrack]
+ :param detections: list[BaseTrack]
+ :param metric:
+ :return: cost_matrix np.ndarray
+ """
+
+ cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float)
+ if cost_matrix.size == 0:
+ return cost_matrix
+ det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float)
+ #for i, track in enumerate(tracks):
+ #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric))
+ track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float)
+ cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features
+ return cost_matrix
+
+
+def gate_cost_matrix(kf, cost_matrix, tracks, detections, only_position=False):
+ if cost_matrix.size == 0:
+ return cost_matrix
+ gating_dim = 2 if only_position else 4
+ gating_threshold = kalman_filter.chi2inv95[gating_dim]
+ measurements = np.asarray([det.to_xyah() for det in detections])
+ for row, track in enumerate(tracks):
+ gating_distance = kf.gating_distance(
+ track.mean, track.covariance, measurements, only_position)
+ cost_matrix[row, gating_distance > gating_threshold] = np.inf
+ return cost_matrix
+
+
+def fuse_motion(kf, cost_matrix, tracks, detections, only_position=False, lambda_=0.98):
+ if cost_matrix.size == 0:
+ return cost_matrix
+ gating_dim = 2 if only_position else 4
+ gating_threshold = kalman_filter.chi2inv95[gating_dim]
+ measurements = np.asarray([det.to_xyah() for det in detections])
+ for row, track in enumerate(tracks):
+ gating_distance = kf.gating_distance(
+ track.mean, track.covariance, measurements, only_position, metric='maha')
+ cost_matrix[row, gating_distance > gating_threshold] = np.inf
+ cost_matrix[row] = lambda_ * cost_matrix[row] + (1 - lambda_) * gating_distance
+ return cost_matrix
+
+
+def fuse_iou(cost_matrix, tracks, detections):
+ if cost_matrix.size == 0:
+ return cost_matrix
+ reid_sim = 1 - cost_matrix
+ iou_dist = iou_distance(tracks, detections)
+ iou_sim = 1 - iou_dist
+ fuse_sim = reid_sim * (1 + iou_sim) / 2
+ det_scores = np.array([det.score for det in detections])
+ det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0)
+ #fuse_sim = fuse_sim * (1 + det_scores) / 2
+ fuse_cost = 1 - fuse_sim
+ return fuse_cost
+
+
+def fuse_score(cost_matrix, detections):
+ if cost_matrix.size == 0:
+ return cost_matrix
+ iou_sim = 1 - cost_matrix
+ det_scores = np.array([det.score for det in detections])
+ det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0)
+ fuse_sim = iou_sim * det_scores
+ fuse_cost = 1 - fuse_sim
+ return fuse_cost
\ No newline at end of file
diff --git a/experimental/yolox/tracking_utils/evaluation.py b/experimental/yolox/tracking_utils/evaluation.py
new file mode 100644
index 0000000..6e2f063
--- /dev/null
+++ b/experimental/yolox/tracking_utils/evaluation.py
@@ -0,0 +1,113 @@
+import os
+import numpy as np
+import copy
+import motmetrics as mm
+mm.lap.default_solver = 'lap'
+
+from yolox.tracking_utils.io import read_results, unzip_objs
+
+
+class Evaluator(object):
+
+ def __init__(self, data_root, seq_name, data_type):
+ self.data_root = data_root
+ self.seq_name = seq_name
+ self.data_type = data_type
+
+ self.load_annotations()
+ self.reset_accumulator()
+
+ def load_annotations(self):
+ assert self.data_type == 'mot'
+
+ gt_filename = os.path.join(self.data_root, self.seq_name, 'gt', 'gt.txt')
+ self.gt_frame_dict = read_results(gt_filename, self.data_type, is_gt=True)
+ self.gt_ignore_frame_dict = read_results(gt_filename, self.data_type, is_ignore=True)
+
+ def reset_accumulator(self):
+ self.acc = mm.MOTAccumulator(auto_id=True)
+
+ def eval_frame(self, frame_id, trk_tlwhs, trk_ids, rtn_events=False):
+ # results
+ trk_tlwhs = np.copy(trk_tlwhs)
+ trk_ids = np.copy(trk_ids)
+
+ # gts
+ gt_objs = self.gt_frame_dict.get(frame_id, [])
+ gt_tlwhs, gt_ids = unzip_objs(gt_objs)[:2]
+
+ # ignore boxes
+ ignore_objs = self.gt_ignore_frame_dict.get(frame_id, [])
+ ignore_tlwhs = unzip_objs(ignore_objs)[0]
+
+ # remove ignored results
+ keep = np.ones(len(trk_tlwhs), dtype=bool)
+ iou_distance = mm.distances.iou_matrix(ignore_tlwhs, trk_tlwhs, max_iou=0.5)
+ if len(iou_distance) > 0:
+ match_is, match_js = mm.lap.linear_sum_assignment(iou_distance)
+ match_is, match_js = map(lambda a: np.asarray(a, dtype=int), [match_is, match_js])
+ match_ious = iou_distance[match_is, match_js]
+
+ match_js = np.asarray(match_js, dtype=int)
+ match_js = match_js[np.logical_not(np.isnan(match_ious))]
+ keep[match_js] = False
+ trk_tlwhs = trk_tlwhs[keep]
+ trk_ids = trk_ids[keep]
+ #match_is, match_js = mm.lap.linear_sum_assignment(iou_distance)
+ #match_is, match_js = map(lambda a: np.asarray(a, dtype=int), [match_is, match_js])
+ #match_ious = iou_distance[match_is, match_js]
+
+ #match_js = np.asarray(match_js, dtype=int)
+ #match_js = match_js[np.logical_not(np.isnan(match_ious))]
+ #keep[match_js] = False
+ #trk_tlwhs = trk_tlwhs[keep]
+ #trk_ids = trk_ids[keep]
+
+ # get distance matrix
+ iou_distance = mm.distances.iou_matrix(gt_tlwhs, trk_tlwhs, max_iou=0.5)
+
+ # acc
+ self.acc.update(gt_ids, trk_ids, iou_distance)
+
+ if rtn_events and iou_distance.size > 0 and hasattr(self.acc, 'last_mot_events'):
+ events = self.acc.last_mot_events # only supported by https://github.com/longcw/py-motmetrics
+ else:
+ events = None
+ return events
+
+ def eval_file(self, filename):
+ self.reset_accumulator()
+
+ result_frame_dict = read_results(filename, self.data_type, is_gt=False)
+ #frames = sorted(list(set(self.gt_frame_dict.keys()) | set(result_frame_dict.keys())))
+ frames = sorted(list(set(result_frame_dict.keys())))
+ for frame_id in frames:
+ trk_objs = result_frame_dict.get(frame_id, [])
+ trk_tlwhs, trk_ids = unzip_objs(trk_objs)[:2]
+ self.eval_frame(frame_id, trk_tlwhs, trk_ids, rtn_events=False)
+
+ return self.acc
+
+ @staticmethod
+ def get_summary(accs, names, metrics=('mota', 'num_switches', 'idp', 'idr', 'idf1', 'precision', 'recall')):
+ names = copy.deepcopy(names)
+ if metrics is None:
+ metrics = mm.metrics.motchallenge_metrics
+ metrics = copy.deepcopy(metrics)
+
+ mh = mm.metrics.create()
+ summary = mh.compute_many(
+ accs,
+ metrics=metrics,
+ names=names,
+ generate_overall=True
+ )
+
+ return summary
+
+ @staticmethod
+ def save_summary(summary, filename):
+ import pandas as pd
+ writer = pd.ExcelWriter(filename)
+ summary.to_excel(writer)
+ writer.save()
\ No newline at end of file
diff --git a/experimental/yolox/tracking_utils/io.py b/experimental/yolox/tracking_utils/io.py
new file mode 100644
index 0000000..5c8b053
--- /dev/null
+++ b/experimental/yolox/tracking_utils/io.py
@@ -0,0 +1,116 @@
+import os
+from typing import Dict
+import numpy as np
+
+
+def write_results(filename, results_dict: Dict, data_type: str):
+ if not filename:
+ return
+ path = os.path.dirname(filename)
+ if not os.path.exists(path):
+ os.makedirs(path)
+
+ if data_type in ('mot', 'mcmot', 'lab'):
+ save_format = '{frame},{id},{x1},{y1},{w},{h},1,-1,-1,-1\n'
+ elif data_type == 'kitti':
+ save_format = '{frame} {id} pedestrian -1 -1 -10 {x1} {y1} {x2} {y2} -1 -1 -1 -1000 -1000 -1000 -10 {score}\n'
+ else:
+ raise ValueError(data_type)
+
+ with open(filename, 'w') as f:
+ for frame_id, frame_data in results_dict.items():
+ if data_type == 'kitti':
+ frame_id -= 1
+ for tlwh, track_id in frame_data:
+ if track_id < 0:
+ continue
+ x1, y1, w, h = tlwh
+ x2, y2 = x1 + w, y1 + h
+ line = save_format.format(frame=frame_id, id=track_id, x1=x1, y1=y1, x2=x2, y2=y2, w=w, h=h, score=1.0)
+ f.write(line)
+
+
+def read_results(filename, data_type: str, is_gt=False, is_ignore=False):
+ if data_type in ('mot', 'lab'):
+ read_fun = read_mot_results
+ else:
+ raise ValueError('Unknown data type: {}'.format(data_type))
+
+ return read_fun(filename, is_gt, is_ignore)
+
+
+"""
+labels={'ped', ... % 1
+'person_on_vhcl', ... % 2
+'car', ... % 3
+'bicycle', ... % 4
+'mbike', ... % 5
+'non_mot_vhcl', ... % 6
+'static_person', ... % 7
+'distractor', ... % 8
+'occluder', ... % 9
+'occluder_on_grnd', ... %10
+'occluder_full', ... % 11
+'reflection', ... % 12
+'crowd' ... % 13
+};
+"""
+
+
+def read_mot_results(filename, is_gt, is_ignore):
+ valid_labels = {1}
+ ignore_labels = {2, 7, 8, 12}
+ results_dict = dict()
+ if os.path.isfile(filename):
+ with open(filename, 'r') as f:
+ for line in f.readlines():
+ linelist = line.split(',')
+ if len(linelist) < 7:
+ continue
+ fid = int(linelist[0])
+ if fid < 1:
+ continue
+ results_dict.setdefault(fid, list())
+
+ box_size = float(linelist[4]) * float(linelist[5])
+
+ if is_gt:
+ if 'MOT16-' in filename or 'MOT17-' in filename:
+ label = int(float(linelist[7]))
+ mark = int(float(linelist[6]))
+ if mark == 0 or label not in valid_labels:
+ continue
+ score = 1
+ elif is_ignore:
+ if 'MOT16-' in filename or 'MOT17-' in filename:
+ label = int(float(linelist[7]))
+ vis_ratio = float(linelist[8])
+ if label not in ignore_labels and vis_ratio >= 0:
+ continue
+ else:
+ continue
+ score = 1
+ else:
+ score = float(linelist[6])
+
+ #if box_size > 7000:
+ #if box_size <= 7000 or box_size >= 15000:
+ #if box_size < 15000:
+ #continue
+
+ tlwh = tuple(map(float, linelist[2:6]))
+ target_id = int(linelist[1])
+
+ results_dict[fid].append((tlwh, target_id, score))
+
+ return results_dict
+
+
+def unzip_objs(objs):
+ if len(objs) > 0:
+ tlwhs, ids, scores = zip(*objs)
+ else:
+ tlwhs, ids, scores = [], [], []
+ tlwhs = np.asarray(tlwhs, dtype=float).reshape(-1, 4)
+
+ return tlwhs, ids, scores
\ No newline at end of file
diff --git a/experimental/yolox/tracking_utils/timer.py b/experimental/yolox/tracking_utils/timer.py
new file mode 100644
index 0000000..c9b15fb
--- /dev/null
+++ b/experimental/yolox/tracking_utils/timer.py
@@ -0,0 +1,37 @@
+import time
+
+
+class Timer(object):
+ """A simple timer."""
+ def __init__(self):
+ self.total_time = 0.
+ self.calls = 0
+ self.start_time = 0.
+ self.diff = 0.
+ self.average_time = 0.
+
+ self.duration = 0.
+
+ def tic(self):
+ # using time.time instead of time.clock because time time.clock
+ # does not normalize for multithreading
+ self.start_time = time.time()
+
+ def toc(self, average=True):
+ self.diff = time.time() - self.start_time
+ self.total_time += self.diff
+ self.calls += 1
+ self.average_time = self.total_time / self.calls
+ if average:
+ self.duration = self.average_time
+ else:
+ self.duration = self.diff
+ return self.duration
+
+ def clear(self):
+ self.total_time = 0.
+ self.calls = 0
+ self.start_time = 0.
+ self.diff = 0.
+ self.average_time = 0.
+ self.duration = 0.
\ No newline at end of file
diff --git a/experimental/yolox/utils/__init__.py b/experimental/yolox/utils/__init__.py
new file mode 100644
index 0000000..a268c1a
--- /dev/null
+++ b/experimental/yolox/utils/__init__.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+from .allreduce_norm import *
+from .boxes import *
+from .checkpoint import load_ckpt, save_checkpoint
+from .demo_utils import *
+from .dist import *
+from .ema import ModelEMA
+from .logger import setup_logger
+from .lr_scheduler import LRScheduler
+from .metric import *
+from .model_utils import *
+from .setup_env import *
+from .visualize import *
diff --git a/experimental/yolox/utils/__pycache__/__init__.cpython-310.pyc b/experimental/yolox/utils/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000..4e40556
Binary files /dev/null and b/experimental/yolox/utils/__pycache__/__init__.cpython-310.pyc differ
diff --git a/experimental/yolox/utils/__pycache__/allreduce_norm.cpython-310.pyc b/experimental/yolox/utils/__pycache__/allreduce_norm.cpython-310.pyc
new file mode 100644
index 0000000..645044d
Binary files /dev/null and b/experimental/yolox/utils/__pycache__/allreduce_norm.cpython-310.pyc differ
diff --git a/experimental/yolox/utils/__pycache__/boxes.cpython-310.pyc b/experimental/yolox/utils/__pycache__/boxes.cpython-310.pyc
new file mode 100644
index 0000000..dafe9e8
Binary files /dev/null and b/experimental/yolox/utils/__pycache__/boxes.cpython-310.pyc differ
diff --git a/experimental/yolox/utils/__pycache__/checkpoint.cpython-310.pyc b/experimental/yolox/utils/__pycache__/checkpoint.cpython-310.pyc
new file mode 100644
index 0000000..816b801
Binary files /dev/null and b/experimental/yolox/utils/__pycache__/checkpoint.cpython-310.pyc differ
diff --git a/experimental/yolox/utils/__pycache__/demo_utils.cpython-310.pyc b/experimental/yolox/utils/__pycache__/demo_utils.cpython-310.pyc
new file mode 100644
index 0000000..b7a8e50
Binary files /dev/null and b/experimental/yolox/utils/__pycache__/demo_utils.cpython-310.pyc differ
diff --git a/experimental/yolox/utils/__pycache__/dist.cpython-310.pyc b/experimental/yolox/utils/__pycache__/dist.cpython-310.pyc
new file mode 100644
index 0000000..a6ca6ad
Binary files /dev/null and b/experimental/yolox/utils/__pycache__/dist.cpython-310.pyc differ
diff --git a/experimental/yolox/utils/__pycache__/ema.cpython-310.pyc b/experimental/yolox/utils/__pycache__/ema.cpython-310.pyc
new file mode 100644
index 0000000..a64001f
Binary files /dev/null and b/experimental/yolox/utils/__pycache__/ema.cpython-310.pyc differ
diff --git a/experimental/yolox/utils/__pycache__/logger.cpython-310.pyc b/experimental/yolox/utils/__pycache__/logger.cpython-310.pyc
new file mode 100644
index 0000000..ad78568
Binary files /dev/null and b/experimental/yolox/utils/__pycache__/logger.cpython-310.pyc differ
diff --git a/experimental/yolox/utils/__pycache__/lr_scheduler.cpython-310.pyc b/experimental/yolox/utils/__pycache__/lr_scheduler.cpython-310.pyc
new file mode 100644
index 0000000..710bddb
Binary files /dev/null and b/experimental/yolox/utils/__pycache__/lr_scheduler.cpython-310.pyc differ
diff --git a/experimental/yolox/utils/__pycache__/metric.cpython-310.pyc b/experimental/yolox/utils/__pycache__/metric.cpython-310.pyc
new file mode 100644
index 0000000..38fc519
Binary files /dev/null and b/experimental/yolox/utils/__pycache__/metric.cpython-310.pyc differ
diff --git a/experimental/yolox/utils/__pycache__/model_utils.cpython-310.pyc b/experimental/yolox/utils/__pycache__/model_utils.cpython-310.pyc
new file mode 100644
index 0000000..f877c02
Binary files /dev/null and b/experimental/yolox/utils/__pycache__/model_utils.cpython-310.pyc differ
diff --git a/experimental/yolox/utils/__pycache__/setup_env.cpython-310.pyc b/experimental/yolox/utils/__pycache__/setup_env.cpython-310.pyc
new file mode 100644
index 0000000..3bc6a51
Binary files /dev/null and b/experimental/yolox/utils/__pycache__/setup_env.cpython-310.pyc differ
diff --git a/experimental/yolox/utils/__pycache__/visualize.cpython-310.pyc b/experimental/yolox/utils/__pycache__/visualize.cpython-310.pyc
new file mode 100644
index 0000000..6d019be
Binary files /dev/null and b/experimental/yolox/utils/__pycache__/visualize.cpython-310.pyc differ
diff --git a/experimental/yolox/utils/allreduce_norm.py b/experimental/yolox/utils/allreduce_norm.py
new file mode 100644
index 0000000..d9b51e2
--- /dev/null
+++ b/experimental/yolox/utils/allreduce_norm.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import torch
+from torch import distributed as dist
+from torch import nn
+
+import pickle
+from collections import OrderedDict
+
+from .dist import _get_global_gloo_group, get_world_size
+
+ASYNC_NORM = (
+ nn.BatchNorm1d,
+ nn.BatchNorm2d,
+ nn.BatchNorm3d,
+ nn.InstanceNorm1d,
+ nn.InstanceNorm2d,
+ nn.InstanceNorm3d,
+)
+
+__all__ = [
+ "get_async_norm_states",
+ "pyobj2tensor",
+ "tensor2pyobj",
+ "all_reduce",
+ "all_reduce_norm",
+]
+
+
+def get_async_norm_states(module):
+ async_norm_states = OrderedDict()
+ for name, child in module.named_modules():
+ if isinstance(child, ASYNC_NORM):
+ for k, v in child.state_dict().items():
+ async_norm_states[".".join([name, k])] = v
+ return async_norm_states
+
+
+def pyobj2tensor(pyobj, device="cuda"):
+ """serialize picklable python object to tensor"""
+ storage = torch.ByteStorage.from_buffer(pickle.dumps(pyobj))
+ return torch.ByteTensor(storage).to(device=device)
+
+
+def tensor2pyobj(tensor):
+ """deserialize tensor to picklable python object"""
+ return pickle.loads(tensor.cpu().numpy().tobytes())
+
+
+def _get_reduce_op(op_name):
+ return {
+ "sum": dist.ReduceOp.SUM,
+ "mean": dist.ReduceOp.SUM,
+ }[op_name.lower()]
+
+
+def all_reduce(py_dict, op="sum", group=None):
+ """
+ Apply all reduce function for python dict object.
+ NOTE: make sure that every py_dict has the same keys and values are in the same shape.
+
+ Args:
+ py_dict (dict): dict to apply all reduce op.
+ op (str): operator, could be "sum" or "mean".
+ """
+ world_size = get_world_size()
+ if world_size == 1:
+ return py_dict
+ if group is None:
+ group = _get_global_gloo_group()
+ if dist.get_world_size(group) == 1:
+ return py_dict
+
+ # all reduce logic across different devices.
+ py_key = list(py_dict.keys())
+ py_key_tensor = pyobj2tensor(py_key)
+ dist.broadcast(py_key_tensor, src=0)
+ py_key = tensor2pyobj(py_key_tensor)
+
+ tensor_shapes = [py_dict[k].shape for k in py_key]
+ tensor_numels = [py_dict[k].numel() for k in py_key]
+
+ flatten_tensor = torch.cat([py_dict[k].flatten() for k in py_key])
+ dist.all_reduce(flatten_tensor, op=_get_reduce_op(op))
+ if op == "mean":
+ flatten_tensor /= world_size
+
+ split_tensors = [
+ x.reshape(shape)
+ for x, shape in zip(torch.split(flatten_tensor, tensor_numels), tensor_shapes)
+ ]
+ return OrderedDict({k: v for k, v in zip(py_key, split_tensors)})
+
+
+def all_reduce_norm(module):
+ """
+ All reduce norm statistics in different devices.
+ """
+ states = get_async_norm_states(module)
+ states = all_reduce(states, op="mean")
+ module.load_state_dict(states, strict=False)
diff --git a/experimental/yolox/utils/boxes.py b/experimental/yolox/utils/boxes.py
new file mode 100644
index 0000000..ac262b9
--- /dev/null
+++ b/experimental/yolox/utils/boxes.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import numpy as np
+
+import torch
+import torchvision
+import torch.nn.functional as F
+
+__all__ = [
+ "filter_box",
+ "postprocess",
+ "bboxes_iou",
+ "matrix_iou",
+ "adjust_box_anns",
+ "xyxy2xywh",
+ "xyxy2cxcywh",
+]
+
+
+def filter_box(output, scale_range):
+ """
+ output: (N, 5+class) shape
+ """
+ min_scale, max_scale = scale_range
+ w = output[:, 2] - output[:, 0]
+ h = output[:, 3] - output[:, 1]
+ keep = (w * h > min_scale * min_scale) & (w * h < max_scale * max_scale)
+ return output[keep]
+
+
+def postprocess(prediction, num_classes, conf_thre=0.7, nms_thre=0.45):
+ box_corner = prediction.new(prediction.shape)
+ box_corner[:, :, 0] = prediction[:, :, 0] - prediction[:, :, 2] / 2
+ box_corner[:, :, 1] = prediction[:, :, 1] - prediction[:, :, 3] / 2
+ box_corner[:, :, 2] = prediction[:, :, 0] + prediction[:, :, 2] / 2
+ box_corner[:, :, 3] = prediction[:, :, 1] + prediction[:, :, 3] / 2
+ prediction[:, :, :4] = box_corner[:, :, :4]
+
+ output = [None for _ in range(len(prediction))]
+ for i, image_pred in enumerate(prediction):
+
+ # If none are remaining => process next image
+ if not image_pred.size(0):
+ continue
+ # Get score and class with highest confidence
+ class_conf, class_pred = torch.max(
+ image_pred[:, 5 : 5 + num_classes], 1, keepdim=True
+ )
+
+ conf_mask = (image_pred[:, 4] * class_conf.squeeze() >= conf_thre).squeeze()
+ # _, conf_mask = torch.topk((image_pred[:, 4] * class_conf.squeeze()), 1000)
+ # Detections ordered as (x1, y1, x2, y2, obj_conf, class_conf, class_pred)
+ detections = torch.cat((image_pred[:, :5], class_conf, class_pred.float()), 1)
+ detections = detections[conf_mask]
+ if not detections.size(0):
+ continue
+
+ nms_out_index = torchvision.ops.batched_nms(
+ detections[:, :4],
+ detections[:, 4] * detections[:, 5],
+ detections[:, 6],
+ nms_thre,
+ )
+ detections = detections[nms_out_index]
+ if output[i] is None:
+ output[i] = detections
+ else:
+ output[i] = torch.cat((output[i], detections))
+
+ return output
+
+
+def bboxes_iou(bboxes_a, bboxes_b, xyxy=True):
+ if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4:
+ raise IndexError
+
+ if xyxy:
+ tl = torch.max(bboxes_a[:, None, :2], bboxes_b[:, :2])
+ br = torch.min(bboxes_a[:, None, 2:], bboxes_b[:, 2:])
+ area_a = torch.prod(bboxes_a[:, 2:] - bboxes_a[:, :2], 1)
+ area_b = torch.prod(bboxes_b[:, 2:] - bboxes_b[:, :2], 1)
+ else:
+ tl = torch.max(
+ (bboxes_a[:, None, :2] - bboxes_a[:, None, 2:] / 2),
+ (bboxes_b[:, :2] - bboxes_b[:, 2:] / 2),
+ )
+ br = torch.min(
+ (bboxes_a[:, None, :2] + bboxes_a[:, None, 2:] / 2),
+ (bboxes_b[:, :2] + bboxes_b[:, 2:] / 2),
+ )
+
+ area_a = torch.prod(bboxes_a[:, 2:], 1)
+ area_b = torch.prod(bboxes_b[:, 2:], 1)
+ en = (tl < br).type(tl.type()).prod(dim=2)
+ area_i = torch.prod(br - tl, 2) * en # * ((tl < br).all())
+ return area_i / (area_a[:, None] + area_b - area_i)
+
+
+def matrix_iou(a, b):
+ """
+ return iou of a and b, numpy version for data augenmentation
+ """
+ lt = np.maximum(a[:, np.newaxis, :2], b[:, :2])
+ rb = np.minimum(a[:, np.newaxis, 2:], b[:, 2:])
+
+ area_i = np.prod(rb - lt, axis=2) * (lt < rb).all(axis=2)
+ area_a = np.prod(a[:, 2:] - a[:, :2], axis=1)
+ area_b = np.prod(b[:, 2:] - b[:, :2], axis=1)
+ return area_i / (area_a[:, np.newaxis] + area_b - area_i + 1e-12)
+
+
+def adjust_box_anns(bbox, scale_ratio, padw, padh, w_max, h_max):
+ #bbox[:, 0::2] = np.clip(bbox[:, 0::2] * scale_ratio + padw, 0, w_max)
+ #bbox[:, 1::2] = np.clip(bbox[:, 1::2] * scale_ratio + padh, 0, h_max)
+ bbox[:, 0::2] = bbox[:, 0::2] * scale_ratio + padw
+ bbox[:, 1::2] = bbox[:, 1::2] * scale_ratio + padh
+ return bbox
+
+
+def xyxy2xywh(bboxes):
+ bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0]
+ bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1]
+ return bboxes
+
+
+def xyxy2cxcywh(bboxes):
+ bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0]
+ bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1]
+ bboxes[:, 0] = bboxes[:, 0] + bboxes[:, 2] * 0.5
+ bboxes[:, 1] = bboxes[:, 1] + bboxes[:, 3] * 0.5
+ return bboxes
diff --git a/experimental/yolox/utils/checkpoint.py b/experimental/yolox/utils/checkpoint.py
new file mode 100644
index 0000000..55903b4
--- /dev/null
+++ b/experimental/yolox/utils/checkpoint.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+from loguru import logger
+
+import torch
+
+import os
+import shutil
+
+
+def load_ckpt(model, ckpt):
+ model_state_dict = model.state_dict()
+ load_dict = {}
+ for key_model, v in model_state_dict.items():
+ if key_model not in ckpt:
+ logger.warning(
+ "{} is not in the ckpt. Please double check and see if this is desired.".format(
+ key_model
+ )
+ )
+ continue
+ v_ckpt = ckpt[key_model]
+ if v.shape != v_ckpt.shape:
+ logger.warning(
+ "Shape of {} in checkpoint is {}, while shape of {} in model is {}.".format(
+ key_model, v_ckpt.shape, key_model, v.shape
+ )
+ )
+ continue
+ load_dict[key_model] = v_ckpt
+
+ model.load_state_dict(load_dict, strict=False)
+ return model
+
+
+def save_checkpoint(state, is_best, save_dir, model_name=""):
+ if not os.path.exists(save_dir):
+ os.makedirs(save_dir)
+ filename = os.path.join(save_dir, model_name + "_ckpt.pth.tar")
+ torch.save(state, filename)
+ if is_best:
+ best_filename = os.path.join(save_dir, "best_ckpt.pth.tar")
+ shutil.copyfile(filename, best_filename)
diff --git a/experimental/yolox/utils/demo_utils.py b/experimental/yolox/utils/demo_utils.py
new file mode 100644
index 0000000..093443c
--- /dev/null
+++ b/experimental/yolox/utils/demo_utils.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import numpy as np
+
+import os
+
+__all__ = ["mkdir", "nms", "multiclass_nms", "demo_postprocess"]
+
+
+def mkdir(path):
+ if not os.path.exists(path):
+ os.makedirs(path)
+
+
+def nms(boxes, scores, nms_thr):
+ """Single class NMS implemented in Numpy."""
+ x1 = boxes[:, 0]
+ y1 = boxes[:, 1]
+ x2 = boxes[:, 2]
+ y2 = boxes[:, 3]
+
+ areas = (x2 - x1 + 1) * (y2 - y1 + 1)
+ order = scores.argsort()[::-1]
+
+ keep = []
+ while order.size > 0:
+ i = order[0]
+ keep.append(i)
+ xx1 = np.maximum(x1[i], x1[order[1:]])
+ yy1 = np.maximum(y1[i], y1[order[1:]])
+ xx2 = np.minimum(x2[i], x2[order[1:]])
+ yy2 = np.minimum(y2[i], y2[order[1:]])
+
+ w = np.maximum(0.0, xx2 - xx1 + 1)
+ h = np.maximum(0.0, yy2 - yy1 + 1)
+ inter = w * h
+ ovr = inter / (areas[i] + areas[order[1:]] - inter)
+
+ inds = np.where(ovr <= nms_thr)[0]
+ order = order[inds + 1]
+
+ return keep
+
+
+def multiclass_nms(boxes, scores, nms_thr, score_thr):
+ """Multiclass NMS implemented in Numpy"""
+ final_dets = []
+ num_classes = scores.shape[1]
+ for cls_ind in range(num_classes):
+ cls_scores = scores[:, cls_ind]
+ valid_score_mask = cls_scores > score_thr
+ if valid_score_mask.sum() == 0:
+ continue
+ else:
+ valid_scores = cls_scores[valid_score_mask]
+ valid_boxes = boxes[valid_score_mask]
+ keep = nms(valid_boxes, valid_scores, nms_thr)
+ if len(keep) > 0:
+ cls_inds = np.ones((len(keep), 1)) * cls_ind
+ dets = np.concatenate(
+ [valid_boxes[keep], valid_scores[keep, None], cls_inds], 1
+ )
+ final_dets.append(dets)
+ if len(final_dets) == 0:
+ return None
+ return np.concatenate(final_dets, 0)
+
+
+def demo_postprocess(outputs, img_size, p6=False):
+
+ grids = []
+ expanded_strides = []
+
+ if not p6:
+ strides = [8, 16, 32]
+ else:
+ strides = [8, 16, 32, 64]
+
+ hsizes = [img_size[0] // stride for stride in strides]
+ wsizes = [img_size[1] // stride for stride in strides]
+
+ for hsize, wsize, stride in zip(hsizes, wsizes, strides):
+ xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize))
+ grid = np.stack((xv, yv), 2).reshape(1, -1, 2)
+ grids.append(grid)
+ shape = grid.shape[:2]
+ expanded_strides.append(np.full((*shape, 1), stride))
+
+ grids = np.concatenate(grids, 1)
+ expanded_strides = np.concatenate(expanded_strides, 1)
+ outputs[..., :2] = (outputs[..., :2] + grids) * expanded_strides
+ outputs[..., 2:4] = np.exp(outputs[..., 2:4]) * expanded_strides
+
+ return outputs
diff --git a/experimental/yolox/utils/dist.py b/experimental/yolox/utils/dist.py
new file mode 100644
index 0000000..691c306
--- /dev/null
+++ b/experimental/yolox/utils/dist.py
@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# This file mainly comes from
+# https://github.com/facebookresearch/detectron2/blob/master/detectron2/utils/comm.py
+# Copyright (c) Facebook, Inc. and its affiliates.
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+"""
+This file contains primitives for multi-gpu communication.
+This is useful when doing distributed training.
+"""
+
+import numpy as np
+
+import torch
+from torch import distributed as dist
+
+import functools
+import logging
+import pickle
+import time
+
+__all__ = [
+ "is_main_process",
+ "synchronize",
+ "get_world_size",
+ "get_rank",
+ "get_local_rank",
+ "get_local_size",
+ "time_synchronized",
+ "gather",
+ "all_gather",
+]
+
+_LOCAL_PROCESS_GROUP = None
+
+
+def synchronize():
+ """
+ Helper function to synchronize (barrier) among all processes when using distributed training
+ """
+ if not dist.is_available():
+ return
+ if not dist.is_initialized():
+ return
+ world_size = dist.get_world_size()
+ if world_size == 1:
+ return
+ dist.barrier()
+
+
+def get_world_size() -> int:
+ if not dist.is_available():
+ return 1
+ if not dist.is_initialized():
+ return 1
+ return dist.get_world_size()
+
+
+def get_rank() -> int:
+ if not dist.is_available():
+ return 0
+ if not dist.is_initialized():
+ return 0
+ return dist.get_rank()
+
+
+def get_local_rank() -> int:
+ """
+ Returns:
+ The rank of the current process within the local (per-machine) process group.
+ """
+ if not dist.is_available():
+ return 0
+ if not dist.is_initialized():
+ return 0
+ assert _LOCAL_PROCESS_GROUP is not None
+ return dist.get_rank(group=_LOCAL_PROCESS_GROUP)
+
+
+def get_local_size() -> int:
+ """
+ Returns:
+ The size of the per-machine process group, i.e. the number of processes per machine.
+ """
+ if not dist.is_available():
+ return 1
+ if not dist.is_initialized():
+ return 1
+ return dist.get_world_size(group=_LOCAL_PROCESS_GROUP)
+
+
+def is_main_process() -> bool:
+ return get_rank() == 0
+
+
+def _get_global_gloo_group():
+ """
+ Return a process group based on gloo backend, containing all the ranks
+ The result is cached.
+ """
+ if dist.get_backend() == "nccl":
+ return dist.new_group(backend="gloo")
+ else:
+ return dist.group.WORLD
+
+
+def _serialize_to_tensor(data, group):
+ backend = dist.get_backend(group)
+ assert backend in ["gloo", "nccl"]
+ device = torch.device("cpu" if backend == "gloo" else "cuda")
+
+ buffer = pickle.dumps(data)
+ if len(buffer) > 1024 ** 3:
+ logger = logging.getLogger(__name__)
+ logger.warning(
+ "Rank {} trying to all-gather {:.2f} GB of data on device {}".format(
+ get_rank(), len(buffer) / (1024 ** 3), device
+ )
+ )
+ storage = torch.ByteStorage.from_buffer(buffer)
+ tensor = torch.ByteTensor(storage).to(device=device)
+ return tensor
+
+
+def _pad_to_largest_tensor(tensor, group):
+ """
+ Returns:
+ list[int]: size of the tensor, on each rank
+ Tensor: padded tensor that has the max size
+ """
+ world_size = dist.get_world_size(group=group)
+ assert (
+ world_size >= 1
+ ), "comm.gather/all_gather must be called from ranks within the given group!"
+ local_size = torch.tensor([tensor.numel()], dtype=torch.int64, device=tensor.device)
+ size_list = [
+ torch.zeros([1], dtype=torch.int64, device=tensor.device)
+ for _ in range(world_size)
+ ]
+ dist.all_gather(size_list, local_size, group=group)
+ size_list = [int(size.item()) for size in size_list]
+
+ max_size = max(size_list)
+
+ # we pad the tensor because torch all_gather does not support
+ # gathering tensors of different shapes
+ if local_size != max_size:
+ padding = torch.zeros(
+ (max_size - local_size,), dtype=torch.uint8, device=tensor.device
+ )
+ tensor = torch.cat((tensor, padding), dim=0)
+ return size_list, tensor
+
+
+def all_gather(data, group=None):
+ """
+ Run all_gather on arbitrary picklable data (not necessarily tensors).
+
+ Args:
+ data: any picklable object
+ group: a torch process group. By default, will use a group which
+ contains all ranks on gloo backend.
+ Returns:
+ list[data]: list of data gathered from each rank
+ """
+ if get_world_size() == 1:
+ return [data]
+ if group is None:
+ group = _get_global_gloo_group()
+ if dist.get_world_size(group) == 1:
+ return [data]
+
+ tensor = _serialize_to_tensor(data, group)
+
+ size_list, tensor = _pad_to_largest_tensor(tensor, group)
+ max_size = max(size_list)
+
+ # receiving Tensor from all ranks
+ tensor_list = [
+ torch.empty((max_size,), dtype=torch.uint8, device=tensor.device)
+ for _ in size_list
+ ]
+ dist.all_gather(tensor_list, tensor, group=group)
+
+ data_list = []
+ for size, tensor in zip(size_list, tensor_list):
+ buffer = tensor.cpu().numpy().tobytes()[:size]
+ data_list.append(pickle.loads(buffer))
+
+ return data_list
+
+
+def gather(data, dst=0, group=None):
+ """
+ Run gather on arbitrary picklable data (not necessarily tensors).
+
+ Args:
+ data: any picklable object
+ dst (int): destination rank
+ group: a torch process group. By default, will use a group which
+ contains all ranks on gloo backend.
+
+ Returns:
+ list[data]: on dst, a list of data gathered from each rank. Otherwise,
+ an empty list.
+ """
+ if get_world_size() == 1:
+ return [data]
+ if group is None:
+ group = _get_global_gloo_group()
+ if dist.get_world_size(group=group) == 1:
+ return [data]
+ rank = dist.get_rank(group=group)
+
+ tensor = _serialize_to_tensor(data, group)
+ size_list, tensor = _pad_to_largest_tensor(tensor, group)
+
+ # receiving Tensor from all ranks
+ if rank == dst:
+ max_size = max(size_list)
+ tensor_list = [
+ torch.empty((max_size,), dtype=torch.uint8, device=tensor.device)
+ for _ in size_list
+ ]
+ dist.gather(tensor, tensor_list, dst=dst, group=group)
+
+ data_list = []
+ for size, tensor in zip(size_list, tensor_list):
+ buffer = tensor.cpu().numpy().tobytes()[:size]
+ data_list.append(pickle.loads(buffer))
+ return data_list
+ else:
+ dist.gather(tensor, [], dst=dst, group=group)
+ return []
+
+
+def shared_random_seed():
+ """
+ Returns:
+ int: a random number that is the same across all workers.
+ If workers need a shared RNG, they can use this shared seed to
+ create one.
+ All workers must call this function, otherwise it will deadlock.
+ """
+ ints = np.random.randint(2 ** 31)
+ all_ints = all_gather(ints)
+ return all_ints[0]
+
+
+def time_synchronized():
+ """pytorch-accurate time"""
+ if torch.cuda.is_available():
+ torch.cuda.synchronize()
+ return time.time()
diff --git a/experimental/yolox/utils/ema.py b/experimental/yolox/utils/ema.py
new file mode 100644
index 0000000..e0d09ba
--- /dev/null
+++ b/experimental/yolox/utils/ema.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+import torch
+import torch.nn as nn
+
+import math
+from copy import deepcopy
+
+
+def is_parallel(model):
+ """check if model is in parallel mode."""
+
+ parallel_type = (
+ nn.parallel.DataParallel,
+ nn.parallel.DistributedDataParallel,
+ )
+ return isinstance(model, parallel_type)
+
+
+def copy_attr(a, b, include=(), exclude=()):
+ # Copy attributes from b to a, options to only include [...] and to exclude [...]
+ for k, v in b.__dict__.items():
+ if (len(include) and k not in include) or k.startswith("_") or k in exclude:
+ continue
+ else:
+ setattr(a, k, v)
+
+
+class ModelEMA:
+ """
+ Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models
+ Keep a moving average of everything in the model state_dict (parameters and buffers).
+ This is intended to allow functionality like
+ https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage
+ A smoothed version of the weights is necessary for some training schemes to perform well.
+ This class is sensitive where it is initialized in the sequence of model init,
+ GPU assignment and distributed training wrappers.
+ """
+
+ def __init__(self, model, decay=0.9999, updates=0):
+ """
+ Args:
+ model (nn.Module): model to apply EMA.
+ decay (float): ema decay reate.
+ updates (int): counter of EMA updates.
+ """
+ # Create EMA(FP32)
+ self.ema = deepcopy(model.module if is_parallel(model) else model).eval()
+ self.updates = updates
+ # decay exponential ramp (to help early epochs)
+ self.decay = lambda x: decay * (1 - math.exp(-x / 2000))
+ for p in self.ema.parameters():
+ p.requires_grad_(False)
+
+ def update(self, model):
+ # Update EMA parameters
+ with torch.no_grad():
+ self.updates += 1
+ d = self.decay(self.updates)
+
+ msd = (
+ model.module.state_dict() if is_parallel(model) else model.state_dict()
+ ) # model state_dict
+ for k, v in self.ema.state_dict().items():
+ if v.dtype.is_floating_point:
+ v *= d
+ v += (1.0 - d) * msd[k].detach()
+
+ def update_attr(self, model, include=(), exclude=("process_group", "reducer")):
+ # Update EMA attributes
+ copy_attr(self.ema, model, include, exclude)
diff --git a/experimental/yolox/utils/logger.py b/experimental/yolox/utils/logger.py
new file mode 100644
index 0000000..4bd51d9
--- /dev/null
+++ b/experimental/yolox/utils/logger.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+from loguru import logger
+
+import inspect
+import os
+import sys
+
+
+def get_caller_name(depth=0):
+ """
+ Args:
+ depth (int): Depth of caller conext, use 0 for caller depth. Default value: 0.
+
+ Returns:
+ str: module name of the caller
+ """
+ # the following logic is a little bit faster than inspect.stack() logic
+ frame = inspect.currentframe().f_back
+ for _ in range(depth):
+ frame = frame.f_back
+
+ return frame.f_globals["__name__"]
+
+
+class StreamToLoguru:
+ """
+ stream object that redirects writes to a logger instance.
+ """
+
+ def __init__(self, level="INFO", caller_names=("apex", "pycocotools")):
+ """
+ Args:
+ level(str): log level string of loguru. Default value: "INFO".
+ caller_names(tuple): caller names of redirected module.
+ Default value: (apex, pycocotools).
+ """
+ self.level = level
+ self.linebuf = ""
+ self.caller_names = caller_names
+
+ def write(self, buf):
+ full_name = get_caller_name(depth=1)
+ module_name = full_name.rsplit(".", maxsplit=-1)[0]
+ if module_name in self.caller_names:
+ for line in buf.rstrip().splitlines():
+ # use caller level log
+ logger.opt(depth=2).log(self.level, line.rstrip())
+ else:
+ sys.__stdout__.write(buf)
+
+ def flush(self):
+ pass
+
+
+def redirect_sys_output(log_level="INFO"):
+ redirect_logger = StreamToLoguru(log_level)
+ sys.stderr = redirect_logger
+ sys.stdout = redirect_logger
+
+
+def setup_logger(save_dir, distributed_rank=0, filename="log.txt", mode="a"):
+ """setup logger for training and testing.
+ Args:
+ save_dir(str): location to save log file
+ distributed_rank(int): device rank when multi-gpu environment
+ filename (string): log save name.
+ mode(str): log file write mode, `append` or `override`. default is `a`.
+
+ Return:
+ logger instance.
+ """
+ loguru_format = (
+ "<green>{time:YYYY-MM-DD HH:mm:ss}</green> | "
+ "<level>{level: <8}</level> | "
+ "<cyan>{name}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>"
+ )
+
+ logger.remove()
+ save_file = os.path.join(save_dir, filename)
+ if mode == "o" and os.path.exists(save_file):
+ os.remove(save_file)
+ # only keep logger in rank0 process
+ if distributed_rank == 0:
+ logger.add(
+ sys.stderr,
+ format=loguru_format,
+ level="INFO",
+ enqueue=True,
+ )
+ logger.add(save_file)
+
+ # redirect stdout/stderr to loguru
+ redirect_sys_output("INFO")
diff --git a/experimental/yolox/utils/lr_scheduler.py b/experimental/yolox/utils/lr_scheduler.py
new file mode 100644
index 0000000..8f85c23
--- /dev/null
+++ b/experimental/yolox/utils/lr_scheduler.py
@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import math
+from functools import partial
+
+
+class LRScheduler:
+ def __init__(self, name, lr, iters_per_epoch, total_epochs, **kwargs):
+ """
+ Supported lr schedulers: [cos, warmcos, multistep]
+
+ Args:
+ lr (float): learning rate.
+ iters_per_peoch (int): number of iterations in one epoch.
+ total_epochs (int): number of epochs in training.
+ kwargs (dict):
+ - cos: None
+ - warmcos: [warmup_epochs, warmup_lr_start (default 1e-6)]
+ - multistep: [milestones (epochs), gamma (default 0.1)]
+ """
+
+ self.lr = lr
+ self.iters_per_epoch = iters_per_epoch
+ self.total_epochs = total_epochs
+ self.total_iters = iters_per_epoch * total_epochs
+
+ self.__dict__.update(kwargs)
+
+ self.lr_func = self._get_lr_func(name)
+
+ def update_lr(self, iters):
+ return self.lr_func(iters)
+
+ def _get_lr_func(self, name):
+ if name == "cos": # cosine lr schedule
+ lr_func = partial(cos_lr, self.lr, self.total_iters)
+ elif name == "warmcos":
+ warmup_total_iters = self.iters_per_epoch * self.warmup_epochs
+ warmup_lr_start = getattr(self, "warmup_lr_start", 1e-6)
+ lr_func = partial(
+ warm_cos_lr,
+ self.lr,
+ self.total_iters,
+ warmup_total_iters,
+ warmup_lr_start,
+ )
+ elif name == "yoloxwarmcos":
+ warmup_total_iters = self.iters_per_epoch * self.warmup_epochs
+ no_aug_iters = self.iters_per_epoch * self.no_aug_epochs
+ warmup_lr_start = getattr(self, "warmup_lr_start", 0)
+ min_lr_ratio = getattr(self, "min_lr_ratio", 0.2)
+ lr_func = partial(
+ yolox_warm_cos_lr,
+ self.lr,
+ min_lr_ratio,
+ self.total_iters,
+ warmup_total_iters,
+ warmup_lr_start,
+ no_aug_iters,
+ )
+ elif name == "yoloxsemiwarmcos":
+ warmup_lr_start = getattr(self, "warmup_lr_start", 0)
+ min_lr_ratio = getattr(self, "min_lr_ratio", 0.2)
+ warmup_total_iters = self.iters_per_epoch * self.warmup_epochs
+ no_aug_iters = self.iters_per_epoch * self.no_aug_epochs
+ normal_iters = self.iters_per_epoch * self.semi_epoch
+ semi_iters = self.iters_per_epoch_semi * (
+ self.total_epochs - self.semi_epoch - self.no_aug_epochs
+ )
+ lr_func = partial(
+ yolox_semi_warm_cos_lr,
+ self.lr,
+ min_lr_ratio,
+ warmup_lr_start,
+ self.total_iters,
+ normal_iters,
+ no_aug_iters,
+ warmup_total_iters,
+ semi_iters,
+ self.iters_per_epoch,
+ self.iters_per_epoch_semi,
+ )
+ elif name == "multistep": # stepwise lr schedule
+ milestones = [
+ int(self.total_iters * milestone / self.total_epochs)
+ for milestone in self.milestones
+ ]
+ gamma = getattr(self, "gamma", 0.1)
+ lr_func = partial(multistep_lr, self.lr, milestones, gamma)
+ else:
+ raise ValueError("Scheduler version {} not supported.".format(name))
+ return lr_func
+
+
+def cos_lr(lr, total_iters, iters):
+ """Cosine learning rate"""
+ lr *= 0.5 * (1.0 + math.cos(math.pi * iters / total_iters))
+ return lr
+
+
+def warm_cos_lr(lr, total_iters, warmup_total_iters, warmup_lr_start, iters):
+ """Cosine learning rate with warm up."""
+ if iters <= warmup_total_iters:
+ lr = (lr - warmup_lr_start) * iters / float(
+ warmup_total_iters
+ ) + warmup_lr_start
+ else:
+ lr *= 0.5 * (
+ 1.0
+ + math.cos(
+ math.pi
+ * (iters - warmup_total_iters)
+ / (total_iters - warmup_total_iters)
+ )
+ )
+ return lr
+
+
+def yolox_warm_cos_lr(
+ lr,
+ min_lr_ratio,
+ total_iters,
+ warmup_total_iters,
+ warmup_lr_start,
+ no_aug_iter,
+ iters,
+):
+ """Cosine learning rate with warm up."""
+ min_lr = lr * min_lr_ratio
+ if iters <= warmup_total_iters:
+ # lr = (lr - warmup_lr_start) * iters / float(warmup_total_iters) + warmup_lr_start
+ lr = (lr - warmup_lr_start) * pow(
+ iters / float(warmup_total_iters), 2
+ ) + warmup_lr_start
+ elif iters >= total_iters - no_aug_iter:
+ lr = min_lr
+ else:
+ lr = min_lr + 0.5 * (lr - min_lr) * (
+ 1.0
+ + math.cos(
+ math.pi
+ * (iters - warmup_total_iters)
+ / (total_iters - warmup_total_iters - no_aug_iter)
+ )
+ )
+ return lr
+
+
+def yolox_semi_warm_cos_lr(
+ lr,
+ min_lr_ratio,
+ warmup_lr_start,
+ total_iters,
+ normal_iters,
+ no_aug_iters,
+ warmup_total_iters,
+ semi_iters,
+ iters_per_epoch,
+ iters_per_epoch_semi,
+ iters,
+):
+ """Cosine learning rate with warm up."""
+ min_lr = lr * min_lr_ratio
+ if iters <= warmup_total_iters:
+ # lr = (lr - warmup_lr_start) * iters / float(warmup_total_iters) + warmup_lr_start
+ lr = (lr - warmup_lr_start) * pow(
+ iters / float(warmup_total_iters), 2
+ ) + warmup_lr_start
+ elif iters >= normal_iters + semi_iters:
+ lr = min_lr
+ elif iters <= normal_iters:
+ lr = min_lr + 0.5 * (lr - min_lr) * (
+ 1.0
+ + math.cos(
+ math.pi
+ * (iters - warmup_total_iters)
+ / (total_iters - warmup_total_iters - no_aug_iters)
+ )
+ )
+ else:
+ lr = min_lr + 0.5 * (lr - min_lr) * (
+ 1.0
+ + math.cos(
+ math.pi
+ * (
+ normal_iters
+ - warmup_total_iters
+ + (iters - normal_iters)
+ * iters_per_epoch
+ * 1.0
+ / iters_per_epoch_semi
+ )
+ / (total_iters - warmup_total_iters - no_aug_iters)
+ )
+ )
+ return lr
+
+
+def multistep_lr(lr, milestones, gamma, iters):
+ """MultiStep learning rate"""
+ for milestone in milestones:
+ lr *= gamma if iters >= milestone else 1.0
+ return lr
diff --git a/experimental/yolox/utils/metric.py b/experimental/yolox/utils/metric.py
new file mode 100644
index 0000000..4840b8d
--- /dev/null
+++ b/experimental/yolox/utils/metric.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+import numpy as np
+
+import torch
+
+import functools
+import os
+import time
+from collections import defaultdict, deque
+
+__all__ = [
+ "AverageMeter",
+ "MeterBuffer",
+ "get_total_and_free_memory_in_Mb",
+ "occupy_mem",
+ "gpu_mem_usage",
+]
+
+
+def get_total_and_free_memory_in_Mb(cuda_device):
+ devices_info_str = os.popen(
+ "nvidia-smi --query-gpu=memory.total,memory.used --format=csv,nounits,noheader"
+ )
+ devices_info = devices_info_str.read().strip().split("\n")
+ total, used = devices_info[int(cuda_device)].split(",")
+ return int(total), int(used)
+
+
+def occupy_mem(cuda_device, mem_ratio=0.95):
+ """
+ pre-allocate gpu memory for training to avoid memory Fragmentation.
+ """
+ total, used = get_total_and_free_memory_in_Mb(cuda_device)
+ max_mem = int(total * mem_ratio)
+ block_mem = max_mem - used
+ x = torch.cuda.FloatTensor(256, 1024, block_mem)
+ del x
+ time.sleep(5)
+
+
+def gpu_mem_usage():
+ """
+ Compute the GPU memory usage for the current device (MB).
+ """
+ mem_usage_bytes = torch.cuda.max_memory_allocated()
+ return mem_usage_bytes / (1024 * 1024)
+
+
+class AverageMeter:
+ """Track a series of values and provide access to smoothed values over a
+ window or the global series average.
+ """
+
+ def __init__(self, window_size=50):
+ self._deque = deque(maxlen=window_size)
+ self._total = 0.0
+ self._count = 0
+
+ def update(self, value):
+ self._deque.append(value)
+ self._count += 1
+ self._total += value
+
+ @property
+ def median(self):
+ d = np.array(list(self._deque))
+ return np.median(d)
+
+ @property
+ def avg(self):
+ # if deque is empty, nan will be returned.
+ d = np.array(list(self._deque))
+ return d.mean()
+
+ @property
+ def global_avg(self):
+ return self._total / max(self._count, 1e-5)
+
+ @property
+ def latest(self):
+ return self._deque[-1] if len(self._deque) > 0 else None
+
+ @property
+ def total(self):
+ return self._total
+
+ def reset(self):
+ self._deque.clear()
+ self._total = 0.0
+ self._count = 0
+
+ def clear(self):
+ self._deque.clear()
+
+
+class MeterBuffer(defaultdict):
+ """Computes and stores the average and current value"""
+
+ def __init__(self, window_size=20):
+ factory = functools.partial(AverageMeter, window_size=window_size)
+ super().__init__(factory)
+
+ def reset(self):
+ for v in self.values():
+ v.reset()
+
+ def get_filtered_meter(self, filter_key="time"):
+ return {k: v for k, v in self.items() if filter_key in k}
+
+ def update(self, values=None, **kwargs):
+ if values is None:
+ values = {}
+ values.update(kwargs)
+ for k, v in values.items():
+ if isinstance(v, torch.Tensor):
+ v = v.detach()
+ self[k].update(v)
+
+ def clear_meters(self):
+ for v in self.values():
+ v.clear()
diff --git a/experimental/yolox/utils/model_utils.py b/experimental/yolox/utils/model_utils.py
new file mode 100644
index 0000000..b4e5812
--- /dev/null
+++ b/experimental/yolox/utils/model_utils.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import torch
+import torch.nn as nn
+from thop import profile
+
+from copy import deepcopy
+
+__all__ = [
+ "fuse_conv_and_bn",
+ "fuse_model",
+ "get_model_info",
+ "replace_module",
+]
+
+
+def get_model_info(model, tsize):
+
+ stride = 64
+ img = torch.zeros((1, 3, stride, stride), device=next(model.parameters()).device)
+ flops, params = profile(deepcopy(model), inputs=(img,), verbose=False)
+ params /= 1e6
+ flops /= 1e9
+ flops *= tsize[0] * tsize[1] / stride / stride * 2 # Gflops
+ info = "Params: {:.2f}M, Gflops: {:.2f}".format(params, flops)
+ return info
+
+
+def fuse_conv_and_bn(conv, bn):
+ # Fuse convolution and batchnorm layers https://tehnokv.com/posts/fusing-batchnorm-and-conv/
+ fusedconv = (
+ nn.Conv2d(
+ conv.in_channels,
+ conv.out_channels,
+ kernel_size=conv.kernel_size,
+ stride=conv.stride,
+ padding=conv.padding,
+ groups=conv.groups,
+ bias=True,
+ )
+ .requires_grad_(False)
+ .to(conv.weight.device)
+ )
+
+ # prepare filters
+ w_conv = conv.weight.clone().view(conv.out_channels, -1)
+ w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var)))
+ fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape))
+
+ # prepare spatial bias
+ b_conv = (
+ torch.zeros(conv.weight.size(0), device=conv.weight.device)
+ if conv.bias is None
+ else conv.bias
+ )
+ b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(
+ torch.sqrt(bn.running_var + bn.eps)
+ )
+ fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn)
+
+ return fusedconv
+
+
+def fuse_model(model):
+ from yolox.models.network_blocks import BaseConv
+
+ for m in model.modules():
+ if type(m) is BaseConv and hasattr(m, "bn"):
+ m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv
+ delattr(m, "bn") # remove batchnorm
+ m.forward = m.fuseforward # update forward
+ return model
+
+
+def replace_module(module, replaced_module_type, new_module_type, replace_func=None):
+ """
+ Replace given type in module to a new type. mostly used in deploy.
+
+ Args:
+ module (nn.Module): model to apply replace operation.
+ replaced_module_type (Type): module type to be replaced.
+ new_module_type (Type)
+ replace_func (function): python function to describe replace logic. Defalut value None.
+
+ Returns:
+ model (nn.Module): module that already been replaced.
+ """
+
+ def default_replace_func(replaced_module_type, new_module_type):
+ return new_module_type()
+
+ if replace_func is None:
+ replace_func = default_replace_func
+
+ model = module
+ if isinstance(module, replaced_module_type):
+ model = replace_func(replaced_module_type, new_module_type)
+ else: # recurrsively replace
+ for name, child in module.named_children():
+ new_child = replace_module(child, replaced_module_type, new_module_type)
+ if new_child is not child: # child is already replaced
+ model.add_module(name, new_child)
+
+ return model
diff --git a/experimental/yolox/utils/setup_env.py b/experimental/yolox/utils/setup_env.py
new file mode 100644
index 0000000..f282b1f
--- /dev/null
+++ b/experimental/yolox/utils/setup_env.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import cv2
+
+import os
+import subprocess
+
+__all__ = ["configure_nccl", "configure_module"]
+
+
+def configure_nccl():
+ """Configure multi-machine environment variables of NCCL."""
+ os.environ["NCCL_LAUNCH_MODE"] = "PARALLEL"
+ os.environ["NCCL_IB_HCA"] = subprocess.getoutput(
+ "pushd /sys/class/infiniband/ > /dev/null; for i in mlx5_*; "
+ "do cat $i/ports/1/gid_attrs/types/* 2>/dev/null "
+ "| grep v >/dev/null && echo $i ; done; popd > /dev/null"
+ )
+ os.environ["NCCL_IB_GID_INDEX"] = "3"
+ os.environ["NCCL_IB_TC"] = "106"
+
+
+def configure_module(ulimit_value=8192):
+ """
+ Configure pytorch module environment. setting of ulimit and cv2 will be set.
+
+ Args:
+ ulimit_value(int): default open file number on linux. Default value: 8192.
+ """
+ # system setting
+ try:
+ import resource
+
+ rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
+ resource.setrlimit(resource.RLIMIT_NOFILE, (ulimit_value, rlimit[1]))
+ except Exception:
+ # Exception might be raised in Windows OS or rlimit reaches max limit number.
+ # However, set rlimit value might not be necessary.
+ pass
+
+ # cv2
+ # multiprocess might be harmful on performance of torch dataloader
+ os.environ["OPENCV_OPENCL_RUNTIME"] = "disabled"
+ try:
+ cv2.setNumThreads(0)
+ cv2.ocl.setUseOpenCL(False)
+ except Exception:
+ # cv2 version mismatch might rasie exceptions.
+ pass
diff --git a/experimental/yolox/utils/visualize.py b/experimental/yolox/utils/visualize.py
new file mode 100644
index 0000000..1d02d47
--- /dev/null
+++ b/experimental/yolox/utils/visualize.py
@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
+
+import cv2
+import numpy as np
+
+__all__ = ["vis"]
+
+
+def vis(img, boxes, scores, cls_ids, conf=0.5, class_names=None):
+
+ for i in range(len(boxes)):
+ box = boxes[i]
+ cls_id = int(cls_ids[i])
+ score = scores[i]
+ if score < conf:
+ continue
+ x0 = int(box[0])
+ y0 = int(box[1])
+ x1 = int(box[2])
+ y1 = int(box[3])
+
+ color = (_COLORS[cls_id] * 255).astype(np.uint8).tolist()
+ text = '{}:{:.1f}%'.format(class_names[cls_id], score * 100)
+ txt_color = (0, 0, 0) if np.mean(_COLORS[cls_id]) > 0.5 else (255, 255, 255)
+ font = cv2.FONT_HERSHEY_SIMPLEX
+
+ txt_size = cv2.getTextSize(text, font, 0.4, 1)[0]
+ cv2.rectangle(img, (x0, y0), (x1, y1), color, 2)
+
+ txt_bk_color = (_COLORS[cls_id] * 255 * 0.7).astype(np.uint8).tolist()
+ cv2.rectangle(
+ img,
+ (x0, y0 + 1),
+ (x0 + txt_size[0] + 1, y0 + int(1.5*txt_size[1])),
+ txt_bk_color,
+ -1
+ )
+ cv2.putText(img, text, (x0, y0 + txt_size[1]), font, 0.4, txt_color, thickness=1)
+
+ return img
+
+
+def get_color(idx):
+ idx = idx * 3
+ color = ((37 * idx) % 255, (17 * idx) % 255, (29 * idx) % 255)
+
+ return color
+
+
+def plot_tracking(image, tlwhs, obj_ids, scores=None, frame_id=0, fps=0., ids2=None):
+ im = np.ascontiguousarray(np.copy(image))
+ im_h, im_w = im.shape[:2]
+
+ top_view = np.zeros([im_w, im_w, 3], dtype=np.uint8) + 255
+
+ #text_scale = max(1, image.shape[1] / 1600.)
+ #text_thickness = 2
+ #line_thickness = max(1, int(image.shape[1] / 500.))
+ text_scale = 2
+ text_thickness = 2
+ line_thickness = 3
+
+ radius = max(5, int(im_w/140.))
+ cv2.putText(im, 'frame: %d fps: %.2f num: %d' % (frame_id, fps, len(tlwhs)),
+ (0, int(15 * text_scale)), cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 255), thickness=2)
+
+ for i, tlwh in enumerate(tlwhs):
+ x1, y1, w, h = tlwh
+ intbox = tuple(map(int, (x1, y1, x1 + w, y1 + h)))
+ obj_id = int(obj_ids[i])
+ id_text = '{}'.format(int(obj_id))
+ if ids2 is not None:
+ id_text = id_text + ', {}'.format(int(ids2[i]))
+ color = get_color(abs(obj_id))
+ cv2.rectangle(im, intbox[0:2], intbox[2:4], color=color, thickness=line_thickness)
+ cv2.putText(im, id_text, (intbox[0], intbox[1]), cv2.FONT_HERSHEY_PLAIN, text_scale, (0, 0, 255),
+ thickness=text_thickness)
+ return im
+
+
+_COLORS = np.array(
+ [
+ 0.000, 0.447, 0.741,
+ 0.850, 0.325, 0.098,
+ 0.929, 0.694, 0.125,
+ 0.494, 0.184, 0.556,
+ 0.466, 0.674, 0.188,
+ 0.301, 0.745, 0.933,
+ 0.635, 0.078, 0.184,
+ 0.300, 0.300, 0.300,
+ 0.600, 0.600, 0.600,
+ 1.000, 0.000, 0.000,
+ 1.000, 0.500, 0.000,
+ 0.749, 0.749, 0.000,
+ 0.000, 1.000, 0.000,
+ 0.000, 0.000, 1.000,
+ 0.667, 0.000, 1.000,
+ 0.333, 0.333, 0.000,
+ 0.333, 0.667, 0.000,
+ 0.333, 1.000, 0.000,
+ 0.667, 0.333, 0.000,
+ 0.667, 0.667, 0.000,
+ 0.667, 1.000, 0.000,
+ 1.000, 0.333, 0.000,
+ 1.000, 0.667, 0.000,
+ 1.000, 1.000, 0.000,
+ 0.000, 0.333, 0.500,
+ 0.000, 0.667, 0.500,
+ 0.000, 1.000, 0.500,
+ 0.333, 0.000, 0.500,
+ 0.333, 0.333, 0.500,
+ 0.333, 0.667, 0.500,
+ 0.333, 1.000, 0.500,
+ 0.667, 0.000, 0.500,
+ 0.667, 0.333, 0.500,
+ 0.667, 0.667, 0.500,
+ 0.667, 1.000, 0.500,
+ 1.000, 0.000, 0.500,
+ 1.000, 0.333, 0.500,
+ 1.000, 0.667, 0.500,
+ 1.000, 1.000, 0.500,
+ 0.000, 0.333, 1.000,
+ 0.000, 0.667, 1.000,
+ 0.000, 1.000, 1.000,
+ 0.333, 0.000, 1.000,
+ 0.333, 0.333, 1.000,
+ 0.333, 0.667, 1.000,
+ 0.333, 1.000, 1.000,
+ 0.667, 0.000, 1.000,
+ 0.667, 0.333, 1.000,
+ 0.667, 0.667, 1.000,
+ 0.667, 1.000, 1.000,
+ 1.000, 0.000, 1.000,
+ 1.000, 0.333, 1.000,
+ 1.000, 0.667, 1.000,
+ 0.333, 0.000, 0.000,
+ 0.500, 0.000, 0.000,
+ 0.667, 0.000, 0.000,
+ 0.833, 0.000, 0.000,
+ 1.000, 0.000, 0.000,
+ 0.000, 0.167, 0.000,
+ 0.000, 0.333, 0.000,
+ 0.000, 0.500, 0.000,
+ 0.000, 0.667, 0.000,
+ 0.000, 0.833, 0.000,
+ 0.000, 1.000, 0.000,
+ 0.000, 0.000, 0.167,
+ 0.000, 0.000, 0.333,
+ 0.000, 0.000, 0.500,
+ 0.000, 0.000, 0.667,
+ 0.000, 0.000, 0.833,
+ 0.000, 0.000, 1.000,
+ 0.000, 0.000, 0.000,
+ 0.143, 0.143, 0.143,
+ 0.286, 0.286, 0.286,
+ 0.429, 0.429, 0.429,
+ 0.571, 0.571, 0.571,
+ 0.714, 0.714, 0.714,
+ 0.857, 0.857, 0.857,
+ 0.000, 0.447, 0.741,
+ 0.314, 0.717, 0.741,
+ 0.50, 0.5, 0
+ ]
+).astype(np.float32).reshape(-1, 3)