Skip to content

Record a sky camera around the clock, keep the footage on your own disk, and find what flew past.

15.4 KB
Raw Blame
"""Follow each object through one recording: link its detections into a track, tell movers from fixed lights,
and, when every frame was decoded, measure it in every frame. Positions are fractions of the frame (0-1). Distances
and speeds are in frame heights (x scaled by the aspect ratio), so they do not depend on the resolution."""
import bisect
import math
from collections import deque

# Calibrated on reviewed sky01 night data: confirmed planes have a median box of 0.017 frame heights (36 of 2160 px),
# and 3,921 plane steps between frames a median speed of 0.012 and a 99.9th percentile of 0.054 frame heights/s.
MAX_SPEED = 0.06   # how far an object whose speed is not yet known may move per second
MAX_GAP = 5.0      # seconds a track may go unseen: missed detections and blinking strobes
STILL = 0.002      # frame heights/s between a fixed light and a mover; a star drifts about 0.0001
JITTER = 0.05      # scatter of a box centre between frames, as a fraction of the box
WINDOW = 2.0       # full-rate samples kept with each frame: this many seconds either side
REACH = 8          # frames either side kept to see what is new in a frame: about 1 s at 8 fps
MIN_PEAK = 6.0     # grey levels of new light that count as the object being visible


def _median(values):
    ordered = sorted(values)
    middle = len(ordered) // 2
    return ordered[middle] if len(ordered) % 2 else (ordered[middle - 1] + ordered[middle]) / 2


def _fit(points):
    """Mean time and position, least-squares velocity, and time spread of points (t, x, y, ...)."""
    n = len(points)
    tm = sum(p[0] for p in points) / n
    xm = sum(p[1] for p in points) / n
    ym = sum(p[2] for p in points) / n
    spread = sum((p[0] - tm) ** 2 for p in points)
    if spread < 1e-9:
        return tm, xm, ym, 0.0, 0.0, spread
    vx = sum((p[0] - tm) * (p[1] - xm) for p in points) / spread
    vy = sum((p[0] - tm) * (p[2] - ym) for p in points) / spread
    return tm, xm, ym, vx, vy, spread


def _expect(track, t):
    """Where a track should be at time t, and how far from there a detection may be and still be it."""
    recent = track[-6:]
    last, size, dt = recent[-1], _median([p[3] for p in recent]), t - recent[-1][0]
    if len(recent) == 1:
        return last[1], last[2], min(0.1, max(1.5 * size, 0.008) + MAX_SPEED * dt)
    tm, xm, ym, vx, vy, _ = _fit(recent)
    return xm + vx * (t - tm), ym + vy * (t - tm), max(size, 0.006) + (0.25 * math.hypot(vx, vy) + 0.003) * dt


def link(frames, aspect):
    """Tracks from (t, annotations) in time order: lists of (t, x, y, size, annotation), x and y in frame heights.
    Each frame's boxes go to the nearest live track that could have reached them, a same-class track first."""
    tracks = []
    for t, annotations in frames:
        found = []
        for a in annotations:
            if a.get('box'):
                x1, y1, x2, y2 = a['box']
                found.append((a, (x1 + x2) / 2 * aspect, (y1 + y2) / 2, max((x2 - x1) * aspect, y2 - y1)))
        live = [track for track in tracks if t - track[-1][0] <= MAX_GAP]
        pairs = []
        for i, track in enumerate(live):
            x, y, reach = _expect(track, t)
            for j, (a, fx, fy, _) in enumerate(found):
                distance = math.hypot(fx - x, fy - y)
                if distance <= reach:
                    pairs.append((distance / reach + (0.5 if a['class_name'] != track[-1][4]['class_name'] else 0.0), i, j))
        linked, taken = set(), set()
        for _, i, j in sorted(pairs):
            if i not in linked and j not in taken:
                linked.add(i); taken.add(j)
                a, fx, fy, size = found[j]
                live[i].append((t, fx, fy, size, a))
        tracks.extend([(t, fx, fy, size, a)] for j, (a, fx, fy, size) in enumerate(found) if j not in taken)
    return tracks


def motion(track):
    """('moving' | 'stationary' | 'brief', speed). Brief means too few or too close detections to tell, and
    the speed is None for a single detection. The verdict allows for each box centre's own scatter."""
    if len(track) < 2:
        return 'brief', None
    tm, xm, ym, vx, vy, spread = _fit(track)
    speed = math.hypot(vx, vy)
    if spread < 1e-9:
        return 'brief', speed
    residual = math.sqrt(sum((p[1] - xm - vx * (p[0] - tm)) ** 2 + (p[2] - ym - vy * (p[0] - tm)) ** 2
                             for p in track) / (2 * (len(track) - 2))) if len(track) > 2 else 0.0
    error = 2 * max(residual, JITTER * _median([p[3] for p in track]), 0.0003) / math.sqrt(spread)
    return ('stationary' if speed + error < STILL else 'moving' if speed - error > STILL else 'brief'), speed


def _centre(box):
    return [(box[0] + box[2]) / 2, (box[1] + box[3]) / 2, box[2] - box[0], box[3] - box[1]]


def describe(frames, aspect, prefix, first=1):
    """Link one model's boxes across a recording. Sets each box's track_id and returns {id: record}.
    frames: (t, annotations) in time order. A record's path lists every detection: t, x, y, width, height
    (fractions of the frame), class_id, confidence. Clipped edges mean the object may extend past the window.
    Track numbers start at `first`, so a job's later models continue its numbering."""
    times = [t for t, _ in frames]
    spacing = _median([b - a for a, b in zip(times, times[1:])]) if len(times) > 1 else 0.0
    records = {}
    for n, track in enumerate(sorted(link(frames, aspect), key=lambda tr: (tr[0][0], tr[0][1], tr[0][2])), first):
        kind, speed = motion(track)
        identifier = f'{prefix}{n}'
        clipped = [edge for edge, gap in (('start', track[0][0] - times[0]), ('end', times[-1] - track[-1][0]))
                   if gap <= 1.5 * spacing]
        records[identifier] = {
            'id': identifier, 'motion': kind, 'speed': None if speed is None else round(speed, 6),
            'start': round(track[0][0], 3), 'end': round(track[-1][0], 3), 'detections': len(track),
            **({'clipped': clipped} if clipped else {}),
            'path': [[round(p[0], 3), *(round(v, 5) for v in _centre(p[4]['box'])), p[4]['class_id'],
                      round(p[4]['confidence'], 4)] for p in track],
        }
        for p in track:
            p[4]['track_id'] = identifier
    return records


def _position(anchors, t):
    """Where a track's detections put it at time t: on the line between the two around t, or past either end at
    the velocity of the nearest few. Also the time to the nearest detection."""
    i = bisect.bisect_left([a[0] for a in anchors], t)
    if 0 < i < len(anchors):
        (ta, xa, ya), (tb, xb, yb) = anchors[i - 1], anchors[i]
        f = (t - ta) / (tb - ta) if tb > ta else 0.0
        return xa + f * (xb - xa), ya + f * (yb - ya), min(t - ta, tb - t)
    near = anchors[:4] if i == 0 else anchors[-4:]
    tm, xm, ym, vx, vy, _ = _fit(near)
    return xm + vx * (t - tm), ym + vy * (t - tm), min(abs(t - a[0]) for a in near)


def measure(image, references, x, y, radius, size, np, cv2):
    """The new light nearest (x, y) within radius (frame heights), as (x, y, peak) in frame fractions and grey
    levels above what the reference frames show there, or None when nothing stands out from the noise."""
    height, width = image.shape
    r = max(4, int(radius * height))
    cx, cy = x * width, y * height
    x0, x1, y0, y1 = max(0, int(cx) - r), min(width, int(cx) + r + 1), max(0, int(cy) - r), min(height, int(cy) + r + 1)
    if x1 - x0 < 3 or y1 - y0 < 3:
        return None

    def difference(left, right, top, bottom):
        background = references[0][top:bottom, left:right]
        for other in references[1:]:
            background = np.maximum(background, other[top:bottom, left:right])
        return cv2.blur((image[top:bottom, left:right].astype(np.int16) - background).astype(np.float32), (3, 3))
    diff = difference(x0, x1, y0, y1)
    level = float(np.median(diff))
    diff -= level
    noise = 1.4826 * float(np.median(np.abs(diff))) + 0.5
    yy, xx = np.ogrid[y0:y1, x0:x1]
    prior = np.exp(-((xx + .5 - cx) ** 2 + (yy + .5 - cy) ** 2) / (2 * (r / 2) ** 2))
    py, px = divmod(int(np.argmax(diff * prior)), x1 - x0)
    peak = float(diff[py, px])
    if peak < max(MIN_PEAK, 5 * noise):
        return None
    # Centre on the light itself, not where the search began: recentring a fresh window settles on a streak's
    # middle even when the search window cut the streak off.
    c = max(2, int(size * height / 2))
    mx, my = float(x0 + px), float(y0 + py)
    for _ in range(4):
        left, top = max(0, int(round(mx)) - c), max(0, int(round(my)) - c)
        patch = np.clip(difference(left, min(width, int(round(mx)) + c + 1), top, min(height, int(round(my)) + c + 1)) - level, 0, None)
        total = float(patch.sum())
        if total <= 0:
            break
        gy, gx = np.indices(patch.shape)
        mx, my = left + float((gx * patch).sum()) / total, top + float((gy * patch).sum()) / total
    return (mx + .5) / width, (my + .5) / height, peak


def _follow_mover(ring, c, plan, fps, np, cv2):
    t, image = ring[c]
    x, y, gap = _position(plan['anchors'], t)
    # References far enough back and ahead that the object has moved off its own pixels.
    step = plan['speed'] / fps
    offset = max(2, min(REACH, math.ceil(2 * plan['size'] / max(step, 1e-6))))
    references = [ring[i][1] for i in (c - offset, c + offset) if 0 <= i < len(ring)]
    if not references:
        return None
    radius = min(0.08, max(2 * plan['size'], 0.01) + 0.3 * plan['speed'] * gap)
    return measure(image, references, x, y, radius, plan['size'], np, cv2)


def _follow_brief(ring, c, plan, np, cv2):
    """Walk out from the detection frame both ways while the object stays visible: a meteor lasts a few frames."""
    others = [ring[i][1] for i in (0, len(ring) - 1) if i != c]
    found = {c: others and measure(ring[c][1], others, *plan['anchors'][0][1:], plan['radius'], plan['size'], np, cv2)}
    for direction in (1, -1):
        seen = [(ring[c][0], plan['anchors'][0][1], plan['anchors'][0][2])]
        if found[c]:
            seen = [(ring[c][0], found[c][0], found[c][1])]
        misses, i = 0, c + direction
        while 0 <= i < len(ring) and misses < 2:
            t = ring[i][0]
            if len(seen) > 1:
                tm, xm, ym, vx, vy, _ = _fit(seen)
                x, y = xm + vx * (t - tm), ym + vy * (t - tm)
            else:
                x, y = seen[-1][1], seen[-1][2]
            references = [ring[k][1] for k in (0, len(ring) - 1) if k != i]
            result = references and measure(ring[i][1], references, x, y, plan['radius'], plan['size'], np, cv2)
            found[i] = result
            if result:
                seen.append((t, result[0], result[1])); misses = 0
            else:
                misses += 1
            i += direction
    return sorted((ring[i][0], *(found[i] or (None, None, 0.0))) for i in found)


def follow(path, plans, start, until, check=lambda: None, recurring=None):
    """Measure planned tracks in every frame from start to until (seconds into the recording).
    plans: {id: {'motion', 'anchors': [(t, x, y)], 'size', 'speed'}}, positions in frame fractions.
    recurring: the scan's mask of light that keeps coming back (a burned-in clock); those pixels are blanked in
    every frame, so they never look new. Returns (fps, {id: [(t, x or None, y or None, peak)]})."""
    import av
    import cv2
    import numpy as np
    samples = {identifier: [] for identifier in plans}
    movers = {i: p for i, p in plans.items() if p['motion'] == 'moving'}
    brief = {i: p for i, p in plans.items() if p['motion'] == 'brief'}
    done = set()
    ring, decoded, first, last, keep = deque(), 0, None, None, None

    def visit(c):
        t = ring[c][0]
        for identifier, plan in movers.items():
            if plan['anchors'][0][0] - 1.0 <= t <= plan['anchors'][-1][0] + 1.0:
                result = _follow_mover(ring, c, plan, fps, np, cv2)
                samples[identifier].append((t, *(result or (None, None, 0.0))))
        for identifier, plan in brief.items():
            if identifier not in done and t >= plan['anchors'][0][0] - 1e-3:
                done.add(identifier)
                samples[identifier] = _follow_brief(ring, c, plan, np, cv2)

    with av.open(str(path)) as container:
        stream = container.streams.video[0]
        # Threaded HEVC decode deadlocks inside PyAV. One thread is slower and finishes.
        try: stream.thread_type = 'NONE'
        except Exception: pass
        origin = stream.start_time if stream.start_time is not None else 0
        fps = float(stream.average_rate) if stream.average_rate else 8.0
        container.seek(int(origin + max(0.0, start - 2) / float(stream.time_base)), backward=True, any_frame=False, stream=stream)
        for frame in container.decode(stream):
            if frame.pts is None:
                continue
            t = float((frame.pts - origin) * frame.time_base)
            if t < start - 1e-6:
                continue
            if t > until + 1e-6:
                break
            decoded += 1
            if decoded % 50 == 0:
                check()
            grey = frame.to_ndarray(format='gray')
            if recurring is not None:
                if keep is None:
                    keep = cv2.resize((recurring == 0).astype(np.uint8) * 255, (grey.shape[1], grey.shape[0]),
                                      interpolation=cv2.INTER_NEAREST)
                grey = cv2.bitwise_and(grey, keep)
            ring.append((t, grey))
            first = t if first is None else first
            last = t
            if len(ring) > 2 * REACH + 1:
                ring.popleft()
            if len(ring) - 1 - REACH >= 0:
                visit(len(ring) - 1 - REACH)
        for c in range(max(0, len(ring) - REACH), len(ring)):
            visit(c)
    if decoded > 1 and last > first:
        fps = (decoded - 1) / (last - first)
    return fps, samples


def plans_for(records, aspect):
    """What to follow at full rate: movers and brief sightings. Fixed lights stay where their detections are."""
    plans = {}
    for identifier, record in records.items():
        path = record['path']
        size = _median([max(p[3] * aspect, p[4]) for p in path])
        anchors = [(p[0], p[1], p[2]) for p in path]
        if record['motion'] == 'moving':
            plans[identifier] = {'motion': 'moving', 'anchors': anchors, 'size': size, 'speed': record['speed']}
        elif record['motion'] == 'brief':
            best = max(path, key=lambda p: p[6])
            plans[identifier] = {'motion': 'brief', 'anchors': [(best[0], best[1], best[2])], 'size': size,
                                 'radius': min(0.15, max(2.5 * size, 0.03))}
    return plans


def summarize(samples, fps, aspect):
    """Whole-track full-rate figures: frames searched and seen, first and last visible time, and the speed
    fitted through the visible positions in frame heights per second (None below two positions)."""
    seen = [s for s in samples if s[1] is not None]
    speed = None
    if len(seen) > 1:
        _, _, _, vx, vy, spread = _fit(seen)
        speed = round(math.hypot(vx * aspect, vy), 6) if spread > 1e-9 else None
    return {'fps': round(fps, 3), 'searched': len(samples), 'seen': len(seen), 'speed': speed,
            'first': round(seen[0][0], 3) if seen else None, 'last': round(seen[-1][0], 3) if seen else None}


def window(samples, t):
    """The full-rate samples within WINDOW seconds of t, rounded for storage."""
    return [[round(s[0], 3), None if s[1] is None else round(s[1], 5), None if s[2] is None else round(s[2], 5),
             round(s[3], 1)] for s in samples if abs(s[0] - t) <= WINDOW]