Skip to content

Record a sky camera around the clock, keep the footage on your own disk, and find what flew past.

12.8 KB
Raw Blame
import hashlib
import io
import http.client
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import threading
import time
import unittest
from datetime import date
from pathlib import Path
from unittest import mock

from skyvision_store.__main__ import RETENTION_DAYS
from skyvision_store.janitor import prune
from skyvision_store.objects import INCOMING
from skyvision_store.server import StoreServer

TOKEN = 't' * 40
KEY = 'raw/cam=sky01/date=2026-09-22/hour=14/sky01_20260922T140500Z.mp4'


def sha(data: bytes) -> str:
    return hashlib.sha256(data).hexdigest()


class StoreTest(unittest.TestCase):
    def setUp(self):
        self.root = Path(tempfile.mkdtemp())
        self.mounted = True
        self.server = StoreServer(('127.0.0.1', 0), self.root, TOKEN, min_free=0, mounted=lambda: self.mounted)
        threading.Thread(target=self.server.serve_forever, daemon=True).start()
        self.connection = http.client.HTTPConnection('127.0.0.1', self.server.server_address[1], timeout=5)

    def tearDown(self):
        self.connection.close()
        self.server.shutdown()
        self.server.server_close()

    def request(self, method, key=KEY, body=None, headers=None, token=TOKEN):
        headers = {'Authorization': f'Bearer {token}', **(headers or {})}
        self.connection.request(method, f'/o/{key}', body=body, headers=headers, encode_chunked='Transfer-Encoding' in headers)
        response = self.connection.getresponse()
        return response, response.read()

    def put(self, data: bytes, key=KEY, digest=None, **options):
        return self.request('PUT', key, data, {'X-Sha256': digest or sha(data)}, **options)

    def stored(self, key=KEY) -> bytes | None:
        path = self.root / key
        return path.read_bytes() if path.exists() else None

    def leftovers(self) -> list:
        return list((self.root / INCOMING).glob('*'))

    # --- Writing ---

    def test_stores_new_bytes_exactly(self):
        data = os.urandom(3_000_000)
        response, body = self.put(data)
        self.assertEqual(response.status, 201)
        self.assertEqual(json.loads(body), {'size': len(data)})
        self.assertEqual(self.stored(), data)
        self.assertEqual(self.leftovers(), [])

    def test_accepts_a_chunked_body(self):
        data = os.urandom(200_000)
        response, _ = self.request('PUT', body=iter([data[:70_000], data[70_000:]]),
                                   headers={'X-Sha256': sha(data), 'Transfer-Encoding': 'chunked'})
        self.assertEqual(response.status, 201)
        self.assertEqual(self.stored(), data)

    def test_same_bytes_again_is_success(self):
        data = os.urandom(10_000)
        self.put(data)
        response, body = self.put(data)
        self.assertEqual(response.status, 200)
        self.assertEqual(json.loads(body), {'size': len(data)})

    def test_refuses_different_bytes_and_keeps_the_connection_usable(self):
        original = os.urandom(10_000)
        self.put(original)
        response, _ = self.put(os.urandom(10_000))
        self.assertEqual(response.status, 409)
        self.assertEqual(self.stored(), original)
        response, body = self.request('GET')  # same connection: the refused body was drained
        self.assertEqual((response.status, body), (200, original))

    def test_refuses_a_body_that_does_not_match_its_checksum(self):
        response, _ = self.put(os.urandom(10_000), digest=sha(b'other'))
        self.assertEqual(response.status, 422)
        self.assertIsNone(self.stored())
        self.assertEqual(self.leftovers(), [])

    def test_refuses_writes_while_the_disk_is_not_mounted(self):
        self.mounted = False
        response, _ = self.put(os.urandom(1_000))
        self.assertEqual(response.status, 503)
        self.assertIsNone(self.stored())

    def test_refuses_writes_when_the_disk_is_nearly_full(self):
        self.server.min_free = 1 << 62
        response, _ = self.put(os.urandom(1_000))
        self.assertEqual(response.status, 507)
        self.assertIsNone(self.stored())

    def test_answers_503_when_the_footage_folder_is_missing(self):
        shutil.rmtree(self.root)
        response, _ = self.put(os.urandom(1_000))
        self.assertEqual(response.status, 503)

    def test_accepts_writes_while_free_space_is_above_the_floor(self):
        # The janitor stops pruning once free space reaches the floor, so a write must not demand more.
        free = shutil.disk_usage(self.root).free
        self.server.min_free = free - 500
        with mock.patch('skyvision_store.server.shutil.disk_usage', return_value=mock.Mock(free=free)):
            response, _ = self.put(os.urandom(1_000))
        self.assertEqual(response.status, 201)

    def test_rejects_a_malformed_checksum(self):
        response, _ = self.put(b'data', digest='abc')
        self.assertEqual(response.status, 400)

    # --- Access ---

    def test_requires_the_token(self):
        response, _ = self.put(b'data', token='wrong')
        self.assertEqual(response.status, 401)
        self.assertIsNone(self.stored())

    def test_rejects_keys_outside_the_footage_tree(self):
        for key in ['raw/../../etc/passwd', f'{INCOMING}/part', 'raw//x.mp4', 'raw/a%20b.mp4']:
            with self.subTest(key=key):
                response, _ = self.put(b'data', key=key)
                self.assertEqual(response.status, 404)
        self.assertFalse(any(self.root.rglob('*.mp4')))

    # --- Reading ---

    def test_reads_whole_files_ranges_and_sizes(self):
        data = bytes(i % 251 for i in range(1_000))
        self.put(data)
        cases = [
            (None, 200, data, None),
            ('bytes=100-199', 206, data[100:200], 'bytes 100-199/1000'),
            ('bytes=990-', 206, data[990:], 'bytes 990-999/1000'),
            ('bytes=-10', 206, data[990:], 'bytes 990-999/1000'),
            ('bytes=900-5000', 206, data[900:], 'bytes 900-999/1000'),
        ]
        for header, status, expected, content_range in cases:
            with self.subTest(range=header):
                response, body = self.request('GET', headers={'Range': header} if header else {})
                self.assertEqual((response.status, body), (status, expected))
                self.assertEqual(response.getheader('Content-Range'), content_range)
                self.assertEqual(response.getheader('Content-Type'), 'video/mp4')
        response, body = self.request('HEAD')
        self.assertEqual((response.status, body, response.getheader('Content-Length')), (200, b'', '1000'))
        self.assertTrue(response.getheader('ETag'))

    def test_answers_unsatisfiable_ranges_with_the_size(self):
        self.put(bytes(1_000))
        for header in ['bytes=1000-', 'bytes=5-3', 'bytes=-0']:
            with self.subTest(range=header):
                response, _ = self.request('GET', headers={'Range': header})
                self.assertEqual(response.status, 416)
                self.assertEqual(response.getheader('Content-Range'), 'bytes */1000')

    def test_missing_recording_is_404(self):
        response, _ = self.request('GET')
        self.assertEqual(response.status, 404)

    # --- Thumbnails ---

    def thumbnail(self, image: bytes, query: str, token=TOKEN):
        self.connection.request('POST', f'/thumbnail?{query}', body=image, headers={'Authorization': f'Bearer {token}'})
        response = self.connection.getresponse()
        return response, response.read()

    def test_crops_the_object_and_shrinks_whole_frames(self):
        from PIL import Image
        # A 4K frame, blue on the left half and green on the right, with a red object at (2000, 1000).
        frame = Image.new('RGB', (3840, 2160), 'blue')
        frame.paste('green', (1920, 0, 3840, 2160))
        frame.paste('red', (2000, 1000, 2060, 1034))
        jpeg = io.BytesIO()
        frame.save(jpeg, 'JPEG', quality=92)
        response, body = self.thumbnail(jpeg.getvalue(), 'size=512&left=1970&top=983&width=120&height=68')
        self.assertEqual((response.status, response.getheader('Content-Type')), (200, 'image/jpeg'))
        crop = Image.open(io.BytesIO(body))
        self.assertEqual(crop.size, (512, 290))
        self.assertGreater(crop.getpixel((256, 145))[0], 200)  # the object fills the middle, enlarged
        self.assertLess(crop.getpixel((5, 5))[0], 60)  # the margin around it is not red
        response, body = self.thumbnail(jpeg.getvalue(), 'size=512')
        whole = Image.open(io.BytesIO(body))
        self.assertEqual(whole.size, (512, 288))
        self.assertGreater(whole.getpixel((10, 144))[2], 200)  # blue left
        self.assertGreater(whole.getpixel((500, 144))[1], 100)  # green right

    def test_never_enlarges_a_whole_frame(self):
        from PIL import Image
        jpeg = io.BytesIO()
        Image.new('RGB', (300, 200), 'blue').save(jpeg, 'JPEG')
        response, body = self.thumbnail(jpeg.getvalue(), 'size=512')
        self.assertEqual(Image.open(io.BytesIO(body)).size, (300, 200))

    def test_refuses_bad_thumbnail_requests(self):
        self.assertEqual(self.thumbnail(b'jpeg', 'size=512', token='wrong')[0].status, 401)
        self.assertEqual(self.thumbnail(b'jpeg', 'size=0')[0].status, 400)
        self.assertEqual(self.thumbnail(b'jpeg', 'size=512&left=0&top=0&width=0&height=5')[0].status, 400)
        self.assertEqual(self.thumbnail(b'not a jpeg', 'size=512')[0].status, 422)


class JanitorTest(unittest.TestCase):
    def setUp(self):
        self.root = Path(tempfile.mkdtemp())
        for day in ['2026-08-01', '2026-08-20', '2026-09-27']:
            for hour in ['00', '01']:
                folder = self.root / f'raw/cam=sky01/date={day}/hour={hour}'
                folder.mkdir(parents=True)
                (folder / 'clip.mp4').write_bytes(b'x')

    def remaining(self) -> list[str]:
        return sorted(str(p.relative_to(self.root / 'raw/cam=sky01')) for p in self.root.glob('raw/cam=*/date=*/hour=*'))

    def test_deletes_days_past_retention(self):
        prune(self.root, date(2026, 9, 28), retention_days=40, min_free=0)
        self.assertEqual(self.remaining(), ['date=2026-08-20/hour=00', 'date=2026-08-20/hour=01',
                                            'date=2026-09-27/hour=00', 'date=2026-09-27/hour=01'])

    def test_deletes_oldest_hours_until_there_is_room(self):
        freed = iter([0, 0, 1])  # room appears after two hours are gone
        prune(self.root, date(2026, 9, 28), retention_days=400, min_free=1, free=lambda: next(freed))
        self.assertEqual(self.remaining()[0], 'date=2026-08-20/hour=00')
        self.assertEqual(len(self.remaining()), 4)

    def test_removes_only_stale_upload_parts(self):
        incoming = self.root / INCOMING
        incoming.mkdir()
        (incoming / 'fresh').write_bytes(b'x')
        stale = incoming / 'stale'
        stale.write_bytes(b'x')
        os.utime(stale, (time.time() - 2 * 86400,) * 2)
        prune(self.root, date(2026, 9, 28), retention_days=400, min_free=0)
        self.assertEqual([p.name for p in incoming.glob('*')], ['fresh'])


class RetentionTest(unittest.TestCase):
    def test_files_outlive_the_workers_retention(self):
        shared = Path(__file__).parents[3] / 'package/shared/src/utils/layoutStorageUtil.ts'
        workers = int(re.search(r'RETENTION_DAYS = (\d+);', shared.read_text()).group(1))
        self.assertGreater(RETENTION_DAYS, workers)


class DeployTest(unittest.TestCase):
    def test_restart_waits_for_the_old_process(self):
        # Booting out by label returns before the process exits, so the bootstrap after it fails
        # and leaves the agent stopped. Only the plist-path form waits.
        script = (Path(__file__).parents[1] / 'deploy.sh').read_text()
        bootouts = re.findall(r'^\s*launchctl bootout (.*)$', script, re.MULTILINE)
        self.assertTrue(bootouts)
        for arguments in bootouts:
            self.assertRegex(arguments, r'^"gui/\$\(id -u\)" "[^"]+\.plist"')

    @unittest.skipUnless(shutil.which('systemd-analyze'), 'needs systemd-analyze')
    def test_linux_unit_takes_any_footage_folder(self):
        # ExecStart= expands % and $ and unescapes \ and ", so the folder is escaped for systemd.
        script = (Path(__file__).parents[1] / 'deploy.sh').read_text()
        quote = re.search(r'^\s*(unit_quote\(\).*)$', script, re.MULTILINE).group(1)
        unit = Path(tempfile.mkdtemp()) / 'store.service'
        root = '/mnt/foot%20age $x "q" back\\slash'
        subprocess.run(['bash', '-c', quote + '; printf "[Service]\\nExecStart=%s -m skyvision_store --root %s\\n" '
                        '"$(unit_quote "$1")" "$(unit_quote "$2")" > "$3"', '-', sys.executable, root, str(unit)], check=True)
        self.assertIn('--root "/mnt/foot%%20age $$x \\"q\\" back\\\\slash"', unit.read_text())
        result = subprocess.run(['systemd-analyze', 'verify', '--man=no', str(unit)], capture_output=True, text=True)
        self.assertEqual(0, result.returncode, result.stderr)


if __name__ == '__main__':
    unittest.main()