Skip to content

The OSINT investigator's workbench. One case, one folder, every tool. Local-first: your media and your investigations never leave your machine.

JavaScript
25.2 KB
Raw Blame
#!/usr/bin/env python3
"""Load, pair and reload the capture extension in a dev browser, without clicks.

A maintainer tool, not part of the app. It drives Chrome's DevTools protocol and
Firefox's remote debugging protocol, so an edit in ``extension/`` reaches both
browsers with one command:

    python3 scripts/devext.py               # both browsers: load/reload + pair
    python3 scripts/devext.py chrome        # just one of them
    python3 scripts/devext.py --watch       # reload on every save, until Ctrl-C
    python3 scripts/devext.py --fresh       # start from an empty dev profile

Each browser gets its own dev profile, kept between runs, so the pairing token,
the last case and anything set in the popup survive a restart. The token is read
from the running app (or from its settings.json) and written straight into the
extension's storage, which is the manual paste this script exists to remove.

The dev browsers are separate instances: the everyday Chrome and Firefox keep
running untouched, with their own profiles and their own copy of the extension.

Stdlib only: it runs with any python3, no venv and no npm.
"""

from __future__ import annotations

import argparse
import base64
import json
import os
import shutil
import socket
import struct
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

REPO = Path(__file__).resolve().parent.parent
EXT = REPO / "extension"
STATE = Path.home() / ".cache" / "azimut-devext"


def addon_id() -> str:
    """The extension's Firefox id, read from its own manifest.

    Firefox keys a temporary add-on by this, and AMO reserves it at the first
    signature, so it is permanent. Read rather than repeated: a copy here would
    drift the day the id moves and this script would simply stop finding the
    add-on it just loaded.
    """
    manifest = json.loads((EXT / "manifest.json").read_text(encoding="utf-8"))
    return manifest["browser_specific_settings"]["gecko"]["id"]


DEFAULT_APP_URL = os.environ.get("AZIMUT_URL", "http://127.0.0.1:8477")
DEFAULT_CHROME_PORT = 9222
DEFAULT_FIREFOX_PORT = 6080

CHROME_NAMES = (
    "google-chrome",
    "google-chrome-stable",
    "chromium",
    "chromium-browser",
    "brave-browser",
    "microsoft-edge",
)
FIREFOX_NAMES = ("firefox", "firefox-esr", "firefox-developer-edition")
MAC_APPS = {
    "chrome": "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
    "firefox": "/Applications/Firefox.app/Contents/MacOS/firefox",
}


@dataclass
class Options:
    """What one sync does, so `relaunch.py --ext` can ask for the same thing."""

    browsers: list[str] = field(default_factory=lambda: ["chrome", "firefox"])
    url: str = DEFAULT_APP_URL
    fresh: bool = False
    reload_tabs: bool = True
    chrome_port: int = DEFAULT_CHROME_PORT
    firefox_port: int = DEFAULT_FIREFOX_PORT


def say(browser: str, message: str) -> None:
    print(f"[{browser}] {message}", flush=True)


BACKGROUND_PROOF = (
    "((api) => JSON.stringify(["
    "api.runtime.getManifest().version, "
    "Math.round(Date.now() - performance.timeOrigin)"
    "]))(typeof browser !== 'undefined' ? browser : chrome)"
)


def report_background(browser: str, answer: str | None) -> None:
    """Say what the background context is, and how long ago it started.

    An age of a second or so is the proof the sync landed: reloading the
    extension restarts its background, so anything older is code from before.
    """
    if not answer:
        say(browser, "background asleep, nothing to read back")
        return
    version, age = json.loads(answer)
    if age > 15000:
        say(browser, f"background is {age // 1000}s old: it kept the code it had")
        return
    say(browser, f"background restarted {age / 1000:.1f}s ago, running v{version}")


def find_binary(env_var: str, names: tuple[str, ...], mac_key: str) -> str | None:
    override = os.environ.get(env_var)
    if override:
        return override
    for name in names:
        found = shutil.which(name)
        if found:
            return found
    mac = MAC_APPS[mac_key]
    return mac if Path(mac).exists() else None


def _detached() -> dict[str, Any]:
    """Popen arguments that let the browser survive this script exiting."""
    if os.name == "nt":
        return {"creationflags": getattr(subprocess, "DETACHED_PROCESS", 0)}
    return {"start_new_session": True}


def port_open(port: int) -> bool:
    with socket.socket() as s:
        s.settimeout(0.4)
        return s.connect_ex(("127.0.0.1", port)) == 0


def wait_for_port(port: int, timeout: float = 30.0) -> bool:
    deadline = time.time() + timeout
    while time.time() < deadline:
        if port_open(port):
            return True
        time.sleep(0.3)
    return False


# --- pairing token ----------------------------------------------------------


def pairing_token(app_url: str) -> str | None:
    """The app's ingest token: from the running app, else from settings.json."""
    request = urllib.request.Request(
        f"{app_url}/api/settings/ingest-token", data=b"", method="POST"
    )
    try:
        with urllib.request.urlopen(request, timeout=3) as response:
            return json.load(response).get("ingest_token")
    except (urllib.error.URLError, OSError, ValueError):
        pass
    root = Path(os.environ.get("AZIMUT_HOME", "~/Azimut")).expanduser()
    settings = root / ".azimut" / "settings" / "settings.json"
    try:
        return json.loads(settings.read_text(encoding="utf-8")).get("ingest_token")
    except (OSError, ValueError):
        return None


# --- a small WebSocket client (Chrome DevTools protocol) --------------------


class WebSocket:
    """Just enough of RFC 6455 to talk CDP: text frames, one connection."""

    def __init__(self, url: str) -> None:
        rest = url.split("://", 1)[1]
        hostport, _, path = rest.partition("/")
        host, _, port = hostport.partition(":")
        self.sock = socket.create_connection((host, int(port or 80)), timeout=30)
        key = base64.b64encode(os.urandom(16)).decode()
        self.sock.sendall(
            (
                f"GET /{path} HTTP/1.1\r\n"
                f"Host: {hostport}\r\n"
                "Upgrade: websocket\r\n"
                "Connection: Upgrade\r\n"
                f"Sec-WebSocket-Key: {key}\r\n"
                "Sec-WebSocket-Version: 13\r\n\r\n"
            ).encode()
        )
        self.buf = b""
        while b"\r\n\r\n" not in self.buf:
            self.buf += self._read(1)
        head, _, self.buf = self.buf.partition(b"\r\n\r\n")
        if b"101" not in head.split(b"\r\n")[0]:
            raise RuntimeError(f"websocket refused: {head.splitlines()[0]!r}")

    def _read(self, n: int) -> bytes:
        chunk = self.sock.recv(max(n, 4096))
        if not chunk:
            raise RuntimeError("websocket closed")
        return chunk

    def _take(self, n: int) -> bytes:
        while len(self.buf) < n:
            self.buf += self._read(n - len(self.buf))
        out, self.buf = self.buf[:n], self.buf[n:]
        return out

    def send(self, text: str) -> None:
        payload = text.encode()
        header = bytearray([0x81])
        length = len(payload)
        if length < 126:
            header.append(0x80 | length)
        elif length < 1 << 16:
            header.append(0x80 | 126)
            header += struct.pack(">H", length)
        else:
            header.append(0x80 | 127)
            header += struct.pack(">Q", length)
        mask = os.urandom(4)
        masked = bytes(b ^ mask[i % 4] for i, b in enumerate(payload))
        self.sock.sendall(bytes(header) + mask + masked)

    def recv(self) -> str:
        while True:
            first, second = self._take(2)
            opcode = first & 0x0F
            length = second & 0x7F
            if length == 126:
                length = struct.unpack(">H", self._take(2))[0]
            elif length == 127:
                length = struct.unpack(">Q", self._take(8))[0]
            payload = self._take(length) if length else b""
            if opcode == 0x9:  # ping: the protocol says answer it
                self.sock.sendall(b"\x8a\x80" + os.urandom(4))
                continue
            if opcode == 0x8:
                raise RuntimeError("websocket closed by browser")
            if opcode in (0x1, 0x2):
                return payload.decode("utf-8", "replace")

    def close(self) -> None:
        try:
            self.sock.close()
        except OSError:
            pass


class Chrome:
    def __init__(self, port: int) -> None:
        with urllib.request.urlopen(
            f"http://127.0.0.1:{port}/json/version", timeout=5
        ) as response:
            endpoint = json.load(response)["webSocketDebuggerUrl"]
        self.ws = WebSocket(endpoint)
        self.n = 0

    def call(
        self, method: str, params: dict[str, Any] | None = None, session: str = ""
    ) -> dict[str, Any]:
        self.n += 1
        message: dict[str, Any] = {"id": self.n, "method": method, "params": params or {}}
        if session:
            message["sessionId"] = session
        self.ws.send(json.dumps(message))
        while True:
            answer = json.loads(self.ws.recv())
            if answer.get("id") != self.n:
                continue  # an event, or a stale answer
            if "error" in answer:
                raise RuntimeError(f"{method}: {answer['error'].get('message')}")
            return answer.get("result", {})

    def close(self) -> None:
        self.ws.close()


def chrome_sync(options: Options, token: str | None) -> None:
    binary = find_binary("AZIMUT_CHROME", CHROME_NAMES, "chrome")
    if not binary:
        say("chrome", "not found. Set AZIMUT_CHROME to its path")
        return
    profile = STATE / "chrome"
    if options.fresh:
        if port_open(options.chrome_port):
            chrome_quit(options.chrome_port, profile)
        shutil.rmtree(profile, ignore_errors=True)
    if not port_open(options.chrome_port):
        profile.mkdir(parents=True, exist_ok=True)
        subprocess.Popen(
            [
                binary,
                f"--user-data-dir={profile}",
                f"--remote-debugging-port={options.chrome_port}",
                # Chrome 137 dropped --load-extension; unpacked loading now goes
                # through the DevTools protocol, which this flag opens up.
                "--enable-unsafe-extension-debugging",
                "--no-first-run",
                "--no-default-browser-check",
                "--disable-search-engine-choice-screen",
                options.url,
            ],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
            # Detached: the browser outlives the sync that started it.
            **_detached(),
        )
        if not wait_for_port(options.chrome_port):
            say("chrome", "no debug port. A dev Chrome may still hold the profile")
            return
        say("chrome", f"launched on a dev profile ({profile})")

    cdp = Chrome(options.chrome_port)
    try:
        extension_id = cdp.call("Extensions.loadUnpacked", {"path": str(EXT)})["id"]
        say("chrome", f"extension loaded from disk ({extension_id})")
        if token:
            chrome_in_extension_page(
                cdp,
                extension_id,
                [
                    "chrome.storage.local.set("
                    f"{json.dumps({'backendUrl': options.url, 'token': token})})"
                ],
            )
            say("chrome", "paired")
        report_background("chrome", chrome_background_proof(cdp, extension_id))
        if options.reload_tabs:
            say("chrome", f"reloaded {chrome_reload_tabs(cdp, options.url)} app tab(s)")
    finally:
        cdp.close()


def chrome_quit(port: int, profile: Path) -> None:
    """Ask the dev Chrome to close, and wait until it has really let go.

    Its lock file outlives the debug port by a moment, and a profile replaced
    before that leaves the next launch talking to the dying instance instead of
    starting its own.
    """
    cdp = Chrome(port)
    try:
        cdp.call("Browser.close")
    except (RuntimeError, OSError):
        pass
    finally:
        cdp.close()
    deadline = time.time() + 15
    lock = profile / "SingletonLock"
    while time.time() < deadline and (port_open(port) or lock.exists()):
        time.sleep(0.2)
    time.sleep(0.5)


def chrome_in_extension_page(
    cdp: Chrome, extension_id: str, expressions: list[str]
) -> list[Any]:
    """Run JS with the extension's own APIs, in a page opened for that alone.

    `chrome.storage` and the extension's files exist only inside an extension
    context, so the options page is opened, used and closed again.
    """
    target = cdp.call(
        "Target.createTarget", {"url": f"chrome-extension://{extension_id}/options.html"}
    )["targetId"]
    values: list[Any] = []
    try:
        time.sleep(0.8)
        session = cdp.call(
            "Target.attachToTarget", {"targetId": target, "flatten": True}
        )["sessionId"]
        for expression in expressions:
            answer = cdp.call(
                "Runtime.evaluate",
                {"expression": expression, "awaitPromise": True, "returnByValue": True},
                session,
            )
            if "exceptionDetails" in answer:
                raise RuntimeError(answer["exceptionDetails"].get("text", "evaluation failed"))
            values.append(answer["result"].get("value"))
    finally:
        cdp.call("Target.closeTarget", {"targetId": target})
    return values


def chrome_background_proof(cdp: Chrome, extension_id: str) -> str | None:
    """Ask the extension's service worker what it is running.

    It is the context the reload restarts, and the only one whose age says
    whether the browser took the new code.
    """
    deadline = time.time() + 5
    while time.time() < deadline:
        for target in cdp.call("Target.getTargets")["targetInfos"]:
            if target["type"] != "service_worker" or extension_id not in target["url"]:
                continue
            session = cdp.call(
                "Target.attachToTarget", {"targetId": target["targetId"], "flatten": True}
            )["sessionId"]
            answer = cdp.call(
                "Runtime.evaluate",
                {"expression": BACKGROUND_PROOF, "returnByValue": True},
                session,
            )
            cdp.call("Target.detachFromTarget", {"sessionId": session})
            value = answer["result"].get("value")
            return str(value) if value is not None else None
        time.sleep(0.3)
    return None


def chrome_reload_tabs(cdp: Chrome, app_url: str) -> int:
    """Reload the app's own tabs: content scripts only re-inject on navigation."""
    reloaded = 0
    for target in cdp.call("Target.getTargets")["targetInfos"]:
        if target["type"] != "page" or not target["url"].startswith(app_url):
            continue
        session = cdp.call(
            "Target.attachToTarget", {"targetId": target["targetId"], "flatten": True}
        )["sessionId"]
        cdp.call("Page.reload", {}, session)
        cdp.call("Target.detachFromTarget", {"sessionId": session})
        reloaded += 1
    return reloaded


# --- Firefox remote debugging protocol --------------------------------------


class Firefox:
    """The `<length>:<json>` protocol behind about:debugging."""

    def __init__(self, port: int) -> None:
        self.sock = socket.create_connection(("127.0.0.1", port), timeout=30)
        self.buf = b""
        self.read()  # the server greets before anything is asked of it

    def read(self) -> dict[str, Any]:
        while True:
            colon = self.buf.find(b":")
            if colon != -1:
                length = int(self.buf[:colon])
                if len(self.buf) >= colon + 1 + length:
                    packet = self.buf[colon + 1 : colon + 1 + length]
                    self.buf = self.buf[colon + 1 + length :]
                    return json.loads(packet)
            chunk = self.sock.recv(65536)
            if not chunk:
                raise RuntimeError("debugger connection closed")
            self.buf += chunk

    def call(
        self, message: dict[str, Any], event: str = "", timeout: float = 20.0
    ) -> dict[str, Any]:
        """Send one request; return its reply, or the awaited event instead."""
        data = json.dumps(message).encode()
        self.sock.sendall(str(len(data)).encode() + b":" + data)
        deadline = time.time() + timeout
        self.sock.settimeout(timeout)
        while time.time() < deadline:
            answer = self.read()
            if event:
                if answer.get("type") == event:
                    return answer
            elif answer.get("from") == message["to"] and "type" not in answer:
                if "error" in answer:
                    raise RuntimeError(f"{message['type']}: {answer['error']}")
                return answer
        raise RuntimeError(f"no answer to {message['type']}")

    def close(self) -> None:
        try:
            self.sock.close()
        except OSError:
            pass


FIREFOX_PREFS = """\
// Written by scripts/devext.py: dev profile for the capture extension.
user_pref("devtools.debugger.remote-enabled", true);
user_pref("devtools.debugger.prompt-connection", false);
user_pref("devtools.chrome.enabled", true);
user_pref("browser.shell.checkDefaultBrowser", false);
user_pref("browser.aboutwelcome.enabled", false);
user_pref("datareporting.policy.dataSubmissionEnabled", false);
user_pref("extensions.autoDisableScopes", 0);
"""


def firefox_profile_dir() -> Path:
    # The snap can only read parts of $HOME; its own writable area always works.
    snap = Path.home() / "snap" / "firefox" / "common"
    return (snap / "azimut-devext") if snap.exists() else (STATE / "firefox")


def firefox_sync(options: Options, token: str | None) -> None:
    binary = find_binary("AZIMUT_FIREFOX", FIREFOX_NAMES, "firefox")
    if not binary:
        say("firefox", "not found. Set AZIMUT_FIREFOX to its path")
        return
    profile = firefox_profile_dir()
    if options.fresh:
        if port_open(options.firefox_port):
            # Firefox has no remote quit, and the profile is open under it.
            say("firefox", "already running. Close it first to start fresh")
            return
        shutil.rmtree(profile, ignore_errors=True)
    if not port_open(options.firefox_port):
        profile.mkdir(parents=True, exist_ok=True)
        (profile / "user.js").write_text(FIREFOX_PREFS, encoding="utf-8")
        subprocess.Popen(
            [
                binary,
                "--profile",
                str(profile),
                "--no-remote",  # a second instance, beside the everyday Firefox
                "--start-debugger-server",
                str(options.firefox_port),
                options.url,
            ],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
            # Detached: the browser outlives the sync that started it.
            **_detached(),
        )
        if not wait_for_port(options.firefox_port, timeout=45):
            say("firefox", "no debug port. A dev Firefox may still hold the profile")
            return
        say("firefox", f"launched on a dev profile ({profile})")

    rdp = Firefox(options.firefox_port)
    try:
        root = rdp.call({"to": "root", "type": "getRoot"})
        rdp.call(
            {
                "to": root["addonsActor"],
                "type": "installTemporaryAddon",
                "addonPath": str(EXT),
                "openDevTools": False,
            }
        )
        say("firefox", "extension installed from disk")
        if not token:
            return
        console = firefox_background_console(rdp)
        if not console:
            say("firefox", "background page not reachable. Pair by hand this once")
            return
        script = (
            "browser.storage.local.set("
            f"{json.dumps({'backendUrl': options.url, 'token': token})})"
        )
        if options.reload_tabs:
            # Filtered here rather than by `tabs.query({url})`: a match pattern
            # cannot carry a port, and the app URL always has one.
            script += (
                ".then(() => browser.tabs.query({}))"
                ".then(tabs => tabs.filter(tab => (tab.url || '').startsWith("
                f"{json.dumps(options.url)})))"
                ".then(tabs => Promise.all(tabs.map(tab => browser.tabs.reload(tab.id)))"
                ".then(() => tabs.length)).catch(() => 0)"
            )
        answer = firefox_evaluate(rdp, console, script)
        if answer is None:
            say("firefox", "could not write the token into the extension")
            return
        say("firefox", "paired")
        report_background("firefox", firefox_evaluate(rdp, console, BACKGROUND_PROOF))
        if options.reload_tabs:
            say("firefox", f"reloaded {answer} app tab(s)")
    finally:
        rdp.close()


def firefox_evaluate(rdp: Firefox, console: str, script: str) -> Any:
    """Run one expression in the add-on's background, or None if it threw."""
    answer = rdp.call(
        {"to": console, "type": "evaluateJSAsync", "text": script, "mapped": {"await": True}},
        event="evaluationResult",
    )
    return None if answer.get("hasException") else answer.get("result")


def firefox_background_console(rdp: Firefox) -> str | None:
    """The console actor of the add-on's background page.

    Watching the add-on announces its frames one by one; the first is devtools'
    own fallback page, the extension's own comes right after.
    """
    addons = rdp.call({"to": "root", "type": "listAddons"})["addons"]
    mine = next((a for a in addons if a.get("id") == addon_id()), None)
    if not mine:
        return None
    watcher = rdp.call({"to": mine["actor"], "type": "getWatcher"})["actor"]
    rdp.call(
        {"to": watcher, "type": "watchTargets", "targetType": "frame"},
        event="target-available-form",
    )
    deadline = time.time() + 10
    rdp.sock.settimeout(10)
    while time.time() < deadline:
        try:
            packet = rdp.read()
        except (OSError, RuntimeError):
            break
        target = packet.get("target") or {}
        if target.get("url", "").startswith("moz-extension://"):
            return target.get("consoleActor")
    return None


# --- watch mode -------------------------------------------------------------


def fingerprint() -> tuple[tuple[str, float], ...]:
    return tuple(
        sorted(
            (str(path), path.stat().st_mtime)
            for path in EXT.rglob("*")
            if path.is_file() and not path.name.startswith(".")
        )
    )


def sync(options: Options) -> None:
    """One pass: load the code from disk into each dev browser, and pair it."""
    token = pairing_token(options.url)
    if not token:
        print("no pairing token yet. Start Azimut once so it mints one", flush=True)
    for browser in options.browsers:
        try:
            (chrome_sync if browser == "chrome" else firefox_sync)(options, token)
        except (RuntimeError, OSError) as error:
            say(browser, f"failed: {error}")


def watch(options: Options, stop: threading.Event | None = None) -> None:
    """Sync again on every save in extension/, until `stop` is set."""
    stop = stop or threading.Event()
    options.fresh = False  # a wipe belongs to the first pass only
    seen = fingerprint()
    while not stop.wait(0.6):
        if fingerprint() == seen:
            continue
        stop.wait(0.3)  # let the editor finish writing
        print(f"change at {time.strftime('%H:%M:%S')}", flush=True)
        sync(options)
        seen = fingerprint()


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument(
        "browsers",
        nargs="*",
        default=[],
        metavar="{chrome,firefox,all}",
        help="which dev browser to sync (default: both)",
    )
    parser.add_argument("--url", default=DEFAULT_APP_URL, help="where the app runs")
    parser.add_argument("--watch", action="store_true", help="reload on every save")
    parser.add_argument("--fresh", action="store_true", help="wipe the dev profile first")
    parser.add_argument(
        "--no-reload-tabs",
        dest="reload_tabs",
        action="store_false",
        help="leave the open app tabs alone",
    )
    parser.add_argument("--chrome-port", type=int, default=DEFAULT_CHROME_PORT)
    parser.add_argument("--firefox-port", type=int, default=DEFAULT_FIREFOX_PORT)
    args = parser.parse_args()

    unknown = set(args.browsers) - {"chrome", "firefox", "all"}
    if unknown:
        parser.error(f"unknown browser: {', '.join(sorted(unknown))}")

    if not (EXT / "manifest.json").exists():
        print(f"no extension at {EXT}", file=sys.stderr)
        return 1

    options = Options(
        browsers=["chrome", "firefox"]
        if "all" in args.browsers or not args.browsers
        else list(dict.fromkeys(args.browsers)),
        url=args.url,
        fresh=args.fresh,
        reload_tabs=args.reload_tabs,
        chrome_port=args.chrome_port,
        firefox_port=args.firefox_port,
    )
    sync(options)
    if not args.watch:
        return 0

    print("watching extension/, Ctrl-C to stop", flush=True)
    try:
        watch(options)
    except KeyboardInterrupt:
        print()
    return 0


if __name__ == "__main__":
    sys.exit(main())