The OSINT investigator's workbench. One case, one folder, every tool. Local-first: your media and your investigations never leave your machine.
JavaScript
"""Turn a raw calibration run into the fixture both test suites read.
python scripts/build_map_fixture.py
`frontend/calibration/record.mjs` drives the map sites in a real browser and
writes down the URLs they wrote about themselves: `tests/fixtures/map-sites.raw.json`,
observation and nothing else. This reads that and produces
`tests/fixtures/map-sites.json`, which carries two derived things the suites
need:
* **the view each URL means**, from ``parse_map_url`` — the app's own parser,
which is the only thing in this project allowed to read a map URL. A recording
whose views were produced any other way would be testing a second parser
against the first;
* **where each site drew its camera**, solved from the two zooms in the
recording, the same way ``extension/mapmath.js`` solves it live.
That second one is a second copy of a formula, and it is on purpose. The JS is
what ships and what the drawing is done with; this is forty lines of Python that
has to agree with it, and `frontend/src/lib/extensionMapFrame.test.js` re-solves
every recording with the real thing and fails if the two ever differ by more
than a pixel. Two independent implementations agreeing on thirteen recordings is
a stronger statement about the arithmetic than either one alone.
Prose, per-recording notes and the `rounding` block are carried over from the
existing fixture: those are what a person knew, and a rebuild must not quietly
delete them.
"""
from __future__ import annotations
import json
import math
import sys
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
from azimut.engine.mapsites import _CAMERA_CENTRE, _mercator_y, parse_map_url # noqa: E402
RAW = ROOT / "tests" / "fixtures" / "map-sites.raw.json"
OUT = ROOT / "tests" / "fixtures" / "map-sites.json"
#: How much the zoom has to have moved between two URLs before the solve below
#: is worth doing. `extension/mapmath.js` states the same floor, and for the same
#: reason: the answer comes out amplified by 1/(1−2^Δz).
MIN_ZOOM_STEP = 0.5
#: How much chrome counts as chrome. The camera offset is measured to about a
#: tenth of a pixel, so a site that draws dead centre lands a hair either side of
#: zero, and calling that a two-pixel header would restate the window's height
#: for nothing. `extension/mapoverlay.js` uses the same floor.
MIN_CHROME_PX = 4
#: WGS84's first eccentricity — the difference between the two Mercators, and
#: the whole of Yandex's disagreement with everyone else.
_E = 0.081819190842621
_MAX_LAT = 85.05112877980659
def _ellipsoidal_y(lat: float) -> float:
"""Northing on the 256-pixel world at zoom 0, elliptical Mercator (Yandex)."""
phi = math.radians(max(-_MAX_LAT, min(_MAX_LAT, lat)))
es = _E * math.sin(phi)
t = math.tan(math.pi / 4 + phi / 2) * ((1 - es) / (1 + es)) ** (_E / 2)
return (1 - math.log(t) / math.pi) / 2 * 256
def _project(lat: float, lon: float, projection: str | None) -> tuple[float, float]:
y = _ellipsoidal_y(lat) if projection == "ellipsoidal" else _mercator_y(lat)
return ((lon + 180) / 360 * 256, y)
def _wrap_near(lon: float, near: float) -> float:
while lon - near > 180:
lon -= 360
while near - lon > 180:
lon += 360
return lon
def solve_centre(before: dict, after: dict, at: dict) -> dict[str, float] | None:
"""The pixel the site draws its camera at, from one zoom about a held point.
With ``s = 2^zoom`` and world coordinates taken at zoom 0, the pixel `p` the
zoom held still is the same before and after::
p = C + s0·(w(P) − w(c0)) = C + s1·(w(P) − w(c1))
which fixes both the point and the camera pixel `C`. Each axis is solved on
its own, so a site that is off-centre one way only says so.
"""
for view in (before, after):
if view.get("zoom") is None or view.get("lat") is None:
return None
if view.get("bearing"):
return None
if before.get("projection") != after.get("projection"):
return None
if abs(after["zoom"] - before["zoom"]) < MIN_ZOOM_STEP:
return None
s0 = 2 ** before["zoom"]
s1 = 2 ** after["zoom"]
c0 = _project(before["lat"], before["lon"], before.get("projection"))
c1 = _project(
after["lat"], _wrap_near(after["lon"], before["lon"]), after.get("projection")
)
out = {}
for axis, pixel in ((0, at["x"]), (1, at["y"])):
point = (s0 * c0[axis] - s1 * c1[axis]) / (s0 - s1)
out["xy"[axis]] = pixel - s0 * (point - c0[axis])
return out if all(math.isfinite(v) for v in out.values()) else None
def _view_of(parsed: dict[str, Any]) -> dict[str, Any]:
return {
"lat": parsed["lat"],
"lon": parsed["lon"],
"zoom": parsed["zoom"],
"bearing": parsed["bearing"] or 0,
"projection": parsed["projection"],
}
def build(entry: dict[str, Any]) -> tuple[dict[str, Any] | None, list[str]]:
"""One raw recording, parsed and solved. Returns the recording and what
went wrong with it."""
window = entry["window"]
trouble: list[str] = []
# The views that state a size rather than a level — Apple's span, Earth's
# distance, Google satellite's metres — are only a scale next to the height the map
# was drawn in, and that height is the window's minus whatever chrome sits
# above or below it. Which is what the solve below measures. So it is run
# twice: once against the window, then again against the map it found.
height = int(window["h"])
recording: dict[str, Any] | None = None
used = height
for _ in range(2):
used = height
views = []
for step in entry["steps"]:
parsed = parse_map_url(step["url"], height_px=height)
if parsed is None:
return None, [f"{step['url']} is not a map URL any more"]
views.append((step["url"], parsed))
solved = []
for zoom in entry["zooms"]:
placed = solve_centre(
_view_of(views[zoom["from"]][1]), _view_of(views[zoom["to"]][1]), zoom["at"]
)
if placed is not None:
solved.append(placed)
if not solved:
return None, ["no zoom in this recording moved a level the URL stated"]
centre = {
"x": round(sum(c["x"] for c in solved) / len(solved), 2),
"y": round(sum(c["y"] for c in solved) / len(solved), 2),
}
spread = max(
(math.dist((a["x"], a["y"]), (b["x"], b["y"])) for a in solved for b in solved),
default=0.0,
)
chrome = 2 * abs(centre["y"] - window["h"] / 2)
# a pixel count, in whole pixels, which is how the site drew it
height = round(window["h"] - chrome) if chrome >= MIN_CHROME_PX else int(window["h"])
recording = {
"label": entry["label"],
"site": entry["site"],
"browser": entry.get("browser", "chromium"),
"page_zoom": entry.get("pageZoom", 1),
"window": window,
"map_height": height,
"centre": centre,
"spread": round(spread, 2),
"canvas": entry.get("canvas"),
"pan": entry["pan"],
"zooms": entry["zooms"],
"steps": [
{
"url": url,
"view": _view_of(parsed),
"scale_source": parsed["scale_source"],
"site": parsed["site"],
}
for url, parsed in views
],
}
assert recording is not None
if recording["map_height"] != used:
# The height the views were parsed at is not the height they imply, so
# the scale and the offset are still chasing each other. One more pass
# would settle it on any layout this has met; a recording that needs
# more than that is one to look at rather than to ship.
trouble.append(
f"the map's height did not settle: parsed at {used}, solved {recording['map_height']}"
)
if recording["spread"] > 2:
trouble.append(
f"its two zooms solved centres {recording['spread']:.1f} px apart"
)
if any(step["view"]["lat"] is None for step in recording["steps"]):
trouble.append("a URL came back with no position in it")
sites = {step["site"] for step in recording["steps"]}
if sites != {entry["site"]}:
trouble.append(f"landed on {sorted(sites)} instead of {entry['site']}")
if recording["canvas"]:
box = recording["canvas"]
inside = (
box["x"] <= centre["x"] <= box["x"] + box["w"]
and box["y"] <= centre["y"] <= box["y"] + box["h"]
)
if not inside:
trouble.append("the camera solved outside the map's own canvas")
return recording, trouble
def build_rounding(block: dict[str, Any], old: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:
"""The one site that rounds the zoom it writes, notch by notch.
Six notches from a whole level, all about one pixel. The whole levels are
the ones the address bar states exactly, so they are what the camera is
solved from; the notches in between are what the extension has to work out
for itself, and what the fixture exists to ask it about.
"""
trouble: list[str] = []
window = block["window"]
parsed = [parse_map_url(step["url"]) for step in block["steps"]]
if any(view is None for view in parsed):
return {}, ["a URL in the rounding run is not a map URL any more"]
views = [view for view in parsed if view is not None]
per_level = max(1, round(1 / block.get("step", 1 / 3)))
solved = []
for first in (0, per_level):
last = first + per_level
if last >= len(views):
continue
placed = solve_centre(_view_of(views[first]), _view_of(views[last]), block["at"])
if placed is not None:
solved.append(placed)
if not solved:
trouble.append("no pair of whole levels in the rounding run could place the camera")
centre = {
"x": round(sum(c["x"] for c in solved) / len(solved), 2) if solved else window["w"] / 2,
"y": round(sum(c["y"] for c in solved) / len(solved), 2) if solved else window["h"] / 2,
}
stated = [view["zoom"] for view in views]
if stated != sorted(stated) or stated[-1] - stated[0] < 1.5:
trouble.append(f"the notches did not climb two levels: {stated}")
return {
"what": old.get("what", ""),
"how": old.get("how", ""),
"site": block["site"],
"browser": block.get("browser", "chromium"),
"window": window,
"at": block["at"],
"centre": centre,
"step": block.get("step", 1 / 3),
"steps": [
{"url": step["url"], "view": _view_of(view)}
for step, view in zip(block["steps"], views)
],
}, trouble
def main() -> int:
raw = json.loads(RAW.read_text(encoding="utf-8"))
old = json.loads(OUT.read_text(encoding="utf-8")) if OUT.exists() else {}
notes = {r["label"]: r.get("note") for r in old.get("recordings", []) if r.get("note")}
recordings: list[dict[str, Any]] = []
dropped: list[str] = []
for entry in raw["recorded"]:
recording, trouble = build(entry)
if recording is None:
dropped.append(f"{entry['label']} ({entry['browser']}): {'; '.join(trouble)}")
continue
if notes.get(entry["label"]):
recording["note"] = notes[entry["label"]]
recordings.append(recording)
offset = (
recording["centre"]["x"] - entry["window"]["w"] / 2,
recording["centre"]["y"] - entry["window"]["h"] / 2,
)
flag = f" ⚠ {'; '.join(trouble)}" if trouble else ""
print(
f"{entry['label']:<40} {entry['browser']:<9} "
f"{entry['window']['w']}×{entry['window']['h']}@{entry.get('pageZoom', 1)} "
f"camera {offset[0]:+7.1f}, {offset[1]:+6.1f} px from the middle "
f"spread {recording['spread']:.2f}{flag}"
)
out = {
"what": raw.get("what", old.get("what", "")),
"why": old.get(
"why",
"The map tools draw on someone else's map from what its URL says. This is the "
"record of what those URLs actually mean.",
),
"centre": old.get(
"centre",
"Where each site drew the coordinate its address bar named, in window pixels, "
"solved from its own two zooms.",
),
"how": "Rebuilt by scripts/build_map_fixture.py from tests/fixtures/map-sites.raw.json.",
# The offset the app ships as each site's starting answer, carried here
# so both suites read one copy of it: `tests/test_mapsites.py` checks it
# against the recordings above, and the extension's engine test checks
# that a fresh install draws from it (`extensionMapEngine.test.js`).
"hints": {
site: {"x": at[0], "y": at[1], "min_w": at[2]}
for site, at in _CAMERA_CENTRE.items()
},
"recordings": recordings,
}
if "rounding" in raw:
block, trouble = build_rounding(raw["rounding"], old.get("rounding", {}))
if block:
out["rounding"] = block
print(
f"\n{block['site']} rounding run: camera "
f"{block['centre']['x'] - block['window']['w'] / 2:+.1f}, "
f"{block['centre']['y'] - block['window']['h'] / 2:+.1f} px from the middle"
)
if trouble:
dropped.append(f"the rounding run: {'; '.join(trouble)}")
OUT.write_text(json.dumps(out, indent=1) + "\n", encoding="utf-8")
print(f"\n{len(recordings)} recordings → {OUT.relative_to(ROOT)}")
if dropped:
print("left out, and why:")
for line in dropped:
print(f" {line}")
return 1 if dropped else 0
if __name__ == "__main__":
raise SystemExit(main())