https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
fix(render): two silent hangs in the turbo capture path (#132)
Zhifei Lic1dae49ba78e
diff --git a/render/src/pixelrag_render/backends/fast_cdp.py b/render/src/pixelrag_render/backends/fast_cdp.pyindex 844b2b6..d2e2f2c 100644--- a/render/src/pixelrag_render/backends/fast_cdp.py+++ b/render/src/pixelrag_render/backends/fast_cdp.py@@ -99,6 +99,42 @@ _WAIT_FONTS_IMGS = (# ---------------------------------------------------------------------------+def _pin_to_cores(cores) -> None:+ """Pool initializer: keep compression workers off the capture cores.++ Module level, not a closure, because the pool is started with "spawn" — the+ initializer has to survive pickling, which a nested function does not.+ """+ try:+ os.sched_setaffinity(0, cores)+ except (OSError, AttributeError):+ pass # not Linux, or the affinity call is unavailable — harmless+++def _raw_scratch_dir() -> Path:+ """Per-user scratch directory on /dev/shm for Chrome's raw BGRA dumps.++ `/dev/shm` is shared by every user on the box, so a fixed path belongs to+ whoever ran first: the directory lands at 0775 owned by them, and for every+ later user `mkdir(exist_ok=True)` still succeeds while Chrome's write into it+ is denied. That combination is silent — capture reports success, the manifest+ claims N tiles, and not one image is written. Scoping the path to the current+ uid keeps users out of each other's way.++ The writability check is here rather than at first write because the failure+ it catches surfaces inside Chrome, whose stderr this backend discards.+ """+ d = Path(f"/dev/shm/pixelrag_render-{os.getuid()}/raw")+ d.mkdir(parents=True, exist_ok=True)+ if not os.access(d, os.W_OK):+ raise RuntimeError(+ f"{d} is not writable by uid {os.getuid()}. Chrome writes raw tiles "+ "there; without write access every capture is silently lost. Remove "+ "the directory and re-run, or set the turbo path aside with turbo=False."+ )+ return d++def compress_tile(raw_path: str, out_path: str, quality: int = 85) -> None:"""Read raw BGRA file, compress to JPEG, delete raw file.@@ -285,8 +321,7 @@ async def _run_render(jpeg_quality: int,n_compressors: int,) -> dict:- raw_dir = Path("/dev/shm/pixelrag_render/raw")- raw_dir.mkdir(parents=True, exist_ok=True)+ raw_dir = _raw_scratch_dir()output_dir.mkdir(parents=True, exist_ok=True)# Semaphore: limit concurrent captures (CPU-bound) to n_workers // 2@@ -296,9 +331,9 @@ async def _run_render(# Compression: dedicated thread with its own multiprocessing.Pool.# Tiles are pushed to a thread-safe queue from capture workers.# The thread runs pool.starmap in batches, fully independent of asyncio.+ import multiprocessingimport queue as _queueimport threading- from multiprocessing import Pool as MPPoolmetrics = {"total_tiles": 0,@@ -312,15 +347,20 @@ async def _run_render(n_cpus = os.cpu_count() or 128compress_cores = set(range(max(0, n_cpus - n_compressors), n_cpus))- def _pool_init():- try:- os.sched_setaffinity(0, compress_cores)- except OSError:- pass-def _compressor_thread():- pool = MPPool(processes=n_compressors, initializer=_pool_init)- # Warm up: ensure all workers are forked and idle before capture starts+ # "spawn", not the platform default. This pool is created from a thread while+ # the capture side already runs its own threads, and forking a multi-threaded+ # process can deadlock the child on a lock that was held at fork time — CPython+ # warns about exactly this since 3.12. In a single-threaded caller the fork+ # happens to work, which is why it survived so long; under any threaded host+ # (a test session that ran async tests first, a web server calling into the+ # renderer) it hangs with no output and no error.+ pool = multiprocessing.get_context("spawn").Pool(+ processes=n_compressors,+ initializer=_pin_to_cores,+ initargs=(compress_cores,),+ )+ # Warm up: ensure all workers are up and idle before capture startspool.map(int, range(n_compressors))async_results = []while True:@@ -328,12 +368,24 @@ async def _run_render(if item is None:breakasync_results.append(pool.apply_async(compress_tile, item))- # Wait for all remaining+ # Wait for all remaining. A compression failure means that tile never+ # reached disk, so it has to be counted and logged: swallowing it let a+ # run report "Done: 13 tiles" with an empty output directory.+ failed = 0for ar in async_results:try:ar.get(timeout=60)- except Exception:- pass+ except Exception as e:+ failed += 1+ if failed <= 3: # one line per failure is enough to diagnose+ logger.error("tile compression failed: %s: %s", type(e).__name__, e)+ if failed:+ logger.error(+ "%d/%d tiles failed to compress and were NOT written to disk",+ failed,+ len(async_results),+ )+ metrics["errors"] += failedpool.close()pool.join()compress_done.set()