Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: deterministic json format (#3706)
tomsideguideef56b5ce4205
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 6ef5b5c73..1f7ac87dc 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -123,7 +123,8 @@"ioredis": "^5.6.1","ipaddr.js": "^2.2.0","joplin-turndown-plugin-gfm": "^1.0.12",- "jsdom": "^26.0.0",+ "jsdom": "^29.1.1",+ "jsonrepair": "^3.14.0","koffi": "^2.9.0","langsmith": "^0.6.0","lodash": "^4.18.0",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex d8ded7e20..376878126 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -180,8 +180,11 @@ importers:specifier: ^1.0.12version: 1.0.12jsdom:- specifier: ^26.0.0- version: 26.0.0([email protected])([email protected])+ specifier: ^29.1.1+ version: 29.1.1(@noble/[email protected])+ jsonrepair:+ specifier: ^3.14.0+ version: 3.14.0koffi:specifier: ^2.9.0version: 2.9.0@@ -544,8 +547,20 @@ packages:'@apm-js-collab/[email protected]':resolution: {integrity: sha512-Vu1CbmPURlN5fTboVuKMoJjbO5qcq9fA5YXpskx3dXe/zTBvjODFoerw+69rVBlRLrJpwPqSDqEuJDEKIrTldw==}- '@asamuzakjp/[email protected]':- resolution: {integrity: sha512-GIc76d9UI1hCvOATjZPyHFmE5qhRccp3/zGfMPapK3jBi+yocEzp6BBB0UnfRYP9NP4FANqUZYb0hnfs3TM3hw==}+ '@asamuzakjp/[email protected]':+ resolution: {integrity: sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg==}+ engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}++ '@asamuzakjp/[email protected]':+ resolution: {integrity: sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ==}+ engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}++ '@asamuzakjp/[email protected]':+ resolution: {integrity: sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg==}+ engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}++ '@asamuzakjp/[email protected]':+ resolution: {integrity: sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==}'@babel/[email protected]':resolution: {integrity: sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==}@@ -716,6 +731,10 @@ packages:'@bcoe/[email protected]':resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==}+ '@bramus/[email protected]':+ resolution: {integrity: sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==}+ hasBin: true+'@bull-board/[email protected]':resolution: {integrity: sha512-UzkvN/wM+1qS73BS43a75LYkRzpBpCCUKlaGq0hp3dM5MNmdF1mx7LMGYgXPt91gqF8j4jq9Y/zCpC3Sqs3RLQ==}peerDependencies:@@ -745,33 +764,41 @@ packages:resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==}engines: {node: '>=12'}- '@csstools/[email protected]':- resolution: {integrity: sha512-MKtmkA0BX87PKaO1NFRTFH+UnkgnmySQOvNxJubsadusqPEC2aJ9MOQiMceZJJ6oitUl/i0L6u0M1IrmAOmgBA==}- engines: {node: '>=18'}+ '@csstools/[email protected]':+ resolution: {integrity: sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q==}+ engines: {node: '>=20.19.0'}- '@csstools/[email protected]':- resolution: {integrity: sha512-rL7kaUnTkL9K+Cvo2pnCieqNpTKgQzy5f+N+5Iuko9HAoasP+xgprVh7KN/MaJVvVL1l0EzQq2MoqBHKSrDrag==}- engines: {node: '>=18'}+ '@csstools/[email protected]':+ resolution: {integrity: sha512-DtdHlgXh5ZkA43cwBcAm+huzgJiwx3ZTWVjBs94kwz2xKqSimDA3lBgCjphYgwgVUMWatSM0pDd8TILB1yrVVg==}+ engines: {node: '>=20.19.0'}peerDependencies:- '@csstools/css-parser-algorithms': ^3.0.4- '@csstools/css-tokenizer': ^3.0.3+ '@csstools/css-parser-algorithms': ^4.0.0+ '@csstools/css-tokenizer': ^4.0.0- '@csstools/[email protected]':- resolution: {integrity: sha512-nkMp2mTICw32uE5NN+EsJ4f5N+IGFeCFu4bGpiKgb2Pq/7J/MpyLBeQ5ry4KKtRFZaYs6sTmcMYrSRIyj5DFKA==}- engines: {node: '>=18'}+ '@csstools/[email protected]':+ resolution: {integrity: sha512-eZ5XOtyhK+mggRafYUWzA0tvaYOFgdY8AkgQiCJF9qNAePnUo/zmsqqYubBBb3sQ8uNUaSKTY9s9klfRaAXL0g==}+ engines: {node: '>=20.19.0'}peerDependencies:- '@csstools/css-parser-algorithms': ^3.0.4- '@csstools/css-tokenizer': ^3.0.3+ '@csstools/css-parser-algorithms': ^4.0.0+ '@csstools/css-tokenizer': ^4.0.0- '@csstools/[email protected]':- resolution: {integrity: sha512-Up7rBoV77rv29d3uKHUIVubz1BTcgyUK72IvCQAbfbMv584xHcGKCKbWh7i8hPrRJ7qU4Y8IO3IY9m+iTB7P3A==}- engines: {node: '>=18'}+ '@csstools/[email protected]':+ resolution: {integrity: sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w==}+ engines: {node: '>=20.19.0'}peerDependencies:- '@csstools/css-tokenizer': ^3.0.3+ '@csstools/css-tokenizer': ^4.0.0- '@csstools/[email protected]':- resolution: {integrity: sha512-UJnjoFsmxfKUdNYdWgOB0mWUypuLvAfQPH1+pyvRJs6euowbFkFC6P13w1l8mJyi3vxYMxc9kld5jZEGRQs6bw==}- engines: {node: '>=18'}+ '@csstools/[email protected]':+ resolution: {integrity: sha512-wgsqt92b7C7tQhIdPNxj0n9zuUbQlvAuI1exyzeNrOKOi62SD7ren8zqszmpVREjAOqg8cD2FqYhQfAuKjk4sw==}+ peerDependencies:+ css-tree: ^3.2.1+ peerDependenciesMeta:+ css-tree:+ optional: true++ '@csstools/[email protected]':+ resolution: {integrity: sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==}+ engines: {node: '>=20.19.0'}'@dabh/[email protected]':resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==}@@ -1100,6 +1127,15 @@ packages:cpu: [x64]os: [win32]+ '@exodus/[email protected]':+ resolution: {integrity: sha512-S6mL0yNB/Abt9Ei4tq8gDhcczc4S3+vQ4ra7vxnAf+YHC02srtqxKKZghx2Dq6p0e66THKwR6r8N6P95wEty7Q==}+ engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}+ peerDependencies:+ '@noble/hashes': ^1.8.0 || ^2.0.0+ peerDependenciesMeta:+ '@noble/hashes':+ optional: true+'@google-cloud/[email protected]':resolution: {integrity: sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==}engines: {node: '>=14.0.0'}@@ -2803,6 +2839,7 @@ packages:'@ungap/[email protected]':resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==}+ deprecated: Potential CWE-502 - Update to 1.3.1 or higher'@unrs/[email protected]':resolution: {integrity: sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==}@@ -3164,6 +3201,9 @@ packages:resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==}+ resolution: {integrity: sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==}+resolution: {integrity: sha512-JocpCSOixzy5XFJi2ub6IMmV/G9i8Lrm2lZvwBv9xPdglmZM0ufDVBbjbrfU/zuLvBfD7Bv2eYxz9i+OHTgkew==}deprecated: pkg version number incorrect@@ -3437,14 +3477,14 @@ packages:resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==}+ resolution: {integrity: sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==}+ engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0}+resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==}engines: {node: '>= 6'}- resolution: {integrity: sha512-9+vem03dMXG7gDmZ62uqmRiMRNtinIZ9ZyuF6BdxzfOD+FdN5hretzynkn0ReS2DO2GSw76RWHs0UmJPI2zUjw==}- engines: {node: '>=18'}-resolution: {integrity: sha512-1+BhOB8ahCn4O0cep0Sh2l9KCOfOdY+BXJnKMHFFzDEouSr/el18QwXEMRlOj9UY5nCeA8UN3a/82rUWRBeyBw==}engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}@@ -3453,9 +3493,9 @@ packages:resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}engines: {node: '>= 12'}- resolution: {integrity: sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==}- engines: {node: '>=18'}+ resolution: {integrity: sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==}+ engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==}@@ -3482,8 +3522,8 @@ packages:supports-color:optional: true- resolution: {integrity: sha512-8vDa8Qxvr/+d94hSh5P3IJwI5t8/c0KsMp+g8bNw9cY2icONa5aPfvKeieW1WlG0WQYwwhJ7mjui2xtiePQSXw==}+ resolution: {integrity: sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==}resolution: {integrity: sha512-+8VxcR21HhTy8nOt6jf20w0c9CADrw1O8d+VZ/YzzCt4bJ3uBjw+D1q2osAB8RnpwwaeYBxy0HyKQxD5JBMuuQ==}@@ -3741,6 +3781,10 @@ packages:resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==}engines: {node: '>=0.12'}+ resolution: {integrity: sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==}+ engines: {node: '>=20.19.0'}+resolution: {integrity: sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==}engines: {node: '>=18'}@@ -4125,9 +4169,9 @@ packages:resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==}hasBin: true- resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==}- engines: {node: '>=18'}+ resolution: {integrity: sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==}+ engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}resolution: {integrity: sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==}@@ -4160,10 +4204,6 @@ packages:resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==}engines: {node: '>= 6'}- resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==}- engines: {node: '>= 14'}-resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}engines: {node: '>= 6'}@@ -4496,9 +4536,9 @@ packages:resolution: {integrity: sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==}- resolution: {integrity: sha512-BZYDGVAIriBWTpIxYzrXjv3E/4u8+/pSG5bQdIYCbNCGOvsPkDQfTVLAIXAf9ETdCpduCVTkDe2NNZ8NIwUVzw==}- engines: {node: '>=18'}+ resolution: {integrity: sha512-ECi4Fi2f7BdJtUKTflYRTiaMxIB0O6zfR1fX0GXpUrf6flp8QIYn1UT20YQqdSOfk2dfkCwS8LAFoJDEppNK5Q==}+ engines: {node: ^20.19.0 || ^22.13.0 || >=24.0.0}peerDependencies:canvas: ^3.0.0peerDependenciesMeta:@@ -4534,6 +4574,10 @@ packages:engines: {node: '>=6'}hasBin: true+ resolution: {integrity: sha512-tWPGKMZf/8UPim+fcW2EfcQ/d/7aKUrP6IECz9G3Tu6Q5dX0orSleqJ9z6sSw7qrQkjF8/Edo4DvsWBZ8H+HNg==}+ hasBin: true+resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==}@@ -4651,13 +4695,14 @@ packages:resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==}hasBin: true- resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==}-resolution: {integrity: sha512-vFrFJkWtJvJnD5hg+hJvVE8Lh/TcMzKnTgCWmtBipwI5yLX/iX+5UB2tfuyODF5E7k9xEzMdYgGqaSb1c0c5Yw==}engines: {node: 20 || >=22}+ resolution: {integrity: sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A==}+ engines: {node: 20 || >=22}+resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==}@@ -4690,6 +4735,9 @@ packages:resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==}+ resolution: {integrity: sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==}+resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==}engines: {node: '>= 0.6'}@@ -4880,9 +4928,6 @@ packages:resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==}- resolution: {integrity: sha512-F1I/bimDpj3ncaNDhfyMWuFqmQDBwDB0Fogc2qpL3BWvkQteFD/8BzWuIRl83rq0DXfm8SGt/HFhLXZyljTXcQ==}-resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==}engines: {node: '>=0.10.0'}@@ -4996,8 +5041,8 @@ packages:resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==}- resolution: {integrity: sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==}+ resolution: {integrity: sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==}resolution: {integrity: sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==}@@ -5313,9 +5358,6 @@ packages:resolution: {integrity: sha512-I+kNjW0udB4Fetr3vvtRuYZJS0PcSPyyvBcH5sDdoV8DFs5E4W2pTr7aiMlKfPxANTClP9RlqCPolj9dd5MsEA==}- resolution: {integrity: sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==}-resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==}@@ -5438,6 +5480,10 @@ packages:resolution: {integrity: sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==}engines: {node: '>= 18'}+ resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==}+ engines: {node: '>=0.10.0'}+resolution: {integrity: sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==}@@ -5611,10 +5657,17 @@ packages:resolution: {integrity: sha512-AOvV5YYIAFFBfransBzSTyztkc3IMfz5Eq3YluaRiEu55nn43Fzaufx70UqEKYr8BoLCach4q8g/bg6e5+/aFw==}+ resolution: {integrity: sha512-nwEyF4vl4RSJjwSjBUmOSxc3BFPoIFdlRthJ6e+5v9P3bHNsoD06UjuqMUspqp7vsEZ1beaHi1km+optiE17yA==}+resolution: {integrity: sha512-+lFzEXhpl7JXgWYaXcB6DqTYXbUArvrWAE/5ioq/X3CdWLbDjpPP4XTrQBmEJ91y3xbe4Fkw7Lxv4P3GWeJaNg==}hasBin: true+ resolution: {integrity: sha512-kCwffuaH8ntKtygnWe1b4BJKWiCUH30n5KfoTr6IchcXOwR7chAOFJxFrH3vjANafUYrIA4a7SDL+nn7SiR4Sw==}+ hasBin: true+resolution: {integrity: sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==}@@ -5630,16 +5683,16 @@ packages:resolution: {integrity: sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==}engines: {node: '>=6'}- resolution: {integrity: sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==}+ resolution: {integrity: sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==}engines: {node: '>=16'}resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==}- resolution: {integrity: sha512-tk2G5R2KRwBd+ZN0zaEXpmzdKyOYksXwywulIX95MBODjSzMIuQnQ3m8JxgbhnL1LeVo7lqQKsYa1O3Htl7K5g==}- engines: {node: '>=18'}+ resolution: {integrity: sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==}+ engines: {node: '>=20'}resolution: {integrity: sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==}@@ -5851,22 +5904,17 @@ packages:resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==}- resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==}- engines: {node: '>=12'}-- resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==}- engines: {node: '>=18'}- deprecated: Use @exodus/bytes instead for a more spec-conformant and faster implementation+ resolution: {integrity: sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==}+ engines: {node: '>=20'}- resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==}- engines: {node: '>=18'}+ resolution: {integrity: sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==}+ engines: {node: '>=20'}- resolution: {integrity: sha512-mDGf9diDad/giZ/Sm9Xi2YcyzaFpbdLpJPr+E9fSkyQ7KpQD4SdFcugkRQYzhmfI4KeV4Qpnn2sKPdo+kmsgRQ==}- engines: {node: '>=18'}+ resolution: {integrity: sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==}+ engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==}@@ -6180,13 +6228,25 @@ snapshots:transitivePeerDependencies:- supports-color- '@asamuzakjp/[email protected]':+ '@asamuzakjp/[email protected]':dependencies:- '@csstools/css-calc': 2.1.1(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])- '@csstools/css-color-parser': 3.0.7(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])- '@csstools/css-parser-algorithms': 3.0.4(@csstools/[email protected])- '@csstools/css-tokenizer': 3.0.3- lru-cache: 10.4.3+ '@asamuzakjp/generational-cache': 1.0.1+ '@csstools/css-calc': 3.2.1(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])+ '@csstools/css-color-parser': 4.1.1(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])+ '@csstools/css-parser-algorithms': 4.0.0(@csstools/[email protected])+ '@csstools/css-tokenizer': 4.0.0++ '@asamuzakjp/[email protected]':+ dependencies:+ '@asamuzakjp/generational-cache': 1.0.1+ '@asamuzakjp/nwsapi': 2.3.9+ bidi-js: 1.0.3+ css-tree: 3.2.1+ is-potential-custom-element-name: 1.0.1++ '@asamuzakjp/[email protected]': {}++ '@asamuzakjp/[email protected]': {}'@babel/[email protected]':dependencies:@@ -6379,6 +6439,10 @@ snapshots:'@bcoe/[email protected]': {}+ '@bramus/[email protected]':+ dependencies:+ css-tree: 3.2.1+'@bull-board/[email protected](@bull-board/[email protected])':dependencies:'@bull-board/ui': 6.14.2@@ -6430,25 +6494,29 @@ snapshots:'@jridgewell/trace-mapping': 0.3.9optional: true- '@csstools/[email protected]': {}+ '@csstools/[email protected]': {}- '@csstools/[email protected](@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])':+ '@csstools/[email protected](@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])':dependencies:- '@csstools/css-parser-algorithms': 3.0.4(@csstools/[email protected])- '@csstools/css-tokenizer': 3.0.3+ '@csstools/css-parser-algorithms': 4.0.0(@csstools/[email protected])+ '@csstools/css-tokenizer': 4.0.0- '@csstools/[email protected](@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])':+ '@csstools/[email protected](@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])':dependencies:- '@csstools/color-helpers': 5.0.1- '@csstools/css-calc': 2.1.1(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])- '@csstools/css-parser-algorithms': 3.0.4(@csstools/[email protected])- '@csstools/css-tokenizer': 3.0.3+ '@csstools/color-helpers': 6.0.2+ '@csstools/css-calc': 3.2.1(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])+ '@csstools/css-parser-algorithms': 4.0.0(@csstools/[email protected])+ '@csstools/css-tokenizer': 4.0.0- '@csstools/[email protected](@csstools/[email protected])':+ '@csstools/[email protected](@csstools/[email protected])':dependencies:- '@csstools/css-tokenizer': 3.0.3+ '@csstools/css-tokenizer': 4.0.0- '@csstools/[email protected]': {}+ '@csstools/[email protected]([email protected])':+ optionalDependencies:+ css-tree: 3.2.1++ '@csstools/[email protected]': {}'@dabh/[email protected]':dependencies:@@ -6629,6 +6697,10 @@ snapshots:'@esbuild/[email protected]':optional: true+ '@exodus/[email protected](@noble/[email protected])':+ optionalDependencies:+ '@noble/hashes': 1.8.0+'@google-cloud/[email protected]':dependencies:arrify: 2.0.1@@ -8914,6 +8986,10 @@ snapshots:+ dependencies:+ require-from-string: 2.0.2+@@ -9203,21 +9279,23 @@ snapshots:domutils: 3.1.0nth-check: 2.1.1- [email protected]: {}-dependencies:- '@asamuzakjp/css-color': 2.8.3- rrweb-cssom: 0.8.0+ mdn-data: 2.27.1+ source-map-js: 1.2.1++ [email protected]: {}+ [email protected](@noble/[email protected]):dependencies:- whatwg-mimetype: 4.0.0- whatwg-url: 14.1.1+ whatwg-mimetype: 5.0.0+ whatwg-url: 16.0.1(@noble/[email protected])+ transitivePeerDependencies:+ - '@noble/hashes'dependencies:@@ -9231,7 +9309,7 @@ snapshots:dependencies:ms: 2.1.3- [email protected]: {}+ [email protected]: {}@@ -9351,6 +9429,8 @@ snapshots:+ [email protected]: {}+@@ -9876,9 +9956,11 @@ snapshots:+ [email protected](@noble/[email protected]):dependencies:- whatwg-encoding: 3.1.1+ '@exodus/bytes': 1.15.1(@noble/[email protected])+ transitivePeerDependencies:+ - '@noble/hashes'@@ -9922,13 +10004,6 @@ snapshots:transitivePeerDependencies:- supports-color- dependencies:- agent-base: 7.1.3- debug: 4.4.3- transitivePeerDependencies:- - supports-color-dependencies:agent-base: 6.0.2@@ -10492,33 +10567,31 @@ snapshots:- dependencies:- cssstyle: 4.2.1- data-urls: 5.0.0- decimal.js: 10.5.0- form-data: 4.0.4- html-encoding-sniffer: 4.0.0- http-proxy-agent: 7.0.2- https-proxy-agent: 7.0.6+ [email protected](@noble/[email protected]):+ dependencies:+ '@asamuzakjp/css-color': 5.1.11+ '@asamuzakjp/dom-selector': 7.1.1+ '@bramus/specificity': 2.4.2+ '@csstools/css-syntax-patches-for-csstree': 1.1.4([email protected])+ '@exodus/bytes': 1.15.1(@noble/[email protected])+ css-tree: 3.2.1+ data-urls: 7.0.0(@noble/[email protected])+ decimal.js: 10.6.0+ html-encoding-sniffer: 6.0.0(@noble/[email protected])is-potential-custom-element-name: 1.0.1- nwsapi: 2.2.16- parse5: 7.2.1- rrweb-cssom: 0.8.0+ lru-cache: 11.5.1+ parse5: 8.0.1saxes: 6.0.0symbol-tree: 3.2.4- tough-cookie: 5.1.2+ tough-cookie: 6.0.1+ undici: 7.24.1w3c-xmlserializer: 5.0.0- webidl-conversions: 7.0.0- whatwg-encoding: 3.1.1- whatwg-mimetype: 4.0.0- whatwg-url: 14.1.1- ws: 8.20.1([email protected])([email protected])+ webidl-conversions: 8.0.1+ whatwg-mimetype: 5.0.0+ whatwg-url: 16.0.1(@noble/[email protected])xml-name-validator: 5.0.0transitivePeerDependencies:- - bufferutil- - supports-color- - utf-8-validate+ - '@noble/hashes'@@ -10538,6 +10611,8 @@ snapshots:+ [email protected]: {}+dependencies:buffer-equal-constant-time: 1.0.1@@ -10671,10 +10746,10 @@ snapshots:dependencies:js-tokens: 4.0.0- [email protected]: {}-+ [email protected]: {}+dependencies:yallist: 3.1.1@@ -10703,6 +10778,8 @@ snapshots:crypt: 0.0.2is-buffer: 1.1.6+ [email protected]: {}+@@ -10851,8 +10928,6 @@ snapshots:dependencies:boolbase: 1.0.0- [email protected]: {}-@@ -10983,9 +11058,9 @@ snapshots:dependencies:entities: 4.5.0dependencies:- entities: 4.5.0+ entities: 8.0.0dependencies:@@ -11290,8 +11365,6 @@ snapshots:bufferutil: 4.0.9utf-8-validate: 5.0.10- [email protected]: {}-dependencies:queue-microtask: 1.2.3@@ -11427,6 +11500,8 @@ snapshots:+ [email protected]: {}+dependencies:buffer-from: 1.1.2@@ -11595,10 +11670,16 @@ snapshots:+ [email protected]: {}+dependencies:tldts-core: 6.1.75+ dependencies:+ tldts-core: 7.4.2+@@ -11614,13 +11695,13 @@ snapshots:universalify: 0.2.0url-parse: 1.5.10dependencies:- tldts: 6.1.75+ tldts: 7.4.2dependencies:punycode: 2.3.1@@ -11827,18 +11908,17 @@ snapshots:- [email protected]: {}-- dependencies:- iconv-lite: 0.6.3+ [email protected]: {}- [email protected]: {}+ [email protected]: {}+ [email protected](@noble/[email protected]):dependencies:- tr46: 5.0.0- webidl-conversions: 7.0.0+ '@exodus/bytes': 1.15.1(@noble/[email protected])+ tr46: 6.0.0+ webidl-conversions: 8.0.1+ transitivePeerDependencies:+ - '@noble/hashes'dependencies:diff --git a/apps/api/src/__tests__/snips/v2/monitor.test.ts b/apps/api/src/__tests__/snips/v2/monitor.test.tsindex 55d99a81c..1bce82397 100644--- a/apps/api/src/__tests__/snips/v2/monitor.test.ts+++ b/apps/api/src/__tests__/snips/v2/monitor.test.ts@@ -252,6 +252,70 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)("/v2/monitor", () => {2 * scrapeTimeout,);+ it(+ "runs a deterministicJson-mode monitor target",+ async () => {+ const jsonSchema = {+ type: "object",+ additionalProperties: false,+ required: ["title"],+ properties: {+ title: {+ type: "string",+ description: "The page title, verbatim.",+ },+ },+ };++ const create = await monitorCreateRaw(+ {+ name: "deterministic-json monitor",+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },+ targets: [+ {+ type: "scrape",+ urls: [createTestIdUrl()],+ scrapeOptions: {+ formats: [+ "markdown",+ {+ type: "deterministicJson",+ prompt: "Extract the page title verbatim.",+ schema: jsonSchema,+ },+ ],+ },+ },+ ],+ },+ identity,+ );+ expect(create.statusCode).toBe(200);++ const monitorId = create.body.data.id;+ const run = await monitorRunRaw(monitorId, identity);+ expect(run.statusCode).toBe(200);+ const checkId = run.body.id;++ let check: any;+ for (let i = 0; i < 90; i++) {+ const raw = await monitorCheckRaw(monitorId, checkId, identity);+ expect(raw.statusCode).toBe(200);+ check = raw.body.data;+ if (["completed", "partial", "failed"].includes(check.status)) break;+ await new Promise(resolve => setTimeout(resolve, 1000));+ }+ // Contract: deterministicJson is accepted, the reusable-json-mode extractor+ // populates document.json via the code-sandbox, and the check flows through+ // the JSON diff path (status "new" on first run) rather than crashing.+ expect(["completed", "partial"]).toContain(check.status);+ expect(check.pages[0].status).toBe("new");++ await monitorDeleteRaw(monitorId, identity);+ },+ 2 * scrapeTimeout,+ );+it("accepts changeTracking-json format on a monitor target",async () => {diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 850973be0..3b6132985 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -286,6 +286,12 @@ const configSchema = z.object({NUQ_PREFETCH_WORKER_HEARTBEAT_URL: z.string().optional(),ZDRCLEANER_HEARTBEAT_URL: z.string().optional(),++ // Deterministic JSON extraction (reusable-json-mode)+ EXTRACT_CODEGEN_MODEL: z.string().default("gemini-3.1-flash-lite"),+ EXTRACT_ANCHOR_MODEL: z.string().default("openai/gpt-oss-120b"),+ EXTRACT_LIGHT_MODEL: z.string().default("openai/gpt-oss-20b"),+ CODE_SANDBOX_URL: z.string().default("ws://code-sandbox:3001"),});export const config = configSchema.parse(process.env);diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 486ab237e..c65684a82 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -350,6 +350,25 @@ const jsonFormatWithOptions = z.strictObject({export type JsonFormatWithOptions = z.output<typeof jsonFormatWithOptions>;+// "Deterministic JSON" — same shape as json mode, but extraction is performed by+// reusable-json-mode (a cached, reusable JS extractor run in the code-sandbox)+// rather than a per-request LLM call. Populates document.json like json mode.+const deterministicJsonFormatWithOptions = z.strictObject({+ type: z.literal("deterministicJson"),+ schema: z+ .any()+ .optional()+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),+ prompt: z.string().max(10000).optional(),+});++type DeterministicJsonFormatWithOptions = z.output<+ typeof deterministicJsonFormatWithOptions+>;+const changeTrackingFormatWithOptions = z.strictObject({type: z.literal("changeTracking"),prompt: z.string().optional(),@@ -431,6 +450,7 @@ export type FormatObject =| { type: "images" }| { type: "summary" }| JsonFormatWithOptions+ | DeterministicJsonFormatWithOptions| ChangeTrackingFormatWithOptions| ScreenshotFormatWithOptions| AttributesFormatWithOptions@@ -605,6 +625,7 @@ const baseScrapeOptions = z.strictObject({z.strictObject({ type: z.literal("images") }),z.strictObject({ type: z.literal("summary") }),jsonFormatWithOptions,+ deterministicJsonFormatWithOptions,changeTrackingFormatWithOptions,screenshotFormatWithOptions,attributesFormatWithOptions,@@ -627,7 +648,14 @@ const baseScrapeOptions = z.strictObject({const hasChangeTracking = x.find(f => f.type === "changeTracking");const hasMarkdown = x.find(f => f.type === "markdown");return !hasChangeTracking || hasMarkdown;- }, "The changeTracking format requires the markdown format to be specified as well"),+ }, "The changeTracking format requires the markdown format to be specified as well")+ .refine(x => {+ // Both json and deterministicJson populate the `json` field, so one would+ // just clobber the other. Require choosing one.+ const hasJson = x.some(f => f.type === "json");+ const hasDeterministicJson = x.some(f => f.type === "deterministicJson");+ return !(hasJson && hasDeterministicJson);+ }, "Cannot specify both json and deterministicJson formats"),headers: z.record(z.string(), z.string()).optional(),includeTags: z.string()diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex d22263724..8dc81b804 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -152,6 +152,32 @@ export const deep_researches = pgTable("deep_researches", {options: jsonb("options"),});+export const deterministic_json_scripts = pgTable(+ "deterministic_json_scripts",+ {+ id: uuid("id").defaultRandom().primaryKey(),+ cache_key: text("cache_key").notNull().unique(),+ code: text("code").notNull(),+ url: text("url"),+ model: text("model"),+ cache_version: integer("cache_version"),+ created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at").notNull().defaultNow(),+ last_used_at: ts("last_used_at").notNull().defaultNow(),+ },+);++export const deterministic_json_llm_cache = pgTable(+ "deterministic_json_llm_cache",+ {+ id: uuid("id").defaultRandom().primaryKey(),+ cache_key: text("cache_key").notNull().unique(),+ response: text("response").notNull(),+ created_at: ts("created_at").notNull().defaultNow(),+ last_used_at: ts("last_used_at").notNull().defaultNow(),+ },+);+export const eb_sync = pgTable("eb-sync", {id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),created_at: ts("created_at").notNull().defaultNow(),diff --git a/apps/api/src/lib/deterministicJson/config.ts b/apps/api/src/lib/deterministicJson/config.tsnew file mode 100644index 000000000..e365a2e7f--- /dev/null+++ b/apps/api/src/lib/deterministicJson/config.ts@@ -0,0 +1,25 @@+import { config } from "../../config";++export const CODEGEN_MODEL = config.EXTRACT_CODEGEN_MODEL;+export const ANCHOR_MODEL = config.EXTRACT_ANCHOR_MODEL;+export const LIGHT_MODEL = config.EXTRACT_LIGHT_MODEL;+export const CODE_SANDBOX_URL = config.CODE_SANDBOX_URL;++// Bump to invalidate every cached extractor at once.+export const CACHE_VERSION = 1;++export const MARKDOWN_BUDGET = 50_000;+export const HTML_BUDGET = 40_000;++export const ANCHOR_PER_BLOCK_BUDGET = 8_000;+export const ANCHOR_TOTAL_BUDGET = 60_000;+export const ANCHOR_MAX_BLOCKS = 16;+export const ANCHOR_MAX_PARENTS = 8;++export const CODEGEN_MAX_TOKENS = 16_384;+export const ANCHOR_PICKER_MAX_TOKENS = 2_000;+export const ASK_LLM_MAX_TOKENS = 2_048;++export const ASK_LLM_MAX_CALLS = 50;++export const SANDBOX_TIMEOUT_MS = 120_000;diff --git a/apps/api/src/lib/deterministicJson/core/cache.ts b/apps/api/src/lib/deterministicJson/core/cache.tsnew file mode 100644index 000000000..604a04a2b--- /dev/null+++ b/apps/api/src/lib/deterministicJson/core/cache.ts@@ -0,0 +1,23 @@+interface ExtractorEntry {+ code: string;+ createdAt: number;+}++interface Entry {+ response: string;+ createdAt: number;+}++interface ExtractorMeta {+ url: string;+ model: string;+ cacheVersion: number;+}++export interface CacheBackend {+ getExtractor(key: string): Promise<ExtractorEntry | undefined>;+ setExtractor(key: string, code: string, meta: ExtractorMeta): Promise<void>;+ getLlm(key: string): Promise<Entry | undefined>;+ setLlm(key: string, response: string): Promise<void>;+ touch?(key: string): Promise<void>;+}diff --git a/apps/api/src/lib/deterministicJson/core/util.ts b/apps/api/src/lib/deterministicJson/core/util.tsnew file mode 100644index 000000000..491066be6--- /dev/null+++ b/apps/api/src/lib/deterministicJson/core/util.ts@@ -0,0 +1,17 @@+import { createHash } from "node:crypto";+import { logger } from "../../logger";++export const log = (...args: unknown[]): void => {+ logger.debug(+ args.map(a => (typeof a === "string" ? a : JSON.stringify(a))).join(" "),+ { module: "deterministic-json" },+ );+};++export function sha(input: string, chars = 32): string {+ return createHash("sha256").update(input).digest("hex").slice(0, chars);+}++export function errorMessage(err: unknown): string {+ return err instanceof Error ? err.message : String(err);+}diff --git a/apps/api/src/lib/deterministicJson/extract.ts b/apps/api/src/lib/deterministicJson/extract.tsnew file mode 100644index 000000000..12549c92f--- /dev/null+++ b/apps/api/src/lib/deterministicJson/extract.ts@@ -0,0 +1,136 @@+import { buildAnchorContext } from "./pipeline/anchor";+import type { CacheBackend } from "./core/cache";+import { CACHE_VERSION, CODEGEN_MODEL, MARKDOWN_BUDGET } from "./config";+import { CostTracking } from "../cost-tracking";+import { generateExtractor } from "./pipeline/generate";+import { type AskLlm, makeAskLlm } from "./llm/client";+import { parseWithSchema } from "./pipeline/postprocess";+import {+ runExtractorInSandbox,+ type SandboxRunner,+} from "./sandbox/runExtractor";+import { tooStrictFeedback, tooStrictSelectors } from "./html/selector-repair";+import { errorMessage, log, sha } from "./core/util";++interface ExtractArgs {+ url: string;+ prompt: string;+ jsonSchema: Record<string, unknown>;+ page: { html: string; markdown: string };+ cache: CacheBackend;+ sandbox: SandboxRunner;+ costTracking: CostTracking;+ maxAskLlmCalls?: number;+ forceRegenerate?: boolean;+}++export async function extractDeterministicJson(+ args: ExtractArgs,+): Promise<unknown> {+ const {+ url,+ prompt,+ forceRegenerate,+ cache,+ sandbox,+ jsonSchema,+ costTracking,+ } = args;++ const schemaJson = JSON.stringify(jsonSchema);+ const key = sha(+ [CACHE_VERSION, CODEGEN_MODEL, url, schemaJson, prompt].join("\0"),+ );+ const meta = { url, model: CODEGEN_MODEL, cacheVersion: CACHE_VERSION };++ const { html, markdown } = args.page;+ const markdownPreview = clipMiddle(markdown.trim(), MARKDOWN_BUDGET);+ const cached = forceRegenerate ? undefined : await cache.getExtractor(key);++ let anchorHtml: string | undefined;++ const generate = async (rejectionFeedback?: string): Promise<string> => {+ anchorHtml ??= await buildAnchorContext(+ {+ html,+ markdownPreview,+ schemaJson,+ prompt,+ },+ costTracking,+ );+ const code = await generateExtractor(+ {+ anchorHtml,+ markdownPreview,+ schemaJson,+ prompt,+ rejectionFeedback,+ },+ costTracking,+ );+ await cache.setExtractor(key, code, meta);+ return code;+ };++ const run = async (code: string): Promise<unknown> => {+ const askLlm: AskLlm = makeAskLlm(cache, costTracking, args.maxAskLlmCalls);+ const value = await runExtractorInSandbox({+ code,+ html,+ url,+ askLlm,+ sandbox,+ });+ return parseWithSchema(value, jsonSchema);+ };++ const runWithRepair = async (code: string): Promise<unknown> => {+ const value = await run(code);+ const broken = tooStrictSelectors(code, html);+ if (broken.length === 0) return value;++ log(+ `extractor has ${broken.length} too-strict selector(s); regenerating once`,+ );+ try {+ const repaired = await generate(tooStrictFeedback(broken));+ if (tooStrictSelectors(repaired, html).length < broken.length) {+ return await run(repaired);+ }+ } catch (err) {+ log(+ "selector repair failed, keeping original result:",+ errorMessage(err),+ );+ }+ await cache.setExtractor(key, code, meta);+ return value;+ };++ let code: string;+ if (cached) {+ await cache.touch?.(key);+ code = cached.code;+ } else {+ code = await generate();+ }++ try {+ return await runWithRepair(code);+ } catch (err) {+ log("extractor run failed, regenerating once:", errorMessage(err));+ // Let a second failure propagate; the caller warns. An empty shape here+ // would instead read as "page had no data" and hide the real failure.+ return await run(await generate(errorMessage(err)));+ }+}++function clipMiddle(text: string, max: number): string {+ if (text.length <= max) return text;+ const marker = `\n\n[...${text.length - max} chars trimmed...]\n\n`;+ const head = Math.ceil((max - marker.length) * 0.7);+ return (+ text.slice(0, head) + marker + text.slice(-(max - marker.length - head))+ );+}diff --git a/apps/api/src/lib/deterministicJson/html/dom.ts b/apps/api/src/lib/deterministicJson/html/dom.tsnew file mode 100644index 000000000..be4e3ba4b--- /dev/null+++ b/apps/api/src/lib/deterministicJson/html/dom.ts@@ -0,0 +1,9 @@+import { JSDOM } from "jsdom";++export function parseDocument(html: string): Document {+ return new JSDOM(html).window.document;+}++export const ELEMENT_NODE = 1;+export const TEXT_NODE = 3;+export const COMMENT_NODE = 8;diff --git a/apps/api/src/lib/deterministicJson/html/selector-repair.ts b/apps/api/src/lib/deterministicJson/html/selector-repair.tsnew file mode 100644index 000000000..caddefcd8--- /dev/null+++ b/apps/api/src/lib/deterministicJson/html/selector-repair.ts@@ -0,0 +1,97 @@+// Detect selectors that are too strict: they match 0 elements, yet a looser+// variant (child combinator `>` relaxed to descendant) matches at least one, so+// we can tell the model exactly what to fix. Only static string/template literals+// are considered.+import { parseDocument } from "./dom";++interface TooStrictSelector {+ selector: string;+ loosened: string;+ count: number;+}++// Pull literal selectors out of querySelector / querySelectorAll calls. Skips+// template literals containing ${...} interpolation (not statically knowable).+const SELECTOR_CALL = /querySelector(?:All)?\(\s*(['"`])((?:\\.|(?!\1).)*)\1/g;++function extractSelectorLiterals(code: string): string[] {+ const out = new Set<string>();+ for (const match of code.matchAll(SELECTOR_CALL)) {+ const selector = match[2];+ if (selector && !selector.includes("${")) out.add(selector);+ }+ return [...out];+}++// Relax child combinators to descendant, but not a `>` inside [attr] or quotes.+function loosenCombinators(selector: string): string {+ let out = "";+ let depth = 0;+ let quote = "";+ for (let i = 0; i < selector.length; i++) {+ const ch = selector[i]!;+ if (quote) {+ out += ch;+ if (ch === quote && selector[i - 1] !== "\\") quote = "";+ } else if (ch === '"' || ch === "'") {+ quote = ch;+ out += ch;+ } else if (ch === "[") {+ depth++;+ out += ch;+ } else if (ch === "]") {+ depth = Math.max(0, depth - 1);+ out += ch;+ } else if (ch === ">" && depth === 0) {+ out = out.replace(/\s+$/, "") + " ";+ while (i + 1 < selector.length && /\s/.test(selector[i + 1]!)) i++;+ } else {+ out += ch;+ }+ }+ return out.trim();+}++export function tooStrictSelectors(+ code: string,+ html: string,+): TooStrictSelector[] {+ const selectors = extractSelectorLiterals(code).filter(s => s.includes(">"));+ if (selectors.length === 0) return [];++ const doc = parseDocument(html);+ const count = (selector: string): number | null => {+ try {+ return doc.querySelectorAll(selector).length;+ } catch {+ return null; // invalid CSS - the sandbox surfaces those on its own+ }+ };++ const out: TooStrictSelector[] = [];+ for (const selector of selectors) {+ if (count(selector) !== 0) continue;+ const loosened = loosenCombinators(selector);+ const loosenedCount = count(loosened);+ if (loosenedCount != null && loosenedCount > 0) {+ out.push({ selector, loosened, count: loosenedCount });+ }+ }+ return out;+}++export function tooStrictFeedback(broken: TooStrictSelector[]): string {+ return (+ `Some selectors matched 0 elements even though the target IS on the page - a ` ++ `child combinator (\`>\`) is breaking on a wrapper element you couldn't see in ` ++ `the sample. Switch these to descendant combinators:\n` ++ broken+ .map(+ b =>+ `- \`${b.selector}\` matched 0; \`${b.loosened}\` matches ${b.count}`,+ )+ .join("\n") ++ `\nRewrite these (and re-check your other selectors for the same issue). Leave ` ++ `genuinely-absent fields empty - never broaden a selector just to match something.`+ );+}diff --git a/apps/api/src/lib/deterministicJson/html/simplify.ts b/apps/api/src/lib/deterministicJson/html/simplify.tsnew file mode 100644index 000000000..1421d3d93--- /dev/null+++ b/apps/api/src/lib/deterministicJson/html/simplify.ts@@ -0,0 +1,274 @@+// Shrink raw HTML into a compact, selector-faithful sample for the codegen model:+// drop noise, keep structure and selector attributes, collapse repeats, clip to a+// budget.+import { COMMENT_NODE, ELEMENT_NODE, parseDocument, TEXT_NODE } from "./dom";++interface SimplifyOptions {+ maxTextLen?: number;+ maxAttrLen?: number;+ maxClassNames?: number;+ keepRepeats?: number;+ maxTotalLen?: number;+}++const DEFAULTS = {+ maxTextLen: 160,+ maxAttrLen: 96,+ maxClassNames: 12,+ keepRepeats: 3,+ maxJsonLdLen: 4_000,+ maxTotalLen: 40_000,+};+type Opts = typeof DEFAULTS;++// Tags whose content never helps a selector. SVG/MATH/CANVAS are kept as empty+// elements (their guts are huge and useless); the rest are removed entirely.+const REMOVE_TAGS = new Set([+ "STYLE",+ "NOSCRIPT",+ "TEMPLATE",+ "LINK",+ "BR",+ "WBR",+ "SOURCE",+ "IFRAME",+]);+const EMPTY_TAGS = new Set(["SVG", "MATH", "CANVAS"]);++const DROP_ATTRS = new Set([+ "style",+ "integrity",+ "nonce",+ "crossorigin",+ "referrerpolicy",+ "decoding",+ "loading",+ "fetchpriority",+ "sizes",+ "autocomplete",+ "spellcheck",+ "tabindex",+]);+const URL_ATTRS = new Set(["href", "src", "data-src", "poster", "action"]);+const TRUNCATE_ATTRS = new Set([+ "src",+ "href",+ "srcset",+ "data-src",+ "poster",+ "action",+ "content",+ "value",+ "alt",+ "title",+ "aria-label",+]);+const KEEP_META = new Set([+ "title",+ "description",+ "author",+ "date",+ "pubdate",+ "article:published_time",+]);++const truncate = (s: string, max: number): string =>+ s.length <= max ? s : `${s.slice(0, max)}...[+${s.length - max}]`;++const collapseSpace = (s: string): string => s.replace(/\s+/g, " ").trim();++// Keep a URL's origin + first path segment (the strongest selector hint) and+// drop the slug tail, rather than blindly cutting the string.+function truncateUrl(value: string, max: number): string {+ if (value.length <= max) return value;+ try {+ const u = new URL(value, "https://x.invalid");+ const seg = u.pathname.split("/").filter(Boolean)[0];+ const head = `${u.origin === "https://x.invalid" ? "" : u.origin}/${seg ?? ""}`;+ if (head.length < max) return `${head}/...[+${value.length - head.length}]`;+ } catch {+ /* fall through */+ }+ return truncate(value, max);+}++const isJsonLd = (el: Element): boolean =>+ el.tagName === "SCRIPT" &&+ (el.getAttribute("type") ?? "").toLowerCase() === "application/ld+json";++function isHidden(el: Element): boolean {+ if (el.hasAttribute("hidden") || el.getAttribute("aria-hidden") === "true")+ return true;+ if (+ el.tagName === "INPUT" &&+ (el.getAttribute("type") ?? "").toLowerCase() === "hidden"+ )+ return true;+ return /display\s*:\s*none|visibility\s*:\s*hidden/i.test(+ el.getAttribute("style") ?? "",+ );+}++function pruneHead(doc: Document): void {+ const head = doc.querySelector("head");+ if (!head) return;+ for (const el of Array.from(head.children)) {+ if (el.tagName === "TITLE" || isJsonLd(el)) continue;+ if (el.tagName === "META") {+ const key = (+ el.getAttribute("name") ??+ el.getAttribute("property") ??+ ""+ ).toLowerCase();+ if (+ KEEP_META.has(key) ||+ key.startsWith("og:") ||+ key.startsWith("twitter:")+ )+ continue;+ }+ el.remove();+ }+}++function cleanAttributes(el: Element, opts: Opts): void {+ for (const name of el.getAttributeNames()) {+ const lower = name.toLowerCase();+ if (+ DROP_ATTRS.has(lower) ||+ lower.startsWith("on") ||+ /^data-(react|next|astro|svelte|vue|emotion|styled|gtm|ga|analytics|tracking|cookie|consent)/.test(+ lower,+ )+ ) {+ el.removeAttribute(name);+ continue;+ }+ const value = el.getAttribute(name) ?? "";+ if (lower === "class") {+ const classes = [...new Set(value.split(/\s+/).filter(Boolean))].slice(+ 0,+ opts.maxClassNames,+ );+ classes.length+ ? el.setAttribute(name, classes.join(" "))+ : el.removeAttribute(name);+ } else if (value.startsWith("data:")) {+ el.setAttribute(name, "data:...");+ } else if (URL_ATTRS.has(lower)) {+ el.setAttribute(name, truncateUrl(value, opts.maxAttrLen));+ } else if (TRUNCATE_ATTRS.has(lower)) {+ el.setAttribute(name, truncate(value, opts.maxAttrLen));+ }+ }+}++// A signature for "same kind of element": tag + sorted classes + child tags.+// Used to collapse long runs of list items / table rows / cards.+function shapeKey(el: Element): string {+ const classes = (el.getAttribute("class") ?? "")+ .split(/\s+/)+ .filter(Boolean)+ .sort()+ .join(".");+ const childTags = Array.from(el.children)+ .slice(0, 5)+ .map(c => c.tagName)+ .join(",");+ return `${el.tagName}|${classes}|${childTags}`;+}++function collapseRepeats(parent: Element, opts: Opts): void {+ const groups = new Map<string, Element[]>();+ for (const el of Array.from(parent.children)) {+ const key = shapeKey(el);+ (groups.get(key) ?? groups.set(key, []).get(key)!).push(el);+ }+ for (const group of groups.values()) {+ // Only collapse a genuinely repeating run, not 2-3 incidental look-alikes.+ if (group.length < Math.max(5, opts.keepRepeats + 1)) continue;+ const extra = group.slice(opts.keepRepeats);+ const tag = group[0]!.tagName.toLowerCase();+ extra[0]!.parentNode?.insertBefore(+ parent.ownerDocument.createComment(+ ` ${extra.length} more <${tag}> with the same shape `,+ ),+ extra[0]!,+ );+ for (const el of extra) el.remove();+ }+}++function walk(node: Element, opts: Opts): void {+ for (const child of Array.from(node.childNodes)) {+ if (+ child.nodeType === COMMENT_NODE ||+ (child.nodeType !== ELEMENT_NODE && child.nodeType !== TEXT_NODE)+ ) {+ child.remove();+ } else if (child.nodeType === TEXT_NODE) {+ const text = collapseSpace(child.textContent ?? "");+ text+ ? (child.textContent = truncate(text, opts.maxTextLen))+ : child.remove();+ } else {+ const el = child as Element;+ if (isHidden(el) || REMOVE_TAGS.has(el.tagName)) {+ el.remove();+ } else if (el.tagName === "SCRIPT") {+ if (isJsonLd(el))+ el.textContent = truncate(+ collapseSpace(el.textContent ?? ""),+ opts.maxJsonLdLen,+ );+ else el.remove();+ } else if (EMPTY_TAGS.has(el.tagName)) {+ cleanAttributes(el, opts);+ el.textContent = "";+ } else {+ cleanAttributes(el, opts);+ walk(el, opts);+ }+ }+ }+ collapseRepeats(node, opts);+}++// Drop attribute-less <div>/<span> wrappers that hold a single element child -+// they add depth without selector value.+function flattenWrappers(el: Element): void {+ for (const child of Array.from(el.children)) flattenWrappers(child);+ if (+ (el.tagName !== "DIV" && el.tagName !== "SPAN") ||+ el.getAttributeNames().length+ )+ return;+ const childElements = Array.from(el.children);+ const hasOwnText = Array.from(el.childNodes).some(+ n => n.nodeType === TEXT_NODE && (n.textContent ?? "").trim(),+ );+ if (childElements.length === 1 && !hasOwnText)+ el.replaceWith(childElements[0]!);+}++function clipMiddle(text: string, max: number): string {+ if (text.length <= max) return text;+ const marker = `\n<!-- ...${text.length - max} chars trimmed... -->\n`;+ const head = Math.ceil((max - marker.length) * 0.65);+ return (+ text.slice(0, head) + marker + text.slice(-(max - marker.length - head))+ );+}++export function simplifyHtml(+ html: string,+ options: SimplifyOptions = {},+): string {+ const opts = { ...DEFAULTS, ...options };+ const doc = parseDocument(html);+ pruneHead(doc);+ walk(doc.documentElement, opts);+ flattenWrappers(doc.documentElement);+ const out = doc.documentElement.outerHTML.replace(/>\s+</g, "><");+ return clipMiddle(out, opts.maxTotalLen);+}diff --git a/apps/api/src/lib/deterministicJson/llm/client.ts b/apps/api/src/lib/deterministicJson/llm/client.tsnew file mode 100644index 000000000..c0d339ab7--- /dev/null+++ b/apps/api/src/lib/deterministicJson/llm/client.ts@@ -0,0 +1,247 @@+import { generateText } from "ai";+import { jsonrepair } from "jsonrepair";+import { getModel } from "../../generic-ai";+import { CostTracking } from "../../cost-tracking";+import {+ ANCHOR_MODEL,+ ANCHOR_PICKER_MAX_TOKENS,+ ASK_LLM_MAX_CALLS,+ ASK_LLM_MAX_TOKENS,+ CODEGEN_MAX_TOKENS,+ CODEGEN_MODEL,+ LIGHT_MODEL,+} from "../config";+import { type CacheBackend } from "../core/cache";+import { modelPrices } from "../../extract/usage/model-prices";+import { ASK_LLM_SYSTEM, type ChatMessage } from "./prompts";+import { errorMessage, log, sha } from "../core/util";++function record(+ costTracking: CostTracking,+ role: string,+ provider: string,+ model: string,+ usage: { inputTokens?: number; outputTokens?: number } | undefined,+): void {+ const input = usage?.inputTokens ?? 0;+ const output = usage?.outputTokens ?? 0;+ const key = `${provider}/${model}`;+ const price = (+ modelPrices as Record<+ string,+ { input_cost_per_token?: number; output_cost_per_token?: number }+ >+ )[key];+ if (!price) log(`no pricing for ${key}`);+ const cost =+ input * (price?.input_cost_per_token ?? 0) ++ output * (price?.output_cost_per_token ?? 0);+ costTracking.addCall({+ type: "other",+ model: key,+ cost,+ tokens: { input, output },+ metadata: { module: "deterministic-json", role },+ });+}++interface Generation {+ content: string;+ truncated: boolean;+}++export async function generateCode(+ messages: ChatMessage[],+ costTracking: CostTracking,+): Promise<Generation> {+ let res;+ try {+ res = await generateText({+ model: getModel(CODEGEN_MODEL, "vertex"),+ messages,+ temperature: 0,+ maxOutputTokens: CODEGEN_MAX_TOKENS,+ });+ } catch (err) {+ throw new Error(`codegen API call failed: ${errorMessage(err)}`);+ }+ record(costTracking, "codegen", "vertex", CODEGEN_MODEL, res.usage);+ return { content: res.text ?? "", truncated: res.finishReason === "length" };+}++export async function pickSnippets(+ messages: ChatMessage[],+ costTracking: CostTracking,+): Promise<string[]> {+ let res;+ try {+ res = await generateText({+ model: getModel(ANCHOR_MODEL, "groq"),+ messages,+ temperature: 0,+ maxOutputTokens: ANCHOR_PICKER_MAX_TOKENS,+ });+ } catch (err) {+ throw new Error(`anchor-picker API call failed: ${errorMessage(err)}`);+ }+ record(costTracking, "anchor", "groq", ANCHOR_MODEL, res.usage);+ return parseSnippets(res.text?.trim() ?? "");+}++function parseSnippets(raw: string): string[] {+ const trimmed = raw+ .replace(/^```(?:json)?\s*/i, "")+ .replace(/```\s*$/i, "")+ .trim();+ let value: unknown;+ try {+ value = JSON.parse(jsonrepair(trimmed));+ } catch {+ return [];+ }+ const list = Array.isArray(value)+ ? value+ : ((value as { snippets?: unknown })?.snippets ?? []);+ return Array.isArray(list)+ ? list.filter((s): s is string => typeof s === "string")+ : [];+}++export type AskLlm = (prompt: string, schema?: unknown) => Promise<unknown>;++const ASK_LLM_RETRIES = 3;++function normalizePlainText(s: string): string {+ return /^(null|none|n\/a|undefined)$/i.test(s.trim()) ? "" : s;+}++function parseJsonLoose(raw: string): unknown {+ try {+ return JSON.parse(raw);+ } catch {+ return JSON.parse(jsonrepair(raw));+ }+}++// askLlm function for one extraction run: caches responses, dedupes identical+// in-flight calls, caps real API calls at maxCalls. Any failure resolves to null+// so one bad inner call can't sink the whole extraction.+export function makeAskLlm(+ cache: CacheBackend,+ costTracking: CostTracking,+ maxCalls = ASK_LLM_MAX_CALLS,+): AskLlm {+ let apiCalls = 0;+ const inFlight = new Map<string, Promise<unknown>>();++ const ask = (prompt: string, schema?: unknown): Promise<unknown> => {+ const key =+ schema == null ? prompt : `${prompt}\0${JSON.stringify(schema)}`;+ const existing = inFlight.get(key);+ if (existing) return existing;+ const pending = askOnce(prompt, schema).catch(err => {+ log(`askLlm failed, returning null: ${errorMessage(err)}`);+ return null;+ });+ inFlight.set(key, pending);+ return pending;+ };++ async function askOnce(prompt: string, schema: unknown): Promise<unknown> {+ const structured = schema != null;+ if (structured && (typeof schema !== "object" || Array.isArray(schema))) {+ throw new Error("askLlm schema must be a JSON Schema object");+ }++ const objectSchema =+ structured && (schema as { type?: unknown }).type === "object";+ const wrap = structured && !objectSchema;++ const userContent = !structured+ ? prompt+ : wrap+ ? `${prompt}\n\nReturn ONLY a JSON object {"value": V} where V matches this JSON Schema (use null if unknown):\n\n${JSON.stringify(schema)}`+ : `${prompt}\n\n### JSON Schema (match this shape; use null for unknown)\n\n${JSON.stringify(schema)}`;++ const finalize = (raw: string): unknown => {+ const parsed = parseJsonLoose(raw);+ let value: unknown = parsed;+ if (+ wrap &&+ parsed &&+ typeof parsed === "object" &&+ !Array.isArray(parsed) &&+ "value" in parsed+ ) {+ value = (parsed as Record<string, unknown>).value;+ }+ return typeof value === "string" ? normalizePlainText(value) : value;+ };++ const cacheKey = sha(+ `${LIGHT_MODEL}\0${ASK_LLM_SYSTEM}\0${userContent}\0${structured ? "json" : "text"}`,+ );+ const cached = await cache.getLlm(cacheKey);+ if (cached) {+ try {+ const value = structured+ ? finalize(cached.response)+ : normalizePlainText(cached.response);+ await cache.touch?.(cacheKey);+ return value;+ } catch (err) {+ // Fall through to a fresh call; otherwise the parse error sinks the+ // whole ask to null via ask()'s catch.+ log(+ `ignoring unparseable cached askLlm response: ${errorMessage(err)}`,+ );+ }+ }++ let lastError = "";+ for (let attempt = 1; attempt <= ASK_LLM_RETRIES; attempt++) {+ if (apiCalls >= maxCalls)+ throw new Error(`askLlm call budget exhausted (${maxCalls})`);+ apiCalls++;++ let raw = "";+ try {+ const res = await generateText({+ model: getModel(LIGHT_MODEL, "groq"),+ temperature: 0,+ maxOutputTokens: ASK_LLM_MAX_TOKENS,+ messages: [+ { role: "system", content: ASK_LLM_SYSTEM },+ { role: "user", content: userContent },+ ],+ });+ raw = res.text?.trim() ?? "";+ record(costTracking, "askLlm", "groq", LIGHT_MODEL, res.usage);+ } catch (err) {+ lastError = errorMessage(err);+ continue;+ }+ if (!raw) {+ lastError = "empty response";+ continue;+ }++ let value: unknown;+ try {+ value = structured ? finalize(raw) : normalizePlainText(raw);+ } catch (err) {+ lastError = `parse: ${errorMessage(err)}`;+ continue;+ }++ await cache.setLlm(cacheKey, raw);+ return value;+ }++ throw new Error(+ `askLlm failed after ${ASK_LLM_RETRIES} attempts: ${lastError}`,+ );+ }++ return ask;+}diff --git a/apps/api/src/lib/deterministicJson/llm/prompts.ts b/apps/api/src/lib/deterministicJson/llm/prompts.tsnew file mode 100644index 000000000..96b8a0ba4--- /dev/null+++ b/apps/api/src/lib/deterministicJson/llm/prompts.ts@@ -0,0 +1,96 @@+export interface ChatMessage {+ role: "system" | "user";+ content: string;+}++// System prompt for inner askLlm calls (the cheap model the extractor calls+// at runtime).+export const ASK_LLM_SYSTEM = [+ "You are a text-processing function in a data pipeline.",+ "Return only the requested value: no preamble, no markdown.",+ "You may summarize, classify, translate, or reformat the supplied text,",+ "but stay grounded in it: never invent facts or do outside lookups",+ 'If the text lacks what you need, return null (or "" / [] per the schema).',+].join(" ");++// Phase 1: pick short verbatim snippets from the markdown that sit next to the+// data the extractor will need. Each snippet is later located in the raw HTML so+// we can hand the codegen model a tight window instead of the whole page.+const ANCHOR_PICKER_SYSTEM = `You are a planning step in a web-extraction pipeline. Given a page's markdown, a target JSON Schema, and a task, pick short text snippets from the markdown that sit next to the data the extractor will need to read.++Rules:+- Each snippet must appear VERBATIM in the markdown.+- Prefer short (5-80 char) distinctive phrases: a label ("Balance:"), a value next to a label, a heading, a unique table cell.+- Pick one snippet near each field the schema asks for. For a list/array schema, pick 2-4 snippets that hit DIFFERENT items.+- Avoid generic phrases ("Home", "Menu", "Read more") and very long quotes.+- Skip fields whose data clearly is not on the page.++Return only JSON of the form {"snippets": ["...", "..."]} - no markdown fences, no commentary.`;++// Phase 2: write the reusable extractor. Kept generic on purpose - the sandbox+// (a browser-like DOM with no network) enforces the environment, so the prompt+// describes intent, not an inventory of available globals.+const EXTRACTOR_SYSTEM = `You write ONE reusable JavaScript function that extracts structured data from a web page.++ async function extract(doc, askLlm) { ... }++- \`doc\` is a standard DOM Document: querySelector / querySelectorAll, document.evaluate (XPath), textContent, getAttribute, etc.+- \`askLlm(prompt)\` returns a string; \`askLlm(prompt, jsonSchema)\` returns a value of that shape. It can fail and return null, so guard its result.++This function is generated once from ONE sample page, cached, then re-run on OTHER pages of the same shape. Therefore:++- Read every value from the DOM at runtime. NEVER return a literal you saw in the sample page - the values differ on other pages. Literals are only allowed in selectors, attribute names, regexes, and the returned object's keys.+- Always return the schema's shape, even on 404 / login / empty pages: use null or [] for anything absent. NEVER throw. A fabricated value is worse than an empty one.+- When a value is present, coerce it to the schema's type (e.g. strip non-numerics for numbers). Keep dates as the page displays them unless the task asks for a specific format.++Selectors:+- Anchor on stable, meaningful things: labels, semantic attributes (itemprop, role, data-*), heading text. Avoid fragile position (:nth-child, sibling chains).+- Prefer descendant combinators over child combinators (\`>\`); an unseen wrapper element breaks \`>\`. A selector that matches 0 elements is usually too strict, not proof the data is absent.+- Read URLs from href/src, not from link text.+- Only standard CSS selectors work. To match by visible text, select structurally then filter in JS, e.g. \`[...doc.querySelectorAll('tr')].find(r => r.textContent.includes('Total'))\`.++askLlm:+- Use it for what selectors cannot do - summarize, classify, translate, disambiguate by meaning. It transforms the text you pass it (no outside lookups like currency conversion; returns null when the text lacks the answer), so pass the DOM text it needs - for a summary, the article body, not a bare title or label. If a field has a fixed value set, include it in the prompt.+- When you call askLlm once per item over a list, issue the calls concurrently with Promise.all rather than awaiting one at a time.++Return only the function source, beginning with \`async function extract\`.`;++export function buildAnchorPickerMessages(args: {+ userPrompt: string;+ schemaJson: string;+ markdownPreview: string;+}): ChatMessage[] {+ return [+ { role: "system", content: ANCHOR_PICKER_SYSTEM },+ {+ role: "user",+ content: `### Page markdown\n\n${args.markdownPreview}\n\n### Target JSON Schema\n\n${args.schemaJson}\n\n### Task\n\n${args.userPrompt}\n\n### Output\n\nReturn {"snippets": [...]} with short verbatim fragments from the markdown above.`,+ },+ ];+}++export function buildExtractorMessages(args: {+ userPrompt: string;+ schemaJson: string;+ markdownPreview: string;+ anchorHtml: string;+ rejectionFeedback?: string;+}): ChatMessage[] {+ // Bulky reference first, the task last - long-context models answer better+ // when the question follows the documents it refers to.+ const markdown = args.markdownPreview.trim()+ ? `### Page markdown (one sample page - the values shown are NOT your output)\n\n${args.markdownPreview}\n\n`+ : "";++ const retry = args.rejectionFeedback?.trim()+ ? `\n\n### Your previous attempt was rejected\n\n${args.rejectionFeedback}\n\nRewrite the function to fix this. Every value must still come from a runtime DOM read or askLlm, and the result must match the schema exactly.`+ : "";++ return [+ { role: "system", content: EXTRACTOR_SYSTEM },+ {+ role: "user",+ content: `${markdown}### HTML snippets around the target fields\n\n${args.anchorHtml}\n\n### Target JSON Schema\n\n${args.schemaJson}\n\n### Task\n\n${args.userPrompt}${retry}`,+ },+ ];+}diff --git a/apps/api/src/lib/deterministicJson/pipeline/anchor.ts b/apps/api/src/lib/deterministicJson/pipeline/anchor.tsnew file mode 100644index 000000000..167200724--- /dev/null+++ b/apps/api/src/lib/deterministicJson/pipeline/anchor.ts@@ -0,0 +1,269 @@+// Build focused HTML context: locate each picked snippet in the raw HTML, climb+// to a budgeted ancestor subtree, simplify it, and stitch the windows together,+// with a document-unique selector hint per snippet. Falls back to simplifying the+// whole page when no snippet lands.+import {+ ANCHOR_MAX_BLOCKS,+ ANCHOR_MAX_PARENTS,+ ANCHOR_PER_BLOCK_BUDGET,+ ANCHOR_TOTAL_BUDGET,+ HTML_BUDGET,+} from "../config";+import { CostTracking } from "../../cost-tracking";+import { parseDocument } from "../html/dom";+import { pickSnippets } from "../llm/client";+import { buildAnchorPickerMessages } from "../llm/prompts";+import { simplifyHtml } from "../html/simplify";+import { errorMessage, log } from "../core/util";++export async function buildAnchorContext(+ args: {+ html: string;+ markdownPreview: string;+ schemaJson: string;+ prompt: string;+ },+ costTracking: CostTracking,+): Promise<string> {+ let snippets: string[] = [];+ try {+ snippets = await pickSnippets(+ buildAnchorPickerMessages({+ userPrompt: args.prompt,+ schemaJson: args.schemaJson,+ markdownPreview: args.markdownPreview,+ }),+ costTracking,+ );+ log(`anchor-picker chose ${snippets.length} snippet(s)`);+ } catch (err) {+ log(+ "anchor-picker failed, falling back to whole-page simplify:",+ errorMessage(err),+ );+ }++ const built = snippets.length ? buildAnchorHtml(args.html, snippets) : "";+ if (built.trim()) return built;++ log("no anchors landed; simplifying whole page");+ return simplifyHtml(args.html, { maxTotalLen: HTML_BUDGET });+}++const normalize = (s: string): string => s.replace(/\s+/g, " ").trim();++// The picker reads markdown, so snippets often carry markdown decoration+// ("- **Foo**", "## Bar", "`baz`") that never appears in rendered HTML text.+// Strip it so the bare text matches the DOM.+function stripMarkdown(s: string): string {+ return s+ .replace(/^\s*(?:[-*+]\s+|\d+\.\s+|>\s+|#+\s+)/, "")+ .replace(/\*\*([^*]+)\*\*/g, "$1")+ .replace(/__([^_]+)__/g, "$1")+ .replace(/(?<!\*)\*([^*\n]+)\*(?!\*)/g, "$1")+ .replace(/`([^`]+)`/g, "$1")+ .replace(/!?\[([^\]]+)\]\([^)]*\)/g, "$1")+ .trim();+}++// Deepest element whose normalized text contains the needle - the leaf that+// actually holds the snippet, so we can climb from a tight starting point.+function findDeepestMatch(root: Element, needle: string): Element | null {+ let best: Element | null = null;+ const visit = (el: Element): boolean => {+ if (!normalize(el.textContent ?? "").includes(needle)) return false;+ let inChild = false;+ for (const child of Array.from(el.children))+ if (visit(child)) inChild = true;+ if (!inChild) best = el;+ return true;+ };+ visit(root);+ return best;+}++function ownText(el: Element): string {+ let out = "";+ for (const child of Array.from(el.childNodes)) {+ out +=+ child.nodeType === 3+ ? (child.textContent ?? "")+ : " " + (child.textContent ?? "");+ }+ return out;+}++// How many distinct (non-nested) elements own text containing the needle -+// signals when a snippet is too generic to anchor on.+function countMatches(root: Element, needle: string): number {+ const matches = Array.from(root.querySelectorAll("*")).filter(el =>+ normalize(ownText(el)).includes(needle),+ );+ return matches.filter(el => !matches.some(o => o !== el && o.contains(el)))+ .length;+}++function climbToBudget(anchor: Element): Element {+ let current = anchor;+ for (let i = 0; i < ANCHOR_MAX_PARENTS; i++) {+ const parent = current.parentElement;+ if (!parent || parent.tagName === "BODY" || parent.tagName === "HTML")+ break;+ if ((parent.outerHTML ?? "").length > ANCHOR_PER_BLOCK_BUDGET) break;+ current = parent;+ }+ return current;+}++const STABLE_ATTRS = ["itemprop", "data-testid", "data-test", "role", "name"];+const cssEscape = (s: string): string => s.replace(/([^a-zA-Z0-9_-])/g, "\\$1");++function isStableClass(c: string): boolean {+ if (/^[a-z-]+-\[?[\d.]/.test(c)) return false; // utility e.g. h-[14px], pr-1.5+ if (/[A-Za-z]_[A-Za-z0-9]{4,}$/.test(c)) return false; // CSS-module hash+ if (/^(is-|has-|js-|active|selected|hidden|open)/.test(c)) return false; // state+ return c.length >= 2;+}++// A selector fragment that targets this element by a stable hook (id, semantic+// attribute, or stable class), or null. Not checked for uniqueness on its own.+function ownHook(el: Element): string | null {+ const tag = el.tagName.toLowerCase();+ const id = el.getAttribute("id");+ if (id && /^[A-Za-z][\w-]*$/.test(id)) return `#${id}`;+ for (const attr of STABLE_ATTRS) {+ const v = el.getAttribute(attr);+ if (v) return `${tag}[${attr}="${v}"]`;+ }+ const classes = Array.from(el.classList).filter(isStableClass);+ if (classes.length)+ return tag + classes.map(c => `.${cssEscape(c)}`).join("");+ return null;+}++// Find a CSS selector that returns exactly this element on the whole document,+// or null. Tries the element's own hook first, then scopes by an ancestor's hook+// (page text often sits in attribute-less spans whose ancestor carries the id).+function uniqueSelector(doc: Document, el: Element): string | null {+ const matchesOne = (sel: string): boolean => {+ try {+ return (+ doc.querySelectorAll(sel).length === 1 && doc.querySelector(sel) === el+ );+ } catch {+ return false;+ }+ };++ const tag = el.tagName.toLowerCase();++ // 1. The element's own hook, narrowing the class chain until it's unique.+ const id = el.getAttribute("id");+ if (id && /^[A-Za-z][\w-]*$/.test(id) && matchesOne(`#${id}`))+ return `#${id}`;+ for (const attr of STABLE_ATTRS) {+ const v = el.getAttribute(attr);+ if (v && matchesOne(`${tag}[${attr}="${v}"]`))+ return `${tag}[${attr}="${v}"]`;+ }+ const classes = Array.from(el.classList).filter(isStableClass);+ for (let n = 1; n <= classes.length; n++) {+ const sel =+ tag ++ classes+ .slice(0, n)+ .map(c => `.${cssEscape(c)}`)+ .join("");+ if (matchesOne(sel)) return sel;+ }++ // 2. Scope by an ancestor's hook: `<ancestorHook> <leafTag(.class)>`.+ const leafForms = [+ tag,+ classes[0] ? `${tag}.${cssEscape(classes[0])}` : "",+ ].filter(Boolean);+ let ancestor = el.parentElement;+ for (+ let depth = 0;+ ancestor && depth < 5;+ depth++, ancestor = ancestor.parentElement+ ) {+ const hook = ownHook(ancestor);+ if (!hook) continue;+ for (const leaf of leafForms) {+ if (matchesOne(`${hook} ${leaf}`)) return `${hook} ${leaf}`;+ }+ }+ return null;+}++function anchorsHeader(hints: { snippet: string; selector: string }[]): string {+ if (!hints.length) return "";+ const lines = hints.map(+ h => ` ${JSON.stringify(h.snippet)} -> ${h.selector}`,+ );+ return (+ `<!-- VERIFIED SELECTORS: each returns exactly one element on the full page.\n` ++ ` Your code runs against the whole document, so prefer these for the\n` ++ ` fields they name. -->\n` ++ lines.join("\n") ++ "\n\n"+ );+}++// simplifyHtml serializes a full <html> shell; we only want the body contents.+function unwrapBody(html: string): string {+ const open = html.match(/<body[^>]*>/i);+ if (!open) return html;+ const start = (open.index ?? 0) + open[0].length;+ const end = html.lastIndexOf("</body>");+ return end < start ? html.slice(start) : html.slice(start, end);+}++function buildAnchorHtml(rawHtml: string, snippets: string[]): string {+ const doc = parseDocument(rawHtml);+ const root = (doc.body ?? doc.documentElement) as Element;++ const emittedSubtrees = new Set<Element>();+ const seenHtml = new Set<string>();+ const hints: { snippet: string; selector: string }[] = [];+ let combined = "";+ let blocks = 0;++ for (const raw of snippets) {+ const snippet = normalize(stripMarkdown(raw));+ if (!snippet) continue;++ const anchor = findDeepestMatch(root, snippet);+ if (!anchor) continue;++ const selector = uniqueSelector(doc, anchor);+ if (selector) hints.push({ snippet: snippet.slice(0, 80), selector });++ // Skip if a window we already emitted contains this anchor.+ if ([...emittedSubtrees].some(prior => prior.contains(anchor))) continue;+ if (blocks >= ANCHOR_MAX_BLOCKS) continue;++ const subtree = climbToBudget(anchor);+ const simplified = unwrapBody(+ simplifyHtml(subtree.outerHTML ?? "", {+ maxTotalLen: ANCHOR_PER_BLOCK_BUDGET,+ }),+ ).trim();+ if (!simplified || seenHtml.has(simplified)) continue;++ const matches = countMatches(root, snippet);+ const note = matches > 1 ? ` (${matches} similar matches on the page)` : "";+ const piece = `<!-- anchor ${blocks + 1}: ${JSON.stringify(snippet.slice(0, 120))}${note} -->\n${simplified}\n`;+ if (combined.length + piece.length > ANCHOR_TOTAL_BUDGET && blocks > 0)+ break;++ emittedSubtrees.add(subtree);+ seenHtml.add(simplified);+ combined += piece;+ blocks += 1;+ }++ log(`anchor-html: ${blocks} window(s), ${hints.length} verified selector(s)`);+ return combined ? anchorsHeader(hints) + combined : "";+}diff --git a/apps/api/src/lib/deterministicJson/pipeline/generate.ts b/apps/api/src/lib/deterministicJson/pipeline/generate.tsnew file mode 100644index 000000000..b40f9e65d--- /dev/null+++ b/apps/api/src/lib/deterministicJson/pipeline/generate.ts@@ -0,0 +1,80 @@+// Prompt the codegen model for the extractor, then statically validate it with+// the TypeScript parser before we bother forking a sandbox: clean the source+// down to the extract function, check it parses and has the right shape, and+// reject references to globals the sandbox can't provide. Each rejection is fed+// back as feedback for one regeneration. Runtime problems (a selector that+// throws, a logic bug) still surface only on execution.+import { generateCode } from "../llm/client";+import { CostTracking } from "../../cost-tracking";+import { buildExtractorMessages } from "../llm/prompts";+import { errorMessage, log } from "../core/util";+import {+ cleanGeneratedCode,+ formatGeneratedCodeIssues,+ validateGeneratedExtractor,+ type GeneratedCodeIssue,+} from "./validate";++const MAX_ATTEMPTS = 2;++export async function generateExtractor(+ args: {+ anchorHtml: string;+ markdownPreview: string;+ schemaJson: string;+ prompt: string;+ rejectionFeedback?: string;+ },+ costTracking: CostTracking,+): Promise<string> {+ let feedback = args.rejectionFeedback;+ let lastIssues: GeneratedCodeIssue[] = [];++ for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {+ const { content, truncated } = await generateCode(+ buildExtractorMessages({+ userPrompt: args.prompt,+ schemaJson: args.schemaJson,+ markdownPreview: args.markdownPreview,+ anchorHtml: args.anchorHtml,+ rejectionFeedback: feedback,+ }),+ costTracking,+ );++ if (truncated) {+ throw new Error("codegen output was truncated; raise CODEGEN_MAX_TOKENS");+ }++ let code = "";+ let issues: GeneratedCodeIssue[] = [];+ try {+ code = cleanGeneratedCode(content);+ issues = validateGeneratedExtractor(code);+ } catch (err) {+ issues = [+ {+ field: "source",+ reason: errorMessage(err),+ excerpt: content.replace(/\s+/g, " ").slice(0, 180),+ },+ ];+ }++ if (issues.length === 0) {+ log(`extractor generated (attempt ${attempt}, ${code.length} chars)`);+ return code;+ }++ lastIssues = issues;+ feedback = formatGeneratedCodeIssues(issues);+ log(+ `rejecting extractor attempt ${attempt}: ${issues.length} issue(s)\n${feedback}`,+ );+ }++ throw new Error(+ "extractor failed validation after retry:\n" ++ formatGeneratedCodeIssues(lastIssues),+ );+}diff --git a/apps/api/src/lib/deterministicJson/pipeline/postprocess.ts b/apps/api/src/lib/deterministicJson/pipeline/postprocess.tsnew file mode 100644index 000000000..985676b2d--- /dev/null+++ b/apps/api/src/lib/deterministicJson/pipeline/postprocess.ts@@ -0,0 +1,199 @@+type JsonObject = Record<string, unknown>;++const isObject = (v: unknown): v is JsonObject =>+ typeof v === "object" && v !== null && !Array.isArray(v);++// Reconcile an extractor's raw output with the target JSON schema: remap key+// casing, fill/omit nulls, strip junk characters, and dedupe over-extracted list+// items. Returns the cleaned value shaped to the schema.+export function parseWithSchema(value: unknown, jsonSchema: unknown): unknown {+ const remapped = remapKeys(value, jsonSchema);+ const coerced = coerceNulls(remapped, jsonSchema);+ const cleaned = sanitizeStrings(coerced);+ return dedupeArrays(cleaned);+}++const normalizeKey = (k: string): string =>+ k.replace(/[^a-z0-9]/gi, "").toLowerCase();++function remapKeys(value: unknown, schema: unknown): unknown {+ if (!isObject(schema)) return value;++ if (Array.isArray(value)) {+ return value.map(v => remapKeys(v, (schema as { items?: unknown }).items));+ }+ if (!isObject(value)) return value;++ const props = (schema as { properties?: JsonObject }).properties;+ if (!props) return value;++ const canonical = new Map(Object.keys(props).map(k => [normalizeKey(k), k]));+ const out: JsonObject = {};+ for (const [key, v] of Object.entries(value)) {+ const target =+ key in props ? key : (canonical.get(normalizeKey(key)) ?? key);+ out[target] = remapKeys(v, props[target]);+ }+ return out;+}++function schemaAllowsNull(schema: JsonObject): boolean {+ const t = schema.type;+ if (+ t === "null" ||+ (Array.isArray(t) && t.includes("null")) ||+ schema.nullable === true+ ) {+ return true;+ }+ for (const key of ["anyOf", "oneOf"] as const) {+ const branches = schema[key];+ if (+ Array.isArray(branches) &&+ branches.some(b => isObject(b) && schemaAllowsNull(b))+ ) {+ return true;+ }+ }+ return false;+}++function typeDefault(schema: JsonObject): unknown {+ const t = schema.type;+ const primary = Array.isArray(t) ? t.find(x => x !== "null") : t;+ switch (primary) {+ case "string":+ return "";+ case "number":+ case "integer":+ return 0;+ case "boolean":+ return false;+ case "array":+ return [];+ case "object": {+ const props = (schema.properties ?? {}) as JsonObject;+ const out: JsonObject = {};+ for (const key of (schema.required as string[] | undefined) ?? []) {+ const sub = props[key];+ out[key] =+ isObject(sub) && !schemaAllowsNull(sub) ? typeDefault(sub) : null;+ }+ return out;+ }+ default:+ return null;+ }+}++// `required` tracks whether the current value sits in a required slot. We only+// fabricate a default for a required, non-nullable null; an optional null is+// dropped so we never emit a fake value for data that simply isn't on the page.+function coerceNulls(+ value: unknown,+ schema: unknown,+ required = true,+): unknown {+ if (!isObject(schema)) return value;++ if (value === null) {+ if (schemaAllowsNull(schema) || !required) return null;+ return typeDefault(schema);+ }++ if (Array.isArray(value)) {+ const items = (schema as { items?: unknown }).items;+ // An item present in the array is a real value: coerce its required subfields.+ return isObject(items)+ ? value.map(v => coerceNulls(v, items, true))+ : value;+ }++ if (isObject(value)) {+ const props = (schema as { properties?: JsonObject }).properties;+ if (!props) return value;+ const requiredKeys = new Set(+ (schema as { required?: string[] }).required ?? [],+ );+ const out: JsonObject = {};+ for (const [key, v] of Object.entries(value)) {+ if (!(key in props)) {+ out[key] = v;+ continue;+ }+ const sub = props[key];+ // Optional field the model returned as null: omit it so a non-nullable+ // optional field still validates, instead of fabricating a default.+ if (+ v === null &&+ !requiredKeys.has(key) &&+ !(isObject(sub) && schemaAllowsNull(sub))+ ) {+ continue;+ }+ out[key] = coerceNulls(v, sub, requiredKeys.has(key));+ }+ // Fill in required keys the model omitted entirely.+ for (const key of requiredKeys) {+ if (!(key in out))+ out[key] = isObject(props[key])+ ? typeDefault(props[key] as JsonObject)+ : null;+ }+ return out;+ }+ return value;+}++// Drop exact-duplicate OBJECTS from arrays (a common over-extraction artifact -+// e.g. a list selector that matches each card twice, or triplicated rows).+// Scalar duplicates are left alone since repeated primitives can be legitimate.+function dedupeArrays(value: unknown): unknown {+ if (Array.isArray(value)) {+ const seen = new Set<string>();+ const out: unknown[] = [];+ for (const item of value.map(dedupeArrays)) {+ if (item && typeof item === "object") {+ const key = JSON.stringify(item);+ if (seen.has(key)) continue;+ seen.add(key);+ }+ out.push(item);+ }+ return out;+ }+ if (isObject(value)) {+ const out: JsonObject = {};+ for (const [k, v] of Object.entries(value)) out[k] = dedupeArrays(v);+ return out;+ }+ return value;+}++// Characters that ride along from the DOM but are never real content: private-+// use icon-font glyphs, zero-width joiners / BOM, and C0/C1 control codes.+// .trim() won't remove them, so we strip them and collapse the doubled spaces+// they leave behind (newlines preserved so multi-line text keeps its shape).+// Built via RegExp(string) so the source stays pure ASCII.+const JUNK_CHARS = new RegExp(+ "[\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u001F\\u007F-\\u009F" ++ "\\u200B-\\u200D\\u2060\\uFEFF\\uE000-\\uF8FF" ++ "\\u{F0000}-\\u{FFFFD}\\u{100000}-\\u{10FFFD}]",+ "gu",+);++function sanitizeStrings(value: unknown): unknown {+ if (typeof value === "string") {+ return value+ .replace(JUNK_CHARS, "")+ .replace(/[^\S\r\n]{2,}/g, " ")+ .trim();+ }+ if (Array.isArray(value)) return value.map(sanitizeStrings);+ if (isObject(value)) {+ const out: JsonObject = {};+ for (const [key, v] of Object.entries(value)) out[key] = sanitizeStrings(v);+ return out;+ }+ return value;+}diff --git a/apps/api/src/lib/deterministicJson/pipeline/validate.ts b/apps/api/src/lib/deterministicJson/pipeline/validate.tsnew file mode 100644index 000000000..8c0eaf6db--- /dev/null+++ b/apps/api/src/lib/deterministicJson/pipeline/validate.ts@@ -0,0 +1,203 @@+import ts from "typescript";++export interface GeneratedCodeIssue {+ field: string;+ reason: string;+ excerpt: string;+}++function parseSource(code: string): ts.SourceFile {+ return ts.createSourceFile(+ "extractor.js",+ code,+ ts.ScriptTarget.Latest,+ true,+ ts.ScriptKind.JS,+ );+}++function stripCodeFences(raw: string): string {+ const text = raw.trim();++ const fenced = text.match(/```(?:javascript|js|ts)?\s*([\s\S]*?)```/i);+ if (fenced) return fenced[1]!.trim();++ return text+ .replace(/^```(?:javascript|js|ts)?\s*/i, "")+ .replace(/```\s*$/i, "")+ .trim();+}++function findTopLevelExtractor(+ source: ts.SourceFile,+): ts.FunctionDeclaration | undefined {+ // On a duplicate `function extract`, the last declaration wins at runtime.+ const matches = source.statements.filter(+ (statement): statement is ts.FunctionDeclaration =>+ ts.isFunctionDeclaration(statement) &&+ statement.name?.text === "extract" &&+ !!statement.body,+ );+ return matches[matches.length - 1];+}++function isAllowedTopLevelDeclaration(statement: ts.Statement): boolean {+ return (+ ts.isFunctionDeclaration(statement) ||+ ts.isVariableStatement(statement) ||+ ts.isClassDeclaration(statement)+ );+}++export function cleanGeneratedCode(raw: string): string {+ const code = stripCodeFences(raw).trim();++ const source = parseSource(code);+ const fn = findTopLevelExtractor(source);+ if (!fn) {+ throw new Error("LLM did not return a function named extract");+ }++ const kept = source.statements.filter(+ statement =>+ statement === fn ||+ (isAllowedTopLevelDeclaration(statement) &&+ !ts.isEmptyStatement(statement)),+ );++ // The sandbox runs this as a plain script, so a surviving `export` would throw.+ return kept+ .map(statement =>+ code+ .slice(statement.getStart(source), statement.end)+ .replace(/^\s*export\s+(?:default\s+)?/i, ""),+ )+ .join("\n\n")+ .trim();+}++function nodeExcerpt(source: ts.SourceFile, node: ts.Node): string {+ const text = node.getText(source).replace(/\s+/g, " ").trim();+ return text.length > 180 ? `${text.slice(0, 177)}...` : text;+}++function validateTopLevelShape(+ source: ts.SourceFile,+ fn: ts.FunctionDeclaration | undefined,+ issues: GeneratedCodeIssue[],+): void {+ const parseDiagnostics =+ (source as ts.SourceFile & { parseDiagnostics?: ts.Diagnostic[] })+ .parseDiagnostics ?? [];+ for (const diagnostic of parseDiagnostics.slice(0, 3)) {+ issues.push({+ field: "source",+ reason: ts.flattenDiagnosticMessageText(diagnostic.messageText, " "),+ excerpt: "",+ });+ }++ if (!fn) {+ issues.push({+ field: "source",+ reason: "missing function declaration named extract",+ excerpt: "",+ });+ return;+ }++ const extras = source.statements.filter(+ statement =>+ statement !== fn &&+ !ts.isEmptyStatement(statement) &&+ !isAllowedTopLevelDeclaration(statement),+ );++ for (const extra of extras) {+ issues.push({+ field: "source",+ reason:+ "top-level code is not allowed; only function/class/const/let/var declarations may sit beside extract",+ excerpt: nodeExcerpt(source, extra),+ });+ }++ const isAsync = fn.modifiers?.some(+ modifier => modifier.kind === ts.SyntaxKind.AsyncKeyword,+ );++ if (!isAsync) {+ issues.push({+ field: "extract",+ reason: "extract must be async",+ excerpt: nodeExcerpt(source, fn),+ });+ }++ if (fn.parameters.length !== 2) {+ issues.push({+ field: "extract",+ reason: "extract must accept exactly two parameters: doc and askLlm",+ excerpt: nodeExcerpt(source, fn),+ });+ }+}++// Crude check for forbidden runtime references. The generated code runs inside+// jsdom's VM context (see sandbox/harness.ts), which has no access to certain+// globals like fetch or XMLHttpRequest.+const FORBIDDEN_GLOBALS = new Set(["fetch", "XMLHttpRequest"]);++function detectForbiddenGlobals(+ source: ts.SourceFile,+ issues: GeneratedCodeIssue[],+): void {+ const seen = new Set<string>();++ const visit = (node: ts.Node): void => {+ if (ts.isIdentifier(node) && FORBIDDEN_GLOBALS.has(node.text)) {+ // Skip identifiers that are property names of access expressions+ // (`foo.fetch` doesn't reference the global fetch) or local bindings.+ const parent = node.parent;+ const isPropertyName =+ parent &&+ ((ts.isPropertyAccessExpression(parent) && parent.name === node) ||+ (ts.isPropertyAssignment(parent) && parent.name === node) ||+ (ts.isBindingElement(parent) && parent.propertyName === node));+ if (!isPropertyName && !seen.has(node.text)) {+ seen.add(node.text);+ issues.push({+ field: "source",+ reason: `references forbidden global '${node.text}' (not available in the sandbox; will throw at runtime)`,+ excerpt: node.text,+ });+ }+ }+ ts.forEachChild(node, visit);+ };++ visit(source);+}++export function validateGeneratedExtractor(code: string): GeneratedCodeIssue[] {+ const source = parseSource(code);+ const fn = findTopLevelExtractor(source);+ const issues: GeneratedCodeIssue[] = [];++ validateTopLevelShape(source, fn, issues);+ detectForbiddenGlobals(source, issues);++ return issues;+}++export function formatGeneratedCodeIssues(+ issues: GeneratedCodeIssue[],+): string {+ return issues+ .slice(0, 16)+ .map(issue => {+ const excerpt = issue.excerpt ? `; ${issue.excerpt}` : "";+ return `- ${issue.field}: ${issue.reason}${excerpt}`;+ })+ .join("\n");+}diff --git a/apps/api/src/lib/deterministicJson/sandbox/harness.ts b/apps/api/src/lib/deterministicJson/sandbox/harness.tsnew file mode 100644index 000000000..2fd8a40c7--- /dev/null+++ b/apps/api/src/lib/deterministicJson/sandbox/harness.ts@@ -0,0 +1,45 @@+// Runs as the body of `async (input, host, require) => { ... }` in the+// code-sandbox jail. input is { code, html, url }: build a jsdom Document, run the+// generated extract(doc, askLlm) in jsdom's VM context, and bridge askLlm back to+// the API worker over host().+export const EXTRACTOR_HARNESS = String.raw`+const { JSDOM, VirtualConsole } = require("jsdom");+const vm = require("node:vm");++const dom = new JSDOM(input.html, {+ url: input.url || "https://example.invalid/",+ runScripts: "outside-only", // page scripts don't run; ours can.+ pretendToBeVisual: true, // enables getComputedStyle, requestAnimationFrame.+ virtualConsole: new VirtualConsole(),+});++// jsdom doesn't implement innerText (it needs layout). Models reach for it+// constantly; alias it to textContent so those reads don't crash.+const proto = dom.window.HTMLElement.prototype;+if (!Object.getOwnPropertyDescriptor(proto, "innerText")) {+ Object.defineProperty(proto, "innerText", {+ get() { return this.textContent; },+ set(value) { this.textContent = value == null ? "" : String(value); },+ configurable: true,+ });+}++const context = dom.getInternalVMContext();++// Bridge askLlm back through the sandbox host-callback to the API worker.+dom.window.askLlm = (prompt, schema) =>+ Promise.resolve()+ .then(() => host("askLlm", { prompt: String(prompt == null ? "" : prompt), schema: schema == null ? null : schema }))+ .catch(() => null);++// Stringify inside the context so the result crosses the realm boundary as a+// plain primitive, and a non-JSON return (DOM node, circular ref) throws here.+const wrapped =+ "(async () => {\n" + input.code + "\n" ++ " const __value = await extract(document, askLlm);\n" ++ " return JSON.stringify(__value === undefined ? null : __value);\n" ++ "})()";++const json = await new vm.Script(wrapped).runInContext(context);+return JSON.parse(typeof json === "string" ? json : "null");+`;diff --git a/apps/api/src/lib/deterministicJson/sandbox/runExtractor.ts b/apps/api/src/lib/deterministicJson/sandbox/runExtractor.tsnew file mode 100644index 000000000..78dd574b6--- /dev/null+++ b/apps/api/src/lib/deterministicJson/sandbox/runExtractor.ts@@ -0,0 +1,30 @@+import type { AskLlm } from "../llm/client";+import { EXTRACTOR_HARNESS } from "./harness";++export type SandboxRunner = (job: {+ code: string;+ input: unknown;+ onHost: (channel: string, payload: unknown) => Promise<unknown>;+}) => Promise<unknown>;++export async function runExtractorInSandbox(args: {+ code: string;+ html: string;+ url: string;+ askLlm: AskLlm;+ sandbox: SandboxRunner;+}): Promise<unknown> {+ return args.sandbox({+ code: EXTRACTOR_HARNESS,+ input: { code: args.code, html: args.html, url: args.url },+ onHost: async (channel, payload) => {+ if (channel !== "askLlm")+ throw new Error(`unknown sandbox channel: ${channel}`);+ const { prompt, schema } = (payload ?? {}) as {+ prompt?: string;+ schema?: unknown;+ };+ return args.askLlm(String(prompt ?? ""), schema ?? undefined);+ },+ });+}diff --git a/apps/api/src/lib/extract/usage/model-prices.ts b/apps/api/src/lib/extract/usage/model-prices.tsindex a5d183a85..c100af736 100644--- a/apps/api/src/lib/extract/usage/model-prices.ts+++ b/apps/api/src/lib/extract/usage/model-prices.ts@@ -19379,8 +19379,8 @@ export const modelPrices = {max_tokens: 32768,max_input_tokens: 131072,max_output_tokens: 32768,- input_cost_per_token: 1e-7,- output_cost_per_token: 5e-7,+ input_cost_per_token: 7.5e-8,+ output_cost_per_token: 3e-7,litellm_provider: "groq",mode: "chat",supports_function_calling: true,@@ -19395,7 +19395,7 @@ export const modelPrices = {max_input_tokens: 131072,max_output_tokens: 32766,input_cost_per_token: 1.5e-7,- output_cost_per_token: 7.5e-7,+ output_cost_per_token: 6e-7,litellm_provider: "groq",mode: "chat",supports_function_calling: true,@@ -19405,6 +19405,15 @@ export const modelPrices = {supports_tool_choice: true,supports_web_search: true,},+ "vertex/gemini-3.1-flash-lite": {+ max_tokens: 65536,+ max_input_tokens: 1048576,+ max_output_tokens: 65536,+ input_cost_per_token: 2.5e-7,+ output_cost_per_token: 1.5e-6,+ litellm_provider: "vertex_ai-language-models",+ mode: "chat",+ },"claude-opus-4-1": {max_tokens: 32000,max_input_tokens: 200000,diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex c033f4680..3efa05045 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -72,6 +72,10 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled = 5;}+ if (hasFormatOfType(options.formats, "deterministicJson")) {+ creditsToBeBilled = 7;+ }+if (internalOptions.v1Agent?.model === "fire-1" ||internalOptions.v1JSONAgent?.model?.toLowerCase() === "fire-1"diff --git a/apps/api/src/scraper/scrapeURL/transformers/deterministicJson.ts b/apps/api/src/scraper/scrapeURL/transformers/deterministicJson.tsnew file mode 100644index 000000000..cfc686fdf--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/transformers/deterministicJson.ts@@ -0,0 +1,199 @@+import WebSocket from "ws";+import { Meta } from "..";+import { Document } from "../../../controllers/v2/types";+import { hasFormatOfType } from "../../../lib/format-utils";+import { eq } from "drizzle-orm";+import { db, dbRr } from "../../../db/connection";+import * as schema from "../../../db/schema";+import { extractDeterministicJson } from "../../../lib/deterministicJson/extract";+import {+ CODE_SANDBOX_URL,+ SANDBOX_TIMEOUT_MS,+} from "../../../lib/deterministicJson/config";+import type { CacheBackend } from "../../../lib/deterministicJson/core/cache";+import type { SandboxRunner } from "../../../lib/deterministicJson/sandbox/runExtractor";++const cache: CacheBackend = {+ async getExtractor(key) {+ const [data] = await dbRr+ .select({+ code: schema.deterministic_json_scripts.code,+ created_at: schema.deterministic_json_scripts.created_at,+ })+ .from(schema.deterministic_json_scripts)+ .where(eq(schema.deterministic_json_scripts.cache_key, key))+ .limit(1);+ return data+ ? { code: data.code, createdAt: new Date(data.created_at).getTime() }+ : undefined;+ },+ async setExtractor(key, code, meta) {+ const now = new Date().toISOString();+ await db+ .insert(schema.deterministic_json_scripts)+ .values({+ cache_key: key,+ code,+ url: meta.url,+ model: meta.model,+ cache_version: meta.cacheVersion,+ updated_at: now,+ last_used_at: now,+ })+ .onConflictDoUpdate({+ target: schema.deterministic_json_scripts.cache_key,+ set: {+ code,+ url: meta.url,+ model: meta.model,+ cache_version: meta.cacheVersion,+ updated_at: now,+ last_used_at: now,+ },+ });+ },+ async getLlm(key) {+ const [data] = await dbRr+ .select({+ response: schema.deterministic_json_llm_cache.response,+ created_at: schema.deterministic_json_llm_cache.created_at,+ })+ .from(schema.deterministic_json_llm_cache)+ .where(eq(schema.deterministic_json_llm_cache.cache_key, key))+ .limit(1);+ return data+ ? {+ response: data.response,+ createdAt: new Date(data.created_at).getTime(),+ }+ : undefined;+ },+ async setLlm(key, response) {+ const now = new Date().toISOString();+ await db+ .insert(schema.deterministic_json_llm_cache)+ .values({ cache_key: key, response, last_used_at: now })+ .onConflictDoUpdate({+ target: schema.deterministic_json_llm_cache.cache_key,+ set: { response, last_used_at: now },+ });+ },+ async touch(key) {+ const now = new Date().toISOString();+ await Promise.all([+ db+ .update(schema.deterministic_json_scripts)+ .set({ last_used_at: now })+ .where(eq(schema.deterministic_json_scripts.cache_key, key)),+ db+ .update(schema.deterministic_json_llm_cache)+ .set({ last_used_at: now })+ .where(eq(schema.deterministic_json_llm_cache.cache_key, key)),+ ]);+ },+};++function sandboxRunner(endpoint: string): SandboxRunner {+ return job =>+ new Promise((resolve, reject) => {+ const ws = new WebSocket(endpoint);+ let done = false;+ let timer: ReturnType<typeof setTimeout>;+ const finish = (err: Error | null, value?: unknown) => {+ if (done) return;+ done = true;+ clearTimeout(timer);+ try {+ ws.close();+ } catch {+ /* closing */+ }+ err ? reject(err) : resolve(value);+ };+ timer = setTimeout(+ () =>+ finish(new Error(`sandbox timed out after ${SANDBOX_TIMEOUT_MS}ms`)),+ SANDBOX_TIMEOUT_MS,+ );+ ws.on("open", () =>+ ws.send(+ JSON.stringify({ type: "run", code: job.code, input: job.input }),+ ),+ );+ ws.on("message", async raw => {+ let frame: any;+ try {+ frame = JSON.parse(raw.toString());+ } catch {+ return;+ }+ if (frame.type === "host") {+ let value: unknown;+ let error: string | undefined;+ try {+ value = await job.onHost(frame.channel, frame.payload);+ } catch (err) {+ error = err instanceof Error ? err.message : String(err);+ }+ if (ws.readyState === WebSocket.OPEN) {+ ws.send(+ JSON.stringify({+ type: "hostResult",+ id: frame.id,+ value,+ error,+ }),+ );+ }+ } else if (frame.type === "result") {+ finish(null, frame.value);+ } else if (frame.type === "error") {+ finish(new Error(frame.message));+ }+ });+ ws.on("error", err =>+ finish(err instanceof Error ? err : new Error(String(err))),+ );+ ws.on("close", () => finish(new Error("sandbox connection closed")));+ });+}++export async function performDeterministicJson(+ meta: Meta,+ document: Document,+): Promise<Document> {+ const format = hasFormatOfType(meta.options.formats, "deterministicJson");+ if (!format) return document;++ const warn = (msg: string) => {+ document.warning = msg + (document.warning ? " " + document.warning : "");+ };++ if (meta.internalOptions.zeroDataRetention) {+ warn("Deterministic JSON mode is not supported with zero data retention.");+ return document;+ }++ const html = document.html ?? document.rawHtml;+ if (!html) {+ warn("Deterministic JSON mode requires page HTML.");+ return document;+ }++ try {+ document.json = await extractDeterministicJson({+ url: document.metadata?.sourceURL ?? meta.url,+ prompt: format.prompt ?? "",+ jsonSchema: (format.schema ?? {}) as Record<string, unknown>,+ page: { html, markdown: document.markdown ?? "" },+ cache,+ sandbox: sandboxRunner(CODE_SANDBOX_URL),+ costTracking: meta.costTracking,+ });+ } catch (error) {+ meta.logger.error("Deterministic JSON extraction failed", { error });+ warn("Deterministic JSON extraction failed.");+ }++ return document;+}diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 6190778a0..e5488d5fe 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -10,6 +10,7 @@ import {performSummary,performCleanContent,} from "./llmExtract";+import { performDeterministicJson } from "./deterministicJson";import { performQuery } from "./query";import { removeBase64Images } from "./removeBase64Images";import { performAgent } from "./agent";@@ -91,7 +92,11 @@ async function deriveMarkdownFromHTML(meta.options.formats,"changeTracking",);- const hasJson = hasFormatOfType(meta.options.formats, "json");+ // deterministicJson populates document.json just like json, so treat it the+ // same here (derive markdown for it; keep the field it produced).+ const hasJson =+ hasFormatOfType(meta.options.formats, "json") ||+ hasFormatOfType(meta.options.formats, "deterministicJson");const hasSummary = hasFormatOfType(meta.options.formats, "summary");const hasQuestion = hasFormatOfType(meta.options.formats, "question");const hasHighlights = hasFormatOfType(meta.options.formats, "highlights");@@ -321,7 +326,11 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {meta.options.formats,"changeTracking",);- const hasJson = hasFormatOfType(meta.options.formats, "json");+ // deterministicJson populates document.json just like json, so treat it the+ // same here (derive markdown for it; keep the field it produced).+ const hasJson =+ hasFormatOfType(meta.options.formats, "json") ||+ hasFormatOfType(meta.options.formats, "deterministicJson");const hasScreenshot = hasFormatOfType(meta.options.formats, "screenshot");const hasSummary = hasFormatOfType(meta.options.formats, "summary");const hasBranding = hasFormatOfType(meta.options.formats, "branding");@@ -576,6 +585,7 @@ const transformerStack: Transformer[] = [...(useIndex ? [sendDocumentToIndex] : []),...(useSearchIndex ? [sendDocumentToSearchIndex] : []), // Add to search index for real-time searchperformLLMExtract,+ performDeterministicJson,performSummary,performQuery,performAttributes,diff --git a/apps/api/src/services/monitoring/diff-orchestrator.ts b/apps/api/src/services/monitoring/diff-orchestrator.tsindex 3521a16aa..b3efbade2 100644--- a/apps/api/src/services/monitoring/diff-orchestrator.ts+++ b/apps/api/src/services/monitoring/diff-orchestrator.ts@@ -14,7 +14,7 @@ import {} from "./diff";import { judgeChange } from "./judgeChange";-type MonitorPageDiffStatus = "same" | "new" | "changed";+type MonitorPageDiffStatus = "same" | "new" | "changed" | "error";type Judgment = {meaningful: boolean;@@ -36,6 +36,7 @@ type MonitorPageDiffResult = {judgment?: Judgment;diffText?: string;diffJson?: Record<string, { previous: unknown; current: unknown }>;+ error?: string;};type PreviousPageRef = {@@ -103,15 +104,17 @@ export async function computeAndPersistPageDiff(params: {? (doc.json as Record<string, unknown>): undefined;- // If the current scrape didn't produce a JSON document we can't- // compute a JSON diff; treat as `changed` to be safe (matches the- // markdown-missing branch's behavior).+ // If the current scrape didn't produce a JSON document we can't compute a+ // JSON diff (e.g. the extraction step failed) - report `error` rather than+ // a false `changed`, so the user isn't alerted to a content change that+ // didn't happen.if (!currentJson) {return {- status: "changed",+ status: "error",diffGcsKey: null,diffTextBytes: null,diffJsonBytes: null,+ error: "JSON extraction produced no result for this check.",};}diff --git a/apps/api/src/services/monitoring/diff.ts b/apps/api/src/services/monitoring/diff.tsindex d5696a85d..200be7912 100644--- a/apps/api/src/services/monitoring/diff.ts+++ b/apps/api/src/services/monitoring/diff.ts@@ -290,7 +290,9 @@ export function formatsRequestJsonExtraction(formats: unknown): boolean {if (typeof entry === "string" && entry === "json") return true;if (entry && typeof entry === "object") {const obj = entry as { type?: unknown; modes?: unknown };- if (obj.type === "json") return true;+ // Both json and deterministicJson populate document.json, so the JSON+ // diff path applies to either.+ if (obj.type === "json" || obj.type === "deterministicJson") return true;if (obj.type === "changeTracking") {const modes = Array.isArray(obj.modes) ? obj.modes : [];if (modes.includes("json")) return true;diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.tsindex aa8fcc120..7ba63ef99 100644--- a/apps/api/src/services/monitoring/results.ts+++ b/apps/api/src/services/monitoring/results.ts@@ -141,6 +141,7 @@ export async function recordMonitorScrapeSuccess(judgment,diffText,diffJson,+ error,} = await computeAndPersistPageDiff({teamId: job.data.team_id,monitorId: monitoring.monitorId,@@ -193,6 +194,7 @@ export async function recordMonitorScrapeSuccess(diff_text_bytes: diffTextBytes,diff_json_bytes: diffJsonBytes,status_code: getDocumentStatusCode(doc),+ ...(error ? { error } : {}),metadata: {title: doc?.metadata?.title ?? null,creditsUsed: doc?.metadata?.creditsUsed ?? null,diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex e4cade567..58354e9fa 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -7,6 +7,7 @@ import { processJobInternal } from "../worker/scrape-worker";import { NuQJob, crawlGroup, scrapeQueue } from "../worker/nuq";import { ScrapeJobData } from "../../types";import { getJobFromGCS } from "../../lib/gcs-jobs";+import { includesFormat } from "../../lib/format-utils";import { computeAndPersistPageDiff } from "./diff-orchestrator";import { normalizeMonitorFormats } from "./diff";import { autumnService } from "../autumn/autumn.service";@@ -216,10 +217,15 @@ function estimateActualCredits(doc: any, options: any): number {return doc.metadata.creditsUsed;}const formats = Array.isArray(options?.formats) ? options.formats : [];- const hasJson = formats.some((format: any) =>- typeof format === "string" ? format === "json" : format?.type === "json",- );- return hasJson ? 5 : 1;+ // Only charge the JSON-extraction premium when extraction actually produced a+ // document.json. When it failed, the page was still scraped, so fall back to+ // the base scrape credit rather than billing for extraction that never ran.+ // Deterministic JSON costs 7 (it generates a reusable extractor); plain JSON 5.+ const producedJson = doc?.json != null;+ if (!producedJson) return 1;+ if (includesFormat(formats, "deterministicJson")) return 7;+ if (includesFormat(formats, "json")) return 5;+ return 1;}async function runSingleScrape(params: {@@ -302,7 +308,7 @@ async function diffAndPersistPage(params: {(f: any) => f?.type === "changeTracking",): undefined;- const { status, diffGcsKey, diffTextBytes, diffJsonBytes, judgment } =+ const { status, diffGcsKey, diffTextBytes, diffJsonBytes, judgment, error } =await computeAndPersistPageDiff({teamId: params.monitor.team_id,monitorId: params.monitor.id,@@ -350,6 +356,7 @@ async function diffAndPersistPage(params: {diff_text_bytes: diffTextBytes,diff_json_bytes: diffJsonBytes,status_code: getDocumentStatusCode(params.doc),+ ...(error ? { error } : {}),metadata: {title: params.doc?.metadata?.title ?? null,},diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex 51f0f7cb9..33cc09872 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -3,6 +3,7 @@ import { v7 as uuidv7 } from "uuid";import { and, asc, count, desc, eq, isNull, ne } from "drizzle-orm";import { db, dbRr } from "../../db/connection";import * as schema from "../../db/schema";+import { includesFormat } from "../../lib/format-utils";import { monitoringClaimDueMonitors } from "../../db/rpc";import {getNextMonitorRunAt,@@ -31,16 +32,37 @@ function ensureTargetIds(targets: Array<Record<string, any>>): MonitorTarget[] {})) as MonitorTarget[];}+// Per-page credit cost for a target's extraction mode. Mirrors+// estimateActualCredits in runner.ts so the reserved/estimated credits match+// what a check actually bills: deterministic JSON = 7, other JSON (plain or+// changeTracking-json) = 5, otherwise the base scrape = 1.+function perPageCredits(target: MonitorTarget): number {+ const formats = Array.isArray(target.scrapeOptions?.formats)+ ? (target.scrapeOptions!.formats as any[])+ : [];+ if (includesFormat(formats, "deterministicJson")) return 7;+ const hasJson =+ includesFormat(formats, "json") ||+ formats.some(+ format =>+ format?.type === "changeTracking" &&+ Array.isArray(format?.modes) &&+ format.modes.includes("json"),+ );+ return hasJson ? 5 : 1;+}+function estimateTargetCredits(target: MonitorTarget): number {+ const perPage = perPageCredits(target);if (target.type === "scrape") {- return target.urls.length;+ return target.urls.length * perPage;}const limit =typeof target.crawlOptions?.limit === "number"? target.crawlOptions.limit: 10000;- return Math.max(1, limit);+ return Math.max(1, limit) * perPage;}export function estimateMonitorCreditsPerRun(