Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: deterministic json format (#3706)

tomsideguideef56b5ce4205
diff --git a/apps/api/package.json b/apps/api/package.json
index 6ef5b5c73..1f7ac87dc 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -123,7 +123,8 @@
"ioredis": "^5.6.1",
"ipaddr.js": "^2.2.0",
"joplin-turndown-plugin-gfm": "^1.0.12",
- "jsdom": "^26.0.0",
+ "jsdom": "^29.1.1",
+ "jsonrepair": "^3.14.0",
"koffi": "^2.9.0",
"langsmith": "^0.6.0",
"lodash": "^4.18.0",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index d8ded7e20..376878126 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -180,8 +180,11 @@ importers:
specifier: ^1.0.12
version: 1.0.12
jsdom:
- specifier: ^26.0.0
+ specifier: ^29.1.1
+ version: 29.1.1(@noble/[email protected])
+ jsonrepair:
+ specifier: ^3.14.0
+ version: 3.14.0
koffi:
specifier: ^2.9.0
version: 2.9.0
@@ -544,8 +547,20 @@ packages:
'@apm-js-collab/[email protected]':
resolution: {integrity: sha512-Vu1CbmPURlN5fTboVuKMoJjbO5qcq9fA5YXpskx3dXe/zTBvjODFoerw+69rVBlRLrJpwPqSDqEuJDEKIrTldw==}
- '@asamuzakjp/[email protected]':
- resolution: {integrity: sha512-GIc76d9UI1hCvOATjZPyHFmE5qhRccp3/zGfMPapK3jBi+yocEzp6BBB0UnfRYP9NP4FANqUZYb0hnfs3TM3hw==}
+ '@asamuzakjp/[email protected]':
+ resolution: {integrity: sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg==}
+ engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}
+
+ '@asamuzakjp/[email protected]':
+ resolution: {integrity: sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ==}
+ engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}
+
+ '@asamuzakjp/[email protected]':
+ resolution: {integrity: sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg==}
+ engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}
+
+ '@asamuzakjp/[email protected]':
+ resolution: {integrity: sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==}
resolution: {integrity: sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==}
@@ -716,6 +731,10 @@ packages:
resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==}
+ '@bramus/[email protected]':
+ resolution: {integrity: sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==}
+ hasBin: true
+
'@bull-board/[email protected]':
resolution: {integrity: sha512-UzkvN/wM+1qS73BS43a75LYkRzpBpCCUKlaGq0hp3dM5MNmdF1mx7LMGYgXPt91gqF8j4jq9Y/zCpC3Sqs3RLQ==}
peerDependencies:
@@ -745,33 +764,41 @@ packages:
resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==}
engines: {node: '>=12'}
- '@csstools/[email protected]':
- resolution: {integrity: sha512-MKtmkA0BX87PKaO1NFRTFH+UnkgnmySQOvNxJubsadusqPEC2aJ9MOQiMceZJJ6oitUl/i0L6u0M1IrmAOmgBA==}
- engines: {node: '>=18'}
+ '@csstools/[email protected]':
+ resolution: {integrity: sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q==}
+ engines: {node: '>=20.19.0'}
- '@csstools/[email protected]':
- resolution: {integrity: sha512-rL7kaUnTkL9K+Cvo2pnCieqNpTKgQzy5f+N+5Iuko9HAoasP+xgprVh7KN/MaJVvVL1l0EzQq2MoqBHKSrDrag==}
- engines: {node: '>=18'}
+ '@csstools/[email protected]':
+ resolution: {integrity: sha512-DtdHlgXh5ZkA43cwBcAm+huzgJiwx3ZTWVjBs94kwz2xKqSimDA3lBgCjphYgwgVUMWatSM0pDd8TILB1yrVVg==}
+ engines: {node: '>=20.19.0'}
peerDependencies:
- '@csstools/css-parser-algorithms': ^3.0.4
- '@csstools/css-tokenizer': ^3.0.3
+ '@csstools/css-parser-algorithms': ^4.0.0
+ '@csstools/css-tokenizer': ^4.0.0
- '@csstools/[email protected]':
- resolution: {integrity: sha512-nkMp2mTICw32uE5NN+EsJ4f5N+IGFeCFu4bGpiKgb2Pq/7J/MpyLBeQ5ry4KKtRFZaYs6sTmcMYrSRIyj5DFKA==}
- engines: {node: '>=18'}
+ '@csstools/[email protected]':
+ resolution: {integrity: sha512-eZ5XOtyhK+mggRafYUWzA0tvaYOFgdY8AkgQiCJF9qNAePnUo/zmsqqYubBBb3sQ8uNUaSKTY9s9klfRaAXL0g==}
+ engines: {node: '>=20.19.0'}
peerDependencies:
- '@csstools/css-parser-algorithms': ^3.0.4
- '@csstools/css-tokenizer': ^3.0.3
+ '@csstools/css-parser-algorithms': ^4.0.0
+ '@csstools/css-tokenizer': ^4.0.0
- '@csstools/[email protected]':
- resolution: {integrity: sha512-Up7rBoV77rv29d3uKHUIVubz1BTcgyUK72IvCQAbfbMv584xHcGKCKbWh7i8hPrRJ7qU4Y8IO3IY9m+iTB7P3A==}
- engines: {node: '>=18'}
+ '@csstools/[email protected]':
+ resolution: {integrity: sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w==}
+ engines: {node: '>=20.19.0'}
peerDependencies:
- '@csstools/css-tokenizer': ^3.0.3
+ '@csstools/css-tokenizer': ^4.0.0
- '@csstools/[email protected]':
- resolution: {integrity: sha512-UJnjoFsmxfKUdNYdWgOB0mWUypuLvAfQPH1+pyvRJs6euowbFkFC6P13w1l8mJyi3vxYMxc9kld5jZEGRQs6bw==}
- engines: {node: '>=18'}
+ '@csstools/[email protected]':
+ resolution: {integrity: sha512-wgsqt92b7C7tQhIdPNxj0n9zuUbQlvAuI1exyzeNrOKOi62SD7ren8zqszmpVREjAOqg8cD2FqYhQfAuKjk4sw==}
+ peerDependencies:
+ css-tree: ^3.2.1
+ peerDependenciesMeta:
+ css-tree:
+ optional: true
+
+ '@csstools/[email protected]':
+ resolution: {integrity: sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==}
+ engines: {node: '>=20.19.0'}
resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==}
@@ -1100,6 +1127,15 @@ packages:
cpu: [x64]
os: [win32]
+ '@exodus/[email protected]':
+ resolution: {integrity: sha512-S6mL0yNB/Abt9Ei4tq8gDhcczc4S3+vQ4ra7vxnAf+YHC02srtqxKKZghx2Dq6p0e66THKwR6r8N6P95wEty7Q==}
+ engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}
+ peerDependencies:
+ '@noble/hashes': ^1.8.0 || ^2.0.0
+ peerDependenciesMeta:
+ '@noble/hashes':
+ optional: true
+
'@google-cloud/[email protected]':
resolution: {integrity: sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==}
engines: {node: '>=14.0.0'}
@@ -2803,6 +2839,7 @@ packages:
resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==}
+ deprecated: Potential CWE-502 - Update to 1.3.1 or higher
resolution: {integrity: sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==}
@@ -3164,6 +3201,9 @@ packages:
resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==}
+ resolution: {integrity: sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==}
+
resolution: {integrity: sha512-JocpCSOixzy5XFJi2ub6IMmV/G9i8Lrm2lZvwBv9xPdglmZM0ufDVBbjbrfU/zuLvBfD7Bv2eYxz9i+OHTgkew==}
deprecated: pkg version number incorrect
@@ -3437,14 +3477,14 @@ packages:
resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==}
+ resolution: {integrity: sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==}
+ engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0}
+
resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==}
engines: {node: '>= 6'}
- resolution: {integrity: sha512-9+vem03dMXG7gDmZ62uqmRiMRNtinIZ9ZyuF6BdxzfOD+FdN5hretzynkn0ReS2DO2GSw76RWHs0UmJPI2zUjw==}
- engines: {node: '>=18'}
-
resolution: {integrity: sha512-1+BhOB8ahCn4O0cep0Sh2l9KCOfOdY+BXJnKMHFFzDEouSr/el18QwXEMRlOj9UY5nCeA8UN3a/82rUWRBeyBw==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
@@ -3453,9 +3493,9 @@ packages:
resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}
engines: {node: '>= 12'}
- resolution: {integrity: sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==}
- engines: {node: '>=18'}
+ resolution: {integrity: sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==}
+ engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}
resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==}
@@ -3482,8 +3522,8 @@ packages:
supports-color:
optional: true
- resolution: {integrity: sha512-8vDa8Qxvr/+d94hSh5P3IJwI5t8/c0KsMp+g8bNw9cY2icONa5aPfvKeieW1WlG0WQYwwhJ7mjui2xtiePQSXw==}
+ resolution: {integrity: sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==}
resolution: {integrity: sha512-+8VxcR21HhTy8nOt6jf20w0c9CADrw1O8d+VZ/YzzCt4bJ3uBjw+D1q2osAB8RnpwwaeYBxy0HyKQxD5JBMuuQ==}
@@ -3741,6 +3781,10 @@ packages:
resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==}
engines: {node: '>=0.12'}
+ resolution: {integrity: sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==}
+ engines: {node: '>=20.19.0'}
+
resolution: {integrity: sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==}
engines: {node: '>=18'}
@@ -4125,9 +4169,9 @@ packages:
resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==}
hasBin: true
- resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==}
- engines: {node: '>=18'}
+ resolution: {integrity: sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==}
+ engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}
resolution: {integrity: sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==}
@@ -4160,10 +4204,6 @@ packages:
resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==}
engines: {node: '>= 6'}
- resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==}
- engines: {node: '>= 14'}
-
resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}
engines: {node: '>= 6'}
@@ -4496,9 +4536,9 @@ packages:
resolution: {integrity: sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==}
- resolution: {integrity: sha512-BZYDGVAIriBWTpIxYzrXjv3E/4u8+/pSG5bQdIYCbNCGOvsPkDQfTVLAIXAf9ETdCpduCVTkDe2NNZ8NIwUVzw==}
- engines: {node: '>=18'}
+ resolution: {integrity: sha512-ECi4Fi2f7BdJtUKTflYRTiaMxIB0O6zfR1fX0GXpUrf6flp8QIYn1UT20YQqdSOfk2dfkCwS8LAFoJDEppNK5Q==}
+ engines: {node: ^20.19.0 || ^22.13.0 || >=24.0.0}
peerDependencies:
canvas: ^3.0.0
peerDependenciesMeta:
@@ -4534,6 +4574,10 @@ packages:
engines: {node: '>=6'}
hasBin: true
+ resolution: {integrity: sha512-tWPGKMZf/8UPim+fcW2EfcQ/d/7aKUrP6IECz9G3Tu6Q5dX0orSleqJ9z6sSw7qrQkjF8/Edo4DvsWBZ8H+HNg==}
+ hasBin: true
+
resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==}
@@ -4651,13 +4695,14 @@ packages:
resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==}
hasBin: true
- resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==}
-
resolution: {integrity: sha512-vFrFJkWtJvJnD5hg+hJvVE8Lh/TcMzKnTgCWmtBipwI5yLX/iX+5UB2tfuyODF5E7k9xEzMdYgGqaSb1c0c5Yw==}
engines: {node: 20 || >=22}
+ resolution: {integrity: sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A==}
+ engines: {node: 20 || >=22}
+
resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==}
@@ -4690,6 +4735,9 @@ packages:
resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==}
+ resolution: {integrity: sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==}
+
resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==}
engines: {node: '>= 0.6'}
@@ -4880,9 +4928,6 @@ packages:
resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==}
- resolution: {integrity: sha512-F1I/bimDpj3ncaNDhfyMWuFqmQDBwDB0Fogc2qpL3BWvkQteFD/8BzWuIRl83rq0DXfm8SGt/HFhLXZyljTXcQ==}
-
resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==}
engines: {node: '>=0.10.0'}
@@ -4996,8 +5041,8 @@ packages:
resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==}
- resolution: {integrity: sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==}
+ resolution: {integrity: sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==}
resolution: {integrity: sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==}
@@ -5313,9 +5358,6 @@ packages:
resolution: {integrity: sha512-I+kNjW0udB4Fetr3vvtRuYZJS0PcSPyyvBcH5sDdoV8DFs5E4W2pTr7aiMlKfPxANTClP9RlqCPolj9dd5MsEA==}
- resolution: {integrity: sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==}
-
resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==}
@@ -5438,6 +5480,10 @@ packages:
resolution: {integrity: sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==}
engines: {node: '>= 18'}
+ resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==}
+ engines: {node: '>=0.10.0'}
+
resolution: {integrity: sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==}
@@ -5611,10 +5657,17 @@ packages:
resolution: {integrity: sha512-AOvV5YYIAFFBfransBzSTyztkc3IMfz5Eq3YluaRiEu55nn43Fzaufx70UqEKYr8BoLCach4q8g/bg6e5+/aFw==}
+ resolution: {integrity: sha512-nwEyF4vl4RSJjwSjBUmOSxc3BFPoIFdlRthJ6e+5v9P3bHNsoD06UjuqMUspqp7vsEZ1beaHi1km+optiE17yA==}
+
resolution: {integrity: sha512-+lFzEXhpl7JXgWYaXcB6DqTYXbUArvrWAE/5ioq/X3CdWLbDjpPP4XTrQBmEJ91y3xbe4Fkw7Lxv4P3GWeJaNg==}
hasBin: true
+ resolution: {integrity: sha512-kCwffuaH8ntKtygnWe1b4BJKWiCUH30n5KfoTr6IchcXOwR7chAOFJxFrH3vjANafUYrIA4a7SDL+nn7SiR4Sw==}
+ hasBin: true
+
resolution: {integrity: sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==}
@@ -5630,16 +5683,16 @@ packages:
resolution: {integrity: sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==}
engines: {node: '>=6'}
- resolution: {integrity: sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==}
+ resolution: {integrity: sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==}
engines: {node: '>=16'}
resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==}
- resolution: {integrity: sha512-tk2G5R2KRwBd+ZN0zaEXpmzdKyOYksXwywulIX95MBODjSzMIuQnQ3m8JxgbhnL1LeVo7lqQKsYa1O3Htl7K5g==}
- engines: {node: '>=18'}
+ resolution: {integrity: sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==}
+ engines: {node: '>=20'}
resolution: {integrity: sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==}
@@ -5851,22 +5904,17 @@ packages:
resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==}
- resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==}
- engines: {node: '>=12'}
-
- resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==}
- engines: {node: '>=18'}
- deprecated: Use @exodus/bytes instead for a more spec-conformant and faster implementation
+ resolution: {integrity: sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==}
+ engines: {node: '>=20'}
- resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==}
- engines: {node: '>=18'}
+ resolution: {integrity: sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==}
+ engines: {node: '>=20'}
- resolution: {integrity: sha512-mDGf9diDad/giZ/Sm9Xi2YcyzaFpbdLpJPr+E9fSkyQ7KpQD4SdFcugkRQYzhmfI4KeV4Qpnn2sKPdo+kmsgRQ==}
- engines: {node: '>=18'}
+ resolution: {integrity: sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==}
+ engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}
resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==}
@@ -6180,13 +6228,25 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@asamuzakjp/[email protected]':
+ '@asamuzakjp/[email protected]':
dependencies:
- '@csstools/css-calc': 2.1.1(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])
- '@csstools/css-color-parser': 3.0.7(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])
- '@csstools/css-parser-algorithms': 3.0.4(@csstools/[email protected])
- '@csstools/css-tokenizer': 3.0.3
- lru-cache: 10.4.3
+ '@asamuzakjp/generational-cache': 1.0.1
+ '@csstools/css-calc': 3.2.1(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])
+ '@csstools/css-color-parser': 4.1.1(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])
+ '@csstools/css-parser-algorithms': 4.0.0(@csstools/[email protected])
+ '@csstools/css-tokenizer': 4.0.0
+
+ '@asamuzakjp/[email protected]':
+ dependencies:
+ '@asamuzakjp/generational-cache': 1.0.1
+ '@asamuzakjp/nwsapi': 2.3.9
+ bidi-js: 1.0.3
+ css-tree: 3.2.1
+ is-potential-custom-element-name: 1.0.1
+
+ '@asamuzakjp/[email protected]': {}
+
+ '@asamuzakjp/[email protected]': {}
dependencies:
@@ -6379,6 +6439,10 @@ snapshots:
'@bcoe/[email protected]': {}
+ '@bramus/[email protected]':
+ dependencies:
+ css-tree: 3.2.1
+
'@bull-board/[email protected](@bull-board/[email protected])':
dependencies:
'@bull-board/ui': 6.14.2
@@ -6430,25 +6494,29 @@ snapshots:
'@jridgewell/trace-mapping': 0.3.9
optional: true
- '@csstools/[email protected]': {}
+ '@csstools/[email protected]': {}
- '@csstools/[email protected](@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])':
+ '@csstools/[email protected](@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])':
dependencies:
- '@csstools/css-parser-algorithms': 3.0.4(@csstools/[email protected])
- '@csstools/css-tokenizer': 3.0.3
+ '@csstools/css-parser-algorithms': 4.0.0(@csstools/[email protected])
+ '@csstools/css-tokenizer': 4.0.0
- '@csstools/[email protected](@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])':
+ '@csstools/[email protected](@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])':
dependencies:
- '@csstools/color-helpers': 5.0.1
- '@csstools/css-calc': 2.1.1(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])
- '@csstools/css-parser-algorithms': 3.0.4(@csstools/[email protected])
- '@csstools/css-tokenizer': 3.0.3
+ '@csstools/color-helpers': 6.0.2
+ '@csstools/css-calc': 3.2.1(@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])
+ '@csstools/css-parser-algorithms': 4.0.0(@csstools/[email protected])
+ '@csstools/css-tokenizer': 4.0.0
- '@csstools/[email protected](@csstools/[email protected])':
+ '@csstools/[email protected](@csstools/[email protected])':
dependencies:
- '@csstools/css-tokenizer': 3.0.3
+ '@csstools/css-tokenizer': 4.0.0
- '@csstools/[email protected]': {}
+ optionalDependencies:
+ css-tree: 3.2.1
+
+ '@csstools/[email protected]': {}
dependencies:
@@ -6629,6 +6697,10 @@ snapshots:
'@esbuild/[email protected]':
optional: true
+ optionalDependencies:
+ '@noble/hashes': 1.8.0
+
'@google-cloud/[email protected]':
dependencies:
arrify: 2.0.1
@@ -8914,6 +8986,10 @@ snapshots:
+ dependencies:
+ require-from-string: 2.0.2
+
@@ -9203,21 +9279,23 @@ snapshots:
domutils: 3.1.0
nth-check: 2.1.1
-
dependencies:
- '@asamuzakjp/css-color': 2.8.3
- rrweb-cssom: 0.8.0
+ mdn-data: 2.27.1
+ source-map-js: 1.2.1
+
dependencies:
- whatwg-mimetype: 4.0.0
- whatwg-url: 14.1.1
+ whatwg-mimetype: 5.0.0
+ whatwg-url: 16.0.1(@noble/[email protected])
+ transitivePeerDependencies:
+ - '@noble/hashes'
dependencies:
@@ -9231,7 +9309,7 @@ snapshots:
dependencies:
ms: 2.1.3
@@ -9351,6 +9429,8 @@ snapshots:
+
@@ -9876,9 +9956,11 @@ snapshots:
dependencies:
- whatwg-encoding: 3.1.1
+ '@exodus/bytes': 1.15.1(@noble/[email protected])
+ transitivePeerDependencies:
+ - '@noble/hashes'
@@ -9922,13 +10004,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- dependencies:
- agent-base: 7.1.3
- debug: 4.4.3
- transitivePeerDependencies:
- - supports-color
-
dependencies:
agent-base: 6.0.2
@@ -10492,33 +10567,31 @@ snapshots:
- dependencies:
- cssstyle: 4.2.1
- data-urls: 5.0.0
- decimal.js: 10.5.0
- form-data: 4.0.4
- html-encoding-sniffer: 4.0.0
- http-proxy-agent: 7.0.2
- https-proxy-agent: 7.0.6
+ dependencies:
+ '@asamuzakjp/css-color': 5.1.11
+ '@asamuzakjp/dom-selector': 7.1.1
+ '@bramus/specificity': 2.4.2
+ '@csstools/css-syntax-patches-for-csstree': 1.1.4([email protected])
+ '@exodus/bytes': 1.15.1(@noble/[email protected])
+ css-tree: 3.2.1
+ data-urls: 7.0.0(@noble/[email protected])
+ decimal.js: 10.6.0
+ html-encoding-sniffer: 6.0.0(@noble/[email protected])
is-potential-custom-element-name: 1.0.1
- nwsapi: 2.2.16
- parse5: 7.2.1
- rrweb-cssom: 0.8.0
+ lru-cache: 11.5.1
+ parse5: 8.0.1
saxes: 6.0.0
symbol-tree: 3.2.4
- tough-cookie: 5.1.2
+ tough-cookie: 6.0.1
+ undici: 7.24.1
w3c-xmlserializer: 5.0.0
- webidl-conversions: 7.0.0
- whatwg-encoding: 3.1.1
- whatwg-mimetype: 4.0.0
- whatwg-url: 14.1.1
+ webidl-conversions: 8.0.1
+ whatwg-mimetype: 5.0.0
+ whatwg-url: 16.0.1(@noble/[email protected])
xml-name-validator: 5.0.0
transitivePeerDependencies:
- - bufferutil
- - supports-color
- - utf-8-validate
+ - '@noble/hashes'
@@ -10538,6 +10611,8 @@ snapshots:
+
dependencies:
buffer-equal-constant-time: 1.0.1
@@ -10671,10 +10746,10 @@ snapshots:
dependencies:
js-tokens: 4.0.0
-
+
dependencies:
yallist: 3.1.1
@@ -10703,6 +10778,8 @@ snapshots:
crypt: 0.0.2
is-buffer: 1.1.6
+
@@ -10851,8 +10928,6 @@ snapshots:
dependencies:
boolbase: 1.0.0
-
@@ -10983,9 +11058,9 @@ snapshots:
dependencies:
entities: 4.5.0
dependencies:
- entities: 4.5.0
+ entities: 8.0.0
dependencies:
@@ -11290,8 +11365,6 @@ snapshots:
bufferutil: 4.0.9
utf-8-validate: 5.0.10
-
dependencies:
queue-microtask: 1.2.3
@@ -11427,6 +11500,8 @@ snapshots:
+
dependencies:
buffer-from: 1.1.2
@@ -11595,10 +11670,16 @@ snapshots:
+
dependencies:
tldts-core: 6.1.75
+ dependencies:
+ tldts-core: 7.4.2
+
@@ -11614,13 +11695,13 @@ snapshots:
universalify: 0.2.0
url-parse: 1.5.10
dependencies:
- tldts: 6.1.75
+ tldts: 7.4.2
dependencies:
punycode: 2.3.1
@@ -11827,18 +11908,17 @@ snapshots:
-
- dependencies:
- iconv-lite: 0.6.3
dependencies:
- tr46: 5.0.0
- webidl-conversions: 7.0.0
+ '@exodus/bytes': 1.15.1(@noble/[email protected])
+ tr46: 6.0.0
+ webidl-conversions: 8.0.1
+ transitivePeerDependencies:
+ - '@noble/hashes'
dependencies:
diff --git a/apps/api/src/__tests__/snips/v2/monitor.test.ts b/apps/api/src/__tests__/snips/v2/monitor.test.ts
index 55d99a81c..1bce82397 100644
--- a/apps/api/src/__tests__/snips/v2/monitor.test.ts
+++ b/apps/api/src/__tests__/snips/v2/monitor.test.ts
@@ -252,6 +252,70 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)("/v2/monitor", () => {
2 * scrapeTimeout,
);
+ it(
+ "runs a deterministicJson-mode monitor target",
+ async () => {
+ const jsonSchema = {
+ type: "object",
+ additionalProperties: false,
+ required: ["title"],
+ properties: {
+ title: {
+ type: "string",
+ description: "The page title, verbatim.",
+ },
+ },
+ };
+
+ const create = await monitorCreateRaw(
+ {
+ name: "deterministic-json monitor",
+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },
+ targets: [
+ {
+ type: "scrape",
+ urls: [createTestIdUrl()],
+ scrapeOptions: {
+ formats: [
+ "markdown",
+ {
+ type: "deterministicJson",
+ prompt: "Extract the page title verbatim.",
+ schema: jsonSchema,
+ },
+ ],
+ },
+ },
+ ],
+ },
+ identity,
+ );
+ expect(create.statusCode).toBe(200);
+
+ const monitorId = create.body.data.id;
+ const run = await monitorRunRaw(monitorId, identity);
+ expect(run.statusCode).toBe(200);
+ const checkId = run.body.id;
+
+ let check: any;
+ for (let i = 0; i < 90; i++) {
+ const raw = await monitorCheckRaw(monitorId, checkId, identity);
+ expect(raw.statusCode).toBe(200);
+ check = raw.body.data;
+ if (["completed", "partial", "failed"].includes(check.status)) break;
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+ // Contract: deterministicJson is accepted, the reusable-json-mode extractor
+ // populates document.json via the code-sandbox, and the check flows through
+ // the JSON diff path (status "new" on first run) rather than crashing.
+ expect(["completed", "partial"]).toContain(check.status);
+ expect(check.pages[0].status).toBe("new");
+
+ await monitorDeleteRaw(monitorId, identity);
+ },
+ 2 * scrapeTimeout,
+ );
+
it(
"accepts changeTracking-json format on a monitor target",
async () => {
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 850973be0..3b6132985 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -286,6 +286,12 @@ const configSchema = z.object({
NUQ_PREFETCH_WORKER_HEARTBEAT_URL: z.string().optional(),
ZDRCLEANER_HEARTBEAT_URL: z.string().optional(),
+
+ // Deterministic JSON extraction (reusable-json-mode)
+ EXTRACT_CODEGEN_MODEL: z.string().default("gemini-3.1-flash-lite"),
+ EXTRACT_ANCHOR_MODEL: z.string().default("openai/gpt-oss-120b"),
+ EXTRACT_LIGHT_MODEL: z.string().default("openai/gpt-oss-20b"),
+ CODE_SANDBOX_URL: z.string().default("ws://code-sandbox:3001"),
});
export const config = configSchema.parse(process.env);
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 486ab237e..c65684a82 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -350,6 +350,25 @@ const jsonFormatWithOptions = z.strictObject({
export type JsonFormatWithOptions = z.output<typeof jsonFormatWithOptions>;
+// "Deterministic JSON" — same shape as json mode, but extraction is performed by
+// reusable-json-mode (a cached, reusable JS extractor run in the code-sandbox)
+// rather than a per-request LLM call. Populates document.json like json mode.
+const deterministicJsonFormatWithOptions = z.strictObject({
+ type: z.literal("deterministicJson"),
+ schema: z
+ .any()
+ .optional()
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
+ prompt: z.string().max(10000).optional(),
+});
+
+type DeterministicJsonFormatWithOptions = z.output<
+ typeof deterministicJsonFormatWithOptions
+>;
+
const changeTrackingFormatWithOptions = z.strictObject({
type: z.literal("changeTracking"),
prompt: z.string().optional(),
@@ -431,6 +450,7 @@ export type FormatObject =
| { type: "images" }
| { type: "summary" }
| JsonFormatWithOptions
+ | DeterministicJsonFormatWithOptions
| ChangeTrackingFormatWithOptions
| ScreenshotFormatWithOptions
| AttributesFormatWithOptions
@@ -605,6 +625,7 @@ const baseScrapeOptions = z.strictObject({
z.strictObject({ type: z.literal("images") }),
z.strictObject({ type: z.literal("summary") }),
jsonFormatWithOptions,
+ deterministicJsonFormatWithOptions,
changeTrackingFormatWithOptions,
screenshotFormatWithOptions,
attributesFormatWithOptions,
@@ -627,7 +648,14 @@ const baseScrapeOptions = z.strictObject({
const hasChangeTracking = x.find(f => f.type === "changeTracking");
const hasMarkdown = x.find(f => f.type === "markdown");
return !hasChangeTracking || hasMarkdown;
- }, "The changeTracking format requires the markdown format to be specified as well"),
+ }, "The changeTracking format requires the markdown format to be specified as well")
+ .refine(x => {
+ // Both json and deterministicJson populate the `json` field, so one would
+ // just clobber the other. Require choosing one.
+ const hasJson = x.some(f => f.type === "json");
+ const hasDeterministicJson = x.some(f => f.type === "deterministicJson");
+ return !(hasJson && hasDeterministicJson);
+ }, "Cannot specify both json and deterministicJson formats"),
headers: z.record(z.string(), z.string()).optional(),
includeTags: z
.string()
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.ts
index d22263724..8dc81b804 100644
--- a/apps/api/src/db/schema/public.ts
+++ b/apps/api/src/db/schema/public.ts
@@ -152,6 +152,32 @@ export const deep_researches = pgTable("deep_researches", {
options: jsonb("options"),
});
+export const deterministic_json_scripts = pgTable(
+ "deterministic_json_scripts",
+ {
+ id: uuid("id").defaultRandom().primaryKey(),
+ cache_key: text("cache_key").notNull().unique(),
+ code: text("code").notNull(),
+ url: text("url"),
+ model: text("model"),
+ cache_version: integer("cache_version"),
+ created_at: ts("created_at").notNull().defaultNow(),
+ updated_at: ts("updated_at").notNull().defaultNow(),
+ last_used_at: ts("last_used_at").notNull().defaultNow(),
+ },
+);
+
+export const deterministic_json_llm_cache = pgTable(
+ "deterministic_json_llm_cache",
+ {
+ id: uuid("id").defaultRandom().primaryKey(),
+ cache_key: text("cache_key").notNull().unique(),
+ response: text("response").notNull(),
+ created_at: ts("created_at").notNull().defaultNow(),
+ last_used_at: ts("last_used_at").notNull().defaultNow(),
+ },
+);
+
export const eb_sync = pgTable("eb-sync", {
id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),
created_at: ts("created_at").notNull().defaultNow(),
diff --git a/apps/api/src/lib/deterministicJson/config.ts b/apps/api/src/lib/deterministicJson/config.ts
new file mode 100644
index 000000000..e365a2e7f
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/config.ts
@@ -0,0 +1,25 @@
+import { config } from "../../config";
+
+export const CODEGEN_MODEL = config.EXTRACT_CODEGEN_MODEL;
+export const ANCHOR_MODEL = config.EXTRACT_ANCHOR_MODEL;
+export const LIGHT_MODEL = config.EXTRACT_LIGHT_MODEL;
+export const CODE_SANDBOX_URL = config.CODE_SANDBOX_URL;
+
+// Bump to invalidate every cached extractor at once.
+export const CACHE_VERSION = 1;
+
+export const MARKDOWN_BUDGET = 50_000;
+export const HTML_BUDGET = 40_000;
+
+export const ANCHOR_PER_BLOCK_BUDGET = 8_000;
+export const ANCHOR_TOTAL_BUDGET = 60_000;
+export const ANCHOR_MAX_BLOCKS = 16;
+export const ANCHOR_MAX_PARENTS = 8;
+
+export const CODEGEN_MAX_TOKENS = 16_384;
+export const ANCHOR_PICKER_MAX_TOKENS = 2_000;
+export const ASK_LLM_MAX_TOKENS = 2_048;
+
+export const ASK_LLM_MAX_CALLS = 50;
+
+export const SANDBOX_TIMEOUT_MS = 120_000;
diff --git a/apps/api/src/lib/deterministicJson/core/cache.ts b/apps/api/src/lib/deterministicJson/core/cache.ts
new file mode 100644
index 000000000..604a04a2b
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/core/cache.ts
@@ -0,0 +1,23 @@
+interface ExtractorEntry {
+ code: string;
+ createdAt: number;
+}
+
+interface Entry {
+ response: string;
+ createdAt: number;
+}
+
+interface ExtractorMeta {
+ url: string;
+ model: string;
+ cacheVersion: number;
+}
+
+export interface CacheBackend {
+ getExtractor(key: string): Promise<ExtractorEntry | undefined>;
+ setExtractor(key: string, code: string, meta: ExtractorMeta): Promise<void>;
+ getLlm(key: string): Promise<Entry | undefined>;
+ setLlm(key: string, response: string): Promise<void>;
+ touch?(key: string): Promise<void>;
+}
diff --git a/apps/api/src/lib/deterministicJson/core/util.ts b/apps/api/src/lib/deterministicJson/core/util.ts
new file mode 100644
index 000000000..491066be6
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/core/util.ts
@@ -0,0 +1,17 @@
+import { createHash } from "node:crypto";
+import { logger } from "../../logger";
+
+export const log = (...args: unknown[]): void => {
+ logger.debug(
+ args.map(a => (typeof a === "string" ? a : JSON.stringify(a))).join(" "),
+ { module: "deterministic-json" },
+ );
+};
+
+export function sha(input: string, chars = 32): string {
+ return createHash("sha256").update(input).digest("hex").slice(0, chars);
+}
+
+export function errorMessage(err: unknown): string {
+ return err instanceof Error ? err.message : String(err);
+}
diff --git a/apps/api/src/lib/deterministicJson/extract.ts b/apps/api/src/lib/deterministicJson/extract.ts
new file mode 100644
index 000000000..12549c92f
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/extract.ts
@@ -0,0 +1,136 @@
+import { buildAnchorContext } from "./pipeline/anchor";
+import type { CacheBackend } from "./core/cache";
+import { CACHE_VERSION, CODEGEN_MODEL, MARKDOWN_BUDGET } from "./config";
+import { CostTracking } from "../cost-tracking";
+import { generateExtractor } from "./pipeline/generate";
+import { type AskLlm, makeAskLlm } from "./llm/client";
+import { parseWithSchema } from "./pipeline/postprocess";
+import {
+ runExtractorInSandbox,
+ type SandboxRunner,
+} from "./sandbox/runExtractor";
+import { tooStrictFeedback, tooStrictSelectors } from "./html/selector-repair";
+import { errorMessage, log, sha } from "./core/util";
+
+interface ExtractArgs {
+ url: string;
+ prompt: string;
+ jsonSchema: Record<string, unknown>;
+ page: { html: string; markdown: string };
+ cache: CacheBackend;
+ sandbox: SandboxRunner;
+ costTracking: CostTracking;
+ maxAskLlmCalls?: number;
+ forceRegenerate?: boolean;
+}
+
+export async function extractDeterministicJson(
+ args: ExtractArgs,
+): Promise<unknown> {
+ const {
+ url,
+ prompt,
+ forceRegenerate,
+ cache,
+ sandbox,
+ jsonSchema,
+ costTracking,
+ } = args;
+
+ const schemaJson = JSON.stringify(jsonSchema);
+ const key = sha(
+ [CACHE_VERSION, CODEGEN_MODEL, url, schemaJson, prompt].join("\0"),
+ );
+ const meta = { url, model: CODEGEN_MODEL, cacheVersion: CACHE_VERSION };
+
+ const { html, markdown } = args.page;
+ const markdownPreview = clipMiddle(markdown.trim(), MARKDOWN_BUDGET);
+ const cached = forceRegenerate ? undefined : await cache.getExtractor(key);
+
+ let anchorHtml: string | undefined;
+
+ const generate = async (rejectionFeedback?: string): Promise<string> => {
+ anchorHtml ??= await buildAnchorContext(
+ {
+ html,
+ markdownPreview,
+ schemaJson,
+ prompt,
+ },
+ costTracking,
+ );
+ const code = await generateExtractor(
+ {
+ anchorHtml,
+ markdownPreview,
+ schemaJson,
+ prompt,
+ rejectionFeedback,
+ },
+ costTracking,
+ );
+ await cache.setExtractor(key, code, meta);
+ return code;
+ };
+
+ const run = async (code: string): Promise<unknown> => {
+ const askLlm: AskLlm = makeAskLlm(cache, costTracking, args.maxAskLlmCalls);
+ const value = await runExtractorInSandbox({
+ code,
+ html,
+ url,
+ askLlm,
+ sandbox,
+ });
+ return parseWithSchema(value, jsonSchema);
+ };
+
+ const runWithRepair = async (code: string): Promise<unknown> => {
+ const value = await run(code);
+ const broken = tooStrictSelectors(code, html);
+ if (broken.length === 0) return value;
+
+ log(
+ `extractor has ${broken.length} too-strict selector(s); regenerating once`,
+ );
+ try {
+ const repaired = await generate(tooStrictFeedback(broken));
+ if (tooStrictSelectors(repaired, html).length < broken.length) {
+ return await run(repaired);
+ }
+ } catch (err) {
+ log(
+ "selector repair failed, keeping original result:",
+ errorMessage(err),
+ );
+ }
+ await cache.setExtractor(key, code, meta);
+ return value;
+ };
+
+ let code: string;
+ if (cached) {
+ await cache.touch?.(key);
+ code = cached.code;
+ } else {
+ code = await generate();
+ }
+
+ try {
+ return await runWithRepair(code);
+ } catch (err) {
+ log("extractor run failed, regenerating once:", errorMessage(err));
+ // Let a second failure propagate; the caller warns. An empty shape here
+ // would instead read as "page had no data" and hide the real failure.
+ return await run(await generate(errorMessage(err)));
+ }
+}
+
+function clipMiddle(text: string, max: number): string {
+ if (text.length <= max) return text;
+ const marker = `\n\n[...${text.length - max} chars trimmed...]\n\n`;
+ const head = Math.ceil((max - marker.length) * 0.7);
+ return (
+ text.slice(0, head) + marker + text.slice(-(max - marker.length - head))
+ );
+}
diff --git a/apps/api/src/lib/deterministicJson/html/dom.ts b/apps/api/src/lib/deterministicJson/html/dom.ts
new file mode 100644
index 000000000..be4e3ba4b
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/html/dom.ts
@@ -0,0 +1,9 @@
+import { JSDOM } from "jsdom";
+
+export function parseDocument(html: string): Document {
+ return new JSDOM(html).window.document;
+}
+
+export const ELEMENT_NODE = 1;
+export const TEXT_NODE = 3;
+export const COMMENT_NODE = 8;
diff --git a/apps/api/src/lib/deterministicJson/html/selector-repair.ts b/apps/api/src/lib/deterministicJson/html/selector-repair.ts
new file mode 100644
index 000000000..caddefcd8
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/html/selector-repair.ts
@@ -0,0 +1,97 @@
+// Detect selectors that are too strict: they match 0 elements, yet a looser
+// variant (child combinator `>` relaxed to descendant) matches at least one, so
+// we can tell the model exactly what to fix. Only static string/template literals
+// are considered.
+import { parseDocument } from "./dom";
+
+interface TooStrictSelector {
+ selector: string;
+ loosened: string;
+ count: number;
+}
+
+// Pull literal selectors out of querySelector / querySelectorAll calls. Skips
+// template literals containing ${...} interpolation (not statically knowable).
+const SELECTOR_CALL = /querySelector(?:All)?\(\s*(['"`])((?:\\.|(?!\1).)*)\1/g;
+
+function extractSelectorLiterals(code: string): string[] {
+ const out = new Set<string>();
+ for (const match of code.matchAll(SELECTOR_CALL)) {
+ const selector = match[2];
+ if (selector && !selector.includes("${")) out.add(selector);
+ }
+ return [...out];
+}
+
+// Relax child combinators to descendant, but not a `>` inside [attr] or quotes.
+function loosenCombinators(selector: string): string {
+ let out = "";
+ let depth = 0;
+ let quote = "";
+ for (let i = 0; i < selector.length; i++) {
+ const ch = selector[i]!;
+ if (quote) {
+ out += ch;
+ if (ch === quote && selector[i - 1] !== "\\") quote = "";
+ } else if (ch === '"' || ch === "'") {
+ quote = ch;
+ out += ch;
+ } else if (ch === "[") {
+ depth++;
+ out += ch;
+ } else if (ch === "]") {
+ depth = Math.max(0, depth - 1);
+ out += ch;
+ } else if (ch === ">" && depth === 0) {
+ out = out.replace(/\s+$/, "") + " ";
+ while (i + 1 < selector.length && /\s/.test(selector[i + 1]!)) i++;
+ } else {
+ out += ch;
+ }
+ }
+ return out.trim();
+}
+
+export function tooStrictSelectors(
+ code: string,
+ html: string,
+): TooStrictSelector[] {
+ const selectors = extractSelectorLiterals(code).filter(s => s.includes(">"));
+ if (selectors.length === 0) return [];
+
+ const doc = parseDocument(html);
+ const count = (selector: string): number | null => {
+ try {
+ return doc.querySelectorAll(selector).length;
+ } catch {
+ return null; // invalid CSS - the sandbox surfaces those on its own
+ }
+ };
+
+ const out: TooStrictSelector[] = [];
+ for (const selector of selectors) {
+ if (count(selector) !== 0) continue;
+ const loosened = loosenCombinators(selector);
+ const loosenedCount = count(loosened);
+ if (loosenedCount != null && loosenedCount > 0) {
+ out.push({ selector, loosened, count: loosenedCount });
+ }
+ }
+ return out;
+}
+
+export function tooStrictFeedback(broken: TooStrictSelector[]): string {
+ return (
+ `Some selectors matched 0 elements even though the target IS on the page - a ` +
+ `child combinator (\`>\`) is breaking on a wrapper element you couldn't see in ` +
+ `the sample. Switch these to descendant combinators:\n` +
+ broken
+ .map(
+ b =>
+ `- \`${b.selector}\` matched 0; \`${b.loosened}\` matches ${b.count}`,
+ )
+ .join("\n") +
+ `\nRewrite these (and re-check your other selectors for the same issue). Leave ` +
+ `genuinely-absent fields empty - never broaden a selector just to match something.`
+ );
+}
diff --git a/apps/api/src/lib/deterministicJson/html/simplify.ts b/apps/api/src/lib/deterministicJson/html/simplify.ts
new file mode 100644
index 000000000..1421d3d93
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/html/simplify.ts
@@ -0,0 +1,274 @@
+// Shrink raw HTML into a compact, selector-faithful sample for the codegen model:
+// drop noise, keep structure and selector attributes, collapse repeats, clip to a
+// budget.
+import { COMMENT_NODE, ELEMENT_NODE, parseDocument, TEXT_NODE } from "./dom";
+
+interface SimplifyOptions {
+ maxTextLen?: number;
+ maxAttrLen?: number;
+ maxClassNames?: number;
+ keepRepeats?: number;
+ maxTotalLen?: number;
+}
+
+const DEFAULTS = {
+ maxTextLen: 160,
+ maxAttrLen: 96,
+ maxClassNames: 12,
+ keepRepeats: 3,
+ maxJsonLdLen: 4_000,
+ maxTotalLen: 40_000,
+};
+type Opts = typeof DEFAULTS;
+
+// Tags whose content never helps a selector. SVG/MATH/CANVAS are kept as empty
+// elements (their guts are huge and useless); the rest are removed entirely.
+const REMOVE_TAGS = new Set([
+ "STYLE",
+ "NOSCRIPT",
+ "TEMPLATE",
+ "LINK",
+ "BR",
+ "WBR",
+ "SOURCE",
+ "IFRAME",
+]);
+const EMPTY_TAGS = new Set(["SVG", "MATH", "CANVAS"]);
+
+const DROP_ATTRS = new Set([
+ "style",
+ "integrity",
+ "nonce",
+ "crossorigin",
+ "referrerpolicy",
+ "decoding",
+ "loading",
+ "fetchpriority",
+ "sizes",
+ "autocomplete",
+ "spellcheck",
+ "tabindex",
+]);
+const URL_ATTRS = new Set(["href", "src", "data-src", "poster", "action"]);
+const TRUNCATE_ATTRS = new Set([
+ "src",
+ "href",
+ "srcset",
+ "data-src",
+ "poster",
+ "action",
+ "content",
+ "value",
+ "alt",
+ "title",
+ "aria-label",
+]);
+const KEEP_META = new Set([
+ "title",
+ "description",
+ "author",
+ "date",
+ "pubdate",
+ "article:published_time",
+]);
+
+const truncate = (s: string, max: number): string =>
+ s.length <= max ? s : `${s.slice(0, max)}...[+${s.length - max}]`;
+
+const collapseSpace = (s: string): string => s.replace(/\s+/g, " ").trim();
+
+// Keep a URL's origin + first path segment (the strongest selector hint) and
+// drop the slug tail, rather than blindly cutting the string.
+function truncateUrl(value: string, max: number): string {
+ if (value.length <= max) return value;
+ try {
+ const u = new URL(value, "https://x.invalid");
+ const seg = u.pathname.split("/").filter(Boolean)[0];
+ const head = `${u.origin === "https://x.invalid" ? "" : u.origin}/${seg ?? ""}`;
+ if (head.length < max) return `${head}/...[+${value.length - head.length}]`;
+ } catch {
+ /* fall through */
+ }
+ return truncate(value, max);
+}
+
+const isJsonLd = (el: Element): boolean =>
+ el.tagName === "SCRIPT" &&
+ (el.getAttribute("type") ?? "").toLowerCase() === "application/ld+json";
+
+function isHidden(el: Element): boolean {
+ if (el.hasAttribute("hidden") || el.getAttribute("aria-hidden") === "true")
+ return true;
+ if (
+ el.tagName === "INPUT" &&
+ (el.getAttribute("type") ?? "").toLowerCase() === "hidden"
+ )
+ return true;
+ return /display\s*:\s*none|visibility\s*:\s*hidden/i.test(
+ el.getAttribute("style") ?? "",
+ );
+}
+
+function pruneHead(doc: Document): void {
+ const head = doc.querySelector("head");
+ if (!head) return;
+ for (const el of Array.from(head.children)) {
+ if (el.tagName === "TITLE" || isJsonLd(el)) continue;
+ if (el.tagName === "META") {
+ const key = (
+ el.getAttribute("name") ??
+ el.getAttribute("property") ??
+ ""
+ ).toLowerCase();
+ if (
+ KEEP_META.has(key) ||
+ key.startsWith("og:") ||
+ key.startsWith("twitter:")
+ )
+ continue;
+ }
+ el.remove();
+ }
+}
+
+function cleanAttributes(el: Element, opts: Opts): void {
+ for (const name of el.getAttributeNames()) {
+ const lower = name.toLowerCase();
+ if (
+ DROP_ATTRS.has(lower) ||
+ lower.startsWith("on") ||
+ /^data-(react|next|astro|svelte|vue|emotion|styled|gtm|ga|analytics|tracking|cookie|consent)/.test(
+ lower,
+ )
+ ) {
+ el.removeAttribute(name);
+ continue;
+ }
+ const value = el.getAttribute(name) ?? "";
+ if (lower === "class") {
+ const classes = [...new Set(value.split(/\s+/).filter(Boolean))].slice(
+ 0,
+ opts.maxClassNames,
+ );
+ classes.length
+ ? el.setAttribute(name, classes.join(" "))
+ : el.removeAttribute(name);
+ } else if (value.startsWith("data:")) {
+ el.setAttribute(name, "data:...");
+ } else if (URL_ATTRS.has(lower)) {
+ el.setAttribute(name, truncateUrl(value, opts.maxAttrLen));
+ } else if (TRUNCATE_ATTRS.has(lower)) {
+ el.setAttribute(name, truncate(value, opts.maxAttrLen));
+ }
+ }
+}
+
+// A signature for "same kind of element": tag + sorted classes + child tags.
+// Used to collapse long runs of list items / table rows / cards.
+function shapeKey(el: Element): string {
+ const classes = (el.getAttribute("class") ?? "")
+ .split(/\s+/)
+ .filter(Boolean)
+ .sort()
+ .join(".");
+ const childTags = Array.from(el.children)
+ .slice(0, 5)
+ .map(c => c.tagName)
+ .join(",");
+ return `${el.tagName}|${classes}|${childTags}`;
+}
+
+function collapseRepeats(parent: Element, opts: Opts): void {
+ const groups = new Map<string, Element[]>();
+ for (const el of Array.from(parent.children)) {
+ const key = shapeKey(el);
+ (groups.get(key) ?? groups.set(key, []).get(key)!).push(el);
+ }
+ for (const group of groups.values()) {
+ // Only collapse a genuinely repeating run, not 2-3 incidental look-alikes.
+ if (group.length < Math.max(5, opts.keepRepeats + 1)) continue;
+ const extra = group.slice(opts.keepRepeats);
+ const tag = group[0]!.tagName.toLowerCase();
+ extra[0]!.parentNode?.insertBefore(
+ parent.ownerDocument.createComment(
+ ` ${extra.length} more <${tag}> with the same shape `,
+ ),
+ extra[0]!,
+ );
+ for (const el of extra) el.remove();
+ }
+}
+
+function walk(node: Element, opts: Opts): void {
+ for (const child of Array.from(node.childNodes)) {
+ if (
+ child.nodeType === COMMENT_NODE ||
+ (child.nodeType !== ELEMENT_NODE && child.nodeType !== TEXT_NODE)
+ ) {
+ child.remove();
+ } else if (child.nodeType === TEXT_NODE) {
+ const text = collapseSpace(child.textContent ?? "");
+ text
+ ? (child.textContent = truncate(text, opts.maxTextLen))
+ : child.remove();
+ } else {
+ const el = child as Element;
+ if (isHidden(el) || REMOVE_TAGS.has(el.tagName)) {
+ el.remove();
+ } else if (el.tagName === "SCRIPT") {
+ if (isJsonLd(el))
+ el.textContent = truncate(
+ collapseSpace(el.textContent ?? ""),
+ opts.maxJsonLdLen,
+ );
+ else el.remove();
+ } else if (EMPTY_TAGS.has(el.tagName)) {
+ cleanAttributes(el, opts);
+ el.textContent = "";
+ } else {
+ cleanAttributes(el, opts);
+ walk(el, opts);
+ }
+ }
+ }
+ collapseRepeats(node, opts);
+}
+
+// Drop attribute-less <div>/<span> wrappers that hold a single element child -
+// they add depth without selector value.
+function flattenWrappers(el: Element): void {
+ for (const child of Array.from(el.children)) flattenWrappers(child);
+ if (
+ (el.tagName !== "DIV" && el.tagName !== "SPAN") ||
+ el.getAttributeNames().length
+ )
+ return;
+ const childElements = Array.from(el.children);
+ const hasOwnText = Array.from(el.childNodes).some(
+ n => n.nodeType === TEXT_NODE && (n.textContent ?? "").trim(),
+ );
+ if (childElements.length === 1 && !hasOwnText)
+ el.replaceWith(childElements[0]!);
+}
+
+function clipMiddle(text: string, max: number): string {
+ if (text.length <= max) return text;
+ const marker = `\n<!-- ...${text.length - max} chars trimmed... -->\n`;
+ const head = Math.ceil((max - marker.length) * 0.65);
+ return (
+ text.slice(0, head) + marker + text.slice(-(max - marker.length - head))
+ );
+}
+
+export function simplifyHtml(
+ html: string,
+ options: SimplifyOptions = {},
+): string {
+ const opts = { ...DEFAULTS, ...options };
+ const doc = parseDocument(html);
+ pruneHead(doc);
+ walk(doc.documentElement, opts);
+ flattenWrappers(doc.documentElement);
+ const out = doc.documentElement.outerHTML.replace(/>\s+</g, "><");
+ return clipMiddle(out, opts.maxTotalLen);
+}
diff --git a/apps/api/src/lib/deterministicJson/llm/client.ts b/apps/api/src/lib/deterministicJson/llm/client.ts
new file mode 100644
index 000000000..c0d339ab7
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/llm/client.ts
@@ -0,0 +1,247 @@
+import { generateText } from "ai";
+import { jsonrepair } from "jsonrepair";
+import { getModel } from "../../generic-ai";
+import { CostTracking } from "../../cost-tracking";
+import {
+ ANCHOR_MODEL,
+ ANCHOR_PICKER_MAX_TOKENS,
+ ASK_LLM_MAX_CALLS,
+ ASK_LLM_MAX_TOKENS,
+ CODEGEN_MAX_TOKENS,
+ CODEGEN_MODEL,
+ LIGHT_MODEL,
+} from "../config";
+import { type CacheBackend } from "../core/cache";
+import { modelPrices } from "../../extract/usage/model-prices";
+import { ASK_LLM_SYSTEM, type ChatMessage } from "./prompts";
+import { errorMessage, log, sha } from "../core/util";
+
+function record(
+ costTracking: CostTracking,
+ role: string,
+ provider: string,
+ model: string,
+ usage: { inputTokens?: number; outputTokens?: number } | undefined,
+): void {
+ const input = usage?.inputTokens ?? 0;
+ const output = usage?.outputTokens ?? 0;
+ const key = `${provider}/${model}`;
+ const price = (
+ modelPrices as Record<
+ string,
+ { input_cost_per_token?: number; output_cost_per_token?: number }
+ >
+ )[key];
+ if (!price) log(`no pricing for ${key}`);
+ const cost =
+ input * (price?.input_cost_per_token ?? 0) +
+ output * (price?.output_cost_per_token ?? 0);
+ costTracking.addCall({
+ type: "other",
+ model: key,
+ cost,
+ tokens: { input, output },
+ metadata: { module: "deterministic-json", role },
+ });
+}
+
+interface Generation {
+ content: string;
+ truncated: boolean;
+}
+
+export async function generateCode(
+ messages: ChatMessage[],
+ costTracking: CostTracking,
+): Promise<Generation> {
+ let res;
+ try {
+ res = await generateText({
+ model: getModel(CODEGEN_MODEL, "vertex"),
+ messages,
+ temperature: 0,
+ maxOutputTokens: CODEGEN_MAX_TOKENS,
+ });
+ } catch (err) {
+ throw new Error(`codegen API call failed: ${errorMessage(err)}`);
+ }
+ record(costTracking, "codegen", "vertex", CODEGEN_MODEL, res.usage);
+ return { content: res.text ?? "", truncated: res.finishReason === "length" };
+}
+
+export async function pickSnippets(
+ messages: ChatMessage[],
+ costTracking: CostTracking,
+): Promise<string[]> {
+ let res;
+ try {
+ res = await generateText({
+ model: getModel(ANCHOR_MODEL, "groq"),
+ messages,
+ temperature: 0,
+ maxOutputTokens: ANCHOR_PICKER_MAX_TOKENS,
+ });
+ } catch (err) {
+ throw new Error(`anchor-picker API call failed: ${errorMessage(err)}`);
+ }
+ record(costTracking, "anchor", "groq", ANCHOR_MODEL, res.usage);
+ return parseSnippets(res.text?.trim() ?? "");
+}
+
+function parseSnippets(raw: string): string[] {
+ const trimmed = raw
+ .replace(/^```(?:json)?\s*/i, "")
+ .replace(/```\s*$/i, "")
+ .trim();
+ let value: unknown;
+ try {
+ value = JSON.parse(jsonrepair(trimmed));
+ } catch {
+ return [];
+ }
+ const list = Array.isArray(value)
+ ? value
+ : ((value as { snippets?: unknown })?.snippets ?? []);
+ return Array.isArray(list)
+ ? list.filter((s): s is string => typeof s === "string")
+ : [];
+}
+
+export type AskLlm = (prompt: string, schema?: unknown) => Promise<unknown>;
+
+const ASK_LLM_RETRIES = 3;
+
+function normalizePlainText(s: string): string {
+ return /^(null|none|n\/a|undefined)$/i.test(s.trim()) ? "" : s;
+}
+
+function parseJsonLoose(raw: string): unknown {
+ try {
+ return JSON.parse(raw);
+ } catch {
+ return JSON.parse(jsonrepair(raw));
+ }
+}
+
+// askLlm function for one extraction run: caches responses, dedupes identical
+// in-flight calls, caps real API calls at maxCalls. Any failure resolves to null
+// so one bad inner call can't sink the whole extraction.
+export function makeAskLlm(
+ cache: CacheBackend,
+ costTracking: CostTracking,
+ maxCalls = ASK_LLM_MAX_CALLS,
+): AskLlm {
+ let apiCalls = 0;
+ const inFlight = new Map<string, Promise<unknown>>();
+
+ const ask = (prompt: string, schema?: unknown): Promise<unknown> => {
+ const key =
+ schema == null ? prompt : `${prompt}\0${JSON.stringify(schema)}`;
+ const existing = inFlight.get(key);
+ if (existing) return existing;
+ const pending = askOnce(prompt, schema).catch(err => {
+ log(`askLlm failed, returning null: ${errorMessage(err)}`);
+ return null;
+ });
+ inFlight.set(key, pending);
+ return pending;
+ };
+
+ async function askOnce(prompt: string, schema: unknown): Promise<unknown> {
+ const structured = schema != null;
+ if (structured && (typeof schema !== "object" || Array.isArray(schema))) {
+ throw new Error("askLlm schema must be a JSON Schema object");
+ }
+
+ const objectSchema =
+ structured && (schema as { type?: unknown }).type === "object";
+ const wrap = structured && !objectSchema;
+
+ const userContent = !structured
+ ? prompt
+ : wrap
+ ? `${prompt}\n\nReturn ONLY a JSON object {"value": V} where V matches this JSON Schema (use null if unknown):\n\n${JSON.stringify(schema)}`
+ : `${prompt}\n\n### JSON Schema (match this shape; use null for unknown)\n\n${JSON.stringify(schema)}`;
+
+ const finalize = (raw: string): unknown => {
+ const parsed = parseJsonLoose(raw);
+ let value: unknown = parsed;
+ if (
+ wrap &&
+ parsed &&
+ typeof parsed === "object" &&
+ !Array.isArray(parsed) &&
+ "value" in parsed
+ ) {
+ value = (parsed as Record<string, unknown>).value;
+ }
+ return typeof value === "string" ? normalizePlainText(value) : value;
+ };
+
+ const cacheKey = sha(
+ `${LIGHT_MODEL}\0${ASK_LLM_SYSTEM}\0${userContent}\0${structured ? "json" : "text"}`,
+ );
+ const cached = await cache.getLlm(cacheKey);
+ if (cached) {
+ try {
+ const value = structured
+ ? finalize(cached.response)
+ : normalizePlainText(cached.response);
+ await cache.touch?.(cacheKey);
+ return value;
+ } catch (err) {
+ // Fall through to a fresh call; otherwise the parse error sinks the
+ // whole ask to null via ask()'s catch.
+ log(
+ `ignoring unparseable cached askLlm response: ${errorMessage(err)}`,
+ );
+ }
+ }
+
+ let lastError = "";
+ for (let attempt = 1; attempt <= ASK_LLM_RETRIES; attempt++) {
+ if (apiCalls >= maxCalls)
+ throw new Error(`askLlm call budget exhausted (${maxCalls})`);
+ apiCalls++;
+
+ let raw = "";
+ try {
+ const res = await generateText({
+ model: getModel(LIGHT_MODEL, "groq"),
+ temperature: 0,
+ maxOutputTokens: ASK_LLM_MAX_TOKENS,
+ messages: [
+ { role: "system", content: ASK_LLM_SYSTEM },
+ { role: "user", content: userContent },
+ ],
+ });
+ raw = res.text?.trim() ?? "";
+ record(costTracking, "askLlm", "groq", LIGHT_MODEL, res.usage);
+ } catch (err) {
+ lastError = errorMessage(err);
+ continue;
+ }
+ if (!raw) {
+ lastError = "empty response";
+ continue;
+ }
+
+ let value: unknown;
+ try {
+ value = structured ? finalize(raw) : normalizePlainText(raw);
+ } catch (err) {
+ lastError = `parse: ${errorMessage(err)}`;
+ continue;
+ }
+
+ await cache.setLlm(cacheKey, raw);
+ return value;
+ }
+
+ throw new Error(
+ `askLlm failed after ${ASK_LLM_RETRIES} attempts: ${lastError}`,
+ );
+ }
+
+ return ask;
+}
diff --git a/apps/api/src/lib/deterministicJson/llm/prompts.ts b/apps/api/src/lib/deterministicJson/llm/prompts.ts
new file mode 100644
index 000000000..96b8a0ba4
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/llm/prompts.ts
@@ -0,0 +1,96 @@
+export interface ChatMessage {
+ role: "system" | "user";
+ content: string;
+}
+
+// System prompt for inner askLlm calls (the cheap model the extractor calls
+// at runtime).
+export const ASK_LLM_SYSTEM = [
+ "You are a text-processing function in a data pipeline.",
+ "Return only the requested value: no preamble, no markdown.",
+ "You may summarize, classify, translate, or reformat the supplied text,",
+ "but stay grounded in it: never invent facts or do outside lookups",
+ 'If the text lacks what you need, return null (or "" / [] per the schema).',
+].join(" ");
+
+// Phase 1: pick short verbatim snippets from the markdown that sit next to the
+// data the extractor will need. Each snippet is later located in the raw HTML so
+// we can hand the codegen model a tight window instead of the whole page.
+const ANCHOR_PICKER_SYSTEM = `You are a planning step in a web-extraction pipeline. Given a page's markdown, a target JSON Schema, and a task, pick short text snippets from the markdown that sit next to the data the extractor will need to read.
+
+Rules:
+- Each snippet must appear VERBATIM in the markdown.
+- Prefer short (5-80 char) distinctive phrases: a label ("Balance:"), a value next to a label, a heading, a unique table cell.
+- Pick one snippet near each field the schema asks for. For a list/array schema, pick 2-4 snippets that hit DIFFERENT items.
+- Avoid generic phrases ("Home", "Menu", "Read more") and very long quotes.
+- Skip fields whose data clearly is not on the page.
+
+Return only JSON of the form {"snippets": ["...", "..."]} - no markdown fences, no commentary.`;
+
+// Phase 2: write the reusable extractor. Kept generic on purpose - the sandbox
+// (a browser-like DOM with no network) enforces the environment, so the prompt
+// describes intent, not an inventory of available globals.
+const EXTRACTOR_SYSTEM = `You write ONE reusable JavaScript function that extracts structured data from a web page.
+
+ async function extract(doc, askLlm) { ... }
+
+- \`doc\` is a standard DOM Document: querySelector / querySelectorAll, document.evaluate (XPath), textContent, getAttribute, etc.
+- \`askLlm(prompt)\` returns a string; \`askLlm(prompt, jsonSchema)\` returns a value of that shape. It can fail and return null, so guard its result.
+
+This function is generated once from ONE sample page, cached, then re-run on OTHER pages of the same shape. Therefore:
+
+- Read every value from the DOM at runtime. NEVER return a literal you saw in the sample page - the values differ on other pages. Literals are only allowed in selectors, attribute names, regexes, and the returned object's keys.
+- Always return the schema's shape, even on 404 / login / empty pages: use null or [] for anything absent. NEVER throw. A fabricated value is worse than an empty one.
+- When a value is present, coerce it to the schema's type (e.g. strip non-numerics for numbers). Keep dates as the page displays them unless the task asks for a specific format.
+
+Selectors:
+- Anchor on stable, meaningful things: labels, semantic attributes (itemprop, role, data-*), heading text. Avoid fragile position (:nth-child, sibling chains).
+- Prefer descendant combinators over child combinators (\`>\`); an unseen wrapper element breaks \`>\`. A selector that matches 0 elements is usually too strict, not proof the data is absent.
+- Read URLs from href/src, not from link text.
+- Only standard CSS selectors work. To match by visible text, select structurally then filter in JS, e.g. \`[...doc.querySelectorAll('tr')].find(r => r.textContent.includes('Total'))\`.
+
+askLlm:
+- Use it for what selectors cannot do - summarize, classify, translate, disambiguate by meaning. It transforms the text you pass it (no outside lookups like currency conversion; returns null when the text lacks the answer), so pass the DOM text it needs - for a summary, the article body, not a bare title or label. If a field has a fixed value set, include it in the prompt.
+- When you call askLlm once per item over a list, issue the calls concurrently with Promise.all rather than awaiting one at a time.
+
+Return only the function source, beginning with \`async function extract\`.`;
+
+export function buildAnchorPickerMessages(args: {
+ userPrompt: string;
+ schemaJson: string;
+ markdownPreview: string;
+}): ChatMessage[] {
+ return [
+ { role: "system", content: ANCHOR_PICKER_SYSTEM },
+ {
+ role: "user",
+ content: `### Page markdown\n\n${args.markdownPreview}\n\n### Target JSON Schema\n\n${args.schemaJson}\n\n### Task\n\n${args.userPrompt}\n\n### Output\n\nReturn {"snippets": [...]} with short verbatim fragments from the markdown above.`,
+ },
+ ];
+}
+
+export function buildExtractorMessages(args: {
+ userPrompt: string;
+ schemaJson: string;
+ markdownPreview: string;
+ anchorHtml: string;
+ rejectionFeedback?: string;
+}): ChatMessage[] {
+ // Bulky reference first, the task last - long-context models answer better
+ // when the question follows the documents it refers to.
+ const markdown = args.markdownPreview.trim()
+ ? `### Page markdown (one sample page - the values shown are NOT your output)\n\n${args.markdownPreview}\n\n`
+ : "";
+
+ const retry = args.rejectionFeedback?.trim()
+ ? `\n\n### Your previous attempt was rejected\n\n${args.rejectionFeedback}\n\nRewrite the function to fix this. Every value must still come from a runtime DOM read or askLlm, and the result must match the schema exactly.`
+ : "";
+
+ return [
+ { role: "system", content: EXTRACTOR_SYSTEM },
+ {
+ role: "user",
+ content: `${markdown}### HTML snippets around the target fields\n\n${args.anchorHtml}\n\n### Target JSON Schema\n\n${args.schemaJson}\n\n### Task\n\n${args.userPrompt}${retry}`,
+ },
+ ];
+}
diff --git a/apps/api/src/lib/deterministicJson/pipeline/anchor.ts b/apps/api/src/lib/deterministicJson/pipeline/anchor.ts
new file mode 100644
index 000000000..167200724
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/pipeline/anchor.ts
@@ -0,0 +1,269 @@
+// Build focused HTML context: locate each picked snippet in the raw HTML, climb
+// to a budgeted ancestor subtree, simplify it, and stitch the windows together,
+// with a document-unique selector hint per snippet. Falls back to simplifying the
+// whole page when no snippet lands.
+import {
+ ANCHOR_MAX_BLOCKS,
+ ANCHOR_MAX_PARENTS,
+ ANCHOR_PER_BLOCK_BUDGET,
+ ANCHOR_TOTAL_BUDGET,
+ HTML_BUDGET,
+} from "../config";
+import { CostTracking } from "../../cost-tracking";
+import { parseDocument } from "../html/dom";
+import { pickSnippets } from "../llm/client";
+import { buildAnchorPickerMessages } from "../llm/prompts";
+import { simplifyHtml } from "../html/simplify";
+import { errorMessage, log } from "../core/util";
+
+export async function buildAnchorContext(
+ args: {
+ html: string;
+ markdownPreview: string;
+ schemaJson: string;
+ prompt: string;
+ },
+ costTracking: CostTracking,
+): Promise<string> {
+ let snippets: string[] = [];
+ try {
+ snippets = await pickSnippets(
+ buildAnchorPickerMessages({
+ userPrompt: args.prompt,
+ schemaJson: args.schemaJson,
+ markdownPreview: args.markdownPreview,
+ }),
+ costTracking,
+ );
+ log(`anchor-picker chose ${snippets.length} snippet(s)`);
+ } catch (err) {
+ log(
+ "anchor-picker failed, falling back to whole-page simplify:",
+ errorMessage(err),
+ );
+ }
+
+ const built = snippets.length ? buildAnchorHtml(args.html, snippets) : "";
+ if (built.trim()) return built;
+
+ log("no anchors landed; simplifying whole page");
+ return simplifyHtml(args.html, { maxTotalLen: HTML_BUDGET });
+}
+
+const normalize = (s: string): string => s.replace(/\s+/g, " ").trim();
+
+// The picker reads markdown, so snippets often carry markdown decoration
+// ("- **Foo**", "## Bar", "`baz`") that never appears in rendered HTML text.
+// Strip it so the bare text matches the DOM.
+function stripMarkdown(s: string): string {
+ return s
+ .replace(/^\s*(?:[-*+]\s+|\d+\.\s+|>\s+|#+\s+)/, "")
+ .replace(/\*\*([^*]+)\*\*/g, "$1")
+ .replace(/__([^_]+)__/g, "$1")
+ .replace(/(?<!\*)\*([^*\n]+)\*(?!\*)/g, "$1")
+ .replace(/`([^`]+)`/g, "$1")
+ .replace(/!?\[([^\]]+)\]\([^)]*\)/g, "$1")
+ .trim();
+}
+
+// Deepest element whose normalized text contains the needle - the leaf that
+// actually holds the snippet, so we can climb from a tight starting point.
+function findDeepestMatch(root: Element, needle: string): Element | null {
+ let best: Element | null = null;
+ const visit = (el: Element): boolean => {
+ if (!normalize(el.textContent ?? "").includes(needle)) return false;
+ let inChild = false;
+ for (const child of Array.from(el.children))
+ if (visit(child)) inChild = true;
+ if (!inChild) best = el;
+ return true;
+ };
+ visit(root);
+ return best;
+}
+
+function ownText(el: Element): string {
+ let out = "";
+ for (const child of Array.from(el.childNodes)) {
+ out +=
+ child.nodeType === 3
+ ? (child.textContent ?? "")
+ : " " + (child.textContent ?? "");
+ }
+ return out;
+}
+
+// How many distinct (non-nested) elements own text containing the needle -
+// signals when a snippet is too generic to anchor on.
+function countMatches(root: Element, needle: string): number {
+ const matches = Array.from(root.querySelectorAll("*")).filter(el =>
+ normalize(ownText(el)).includes(needle),
+ );
+ return matches.filter(el => !matches.some(o => o !== el && o.contains(el)))
+ .length;
+}
+
+function climbToBudget(anchor: Element): Element {
+ let current = anchor;
+ for (let i = 0; i < ANCHOR_MAX_PARENTS; i++) {
+ const parent = current.parentElement;
+ if (!parent || parent.tagName === "BODY" || parent.tagName === "HTML")
+ break;
+ if ((parent.outerHTML ?? "").length > ANCHOR_PER_BLOCK_BUDGET) break;
+ current = parent;
+ }
+ return current;
+}
+
+const STABLE_ATTRS = ["itemprop", "data-testid", "data-test", "role", "name"];
+const cssEscape = (s: string): string => s.replace(/([^a-zA-Z0-9_-])/g, "\\$1");
+
+function isStableClass(c: string): boolean {
+ if (/^[a-z-]+-\[?[\d.]/.test(c)) return false; // utility e.g. h-[14px], pr-1.5
+ if (/[A-Za-z]_[A-Za-z0-9]{4,}$/.test(c)) return false; // CSS-module hash
+ if (/^(is-|has-|js-|active|selected|hidden|open)/.test(c)) return false; // state
+ return c.length >= 2;
+}
+
+// A selector fragment that targets this element by a stable hook (id, semantic
+// attribute, or stable class), or null. Not checked for uniqueness on its own.
+function ownHook(el: Element): string | null {
+ const tag = el.tagName.toLowerCase();
+ const id = el.getAttribute("id");
+ if (id && /^[A-Za-z][\w-]*$/.test(id)) return `#${id}`;
+ for (const attr of STABLE_ATTRS) {
+ const v = el.getAttribute(attr);
+ if (v) return `${tag}[${attr}="${v}"]`;
+ }
+ const classes = Array.from(el.classList).filter(isStableClass);
+ if (classes.length)
+ return tag + classes.map(c => `.${cssEscape(c)}`).join("");
+ return null;
+}
+
+// Find a CSS selector that returns exactly this element on the whole document,
+// or null. Tries the element's own hook first, then scopes by an ancestor's hook
+// (page text often sits in attribute-less spans whose ancestor carries the id).
+function uniqueSelector(doc: Document, el: Element): string | null {
+ const matchesOne = (sel: string): boolean => {
+ try {
+ return (
+ doc.querySelectorAll(sel).length === 1 && doc.querySelector(sel) === el
+ );
+ } catch {
+ return false;
+ }
+ };
+
+ const tag = el.tagName.toLowerCase();
+
+ // 1. The element's own hook, narrowing the class chain until it's unique.
+ const id = el.getAttribute("id");
+ if (id && /^[A-Za-z][\w-]*$/.test(id) && matchesOne(`#${id}`))
+ return `#${id}`;
+ for (const attr of STABLE_ATTRS) {
+ const v = el.getAttribute(attr);
+ if (v && matchesOne(`${tag}[${attr}="${v}"]`))
+ return `${tag}[${attr}="${v}"]`;
+ }
+ const classes = Array.from(el.classList).filter(isStableClass);
+ for (let n = 1; n <= classes.length; n++) {
+ const sel =
+ tag +
+ classes
+ .slice(0, n)
+ .map(c => `.${cssEscape(c)}`)
+ .join("");
+ if (matchesOne(sel)) return sel;
+ }
+
+ // 2. Scope by an ancestor's hook: `<ancestorHook> <leafTag(.class)>`.
+ const leafForms = [
+ tag,
+ classes[0] ? `${tag}.${cssEscape(classes[0])}` : "",
+ ].filter(Boolean);
+ let ancestor = el.parentElement;
+ for (
+ let depth = 0;
+ ancestor && depth < 5;
+ depth++, ancestor = ancestor.parentElement
+ ) {
+ const hook = ownHook(ancestor);
+ if (!hook) continue;
+ for (const leaf of leafForms) {
+ if (matchesOne(`${hook} ${leaf}`)) return `${hook} ${leaf}`;
+ }
+ }
+ return null;
+}
+
+function anchorsHeader(hints: { snippet: string; selector: string }[]): string {
+ if (!hints.length) return "";
+ const lines = hints.map(
+ h => ` ${JSON.stringify(h.snippet)} -> ${h.selector}`,
+ );
+ return (
+ `<!-- VERIFIED SELECTORS: each returns exactly one element on the full page.\n` +
+ ` Your code runs against the whole document, so prefer these for the\n` +
+ ` fields they name. -->\n` +
+ lines.join("\n") +
+ "\n\n"
+ );
+}
+
+// simplifyHtml serializes a full <html> shell; we only want the body contents.
+function unwrapBody(html: string): string {
+ const open = html.match(/<body[^>]*>/i);
+ if (!open) return html;
+ const start = (open.index ?? 0) + open[0].length;
+ const end = html.lastIndexOf("</body>");
+ return end < start ? html.slice(start) : html.slice(start, end);
+}
+
+function buildAnchorHtml(rawHtml: string, snippets: string[]): string {
+ const doc = parseDocument(rawHtml);
+ const root = (doc.body ?? doc.documentElement) as Element;
+
+ const emittedSubtrees = new Set<Element>();
+ const seenHtml = new Set<string>();
+ const hints: { snippet: string; selector: string }[] = [];
+ let combined = "";
+ let blocks = 0;
+
+ for (const raw of snippets) {
+ const snippet = normalize(stripMarkdown(raw));
+ if (!snippet) continue;
+
+ const anchor = findDeepestMatch(root, snippet);
+ if (!anchor) continue;
+
+ const selector = uniqueSelector(doc, anchor);
+ if (selector) hints.push({ snippet: snippet.slice(0, 80), selector });
+
+ // Skip if a window we already emitted contains this anchor.
+ if ([...emittedSubtrees].some(prior => prior.contains(anchor))) continue;
+ if (blocks >= ANCHOR_MAX_BLOCKS) continue;
+
+ const subtree = climbToBudget(anchor);
+ const simplified = unwrapBody(
+ simplifyHtml(subtree.outerHTML ?? "", {
+ maxTotalLen: ANCHOR_PER_BLOCK_BUDGET,
+ }),
+ ).trim();
+ if (!simplified || seenHtml.has(simplified)) continue;
+
+ const matches = countMatches(root, snippet);
+ const note = matches > 1 ? ` (${matches} similar matches on the page)` : "";
+ const piece = `<!-- anchor ${blocks + 1}: ${JSON.stringify(snippet.slice(0, 120))}${note} -->\n${simplified}\n`;
+ if (combined.length + piece.length > ANCHOR_TOTAL_BUDGET && blocks > 0)
+ break;
+
+ emittedSubtrees.add(subtree);
+ seenHtml.add(simplified);
+ combined += piece;
+ blocks += 1;
+ }
+
+ log(`anchor-html: ${blocks} window(s), ${hints.length} verified selector(s)`);
+ return combined ? anchorsHeader(hints) + combined : "";
+}
diff --git a/apps/api/src/lib/deterministicJson/pipeline/generate.ts b/apps/api/src/lib/deterministicJson/pipeline/generate.ts
new file mode 100644
index 000000000..b40f9e65d
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/pipeline/generate.ts
@@ -0,0 +1,80 @@
+// Prompt the codegen model for the extractor, then statically validate it with
+// the TypeScript parser before we bother forking a sandbox: clean the source
+// down to the extract function, check it parses and has the right shape, and
+// reject references to globals the sandbox can't provide. Each rejection is fed
+// back as feedback for one regeneration. Runtime problems (a selector that
+// throws, a logic bug) still surface only on execution.
+import { generateCode } from "../llm/client";
+import { CostTracking } from "../../cost-tracking";
+import { buildExtractorMessages } from "../llm/prompts";
+import { errorMessage, log } from "../core/util";
+import {
+ cleanGeneratedCode,
+ formatGeneratedCodeIssues,
+ validateGeneratedExtractor,
+ type GeneratedCodeIssue,
+} from "./validate";
+
+const MAX_ATTEMPTS = 2;
+
+export async function generateExtractor(
+ args: {
+ anchorHtml: string;
+ markdownPreview: string;
+ schemaJson: string;
+ prompt: string;
+ rejectionFeedback?: string;
+ },
+ costTracking: CostTracking,
+): Promise<string> {
+ let feedback = args.rejectionFeedback;
+ let lastIssues: GeneratedCodeIssue[] = [];
+
+ for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+ const { content, truncated } = await generateCode(
+ buildExtractorMessages({
+ userPrompt: args.prompt,
+ schemaJson: args.schemaJson,
+ markdownPreview: args.markdownPreview,
+ anchorHtml: args.anchorHtml,
+ rejectionFeedback: feedback,
+ }),
+ costTracking,
+ );
+
+ if (truncated) {
+ throw new Error("codegen output was truncated; raise CODEGEN_MAX_TOKENS");
+ }
+
+ let code = "";
+ let issues: GeneratedCodeIssue[] = [];
+ try {
+ code = cleanGeneratedCode(content);
+ issues = validateGeneratedExtractor(code);
+ } catch (err) {
+ issues = [
+ {
+ field: "source",
+ reason: errorMessage(err),
+ excerpt: content.replace(/\s+/g, " ").slice(0, 180),
+ },
+ ];
+ }
+
+ if (issues.length === 0) {
+ log(`extractor generated (attempt ${attempt}, ${code.length} chars)`);
+ return code;
+ }
+
+ lastIssues = issues;
+ feedback = formatGeneratedCodeIssues(issues);
+ log(
+ `rejecting extractor attempt ${attempt}: ${issues.length} issue(s)\n${feedback}`,
+ );
+ }
+
+ throw new Error(
+ "extractor failed validation after retry:\n" +
+ formatGeneratedCodeIssues(lastIssues),
+ );
+}
diff --git a/apps/api/src/lib/deterministicJson/pipeline/postprocess.ts b/apps/api/src/lib/deterministicJson/pipeline/postprocess.ts
new file mode 100644
index 000000000..985676b2d
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/pipeline/postprocess.ts
@@ -0,0 +1,199 @@
+type JsonObject = Record<string, unknown>;
+
+const isObject = (v: unknown): v is JsonObject =>
+ typeof v === "object" && v !== null && !Array.isArray(v);
+
+// Reconcile an extractor's raw output with the target JSON schema: remap key
+// casing, fill/omit nulls, strip junk characters, and dedupe over-extracted list
+// items. Returns the cleaned value shaped to the schema.
+export function parseWithSchema(value: unknown, jsonSchema: unknown): unknown {
+ const remapped = remapKeys(value, jsonSchema);
+ const coerced = coerceNulls(remapped, jsonSchema);
+ const cleaned = sanitizeStrings(coerced);
+ return dedupeArrays(cleaned);
+}
+
+const normalizeKey = (k: string): string =>
+ k.replace(/[^a-z0-9]/gi, "").toLowerCase();
+
+function remapKeys(value: unknown, schema: unknown): unknown {
+ if (!isObject(schema)) return value;
+
+ if (Array.isArray(value)) {
+ return value.map(v => remapKeys(v, (schema as { items?: unknown }).items));
+ }
+ if (!isObject(value)) return value;
+
+ const props = (schema as { properties?: JsonObject }).properties;
+ if (!props) return value;
+
+ const canonical = new Map(Object.keys(props).map(k => [normalizeKey(k), k]));
+ const out: JsonObject = {};
+ for (const [key, v] of Object.entries(value)) {
+ const target =
+ key in props ? key : (canonical.get(normalizeKey(key)) ?? key);
+ out[target] = remapKeys(v, props[target]);
+ }
+ return out;
+}
+
+function schemaAllowsNull(schema: JsonObject): boolean {
+ const t = schema.type;
+ if (
+ t === "null" ||
+ (Array.isArray(t) && t.includes("null")) ||
+ schema.nullable === true
+ ) {
+ return true;
+ }
+ for (const key of ["anyOf", "oneOf"] as const) {
+ const branches = schema[key];
+ if (
+ Array.isArray(branches) &&
+ branches.some(b => isObject(b) && schemaAllowsNull(b))
+ ) {
+ return true;
+ }
+ }
+ return false;
+}
+
+function typeDefault(schema: JsonObject): unknown {
+ const t = schema.type;
+ const primary = Array.isArray(t) ? t.find(x => x !== "null") : t;
+ switch (primary) {
+ case "string":
+ return "";
+ case "number":
+ case "integer":
+ return 0;
+ case "boolean":
+ return false;
+ case "array":
+ return [];
+ case "object": {
+ const props = (schema.properties ?? {}) as JsonObject;
+ const out: JsonObject = {};
+ for (const key of (schema.required as string[] | undefined) ?? []) {
+ const sub = props[key];
+ out[key] =
+ isObject(sub) && !schemaAllowsNull(sub) ? typeDefault(sub) : null;
+ }
+ return out;
+ }
+ default:
+ return null;
+ }
+}
+
+// `required` tracks whether the current value sits in a required slot. We only
+// fabricate a default for a required, non-nullable null; an optional null is
+// dropped so we never emit a fake value for data that simply isn't on the page.
+function coerceNulls(
+ value: unknown,
+ schema: unknown,
+ required = true,
+): unknown {
+ if (!isObject(schema)) return value;
+
+ if (value === null) {
+ if (schemaAllowsNull(schema) || !required) return null;
+ return typeDefault(schema);
+ }
+
+ if (Array.isArray(value)) {
+ const items = (schema as { items?: unknown }).items;
+ // An item present in the array is a real value: coerce its required subfields.
+ return isObject(items)
+ ? value.map(v => coerceNulls(v, items, true))
+ : value;
+ }
+
+ if (isObject(value)) {
+ const props = (schema as { properties?: JsonObject }).properties;
+ if (!props) return value;
+ const requiredKeys = new Set(
+ (schema as { required?: string[] }).required ?? [],
+ );
+ const out: JsonObject = {};
+ for (const [key, v] of Object.entries(value)) {
+ if (!(key in props)) {
+ out[key] = v;
+ continue;
+ }
+ const sub = props[key];
+ // Optional field the model returned as null: omit it so a non-nullable
+ // optional field still validates, instead of fabricating a default.
+ if (
+ v === null &&
+ !requiredKeys.has(key) &&
+ !(isObject(sub) && schemaAllowsNull(sub))
+ ) {
+ continue;
+ }
+ out[key] = coerceNulls(v, sub, requiredKeys.has(key));
+ }
+ // Fill in required keys the model omitted entirely.
+ for (const key of requiredKeys) {
+ if (!(key in out))
+ out[key] = isObject(props[key])
+ ? typeDefault(props[key] as JsonObject)
+ : null;
+ }
+ return out;
+ }
+ return value;
+}
+
+// Drop exact-duplicate OBJECTS from arrays (a common over-extraction artifact -
+// e.g. a list selector that matches each card twice, or triplicated rows).
+// Scalar duplicates are left alone since repeated primitives can be legitimate.
+function dedupeArrays(value: unknown): unknown {
+ if (Array.isArray(value)) {
+ const seen = new Set<string>();
+ const out: unknown[] = [];
+ for (const item of value.map(dedupeArrays)) {
+ if (item && typeof item === "object") {
+ const key = JSON.stringify(item);
+ if (seen.has(key)) continue;
+ seen.add(key);
+ }
+ out.push(item);
+ }
+ return out;
+ }
+ if (isObject(value)) {
+ const out: JsonObject = {};
+ for (const [k, v] of Object.entries(value)) out[k] = dedupeArrays(v);
+ return out;
+ }
+ return value;
+}
+
+// Characters that ride along from the DOM but are never real content: private-
+// use icon-font glyphs, zero-width joiners / BOM, and C0/C1 control codes.
+// .trim() won't remove them, so we strip them and collapse the doubled spaces
+// they leave behind (newlines preserved so multi-line text keeps its shape).
+// Built via RegExp(string) so the source stays pure ASCII.
+const JUNK_CHARS = new RegExp(
+ "[\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u001F\\u007F-\\u009F" +
+ "\\u200B-\\u200D\\u2060\\uFEFF\\uE000-\\uF8FF" +
+ "\\u{F0000}-\\u{FFFFD}\\u{100000}-\\u{10FFFD}]",
+ "gu",
+);
+
+function sanitizeStrings(value: unknown): unknown {
+ if (typeof value === "string") {
+ return value
+ .replace(JUNK_CHARS, "")
+ .replace(/[^\S\r\n]{2,}/g, " ")
+ .trim();
+ }
+ if (Array.isArray(value)) return value.map(sanitizeStrings);
+ if (isObject(value)) {
+ const out: JsonObject = {};
+ for (const [key, v] of Object.entries(value)) out[key] = sanitizeStrings(v);
+ return out;
+ }
+ return value;
+}
diff --git a/apps/api/src/lib/deterministicJson/pipeline/validate.ts b/apps/api/src/lib/deterministicJson/pipeline/validate.ts
new file mode 100644
index 000000000..8c0eaf6db
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/pipeline/validate.ts
@@ -0,0 +1,203 @@
+import ts from "typescript";
+
+export interface GeneratedCodeIssue {
+ field: string;
+ reason: string;
+ excerpt: string;
+}
+
+function parseSource(code: string): ts.SourceFile {
+ return ts.createSourceFile(
+ "extractor.js",
+ code,
+ ts.ScriptTarget.Latest,
+ true,
+ ts.ScriptKind.JS,
+ );
+}
+
+function stripCodeFences(raw: string): string {
+ const text = raw.trim();
+
+ const fenced = text.match(/```(?:javascript|js|ts)?\s*([\s\S]*?)```/i);
+ if (fenced) return fenced[1]!.trim();
+
+ return text
+ .replace(/^```(?:javascript|js|ts)?\s*/i, "")
+ .replace(/```\s*$/i, "")
+ .trim();
+}
+
+function findTopLevelExtractor(
+ source: ts.SourceFile,
+): ts.FunctionDeclaration | undefined {
+ // On a duplicate `function extract`, the last declaration wins at runtime.
+ const matches = source.statements.filter(
+ (statement): statement is ts.FunctionDeclaration =>
+ ts.isFunctionDeclaration(statement) &&
+ statement.name?.text === "extract" &&
+ !!statement.body,
+ );
+ return matches[matches.length - 1];
+}
+
+function isAllowedTopLevelDeclaration(statement: ts.Statement): boolean {
+ return (
+ ts.isFunctionDeclaration(statement) ||
+ ts.isVariableStatement(statement) ||
+ ts.isClassDeclaration(statement)
+ );
+}
+
+export function cleanGeneratedCode(raw: string): string {
+ const code = stripCodeFences(raw).trim();
+
+ const source = parseSource(code);
+ const fn = findTopLevelExtractor(source);
+ if (!fn) {
+ throw new Error("LLM did not return a function named extract");
+ }
+
+ const kept = source.statements.filter(
+ statement =>
+ statement === fn ||
+ (isAllowedTopLevelDeclaration(statement) &&
+ !ts.isEmptyStatement(statement)),
+ );
+
+ // The sandbox runs this as a plain script, so a surviving `export` would throw.
+ return kept
+ .map(statement =>
+ code
+ .slice(statement.getStart(source), statement.end)
+ .replace(/^\s*export\s+(?:default\s+)?/i, ""),
+ )
+ .join("\n\n")
+ .trim();
+}
+
+function nodeExcerpt(source: ts.SourceFile, node: ts.Node): string {
+ const text = node.getText(source).replace(/\s+/g, " ").trim();
+ return text.length > 180 ? `${text.slice(0, 177)}...` : text;
+}
+
+function validateTopLevelShape(
+ source: ts.SourceFile,
+ fn: ts.FunctionDeclaration | undefined,
+ issues: GeneratedCodeIssue[],
+): void {
+ const parseDiagnostics =
+ (source as ts.SourceFile & { parseDiagnostics?: ts.Diagnostic[] })
+ .parseDiagnostics ?? [];
+ for (const diagnostic of parseDiagnostics.slice(0, 3)) {
+ issues.push({
+ field: "source",
+ reason: ts.flattenDiagnosticMessageText(diagnostic.messageText, " "),
+ excerpt: "",
+ });
+ }
+
+ if (!fn) {
+ issues.push({
+ field: "source",
+ reason: "missing function declaration named extract",
+ excerpt: "",
+ });
+ return;
+ }
+
+ const extras = source.statements.filter(
+ statement =>
+ statement !== fn &&
+ !ts.isEmptyStatement(statement) &&
+ !isAllowedTopLevelDeclaration(statement),
+ );
+
+ for (const extra of extras) {
+ issues.push({
+ field: "source",
+ reason:
+ "top-level code is not allowed; only function/class/const/let/var declarations may sit beside extract",
+ excerpt: nodeExcerpt(source, extra),
+ });
+ }
+
+ const isAsync = fn.modifiers?.some(
+ modifier => modifier.kind === ts.SyntaxKind.AsyncKeyword,
+ );
+
+ if (!isAsync) {
+ issues.push({
+ field: "extract",
+ reason: "extract must be async",
+ excerpt: nodeExcerpt(source, fn),
+ });
+ }
+
+ if (fn.parameters.length !== 2) {
+ issues.push({
+ field: "extract",
+ reason: "extract must accept exactly two parameters: doc and askLlm",
+ excerpt: nodeExcerpt(source, fn),
+ });
+ }
+}
+
+// Crude check for forbidden runtime references. The generated code runs inside
+// jsdom's VM context (see sandbox/harness.ts), which has no access to certain
+// globals like fetch or XMLHttpRequest.
+const FORBIDDEN_GLOBALS = new Set(["fetch", "XMLHttpRequest"]);
+
+function detectForbiddenGlobals(
+ source: ts.SourceFile,
+ issues: GeneratedCodeIssue[],
+): void {
+ const seen = new Set<string>();
+
+ const visit = (node: ts.Node): void => {
+ if (ts.isIdentifier(node) && FORBIDDEN_GLOBALS.has(node.text)) {
+ // Skip identifiers that are property names of access expressions
+ // (`foo.fetch` doesn't reference the global fetch) or local bindings.
+ const parent = node.parent;
+ const isPropertyName =
+ parent &&
+ ((ts.isPropertyAccessExpression(parent) && parent.name === node) ||
+ (ts.isPropertyAssignment(parent) && parent.name === node) ||
+ (ts.isBindingElement(parent) && parent.propertyName === node));
+ if (!isPropertyName && !seen.has(node.text)) {
+ seen.add(node.text);
+ issues.push({
+ field: "source",
+ reason: `references forbidden global '${node.text}' (not available in the sandbox; will throw at runtime)`,
+ excerpt: node.text,
+ });
+ }
+ }
+ ts.forEachChild(node, visit);
+ };
+
+ visit(source);
+}
+
+export function validateGeneratedExtractor(code: string): GeneratedCodeIssue[] {
+ const source = parseSource(code);
+ const fn = findTopLevelExtractor(source);
+ const issues: GeneratedCodeIssue[] = [];
+
+ validateTopLevelShape(source, fn, issues);
+ detectForbiddenGlobals(source, issues);
+
+ return issues;
+}
+
+export function formatGeneratedCodeIssues(
+ issues: GeneratedCodeIssue[],
+): string {
+ return issues
+ .slice(0, 16)
+ .map(issue => {
+ const excerpt = issue.excerpt ? `; ${issue.excerpt}` : "";
+ return `- ${issue.field}: ${issue.reason}${excerpt}`;
+ })
+ .join("\n");
+}
diff --git a/apps/api/src/lib/deterministicJson/sandbox/harness.ts b/apps/api/src/lib/deterministicJson/sandbox/harness.ts
new file mode 100644
index 000000000..2fd8a40c7
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/sandbox/harness.ts
@@ -0,0 +1,45 @@
+// Runs as the body of `async (input, host, require) => { ... }` in the
+// code-sandbox jail. input is { code, html, url }: build a jsdom Document, run the
+// generated extract(doc, askLlm) in jsdom's VM context, and bridge askLlm back to
+// the API worker over host().
+export const EXTRACTOR_HARNESS = String.raw`
+const { JSDOM, VirtualConsole } = require("jsdom");
+const vm = require("node:vm");
+
+const dom = new JSDOM(input.html, {
+ url: input.url || "https://example.invalid/",
+ runScripts: "outside-only", // page scripts don't run; ours can.
+ pretendToBeVisual: true, // enables getComputedStyle, requestAnimationFrame.
+ virtualConsole: new VirtualConsole(),
+});
+
+// jsdom doesn't implement innerText (it needs layout). Models reach for it
+// constantly; alias it to textContent so those reads don't crash.
+const proto = dom.window.HTMLElement.prototype;
+if (!Object.getOwnPropertyDescriptor(proto, "innerText")) {
+ Object.defineProperty(proto, "innerText", {
+ get() { return this.textContent; },
+ set(value) { this.textContent = value == null ? "" : String(value); },
+ configurable: true,
+ });
+}
+
+const context = dom.getInternalVMContext();
+
+// Bridge askLlm back through the sandbox host-callback to the API worker.
+dom.window.askLlm = (prompt, schema) =>
+ Promise.resolve()
+ .then(() => host("askLlm", { prompt: String(prompt == null ? "" : prompt), schema: schema == null ? null : schema }))
+ .catch(() => null);
+
+// Stringify inside the context so the result crosses the realm boundary as a
+// plain primitive, and a non-JSON return (DOM node, circular ref) throws here.
+const wrapped =
+ "(async () => {\n" + input.code + "\n" +
+ " const __value = await extract(document, askLlm);\n" +
+ " return JSON.stringify(__value === undefined ? null : __value);\n" +
+ "})()";
+
+const json = await new vm.Script(wrapped).runInContext(context);
+return JSON.parse(typeof json === "string" ? json : "null");
+`;
diff --git a/apps/api/src/lib/deterministicJson/sandbox/runExtractor.ts b/apps/api/src/lib/deterministicJson/sandbox/runExtractor.ts
new file mode 100644
index 000000000..78dd574b6
--- /dev/null
+++ b/apps/api/src/lib/deterministicJson/sandbox/runExtractor.ts
@@ -0,0 +1,30 @@
+import type { AskLlm } from "../llm/client";
+import { EXTRACTOR_HARNESS } from "./harness";
+
+export type SandboxRunner = (job: {
+ code: string;
+ input: unknown;
+ onHost: (channel: string, payload: unknown) => Promise<unknown>;
+}) => Promise<unknown>;
+
+export async function runExtractorInSandbox(args: {
+ code: string;
+ html: string;
+ url: string;
+ askLlm: AskLlm;
+ sandbox: SandboxRunner;
+}): Promise<unknown> {
+ return args.sandbox({
+ code: EXTRACTOR_HARNESS,
+ input: { code: args.code, html: args.html, url: args.url },
+ onHost: async (channel, payload) => {
+ if (channel !== "askLlm")
+ throw new Error(`unknown sandbox channel: ${channel}`);
+ const { prompt, schema } = (payload ?? {}) as {
+ prompt?: string;
+ schema?: unknown;
+ };
+ return args.askLlm(String(prompt ?? ""), schema ?? undefined);
+ },
+ });
+}
diff --git a/apps/api/src/lib/extract/usage/model-prices.ts b/apps/api/src/lib/extract/usage/model-prices.ts
index a5d183a85..c100af736 100644
--- a/apps/api/src/lib/extract/usage/model-prices.ts
+++ b/apps/api/src/lib/extract/usage/model-prices.ts
@@ -19379,8 +19379,8 @@ export const modelPrices = {
max_tokens: 32768,
max_input_tokens: 131072,
max_output_tokens: 32768,
- input_cost_per_token: 1e-7,
- output_cost_per_token: 5e-7,
+ input_cost_per_token: 7.5e-8,
+ output_cost_per_token: 3e-7,
litellm_provider: "groq",
mode: "chat",
supports_function_calling: true,
@@ -19395,7 +19395,7 @@ export const modelPrices = {
max_input_tokens: 131072,
max_output_tokens: 32766,
input_cost_per_token: 1.5e-7,
- output_cost_per_token: 7.5e-7,
+ output_cost_per_token: 6e-7,
litellm_provider: "groq",
mode: "chat",
supports_function_calling: true,
@@ -19405,6 +19405,15 @@ export const modelPrices = {
supports_tool_choice: true,
supports_web_search: true,
},
+ "vertex/gemini-3.1-flash-lite": {
+ max_tokens: 65536,
+ max_input_tokens: 1048576,
+ max_output_tokens: 65536,
+ input_cost_per_token: 2.5e-7,
+ output_cost_per_token: 1.5e-6,
+ litellm_provider: "vertex_ai-language-models",
+ mode: "chat",
+ },
"claude-opus-4-1": {
max_tokens: 32000,
max_input_tokens: 200000,
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index c033f4680..3efa05045 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -72,6 +72,10 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled = 5;
}
+ if (hasFormatOfType(options.formats, "deterministicJson")) {
+ creditsToBeBilled = 7;
+ }
+
if (
internalOptions.v1Agent?.model === "fire-1" ||
internalOptions.v1JSONAgent?.model?.toLowerCase() === "fire-1"
diff --git a/apps/api/src/scraper/scrapeURL/transformers/deterministicJson.ts b/apps/api/src/scraper/scrapeURL/transformers/deterministicJson.ts
new file mode 100644
index 000000000..cfc686fdf
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/transformers/deterministicJson.ts
@@ -0,0 +1,199 @@
+import WebSocket from "ws";
+import { Meta } from "..";
+import { Document } from "../../../controllers/v2/types";
+import { hasFormatOfType } from "../../../lib/format-utils";
+import { eq } from "drizzle-orm";
+import { db, dbRr } from "../../../db/connection";
+import * as schema from "../../../db/schema";
+import { extractDeterministicJson } from "../../../lib/deterministicJson/extract";
+import {
+ CODE_SANDBOX_URL,
+ SANDBOX_TIMEOUT_MS,
+} from "../../../lib/deterministicJson/config";
+import type { CacheBackend } from "../../../lib/deterministicJson/core/cache";
+import type { SandboxRunner } from "../../../lib/deterministicJson/sandbox/runExtractor";
+
+const cache: CacheBackend = {
+ async getExtractor(key) {
+ const [data] = await dbRr
+ .select({
+ code: schema.deterministic_json_scripts.code,
+ created_at: schema.deterministic_json_scripts.created_at,
+ })
+ .from(schema.deterministic_json_scripts)
+ .where(eq(schema.deterministic_json_scripts.cache_key, key))
+ .limit(1);
+ return data
+ ? { code: data.code, createdAt: new Date(data.created_at).getTime() }
+ : undefined;
+ },
+ async setExtractor(key, code, meta) {
+ const now = new Date().toISOString();
+ await db
+ .insert(schema.deterministic_json_scripts)
+ .values({
+ cache_key: key,
+ code,
+ url: meta.url,
+ model: meta.model,
+ cache_version: meta.cacheVersion,
+ updated_at: now,
+ last_used_at: now,
+ })
+ .onConflictDoUpdate({
+ target: schema.deterministic_json_scripts.cache_key,
+ set: {
+ code,
+ url: meta.url,
+ model: meta.model,
+ cache_version: meta.cacheVersion,
+ updated_at: now,
+ last_used_at: now,
+ },
+ });
+ },
+ async getLlm(key) {
+ const [data] = await dbRr
+ .select({
+ response: schema.deterministic_json_llm_cache.response,
+ created_at: schema.deterministic_json_llm_cache.created_at,
+ })
+ .from(schema.deterministic_json_llm_cache)
+ .where(eq(schema.deterministic_json_llm_cache.cache_key, key))
+ .limit(1);
+ return data
+ ? {
+ response: data.response,
+ createdAt: new Date(data.created_at).getTime(),
+ }
+ : undefined;
+ },
+ async setLlm(key, response) {
+ const now = new Date().toISOString();
+ await db
+ .insert(schema.deterministic_json_llm_cache)
+ .values({ cache_key: key, response, last_used_at: now })
+ .onConflictDoUpdate({
+ target: schema.deterministic_json_llm_cache.cache_key,
+ set: { response, last_used_at: now },
+ });
+ },
+ async touch(key) {
+ const now = new Date().toISOString();
+ await Promise.all([
+ db
+ .update(schema.deterministic_json_scripts)
+ .set({ last_used_at: now })
+ .where(eq(schema.deterministic_json_scripts.cache_key, key)),
+ db
+ .update(schema.deterministic_json_llm_cache)
+ .set({ last_used_at: now })
+ .where(eq(schema.deterministic_json_llm_cache.cache_key, key)),
+ ]);
+ },
+};
+
+function sandboxRunner(endpoint: string): SandboxRunner {
+ return job =>
+ new Promise((resolve, reject) => {
+ const ws = new WebSocket(endpoint);
+ let done = false;
+ let timer: ReturnType<typeof setTimeout>;
+ const finish = (err: Error | null, value?: unknown) => {
+ if (done) return;
+ done = true;
+ clearTimeout(timer);
+ try {
+ ws.close();
+ } catch {
+ /* closing */
+ }
+ err ? reject(err) : resolve(value);
+ };
+ timer = setTimeout(
+ () =>
+ finish(new Error(`sandbox timed out after ${SANDBOX_TIMEOUT_MS}ms`)),
+ SANDBOX_TIMEOUT_MS,
+ );
+ ws.on("open", () =>
+ ws.send(
+ JSON.stringify({ type: "run", code: job.code, input: job.input }),
+ ),
+ );
+ ws.on("message", async raw => {
+ let frame: any;
+ try {
+ frame = JSON.parse(raw.toString());
+ } catch {
+ return;
+ }
+ if (frame.type === "host") {
+ let value: unknown;
+ let error: string | undefined;
+ try {
+ value = await job.onHost(frame.channel, frame.payload);
+ } catch (err) {
+ error = err instanceof Error ? err.message : String(err);
+ }
+ if (ws.readyState === WebSocket.OPEN) {
+ ws.send(
+ JSON.stringify({
+ type: "hostResult",
+ id: frame.id,
+ value,
+ error,
+ }),
+ );
+ }
+ } else if (frame.type === "result") {
+ finish(null, frame.value);
+ } else if (frame.type === "error") {
+ finish(new Error(frame.message));
+ }
+ });
+ ws.on("error", err =>
+ finish(err instanceof Error ? err : new Error(String(err))),
+ );
+ ws.on("close", () => finish(new Error("sandbox connection closed")));
+ });
+}
+
+export async function performDeterministicJson(
+ meta: Meta,
+ document: Document,
+): Promise<Document> {
+ const format = hasFormatOfType(meta.options.formats, "deterministicJson");
+ if (!format) return document;
+
+ const warn = (msg: string) => {
+ document.warning = msg + (document.warning ? " " + document.warning : "");
+ };
+
+ if (meta.internalOptions.zeroDataRetention) {
+ warn("Deterministic JSON mode is not supported with zero data retention.");
+ return document;
+ }
+
+ const html = document.html ?? document.rawHtml;
+ if (!html) {
+ warn("Deterministic JSON mode requires page HTML.");
+ return document;
+ }
+
+ try {
+ document.json = await extractDeterministicJson({
+ url: document.metadata?.sourceURL ?? meta.url,
+ prompt: format.prompt ?? "",
+ jsonSchema: (format.schema ?? {}) as Record<string, unknown>,
+ page: { html, markdown: document.markdown ?? "" },
+ cache,
+ sandbox: sandboxRunner(CODE_SANDBOX_URL),
+ costTracking: meta.costTracking,
+ });
+ } catch (error) {
+ meta.logger.error("Deterministic JSON extraction failed", { error });
+ warn("Deterministic JSON extraction failed.");
+ }
+
+ return document;
+}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 6190778a0..e5488d5fe 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -10,6 +10,7 @@ import {
performSummary,
performCleanContent,
} from "./llmExtract";
+import { performDeterministicJson } from "./deterministicJson";
import { performQuery } from "./query";
import { removeBase64Images } from "./removeBase64Images";
import { performAgent } from "./agent";
@@ -91,7 +92,11 @@ async function deriveMarkdownFromHTML(
meta.options.formats,
"changeTracking",
);
- const hasJson = hasFormatOfType(meta.options.formats, "json");
+ // deterministicJson populates document.json just like json, so treat it the
+ // same here (derive markdown for it; keep the field it produced).
+ const hasJson =
+ hasFormatOfType(meta.options.formats, "json") ||
+ hasFormatOfType(meta.options.formats, "deterministicJson");
const hasSummary = hasFormatOfType(meta.options.formats, "summary");
const hasQuestion = hasFormatOfType(meta.options.formats, "question");
const hasHighlights = hasFormatOfType(meta.options.formats, "highlights");
@@ -321,7 +326,11 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {
meta.options.formats,
"changeTracking",
);
- const hasJson = hasFormatOfType(meta.options.formats, "json");
+ // deterministicJson populates document.json just like json, so treat it the
+ // same here (derive markdown for it; keep the field it produced).
+ const hasJson =
+ hasFormatOfType(meta.options.formats, "json") ||
+ hasFormatOfType(meta.options.formats, "deterministicJson");
const hasScreenshot = hasFormatOfType(meta.options.formats, "screenshot");
const hasSummary = hasFormatOfType(meta.options.formats, "summary");
const hasBranding = hasFormatOfType(meta.options.formats, "branding");
@@ -576,6 +585,7 @@ const transformerStack: Transformer[] = [
...(useIndex ? [sendDocumentToIndex] : []),
...(useSearchIndex ? [sendDocumentToSearchIndex] : []), // Add to search index for real-time search
performLLMExtract,
+ performDeterministicJson,
performSummary,
performQuery,
performAttributes,
diff --git a/apps/api/src/services/monitoring/diff-orchestrator.ts b/apps/api/src/services/monitoring/diff-orchestrator.ts
index 3521a16aa..b3efbade2 100644
--- a/apps/api/src/services/monitoring/diff-orchestrator.ts
+++ b/apps/api/src/services/monitoring/diff-orchestrator.ts
@@ -14,7 +14,7 @@ import {
} from "./diff";
import { judgeChange } from "./judgeChange";
-type MonitorPageDiffStatus = "same" | "new" | "changed";
+type MonitorPageDiffStatus = "same" | "new" | "changed" | "error";
type Judgment = {
meaningful: boolean;
@@ -36,6 +36,7 @@ type MonitorPageDiffResult = {
judgment?: Judgment;
diffText?: string;
diffJson?: Record<string, { previous: unknown; current: unknown }>;
+ error?: string;
};
type PreviousPageRef = {
@@ -103,15 +104,17 @@ export async function computeAndPersistPageDiff(params: {
? (doc.json as Record<string, unknown>)
: undefined;
- // If the current scrape didn't produce a JSON document we can't
- // compute a JSON diff; treat as `changed` to be safe (matches the
- // markdown-missing branch's behavior).
+ // If the current scrape didn't produce a JSON document we can't compute a
+ // JSON diff (e.g. the extraction step failed) - report `error` rather than
+ // a false `changed`, so the user isn't alerted to a content change that
+ // didn't happen.
if (!currentJson) {
return {
- status: "changed",
+ status: "error",
diffGcsKey: null,
diffTextBytes: null,
diffJsonBytes: null,
+ error: "JSON extraction produced no result for this check.",
};
}
diff --git a/apps/api/src/services/monitoring/diff.ts b/apps/api/src/services/monitoring/diff.ts
index d5696a85d..200be7912 100644
--- a/apps/api/src/services/monitoring/diff.ts
+++ b/apps/api/src/services/monitoring/diff.ts
@@ -290,7 +290,9 @@ export function formatsRequestJsonExtraction(formats: unknown): boolean {
if (typeof entry === "string" && entry === "json") return true;
if (entry && typeof entry === "object") {
const obj = entry as { type?: unknown; modes?: unknown };
- if (obj.type === "json") return true;
+ // Both json and deterministicJson populate document.json, so the JSON
+ // diff path applies to either.
+ if (obj.type === "json" || obj.type === "deterministicJson") return true;
if (obj.type === "changeTracking") {
const modes = Array.isArray(obj.modes) ? obj.modes : [];
if (modes.includes("json")) return true;
diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.ts
index aa8fcc120..7ba63ef99 100644
--- a/apps/api/src/services/monitoring/results.ts
+++ b/apps/api/src/services/monitoring/results.ts
@@ -141,6 +141,7 @@ export async function recordMonitorScrapeSuccess(
judgment,
diffText,
diffJson,
+ error,
} = await computeAndPersistPageDiff({
teamId: job.data.team_id,
monitorId: monitoring.monitorId,
@@ -193,6 +194,7 @@ export async function recordMonitorScrapeSuccess(
diff_text_bytes: diffTextBytes,
diff_json_bytes: diffJsonBytes,
status_code: getDocumentStatusCode(doc),
+ ...(error ? { error } : {}),
metadata: {
title: doc?.metadata?.title ?? null,
creditsUsed: doc?.metadata?.creditsUsed ?? null,
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index e4cade567..58354e9fa 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -7,6 +7,7 @@ import { processJobInternal } from "../worker/scrape-worker";
import { NuQJob, crawlGroup, scrapeQueue } from "../worker/nuq";
import { ScrapeJobData } from "../../types";
import { getJobFromGCS } from "../../lib/gcs-jobs";
+import { includesFormat } from "../../lib/format-utils";
import { computeAndPersistPageDiff } from "./diff-orchestrator";
import { normalizeMonitorFormats } from "./diff";
import { autumnService } from "../autumn/autumn.service";
@@ -216,10 +217,15 @@ function estimateActualCredits(doc: any, options: any): number {
return doc.metadata.creditsUsed;
}
const formats = Array.isArray(options?.formats) ? options.formats : [];
- const hasJson = formats.some((format: any) =>
- typeof format === "string" ? format === "json" : format?.type === "json",
- );
- return hasJson ? 5 : 1;
+ // Only charge the JSON-extraction premium when extraction actually produced a
+ // document.json. When it failed, the page was still scraped, so fall back to
+ // the base scrape credit rather than billing for extraction that never ran.
+ // Deterministic JSON costs 7 (it generates a reusable extractor); plain JSON 5.
+ const producedJson = doc?.json != null;
+ if (!producedJson) return 1;
+ if (includesFormat(formats, "deterministicJson")) return 7;
+ if (includesFormat(formats, "json")) return 5;
+ return 1;
}
async function runSingleScrape(params: {
@@ -302,7 +308,7 @@ async function diffAndPersistPage(params: {
(f: any) => f?.type === "changeTracking",
)
: undefined;
- const { status, diffGcsKey, diffTextBytes, diffJsonBytes, judgment } =
+ const { status, diffGcsKey, diffTextBytes, diffJsonBytes, judgment, error } =
await computeAndPersistPageDiff({
teamId: params.monitor.team_id,
monitorId: params.monitor.id,
@@ -350,6 +356,7 @@ async function diffAndPersistPage(params: {
diff_text_bytes: diffTextBytes,
diff_json_bytes: diffJsonBytes,
status_code: getDocumentStatusCode(params.doc),
+ ...(error ? { error } : {}),
metadata: {
title: params.doc?.metadata?.title ?? null,
},
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index 51f0f7cb9..33cc09872 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -3,6 +3,7 @@ import { v7 as uuidv7 } from "uuid";
import { and, asc, count, desc, eq, isNull, ne } from "drizzle-orm";
import { db, dbRr } from "../../db/connection";
import * as schema from "../../db/schema";
+import { includesFormat } from "../../lib/format-utils";
import { monitoringClaimDueMonitors } from "../../db/rpc";
import {
getNextMonitorRunAt,
@@ -31,16 +32,37 @@ function ensureTargetIds(targets: Array<Record<string, any>>): MonitorTarget[] {
})) as MonitorTarget[];
}
+// Per-page credit cost for a target's extraction mode. Mirrors
+// estimateActualCredits in runner.ts so the reserved/estimated credits match
+// what a check actually bills: deterministic JSON = 7, other JSON (plain or
+// changeTracking-json) = 5, otherwise the base scrape = 1.
+function perPageCredits(target: MonitorTarget): number {
+ const formats = Array.isArray(target.scrapeOptions?.formats)
+ ? (target.scrapeOptions!.formats as any[])
+ : [];
+ if (includesFormat(formats, "deterministicJson")) return 7;
+ const hasJson =
+ includesFormat(formats, "json") ||
+ formats.some(
+ format =>
+ format?.type === "changeTracking" &&
+ Array.isArray(format?.modes) &&
+ format.modes.includes("json"),
+ );
+ return hasJson ? 5 : 1;
+}
+
function estimateTargetCredits(target: MonitorTarget): number {
+ const perPage = perPageCredits(target);
if (target.type === "scrape") {
- return target.urls.length;
+ return target.urls.length * perPage;
}
const limit =
typeof target.crawlOptions?.limit === "number"
? target.crawlOptions.limit
: 10000;
- return Math.max(1, limit);
+ return Math.max(1, limit) * perPage;
}
export function estimateMonitorCreditsPerRun(