Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): avoid shelling out for change tracking diffs
mogeryf01827df2958
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 46f6e0de8..0ca1c9174 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -116,7 +116,7 @@"express-ws": "^5.0.2","foundationdb": "^2.0.1","geoip-country": "^5.0.202510312342",- "git-diff": "^2.0.6",+ "diff": "^8.0.3","he": "^1.2.0","http-cookie-agent": "^7.0.1","ioredis": "^5.6.1",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex b0972a982..8a07f312f 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -154,6 +154,9 @@ importers:culori:specifier: ^4.0.2version: 4.0.2+ diff:+ specifier: ^8.0.3+ version: 8.0.3dotenv:specifier: ^16.3.1version: 16.4.5@@ -178,9 +181,6 @@ importers:geoip-country:specifier: ^5.0.202510312342version: 5.0.202510312342- git-diff:- specifier: ^2.0.6- version: 2.0.6he:specifier: ^1.2.0version: 1.2.0@@ -2698,10 +2698,6 @@ packages:resolution: {integrity: sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==}engines: {node: '>=12'}- resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==}- engines: {node: '>=4'}-resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==}engines: {node: '>=8'}@@ -2876,10 +2872,6 @@ packages:resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==}engines: {node: '>=18'}- resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==}- engines: {node: '>=4'}-resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}engines: {node: '>=10'}@@ -3347,10 +3339,6 @@ packages:resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==}- resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==}- engines: {node: '>=0.8.0'}-resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==}@@ -3537,9 +3525,6 @@ packages:resolution: {integrity: sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==}- resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==}-resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}@@ -3583,10 +3568,6 @@ packages:resolution: {integrity: sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==}- resolution: {integrity: sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==}- engines: {node: '>= 4.8.0'}-resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}engines: {node: '>= 6'}@@ -3595,10 +3576,6 @@ packages:resolution: {integrity: sha512-B7U/vJpE3DkJ5WXTgTpTRN63uV42DseiXXKMwG14LQBXmsdeIoHAPbU/MEo6II0k5ED74uc2ZGTC6MwHFQhF6w==}engines: {node: 20 || >=22}- resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==}- deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting [email protected]-resolution: {integrity: sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==}@@ -3630,10 +3607,6 @@ packages:resolution: {integrity: sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==}engines: {node: '>=18'}- resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==}- engines: {node: '>=4'}-resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==}engines: {node: '>=8'}@@ -3646,10 +3619,6 @@ packages:resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==}engines: {node: '>= 0.4'}- resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==}- engines: {node: '>= 0.4'}-resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==}engines: {node: '>= 0.4'}@@ -3728,20 +3697,12 @@ packages:resolution: {integrity: sha512-yNZhyQYqXpkT0AKq3F3KLasUSK4fHvebNH5hOsKQw2dhGSALvQ4U0BqUc5suziKvydO5u5hgN2hy1RJaho8U5A==}- resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==}- deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.-resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}- resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==}- engines: {node: '>= 0.10'}-resolution: {integrity: sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==}engines: {node: '>=12.22.0'}@@ -3764,9 +3725,6 @@ packages:resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==}- resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==}-resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}engines: {node: '>=0.10.0'}@@ -4059,10 +4017,6 @@ packages:resolution: {integrity: sha512-CdaO738xRapbKIMVn2m4F6KTj4j7ooJ8POVnebSgKo3KBz5axNXRAL7ZdRjIV6NOr2Uf4vjtRkxrFETOioCqSA==}engines: {node: '>= 12.0.0'}- resolution: {integrity: sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==}- engines: {node: '>= 0.6.0'}-resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==}hasBin: true@@ -4370,17 +4324,10 @@ packages:resolution: {integrity: sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==}engines: {node: '>=14.0.0'}- resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==}- engines: {node: '>=0.10.0'}-resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==}engines: {node: '>=8'}- resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==}-resolution: {integrity: sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==}engines: {node: 20 || >=22}@@ -4569,10 +4516,6 @@ packages:resolution: {integrity: sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==}engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}- resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==}- engines: {node: '>= 0.10'}-resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==}engines: {node: '>=4'}@@ -4606,10 +4549,6 @@ packages:resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==}- resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==}- hasBin: true-resolution: {integrity: sha512-fiyq1RvW5/Br6iAtT8jN1XrNY8WPu2+yEypLbaijWry8WDZmn12azG9p/+c+qpEebURLlQmqCB8BNSu7ji+xQQ==}engines: {node: '>= 0.8.0'}@@ -4715,15 +4654,6 @@ packages:resolution: {integrity: sha512-VsC6n6vz1ihYYyZZwX7YZSF5l5x36ca17OC+a69h94YqB7X6XLwf+5MOgynYir2SLFUbl8gIYvBo8K8RoNQ6bQ==}engines: {node: '>= 0.4'}- resolution: {integrity: sha512-vFILCw+lzUtiwBAHV8/Ex8JsFjelFMdhONIsgKNLgTzeRckp2AOYRQtHJE/9LhNvdMmE27AGtzWx0+DHpwIwSw==}- engines: {node: '>= 4.0.0'}-- resolution: {integrity: sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==}- engines: {node: '>=4'}- hasBin: true-resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==}engines: {node: '>= 0.4'}@@ -4875,18 +4805,10 @@ packages:engines: {node: '>=6.4.0'}deprecated: Please upgrade to supertest v7.1.3+, see release notes at https://github.com/forwardemail/supertest/releases/tag/v7.1.3 - maintenance is supported by Forward Email @ https://forwardemail.net- resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==}- engines: {node: '>=4'}-resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==}engines: {node: '>=8'}- resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==}- engines: {node: '>= 0.4'}-resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==}@@ -7633,10 +7555,6 @@ snapshots:- dependencies:- color-convert: 1.9.3-dependencies:color-convert: 2.0.1@@ -7820,12 +7738,6 @@ snapshots:- dependencies:- ansi-styles: 3.2.1- escape-string-regexp: 1.0.5- supports-color: 5.5.0-dependencies:ansi-styles: 4.3.0@@ -8167,8 +8079,6 @@ snapshots:- [email protected]: {}-dependencies:'@types/estree': 1.0.9@@ -8388,8 +8298,6 @@ snapshots:- [email protected]: {}-optional: true@@ -8463,14 +8371,6 @@ snapshots:dependencies:resolve-pkg-maps: 1.0.0- dependencies:- chalk: 2.4.2- diff: 8.0.3- loglevel: 1.9.1- shelljs: 0.8.5- shelljs.exec: 1.1.8-dependencies:is-glob: 4.0.3@@ -8481,15 +8381,6 @@ snapshots:minipass: 7.1.2path-scurry: 2.0.1- dependencies:- fs.realpath: 1.0.0- inflight: 1.0.6- inherits: 2.0.4- minimatch: 10.2.4- once: 1.4.0- path-is-absolute: 1.0.1-@@ -8537,8 +8428,6 @@ snapshots:transitivePeerDependencies:- supports-color- [email protected]: {}-@@ -8547,10 +8436,6 @@ snapshots:dependencies:has-symbols: 1.1.0- dependencies:- function-bind: 1.1.2-dependencies:function-bind: 1.1.2@@ -8652,17 +8537,10 @@ snapshots:cjs-module-lexer: 1.4.3module-details-from-path: 1.0.4- dependencies:- once: 1.4.0- wrappy: 1.0.2-- [email protected]: {}-dependencies:'@ioredis/commands': 1.2.0@@ -8695,10 +8573,6 @@ snapshots:- dependencies:- hasown: 2.0.2-@@ -8990,8 +8864,6 @@ snapshots:safe-stable-stringify: 2.4.3triple-beam: 1.4.1- [email protected]: {}-dependencies:js-tokens: 4.0.0@@ -9275,12 +9147,8 @@ snapshots:- [email protected]: {}-- [email protected]: {}-dependencies:lru-cache: 11.2.5@@ -9466,10 +9334,6 @@ snapshots:process: 0.11.10string_decoder: 1.3.0- dependencies:- resolve: 1.22.8-@@ -9504,12 +9368,6 @@ snapshots:- dependencies:- is-core-module: 2.13.1- path-parse: 1.0.7- supports-preserve-symlinks-flag: 1.0.0-dependencies:depd: 2.0.0@@ -9650,14 +9508,6 @@ snapshots:- [email protected]: {}-- dependencies:- glob: 7.2.3- interpret: 1.4.0- rechoir: 0.6.2-dependencies:es-errors: 1.3.0@@ -9817,16 +9667,10 @@ snapshots:transitivePeerDependencies:- supports-color- dependencies:- has-flag: 3.0.0-dependencies:has-flag: 4.0.0- [email protected]: {}-diff --git a/apps/api/src/lib/change-tracking-diff.ts b/apps/api/src/lib/change-tracking-diff.tsnew file mode 100644index 000000000..f447df3c1--- /dev/null+++ b/apps/api/src/lib/change-tracking-diff.ts@@ -0,0 +1,96 @@+import { createTwoFilesPatch } from "diff";+import parseDiff from "parse-diff";++type StructuredMarkdownDiff = {+ files: Array<{+ from: string | null;+ to: string | null;+ chunks: Array<{+ content: string;+ changes: Array<{+ type: string;+ normal?: boolean;+ add?: boolean;+ del?: boolean;+ ln?: number;+ ln1?: number;+ ln2?: number;+ content: string;+ }>;+ }>;+ }>;+};++export function createMarkdownChangeDiff(+ previousMarkdown: string,+ currentMarkdown: string,+): { text: string; json: StructuredMarkdownDiff } | undefined {+ const text = createTwoFilesPatch(+ "previous.md",+ "current.md",+ previousMarkdown,+ currentMarkdown,+ "",+ "",+ { context: 3 },+ );++ const structured = parseDiff(text);+ const hasChanges = structured.some(file =>+ file.chunks.some(chunk =>+ chunk.changes.some(+ change => change.type === "add" || change.type === "del",+ ),+ ),+ );++ if (!hasChanges) return undefined;++ return {+ text,+ json: {+ files: structured.map(file => ({+ from: file.from || null,+ to: file.to || null,+ chunks: file.chunks.map(chunk => ({+ content: chunk.content,+ changes: chunk.changes.map(change => {+ const baseChange = {+ type: change.type,+ content: change.content,+ };++ if (+ change.type === "normal" &&+ "ln1" in change &&+ "ln2" in change+ ) {+ return {+ ...baseChange,+ normal: true,+ ln1: change.ln1,+ ln2: change.ln2,+ };+ }+ if (change.type === "add" && "ln" in change) {+ return {+ ...baseChange,+ add: true,+ ln: change.ln,+ };+ }+ if (change.type === "del" && "ln" in change) {+ return {+ ...baseChange,+ del: true,+ ln: change.ln,+ };+ }++ return baseChange;+ }),+ })),+ })),+ },+ };+}diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.tsindex a36ed8ffc..4004bb766 100644--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts@@ -1,11 +1,10 @@import { diffGetLastScrape } from "../../../db/rpc";import { Document } from "../../../controllers/v1/types";import { Meta } from "../index";-import gitDiff from "git-diff";-import parseDiff from "parse-diff";import { generateCompletions } from "./llmExtract";import { hasFormatOfType } from "../../../lib/format-utils";import { getJobFromGCS } from "../../../lib/gcs-jobs";+import { createMarkdownChangeDiff } from "../../../lib/change-tracking-diff";async function extractDataWithSchema(content: string,@@ -161,58 +160,15 @@ export async function deriveDiff(changeTrackingFormat?.modes?.includes("git-diff") &&changeStatus === "changed") {- const diffText = gitDiff(previousMarkdown, currentMarkdown, {- color: false,- wordDiff: false,- });- // meta.logger.debug("Diff text", { diffText });- if (diffText) {- const diffStructured = parseDiff(diffText);- // meta.logger.debug("Diff structured", { diffStructured });+ const diff = createMarkdownChangeDiff(+ previousMarkdown,+ currentMarkdown,+ );+ // meta.logger.debug("Diff text", { diffText: diff?.text });+ if (diff) {document.changeTracking.diff = {- text: diffText,- json: {- files: diffStructured.map(file => ({- from: file.from || null,- to: file.to || null,- chunks: file.chunks.map(chunk => ({- content: chunk.content,- changes: chunk.changes.map(change => {- const baseChange = {- type: change.type,- content: change.content,- };-- if (- change.type === "normal" &&- "ln1" in change &&- "ln2" in change- ) {- return {- ...baseChange,- normal: true,- ln1: change.ln1,- ln2: change.ln2,- };- } else if (change.type === "add" && "ln" in change) {- return {- ...baseChange,- add: true,- ln: change.ln,- };- } else if (change.type === "del" && "ln" in change) {- return {- ...baseChange,- del: true,- ln: change.ln,- };- }-- return baseChange;- }),- })),- })),- },+ text: diff.text,+ json: diff.json,};}}diff --git a/apps/api/src/services/monitoring/diff.test.ts b/apps/api/src/services/monitoring/diff.test.tsindex df707ec0b..9c069d8de 100644--- a/apps/api/src/services/monitoring/diff.test.ts+++ b/apps/api/src/services/monitoring/diff.test.ts@@ -1,4 +1,55 @@-import { diffMonitorJson } from "./diff";+import { existsSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { createMarkdownChangeDiff } from "../../lib/change-tracking-diff";+import { diffMonitorJson, diffMonitorMarkdown } from "./diff";++describe("markdown change tracking diff", () => {+ it("treats command substitution syntax as text", () => {+ const marker = join(+ tmpdir(),+ `firecrawl-change-diff-${process.pid}-${Date.now()}`,+ );+ const current = `visible text $(touch ${marker}) and \`touch ${marker}.bt\``;++ try {+ const result = createMarkdownChangeDiff("visible text", current);++ expect(result).toBeDefined();+ expect(result?.text).toContain("$(touch");+ expect(result?.text).toContain("`touch");+ expect(result?.json.files).toBeInstanceOf(Array);+ expect(existsSync(marker)).toBe(false);+ expect(existsSync(`${marker}.bt`)).toBe(false);+ } finally {+ rmSync(marker, { force: true });+ rmSync(`${marker}.bt`, { force: true });+ }+ });++ it("uses the shell-free diff path for monitor markdown diffs", () => {+ const marker = join(+ tmpdir(),+ `firecrawl-monitor-diff-${process.pid}-${Date.now()}`,+ );++ try {+ const result = diffMonitorMarkdown(+ "baseline",+ `changed $(touch ${marker})`,+ );++ if (result.status !== "changed") {+ throw new Error("Expected monitor markdown diff to change");+ }+ expect(result.text).toContain("$(touch");+ expect(result.json.files).toBeInstanceOf(Array);+ expect(existsSync(marker)).toBe(false);+ } finally {+ rmSync(marker, { force: true });+ }+ });+});describe("diffMonitorJson", () => {it("returns `same` when every value matches", () => {@@ -11,9 +62,10 @@ describe("diffMonitorJson", () => {});it("returns `same` regardless of object key order", () => {- expect(- diffMonitorJson({ a: 1, b: 2 }, { b: 2, a: 1 }),- ).toEqual({ kind: "json", status: "same" });+ expect(diffMonitorJson({ a: 1, b: 2 }, { b: 2, a: 1 })).toEqual({+ kind: "json",+ status: "same",+ });});it("reports a single change for a top-level primitive that differs", () => {diff --git a/apps/api/src/services/monitoring/diff.ts b/apps/api/src/services/monitoring/diff.tsindex 200be7912..b56527bea 100644--- a/apps/api/src/services/monitoring/diff.ts+++ b/apps/api/src/services/monitoring/diff.ts@@ -1,5 +1,4 @@-import gitDiff from "git-diff";-import parseDiff from "parse-diff";+import { createMarkdownChangeDiff } from "../../lib/change-tracking-diff";type MonitorMarkdownDiffResult =| {@@ -60,60 +59,13 @@ export function diffMonitorMarkdown(return { kind: "markdown", status: "same" };}- const text = gitDiff(previousMarkdown, currentMarkdown, {- color: false,- wordDiff: false,- });- const structured = parseDiff(text);+ const diff = createMarkdownChangeDiff(previousMarkdown, currentMarkdown);return {kind: "markdown",status: "changed",- text,- json: {- files: structured.map(file => ({- from: file.from || null,- to: file.to || null,- chunks: file.chunks.map(chunk => ({- content: chunk.content,- changes: chunk.changes.map(change => {- const base = {- type: change.type,- content: change.content,- };-- if (- change.type === "normal" &&- "ln1" in change &&- "ln2" in change- ) {- return {- ...base,- normal: true,- ln1: change.ln1,- ln2: change.ln2,- };- }- if (change.type === "add" && "ln" in change) {- return {- ...base,- add: true,- ln: change.ln,- };- }- if (change.type === "del" && "ln" in change) {- return {- ...base,- del: true,- ln: change.ln,- };- }-- return base;- }),- })),- })),- },+ text: diff?.text ?? "",+ json: diff?.json ?? { files: [] },};}