Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): avoid shelling out for change tracking diffs

mogeryf01827df2958
diff --git a/apps/api/package.json b/apps/api/package.json
index 46f6e0de8..0ca1c9174 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -116,7 +116,7 @@
"express-ws": "^5.0.2",
"foundationdb": "^2.0.1",
"geoip-country": "^5.0.202510312342",
- "git-diff": "^2.0.6",
+ "diff": "^8.0.3",
"he": "^1.2.0",
"http-cookie-agent": "^7.0.1",
"ioredis": "^5.6.1",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index b0972a982..8a07f312f 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -154,6 +154,9 @@ importers:
culori:
specifier: ^4.0.2
version: 4.0.2
+ diff:
+ specifier: ^8.0.3
+ version: 8.0.3
dotenv:
specifier: ^16.3.1
version: 16.4.5
@@ -178,9 +181,6 @@ importers:
geoip-country:
specifier: ^5.0.202510312342
version: 5.0.202510312342
- git-diff:
- specifier: ^2.0.6
- version: 2.0.6
he:
specifier: ^1.2.0
version: 1.2.0
@@ -2698,10 +2698,6 @@ packages:
resolution: {integrity: sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==}
engines: {node: '>=12'}
- resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==}
- engines: {node: '>=4'}
-
resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==}
engines: {node: '>=8'}
@@ -2876,10 +2872,6 @@ packages:
resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==}
engines: {node: '>=18'}
- resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==}
- engines: {node: '>=4'}
-
resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
engines: {node: '>=10'}
@@ -3347,10 +3339,6 @@ packages:
resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==}
- resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==}
- engines: {node: '>=0.8.0'}
-
resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==}
@@ -3537,9 +3525,6 @@ packages:
resolution: {integrity: sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==}
- resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==}
-
resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}
engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
@@ -3583,10 +3568,6 @@ packages:
resolution: {integrity: sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==}
- resolution: {integrity: sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA==}
- engines: {node: '>= 4.8.0'}
-
resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}
engines: {node: '>= 6'}
@@ -3595,10 +3576,6 @@ packages:
resolution: {integrity: sha512-B7U/vJpE3DkJ5WXTgTpTRN63uV42DseiXXKMwG14LQBXmsdeIoHAPbU/MEo6II0k5ED74uc2ZGTC6MwHFQhF6w==}
engines: {node: 20 || >=22}
- resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==}
- deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting [email protected]
-
resolution: {integrity: sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==}
@@ -3630,10 +3607,6 @@ packages:
resolution: {integrity: sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==}
engines: {node: '>=18'}
- resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==}
- engines: {node: '>=4'}
-
resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==}
engines: {node: '>=8'}
@@ -3646,10 +3619,6 @@ packages:
resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==}
engines: {node: '>= 0.4'}
- resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==}
- engines: {node: '>= 0.4'}
-
resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==}
engines: {node: '>= 0.4'}
@@ -3728,20 +3697,12 @@ packages:
resolution: {integrity: sha512-yNZhyQYqXpkT0AKq3F3KLasUSK4fHvebNH5hOsKQw2dhGSALvQ4U0BqUc5suziKvydO5u5hgN2hy1RJaho8U5A==}
- resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==}
- deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.
-
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}
- resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==}
- engines: {node: '>= 0.10'}
-
resolution: {integrity: sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==}
engines: {node: '>=12.22.0'}
@@ -3764,9 +3725,6 @@ packages:
resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==}
- resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==}
-
resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}
engines: {node: '>=0.10.0'}
@@ -4059,10 +4017,6 @@ packages:
resolution: {integrity: sha512-CdaO738xRapbKIMVn2m4F6KTj4j7ooJ8POVnebSgKo3KBz5axNXRAL7ZdRjIV6NOr2Uf4vjtRkxrFETOioCqSA==}
engines: {node: '>= 12.0.0'}
- resolution: {integrity: sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==}
- engines: {node: '>= 0.6.0'}
-
resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==}
hasBin: true
@@ -4370,17 +4324,10 @@ packages:
resolution: {integrity: sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==}
engines: {node: '>=14.0.0'}
- resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==}
- engines: {node: '>=0.10.0'}
-
resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==}
engines: {node: '>=8'}
- resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==}
-
resolution: {integrity: sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==}
engines: {node: 20 || >=22}
@@ -4569,10 +4516,6 @@ packages:
resolution: {integrity: sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
- resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==}
- engines: {node: '>= 0.10'}
-
resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==}
engines: {node: '>=4'}
@@ -4606,10 +4549,6 @@ packages:
resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==}
- resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==}
- hasBin: true
-
resolution: {integrity: sha512-fiyq1RvW5/Br6iAtT8jN1XrNY8WPu2+yEypLbaijWry8WDZmn12azG9p/+c+qpEebURLlQmqCB8BNSu7ji+xQQ==}
engines: {node: '>= 0.8.0'}
@@ -4715,15 +4654,6 @@ packages:
resolution: {integrity: sha512-VsC6n6vz1ihYYyZZwX7YZSF5l5x36ca17OC+a69h94YqB7X6XLwf+5MOgynYir2SLFUbl8gIYvBo8K8RoNQ6bQ==}
engines: {node: '>= 0.4'}
- resolution: {integrity: sha512-vFILCw+lzUtiwBAHV8/Ex8JsFjelFMdhONIsgKNLgTzeRckp2AOYRQtHJE/9LhNvdMmE27AGtzWx0+DHpwIwSw==}
- engines: {node: '>= 4.0.0'}
-
- resolution: {integrity: sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==}
- engines: {node: '>=4'}
- hasBin: true
-
resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==}
engines: {node: '>= 0.4'}
@@ -4875,18 +4805,10 @@ packages:
engines: {node: '>=6.4.0'}
deprecated: Please upgrade to supertest v7.1.3+, see release notes at https://github.com/forwardemail/supertest/releases/tag/v7.1.3 - maintenance is supported by Forward Email @ https://forwardemail.net
- resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==}
- engines: {node: '>=4'}
-
resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==}
engines: {node: '>=8'}
- resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==}
- engines: {node: '>= 0.4'}
-
resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==}
@@ -7633,10 +7555,6 @@ snapshots:
- dependencies:
- color-convert: 1.9.3
-
dependencies:
color-convert: 2.0.1
@@ -7820,12 +7738,6 @@ snapshots:
- dependencies:
- ansi-styles: 3.2.1
- escape-string-regexp: 1.0.5
- supports-color: 5.5.0
-
dependencies:
ansi-styles: 4.3.0
@@ -8167,8 +8079,6 @@ snapshots:
-
dependencies:
'@types/estree': 1.0.9
@@ -8388,8 +8298,6 @@ snapshots:
-
optional: true
@@ -8463,14 +8371,6 @@ snapshots:
dependencies:
resolve-pkg-maps: 1.0.0
- dependencies:
- chalk: 2.4.2
- diff: 8.0.3
- loglevel: 1.9.1
- shelljs: 0.8.5
- shelljs.exec: 1.1.8
-
dependencies:
is-glob: 4.0.3
@@ -8481,15 +8381,6 @@ snapshots:
minipass: 7.1.2
path-scurry: 2.0.1
- dependencies:
- fs.realpath: 1.0.0
- inflight: 1.0.6
- inherits: 2.0.4
- minimatch: 10.2.4
- once: 1.4.0
- path-is-absolute: 1.0.1
-
@@ -8537,8 +8428,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
-
@@ -8547,10 +8436,6 @@ snapshots:
dependencies:
has-symbols: 1.1.0
- dependencies:
- function-bind: 1.1.2
-
dependencies:
function-bind: 1.1.2
@@ -8652,17 +8537,10 @@ snapshots:
cjs-module-lexer: 1.4.3
module-details-from-path: 1.0.4
- dependencies:
- once: 1.4.0
- wrappy: 1.0.2
-
-
dependencies:
'@ioredis/commands': 1.2.0
@@ -8695,10 +8573,6 @@ snapshots:
- dependencies:
- hasown: 2.0.2
-
@@ -8990,8 +8864,6 @@ snapshots:
safe-stable-stringify: 2.4.3
triple-beam: 1.4.1
-
dependencies:
js-tokens: 4.0.0
@@ -9275,12 +9147,8 @@ snapshots:
-
-
dependencies:
lru-cache: 11.2.5
@@ -9466,10 +9334,6 @@ snapshots:
process: 0.11.10
string_decoder: 1.3.0
- dependencies:
- resolve: 1.22.8
-
@@ -9504,12 +9368,6 @@ snapshots:
- dependencies:
- is-core-module: 2.13.1
- path-parse: 1.0.7
- supports-preserve-symlinks-flag: 1.0.0
-
dependencies:
depd: 2.0.0
@@ -9650,14 +9508,6 @@ snapshots:
-
- dependencies:
- glob: 7.2.3
- interpret: 1.4.0
- rechoir: 0.6.2
-
dependencies:
es-errors: 1.3.0
@@ -9817,16 +9667,10 @@ snapshots:
transitivePeerDependencies:
- supports-color
- dependencies:
- has-flag: 3.0.0
-
dependencies:
has-flag: 4.0.0
-
diff --git a/apps/api/src/lib/change-tracking-diff.ts b/apps/api/src/lib/change-tracking-diff.ts
new file mode 100644
index 000000000..f447df3c1
--- /dev/null
+++ b/apps/api/src/lib/change-tracking-diff.ts
@@ -0,0 +1,96 @@
+import { createTwoFilesPatch } from "diff";
+import parseDiff from "parse-diff";
+
+type StructuredMarkdownDiff = {
+ files: Array<{
+ from: string | null;
+ to: string | null;
+ chunks: Array<{
+ content: string;
+ changes: Array<{
+ type: string;
+ normal?: boolean;
+ add?: boolean;
+ del?: boolean;
+ ln?: number;
+ ln1?: number;
+ ln2?: number;
+ content: string;
+ }>;
+ }>;
+ }>;
+};
+
+export function createMarkdownChangeDiff(
+ previousMarkdown: string,
+ currentMarkdown: string,
+): { text: string; json: StructuredMarkdownDiff } | undefined {
+ const text = createTwoFilesPatch(
+ "previous.md",
+ "current.md",
+ previousMarkdown,
+ currentMarkdown,
+ "",
+ "",
+ { context: 3 },
+ );
+
+ const structured = parseDiff(text);
+ const hasChanges = structured.some(file =>
+ file.chunks.some(chunk =>
+ chunk.changes.some(
+ change => change.type === "add" || change.type === "del",
+ ),
+ ),
+ );
+
+ if (!hasChanges) return undefined;
+
+ return {
+ text,
+ json: {
+ files: structured.map(file => ({
+ from: file.from || null,
+ to: file.to || null,
+ chunks: file.chunks.map(chunk => ({
+ content: chunk.content,
+ changes: chunk.changes.map(change => {
+ const baseChange = {
+ type: change.type,
+ content: change.content,
+ };
+
+ if (
+ change.type === "normal" &&
+ "ln1" in change &&
+ "ln2" in change
+ ) {
+ return {
+ ...baseChange,
+ normal: true,
+ ln1: change.ln1,
+ ln2: change.ln2,
+ };
+ }
+ if (change.type === "add" && "ln" in change) {
+ return {
+ ...baseChange,
+ add: true,
+ ln: change.ln,
+ };
+ }
+ if (change.type === "del" && "ln" in change) {
+ return {
+ ...baseChange,
+ del: true,
+ ln: change.ln,
+ };
+ }
+
+ return baseChange;
+ }),
+ })),
+ })),
+ },
+ };
+}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.ts
index a36ed8ffc..4004bb766 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts
@@ -1,11 +1,10 @@
import { diffGetLastScrape } from "../../../db/rpc";
import { Document } from "../../../controllers/v1/types";
import { Meta } from "../index";
-import gitDiff from "git-diff";
-import parseDiff from "parse-diff";
import { generateCompletions } from "./llmExtract";
import { hasFormatOfType } from "../../../lib/format-utils";
import { getJobFromGCS } from "../../../lib/gcs-jobs";
+import { createMarkdownChangeDiff } from "../../../lib/change-tracking-diff";
async function extractDataWithSchema(
content: string,
@@ -161,58 +160,15 @@ export async function deriveDiff(
changeTrackingFormat?.modes?.includes("git-diff") &&
changeStatus === "changed"
) {
- const diffText = gitDiff(previousMarkdown, currentMarkdown, {
- color: false,
- wordDiff: false,
- });
- // meta.logger.debug("Diff text", { diffText });
- if (diffText) {
- const diffStructured = parseDiff(diffText);
- // meta.logger.debug("Diff structured", { diffStructured });
+ const diff = createMarkdownChangeDiff(
+ previousMarkdown,
+ currentMarkdown,
+ );
+ // meta.logger.debug("Diff text", { diffText: diff?.text });
+ if (diff) {
document.changeTracking.diff = {
- text: diffText,
- json: {
- files: diffStructured.map(file => ({
- from: file.from || null,
- to: file.to || null,
- chunks: file.chunks.map(chunk => ({
- content: chunk.content,
- changes: chunk.changes.map(change => {
- const baseChange = {
- type: change.type,
- content: change.content,
- };
-
- if (
- change.type === "normal" &&
- "ln1" in change &&
- "ln2" in change
- ) {
- return {
- ...baseChange,
- normal: true,
- ln1: change.ln1,
- ln2: change.ln2,
- };
- } else if (change.type === "add" && "ln" in change) {
- return {
- ...baseChange,
- add: true,
- ln: change.ln,
- };
- } else if (change.type === "del" && "ln" in change) {
- return {
- ...baseChange,
- del: true,
- ln: change.ln,
- };
- }
-
- return baseChange;
- }),
- })),
- })),
- },
+ text: diff.text,
+ json: diff.json,
};
}
}
diff --git a/apps/api/src/services/monitoring/diff.test.ts b/apps/api/src/services/monitoring/diff.test.ts
index df707ec0b..9c069d8de 100644
--- a/apps/api/src/services/monitoring/diff.test.ts
+++ b/apps/api/src/services/monitoring/diff.test.ts
@@ -1,4 +1,55 @@
-import { diffMonitorJson } from "./diff";
+import { existsSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createMarkdownChangeDiff } from "../../lib/change-tracking-diff";
+import { diffMonitorJson, diffMonitorMarkdown } from "./diff";
+
+describe("markdown change tracking diff", () => {
+ it("treats command substitution syntax as text", () => {
+ const marker = join(
+ tmpdir(),
+ `firecrawl-change-diff-${process.pid}-${Date.now()}`,
+ );
+ const current = `visible text $(touch ${marker}) and \`touch ${marker}.bt\``;
+
+ try {
+ const result = createMarkdownChangeDiff("visible text", current);
+
+ expect(result).toBeDefined();
+ expect(result?.text).toContain("$(touch");
+ expect(result?.text).toContain("`touch");
+ expect(result?.json.files).toBeInstanceOf(Array);
+ expect(existsSync(marker)).toBe(false);
+ expect(existsSync(`${marker}.bt`)).toBe(false);
+ } finally {
+ rmSync(marker, { force: true });
+ rmSync(`${marker}.bt`, { force: true });
+ }
+ });
+
+ it("uses the shell-free diff path for monitor markdown diffs", () => {
+ const marker = join(
+ tmpdir(),
+ `firecrawl-monitor-diff-${process.pid}-${Date.now()}`,
+ );
+
+ try {
+ const result = diffMonitorMarkdown(
+ "baseline",
+ `changed $(touch ${marker})`,
+ );
+
+ if (result.status !== "changed") {
+ throw new Error("Expected monitor markdown diff to change");
+ }
+ expect(result.text).toContain("$(touch");
+ expect(result.json.files).toBeInstanceOf(Array);
+ expect(existsSync(marker)).toBe(false);
+ } finally {
+ rmSync(marker, { force: true });
+ }
+ });
+});
describe("diffMonitorJson", () => {
it("returns `same` when every value matches", () => {
@@ -11,9 +62,10 @@ describe("diffMonitorJson", () => {
});
it("returns `same` regardless of object key order", () => {
- expect(
- diffMonitorJson({ a: 1, b: 2 }, { b: 2, a: 1 }),
- ).toEqual({ kind: "json", status: "same" });
+ expect(diffMonitorJson({ a: 1, b: 2 }, { b: 2, a: 1 })).toEqual({
+ kind: "json",
+ status: "same",
+ });
});
it("reports a single change for a top-level primitive that differs", () => {
diff --git a/apps/api/src/services/monitoring/diff.ts b/apps/api/src/services/monitoring/diff.ts
index 200be7912..b56527bea 100644
--- a/apps/api/src/services/monitoring/diff.ts
+++ b/apps/api/src/services/monitoring/diff.ts
@@ -1,5 +1,4 @@
-import gitDiff from "git-diff";
-import parseDiff from "parse-diff";
+import { createMarkdownChangeDiff } from "../../lib/change-tracking-diff";
type MonitorMarkdownDiffResult =
| {
@@ -60,60 +59,13 @@ export function diffMonitorMarkdown(
return { kind: "markdown", status: "same" };
}
- const text = gitDiff(previousMarkdown, currentMarkdown, {
- color: false,
- wordDiff: false,
- });
- const structured = parseDiff(text);
+ const diff = createMarkdownChangeDiff(previousMarkdown, currentMarkdown);
return {
kind: "markdown",
status: "changed",
- text,
- json: {
- files: structured.map(file => ({
- from: file.from || null,
- to: file.to || null,
- chunks: file.chunks.map(chunk => ({
- content: chunk.content,
- changes: chunk.changes.map(change => {
- const base = {
- type: change.type,
- content: change.content,
- };
-
- if (
- change.type === "normal" &&
- "ln1" in change &&
- "ln2" in change
- ) {
- return {
- ...base,
- normal: true,
- ln1: change.ln1,
- ln2: change.ln2,
- };
- }
- if (change.type === "add" && "ln" in change) {
- return {
- ...base,
- add: true,
- ln: change.ln,
- };
- }
- if (change.type === "del" && "ln" in change) {
- return {
- ...base,
- del: true,
- ln: change.ln,
- };
- }
-
- return base;
- }),
- })),
- })),
- },
+ text: diff?.text ?? "",
+ json: diff?.json ?? { files: [] },
};
}