Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(server): fail a compressed page cut off part way instead of hanging, and move an inactive workspace's catalog sources to the back of the daily run (#1512)
Gökhan Geyik2bb83463912d
diff --git a/server/src/lib/catalog/sources.js b/server/src/lib/catalog/sources.jsindex 55bbab8ff..1ef4fa701 100644--- a/server/src/lib/catalog/sources.js+++ b/server/src/lib/catalog/sources.js@@ -211,10 +211,19 @@ async function syncOnce(return outcome;}+/** What a source of a workspace without an active subscription records, instead of a read. */+export const NOT_READ_INACTIVE = {+ status: 'failed',+ error: "Not read: this workspace's subscription isn't active.",+};+/*** The daily run: feeds and stores not read for SYNC_EVERY_MS, the longest* waiting first, SWEEP_LIMIT a run, one at a time, until SWEEP_DEADLINE_MS.- * On the managed cloud, only workspaces with an active subscription.+ * On the managed cloud, only workspaces with an active subscription are+ * read. A source of any other workspace records that it wasn't read, which+ * also moves it to the back of the line: left untouched, those sources would+ * stay first in line every day and could fill a run's every slot.*/export async function sweepCatalogSources({db = supabaseAdmin,@@ -236,10 +245,15 @@ export async function sweepCatalogSources({.limit(SWEEP_LIMIT);if (error) throw new Error(`catalog_sources read: ${error.message}`);let synced = 0;+ let inactive = 0;for (const source of data ?? []) {if (Date.now() - started > SWEEP_DEADLINE_MS) break;- if (!isSubscriptionActive(source.brands?.organizations?.subscription_status)) continue;try {+ if (!isSubscriptionActive(source.brands?.organizations?.subscription_status)) {+ await record(db, source.id, now.toISOString(), NOT_READ_INACTIVE);+ inactive++;+ continue;+ }await syncSource(source, { db, now, fetch, request, connectors });synced++;} catch (err) {@@ -249,5 +263,5 @@ export async function sweepCatalogSources({);}}- return { due: (data ?? []).length, synced };+ return { due: (data ?? []).length, synced, inactive };}diff --git a/server/src/lib/catalog/sources.test.js b/server/src/lib/catalog/sources.test.jsindex 309ac3da0..edc474f1e 100644--- a/server/src/lib/catalog/sources.test.js+++ b/server/src/lib/catalog/sources.test.js@@ -325,7 +325,7 @@ describe('redact', () => {});describe('sweepCatalogSources', () => {- it('syncs the due feeds, skipping workspaces without an active subscription on the cloud', async () => {+ it("syncs the due feeds, and records that an inactive workspace's were not read", async () => {process.env.IS_CLOUD = 'true';const [first] = fake.rows('catalog_sources');first.brands = { organizations: { subscription_status: 'active' } };@@ -340,9 +340,29 @@ describe('sweepCatalogSources', () => {});const fetch = vi.fn().mockResolvedValue({ html: fixture('google-feed.xml') });const result = await sweepCatalogSources({ db: fake.db, now: NOW, fetch });- expect(result).toEqual({ due: 2, synced: 1 });+ expect(result).toEqual({ due: 2, synced: 1, inactive: 1 });expect(fetch).toHaveBeenCalledTimes(1);expect(fetch.mock.calls[0][0]).toBe('https://shop.example/feed.xml');+ // Not read, and said so: it goes to the back of the line instead of+ // staying first every day.+ expect(fake.rows('catalog_sources').find((s) => s.brand_id === 'brand-2')).toMatchObject({+ last_synced_at: NOW.toISOString(),+ last_status: 'failed',+ last_error: "Not read: this workspace's subscription isn't active.",+ last_counts: null,+ });+ expect(fake.rows('catalog_products').every((p) => p.brand_id === 'brand-1')).toBe(true);+ });++ it("keeps a workspace's own catalog for when its subscription is active again", async () => {+ process.env.IS_CLOUD = 'true';+ const [first] = fake.rows('catalog_sources');+ first.brands = { organizations: { subscription_status: 'past_due' } };+ const fetch = vi.fn();+ await sweepCatalogSources({ db: fake.db, now: NOW, fetch });+ expect(fetch).not.toHaveBeenCalled();+ // Recorded as not read; its products are left as they were, not marked removed.+ expect(fake.writes.filter((w) => w.table !== 'catalog_sources')).toEqual([]);});it('reads stores too, and never an upload', async () => {@@ -371,7 +391,7 @@ describe('sweepCatalogSources', () => {fetch,connectors: { feed_url: feedUrl },});- expect(result).toEqual({ due: 2, synced: 2 });+ expect(result).toEqual({ due: 2, synced: 2, inactive: 0 });expect(fetch).toHaveBeenCalledTimes(1);const tried = fake.rows('catalog_sources').filter((s) => s.last_status);expect(tried.map((s) => [s.kind, s.last_status]).sort()).toEqual([diff --git a/server/src/lib/safe-fetch.js b/server/src/lib/safe-fetch.jsindex 5d5f46968..5e14b07e5 100644--- a/server/src/lib/safe-fetch.js+++ b/server/src/lib/safe-fetch.js@@ -20,6 +20,7 @@ import dns from 'node:dns';import http from 'node:http';import https from 'node:https';import net from 'node:net';+import { pipeline } from 'node:stream';import { TextDecoder } from 'node:util';import zlib from 'node:zlib';@@ -200,10 +201,20 @@ function requestOnce(url, { signal, lookup, maxBytes, truncate, types, accept })return;}const encoding = String(res.headers['content-encoding'] ?? '').toLowerCase();- let body = res;- if (encoding === 'gzip' || encoding === 'x-gzip') body = res.pipe(zlib.createGunzip());- else if (encoding === 'deflate') body = res.pipe(zlib.createInflate());- else if (encoding === 'br') body = res.pipe(zlib.createBrotliDecompress());+ let decoder = null;+ if (encoding === 'gzip' || encoding === 'x-gzip') decoder = zlib.createGunzip();+ else if (encoding === 'deflate') decoder = zlib.createInflate();+ else if (encoding === 'br') decoder = zlib.createBrotliDecompress();+ // pipeline, not pipe: a page cut off part way fails the read instead+ // of leaving its decoder waiting for an end that never comes, past+ // the timeout (the request is over by then).+ if (decoder) pipeline(res, decoder, (err) => err && reject(err));+ const body = decoder ?? res;+ res.on('close', () => {+ if (!res.complete) {+ reject(new PageFetchError('unreachable', 'The page stopped part way through'));+ }+ });const chunks = [];let size = 0;diff --git a/server/src/lib/safe-fetch.test.js b/server/src/lib/safe-fetch.test.jsindex 3aec610f3..e537774db 100644--- a/server/src/lib/safe-fetch.test.js+++ b/server/src/lib/safe-fetch.test.js@@ -145,6 +145,17 @@ describe('fetchPage against a local site', () => {res.end('late');}, 300);},+ // The connection drops part way through the body.+ '/cut': (res) => {+ res.writeHead(200, { 'content-type': 'text/html', 'content-length': '1000' });+ res.write('<p>part');+ setTimeout(() => res.socket.destroy(), 10);+ },+ '/cut-gzip': (res) => {+ res.writeHead(200, { 'content-type': 'text/html', 'content-encoding': 'gzip' });+ res.write(zlib.gzipSync(`<p>${'x'.repeat(10_000)}</p>`).subarray(0, 40));+ setTimeout(() => res.socket.destroy(), 10);+ },};beforeAll(async () => {@@ -205,6 +216,15 @@ describe('fetchPage against a local site', () => {).resolves.toMatchObject({ html: '<rss version="2.0"><channel></channel></rss>' });});+ it('fails a page cut off part way, compressed or not, instead of waiting for it', async () => {+ // Past its own timeout, a compressed page cut off would wait forever: the+ // request is already over, so the timeout has nothing left to stop.+ await expect(get('/cut', { timeoutMs: 2000 })).rejects.toMatchObject({ code: 'unreachable' });+ await expect(get('/cut-gzip', { timeoutMs: 2000 })).rejects.toMatchObject({+ code: 'unreachable',+ });+ });+it('reads the start of a page over the cap when asked to, instead of refusing it', async () => {await expect(get('/big', { maxBytes: 1_000, truncate: true })).resolves.toMatchObject({status: 200,