Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: preserve gid parameter when rewriting Google Sheets URLs (#2693)
devin-ai-integration[bot]121fa3cab686
diff --git a/apps/api/src/scraper/scrapeURL/lib/__tests__/rewriteUrl.test.ts b/apps/api/src/scraper/scrapeURL/lib/__tests__/rewriteUrl.test.tsindex c115c0091..08c5fe600 100644--- a/apps/api/src/scraper/scrapeURL/lib/__tests__/rewriteUrl.test.ts+++ b/apps/api/src/scraper/scrapeURL/lib/__tests__/rewriteUrl.test.ts@@ -64,6 +64,30 @@ describe("rewriteUrl", () => {);});+ it("should preserve gid parameter from query string for specific sheet tab", () => {+ const url =+ "https://docs.google.com/spreadsheets/d/1dhyxGttUbI2RlTxPXF4CQcY4TD2k6Jp7-hcqS9PP5uc/edit?gid=89683736";+ expect(rewriteUrl(url)).toBe(+ "https://docs.google.com/spreadsheets/d/1dhyxGttUbI2RlTxPXF4CQcY4TD2k6Jp7-hcqS9PP5uc/gviz/tq?tqx=out:html&gid=89683736",+ );+ });++ it("should preserve gid parameter from hash fragment for specific sheet tab", () => {+ const url =+ "https://docs.google.com/spreadsheets/d/1dhyxGttUbI2RlTxPXF4CQcY4TD2k6Jp7-hcqS9PP5uc/edit#gid=89683736";+ expect(rewriteUrl(url)).toBe(+ "https://docs.google.com/spreadsheets/d/1dhyxGttUbI2RlTxPXF4CQcY4TD2k6Jp7-hcqS9PP5uc/gviz/tq?tqx=out:html&gid=89683736",+ );+ });++ it("should preserve gid parameter when both query and hash have gid (uses first match)", () => {+ const url =+ "https://docs.google.com/spreadsheets/d/1dhyxGttUbI2RlTxPXF4CQcY4TD2k6Jp7-hcqS9PP5uc/edit?gid=89683736#gid=89683736";+ expect(rewriteUrl(url)).toBe(+ "https://docs.google.com/spreadsheets/d/1dhyxGttUbI2RlTxPXF4CQcY4TD2k6Jp7-hcqS9PP5uc/gviz/tq?tqx=out:html&gid=89683736",+ );+ });+it("should NOT rewrite published Google Sheets URLs (/d/e/)", () => {const url ="https://docs.google.com/spreadsheets/d/e/2PACX-1vSomePublishId/pubhtml";diff --git a/apps/api/src/scraper/scrapeURL/lib/rewriteUrl.ts b/apps/api/src/scraper/scrapeURL/lib/rewriteUrl.tsindex 1a56429f7..0b817f801 100644--- a/apps/api/src/scraper/scrapeURL/lib/rewriteUrl.ts+++ b/apps/api/src/scraper/scrapeURL/lib/rewriteUrl.ts@@ -43,7 +43,10 @@ export function rewriteUrl(url: string): string | undefined {}const id = url.match(/\/spreadsheets\/d\/([-\w]+)/)?.[1];if (id) {- return `https://docs.google.com/spreadsheets/d/${id}/gviz/tq?tqx=out:html`;+ // Extract gid parameter from query string or hash fragment to preserve the selected tab+ const gidMatch = url.match(/[?&#]gid=(\d+)/);+ const gidParam = gidMatch ? `&gid=${gidMatch[1]}` : "";+ return `https://docs.google.com/spreadsheets/d/${id}/gviz/tq?tqx=out:html${gidParam}`;}}