Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(crawl-redis): reduce `crawl:<id>:visited` size in Redis by 16x (#1936)
Gergő Móricz7611f819ca37
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex e0a822217..65bd96878 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -223,6 +223,19 @@ export function normalizeURL(url: string, sc: StoredCrawl): string {return urlO.href;}+// For this function and the infrastructure surrounding it to work correctly, this function must:+// 1. Return the a non-zero number of permutations for all valid URLs.+// generateURLPermutations(url).length > 0+// 2. The generated permutations of the returned array's members must be the same as the original generated permutations.+// generateURLPermutations(url) == generateURLPermutations(generateURLPermutations(url)[n])+// Obviously this is not valid in JS, but you get the idea.+// 3. Two generated permutations of signficantly different URLs may not have any overlap.+// In practice, this means that if there is a generated array of permutations, there must be no URL that is+// 1. not included in that array, and+// 2. has a permutation that is included in that array.+//+// Points 1 and 2 are proven in permu-refactor.test.ts, point 3 is not as proving a negative is hard and outside the scope of a web crawler.+// - mogeryexport function generateURLPermutations(url: string | URL): URL[] {const urlO = new URL(url);@@ -302,29 +315,9 @@ export async function lockURL(if (!sc.crawlerOptions?.deduplicateSimilarURLs) {res = (await redisEvictConnection.sadd("crawl:" + id + ":visited", url)) !== 0;} else {- const permutations = generateURLPermutations(url).map((x) => x.href);- const x = await redisEvictConnection.sadd(- "crawl:" + id + ":visited",- ...permutations,- );- // TEMP: proving ENG-3085- if (x === permutations.length || x === 0) {- logger.debug("proof OK", {- proof: "ENG-3085",- permutations: permutations.length,- result: x,- zeroDataRetention: sc.zeroDataRetention,- });- } else {- logger.warn("proof failed", {- proof: "ENG-3085",- permutations: permutations.length,- result: x,- url,- zeroDataRetention: sc.zeroDataRetention,- });- }- res = x === permutations.length;+ const permutation = generateURLPermutations(url)[0].href;+ const x = await redisEvictConnection.sadd("crawl:" + id + ":visited", permutation);+ res = x !== 0;}await redisEvictConnection.expire("crawl:" + id + ":visited", 24 * 60 * 60);@@ -370,9 +363,7 @@ export async function lockURLs(const x = await redisEvictConnection.sadd("crawl:" + id + ":visited", ...urls);res = x === urls.length;} else {- const allPermutations = urls.flatMap((url) =>- generateURLPermutations(url).map((x) => x.href),- );+ const allPermutations = urls.map(url => generateURLPermutations(url)[0].href);logger.debug("Adding " + allPermutations.length + " URL permutations...");const x = await redisEvictConnection.sadd("crawl:" + id + ":visited",diff --git a/apps/api/src/lib/permu-refactor.test.ts b/apps/api/src/lib/permu-refactor.test.tsnew file mode 100644index 000000000..58ff05327--- /dev/null+++ b/apps/api/src/lib/permu-refactor.test.ts@@ -0,0 +1,300 @@+function generateURLPermutations(url: string | URL): URL[] {+ const urlO = new URL(url);++ // Construct two versions, one with www., one without+ const urlWithWWW = new URL(urlO);+ const urlWithoutWWW = new URL(urlO);+ if (urlO.hostname.startsWith("www.")) {+ urlWithoutWWW.hostname = urlWithWWW.hostname.slice(4);+ } else {+ urlWithWWW.hostname = "www." + urlWithoutWWW.hostname;+ }++ let permutations = [urlWithWWW, urlWithoutWWW];++ // Construct more versions for http/https+ permutations = permutations.flatMap((urlO) => {+ if (!["http:", "https:"].includes(urlO.protocol)) {+ return [urlO];+ }++ const urlWithHTTP = new URL(urlO);+ const urlWithHTTPS = new URL(urlO);+ urlWithHTTP.protocol = "http:";+ urlWithHTTPS.protocol = "https:";++ return [urlWithHTTP, urlWithHTTPS];+ });++ // Construct more versions for index.html/index.php+ permutations = permutations.flatMap((urlO) => {+ const urlWithHTML = new URL(urlO);+ const urlWithPHP = new URL(urlO);+ const urlWithBare = new URL(urlO);+ const urlWithSlash = new URL(urlO);++ if (urlO.pathname.endsWith("/")) {+ urlWithBare.pathname = urlWithBare.pathname.length === 1 ? urlWithBare.pathname : urlWithBare.pathname.slice(0, -1);+ urlWithHTML.pathname += "index.html";+ urlWithPHP.pathname += "index.php";+ } else if (urlO.pathname.endsWith("/index.html")) {+ urlWithPHP.pathname = urlWithPHP.pathname.slice(0, -"index.html".length) + "index.php";+ urlWithSlash.pathname = urlWithSlash.pathname.slice(0, -"index.html".length);+ urlWithBare.pathname = urlWithBare.pathname.slice(0, -"/index.html".length);+ } else if (urlO.pathname.endsWith("/index.php")) {+ urlWithHTML.pathname = urlWithHTML.pathname.slice(0, -"index.php".length) + "index.html";+ urlWithSlash.pathname = urlWithSlash.pathname.slice(0, -"index.php".length);+ urlWithBare.pathname = urlWithBare.pathname.slice(0, -"/index.php".length);+ } else {+ urlWithSlash.pathname += "/";+ urlWithHTML.pathname += "/index.html";+ urlWithPHP.pathname += "/index.php";+ }++ return [urlWithHTML, urlWithPHP, urlWithSlash, urlWithBare];+ });++ return [...new Set(permutations.map(x => x.href))].map(x => new URL(x));+}++// Random URLs generated by Claude.+// This should suffice. - mogery+const urls = `http://www.example.com/index.html+https://subdomain.test.co.uk/path/to/resource?id=123&name=test+https://api.github.com/users/john-doe/repos?sort=created&direction=desc+http://shop.example.net/products/category/subcategory/item-name?color=blue&size=large&discount=20#reviews+https://www.google.com/search?q=url+parsing&hl=en&source=hp&ei=abc123+https://en.wikipedia.org/wiki/URL#History+http://mail.google.com/mail/u/0/#inbox/FMfcgzGpg+https://www.amazon.co.jp/dp/B08N5WRWNW/ref=sr_1_1?keywords=echo&qid=1614556+https://subdomain1.subdomain2.example.com/path/index.html?param1=value1¶m2=value2+https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLx123&index=5+https://docs.google.com/spreadsheets/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/edit#gid=0+http://stackoverflow.com/questions/3809401/what-is-a-good-regular-expression-to-match-a-url+https://www.linkedin.com/in/john-doe-123456789/+https://app.slack.com/client/T0123456/C0123456789/thread/C0123456789-1614556789.123456+http://www.example.com/page.html?utm_source=google&utm_medium=cpc&utm_campaign=spring_sale+https://github.com/user/repo/blob/main/src/components/Header.jsx#L42+https://www.reddit.com/r/programming/comments/abc123/title_of_post/?sort=top+http://bit.ly/3abc123+https://tinyurl.com/yckten7u+https://www.example.com/index.php?id=123&action=view&session=abc123def456+http://sub-domain.example-site.co.uk/path/to/file.html+https://www.airbnb.com/rooms/123456?check_in=2024-03-15&check_out=2024-03-20&guests=2+https://twitter.com/username/status/1234567890123456789+http://www.example.com/search?q=test+query+with+spaces&filter=recent+https://drive.google.com/file/d/1a2b3c4d5e6f7g8h9i0j/view?usp=sharing+http://example.com/index.html+https://example.com/secure/page.html+https://www.example.com/api/v2/users/123/posts?limit=10&offset=20&sort=date:desc+https://maps.google.com/maps?q=40.7128,-74.0060&z=15+http://news.ycombinator.com/item?id=123456+https://www.facebook.com/photo.php?fbid=123456789&set=a.987654321&type=3+http://example.org/path/to/deeply/nested/resource/index.html+https://medium.com/@username/article-title-here-abc123def456+http://localhost/api/v1/users+https://www.instagram.com/p/ABC123def456/+http://test.example.com/~user/personal/page.html+https://www.npmjs.com/package/express?activeTab=readme+http://blog.example.net/2024/03/15/post-title-with-dashes/+https://codepen.io/username/pen/abcDEF?editors=1111+http://www.example.edu/students/enrollment.php?semester=fall2024&course=CS101+https://www.ebay.com/itm/123456789012?var=987654321098+http://archive.org/web/20210301000000/http://example.com/+https://www.twitch.tv/username/clip/ClipNameHere-abc123def456+http://example.com/products/electronics/phones/iphone.html?model=15&color=blue+https://gitlab.com/group/project/-/merge_requests/123+http://www.example.com/blog/index.html+https://store.steampowered.com/app/730/Counter-Strike_Global_Offensive/+http://example.net/api/users.json?page=1&per_page=50+https://www.spotify.com/us/track/4u7EnebtmKWzUH433cf5Qv+http://subdomain.example.com/path%20with%20spaces/file.html+https://www.goodreads.com/book/show/12345.Book_Title_Here+http://example.com/search?q=test&lang=en&safe=on&start=10+https://marketplace.visualstudio.com/items?itemName=ms-python.python+http://www.example.org/downloads/software-v1.2.3.tar.gz+https://codesandbox.io/s/react-example-abc123+http://example.com/category/subcategory/index.html?sort=price&order=asc+https://www.udemy.com/course/course-name-here/?couponCode=DISCOUNT50+http://api.example.com/v3/data.xml?format=json&key=public123+https://www.etsy.com/listing/123456789/handmade-item-title+http://cdn.example.net/images/photo_2024_03_15.jpg?w=800&h=600&crop=true+https://dev.to/username/article-slug-here-abc1+http://example.com/path/../another/path/file.html+https://www.behance.net/gallery/123456789/Project-Title+http://www.example.com/form.html?name=John+Doe&email=john%40example.com+https://stackexchange.com/questions/123456/question-title-with-words+http://staging.example.com/test/environment/page.html+https://www.coursera.org/learn/machine-learning?action=enroll+http://example.net/~user/public_html/index.html+https://vimeo.com/123456789+http://blog.example.com/feed.rss+https://www.pinterest.com/pin/123456789012345678/+http://example.org/cgi-bin/script.cgi?action=list&type=all+https://unsplash.com/photos/abc123def456+http://www.example.com/products.php?category=1&brand=2&price_min=10&price_max=100+https://jsfiddle.net/username/abc123/+http://mirror.example.net/pub/software/package-1.0.0.rpm+https://www.figma.com/file/abc123def456/Design-File-Name+http://example.com/api/endpoint?callback=jsonpCallback123+https://open.spotify.com/playlist/37i9dQZF1DXcBWIGoYBM5M+http://download.example.org/files/document.pdf?token=temp123+https://chrome.google.com/webstore/detail/extension-name/abcdefghijklmnop+http://www.example.com/path/to/page.html#section-heading+https://hub.docker.com/r/library/nginx+http://example.net/search?q=query&filters[price][min]=10&filters[price][max]=100+https://www.patreon.com/username/posts?tag=tutorial+http://static.example.com/assets/css/style.min.css?v=1.2.3+https://soundcloud.com/artist-name/track-name+http://www.example.org/docs/api/reference.html#authentication+https://codeberg.org/username/repository+http://example.com/product/item-name/reviews?page=2&rating=5+https://www.kickstarter.com/projects/creator/project-name+http://api.example.net/graphql?query={user(id:123){name,email}}+https://gist.github.com/username/abc123def456789012345678901234567890+http://www.example.com/articles/2024/march/article-title.html+https://discord.com/channels/123456789/987654321+http://example.org/download.php?file=document.docx&version=2+https://observablehq.com/@username/notebook-title+http://test.example.com/api/v1/status.json+https://www.deviantart.com/username/art/Art-Title-123456789+http://example.net/path/to/resource/index.html?debug=true+https://repl.it/@username/project-name+http://www.example.com/shop/cart.php?action=add&product_id=456+https://www.last.fm/music/Artist+Name/Album+Name+http://cdn.example.org/media/video.mp4?quality=1080p+https://exercism.io/tracks/javascript/exercises/hello-world+http://example.com/api/search?query=test&fields=id,name,description+https://www.meetup.com/group-name/events/123456789/+http://archive.example.net/2024/03/15/snapshot.html+https://airtable.com/shrABC123DEF456+http://www.example.com/form?step=1&ref=homepage+https://www.producthunt.com/posts/product-name+http://api.example.org/rest/v2/users/123.json+https://glitch.com/~project-name+http://example.net/catalog/item.html?id=789&variant=red+https://www.skillshare.com/classes/Class-Title/123456789+http://demo.example.com/preview/template-1/index.html+https://dribbble.com/shots/123456-Shot-Title+http://www.example.org/newsletter/subscribe.html?email=test%40test.com+https://www.npmjs.com/search?q=keywords:react+http://example.com/api/data?format=csv&delimiter=comma+https://colab.research.google.com/drive/1abc123def456+http://media.example.net/thumbnails/image_300x200.jpg+https://www.notion.so/Page-Title-abc123def456+http://example.org/path/to/file.html?lang=en-US+https://www.canva.com/design/DAE123abc/edit+http://beta.example.com/new-features/test.html+https://www.freecodecamp.org/learn/javascript-algorithms-and-data-structures/+http://example.net/api/v1/auth/login.json+https://www.hackerrank.com/challenges/challenge-name/problem+http://www.example.com/blog/tags/javascript/page/2+https://replit.com/@username/project-name#file.js+http://assets.example.org/fonts/font-family.woff2+https://www.kaggle.com/datasets/username/dataset-name+http://example.com/products/index.html?sort=newest&limit=20+https://zenodo.org/record/123456+http://www.example.net/documentation/v2.0/getting-started.html+https://www.openstreetmap.org/#map=15/40.7128/-74.0060+http://example.org/api/export?format=pdf&orientation=landscape+https://codewars.com/kata/kata-id-here+http://staging.example.com/test?env=development&debug=1+https://www.atlassian.com/software/jira/pricing+http://example.com/path/to/page.aspx?id=123&mode=edit+https://bundlephobia.com/package/[email protected]+http://www.example.org/events/2024/conference.html#speakers+https://regex101.com/r/abc123/1+http://api.example.net/v1/search?q=keyword&limit=10&offset=0+https://www.typescriptlang.org/docs/handbook/intro.html+http://example.com/user/profile.php?username=johndoe+https://marketplace.zoom.us/apps/abc123+http://www.example.net/resources/whitepaper.pdf?download=true+https://www.pluralsight.com/courses/course-name+http://example.org/api/webhook?event=user.created+https://httpbin.org/get?test=value+http://demo.example.com/sandbox/index.html+https://www.jetbrains.com/idea/download/#section=windows+http://example.net/path/to/deeply/nested/folder/file.html+https://www.heroku.com/platform+http://www.example.com/contact.html?subject=inquiry&priority=high+https://material-ui.com/components/buttons/+http://api.example.org/oauth/authorize?client_id=123&redirect_uri=http://example.com+https://www.digitalocean.com/community/tutorials/tutorial-slug+http://example.com/search/results.php?query=test&page=3+https://vercel.com/new/templates+http://www.example.net/downloads/index.html?category=software+https://www.postman.com/downloads/+http://example.org/api/v2/data.json?include=metadata+https://tailwindcss.com/docs/installation+http://media.example.com/stream/video.m3u8+https://www.elastic.co/guide/index.html+http://example.net/admin/dashboard/index.html+https://nodejs.org/dist/latest-v18.x/docs/api/+http://www.example.com/shop/checkout.html?coupon=SAVE20+https://www.mongodb.com/cloud/atlas/register+http://example.org/files/archive.zip?extract=true+https://bundler.io/guides/getting_started.html+http://api.example.net/graphql/playground+https://www.selenium.dev/documentation/+http://example.com/blog/post.html?id=456&comments=true+https://flutter.dev/docs/get-started/install+http://www.example.org/gallery/photos/2024/index.html+https://www.djangoproject.com/start/+http://example.net/api/status?format=json&verbose=true+https://reactjs.org/tutorial/tutorial.html+http://cdn.example.com/js/app.bundle.js?v=3.2.1+https://www.ruby-lang.org/en/documentation/+http://example.org/search?facets[category][]=tech&facets[category][]=science+https://pytorch.org/get-started/locally/+http://www.example.com/faq.html#payment-methods+https://www.tensorflow.org/tutorials+http://example.net/export/data.csv?encoding=utf-8+https://vuejs.org/guide/introduction.html+http://api.example.org/v3/metrics?period=daily&start=2024-01-01+https://symfony.com/doc/current/index.html+http://example.com/products/sale/index.html?discount=50+https://laravel.com/docs/10.x/installation+http://www.example.net/api/batch?operations=create,update,delete+https://angular.io/guide/setup-local+http://example.org/tools/calculator.html?mode=scientific+https://expressjs.com/en/starter/installing.html+http://static.example.com/images/logo.svg?color=blue+https://spring.io/guides/gs/spring-boot/+http://example.net/path/to/file.xml?transform=json+https://www.rust-lang.org/learn/get-started+http://www.example.com/newsletter/archive/2024/march.html+https://kotlinlang.org/docs/getting-started.html+http://example.org/api/search?facets=true&hl=true&hl.fl=title,description+https://dart.dev/guides/language/language-tour+http://media.example.net/audio/podcast-episode-123.mp3+https://golang.org/doc/install+http://example.com/user/settings.html?tab=privacy+https://www.scala-lang.org/download/+http://www.example.org/reports/annual/2024/index.html+https://elixir-lang.org/getting-started/introduction.html+http://api.example.net/v1/aggregate?groupby=category&metric=count+https://www.haskell.org/platform/+http://example.org/documentation/api/v2/index.html#endpoints+https://julialang.org/downloads/+http://www.example.com/store/product.php?sku=ABC123&reviews=show+https://www.erlang.org/downloads+http://example.net/calendar/events.html?month=march&year=2024+https://clojure.org/guides/getting_started+http://cdn.example.org/videos/tutorial-01.webm+https://fsharp.org/learn/`.split("\n");++describe("Proof that generateURLPermutations is stable", () => {+ describe.each(urls)("%s", (url) => {+ it("any permutation of a URL will produce the same permutations (content, length, order!) as the permutations of the original URL", () => {+ const permutations = generateURLPermutations(new URL(url));+ expect(permutations).toBeInstanceOf(Array);+ expect(permutations.length).toBeGreaterThan(0);++ permutations.forEach(newBase => {+ const newPermutations = generateURLPermutations(newBase);+ expect(newPermutations.length).toBe(permutations.length);+ expect(newPermutations.every((p, i) => p.href === permutations[i].href)).toBe(true);+ });+ });+ });+});\ No newline at end of file