Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

fix(engine): P4 audit loop 3: photo URLs capped and plain; long CSV cells read everywhere

Mahan Jafari887e2f1aefdf
diff --git a/engine/details.py b/engine/details.py
index c472be0..9dbe552 100644
--- a/engine/details.py
+++ b/engine/details.py
@@ -100,8 +100,12 @@ def open_hours(days):
return json.dumps({name: spans for _, (name, spans) in sorted(week.items())}, ensure_ascii=False)
+IMAGES_MAX, IMAGE_URL_MAX = 20, 1000 # keeps the cell far below csv's 128 KB field limit
+
+
def images(entity):
- """Photo URLs already supplied by Maps, as the existing images column's JSON objects."""
+ """Photo URLs already supplied by Maps, as the existing images column's JSON objects: plain https
+ links (no login, no whitespace), at most IMAGES_MAX of them."""
found, seen = [], set()
# Search cards and panels use different gallery layouts. Never include reviewer profiles.
groups = [(dig(entity, (171, 0)), (3, 0, 6, 0), (2,)),
@@ -112,17 +116,20 @@ def images(entity):
continue
for item in items:
url = dig(item, path)
- if not isinstance(url, str) or url in seen:
+ if (not isinstance(url, str) or url in seen or len(url) > IMAGE_URL_MAX
+ or any(c.isspace() or ord(c) < 32 for c in url)):
continue
try:
parsed = urllib.parse.urlsplit(url)
- if parsed.scheme != "https" or not parsed.hostname:
+ if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password:
continue
except ValueError:
continue
seen.add(url)
title = _text(dig(item, title_path)) if title_path else ""
found.append({"title": title, "image": url})
+ if len(found) == IMAGES_MAX:
+ return json.dumps(found, ensure_ascii=False)
return json.dumps(found, ensure_ascii=False) if found else ""
diff --git a/engine/jobs.py b/engine/jobs.py
index b6e7498..ce90f76 100644
--- a/engine/jobs.py
+++ b/engine/jobs.py
@@ -18,6 +18,8 @@ from datetime import datetime, timezone
import browser, details, maps
from cdp import CDPError, Page
+csv.field_size_limit(2 ** 31 - 1) # a long cell (a description, a 1.x file) can pass csv's 128 KB default
+
# Kit 1.x's CSV header, in its order, so files and scripts that read the old engine's output keep
# working. Our own columns follow.
COLUMNS = ["input_id", "link", "title", "category", "address", "open_hours", "popular_times", "website",
@@ -305,7 +307,6 @@ class Store:
old.close()
finally:
shutil.rmtree(tmp, ignore_errors=True)
- csv.field_size_limit(2 ** 31 - 1) # a 1.x cell (a long description) can pass csv's 128 KB default
n = 0
for jid, name, status, data, created in rows:
try:
diff --git a/engine/test_engine.py b/engine/test_engine.py
index 32fb796..c519ead 100644
--- a/engine/test_engine.py
+++ b/engine/test_engine.py
@@ -993,6 +993,12 @@ class ExtraDetailsParser(unittest.TestCase):
photo[6] = ["javascript:alert(1)"]
e[171] = None
self.assertNotIn("images", details.parse_entity(e))
+ for bad in ("https://u:[email protected]/x", "https://a.com/x\ny", "https://a.com/" + "x" * 1000):
+ photo[6] = [bad]
+ self.assertNotIn("images", details.parse_entity(e))
+ many = [[None] * 6 + [[f"https://lh3.googleusercontent.com/{n}"]] for n in range(50)]
+ e[37] = [many]
+ self.assertEqual(len(json.loads(details.parse_entity(e)["images"])), details.IMAGES_MAX)
class FullView(unittest.TestCase):
"""details.py fields that only Google's full view carries (a Chrome window), from made-up data."""
diff --git a/scripts/scrape.py b/scripts/scrape.py
index ef55693..0f32dc6 100755
--- a/scripts/scrape.py
+++ b/scripts/scrape.py
@@ -25,6 +25,8 @@ Notes:
"""
import argparse, csv, functools, io, ipaddress, json, math, os, re, socket, sys, time
import html as _html, http.client, urllib.request, urllib.parse, urllib.error
+
+csv.field_size_limit(2 ** 31 - 1) # a long cell (a description) can pass csv's 128 KB default
from concurrent.futures import ThreadPoolExecutor
# The status lines below use ▶ ✓ ℹ •, and every file this script writes can hold accented business