Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

fix(kit): P5 audit loop 3: broken preset files leave the engine up, fixtures off example domains

Mahan Jafari11dabc1767d5
diff --git a/engine/serve.py b/engine/serve.py
index 82a1471..24773fc 100644
--- a/engine/serve.py
+++ b/engine/serve.py
@@ -43,8 +43,23 @@ SPONSORS = "https://raw.githubusercontent.com/Mahanaicoach/google-maps-scraper-k
CATEGORIES = ("https://raw.githubusercontent.com/lobstrio/google-business-categories/"
"4ee1c4788d07c812a32986177c2a51c28957062f/data/google-business-categories.txt")
PRESETS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "presets")
-with open(os.path.join(PRESETS, "business-types.json"), encoding="utf-8") as _f:
- TYPES = json.load(_f)
+
+
+def load_types(folder=None):
+ """The What list; [] with one warning when the file is missing or broken, so the API and the scripts
+ still work (only the web UI's What box is empty)."""
+ try:
+ with open(os.path.join(folder or PRESETS, "business-types.json"), encoding="utf-8") as f:
+ types = json.load(f)
+ if not isinstance(types, list):
+ raise ValueError("not a list")
+ return types
+ except (OSError, ValueError) as e:
+ print(f"presets/business-types.json unusable ({e.__class__.__name__}): the What box will be empty", file=sys.stderr)
+ return []
+
+
+TYPES = load_types()
TYPES_JS = json.dumps(TYPES).replace("</", "<\\/") # the What list, inlined in the page script
_cities, _cities_lock = None, threading.Lock()
ENV_KEYS = ("SCRAPER_PORT", "SCRAPER_API_KEY", "SCRAPER_HEADLESS")
@@ -83,11 +98,14 @@ def cities():
with _cities_lock:
if _cities is None:
rows = []
- with gzip.open(os.path.join(PRESETS, "cities.tsv.gz"), "rt", encoding="utf-8") as f:
- for line in f:
- name, admin, cc, lat, lon = line.rstrip("\n").split("\t")
- label = ", ".join([name] + ([admin] if admin and admin != name else []) + [cc])
- rows.append("\t".join((fold(label), label, lat, lon, cc)))
+ try:
+ with gzip.open(os.path.join(PRESETS, "cities.tsv.gz"), "rt", encoding="utf-8") as f:
+ for line in f:
+ name, admin, cc, lat, lon = line.rstrip("\n").split("\t")
+ label = ", ".join([name] + ([admin] if admin and admin != name else []) + [cc])
+ rows.append("\t".join((fold(label), label, lat, lon, cc)))
+ except (OSError, EOFError, ValueError) as e: # missing, truncated or odd file: what was read, once
+ log(f"presets/cities.tsv.gz unusable after {len(rows)} cities ({e.__class__.__name__})")
_cities = rows
return _cities
diff --git a/engine/test_engine.py b/engine/test_engine.py
index a25de61..7da41be 100644
--- a/engine/test_engine.py
+++ b/engine/test_engine.py
@@ -1292,6 +1292,22 @@ class Presets(Server):
self.assertEqual(code, 200)
return json.loads(body)
+ def test_broken_preset_files_do_not_take_the_engine_down(self):
+ tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, tmp, True)
+ self.assertEqual(serve.load_types(tmp), []) # no file
+ with open(os.path.join(tmp, "business-types.json"), "w") as f:
+ f.write("{not json")
+ self.assertEqual(serve.load_types(tmp), [])
+ with open(os.path.join(tmp, "cities.tsv.gz"), "wb") as f: # the real file cut short
+ with open(os.path.join(serve.PRESETS, "cities.tsv.gz"), "rb") as g:
+ f.write(g.read(5000))
+ with unittest.mock.patch.object(serve, "PRESETS", tmp), unittest.mock.patch.object(serve, "_cities", None):
+ code, body, _ = self.call("GET", "/api/v1/cities?q=aus")
+ self.assertEqual(code, 200)
+ self.assertIsInstance(json.loads(body), list)
+ self.assertEqual(self.call("GET", "/api/v1/jobs")[0], 200)
+
def test_where(self):
top = self.cities("") # nothing typed: the biggest US cities
self.assertEqual((len(top), top[0]["label"]), (20, "New York City, New York, US"))
diff --git a/scripts/scrape.py b/scripts/scrape.py
index 5c5361b..674ef33 100755
--- a/scripts/scrape.py
+++ b/scripts/scrape.py
@@ -82,7 +82,7 @@ KEY = os.environ.get("SCRAPER_API_KEY", "")
# The 8 lead fields: what you use to contact and qualify a lead.
# Everything else (geo coordinates, IDs, hours, images, review blobs) is dropped by default.
LEAD = ["title", "phone", "emails", "website", "category", "address", "review_rating", "review_count"]
-UA = "google-maps-scraper-kit/1.6 (https://github.com/Mahanaicoach/google-maps-scraper-kit)"
+UA = "google-maps-scraper-kit/2.0 (https://github.com/Mahanaicoach/google-maps-scraper-kit)"
# Business websites fetched at once by the email crawl, --socials and --tech. They are many
# different small sites, not Google, so this does not raise the rate-limit risk.
SITE_WORKERS = 16
diff --git a/scripts/test_scrape.py b/scripts/test_scrape.py
index 0731f5e..c41c310 100644
--- a/scripts/test_scrape.py
+++ b/scripts/test_scrape.py
@@ -381,10 +381,10 @@ class TechSignals(unittest.TestCase):
class EmailCrawl(unittest.TestCase):
def test_template_junk_dropped(self):
# Placeholders and a JSON-escaped prefix seen on real Austin coffee-shop sites.
- self.assertEqual(scrape.extract_emails(html, "funkysolcoffee.com"),
+ self.assertEqual(scrape.extract_emails(html, "cafe-one.test"),
# Seen in a new-user test run: an image name and a site-builder placeholder.
self.assertEqual(scrape.extract_emails("[email protected] [email protected]", "x.com"), [])
@@ -421,7 +421,7 @@ class IsRealEmail(unittest.TestCase):
"[email protected]", "[email protected]"): # seen in audit runs
self.assertFalse(scrape.is_real_email(e), e)
self.assertTrue(scrape.is_real_email(e), e)