High performance, self-hosted, newsletter and mailing list manager with a modern dashboard. Single binary app.
JavaScript campaigncampaign-managementemail-marketingemail-subscriptionlistmonkmailing-listnewsletternewsletter-managementnewsletter-softwareself-hostedsms-gatewaysmtptransactional-emails
Add a script to populate an empty DB with dummy data. Fully LLM-generated.
Kailash Nadh321391931af7
diff --git a/scripts/populate-dummy.py b/scripts/populate-dummy.pynew file mode 100755index 00000000..f69a74e7--- /dev/null+++ b/scripts/populate-dummy.py@@ -0,0 +1,256 @@+#!/usr/bin/env python3+"""Populates a listmonk database with random dummy data."""++import argparse+import json+import os+import random+import uuid+from datetime import datetime, timedelta, timezone+from urllib.parse import urlparse++import pg8000.dbapi++NUM_LISTS = 20+NUM_SUBSCRIBERS = 10000+NUM_CAMPAIGNS = 30+NUM_LINKS = 10+NUM_USERS = 10+NUM_ROLES = 3+NUM_CLICKS = 10000+NUM_VIEWS = 10000++# Number of campaigns and links that clicks/views are spread across.+SAMPLE_CAMPAIGNS = 10+SAMPLE_LINKS = 10++MAX_SUBSCRIPTIONS = 5+MAX_CAMPAIGN_LISTS = 3+MAX_TAGS = 3+MAX_ROLE_LISTS = 5+ACTIVITY_DAYS = 90++ADMIN_USER = "admin"+ADMIN_PASSWORD = "listmonk"++# Local mailhog (or equivalent) that outgoing e-mails are pointed at.+SMTP = {"name": "mailhog", "enabled": True, "host": "localhost", "port": 1025,+ "auth_protocol": "none", "username": "", "password": "", "hello_hostname": "",+ "max_conns": 10, "idle_timeout": "15s", "wait_timeout": "5s", "max_msg_retries": 2,+ "msg_retry_delay": "10ms", "tls_type": "none", "tls_skip_verify": False,+ "email_headers": [], "from_addresses": []}++# Per-list permissions that list roles are made of.+LIST_PERMS = ["list:get", "list:manage"]++VERBS = ["run", "jump", "build", "wander", "gather", "carve", "float", "sketch", "brew",+ "forge", "drift", "climb", "polish", "whistle", "scatter", "mend", "trace",+ "kindle", "ramble", "harvest"]++NOUNS = ["mango", "harbour", "lantern", "meadow", "compass", "river", "thunder", "orchard",+ "pebble", "canyon", "willow", "beacon", "anchor", "sparrow", "cobble", "marble",+ "prairie", "juniper", "cinder", "quarry"]++WORDS = VERBS + NOUNS++CAMPAIGN_STATUSES = ["draft", "finished", "cancelled", "scheduled"]++# (status, weight). Blocklisted is ~7% of subscribers.+SUBSCRIBER_STATUSES = [("enabled", 83), ("disabled", 10), ("blocklisted", 7)]+++def sentence_name(n):+ words = random.sample(WORDS, n)+ return " ".join([words[0].capitalize()] + words[1:])+++def camel_name():+ return "".join(w.capitalize() for w in random.sample(WORDS, 2))+++def rand_tags():+ return random.sample(WORDS, random.randint(0, MAX_TAGS))+++def sub_state():+ statuses, weights = zip(*SUBSCRIBER_STATUSES)+ return random.choices(statuses, weights=weights)[0]+++def sub_status(optin):+ if optin == "double":+ return random.choice(["unconfirmed", "confirmed", "unsubscribed"])+ return random.choice(["unconfirmed", "unconfirmed", "unsubscribed"])+++def all_perms():+ """Read the global permissions from the repo's permissions.json."""+ f = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "permissions.json")+ with open(f) as fp:+ return [p for group in json.load(fp) for p in group["permissions"]]+++def rand_time(now):+ return now - timedelta(seconds=random.randint(0, ACTIVITY_DAYS * 86400))+++def connect(dsn):+ u = urlparse(dsn)+ return pg8000.dbapi.connect(user=u.username or "listmonk", password=u.password,+ host=u.hostname or "localhost", port=u.port or 5432,+ database=u.path.lstrip("/") or "listmonk")+++def insert(cur, sql, rows, fetch=False, chunk=1000):+ """Bulk INSERT rows in chunks. sql should end with a `{}` placeholder for VALUES."""+ out = []+ for i in range(0, len(rows), chunk):+ batch = rows[i:i + chunk]+ tpl = "(" + ",".join(["%s"] * len(batch[0])) + ")"+ cur.execute(sql.format(",".join([tpl] * len(batch))), [v for r in batch for v in r])+ if fetch:+ out += cur.fetchall()+ return out+++def main():+ p = argparse.ArgumentParser(description="Populate a listmonk DB with dummy data.")+ p.add_argument("--db", default="postgres://listmonk:listmonk@localhost:5432/listmonk?sslmode=disable", help="Postgres DSN")+ args = p.parse_args()++ now = datetime.now(timezone.utc)+ db = connect(args.db)+ cur = db.cursor()++ cur.execute("UPDATE settings SET value=%s WHERE key='smtp'",+ [json.dumps([dict(SMTP, uuid=str(uuid.uuid4()))])])+ print(f"smtp -> {SMTP['host']}:{SMTP['port']}")++ cur.execute("SELECT id FROM templates WHERE type='campaign' ORDER BY id LIMIT 1")+ row = cur.fetchone()+ template_id = row[0] if row else None++ lists = [(str(uuid.uuid4()), sentence_name(3), random.choice(["public", "private"]),+ random.choice(["single", "double"]), rand_tags()) for _ in range(NUM_LISTS)]+ list_ids = insert(cur, """+ INSERT INTO lists (uuid, name, type, optin, tags) VALUES {} RETURNING id, optin, name+ """, lists, fetch=True)+ print(f"{len(list_ids)} lists")++ subs = [(str(uuid.uuid4()), f"{camel_name().lower()}{i}@example.com", camel_name(),+ sub_state()) for i in range(NUM_SUBSCRIBERS)]+ sub_ids = [r[0] for r in insert(cur, """+ INSERT INTO subscribers (uuid, email, name, status) VALUES {} RETURNING id+ """, subs, fetch=True)]+ print(f"{len(sub_ids)} subscribers")++ subscriptions = []+ for sid in sub_ids:+ for l in random.sample(list_ids, random.randint(1, MAX_SUBSCRIPTIONS)):+ subscriptions.append((sid, l[0], sub_status(l[1])))+ insert(cur, """+ INSERT INTO subscriber_lists (subscriber_id, list_id, status) VALUES {}+ """, subscriptions)+ print(f"{len(subscriptions)} subscriptions")++ perms = all_perms()++ # ID 1 is the primordial super admin role that has all permissions.+ cur.execute("""+ INSERT INTO roles (id, name, type, permissions) VALUES (1, 'Super Admin', 'user', %s)+ """, [perms])+ cur.execute("SELECT SETVAL('roles_id_seq', 1)")++ # Random user roles, each with a random subset of the global permissions.+ user_role_ids = [r[0] for r in insert(cur, """+ INSERT INTO roles (name, type, permissions) VALUES {} RETURNING id+ """, [(f"{sentence_name(2)} {i}", "user", random.sample(perms, random.randint(2, len(perms))))+ for i in range(NUM_ROLES)], fetch=True)]++ # Random list roles. A list role is a parent row with one child row per list+ # carrying that list's permissions.+ list_role_ids = [r[0] for r in insert(cur, """+ INSERT INTO roles (name, type) VALUES {} RETURNING id+ """, [(f"{sentence_name(2)} {i}", "list") for i in range(NUM_ROLES)], fetch=True)]++ role_lists = []+ for rid in list_role_ids:+ for l in random.sample(list_ids, random.randint(1, MAX_ROLE_LISTS)):+ role_lists.append((rid, l[0], "list",+ random.sample(LIST_PERMS, random.randint(1, len(LIST_PERMS)))))+ insert(cur, """+ INSERT INTO roles (parent_id, list_id, type, permissions) VALUES {}+ """, role_lists)+ print(f"{len(user_role_ids)} user roles, {len(list_role_ids)} list roles")++ # `{}` is the row's ID expression, as the super admin user needs a fixed ID.+ user_sql = """+ INSERT INTO users (id, username, password_login, password, email, name, type,+ user_role_id, list_role_id, status)+ VALUES ({}, %s, TRUE, CRYPT(%s, GEN_SALT('bf')), %s, %s, 'user', %s, %s, 'enabled')+ """++ # ID 1 is the super admin user.+ cur.execute(user_sql.format(1), [ADMIN_USER, ADMIN_PASSWORD, f"{ADMIN_USER}@example.com",+ ADMIN_USER, 1, None])+ cur.execute("SELECT SETVAL('users_id_seq', 1)")++ for i in range(NUM_USERS):+ u = camel_name().lower() + str(i)+ cur.execute(user_sql.format("DEFAULT"),+ [u, ADMIN_PASSWORD, f"{u}@example.com", camel_name(),+ random.choice(user_role_ids), random.choice(list_role_ids)])+ print(f"{NUM_USERS + 1} users ('{ADMIN_USER}' / '{ADMIN_PASSWORD}')")++ camps = []+ for _ in range(NUM_CAMPAIGNS):+ status = random.choice(CAMPAIGN_STATUSES)+ name = sentence_name(4)+ camps.append((str(uuid.uuid4()), name, name, "[email protected]",+ f"<p>{sentence_name(6)}</p>", "richtext", status,+ now + timedelta(days=3650) if status == "scheduled" else None,+ "email", template_id, rand_tags()))+ camp_ids = [r[0] for r in insert(cur, """+ INSERT INTO campaigns (uuid, name, subject, from_email, body, content_type, status,+ send_at, messenger, template_id, tags) VALUES {} RETURNING id+ """, camps, fetch=True)]+ print(f"{len(camp_ids)} campaigns")++ camp_lists = []+ for cid in camp_ids:+ for l in random.sample(list_ids, random.randint(1, MAX_CAMPAIGN_LISTS)):+ camp_lists.append((cid, l[0], l[2]))+ insert(cur, """+ INSERT INTO campaign_lists (campaign_id, list_id, list_name) VALUES {}+ """, camp_lists)++ links = [(str(uuid.uuid4()), f"https://example.com/{sentence_name(2).lower().replace(' ', '-')}-{i}")+ for i in range(NUM_LINKS)]+ link_ids = [r[0] for r in insert(cur, """+ INSERT INTO links (uuid, url) VALUES {} RETURNING id+ """, links, fetch=True)]++ click_camps = random.sample(camp_ids, min(SAMPLE_CAMPAIGNS, len(camp_ids)))+ click_links = random.sample(link_ids, min(SAMPLE_LINKS, len(link_ids)))++ clicks = [(random.choice(click_camps), random.choice(click_links),+ random.choice(sub_ids), rand_time(now)) for _ in range(NUM_CLICKS)]+ insert(cur, """+ INSERT INTO link_clicks (campaign_id, link_id, subscriber_id, created_at) VALUES {}+ """, clicks)+ print(f"{len(clicks)} link clicks")++ views = [(random.choice(click_camps), random.choice(sub_ids), rand_time(now))+ for _ in range(NUM_VIEWS)]+ insert(cur, """+ INSERT INTO campaign_views (campaign_id, subscriber_id, created_at) VALUES {}+ """, views)+ print(f"{len(views)} campaign views")++ db.commit()+ cur.close()+ db.close()+++if __name__ == "__main__":+ main()diff --git a/scripts/requirements.txt b/scripts/requirements.txtnew file mode 100644index 00000000..5e29a933--- /dev/null+++ b/scripts/requirements.txt@@ -0,0 +1,2 @@+openai # translate-i18n.py+pg8000 # populate-dummy.py