Skip to content

High performance, self-hosted, newsletter and mailing list manager with a modern dashboard. Single binary app.

10.0 KB
Raw Blame
#!/usr/bin/env python3
"""Populates a listmonk database with random dummy data."""

import argparse
import json
import os
import random
import uuid
from datetime import datetime, timedelta, timezone
from urllib.parse import urlparse

import pg8000.dbapi

NUM_LISTS = 20
NUM_SUBSCRIBERS = 10000
NUM_CAMPAIGNS = 30
NUM_LINKS = 10
NUM_USERS = 10
NUM_ROLES = 3
NUM_CLICKS = 10000
NUM_VIEWS = 10000
NUM_BOUNCES = 100

# Number of campaigns and links that clicks/views/bounces are spread across.
SAMPLE_CAMPAIGNS = 10
SAMPLE_LINKS = 10

MAX_SUBSCRIPTIONS = 5
MAX_CAMPAIGN_LISTS = 3
MAX_TAGS = 3
MAX_ROLE_LISTS = 5
ACTIVITY_DAYS = 90

ADMIN_USER = "admin"
ADMIN_PASSWORD = "listmonk"

# Local mailhog (or equivalent) that outgoing e-mails are pointed at.
SMTP = {"name": "mailhog", "enabled": True, "host": "localhost", "port": 1025,
        "auth_protocol": "none", "username": "", "password": "", "hello_hostname": "",
        "max_conns": 10, "idle_timeout": "15s", "wait_timeout": "5s", "max_msg_retries": 2,
        "msg_retry_delay": "10ms", "tls_type": "none", "tls_skip_verify": False,
        "email_headers": [], "from_addresses": []}

# Per-list permissions that list roles are made of.
LIST_PERMS = ["list:get", "list:manage"]

VERBS = ["run", "jump", "build", "wander", "gather", "carve", "float", "sketch", "brew",
         "forge", "drift", "climb", "polish", "whistle", "scatter", "mend", "trace",
         "kindle", "ramble", "harvest"]

NOUNS = ["mango", "harbour", "lantern", "meadow", "compass", "river", "thunder", "orchard",
         "pebble", "canyon", "willow", "beacon", "anchor", "sparrow", "cobble", "marble",
         "prairie", "juniper", "cinder", "quarry"]

WORDS = VERBS + NOUNS

CAMPAIGN_STATUSES = ["draft", "finished", "cancelled", "scheduled"]

# (status, weight). Blocklisted is ~7% of subscribers.
SUBSCRIBER_STATUSES = [("enabled", 83), ("disabled", 10), ("blocklisted", 7)]


def sentence_name(n):
    words = random.sample(WORDS, n)
    return " ".join([words[0].capitalize()] + words[1:])


def camel_name():
    return "".join(w.capitalize() for w in random.sample(WORDS, 2))


def rand_tags():
    return random.sample(WORDS, random.randint(0, MAX_TAGS))


def sub_state():
    statuses, weights = zip(*SUBSCRIBER_STATUSES)
    return random.choices(statuses, weights=weights)[0]


def sub_status(optin):
    if optin == "double":
        return random.choice(["unconfirmed", "confirmed", "unsubscribed"])
    return random.choice(["unconfirmed", "unconfirmed", "unsubscribed"])


def all_perms():
    """Read the global permissions from the repo's permissions.json."""
    f = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "permissions.json")
    with open(f) as fp:
        return [p for group in json.load(fp) for p in group["permissions"]]


def rand_time(now):
    return now - timedelta(seconds=random.randint(0, ACTIVITY_DAYS * 86400))


def connect(dsn):
    u = urlparse(dsn)
    return pg8000.dbapi.connect(user=u.username or "listmonk", password=u.password,
                                host=u.hostname or "localhost", port=u.port or 5432,
                                database=u.path.lstrip("/") or "listmonk")


def insert(cur, sql, rows, fetch=False, chunk=1000):
    """Bulk INSERT rows in chunks. sql should end with a `{}` placeholder for VALUES."""
    out = []
    for i in range(0, len(rows), chunk):
        batch = rows[i:i + chunk]
        tpl = "(" + ",".join(["%s"] * len(batch[0])) + ")"
        cur.execute(sql.format(",".join([tpl] * len(batch))), [v for r in batch for v in r])
        if fetch:
            out += cur.fetchall()
    return out


def main():
    p = argparse.ArgumentParser(description="Populate a listmonk DB with dummy data.")
    p.add_argument("--db", default="postgres://listmonk:listmonk@localhost:5432/listmonk?sslmode=disable", help="Postgres DSN")
    args = p.parse_args()

    now = datetime.now(timezone.utc)
    db = connect(args.db)
    cur = db.cursor()

    cur.execute("UPDATE settings SET value=%s WHERE key='smtp'",
                [json.dumps([dict(SMTP, uuid=str(uuid.uuid4()))])])
    print(f"smtp -> {SMTP['host']}:{SMTP['port']}")

    cur.execute("SELECT id FROM templates WHERE type='campaign' ORDER BY id LIMIT 1")
    row = cur.fetchone()
    template_id = row[0] if row else None

    lists = [(str(uuid.uuid4()), sentence_name(3), random.choice(["public", "private"]),
              random.choice(["single", "double"]), rand_tags()) for _ in range(NUM_LISTS)]
    list_ids = insert(cur, """
        INSERT INTO lists (uuid, name, type, optin, tags) VALUES {} RETURNING id, optin, name
    """, lists, fetch=True)
    print(f"{len(list_ids)} lists")

    subs = [(str(uuid.uuid4()), f"{camel_name().lower()}{i}@example.com", camel_name(),
             sub_state()) for i in range(NUM_SUBSCRIBERS)]
    sub_ids = [r[0] for r in insert(cur, """
        INSERT INTO subscribers (uuid, email, name, status) VALUES {} RETURNING id
    """, subs, fetch=True)]
    print(f"{len(sub_ids)} subscribers")

    subscriptions = []
    for sid in sub_ids:
        for l in random.sample(list_ids, random.randint(1, MAX_SUBSCRIPTIONS)):
            subscriptions.append((sid, l[0], sub_status(l[1])))
    insert(cur, """
        INSERT INTO subscriber_lists (subscriber_id, list_id, status) VALUES {}
    """, subscriptions)
    print(f"{len(subscriptions)} subscriptions")

    perms = all_perms()

    # ID 1 is the primordial super admin role that has all permissions.
    cur.execute("""
        INSERT INTO roles (id, name, type, permissions) VALUES (1, 'Super Admin', 'user', %s)
    """, [perms])
    cur.execute("SELECT SETVAL('roles_id_seq', 1)")

    # Random user roles, each with a random subset of the global permissions.
    user_role_ids = [r[0] for r in insert(cur, """
        INSERT INTO roles (name, type, permissions) VALUES {} RETURNING id
    """, [(f"{sentence_name(2)} {i}", "user", random.sample(perms, random.randint(2, len(perms))))
          for i in range(NUM_ROLES)], fetch=True)]

    # Random list roles. A list role is a parent row with one child row per list
    # carrying that list's permissions.
    list_role_ids = [r[0] for r in insert(cur, """
        INSERT INTO roles (name, type) VALUES {} RETURNING id
    """, [(f"{sentence_name(2)} {i}", "list") for i in range(NUM_ROLES)], fetch=True)]

    role_lists = []
    for rid in list_role_ids:
        for l in random.sample(list_ids, random.randint(1, MAX_ROLE_LISTS)):
            role_lists.append((rid, l[0], "list",
                               random.sample(LIST_PERMS, random.randint(1, len(LIST_PERMS)))))
    insert(cur, """
        INSERT INTO roles (parent_id, list_id, type, permissions) VALUES {}
    """, role_lists)
    print(f"{len(user_role_ids)} user roles, {len(list_role_ids)} list roles")

    # `{}` is the row's ID expression, as the super admin user needs a fixed ID.
    user_sql = """
        INSERT INTO users (id, username, password_login, password, email, name, type,
            user_role_id, list_role_id, status)
        VALUES ({}, %s, TRUE, CRYPT(%s, GEN_SALT('bf')), %s, %s, 'user', %s, %s, 'enabled')
    """

    # ID 1 is the super admin user.
    cur.execute(user_sql.format(1), [ADMIN_USER, ADMIN_PASSWORD, f"{ADMIN_USER}@example.com",
                                     ADMIN_USER, 1, None])
    cur.execute("SELECT SETVAL('users_id_seq', 1)")

    for i in range(NUM_USERS):
        u = camel_name().lower() + str(i)
        cur.execute(user_sql.format("DEFAULT"),
                    [u, ADMIN_PASSWORD, f"{u}@example.com", camel_name(),
                     random.choice(user_role_ids), random.choice(list_role_ids)])
    print(f"{NUM_USERS + 1} users ('{ADMIN_USER}' / '{ADMIN_PASSWORD}')")

    camps = []
    for _ in range(NUM_CAMPAIGNS):
        status = random.choice(CAMPAIGN_STATUSES)
        name = sentence_name(4)
        camps.append((str(uuid.uuid4()), name, name, "[email protected]",
                      f"<p>{sentence_name(6)}</p>", "richtext", status,
                      now + timedelta(days=3650) if status == "scheduled" else None,
                      "email", template_id, rand_tags()))
    camp_ids = [r[0] for r in insert(cur, """
        INSERT INTO campaigns (uuid, name, subject, from_email, body, content_type, status,
            send_at, messenger, template_id, tags) VALUES {} RETURNING id
    """, camps, fetch=True)]
    print(f"{len(camp_ids)} campaigns")

    camp_lists = []
    for cid in camp_ids:
        for l in random.sample(list_ids, random.randint(1, MAX_CAMPAIGN_LISTS)):
            camp_lists.append((cid, l[0], l[2]))
    insert(cur, """
        INSERT INTO campaign_lists (campaign_id, list_id, list_name) VALUES {}
    """, camp_lists)

    links = [(str(uuid.uuid4()), f"https://example.com/{sentence_name(2).lower().replace(' ', '-')}-{i}")
             for i in range(NUM_LINKS)]
    link_ids = [r[0] for r in insert(cur, """
        INSERT INTO links (uuid, url) VALUES {} RETURNING id
    """, links, fetch=True)]

    click_camps = random.sample(camp_ids, min(SAMPLE_CAMPAIGNS, len(camp_ids)))
    click_links = random.sample(link_ids, min(SAMPLE_LINKS, len(link_ids)))

    clicks = [(random.choice(click_camps), random.choice(click_links),
               random.choice(sub_ids), rand_time(now)) for _ in range(NUM_CLICKS)]
    insert(cur, """
        INSERT INTO link_clicks (campaign_id, link_id, subscriber_id, created_at) VALUES {}
    """, clicks)
    print(f"{len(clicks)} link clicks")

    views = [(random.choice(click_camps), random.choice(sub_ids), rand_time(now))
             for _ in range(NUM_VIEWS)]
    insert(cur, """
        INSERT INTO campaign_views (campaign_id, subscriber_id, created_at) VALUES {}
    """, views)
    print(f"{len(views)} campaign views")

    bounces = [(random.choice(click_camps), random.choice(sub_ids),
                random.choice(["soft", "hard", "complaint"]), "demo", rand_time(now))
               for _ in range(NUM_BOUNCES)]
    insert(cur, """
        INSERT INTO bounces (campaign_id, subscriber_id, type, source, created_at) VALUES {}
    """, bounces)
    print(f"{len(bounces)} bounces")

    db.commit()
    cur.close()
    db.close()


if __name__ == "__main__":
    main()