Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Refactor Filters component to improve parameter handling and UI clarity

Bill Yangdbe66cf5c84f
diff --git a/mockdata/README.md b/mockdata/README.md
new file mode 100644
index 000000000..79778b157
--- /dev/null
+++ b/mockdata/README.md
@@ -0,0 +1,101 @@
+# E-commerce Mock Data Generator for Frogstats
+
+This script generates realistic mock analytics data for Frogstats, simulating user behavior on an e-commerce website called ShopEase.
+
+## Features
+
+- Generates approximately 5 million events per day over a specified number of days
+- Creates realistic user shopping sessions with multiple pageviews and custom events
+- Models realistic e-commerce user journeys (browsing, product views, add to cart, checkout, purchase)
+- Sessions end after 30 minutes of inactivity
+- Distributes traffic realistically throughout the day (with peak hours)
+- Uses Faker.js to generate realistic product data, location information, and more
+- Uses weighted distributions for pages, browsers, operating systems, screen sizes, and more
+- Varies the number of events per day to be more realistic
+
+## Requirements
+
+- Node.js 14+
+- ClickHouse database set up (configuration in .env file)
+
+## Installation
+
+1. Make sure all dependencies are installed:
+
+```bash
+npm install
+```
+
+## Usage
+
+Run the script with optional parameters:
+
+```bash
+# Default: 30 days with ~5 million events per day
+npm run generate
+
+# Custom parameters (days, events per day)
+node index.js 15 2000000 # 15 days with ~2 million events per day
+```
+
+### Parameters
+
+1. `daysInPast`: Number of days in the past to generate data for (default: 30)
+2. `eventsPerDay`: Approximate number of events per day (default: 5,000,000)
+
+## Data Distribution
+
+The script creates realistic distributions of:
+
+- Page views with appropriate e-commerce user flows
+- Shopping behavior patterns (browsing, cart additions, checkouts, purchases)
+- Geographic distribution (countries and regions using Faker.js)
+- Browser types and versions
+- Operating systems
+- Screen resolutions
+- Referrers (including social media and other e-commerce sites)
+- Custom events specific to e-commerce (product views, add to cart, checkout steps, etc.)
+- Time of day (peak hours vs. off hours)
+
+## E-commerce Event Types
+
+The generator includes these e-commerce-specific events:
+
+- page-view - Basic page view tracking
+- product-view - When a user views a product detail page
+- add-to-cart - When a user adds an item to their cart
+- remove-from-cart - When a user removes an item from their cart
+- begin-checkout - When a user starts the checkout process
+- checkout-step - Tracking each step of the checkout funnel
+- purchase - When a user completes a purchase
+- search - When a user searches for products
+- filter-products - When a user applies filters to product listings
+- add-to-wishlist - When a user adds items to their wishlist
+- And more...
+
+## Data Schema
+
+Events are inserted into the ClickHouse `pageviews` table with the following schema:
+
+- site_id
+- timestamp
+- session_id
+- user_id
+- hostname
+- pathname
+- querystring
+- page_title
+- referrer
+- browser
+- browser_version
+- operating_system
+- operating_system_version
+- language
+- country
+- iso_3166_2
+- screen_width
+- screen_height
+- device_type
+- type (pageview or custom_event)
+- event_name (for custom events)
+- properties (JSON string for custom events)
diff --git a/mockdata/index.js b/mockdata/index.js
new file mode 100644
index 000000000..334c25e99
--- /dev/null
+++ b/mockdata/index.js
@@ -0,0 +1,1217 @@
+const { createClient } = require("@clickhouse/client");
+const crypto = require("crypto");
+const { DateTime } = require("luxon");
+const { faker } = require("@faker-js/faker");
+require("dotenv").config();
+
+const clickhouse = createClient({
+ host: process.env.CLICKHOUSE_HOST,
+ database: process.env.CLICKHOUSE_DB,
+ password: process.env.CLICKHOUSE_PASSWORD,
+});
+
+// Command line arguments
+const args = process.argv.slice(2);
+const daysInPast = parseInt(args[0] || "30", 10);
+const eventsPerDay = parseInt(args[1] || "5000000", 10);
+
+console.log(
+ `Generating mock data for ${daysInPast} days with approximately ${eventsPerDay} events per day`
+);
+
+// Site ID to use
+const SITE_ID = 5;
+
+// Generate 10,000 unique user IDs
+const userIds = Array.from({ length: 10000 }, () => faker.string.uuid());
+
+// Website details
+const SITE_NAME = "ShopEase";
+const SITE_DOMAIN = "shopease.com";
+
+// Generate products for consistent paths
+const products = Array.from({ length: 50 }, (_, index) => {
+ const productId = 1000 + index;
+ const name = faker.commerce.productName();
+ const category = faker.commerce.department();
+ const subcategory = faker.commerce.productAdjective();
+ const slug = name.toLowerCase().replace(/[^a-z0-9]+/g, "-");
+
+ return {
+ id: productId,
+ name,
+ category,
+ subcategory,
+ slug,
+ price: faker.commerce.price({ min: 10, max: 1000 }),
+ };
+});
+
+// Generate categories
+const categories = [...new Set(products.map((p) => p.category))];
+const subcategories = [...new Set(products.map((p) => p.subcategory))];
+
+// Realistic page paths with weighted distribution
+const pagePaths = [
+ { path: "/", weight: 20, title: `${SITE_NAME} - Online Shopping` },
+ { path: "/products", weight: 15, title: `All Products - ${SITE_NAME}` },
+ { path: "/categories", weight: 10, title: `Shop by Category - ${SITE_NAME}` },
+ { path: "/sale", weight: 12, title: `Sale Items - ${SITE_NAME}` },
+ { path: "/new-arrivals", weight: 8, title: `New Arrivals - ${SITE_NAME}` },
+ { path: "/cart", weight: 7, title: `Your Shopping Cart - ${SITE_NAME}` },
+ { path: "/checkout", weight: 5, title: `Checkout - ${SITE_NAME}` },
+ { path: "/account", weight: 5, title: `Your Account - ${SITE_NAME}` },
+ { path: "/wishlist", weight: 4, title: `Your Wishlist - ${SITE_NAME}` },
+ { path: "/order-history", weight: 3, title: `Order History - ${SITE_NAME}` },
+ { path: "/about", weight: 2, title: `About Us - ${SITE_NAME}` },
+ { path: "/contact", weight: 2, title: `Contact Us - ${SITE_NAME}` },
+ { path: "/faq", weight: 2, title: `FAQ - ${SITE_NAME}` },
+ { path: "/shipping", weight: 2, title: `Shipping Policy - ${SITE_NAME}` },
+ { path: "/returns", weight: 2, title: `Returns & Refunds - ${SITE_NAME}` },
+];
+
+// Add category pages
+categories.forEach((category) => {
+ const slug = category.toLowerCase().replace(/[^a-z0-9]+/g, "-");
+ pagePaths.push({
+ path: `/category/${slug}`,
+ weight: 6,
+ title: `${category} - ${SITE_NAME}`,
+ category,
+ });
+});
+
+// Add product pages (high weight as these are important for e-commerce)
+products.forEach((product) => {
+ pagePaths.push({
+ path: `/product/${product.id}-${product.slug}`,
+ weight: 9,
+ title: `${product.name} - ${SITE_NAME}`,
+ product,
+ });
+});
+
+// Custom events with weighted distribution
+const customEvents = [
+ { name: "page-view", weight: 25, properties: { page: "home" } },
+ { name: "product-view", weight: 20, properties: {} }, // Will be filled with product details
+ { name: "add-to-cart", weight: 15, properties: {} }, // Will be filled with product details
+ { name: "remove-from-cart", weight: 5, properties: {} }, // Will be filled with product details
+ {
+ name: "begin-checkout",
+ weight: 8,
+ properties: { items_count: 1, value: 0 },
+ },
+ { name: "checkout-step", weight: 7, properties: { step: 1, option: "" } },
+ {
+ name: "purchase",
+ weight: 5,
+ properties: { transaction_id: "", value: 0, currency: "USD", items: [] },
+ },
+ {
+ name: "product-click",
+ weight: 12,
+ properties: { list_name: "recommended" },
+ },
+ { name: "search", weight: 10, properties: { search_term: "" } },
+ {
+ name: "filter-products",
+ weight: 8,
+ properties: { filter: "price", value: "100-200" },
+ },
+ { name: "add-to-wishlist", weight: 6, properties: {} }, // Will be filled with product details
+ { name: "share-product", weight: 4, properties: { method: "email" } },
+ {
+ name: "select-promotion",
+ weight: 5,
+ properties: { promotion_name: "Summer Sale" },
+ },
+ { name: "sign-up", weight: 3, properties: { method: "email" } },
+ { name: "login", weight: 6, properties: { method: "site" } },
+ { name: "view-cart", weight: 10, properties: { items_count: 0, value: 0 } },
+];
+
+// Tab/query parameters for certain pages
+const tabParams = {
+ "/product/": [
+ { param: "?variant=large", weight: 30 },
+ { param: "?variant=medium", weight: 25 },
+ { param: "?variant=small", weight: 20 },
+ { param: "?color=blue", weight: 15 },
+ { param: "?color=black", weight: 15 },
+ { param: "", weight: 20 },
+ ],
+ "/products": [
+ { param: "?sort=price-asc", weight: 25 },
+ { param: "?sort=price-desc", weight: 20 },
+ { param: "?sort=newest", weight: 20 },
+ { param: "?sort=popular", weight: 20 },
+ { param: "", weight: 15 },
+ ],
+ "/category/": [
+ { param: "?filter=new", weight: 20 },
+ { param: "?filter=sale", weight: 25 },
+ { param: "?filter=in-stock", weight: 20 },
+ { param: "?sort=price-low", weight: 15 },
+ { param: "", weight: 20 },
+ ],
+};
+
+// Referrers with weighted distribution
+const referrers = [
+ { url: "", weight: 30 }, // Direct
+ { url: "https://www.google.com/", weight: 25 },
+ { url: "https://www.facebook.com/", weight: 10 },
+ { url: "https://www.instagram.com/", weight: 12 },
+ { url: "https://pinterest.com/", weight: 8 },
+ { url: "https://www.amazon.com/", weight: 5 },
+ { url: "https://www.youtube.com/", weight: 5 },
+ { url: "https://www.tiktok.com/", weight: 4 },
+ { url: "https://twitter.com/", weight: 3 },
+ { url: "https://www.bing.com/", weight: 3 },
+ { url: "https://www.retailmenot.com/", weight: 2 },
+ { url: "https://slickdeals.net/", weight: 2 },
+ { url: "https://www.dealsplus.com/", weight: 1 },
+ { url: `https://email.${SITE_DOMAIN}/`, weight: 10 }, // Email campaigns
+ { url: "https://www.pricegrabber.com/", weight: 1 },
+];
+
+// Browsers with weighted distribution
+const browsers = [
+ { name: "Chrome", weight: 60, versions: ["133", "134", "135", "132"] },
+ { name: "Firefox", weight: 15, versions: ["124", "123", "122"] },
+ { name: "Safari", weight: 10, versions: ["17", "16", "15"] },
+ { name: "Edge", weight: 8, versions: ["122", "121", "120"] },
+ { name: "Opera", weight: 4, versions: ["105", "104", "103"] },
+ { name: "Yandex", weight: 3, versions: ["25", "24", "23"] },
+];
+
+// Operating systems with weighted distribution
+const operatingSystems = [
+ { name: "Windows", weight: 65, versions: ["10", "11", "7"] },
+ { name: "Android", weight: 15, versions: ["13", "12", "11", "10"] },
+ { name: "iOS", weight: 10, versions: ["17", "16", "15"] },
+ { name: "macOS", weight: 7, versions: ["14", "13", "12"] },
+ { name: "Linux", weight: 3, versions: ["5.15", "5.10", "5.8"] },
+];
+
+// Screen resolutions with weighted distribution
+const screenResolutions = [
+ { width: 1920, height: 1080, weight: 30 },
+ { width: 1366, height: 768, weight: 20 },
+ { width: 1536, height: 864, weight: 15 },
+ { width: 2560, height: 1440, weight: 10 },
+ { width: 1440, height: 900, weight: 8 },
+ { width: 1024, height: 768, weight: 5 },
+ { width: 3840, height: 2160, weight: 5 },
+ { width: 1280, height: 720, weight: 3 },
+ // Mobile resolutions
+ { width: 414, height: 896, weight: 3 },
+ { width: 375, height: 667, weight: 3 },
+ { width: 360, height: 740, weight: 3 },
+ { width: 412, height: 915, weight: 2 },
+ { width: 390, height: 844, weight: 2 },
+ { width: 768, height: 1024, weight: 1 }, // iPad
+];
+
+// Languages with weighted distribution
+const languages = [
+ { code: "en-US", weight: 30 },
+ { code: "en", weight: 15 },
+ { code: "ru", weight: 12 },
+ { code: "de", weight: 8 },
+ { code: "pl", weight: 7 },
+ { code: "es", weight: 6 },
+ { code: "fr", weight: 6 },
+ { code: "zh-CN", weight: 5 },
+ { code: "it", weight: 4 },
+ { code: "cs", weight: 3 },
+ { code: "pt-BR", weight: 2 },
+ { code: "ja", weight: 2 },
+];
+
+// Countries and regions
+const geoData = [
+ {
+ country: "US",
+ weight: 25,
+ regions: [
+ { code: "US-NY", weight: 20 },
+ { code: "US-CA", weight: 18 },
+ { code: "US-TX", weight: 15 },
+ { code: "US-FL", weight: 12 },
+ { code: "US-IL", weight: 10 },
+ { code: "US-PA", weight: 8 },
+ { code: "US-OH", weight: 7 },
+ { code: "US-MI", weight: 5 },
+ { code: "US-GA", weight: 5 },
+ ],
+ },
+ {
+ country: "RU",
+ weight: 20,
+ regions: [
+ { code: "RU-MOW", weight: 25 },
+ { code: "RU-SPE", weight: 15 },
+ { code: "RU-KDA", weight: 10 },
+ { code: "RU-KYA", weight: 10 },
+ { code: "RU-NVS", weight: 8 },
+ { code: "RU-SVE", weight: 8 },
+ { code: "RU-IRK", weight: 5 },
+ { code: "RU-YAR", weight: 5 },
+ ],
+ },
+ {
+ country: "DE",
+ weight: 10,
+ regions: [
+ { code: "DE-BY", weight: 20 },
+ { code: "DE-NW", weight: 18 },
+ { code: "DE-BW", weight: 15 },
+ { code: "DE-NI", weight: 12 },
+ { code: "DE-HE", weight: 10 },
+ { code: "DE-BE", weight: 8 },
+ ],
+ },
+ {
+ country: "PL",
+ weight: 8,
+ regions: [
+ { code: "PL-MZ", weight: 20 },
+ { code: "PL-DS", weight: 15 },
+ { code: "PL-WP", weight: 15 },
+ { code: "PL-SL", weight: 12 },
+ { code: "PL-MA", weight: 10 },
+ ],
+ },
+ {
+ country: "GB",
+ weight: 7,
+ regions: [
+ { code: "GB-ENG", weight: 80 },
+ { code: "GB-SCT", weight: 10 },
+ { code: "GB-WLS", weight: 7 },
+ { code: "GB-NIR", weight: 3 },
+ ],
+ },
+ {
+ country: "FR",
+ weight: 6,
+ regions: [
+ { code: "FR-IDF", weight: 25 },
+ { code: "FR-ARA", weight: 15 },
+ { code: "FR-HDF", weight: 12 },
+ { code: "FR-NAQ", weight: 10 },
+ { code: "FR-OCC", weight: 10 },
+ ],
+ },
+ {
+ country: "CA",
+ weight: 5,
+ regions: [
+ { code: "CA-ON", weight: 40 },
+ { code: "CA-QC", weight: 25 },
+ { code: "CA-BC", weight: 15 },
+ { code: "CA-AB", weight: 10 },
+ ],
+ },
+ {
+ country: "CN",
+ weight: 4,
+ regions: [
+ { code: "CN-11", weight: 20 }, // Beijing
+ { code: "CN-31", weight: 20 }, // Shanghai
+ { code: "CN-44", weight: 15 }, // Guangdong
+ { code: "CN-51", weight: 10 }, // Sichuan
+ { code: "CN-33", weight: 10 }, // Zhejiang
+ { code: "CN-32", weight: 10 }, // Jiangsu
+ ],
+ },
+ {
+ country: "AU",
+ weight: 3,
+ regions: [
+ { code: "AU-NSW", weight: 35 },
+ { code: "AU-VIC", weight: 30 },
+ { code: "AU-QLD", weight: 20 },
+ { code: "AU-WA", weight: 15 },
+ ],
+ },
+ {
+ country: "ES",
+ weight: 3,
+ regions: [
+ { code: "ES-MD", weight: 25 },
+ { code: "ES-CT", weight: 20 },
+ { code: "ES-AN", weight: 18 },
+ { code: "ES-VA", weight: 12 },
+ ],
+ },
+ {
+ country: "IT",
+ weight: 2,
+ regions: [
+ { code: "IT-25", weight: 20 }, // Lombardy
+ { code: "IT-62", weight: 15 }, // Lazio
+ { code: "IT-21", weight: 15 }, // Piedmont
+ { code: "IT-34", weight: 12 }, // Veneto
+ ],
+ },
+ {
+ country: "CZ",
+ weight: 2,
+ regions: [
+ { code: "CZ-PR", weight: 35 },
+ { code: "CZ-JM", weight: 20 },
+ { code: "CZ-ST", weight: 15 },
+ ],
+ },
+ {
+ country: "AT",
+ weight: 2,
+ regions: [
+ { code: "AT-9", weight: 35 }, // Vienna
+ { code: "AT-6", weight: 20 }, // Styria
+ { code: "AT-4", weight: 15 }, // Upper Austria
+ ],
+ },
+ {
+ country: "JP",
+ weight: 2,
+ regions: [
+ { code: "JP-13", weight: 30 }, // Tokyo
+ { code: "JP-27", weight: 15 }, // Osaka
+ { code: "JP-14", weight: 10 }, // Kanagawa
+ ],
+ },
+ {
+ country: "BR",
+ weight: 1,
+ regions: [
+ { code: "BR-SP", weight: 35 },
+ { code: "BR-RJ", weight: 20 },
+ { code: "BR-MG", weight: 15 },
+ ],
+ },
+];
+
+// Helper function to select a random item based on weights
+function weightedRandom(items) {
+ const totalWeight = items.reduce((sum, item) => sum + item.weight, 0);
+ let random = Math.random() * totalWeight;
+
+ for (const item of items) {
+ random -= item.weight;
+ if (random <= 0) {
+ return item;
+ }
+ }
+
+ return items[0]; // Fallback
+}
+
+// Helper function to generate a UUID
+function generateUUID() {
+ return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function (c) {
+ const r = (Math.random() * 16) | 0;
+ const v = c === "x" ? r : (r & 0x3) | 0x8;
+ return v.toString(16);
+ });
+}
+
+// Function to generate a realistic timestamp for a given day with more traffic during peak hours
+function generateTimeForDay(date) {
+ // Define peak hours (in 24-hour format)
+ const peakHours = [
+ { hour: 9, weight: 5 }, // 9 AM
+ { hour: 10, weight: 6 }, // 10 AM
+ { hour: 11, weight: 7 }, // 11 AM
+ { hour: 12, weight: 8 }, // 12 PM
+ { hour: 13, weight: 9 }, // 1 PM
+ { hour: 14, weight: 10 }, // 2 PM
+ { hour: 15, weight: 10 }, // 3 PM
+ { hour: 16, weight: 9 }, // 4 PM
+ { hour: 17, weight: 8 }, // 5 PM
+ { hour: 18, weight: 9 }, // 6 PM
+ { hour: 19, weight: 10 }, // 7 PM
+ { hour: 20, weight: 10 }, // 8 PM
+ { hour: 21, weight: 9 }, // 9 PM
+ { hour: 22, weight: 7 }, // 10 PM
+ { hour: 23, weight: 5 }, // 11 PM
+ { hour: 0, weight: 3 }, // 12 AM
+ { hour: 1, weight: 2 }, // 1 AM
+ { hour: 2, weight: 1 }, // 2 AM
+ { hour: 3, weight: 1 }, // 3 AM
+ { hour: 4, weight: 1 }, // 4 AM
+ { hour: 5, weight: 1 }, // 5 AM
+ { hour: 6, weight: 2 }, // 6 AM
+ { hour: 7, weight: 3 }, // 7 AM
+ { hour: 8, weight: 4 }, // 8 AM
+ ];
+
+ const hour = weightedRandom(peakHours).hour;
+ const minute = Math.floor(Math.random() * 60);
+ const second = Math.floor(Math.random() * 60);
+
+ return date.set({ hour, minute, second });
+}
+
+// Helper function to format estimated time remaining
+function formatETA(currentCount, targetCount, eventsPerSecond, elapsedSeconds) {
+ if (eventsPerSecond <= 0 || elapsedSeconds < 1) return "calculating...";
+
+ const remainingEvents = targetCount - currentCount;
+
+ // Use a dynamic slowdown factor based on how far along we are
+ // The more events we've generated, the more we expect slowdown
+ const progress = currentCount / targetCount;
+ const slowdownFactor = 1 + progress * 0.3; // Gradually add up to 30% buffer as we progress
+
+ // Calculate seconds remaining with the slowdown factor applied
+ const adjustedRate = eventsPerSecond / slowdownFactor;
+ const secondsRemaining = remainingEvents / adjustedRate;
+
+ // Format the ETA string
+ if (secondsRemaining < 60) {
+ return `${Math.ceil(secondsRemaining)}s`;
+ } else if (secondsRemaining < 3600) {
+ return `${Math.floor(secondsRemaining / 60)}m ${Math.ceil(
+ secondsRemaining % 60
+ )}s`;
+ } else {
+ const hours = Math.floor(secondsRemaining / 3600);
+ const minutes = Math.floor((secondsRemaining % 3600) / 60);
+ return `${hours}h ${minutes}m`;
+ }
+}
+
+// Helper function to format elapsed time
+function formatTime(seconds) {
+ if (seconds < 60) {
+ return `${Math.floor(seconds)}s`;
+ } else if (seconds < 3600) {
+ const minutes = Math.floor(seconds / 60);
+ const remainingSeconds = Math.floor(seconds % 60);
+ return `${minutes}m ${remainingSeconds}s`;
+ } else {
+ const hours = Math.floor(seconds / 3600);
+ const minutes = Math.floor((seconds % 3600) / 60);
+ const remainingSeconds = Math.floor(seconds % 60);
+ return `${hours}h ${minutes}m ${remainingSeconds}s`;
+ }
+}
+
+// Function to generate events for a specific day
+async function generateEventsForDay(date, targetEventsCount) {
+ const events = [];
+
+ // Active sessions map to track ongoing sessions
+ const activeSessions = new Map();
+
+ // Vary the count by ±15% to make it more realistic
+ const variation = Math.random() * 0.3 - 0.15; // -15% to +15%
+ const actualEventsCount = Math.round(targetEventsCount * (1 + variation));
+
+ console.log(
+ `Generating ${actualEventsCount.toLocaleString()} events for ${date.toFormat(
+ "yyyy-MM-dd"
+ )}`
+ );
+
+ // Generate 4-10x more events than sessions (averaging 5-6 events per session)
+ const eventsPerSession = 6;
+ // Increase initial allocation for better performance
+ const initialCapacity = Math.min(actualEventsCount, 2000000);
+ events.length = initialCapacity;
+ let eventCount = 0;
+
+ const sessionsToGenerate = Math.ceil(actualEventsCount / eventsPerSession);
+
+ console.log(
+ `Preparing ${sessionsToGenerate.toLocaleString()} sessions with timestamps...`
+ );
+
+ // Pre-generate timestamps for the day with realistic distribution in batches
+ // This reduces memory pressure while maintaining chronological order
+ const TIMESTAMP_BATCH_SIZE = 100000;
+ let totalTimestampsGenerated = 0;
+ let timestampBatch = [];
+
+ // For tracking progress
+ const progressInterval = Math.max(Math.ceil(sessionsToGenerate / 50), 500); // Show progress more frequently
+ const startGenerationTime = Date.now();
+ let lastProgressTime = startGenerationTime;
+
+ // For improved ETA calculation
+ let recentRates = [];
+ const MAX_RATES_TO_TRACK = 5; // Keep track of the last 5 generation rates
+
+ console.log(`Beginning event generation in optimized batches...`);
+
+ // Process in batches to optimize memory usage and performance
+ while (totalTimestampsGenerated < sessionsToGenerate) {
+ // Generate next batch of timestamps
+ const batchSize = Math.min(
+ TIMESTAMP_BATCH_SIZE,
+ sessionsToGenerate - totalTimestampsGenerated
+ );
+ timestampBatch = [];
+
+ // Generate timestamps
+ for (let i = 0; i < batchSize; i++) {
+ timestampBatch.push(
+ generateTimeForDay(DateTime.fromJSDate(date.toJSDate()))
+ );
+ }
+
+ // Sort this batch chronologically
+ timestampBatch.sort((a, b) => a.toMillis() - b.toMillis());
+
+ // Process each timestamp to generate sessions and events
+ for (let idx = 0; idx < timestampBatch.length; idx++) {
+ const timestamp = timestampBatch[idx];
+ const sessionIdx = totalTimestampsGenerated + idx;
+
+ // Display progress updates periodically
+ if (
+ sessionIdx % progressInterval === 0 ||
+ sessionIdx === sessionsToGenerate - 1
+ ) {
+ const currentTime = Date.now();
+ const elapsedSeconds = (currentTime - startGenerationTime) / 1000;
+ const timeSinceLastProgress = (currentTime - lastProgressTime) / 1000;
+
+ // Calculate progress based on actual events generated, not sessions
+ const percentComplete = Math.min(
+ ((eventCount / actualEventsCount) * 100).toFixed(1),
+ 100.0
+ );
+
+ // Format elapsed time
+ const elapsedFormatted = formatTime(elapsedSeconds);
+
+ // Calculate current generation rate for this interval
+ const currentIntervalRate =
+ eventCount > 0 ? Math.round(eventCount / elapsedSeconds) : 0;
+
+ // Store recent rates for weighted average calculation
+ if (timeSinceLastProgress > 0) {
+ // Calculate rate just for this most recent interval
+ const newEventsInThisInterval =
+ eventCount -
+ (recentRates.length > 0
+ ? eventCount -
+ (elapsedSeconds - timeSinceLastProgress) * currentIntervalRate
+ : 0);
+ const intervalRate = Math.round(
+ newEventsInThisInterval / timeSinceLastProgress
+ );
+
+ // Add to recent rates, keeping only the most recent ones
+ recentRates.push(intervalRate);
+ if (recentRates.length > MAX_RATES_TO_TRACK) {
+ recentRates.shift(); // Remove oldest rate
+ }
+ }
+
+ // Calculate weighted average of recent rates (more weight to recent rates)
+ let weightedRate = currentIntervalRate;
+ if (recentRates.length > 0) {
+ let totalWeight = 0;
+ let weightedSum = 0;
+ for (let i = 0; i < recentRates.length; i++) {
+ const weight = i + 1; // More recent rates get higher weights
+ weightedSum += recentRates[i] * weight;
+ totalWeight += weight;
+ }
+ weightedRate = Math.round(weightedSum / totalWeight);
+ }
+
+ console.log(
+ `Progress: ${percentComplete}% | Generated ${eventCount.toLocaleString()} of ${actualEventsCount.toLocaleString()} events ` +
+ `| ${weightedRate.toLocaleString()} events/sec | Time: ${elapsedFormatted} | ETA: ${formatETA(
+ eventCount,
+ actualEventsCount,
+ weightedRate,
+ elapsedSeconds
+ )}`
+ );
+
+ lastProgressTime = currentTime;
+ }
+
+ // Optimized session generation logic
+ // Decide whether to continue an existing session or create a new one
+ const now = timestamp.toMillis();
+ let sessionId, userId, sessionData;
+
+ // First expire old sessions (more than 30 minutes old)
+ if (activeSessions.size > 100) {
+ // Only check when we have many active sessions
+ for (const [sid, sessionInfo] of activeSessions.entries()) {
+ if (now - sessionInfo.lastActivity > 30 * 60 * 1000) {
+ activeSessions.delete(sid);
+ }
+ }
+ }
+
+ // 20% chance to continue an existing session if any are active and not too many sessions
+ if (
+ activeSessions.size > 0 &&
+ activeSessions.size < 1000 &&
+ Math.random() < 0.2
+ ) {
+ // Continue an existing session (select one of the first 50 to keep it fast)
+ const activeSids = Array.from(activeSessions.keys()).slice(0, 50);
+ sessionId = activeSids[Math.floor(Math.random() * activeSids.length)];
+ const sessionInfo = activeSessions.get(sessionId);
+ userId = sessionInfo.userId;
+ sessionData = sessionInfo.data;
+
+ // Update last activity timestamp
+ sessionInfo.lastActivity = now;
+ } else {
+ // Create a new session
+ sessionId = generateUUID();
+ userId = userIds[Math.floor(Math.random() * userIds.length)];
+ sessionData = generateSessionData();
+
+ // Add to active sessions
+ activeSessions.set(sessionId, {
+ userId,
+ data: sessionData,
+ lastActivity: now,
+ });
+ }
+
+ // Generate events for this session using fast, optimized event generation
+ const sessionEvents = generateSessionEventsOptimized(
+ userId,
+ sessionId,
+ timestamp,
+ sessionData
+ );
+
+ // Add these events to our events array
+ const newEventCount = sessionEvents.length;
+
+ // Ensure we have capacity
+ if (eventCount + newEventCount > events.length) {
+ // Need more room, increase array size
+ events.length = Math.max(
+ events.length * 1.5,
+ eventCount + newEventCount
+ );
+ }
+
+ // Copy events
+ for (let i = 0; i < newEventCount; i++) {
+ events[eventCount++] = sessionEvents[i];
+ }
+
+ // Check if we've reached our target event count
+ if (eventCount >= actualEventsCount) {
+ break;
+ }
+ }
+
+ totalTimestampsGenerated += timestampBatch.length;
+
+ // If we've reached our target event count, break out
+ if (eventCount >= actualEventsCount) {
+ break;
+ }
+ }
+
+ // Trim to the actual event count
+ const finalEvents = events.slice(0, Math.min(eventCount, actualEventsCount));
+
+ const generationEndTime = Date.now();
+ const generationSeconds = (
+ (generationEndTime - startGenerationTime) /
+ 1000
+ ).toFixed(2);
+ const genEventsPerSecond = Math.round(finalEvents.length / generationSeconds);
+
+ console.log(
+ `Generated ${finalEvents.length.toLocaleString()} events in ${generationSeconds} seconds ` +
+ `(${genEventsPerSecond.toLocaleString()} events/sec). Starting insertion...`
+ );
+
+ // Add timing measurement
+ const startTime = Date.now();
+ let totalInserted = 0;
+
+ // Increase batch size for higher throughput
+ const BATCH_SIZE = 50000; // Increased from 10000 to 50000
+
+ // Create batches
+ const batches = [];
+ for (let i = 0; i < finalEvents.length; i += BATCH_SIZE) {
+ batches.push(finalEvents.slice(i, i + BATCH_SIZE));
+ }
+
+ // Maximum number of parallel inserts (based on CPU cores)
+ const MAX_PARALLEL = 3; // Using 3 of the 4 cores for parallelism
+
+ // Process batches with controlled parallelism
+ for (let i = 0; i < batches.length; i += MAX_PARALLEL) {
+ const batchPromises = [];
+
+ // Create a set of promises for parallel execution
+ for (let j = 0; j < MAX_PARALLEL && i + j < batches.length; j++) {
+ const batchIndex = i + j;
+ const batch = batches[batchIndex];
+
+ batchPromises.push(
+ (async () => {
+ const batchStartTime = Date.now();
+
+ try {
+ await clickhouse.insert({
+ table: "pageviews",
+ values: batch,
+ format: "JSONEachRow",
+ });
+
+ const batchEndTime = Date.now();
+ const batchSeconds = (batchEndTime - batchStartTime) / 1000;
+ const batchSpeed = Math.round(batch.length / batchSeconds);
+
+ return {
+ batchIndex,
+ batchSize: batch.length,
+ batchSpeed,
+ success: true,
+ };
+ } catch (error) {
+ console.error(`Error inserting batch ${batchIndex + 1}:`, error);
+ throw error;
+ }
+ })()
+ );
+ }
+
+ // Wait for this group of batches to complete
+ try {
+ const results = await Promise.all(batchPromises);
+
+ // Update metrics
+ for (const result of results) {
+ if (result.success) {
+ totalInserted += result.batchSize;
+
+ const totalElapsed = (Date.now() - startTime) / 1000;
+ const averageSpeed = Math.round(totalInserted / totalElapsed);
+
+ console.log(
+ `Inserted batch ${result.batchIndex + 1} of ${batches.length} | ` +
+ `Batch speed: ${result.batchSpeed.toLocaleString()} events/sec | ` +
+ `Avg speed: ${averageSpeed.toLocaleString()} events/sec`
+ );
+ }
+ }
+ } catch (error) {
+ console.error("Error in batch processing:", error);
+ throw error;
+ }
+ }
+
+ const endTime = Date.now();
+ const totalSeconds = (endTime - startTime) / 1000;
+ const overallSpeed = Math.round(finalEvents.length / totalSeconds);
+
+ console.log(
+ `Day completed in ${totalSeconds.toFixed(2)} seconds | ` +
+ `Overall insertion speed: ${overallSpeed.toLocaleString()} events/sec | ` +
+ `CPU cores utilized: ${MAX_PARALLEL} of 4`
+ );
+
+ return finalEvents.length;
+}
+
+// Optimized version of generateSessionEvents for speed
+function generateSessionEventsOptimized(
+ userId,
+ sessionId,
+ startTime,
+ sessionData
+) {
+ const events = [];
+ let currentTime = startTime;
+
+ // Use simpler random number generation for speed
+ const sessionDuration = Math.floor(Math.random() * 1800) + 60; // 1-30 minutes in seconds
+ const numberOfPageviews = Math.floor(Math.random() * 12) + 1; // 1-12 pageviews per session
+
+ let currentPath = null;
+ let currentProduct = null;
+ let cartItems = [];
+ let hasPurchased = false;
+
+ // Pre-allocate array for better performance
+ events.length = Math.min(numberOfPageviews * 3, 30); // Approximate - allow space for multiple events per page
+ let eventCount = 0;
+
+ // Avoid repeated code by creating a template event object
+ const baseEvent = {
+ site_id: SITE_ID,
+ session_id: sessionId,
+ user_id: userId,
+ hostname: SITE_DOMAIN,
+ browser: sessionData.browser,
+ browser_version: sessionData.browserVersion,
+ operating_system: sessionData.os,
+ operating_system_version: sessionData.osVersion,
+ language: sessionData.language,
+ screen_width: sessionData.screenWidth,
+ screen_height: sessionData.screenHeight,
+ device_type: sessionData.deviceType,
+ country: sessionData.country,
+ iso_3166_2: sessionData.iso3166,
+ };
+
+ // Simplified page selection logic for speed
+ for (let i = 0; i < numberOfPageviews; i++) {
+ // Select page
+ if (i === 0) {
+ // First page - entry point
+ const entryPages = [
+ pagePaths[0], // homepage
+ pagePaths[1], // products
+ // A few random category pages
+ pagePaths[Math.floor(Math.random() * 5) + 15],
+ pagePaths[Math.floor(Math.random() * 5) + 15],
+ ];
+ currentPath = entryPages[Math.floor(Math.random() * entryPages.length)];
+ } else {
+ // Follow simplified, faster pathing logic
+ if (currentPath.path.startsWith("/product/") && Math.random() < 0.4) {
+ // From product, might add to cart and go to cart
+ if (Math.random() < 0.4 && currentPath.product) {
+ cartItems.push(currentPath.product);
+ }
+
+ if (cartItems.length > 0 && Math.random() < 0.5) {
+ // Go to cart
+ currentPath =
+ pagePaths.find((p) => p.path === "/cart") || pagePaths[0];
+ } else {
+ // Go to another product or category
+ const idx = Math.floor(Math.random() * pagePaths.length);
+ currentPath = pagePaths[idx];
+ }
+ } else if (currentPath.path === "/cart" && cartItems.length > 0) {
+ // From cart, might go to checkout
+ if (Math.random() < 0.6) {
+ currentPath =
+ pagePaths.find((p) => p.path === "/checkout") || pagePaths[0];
+ } else {
+ // Continue shopping
+ const idx = Math.floor(Math.random() * pagePaths.length);
+ currentPath = pagePaths[idx];
+ }
+ } else if (
+ currentPath.path === "/checkout" &&
+ cartItems.length > 0 &&
+ !hasPurchased
+ ) {
+ if (Math.random() < 0.7) {
+ // Purchase completed
+ hasPurchased = true;
+ currentPath =
+ pagePaths.find((p) => p.path === "/order-history") || pagePaths[0];
+ } else {
+ // Abandoned
+ currentPath =
+ pagePaths.find((p) => p.path === "/cart") || pagePaths[0];
+ }
+ } else {
+ // Random navigation
+ const idx = Math.floor(Math.random() * pagePaths.length);
+ currentPath = pagePaths[idx];
+ }
+ }
+
+ // Update product if on product page
+ if (currentPath.path.startsWith("/product/")) {
+ currentProduct = currentPath.product;
+ }
+
+ // Determine querystring (simplified)
+ let querystring = "";
+ if (Math.random() < 0.3) {
+ for (const prefix in tabParams) {
+ if (currentPath.path.includes(prefix)) {
+ const params = tabParams[prefix];
+ querystring = params[Math.floor(Math.random() * params.length)].param;
+ break;
+ }
+ }
+ }
+
+ // Add pageview event
+ currentTime = startTime.plus({
+ seconds: i * 30 + Math.floor(Math.random() * 30),
+ });
+ events[eventCount++] = {
+ ...baseEvent,
+ timestamp: currentTime.toFormat("yyyy-MM-dd HH:mm:ss"),
+ pathname: currentPath.path,
+ querystring: querystring,
+ page_title: currentPath.title,
+ referrer: i === 0 ? sessionData.referrer : "",
+ type: "pageview",
+ event_name: "",
+ properties: "",
+ };
+
+ // Add custom event based on page type (simplified for speed)
+ if (currentPath.path.startsWith("/product/") && currentProduct) {
+ // Product view
+ currentTime = currentTime.plus({
+ seconds: Math.floor(Math.random() * 5) + 2,
+ });
+ events[eventCount++] = {
+ ...baseEvent,
+ timestamp: currentTime.toFormat("yyyy-MM-dd HH:mm:ss"),
+ pathname: currentPath.path,
+ querystring: querystring,
+ page_title: currentPath.title,
+ referrer: "",
+ type: "custom_event",
+ event_name: "product-view",
+ properties: JSON.stringify({
+ product_id: currentProduct.id,
+ product_name: currentProduct.name,
+ category: currentProduct.category,
+ price: currentProduct.price,
+ currency: "USD",
+ }),
+ };
+
+ // Possible add to cart (30% chance)
+ if (Math.random() < 0.3 && !cartItems.includes(currentProduct)) {
+ currentTime = currentTime.plus({
+ seconds: Math.floor(Math.random() * 10) + 5,
+ });
+ cartItems.push(currentProduct);
+
+ events[eventCount++] = {
+ ...baseEvent,
+ timestamp: currentTime.toFormat("yyyy-MM-dd HH:mm:ss"),
+ pathname: currentPath.path,
+ querystring: querystring,
+ page_title: currentPath.title,
+ referrer: "",
+ type: "custom_event",
+ event_name: "add-to-cart",
+ properties: JSON.stringify({
+ product_id: currentProduct.id,
+ product_name: currentProduct.name,
+ category: currentProduct.category,
+ price: currentProduct.price,
+ quantity: 1,
+ currency: "USD",
+ }),
+ };
+ }
+ } else if (currentPath.path === "/cart" && cartItems.length > 0) {
+ // View cart
+ const cartTotal = cartItems.reduce(
+ (sum, item) => sum + parseFloat(item.price),
+ 0
+ );
+
+ events[eventCount++] = {
+ ...baseEvent,
+ timestamp: currentTime.toFormat("yyyy-MM-dd HH:mm:ss"),
+ pathname: currentPath.path,
+ querystring: querystring,
+ page_title: currentPath.title,
+ referrer: "",
+ type: "custom_event",
+ event_name: "view-cart",
+ properties: JSON.stringify({
+ items_count: cartItems.length,
+ value: cartTotal.toFixed(2),
+ currency: "USD",
+ }),
+ };
+ } else if (
+ currentPath.path === "/checkout" &&
+ cartItems.length > 0 &&
+ !hasPurchased
+ ) {
+ // Begin checkout
+ const cartTotal = cartItems.reduce(
+ (sum, item) => sum + parseFloat(item.price),
+ 0
+ );
+
+ events[eventCount++] = {
+ ...baseEvent,
+ timestamp: currentTime.toFormat("yyyy-MM-dd HH:mm:ss"),
+ pathname: currentPath.path,
+ querystring: querystring,
+ page_title: currentPath.title,
+ referrer: "",
+ type: "custom_event",
+ event_name: "begin-checkout",
+ properties: JSON.stringify({
+ items_count: cartItems.length,
+ value: cartTotal.toFixed(2),
+ currency: "USD",
+ }),
+ };
+
+ // Add purchase if this is the last pageview or randomly
+ if (
+ (i === numberOfPageviews - 1 || Math.random() < 0.7) &&
+ !hasPurchased
+ ) {
+ currentTime = currentTime.plus({
+ seconds: Math.floor(Math.random() * 30) + 20,
+ });
+ const transactionId = Math.random()
+ .toString(36)
+ .substring(2, 10)
+ .toUpperCase();
+
+ events[eventCount++] = {
+ ...baseEvent,
+ timestamp: currentTime.toFormat("yyyy-MM-dd HH:mm:ss"),
+ pathname: currentPath.path,
+ querystring: querystring,
+ page_title: currentPath.title,
+ referrer: "",
+ type: "custom_event",
+ event_name: "purchase",
+ properties: JSON.stringify({
+ transaction_id: transactionId,
+ value: cartTotal.toFixed(2),
+ tax: (cartTotal * 0.08).toFixed(2),
+ shipping: (Math.random() * 10 + 5).toFixed(2),
+ currency: "USD",
+ coupon: Math.random() < 0.3 ? "SAVE20" : "",
+ items: cartItems.map((item) => ({
+ product_id: item.id,
+ product_name: item.name,
+ category: item.category,
+ price: item.price,
+ quantity: 1,
+ })),
+ }),
+ };
+
+ hasPurchased = true;
+ }
+ }
+
+ // Return if we exceed session duration
+ if (currentTime > startTime.plus({ seconds: sessionDuration })) {
+ break;
+ }
+ }
+
+ // Return only the populated part of the array
+ return events.slice(0, eventCount);
+}
+
+// Function to generate session data (browser, OS, screen resolution, etc.)
+function generateSessionData() {
+ // Select browser and version
+ const browser = weightedRandom(browsers);
+ const browserVersion =
+ browser.versions[Math.floor(Math.random() * browser.versions.length)];
+
+ // Select OS and version
+ const os = weightedRandom(operatingSystems);
+ const osVersion = os.versions[Math.floor(Math.random() * os.versions.length)];
+
+ // Select screen resolution
+ const resolution = weightedRandom(screenResolutions);
+
+ // Generate language code (instead of using faker.locale)
+ const languageCodes = [
+ "en",
+ "es",
+ "fr",
+ "de",
+ "it",
+ "ru",
+ "zh",
+ "ja",
+ "pt",
+ "nl",
+ ];
+ const language =
+ languageCodes[Math.floor(Math.random() * languageCodes.length)] +
+ (Math.random() < 0.5 ? "" : "-" + faker.location.countryCode());
+
+ // Select referrer
+ const referrer = weightedRandom(referrers).url;
+
+ // Use Faker for location data
+ const country = faker.location.countryCode();
+ let region = "";
+
+ // Get region code based on country
+ if (country === "US") {
+ region = faker.location.state({ abbreviated: true });
+ } else {
+ // For non-US, we'll use a simple region code
+ region = faker.location.county().slice(0, 3).toUpperCase();
+ }
+
+ // Determine device type based on screen resolution and OS
+ let deviceType = "Desktop";
+ if (os.name === "Android" || os.name === "iOS") {
+ deviceType = "Mobile";
+ if (resolution.width > 768) {
+ deviceType = "Tablet";
+ }
+ } else if (resolution.width <= 1024) {
+ deviceType = "Mobile";
+ }
+
+ return {
+ browser: browser.name,
+ browserVersion: browserVersion,
+ os: os.name,
+ osVersion: osVersion,
+ screenWidth: resolution.width,
+ screenHeight: resolution.height,
+ language: language,
+ referrer: referrer,
+ country: country,
+ iso3166: country && region ? `${country}-${region}` : country,
+ deviceType: deviceType,
+ };
+}
+
+// Main function to generate all the data
+async function generateMockData() {
+ let totalEvents = 0;
+
+ // Generate data for each day, starting from the most recent
+ for (let day = 0; day < daysInPast; day++) {
+ const date = DateTime.now().minus({ days: day });
+ const eventsCount = await generateEventsForDay(date, eventsPerDay);
+ totalEvents += eventsCount;
+ console.log(
+ `Completed day ${day + 1} of ${daysInPast}. Total events: ${totalEvents}`
+ );
+ }
+
+ console.log(
+ `Mock data generation complete. Generated ${totalEvents} events across ${daysInPast} days.`
+ );
+ process.exit(0);
+}
+
+// Start the data generation
+generateMockData().catch((error) => {
+ console.error("Error generating mock data:", error);
+ process.exit(1);
+});
diff --git a/mockdata/package-lock.json b/mockdata/package-lock.json
new file mode 100644
index 000000000..abcde73a7
--- /dev/null
+++ b/mockdata/package-lock.json
@@ -0,0 +1,279 @@
+{
+ "name": "mockdata",
+ "version": "1.0.0",
+ "lockfileVersion": 3,
+ "requires": true,
+ "packages": {
+ "": {
+ "name": "mockdata",
+ "version": "1.0.0",
+ "license": "ISC",
+ "dependencies": {
+ "@clickhouse/client": "^0.2.0",
+ "@faker-js/faker": "^8.0.2",
+ "dotenv": "^16.3.1",
+ "luxon": "^3.4.0"
+ },
+ "devDependencies": {
+ "@types/luxon": "^3.4.2",
+ "@types/node": "^20.17.25",
+ "ts-node": "^10.9.1",
+ "typescript": "^5.2.2"
+ }
+ },
+ "node_modules/@clickhouse/client": {
+ "version": "0.2.10",
+ "resolved": "https://registry.npmjs.org/@clickhouse/client/-/client-0.2.10.tgz",
+ "integrity": "sha512-ZwBgzjEAFN/ogS0ym5KHVbR7Hx/oYCX01qGp2baEyfN2HM73kf/7Vp3GvMHWRy+zUXISONEtFv7UTViOXnmFrg==",
+ "dependencies": {
+ "@clickhouse/client-common": "0.2.10"
+ },
+ "engines": {
+ "node": ">=16"
+ }
+ },
+ "node_modules/@clickhouse/client-common": {
+ "version": "0.2.10",
+ "resolved": "https://registry.npmjs.org/@clickhouse/client-common/-/client-common-0.2.10.tgz",
+ "integrity": "sha512-BvTY0IXS96y9RUeNCpKL4HUzHmY80L0lDcGN0lmUD6zjOqYMn78+xyHYJ/AIAX7JQsc+/KwFt2soZutQTKxoGQ=="
+ },
+ "node_modules/@cspotcode/source-map-support": {
+ "version": "0.8.1",
+ "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
+ "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==",
+ "dev": true,
+ "dependencies": {
+ "@jridgewell/trace-mapping": "0.3.9"
+ },
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/@faker-js/faker": {
+ "version": "8.4.1",
+ "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-8.4.1.tgz",
+ "integrity": "sha512-XQ3cU+Q8Uqmrbf2e0cIC/QN43sTBSC8KF12u29Mb47tWrt2hAgBXSgpZMj4Ao8Uk0iJcU99QsOCaIL8934obCg==",
+ "funding": [
+ {
+ "type": "opencollective",
+ "url": "https://opencollective.com/fakerjs"
+ }
+ ],
+ "engines": {
+ "node": "^14.17.0 || ^16.13.0 || >=18.0.0",
+ "npm": ">=6.14.13"
+ }
+ },
+ "node_modules/@jridgewell/resolve-uri": {
+ "version": "3.1.2",
+ "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz",
+ "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==",
+ "dev": true,
+ "engines": {
+ "node": ">=6.0.0"
+ }
+ },
+ "node_modules/@jridgewell/sourcemap-codec": {
+ "version": "1.5.0",
+ "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz",
+ "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==",
+ "dev": true
+ },
+ "node_modules/@jridgewell/trace-mapping": {
+ "version": "0.3.9",
+ "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz",
+ "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==",
+ "dev": true,
+ "dependencies": {
+ "@jridgewell/resolve-uri": "^3.0.3",
+ "@jridgewell/sourcemap-codec": "^1.4.10"
+ }
+ },
+ "node_modules/@tsconfig/node10": {
+ "version": "1.0.11",
+ "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz",
+ "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==",
+ "dev": true
+ },
+ "node_modules/@tsconfig/node12": {
+ "version": "1.0.11",
+ "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz",
+ "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==",
+ "dev": true
+ },
+ "node_modules/@tsconfig/node14": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz",
+ "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==",
+ "dev": true
+ },
+ "node_modules/@tsconfig/node16": {
+ "version": "1.0.4",
+ "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz",
+ "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==",
+ "dev": true
+ },
+ "node_modules/@types/luxon": {
+ "version": "3.4.2",
+ "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.4.2.tgz",
+ "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==",
+ "dev": true
+ },
+ "node_modules/@types/node": {
+ "version": "20.17.25",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.25.tgz",
+ "integrity": "sha512-bT+r2haIlplJUYtlZrEanFHdPIZTeiMeh/fSOEbOOfWf9uTn+lg8g0KU6Q3iMgjd9FLuuMAgfCNSkjUbxL6E3Q==",
+ "dev": true,
+ "dependencies": {
+ "undici-types": "~6.19.2"
+ }
+ },
+ "node_modules/acorn": {
+ "version": "8.14.1",
+ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz",
+ "integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==",
+ "dev": true,
+ "bin": {
+ "acorn": "bin/acorn"
+ },
+ "engines": {
+ "node": ">=0.4.0"
+ }
+ },
+ "node_modules/acorn-walk": {
+ "version": "8.3.4",
+ "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz",
+ "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==",
+ "dev": true,
+ "dependencies": {
+ "acorn": "^8.11.0"
+ },
+ "engines": {
+ "node": ">=0.4.0"
+ }
+ },
+ "node_modules/arg": {
+ "version": "4.1.3",
+ "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
+ "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
+ "dev": true
+ },
+ "node_modules/create-require": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz",
+ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
+ "dev": true
+ },
+ "node_modules/diff": {
+ "version": "4.0.2",
+ "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
+ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
+ "dev": true,
+ "engines": {
+ "node": ">=0.3.1"
+ }
+ },
+ "node_modules/dotenv": {
+ "version": "16.4.7",
+ "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz",
+ "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==",
+ "engines": {
+ "node": ">=12"
+ },
+ "funding": {
+ "url": "https://dotenvx.com"
+ }
+ },
+ "node_modules/luxon": {
+ "version": "3.5.0",
+ "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz",
+ "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==",
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/make-error": {
+ "version": "1.3.6",
+ "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
+ "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
+ "dev": true
+ },
+ "node_modules/ts-node": {
+ "version": "10.9.2",
+ "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz",
+ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
+ "dev": true,
+ "dependencies": {
+ "@cspotcode/source-map-support": "^0.8.0",
+ "@tsconfig/node10": "^1.0.7",
+ "@tsconfig/node12": "^1.0.7",
+ "@tsconfig/node14": "^1.0.0",
+ "@tsconfig/node16": "^1.0.2",
+ "acorn": "^8.4.1",
+ "acorn-walk": "^8.1.1",
+ "arg": "^4.1.0",
+ "create-require": "^1.1.0",
+ "diff": "^4.0.1",
+ "make-error": "^1.1.1",
+ "v8-compile-cache-lib": "^3.0.1",
+ "yn": "3.1.1"
+ },
+ "bin": {
+ "ts-node": "dist/bin.js",
+ "ts-node-cwd": "dist/bin-cwd.js",
+ "ts-node-esm": "dist/bin-esm.js",
+ "ts-node-script": "dist/bin-script.js",
+ "ts-node-transpile-only": "dist/bin-transpile.js",
+ "ts-script": "dist/bin-script-deprecated.js"
+ },
+ "peerDependencies": {
+ "@swc/core": ">=1.2.50",
+ "@swc/wasm": ">=1.2.50",
+ "@types/node": "*",
+ "typescript": ">=2.7"
+ },
+ "peerDependenciesMeta": {
+ "@swc/core": {
+ "optional": true
+ },
+ "@swc/wasm": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/typescript": {
+ "version": "5.8.2",
+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz",
+ "integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==",
+ "dev": true,
+ "bin": {
+ "tsc": "bin/tsc",
+ "tsserver": "bin/tsserver"
+ },
+ "engines": {
+ "node": ">=14.17"
+ }
+ },
+ "node_modules/undici-types": {
+ "version": "6.19.8",
+ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
+ "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
+ "dev": true
+ },
+ "node_modules/v8-compile-cache-lib": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
+ "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
+ "dev": true
+ },
+ "node_modules/yn": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",
+ "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==",
+ "dev": true,
+ "engines": {
+ "node": ">=6"
+ }
+ }
+ }
+}
diff --git a/mockdata/package.json b/mockdata/package.json
new file mode 100644
index 000000000..b903e3959
--- /dev/null
+++ b/mockdata/package.json
@@ -0,0 +1,18 @@
+{
+ "name": "mockdata",
+ "version": "1.0.0",
+ "main": "index.js",
+ "scripts": {
+ "generate": "node index.js",
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": "",
+ "license": "ISC",
+ "description": "",
+ "dependencies": {
+ "@clickhouse/client": "^1.11.0",
+ "@faker-js/faker": "^9.6.0",
+ "dotenv": "^16.3.1",
+ "luxon": "^3.4.4"
+ }
+}