Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

update importer script

Carl-Gerhard Lindesvärd65c464a63c6f
diff --git a/packages/cli/src/importer/importer.ts b/packages/cli/src/importer/importer.ts
index 0fe61ae23..b83629653 100644
--- a/packages/cli/src/importer/importer.ts
+++ b/packages/cli/src/importer/importer.ts
@@ -1,15 +1,16 @@
import { randomUUID } from 'crypto';
import fs from 'fs';
import readline from 'readline';
-import { glob } from 'glob';
+import zlib from 'zlib';
import Progress from 'progress';
import { assocPath, prop, uniqBy } from 'ramda';
-import { parsePath } from '@openpanel/common';
+import { isSameDomain, parsePath } from '@openpanel/common';
import type { IImportedEvent } from '@openpanel/db';
-const BATCH_SIZE = 1000;
+const BATCH_SIZE = 30_000;
const SLEEP_TIME = 20;
+const MAX_CONCURRENT_REQUESTS = 8;
type IMixpanelEvent = {
event: string;
@@ -179,6 +180,21 @@ function generateSessionEvents(events: IImportedEvent[]): Session[] {
}
function createEventObject(event: IMixpanelEvent): IImportedEvent {
+ const getReferrer = (referrer: string | undefined) => {
+ if (!referrer) {
+ return '';
+ }
+
+ if (referrer === '$direct') {
+ return '';
+ }
+
+ if (isSameDomain(referrer, event.properties.$current_url)) {
+ return '';
+ }
+
+ return referrer;
+ };
const url = parsePath(event.properties.$current_url);
return {
profile_id: event.properties.distinct_id
@@ -203,7 +219,7 @@ function createEventObject(event: IMixpanelEvent): IImportedEvent {
browser_version: event.properties.$browser_version
? String(event.properties.$browser_version)
: '',
- referrer: event.properties.$initial_referrer ?? '',
+ referrer: getReferrer(event.properties.$initial_referrer),
referrer_type: event.properties.$search_engine ? 'search' : '',
referrer_name: event.properties.$search_engine ?? '',
device_id: event.properties.$device_id ?? '',
@@ -295,17 +311,33 @@ function processEvents(events: IImportedEvent[]): IImportedEvent[] {
];
}
-async function sendBatchToAPI(batch: IImportedEvent[]) {
+async function sendBatchToAPI(
+ batch: IImportedEvent[],
+ {
+ apiUrl,
+ clientId,
+ clientSecret,
+ }: {
+ apiUrl: string;
+ clientId: string;
+ clientSecret: string;
+ }
+) {
try {
- const res = await fetch('http://localhost:3333/import/events', {
+ const res = await fetch(`${apiUrl}/import/events`, {
method: 'POST',
headers: {
+ 'Content-Encoding': 'gzip',
'Content-Type': 'application/json',
- 'openpanel-client-id': 'dd3db204-dcf6-49e2-9e82-de01cba7e585',
- 'openpanel-client-secret': 'sec_293b903816e327e10c9d',
+ 'openpanel-client-id': clientId,
+ 'openpanel-client-secret': clientSecret,
},
- body: JSON.stringify(batch),
+ body: zlib.gzipSync(JSON.stringify(batch)),
});
+ if (!res.ok) {
+ console.log('Failed to send batch to API');
+ console.log(await res.text());
+ }
await new Promise((resolve) => setTimeout(resolve, SLEEP_TIME));
} catch (e) {
console.log('sendBatchToAPI failed');
@@ -313,7 +345,17 @@ async function sendBatchToAPI(batch: IImportedEvent[]) {
}
}
-async function processFiles(files: string[]) {
+async function processFiles({
+ files,
+ apiUrl,
+ clientId,
+ clientSecret,
+}: {
+ files: string[];
+ apiUrl: string;
+ clientId: string;
+ clientSecret: string;
+}) {
const progress = new Progress(
'Processing (:current/:total) :file [:bar] :percent | :savedEvents saved events | :status',
{
@@ -347,34 +389,57 @@ async function processFiles(files: string[]) {
currentBatch = [];
}
- if (apiBatching.length >= 10) {
- await Promise.all(apiBatching.map(sendBatchToAPI));
+ if (apiBatching.length >= MAX_CONCURRENT_REQUESTS) {
+ await Promise.all(
+ apiBatching.map((batch) =>
+ sendBatchToAPI(batch, {
+ apiUrl,
+ clientId,
+ clientSecret,
+ })
+ )
+ );
apiBatching = [];
}
}
}
if (currentBatch.length > 0) {
- await sendBatchToAPI(currentBatch);
+ await sendBatchToAPI(currentBatch, {
+ apiUrl,
+ clientId,
+ clientSecret,
+ });
savedEvents += currentBatch.length;
progress.render({ file: 'Complete', savedEvents, status: 'Complete' });
}
}
-export async function importFiles(matcher: string) {
- const files = await glob([matcher], { root: '/' });
-
+export async function importFiles({
+ files,
+ apiUrl,
+ clientId,
+ clientSecret,
+}: {
+ files: string[];
+ apiUrl: string;
+ clientId: string;
+ clientSecret: string;
+}) {
if (files.length === 0) {
console.log('No files found');
return;
}
- files.sort((a, b) => a.localeCompare(b));
-
console.log(`Found ${files.length} files to process`);
const startTime = Date.now();
- await processFiles(files);
+ await processFiles({
+ files,
+ apiUrl,
+ clientId,
+ clientSecret,
+ });
const endTime = Date.now();
console.log(
diff --git a/packages/cli/src/importer/index.ts b/packages/cli/src/importer/index.ts
index ea1ef9ac2..3cc88e1ff 100644
--- a/packages/cli/src/importer/index.ts
+++ b/packages/cli/src/importer/index.ts
@@ -1,12 +1,19 @@
import path from 'path';
import arg from 'arg';
+import { glob } from 'glob';
import { importFiles } from './importer';
-export default function importer() {
+export default async function importer() {
const args = arg(
{
'--glob': String,
+ '--api-url': String,
+ '--client-id': String,
+ '--client-secret': String,
+ '--dry-run': Boolean,
+ '--from': Number,
+ '--to': Number,
},
{
permissive: true,
@@ -17,9 +24,36 @@ export default function importer() {
throw new Error('Missing --glob argument');
}
+ if (!args['--client-id']) {
+ throw new Error('Missing --client-id argument');
+ }
+
+ if (!args['--client-secret']) {
+ throw new Error('Missing --client-secret argument');
+ }
+
const cwd = process.cwd();
- const filePath = path.resolve(cwd, args['--glob']);
+ const fileMatcher = path.resolve(cwd, args['--glob']);
+ const allFiles = await glob([fileMatcher], { root: '/' });
+ allFiles.sort((a, b) => a.localeCompare(b));
+
+ const files = allFiles.slice(
+ args['--from'] ?? 0,
+ args['--to'] ?? Number.MAX_SAFE_INTEGER
+ );
+
+ if (args['--dry-run']) {
+ files.forEach((file, index) => {
+ console.log(`Would import (index: ${index}): ${file}`);
+ });
+ return;
+ }
- return importFiles(filePath);
+ return importFiles({
+ files,
+ clientId: args['--client-id'],
+ clientSecret: args['--client-secret'],
+ apiUrl: args['--api-url'] ?? 'https://api.openpanel.dev',
+ });
}