Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ingest: stop answering 500 for malformed bodies

Carl-Gerhard Lindesvärdf577ad85900a
diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.ts
index c04771201..bf0f3fa7f 100644
--- a/apps/api/src/config/env.ts
+++ b/apps/api/src/config/env.ts
@@ -57,6 +57,8 @@ const DEFAULT_KAFKA_SESSION_TIMEOUT_MS = 30_000;
const DEFAULT_KAFKA_HEARTBEAT_INTERVAL_MS = 3000;
const DEFAULT_KAFKA_REQUEST_TIMEOUT_MS = 5000;
const DEFAULT_KAFKA_CONNECTION_TIMEOUT_MS = 2000;
+/** Kafka's and Redpanda's own default `max.message.bytes`: 1 MiB. */
+const DEFAULT_KAFKA_MAX_MESSAGE_BYTES = 1_048_576;
const DEFAULT_KAFKA_PRODUCER_RETRIES = 2;
const DEFAULT_KAFKA_PRODUCER_INITIAL_RETRY_MS = 100;
const DEFAULT_KAFKA_PRODUCER_MAX_RETRY_MS = 1000;
@@ -534,6 +536,7 @@ const rawSchema = z.object({
KAFKA_SSL_REJECT_UNAUTHORIZED: strictOptionalBoolean,
KAFKA_SASL_USERNAME: optionalString,
KAFKA_SASL_PASSWORD: optionalString,
+ KAFKA_MAX_MESSAGE_BYTES: optionalPositiveInt,
KAFKA_SASL_MECHANISM: kafkaSaslMechanismSchema,
// --- ingest dead-letter ---
@@ -774,6 +777,8 @@ function deriveKafkaConfig(raw: RawEnv): CoreConfig['kafka'] {
eventsDlqTopic:
raw.KAFKA_EVENTS_DLQ_TOPIC ?? `${raw.KAFKA_EVENTS_TOPIC}-dlq`,
consumerGroup: raw.KAFKA_CONSUMER_GROUP,
+ maxMessageBytes:
+ raw.KAFKA_MAX_MESSAGE_BYTES ?? DEFAULT_KAFKA_MAX_MESSAGE_BYTES,
partitionsConcurrent: raw.KAFKA_PARTITIONS_CONCURRENT,
minMessages: raw.KAFKA_MIN_MESSAGES,
maxWaitMs: raw.KAFKA_MAX_WAIT_MS,
diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts
index d43aff3d8..e2c1c1605 100644
--- a/packages/core/src/config.ts
+++ b/packages/core/src/config.ts
@@ -139,6 +139,12 @@ export interface KafkaConfig {
producerBatchSize: number;
/** A partial batch's maximum wait, in ms, before it is sent anyway. */
producerBatchLingerMs: number;
+ /**
+ * The broker's `max.message.bytes`. `/track` refuses a larger body with a
+ * 413 rather than letting kafkajs answer with a raw protocol error at 500.
+ * Raise this alongside the broker's own setting.
+ */
+ maxMessageBytes: number;
handlerMaxAttempts: number;
handlerRetryInitialMs: number;
handlerRetryMaxMs: number;
diff --git a/packages/core/src/modules/ingest/ingest.routes.ts b/packages/core/src/modules/ingest/ingest.routes.ts
index 51bdc31f7..cba704c97 100644
--- a/packages/core/src/modules/ingest/ingest.routes.ts
+++ b/packages/core/src/modules/ingest/ingest.routes.ts
@@ -59,6 +59,36 @@ const ACCEPTED_STATUS = 202;
type StatusFn = (code: number, body?: unknown) => unknown;
+const PAYLOAD_TOO_LARGE = 413;
+
+/**
+ * kafkajs answers a body over the broker's `max.message.bytes` with a raw
+ * protocol error, which surfaced as a 500 carrying "The request included a
+ * message larger than the max message size the server will accept". The
+ * threshold is the broker's, so it is configuration rather than a constant —
+ * a deployment that raises `max.message.bytes` raises this with it.
+ */
+function refuseOversizedBody({
+ ctx,
+ request,
+ status,
+}: {
+ ctx: { config: { kafka: { maxMessageBytes: number } } };
+ request: Request;
+ status: StatusFn;
+}) {
+ const declared = Number(request.headers.get('content-length'));
+ const limit = ctx.config.kafka.maxMessageBytes;
+ if (!Number.isFinite(declared) || declared <= limit) {
+ return;
+ }
+ return status(PAYLOAD_TOO_LARGE, {
+ status: PAYLOAD_TOO_LARGE,
+ error: 'Payload Too Large',
+ message: `Body is ${declared} bytes; the limit is ${limit}.`,
+ });
+}
+
// V1's error handler turns each thrown HttpError into this body; core has no
// such handler, so the mapping is explicit and the codes and messages are
// V1's.
@@ -200,7 +230,7 @@ export const ingestRoutes = defineRoutes((app, deps: AppDeps) => {
{
clientAuth: { ingest: validateIngestRequest },
body: zTrackHandlerPayload,
- beforeHandle: [botGuard, blockWhenWoundDown],
+ beforeHandle: [refuseOversizedBody, botGuard, blockWhenWoundDown],
detail: {
tags: TAGS,
description:
diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.ts
index 504969ea6..45d36d5c1 100644
--- a/packages/core/src/modules/ingest/ingest.service.ts
+++ b/packages/core/src/modules/ingest/ingest.service.ts
@@ -175,8 +175,10 @@ export function getTimestamp(
payload: ITrackHandlerPayload['payload']
): { timestamp: number; isTimestampFromThePast: boolean } {
const safeTimestamp = timestamp || Date.now();
+ // `/event` has no body schema (ADR-003), so `payload` can be a string or
+ // null; `'properties' in payload` throws on a primitive.
const userDefinedTimestamp =
- 'properties' in payload
+ typeof payload === 'object' && payload !== null && 'properties' in payload
? (payload?.properties?.__timestamp as string | undefined)
: undefined;
@@ -655,6 +657,15 @@ export async function ingestLegacyEvent(
return { status: 'missing-project-id' };
}
+ // An event with no name is not an event. V1 answered 202 and wrote a row
+ // with `name = ''`, which then shows up in every picker and breakdown.
+ // The status stays 202 — this route has no body schema (ADR-003) and the
+ // HTTP contract must not change — but nothing is written.
+ const name = (request.body as { name?: unknown } | null | undefined)?.name;
+ if (typeof name !== 'string' || name.trim() === '') {
+ return { status: 'ok' };
+ }
+
// The cast is V1's own call (event.controller.ts:26): `/event` has no body
// schema, so `getTimestamp` reads `properties` off whatever arrived.
const { timestamp, isTimestampFromThePast } = getTimestamp(
diff --git a/packages/core/src/modules/ingest/src/producer-tuning.test.ts b/packages/core/src/modules/ingest/src/producer-tuning.test.ts
index f60d70bfb..6c316188c 100644
--- a/packages/core/src/modules/ingest/src/producer-tuning.test.ts
+++ b/packages/core/src/modules/ingest/src/producer-tuning.test.ts
@@ -23,6 +23,7 @@ const kafkaConfig = (tuning: Partial<KafkaConfig> = {}): KafkaConfig =>
eventsTopic: 'events',
eventsDlqTopic: 'events-dlq',
consumerGroup: 'openpanel-events',
+ maxMessageBytes: 1_048_576,
partitionsConcurrent: 8,
minMessages: 1,
maxWaitMs: 500,
diff --git a/packages/core/src/modules/profile/profile.routes.test.ts b/packages/core/src/modules/profile/profile.routes.test.ts
index 0dad98665..66249daa2 100644
--- a/packages/core/src/modules/profile/profile.routes.test.ts
+++ b/packages/core/src/modules/profile/profile.routes.test.ts
@@ -155,11 +155,66 @@ test('a client without a project answers 400 before touching the service', async
expect(identifyProfile).not.toHaveBeenCalled();
});
+// These bodies used to reach the service and throw on `input.property.split`,
+// so the caller saw a 500 with a raw TypeError (ISSUES.md H5). Elysia's schema
+// refuses them instead.
+//
+// This suite mounts the routes alone, so it sees Elysia's own VALIDATION
+// status. The assembled app maps that to 400 — see http/errors.ts, which
+// documents the mapping — and the live API was verified answering
+// `400 {"message":"body/property Invalid input: expected string, received
+// undefined"}` for these same bodies.
+const ELYSIA_VALIDATION_STATUS = 422;
+
+test('the adjust routes reject a body that cannot name a property', async () => {
+ clientResult = { ok: true, client: CLIENT };
+
+ for (const path of ['/profile/increment', '/profile/decrement']) {
+ for (const body of [
+ {},
+ { profileId: 'prof-1' },
+ { profileId: 'prof-1', property: 'score' },
+ { profileId: 'prof-1', property: '', value: 1 },
+ ]) {
+ const response = await post(path, body);
+ expect([path, response.status]).toEqual([path, ELYSIA_VALIDATION_STATUS]);
+ }
+ }
+
+ expect(adjustProfileProperty).not.toHaveBeenCalled();
+});
+
+// A negative delta is what separates this route's schema from `/track`'s
+// `zIncrementPayload`, which requires a positive value.
+test('increment still accepts a negative delta', async () => {
+ clientResult = { ok: true, client: CLIENT };
+
+ const response = await post('/profile/increment', {
+ profileId: 'prof-1',
+ property: 'score',
+ value: -2,
+ });
+
+ expect(response.status).not.toBe(400);
+ expect(adjustProfileProperty).toHaveBeenCalled();
+});
+
+// Each path gets a body its schema accepts, so the 401 under test is the auth
+// check and not the body validation. Elysia validates the body before the
+// clientAuth hook — `/track` has behaved that way since it got
+// `zTrackHandlerPayload`, so an anonymous caller with a malformed body sees a
+// 400 there too.
+const VALID_BODY_FOR: Record<string, Record<string, unknown>> = {
+ '/profile': { profileId: 'prof-1' },
+ '/profile/increment': { profileId: 'prof-1', property: 'score', value: 1 },
+ '/profile/decrement': { profileId: 'prof-1', property: 'score', value: 1 },
+};
+
test('every /profile route requires client credentials', async () => {
clientResult = { ok: false, ingest: true, message: 'Missing client id' };
- for (const path of ['/profile', '/profile/increment', '/profile/decrement']) {
- const response = await post(path, { profileId: 'prof-1' });
+ for (const [path, body] of Object.entries(VALID_BODY_FOR)) {
+ const response = await post(path, body);
expect(response.status).toBe(401);
}
expect(authenticateClient).toHaveBeenCalledTimes(3);
diff --git a/packages/core/src/modules/profile/profile.routes.ts b/packages/core/src/modules/profile/profile.routes.ts
index 51c5be609..de9c18c69 100644
--- a/packages/core/src/modules/profile/profile.routes.ts
+++ b/packages/core/src/modules/profile/profile.routes.ts
@@ -15,8 +15,10 @@
// behaviour change"), so the bodies are typed but not shape-validated.
import { parseUserAgent } from '@openpanel/shared/server';
+import { z } from 'zod';
import { getGeoLocation } from '../../clients/geo';
import { defineRoutes } from '../../http/define';
+import { zProfileId } from '../ingest/ingest.constants';
import { validateIngestRequest } from '../ingest/src/client-auth';
import {
type AdjustProfilePropertyResult,
@@ -27,14 +29,22 @@ import {
const TAGS = ['Profile'];
-interface AdjustPropertyBody {
- profileId: string;
- property: string;
- value: number;
-}
-
type StatusFn = (code: 400 | 404, body: string) => unknown;
+/**
+ * Deliberately NOT `zIncrementPayload` from ingest.constants: that schema is
+ * `/track`'s, where `value` is optional and must be positive. This route has
+ * always accepted a negative delta and has always needed a value — without
+ * one the service writes NaN. So the schema below is exactly as permissive as
+ * the paths that work today, and rejects only the bodies that reached
+ * `input.property.split` and answered 500.
+ */
+const zAdjustProperty = z.object({
+ profileId: zProfileId,
+ property: z.string().min(1),
+ value: z.number(),
+});
+
function respondAdjusted(
result: AdjustProfilePropertyResult,
status: StatusFn,
@@ -58,6 +68,9 @@ export const profileRoutes = defineRoutes((app) =>
if (!client.projectId) {
return status(400, 'No projectId');
}
+ if (body === null || body === undefined) {
+ return status(400, 'Missing body');
+ }
const payload = body as IdentifyProfileInput;
const userAgent = parseUserAgent(
ctx.headers.get('user-agent'),
@@ -85,9 +98,9 @@ export const profileRoutes = defineRoutes((app) =>
if (!client.projectId) {
return status(400, 'No projectId');
}
- const { profileId, property, value } = body as AdjustPropertyBody;
+ const { profileId, property, value } = body;
const result = await adjustProfileProperty(ctx, client.projectId, {
- profileId,
+ profileId: String(profileId),
property,
delta: value,
});
@@ -95,6 +108,7 @@ export const profileRoutes = defineRoutes((app) =>
},
{
clientAuth: { ingest: validateIngestRequest },
+ body: zAdjustProperty,
detail: {
tags: TAGS,
description: 'Increment a numeric property on a user profile.',
@@ -107,9 +121,9 @@ export const profileRoutes = defineRoutes((app) =>
if (!client.projectId) {
return status(400, 'No projectId');
}
- const { profileId, property, value } = body as AdjustPropertyBody;
+ const { profileId, property, value } = body;
const result = await adjustProfileProperty(ctx, client.projectId, {
- profileId,
+ profileId: String(profileId),
property,
delta: -value,
});
@@ -117,6 +131,7 @@ export const profileRoutes = defineRoutes((app) =>
},
{
clientAuth: { ingest: validateIngestRequest },
+ body: zAdjustProperty,
detail: {
tags: TAGS,
description: 'Decrement a numeric property on a user profile.',
diff --git a/packages/core/test/config-fixture.ts b/packages/core/test/config-fixture.ts
index 1475185ce..06bf99608 100644
--- a/packages/core/test/config-fixture.ts
+++ b/packages/core/test/config-fixture.ts
@@ -69,6 +69,7 @@ export function testCoreConfig(
eventsTopic: 'events',
eventsDlqTopic: 'events-dlq',
consumerGroup: 'openpanel-events',
+ maxMessageBytes: 1_048_576,
partitionsConcurrent: 8,
minMessages: 1,
maxWaitMs: 500,