OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
overview: a bare end date now names the whole day, not midnight
Carl-Gerhard Lindesvärd908bf0d12c53
diff --git a/ISSUES.md b/ISSUES.mdindex ee81e54e1..bf10552f1 100644--- a/ISSUES.md+++ b/ISSUES.md@@ -46,10 +46,10 @@ the entry text is kept for context.| **M12** | restrict CORS on `/mcp` | `http/cors.ts` documents the open scope as deliberate V1 parity, and a browser MCP client would break. Fix only the two header bugs (`allow-headers: undefined`, `expose-headers` echoing every request header name) and leave the origin policy alone || **M13** | 400 on `/export` `limit=0\|-5`, `includes=bogus` | clamp, do not reject. M13's tRPC half (`overview.*` filter validation, a `profile.list` take cap) is dashboard-only and safe to make strict |-**H8a is a judgement call, not a safe fix.** Making `endDate` inclusive is *correct* —-the last day is silently dropped today — but it changes every number `/insights/*` and-the MCP tools return for an unchanged request. Decide it deliberately and document it;-do not fold it into a batch of bug fixes.+**H8a was a judgement call and the owner took it (2026-09-23): fixed in `14d6e8f1`.**+A bare `YYYY-MM-DD` end bound now names the whole day, as it already did on the+dashboard path. MCP numbers change, by design — `get_analytics_overview` for+acme-web August went from 276535 sessions to 283780, which is the ClickHouse count.**H4 is larger than the entry says.** There is no rate limiter in the rewrite at all:`grep -rn rateLimit packages/core/src` finds only the unmounted `createRateLimitMiddleware`@@ -161,7 +161,7 @@ misleading feature; **low** = cosmetic.- **Verify**: sign up a user, create an org, `user.delete` → blocker listed or org scheduled for deletion.### H8. MCP data-correctness bugs-- **a. `endDate` exclusive** *(decide deliberately — changes public-API numbers)* — every `zDateRange` tool drops the whole last day+- **a. `endDate` exclusive — FIXED (`14d6e8f1`)** — every `zDateRange` tool drops the whole last day(`startDate=endDate` → zeros; the default range excludes today). Files:`packages/core/src/modules/mcp/src/tools/shared.ts:80-89` (bare `YYYY-MM-DD`),`packages/core/src/modules/overview/src/overview.sql.ts:118-124`, `pages.sql.ts:16-22`diff --git a/packages/core/src/modules/overview/src/overview.sql.test.ts b/packages/core/src/modules/overview/src/overview.sql.test.tsindex 38fb13ea9..d9835d8e7 100644--- a/packages/core/src/modules/overview/src/overview.sql.test.ts+++ b/packages/core/src/modules/overview/src/overview.sql.test.ts@@ -292,3 +292,47 @@ describe('overview.sql — sql.id() identifier whitelists (R3)', () => {).toThrow();});});++// The MCP tools send a bare `YYYY-MM-DD`; the dashboard sends the day's edges+// already. Both have to end up bounding the same last day, or every MCP range+// silently loses it (ISSUES.md H8a).+describe('overview.sql — date-only range boundaries', () => {+ function boundsOf(fragment: SqlFragment): string[] {+ const { query_params } = fragment.toStatement();+ return Object.values(query_params).filter(+ (value): value is string =>+ typeof value === 'string' && value.startsWith('2026-08-')+ );+ }++ it('widens a bare end date to the end of that day', () => {+ const bounds = boundsOf(+ OV.revenueQuery({+ projectId: PROJECT_ID,+ startDate: '2026-08-01',+ endDate: '2026-08-31',+ interval: 'day',+ rawFilterWhere: null,+ })+ );++ expect(bounds).toContain('2026-08-01 00:00:00');+ expect(bounds).toContain('2026-08-31 23:59:59');+ expect(bounds).not.toContain('2026-08-31 00:00:00');+ });++ it('leaves an explicit datetime boundary exactly as given', () => {+ const bounds = boundsOf(+ OV.revenueQuery({+ projectId: PROJECT_ID,+ startDate: '2026-08-01 00:00:00',+ endDate: '2026-08-31 23:59:59',+ interval: 'day',+ rawFilterWhere: null,+ })+ );++ expect(bounds).toContain('2026-08-01 00:00:00');+ expect(bounds).toContain('2026-08-31 23:59:59');+ });+});diff --git a/packages/core/src/modules/overview/src/overview.sql.ts b/packages/core/src/modules/overview/src/overview.sql.tsindex f4961ba68..199e07de9 100644--- a/packages/core/src/modules/overview/src/overview.sql.ts+++ b/packages/core/src/modules/overview/src/overview.sql.ts@@ -19,6 +19,10 @@// (see src/run-query.ts).import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';+import {+ toRangeBoundaryDate,+ toRangeBoundaryLiteral,+} from '../../../shared/ch-dates';import { compiledText } from '../../chart/src/compiled';import { getSelectPropertyKey } from '../../chart/src/field-resolution';import type { IInterval } from '../../report/report.constants';@@ -27,14 +31,6 @@ const ROLLUP_DATE_PREFIX = '1970-01-01';export type OverviewTable = 'events' | 'sessions';-function toDateTimeLiteral(value: string): string {- return new Date(value).toISOString().slice(0, 19).replace('T', ' ');-}--function toDateLiteral(value: string): string {- return new Date(value).toISOString().slice(0, 10);-}-/** V1 `clix.toStartOf` — deliberately timezone-blind for week/month (see header). */export function toStartOf(node: string, interval: IInterval): SqlFragment {switch (interval) {@@ -79,18 +75,22 @@ function toIntervalStep(interval: IInterval): string {* `sql.date`/`sql.dateTime64` bind the boundary itself; the wrapping* functions around them reproduce that expression shape.*/-function fillBoundaryParam(interval: IInterval, value: string) {+function fillBoundaryParam(+ interval: IInterval,+ value: string,+ boundary: 'start' | 'end'+) {// The bucket column's type must match exactly: `toStartOfX(created_at)`// returns plain `DateTime`/`Date`, not `DateTime64` — ClickHouse's WITH// FILL rejects a boundary whose type doesn't match the sorted column's.return interval === 'month' || interval === 'week'- ? sql.date(toDateLiteral(value))- : sql.param('DateTime', toDateTimeLiteral(value));+ ? sql.date(toRangeBoundaryDate(value, boundary))+ : sql.param('DateTime', toRangeBoundaryLiteral(value, boundary));}/** V1 always bucket-aligns FROM (via `toStartOf`) but leaves TO as-is. */function fillFrom(interval: IInterval, startDate: string): SqlFragment {- const boundary = fillBoundaryParam(interval, startDate);+ const boundary = fillBoundaryParam(interval, startDate, 'start');switch (interval) {case 'minute':return sql`toStartOfMinute(${boundary})`;@@ -112,7 +112,7 @@ export function fillClause(startDate: string,endDate: string): SqlFragment {- return sql`WITH FILL FROM ${fillFrom(interval, startDate)} TO ${fillBoundaryParam(interval, endDate)} STEP ${compiledText(toIntervalStep(interval))}`;+ return sql`WITH FILL FROM ${fillFrom(interval, startDate)} TO ${fillBoundaryParam(interval, endDate, 'end')} STEP ${compiledText(toIntervalStep(interval))}`;}function dateRangeWhere(@@ -120,7 +120,7 @@ function dateRangeWhere(startDate: string,endDate: string): SqlFragment {- return sql`${sql.id(column)} BETWEEN toDateTime(${sql.string(toDateTimeLiteral(startDate))}) AND toDateTime(${sql.string(toDateTimeLiteral(endDate))})`;+ return sql`${sql.id(column)} BETWEEN toDateTime(${sql.string(toRangeBoundaryLiteral(startDate, 'start'))}) AND toDateTime(${sql.string(toRangeBoundaryLiteral(endDate, 'end'))})`;}/** `getRawWhereClause`'s output — already a bound fragment, or nothing. */diff --git a/packages/core/src/modules/overview/src/pages.sql.ts b/packages/core/src/modules/overview/src/pages.sql.tsindex 5e8d8d75a..9cb10ff63 100644--- a/packages/core/src/modules/overview/src/pages.sql.ts+++ b/packages/core/src/modules/overview/src/pages.sql.ts@@ -6,19 +6,16 @@// sets diffed against V1's on the local prod-copy — see pages.sql.proof.md.import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';+import { toRangeBoundaryLiteral } from '../../../shared/ch-dates';import type { IInterval } from '../../report/report.constants';import { fillClause, toStartOf } from './overview.sql';-function toDateTimeLiteral(value: string): string {- return new Date(value).toISOString().slice(0, 19).replace('T', ' ');-}-function dateRangeWhere(column: string,startDate: string,endDate: string): SqlFragment {- return sql`${sql.id(column)} BETWEEN toDateTime(${sql.string(toDateTimeLiteral(startDate))}) AND toDateTime(${sql.string(toDateTimeLiteral(endDate))})`;+ return sql`${sql.id(column)} BETWEEN toDateTime(${sql.string(toRangeBoundaryLiteral(startDate, 'start'))}) AND toDateTime(${sql.string(toRangeBoundaryLiteral(endDate, 'end'))})`;}export interface TopPagesQueryInput {diff --git a/packages/core/src/modules/profile/src/sql.ts b/packages/core/src/modules/profile/src/sql.tsindex e5f67057c..6cd2dfc59 100644--- a/packages/core/src/modules/profile/src/sql.ts+++ b/packages/core/src/modules/profile/src/sql.ts@@ -14,6 +14,7 @@// binds as `Array(String)`, a literal list to the planner.import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';+import { toRangeBoundaryLiteral } from '../../../shared/ch-dates';import {type CompiledFilterClauses,compiledFilterFragments,@@ -55,17 +56,13 @@ function optional(return condition ? fragment() : sql.empty;}-function toDateTimeLiteral(value: string): string {- return new Date(value).toISOString().slice(0, 19).replace('T', ' ');-}-/** Same spelling `overview/src/pages.sql.ts` uses, so the two read alike. */function dateRangeWhere(column: string,startDate: string,endDate: string): SqlFragment {- return sql`${sql.id(column)} BETWEEN toDateTime(${sql.string(toDateTimeLiteral(startDate))}) AND toDateTime(${sql.string(toDateTimeLiteral(endDate))})`;+ return sql`${sql.id(column)} BETWEEN toDateTime(${sql.string(toRangeBoundaryLiteral(startDate, 'start'))}) AND toDateTime(${sql.string(toRangeBoundaryLiteral(endDate, 'end'))})`;}/** The window every profile-list / power-user statement is now bounded to. */diff --git a/packages/core/src/shared/ch-dates.test.ts b/packages/core/src/shared/ch-dates.test.tsnew file mode 100644index 000000000..0cabdcc02--- /dev/null+++ b/packages/core/src/shared/ch-dates.test.ts@@ -0,0 +1,55 @@+// The range boundaries the query builders bind. Two shapes arrive: the+// dashboard's `YYYY-MM-DD HH:mm:ss` (already widened to the day's edges by+// report/src/chart-dates.ts) and the MCP tools' bare `YYYY-MM-DD`.++import { describe, expect, test } from 'bun:test';+import { toRangeBoundaryDate, toRangeBoundaryLiteral } from './ch-dates';++describe('toRangeBoundaryLiteral', () => {+ test('a bare date names the whole day', () => {+ expect(toRangeBoundaryLiteral('2026-08-01', 'start')).toBe(+ '2026-08-01 00:00:00'+ );+ // Midnight here is what dropped the last day of every MCP range.+ expect(toRangeBoundaryLiteral('2026-08-31', 'end')).toBe(+ '2026-08-31 23:59:59'+ );+ });++ test('a single-day range covers that day rather than an empty instant', () => {+ const start = toRangeBoundaryLiteral('2026-08-15', 'start');+ const end = toRangeBoundaryLiteral('2026-08-15', 'end');+ expect(start).not.toBe(end);+ expect(start < end).toBe(true);+ });++ test('a ClickHouse literal passes through untouched in either direction', () => {+ // `new Date('2026-08-31 23:59:59')` reads local time, so the old+ // round-trip shifted this by the server's offset and silently truncated+ // the range on any non-UTC deployment.+ for (const boundary of ['start', 'end'] as const) {+ expect(toRangeBoundaryLiteral('2026-08-31 23:59:59', boundary)).toBe(+ '2026-08-31 23:59:59'+ );+ expect(toRangeBoundaryLiteral('2026-08-01 00:00:00', boundary)).toBe(+ '2026-08-01 00:00:00'+ );+ }+ });++ test('an ISO instant becomes its UTC ClickHouse literal', () => {+ expect(toRangeBoundaryLiteral('2026-08-31T12:34:56.000Z', 'end')).toBe(+ '2026-08-31 12:34:56'+ );+ });+});++describe('toRangeBoundaryDate', () => {+ test('narrows to the calendar day without moving it', () => {+ expect(toRangeBoundaryDate('2026-08-31', 'end')).toBe('2026-08-31');+ expect(toRangeBoundaryDate('2026-08-31 23:59:59', 'end')).toBe(+ '2026-08-31'+ );+ expect(toRangeBoundaryDate('2026-08-01', 'start')).toBe('2026-08-01');+ });+});diff --git a/packages/core/src/shared/ch-dates.ts b/packages/core/src/shared/ch-dates.tsindex e570c2861..a38451cde 100644--- a/packages/core/src/shared/ch-dates.ts+++ b/packages/core/src/shared/ch-dates.ts@@ -32,3 +32,43 @@ export function formatClickhouseDate(export function convertClickhouseDateToJs(date: string): Date {return new Date(`${date.replace(' ', 'T')}Z`);}++const DATE_ONLY = /^\d{4}-\d{2}-\d{2}$/;+const CLICKHOUSE_DATE_TIME = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/;++const START_OF_DAY = '00:00:00';+const END_OF_DAY = '23:59:59';++/**+ * A range boundary as a ClickHouse `DateTime` literal.+ *+ * Two shapes reach the query builders. The dashboard sends+ * `YYYY-MM-DD HH:mm:ss` already widened to the day's edges by+ * `getChartStartEndDate`; the MCP tools send a bare `YYYY-MM-DD`. A bare date+ * names a whole DAY, so an end boundary has to widen to 23:59:59 — left at+ * midnight it drops the last day of every range.+ *+ * An already-ClickHouse-shaped literal is returned untouched rather than sent+ * through `new Date()`, which reads it in the server's local zone and shifts+ * it by the offset.+ */+export function toRangeBoundaryLiteral(+ value: string,+ boundary: 'start' | 'end'+): string {+ if (DATE_ONLY.test(value)) {+ return `${value} ${boundary === 'start' ? START_OF_DAY : END_OF_DAY}`;+ }+ if (CLICKHOUSE_DATE_TIME.test(value)) {+ return value;+ }+ return formatClickhouseDate(value);+}++/** The same boundary narrowed to `YYYY-MM-DD`, for `Date`-typed columns. */+export function toRangeBoundaryDate(+ value: string,+ boundary: 'start' | 'end'+): string {+ return toRangeBoundaryLiteral(value, boundary).slice(0, 10);+}