🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/**
* Logical events routing. Runs right after the CTE name check in every
* compile.
*
* Call sites read the logical `events` relation; this pass replaces each
* FROM/JOIN reference to it with `events_core` or `events_full`, one choice
* per relation. A relation goes to `events_full` only when a column reference
* resolves to it and names a column in {@link EVENTS_CORE_TRUNCATED_COLUMNS},
* outside the metadata predicates `metadataFilterIsEventsCoreSafe` accepts:
*
* e.metadata[k] = 'short' → events_core (< 200 code points)
* startsWith(e.metadata[k], 'p') → events_core (≤ 200 code points)
* e.input = 'x', select e.input → events_full
*
* A qualifier resolves to the innermost FROM/JOIN alias (or unaliased table
* name) of that name. Shapes that cannot be resolved to one relation are rejected
* rather than routed to `events_full`, so `events_full` is read only where a
* query needs it:
* - `*`, `e.*` or `COLUMNS(…)` over a logical relation (`count(*)` reads no
* column);
* - an unqualified truncated column while a logical relation is in scope;
* - raw SQL text or identifiers (`sql` templates, `sql.raw`, `sql.id`,
* `sql.table`) in or under a select that reads a logical relation.
* `sql.ref` and bound values resolve like builder references.
*
* Callers that want the truncated copy on purpose read `events_truncated`,
* which always routes to `events_core`, whatever columns it reads.
*
* Only this pass names the physical tables. A call site that does — as a
* FROM/JOIN table, an alias, a column qualifier, or in raw SQL — is
* rejected: ClickHouse resolves a qualifier by table name even when the
* table is aliased, so a physical name could reach a routed relation.
*/
import {
AggregateFunctionNode,
AliasNode,
BinaryOperationNode,
ColumnNode,
FromNode,
FunctionNode,
IdentifierNode,
JoinNode,
OperatorNode,
RawNode,
SelectAllNode,
SelectionNode,
TableNode,
ValueNode,
type KyselyPlugin,
type OperationNode,
type OrderByItemNode,
type PluginTransformQueryArgs,
type PluginTransformResultArgs,
type QueryId,
ReferenceNode,
type RootOperationNode,
type SelectQueryNode,
} from "kysely";
import { metadataFilterIsEventsCoreSafe } from "../queries/clickhouse-sql/events-core-truncation";
import { QueryCompileError } from "./errors";
import { ArrayIndexNode, FinalTableNode, walkNodes } from "./nodes";
import { describeRelation, selectRelations, type Relation } from "./relations";
import {
EVENTS_CORE_TRUNCATED_COLUMNS,
LOGICAL_EVENTS_TABLES,
PHYSICAL_EVENTS_TABLES,
} from "./schema";
import { ClickHouseOperationNodeTransformer } from "./transformer";
const LOGICAL_EVENTS = "events";
const PHYSICAL_EVENT_NAME = /\bevents_(core|full)\b/;
export class EventsTableRoutingPlugin implements KyselyPlugin {
transformQuery(args: PluginTransformQueryArgs): RootOperationNode {
// Builders share unchanged child nodes, so one relation node can sit at
// two places in the tree. A plain transform rebuilds every node, giving
// each place its own node to key routing on.
const tree = new ClickHouseOperationNodeTransformer().transformNode(
args.node,
);
const analysis = new EventsReadAnalysis();
analysis.transformNode(tree);
if (!analysis.readsLogicalEvents) return args.node;
return new EventsTableRouter(analysis.fullRelations).transformNode(
tree,
) as RootOperationNode;
}
async transformResult(args: PluginTransformResultArgs) {
return args.result;
}
}
/**
* Read-only walk that resolves column references to relations; the
* transformer is used for its traversal only.
*/
class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {
readsLogicalEvents = false;
/** Logical relations that read a truncated column. */
readonly fullRelations = new Set<OperationNode>();
/** Relations of each enclosing select, innermost last. */
private readonly scopes: RoutingRelation[][] = [];
protected override transformNodeImpl<T extends OperationNode>(
node: T,
queryId?: QueryId,
): T {
const lookup = coreSafeMetadataLookup(node);
if (lookup) {
// The lookup's columns read no truncated value, but their qualifiers
// must still not name a physical table.
rejectPhysicalQualifiers(lookup.array);
this.transformNode(lookup.index, queryId);
return node;
}
return super.transformNodeImpl(node, queryId);
}
protected override transformSelectQuery(
node: SelectQueryNode,
queryId?: QueryId,
): SelectQueryNode {
const described = selectRelations(node);
described.forEach(rejectPhysicalRelation);
const relations = described.map(routingRelation);
if (relations.some((relation) => relation.routed)) {
this.readsLogicalEvents = true;
}
this.scopes.push(relations);
try {
return super.transformSelectQuery(node, queryId);
} finally {
this.scopes.pop();
}
}
protected override transformReference(
node: ReferenceNode,
_queryId?: QueryId,
): ReferenceNode {
const qualifier = node.table?.table.identifier.name;
rejectPhysicalName(qualifier);
// `sql.ref("e.*")` parses the star as a column named `*`.
if (SelectAllNode.is(node.column) || node.column.column.name === "*") {
if (qualifier === undefined) this.rejectStar("*");
else if (this.resolveQualified(qualifier)?.logical) {
throw starError(`${qualifier}.*`);
}
} else {
this.readColumn(node.column.column.name, qualifier);
}
return node;
}
/** Reached only outside a reference: `selectAll()`, `*` in raw SQL. */
protected override transformSelectAll(node: SelectAllNode): SelectAllNode {
this.rejectStar("*");
return node;
}
/** Reached only outside a reference. */
protected override transformColumn(node: ColumnNode): ColumnNode {
if (node.column.name === "*") this.rejectStar("*");
else this.readColumn(node.column.name, undefined);
return node;
}
/**
* Rejects `columns` like any function ({@link rejectColumnsMatcher}). `count(*)`
* reads no column; its clauses are walked as usual.
*/
protected override transformAggregateFunction(
node: AggregateFunctionNode,
queryId?: QueryId,
): AggregateFunctionNode {
this.rejectColumnsMatcher(node.func);
if (isCountStar(node)) {
super.transformAggregateFunction({ ...node, aggregated: [] }, queryId);
return node;
}
return super.transformAggregateFunction(node, queryId);
}
/**
* The direction is raw `asc` / `desc`, so it is not walked: the
* `no-deprecated` lint rule keeps `orderBy` from taking an expression.
*/
protected override transformOrderByItem(
node: OrderByItemNode,
queryId?: QueryId,
): OrderByItemNode {
this.transformNode(node.orderBy, queryId);
return node;
}
protected override transformFunction(
node: FunctionNode,
queryId?: QueryId,
): FunctionNode {
this.rejectColumnsMatcher(node.func);
return super.transformFunction(node, queryId);
}
protected override transformRaw(node: RawNode, queryId?: QueryId): RawNode {
rejectPhysicalRawNames(node);
if (
this.readsLogicalInScope() &&
(node.sqlFragments.some((fragment) => fragment.trim() !== "") ||
node.parameters.some((p) => IdentifierNode.is(p) || TableNode.is(p)))
) {
throw new QueryCompileError(
`Raw SQL text or identifiers cannot appear in a select that reads the logical "${LOGICAL_EVENTS}" relation: they may read a truncated column the routing pass cannot see. Use builder expressions, sql.ref() or bound values.`,
);
}
return super.transformRaw(node, queryId);
}
private readColumn(column: string, qualifier: string | undefined): void {
if (!EVENTS_CORE_TRUNCATED_COLUMNS.has(column)) return;
const relation =
qualifier === undefined
? this.resolveUnqualified(column)
: this.resolveQualified(qualifier);
if (relation?.logical) this.fullRelations.add(relation.node);
}
private resolveQualified(qualifier: string): RoutingRelation | undefined {
for (const relations of this.scopes.toReversed()) {
const match = relations.find((relation) => relation.name === qualifier);
if (match) return match;
}
return undefined;
}
private resolveUnqualified(column: string): RoutingRelation | undefined {
if (this.readsLogicalInScope()) {
throw new QueryCompileError(
`Qualify "${column}": it is truncated in events_core, so the routing pass needs to know which relation reads it while the logical "${LOGICAL_EVENTS}" relation is in scope.`,
);
}
return undefined;
}
/** `columns(…)` expands like `*`. */
private rejectColumnsMatcher(func: string): void {
if (func.toLowerCase() === "columns") this.rejectStar(`${func}(…)`);
}
/** Whether the current select or an enclosing one reads a logical relation. */
private readsLogicalInScope(): boolean {
return this.scopes.some((relations) => relations.some((r) => r.logical));
}
private rejectStar(star: string): void {
if (this.scopes.at(-1)?.some((relation) => relation.logical)) {
throw starError(star);
}
}
}
function rejectPhysicalName(name: string | undefined): void {
if (name === undefined || !PHYSICAL_EVENTS_TABLES.has(name)) return;
throw new QueryCompileError(
`Read the logical "${LOGICAL_EVENTS}" relation instead of "${name}": the routing pass picks events_core or events_full from the columns read. Read "events_truncated" to get the truncated events_core copy on purpose.`,
);
}
/** Column qualifiers anywhere under `node`. */
function rejectPhysicalQualifiers(node: OperationNode): void {
walkNodes(node, (child) => {
if (ReferenceNode.is(child)) {
rejectPhysicalName(child.table?.table.identifier.name);
}
});
}
/** Raw SQL text, and `sql.table` / `sql.id` parameters. */
function rejectPhysicalRawNames(node: RawNode): void {
for (const fragment of node.sqlFragments) {
rejectPhysicalName(PHYSICAL_EVENT_NAME.exec(fragment)?.[0]);
}
for (const parameter of node.parameters) {
rejectPhysicalName(rawParameterName(parameter));
}
}
/** The name a `sql.table` / `sql.id` parameter writes into raw SQL. */
function rawParameterName(node: OperationNode): string | undefined {
if (TableNode.is(node)) return node.table.identifier.name;
if (IdentifierNode.is(node)) return node.name;
return undefined;
}
/** A physical events table as FROM/JOIN table or alias. */
function rejectPhysicalRelation({ table, alias }: Relation): void {
rejectPhysicalName(table?.table.identifier.name);
rejectPhysicalName(alias);
}
function starError(star: string): QueryCompileError {
return new QueryCompileError(
`${star} cannot expand the logical "${LOGICAL_EVENTS}" relation: list the columns, so the routing pass can tell whether it reads a truncated one.`,
);
}
class EventsTableRouter extends ClickHouseOperationNodeTransformer {
constructor(private readonly fullRelations: ReadonlySet<OperationNode>) {
super();
}
protected override transformFrom(
node: FromNode,
queryId?: QueryId,
): FromNode {
return FromNode.create(
node.froms.map((from) => this.routeRelation(from, queryId)),
);
}
protected override transformJoin(
node: JoinNode,
queryId?: QueryId,
): JoinNode {
return {
...super.transformJoin(node, queryId),
table: this.routeRelation(node.table, queryId),
};
}
/**
* `events as e` → `events_core as e`. An unaliased relation keeps its
* logical name as the alias (`events_core as events`), so `events.col`
* references and the qualifiers later passes add still resolve.
*/
private routeRelation(node: OperationNode, queryId?: QueryId): OperationNode {
const table = TableNode.create(
this.fullRelations.has(node) ? "events_full" : "events_core",
);
const { source, alias } = describeRelation(node);
const logicalName = logicalEventsTableName(source);
if (logicalName === undefined) {
return this.transformNode(node, queryId);
}
if (FinalTableNode.is(node)) {
throw new QueryCompileError(
`FINAL cannot wrap the logical "${logicalName}" relation: events_core and events_full are immutable at read time. Drop the FINAL.`,
);
}
return AliasNode.create(table, IdentifierNode.create(alias ?? logicalName));
}
}
/**
* Named by its alias, else by its table name: `events as e` → `e`,
* `events` → `events`. Schema-qualified names never get here (`ctes.ts`).
*/
function routingRelation({
node,
source,
table,
alias,
}: Relation): RoutingRelation {
const logicalName = logicalEventsTableName(source);
return {
node,
name: alias ?? table?.table.identifier.name,
logical: logicalName === LOGICAL_EVENTS,
routed: logicalName !== undefined,
};
}
/** `events` or `events_truncated`, when `node` reads one of them. */
function logicalEventsTableName(node: OperationNode): string | undefined {
if (!TableNode.is(node)) return undefined;
const name = node.table.identifier.name;
return LOGICAL_EVENTS_TABLES.has(name) ? name : undefined;
}
/** `count(*)` with any `distinct`, `filter`, `over`, … clause. */
function isCountStar(node: AggregateFunctionNode): boolean {
if (node.func !== "count" || node.aggregated.length !== 1) return false;
const [arg] = node.aggregated;
return (
SelectAllNode.is(arg) ||
(SelectionNode.is(arg) && SelectAllNode.is(arg.selection))
);
}
/**
* The metadata lookup of `metadataValue(…, k) = v` or
* `startsWith(metadataValue(…, k), v)`, for a bound `v` that
* `metadataFilterIsEventsCoreSafe` accepts.
*/
function coreSafeMetadataLookup(
node: OperationNode,
): ArrayIndexNode | undefined {
if (
BinaryOperationNode.is(node) &&
OperatorNode.is(node.operator) &&
node.operator.operator === "="
) {
return coreSafe(node.leftOperand, "=", node.rightOperand);
}
if (
FunctionNode.is(node) &&
node.func === "startsWith" &&
node.arguments.length === 2
) {
return coreSafe(node.arguments[0], "starts with", node.arguments[1]);
}
return undefined;
}
// `lookup` is typed `{ kind: string }`: `ArrayIndexNode` is outside Kysely's
// closed `OperationNode` union, so narrowing an `OperationNode` yields `never`.
function coreSafe(
lookup: { kind: string },
operator: "=" | "starts with",
value: OperationNode,
): ArrayIndexNode | undefined {
return ArrayIndexNode.is(lookup) &&
lookup.metadataLookup &&
ValueNode.is(value) &&
metadataFilterIsEventsCoreSafe(operator, value.value)
? lookup
: undefined;
}
/** A FROM/JOIN source of one select. */
type RoutingRelation = {
/** The FROM item or join table, as the router meets it. */
node: OperationNode;
/** Alias, or the table name of an unaliased table. */
name?: string;
/** Reads `events`, so its truncated-column reads pick the table. */
logical: boolean;
/** Reads `events` or `events_truncated`, so the router replaces it. */
routed: boolean;
};