:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
refactor(hog): check answers through a schema interface hog owns (#130241)
Rafael Audibert3574f1e87a67
diff --git a/hog/AGENTS.md b/hog/AGENTS.mdindex 84b72cad9abe..533327b89db8 100644--- a/hog/AGENTS.md+++ b/hog/AGENTS.md@@ -72,7 +72,7 @@ export default defineCommand({- A failure throws a `HogError` with a stable `code` and a `hint` that names the next step. `cli.ts` prints it as text or as JSON on stderr. Do not catch errors to print them yourself.- Ask questions only through `ctx.prompts`. It is undefined for agents, pipes and `-o json`, so a command that needs an answer must take it as an option there, or fail with a hint that names the option. Never block an agent on a person: split the work into a call that returns at once and a call that finishes it, like `hog auth login` and `hog auth login --wait`. A command that waits for a person to approve something in a browser uses `runApproval` and `resumeApproval` from `src/core/approval.ts`, so every such command starts, waits and resumes the same way.- `exitCode` is for results that are not failures but that a script must branch on, like `hog query check` finding errors.-- `--raw` is global: it prints the result of `run` as JSON and skips both views. Do not add per-command `--raw` or `--debug` flags; return enough data from `run` instead.+- `--raw` is global: it prints the result of `run` as JSON and skips both views. A command whose result is a trimmed view of a PostHog answer, like `hog query run`, keeps the whole answer in it and returns that from `raw`. Do not add per-command `--raw` or `--debug` flags; return enough data from `run` instead.- An argument with `stdin: true` reads stdin when the caller passes `-`, and fails when it is empty. `hog commands` tells agents which arguments accept it.- Data that enters hog, such as flags, env vars, files, keychain items and answers from PostHog, is parsed into a typed value at the boundary with a zod schema. It is not checked and then passed on as a plain string. A branded value, such as a `Host`, a `ProjectId` or a token kind, comes only from its factory, such as `parseHost` or `parseProjectId`. Semgrep blocks `JSON.parse` outside `src/core/json.ts` and a cast to a brand outside the module that defines it (`.semgrep/rules/devex/hog-typed-boundaries.yaml`).- Commands reach the outside world only through `ctx` (`ctx.posthog()`, `ctx.runtime`), so tests can fake it. Do not read `process` or call `fetch` directly.diff --git a/hog/src/cli.ts b/hog/src/cli.tsindex 549a8305aeae..7d38128b62b0 100644--- a/hog/src/cli.ts+++ b/hog/src/cli.ts@@ -224,7 +224,7 @@ function createProgram(runtime: Runtime, mode: Mode, state: RunState, telemetry:if (render && !globals.raw) {render({ result, out, args, options })} else if (result !== undefined) {- out.json(result)+ out.json(globals.raw && node.raw ? node.raw(result) : result)}state.exitCode = node.exitCode?.(result) ?? 0state.properties = {diff --git a/hog/src/commands/query/query.test.ts b/hog/src/commands/query/query.test.tsindex f13e303c6a71..ae6c8bbff68a 100644--- a/hog/src/commands/query/query.test.ts+++ b/hog/src/commands/query/query.test.ts@@ -10,6 +10,7 @@ const ROWS = {['signup', null],],hasMore: false,+ clickhouse: 'SELECT 1',}describe('hog query', () => {@@ -37,7 +38,7 @@ describe('hog query', () => {'--raw prints the full response',false,['--raw'],- '{"columns":["event","total"],"hasMore":false,"results":[["$pageview",70],["signup",null]]}\n',+ '{"clickhouse":"SELECT 1","columns":["event","total"],"hasMore":false,"results":[["$pageview",70],["signup",null]]}\n',],])('run: %s', async (_name, tty, flags, expected) => {const result = await runHog(['query', 'run', 'select 1', ...flags], {diff --git a/hog/src/commands/query/run.ts b/hog/src/commands/query/run.tsindex c1cd5ce7cd16..97b8eb4d1190 100644--- a/hog/src/commands/query/run.ts+++ b/hog/src/commands/query/run.ts@@ -5,10 +5,11 @@ import type { ColumnKind } from '@/core/output'import { plural } from '@/core/text'import { ApiError } from '@/posthog/api'+// HogQLQueryResponse declares `columns` and `types` optional.const QueryResponse = z.looseObject({columns: z.array(z.string()).nullish(),- results: z.array(z.array(z.unknown())).nullish(),types: z.array(z.tuple([z.string(), z.string()])).nullish(),+ results: z.array(z.array(z.unknown())),hasMore: z.boolean().nullish(),})@@ -40,37 +41,42 @@ export default defineCommand({async run({ args, ctx }) {const api = await ctx.posthog()+ let response: z.output<typeof QueryResponse>try {- return await api.post(- 'query',- { query: { kind: 'HogQLQuery', query: args.query }, refresh: 'blocking' },- QueryResponse- )+ const body = { query: { kind: 'HogQLQuery', query: args.query }, refresh: 'blocking' }+ response = await api.post('query', body, QueryResponse)} catch (error) {if (error instanceof ApiError && error.type === 'validation_error') {throw error.withHint('Run `hog query check` on the query to see where it fails.')}throw error}+ return {+ columns: (response.columns ?? []).map((name, index) => ({ name, type: response.types?.[index]?.[1] })),+ rows: response.results,+ hasMore: response.hasMore === true,+ response,+ }},+ raw: (result) => result.response,+human({ result, out }) {- const rows = result.results ?? []out.table(- (result.columns ?? []).map((name, index) => ({ name, kind: kindOf(result.types?.[index]?.[1]) })),- rows+ result.columns.map((column) => ({ name: column.name, kind: kindOf(column.type) })),+ result.rows)- const count = plural(rows.length, 'row')+ const count = plural(result.rows.length, 'row')out.line(out.style(result.hasMore ? `${count}, more available` : count, 'dim'))},agent({ result, out }) {- out.jsonLines(result.results ?? [])+ out.jsonLines(result.rows)},telemetry: ({ result }) => ({- row_count: result.results?.length ?? 0,- column_count: result.columns?.length ?? 0,- has_more: result.hasMore === true,+ row_count: result.rows.length,+ column_count: result.columns.length,+ has_more: result.hasMore,}),})diff --git a/hog/src/core/approval.ts b/hog/src/core/approval.tsindex 5b63beaad625..74b89f745948 100644--- a/hog/src/core/approval.ts+++ b/hog/src/core/approval.ts@@ -7,6 +7,7 @@ import { escapeControls, type Output } from './output'import { hogFile } from './paths'import type { Spinner } from './prompts'import type { Runtime } from './runtime'+import type { Schema } from './schema'// Something a person approves in a browser, saved in hog's folder while hog waits for them.export const PendingApprovalSchema = z.object({@@ -38,7 +39,7 @@ export interface ApprovalFlow<Pending extends PendingApproval, Value> {// What the person approves, in copy and in error codes, like `login`.noun: string// Reads the pending item back from its file. An item hog can't read counts as none.- pending: z.ZodType<Pending>+ pending: Schema<Pending>start: () => Promise<Pending>// Whether a pending item from an earlier run can stand in for a new one, so an agent that// retries hands the person the link it already shared.diff --git a/hog/src/core/command.ts b/hog/src/core/command.tsindex 12415cba35a7..dd5ffa0aa5c1 100644--- a/hog/src/core/command.ts+++ b/hog/src/core/command.ts@@ -92,6 +92,8 @@ interface CommandDefinition<Args, Options, Result> {// Extra properties for the anonymous usage event: counts and names from the result, never// values, paths or text the caller typed.telemetry?(input: CommandInput<Args, Options> & { result: Result }): Record<string, unknown>+ // What --raw prints, for a result that holds more than its views show. Defaults to the result.+ raw?(result: Result): unknown// The exit code for a result. Defaults to 0. Failures throw a HogError instead.exitCode?(result: Result): number}diff --git a/hog/src/core/json.ts b/hog/src/core/json.tsindex f79a5b9ed5e1..4bf7494b5ede 100644--- a/hog/src/core/json.ts+++ b/hog/src/core/json.ts@@ -1,13 +1,13 @@import { z } from 'zod'-import { failWith, parseOrThrow, type ParseFailure } from './schema'+import { failWith, parseOrThrow, type ParseFailure, type Schema } from './schema'// The only module that calls JSON.parse. Its result is `unknown` until a schema checks it, so no// caller casts JSON to the type it hopes for.export const JsonObject = z.record(z.string(), z.unknown())-export function parseJson<S extends z.ZodType>(text: string, schema: S, failure: ParseFailure): z.output<S> {+export function parseJson<T>(text: string, schema: Schema<T>, failure: ParseFailure): T {let raw: unknowntry {raw = JSON.parse(text)@@ -18,9 +18,10 @@ export function parseJson<S extends z.ZodType>(text: string, schema: S, failure:}// For files hog wrote itself and data it can do without: anything it can't read counts as missing.-export function parseJsonOrUndefined<S extends z.ZodType>(text: string, schema: S): z.output<S> | undefined {+export function parseJsonOrUndefined<T>(text: string, schema: Schema<T>): T | undefined {try {- return schema.safeParse(JSON.parse(text)).data+ const parsed = schema.safeParse(JSON.parse(text))+ return parsed.success ? parsed.data : undefined} catch {return undefined}diff --git a/hog/src/core/schema.ts b/hog/src/core/schema.tsindex 6af07bff672c..53cb7b1d7bdd 100644--- a/hog/src/core/schema.ts+++ b/hog/src/core/schema.ts@@ -1,7 +1,15 @@-import type { z } from 'zod'-import { HogError } from './errors'+// zod schemas have this shape, so hog's parsing helpers don't depend on zod's types.+export interface Schema<T> {+ safeParse(raw: unknown): { success: true; data: T } | { success: false; error: { issues: readonly SchemaIssue[] } }+}++export interface SchemaIssue {+ readonly path: readonly PropertyKey[]+ readonly message: string+}+export interface ParseFailure {code: stringhint?: string@@ -13,7 +21,7 @@ const MAX_LISTED_ERRORS = 5// Every issue zod found, so a person can fix them all in one go. The schemas in hog set their own// messages, and an issue inside a document, such as an API answer, starts with its field.-export function getErrorMessage(error: z.ZodError): string {+export function getErrorMessage(error: { issues: readonly SchemaIssue[] }): string {const messages = error.issues.map((issue) =>issue.path.length > 0 ? `${issue.path.map(String).join('.')}: ${issue.message}` : issue.message)@@ -22,12 +30,12 @@ export function getErrorMessage(error: z.ZodError): string {return more > 0 ? `${listed}; and ${more} more` : listed || 'The value is not valid.'}-export function validationError(schema: z.ZodType, raw: unknown): string | undefined {+export function validationError(schema: Schema<unknown>, raw: unknown): string | undefined {const parsed = schema.safeParse(raw)return parsed.success ? undefined : getErrorMessage(parsed.error)}-export function parseOrThrow<S extends z.ZodType>(schema: S, raw: unknown, failure: ParseFailure): z.output<S> {+export function parseOrThrow<T>(schema: Schema<T>, raw: unknown, failure: ParseFailure): T {const parsed = schema.safeParse(raw)if (parsed.success) {return parsed.datadiff --git a/hog/src/core/telemetry.ts b/hog/src/core/telemetry.tsindex 14b3cc269aaf..f691a1513126 100644--- a/hog/src/core/telemetry.ts+++ b/hog/src/core/telemetry.ts@@ -8,6 +8,7 @@ import { VERSION } from '@/version'import type { Mode } from './agent'import { parseJsonOrUndefined } from './json'import type { Env, Runtime } from './runtime'+import type { Schema } from './schema'// PostHog's own project, where posthog-cli release builds send their events too.export const DEFAULT_TELEMETRY_TOKEN = 'sTMFPsFhdP1Ssg'@@ -255,7 +256,7 @@ export class Telemetry {}private async lookup(key: AccessKey): Promise<Identity> {- const get = async <S extends z.ZodType>(path: string, schema: S): Promise<z.output<S> | undefined> => {+ const get = async <T>(path: string, schema: Schema<T>): Promise<T | undefined> => {try {const response = await this.runtime.fetch(new URL(path, key.host), {headers: { Authorization: `Bearer ${key.token}` },diff --git a/hog/src/error-tracking/upload.ts b/hog/src/error-tracking/upload.tsindex 2838255d1ef2..698e6bc870e9 100644--- a/hog/src/error-tracking/upload.ts+++ b/hog/src/error-tracking/upload.ts@@ -1,10 +1,10 @@import { z } from 'zod'import { HogError } from '@/core/errors'-import { parseJson, parseJsonOrUndefined } from '@/core/json'+import { parseJsonOrUndefined } from '@/core/json'import type { Runtime } from '@/core/runtime'-import { parseOrThrow } from '@/core/schema'-import { USER_AGENT } from '@/posthog/api'+import { failWith, parseOrThrow, type Schema } from '@/core/schema'+import { sendRequest } from '@/posthog/api'import { hostUrl } from '@/posthog/host'import type { KeySession } from '@/posthog/key-session'@@ -57,15 +57,11 @@ export function errorCode(body: string): string | undefined {class ApiFailure extends Error {constructor(- readonly status: number | undefined,+ readonly status: number,readonly body: string,readonly url: string) {- super(- status === undefined- ? `Request to ${url} failed: ${body}`- : `PostHog answered ${status} for ${url}: ${body}`- )+ super(`PostHog answered ${status} for ${url}: ${body}`)}// posthog-cli matches error codes in the parsed body, or anywhere in the raw text as a fallback.@@ -169,41 +165,24 @@ export class SymbolSetClient {return new URL(path, hostUrl(host, `api/projects/${projectId}/`))}- async api<S extends z.ZodType>(- method: 'GET' | 'POST',- path: string,- schema: S,- body?: unknown- ): Promise<z.output<S>> {+ async api<T>(method: 'GET' | 'POST', path: string, schema: Schema<T>, body?: unknown): Promise<T> {await this.throttle.wait()const url = this.projectUrl(path)- // Only a failed request becomes an ApiFailure. A renewal that fails keeps its own error and hint.- const send = async (token: string): Promise<Response> => {- try {- return await this.runtime.fetch(url, {- method,- headers: {- Authorization: `Bearer ${token}`,- 'Content-Type': 'application/json',- 'User-Agent': USER_AGENT,- },- body: body === undefined ? undefined : JSON.stringify(body),- timeoutMs: REQUEST_TIMEOUT_MS,- })- } catch (error) {- throw new ApiFailure(undefined, error instanceof Error ? error.message : String(error), url.toString())- }- }- const response = await this.session.send(send, (answer) => answer.status === 401)- const text = await response.text()+ const response = await this.session.send(+ (token) => sendRequest(this.runtime.fetch, url, { method, body, token, timeoutMs: REQUEST_TIMEOUT_MS }),+ (answer) => answer.status === 401+ )if (!response.ok) {- throw new ApiFailure(response.status, text, url.toString())+ throw new ApiFailure(response.status, response.text, url.toString())}const failure = {code: 'unexpected_response',context: `PostHog's answer to ${method} ${url.pathname} isn't what hog expects.`,}- return text ? parseJson(text, schema, failure) : parseOrThrow(schema, undefined, failure)+ if (response.text && response.json === undefined) {+ throw failWith(failure, "The answer isn't JSON.")+ }+ return parseOrThrow(schema, response.json, failure)}async upload(uploads: SymbolSetUpload[], options: UploadOptions): Promise<UploadOutcome> {@@ -290,13 +269,7 @@ export class SymbolSetClient {force: options.force,skip_on_conflict: options.skipOnConflict,}),- (error) =>- !(- error instanceof ApiFailure &&- error.status !== undefined &&- error.status >= 400 &&- error.status < 500- )+ (error) => !(error instanceof ApiFailure && error.status >= 400 && error.status < 500))const response = parseOrThrow(CheckResponse, answer, {code: 'unexpected_response',diff --git a/hog/src/posthog/api.ts b/hog/src/posthog/api.tsindex ce4892ab4b16..7ac896a5faab 100644--- a/hog/src/posthog/api.ts+++ b/hog/src/posthog/api.ts@@ -3,7 +3,7 @@ import { z } from 'zod'import { HogError } from '@/core/errors'import { parseLosslessJson } from '@/core/json'import type { RuntimeFetch } from '@/core/runtime'-import { parseOrThrow } from '@/core/schema'+import { parseOrThrow, type Schema } from '@/core/schema'import { VERSION } from '@/version'import { hostUrl } from './host'@@ -120,24 +120,19 @@ export class PostHogApi {// Each call names the schema of the answer it expects, so a command never works with a shape// PostHog didn't send.- get<S extends z.ZodType>(path: string, schema: S): Promise<z.output<S>> {+ get<T>(path: string, schema: Schema<T>): Promise<T> {return this.request('GET', path, schema)}- post<S extends z.ZodType>(path: string, body: unknown, schema: S): Promise<z.output<S>> {+ post<T>(path: string, body: unknown, schema: Schema<T>): Promise<T> {return this.request('POST', path, schema, body)}- patch<S extends z.ZodType>(path: string, body: unknown, schema: S): Promise<z.output<S>> {+ patch<T>(path: string, body: unknown, schema: Schema<T>): Promise<T> {return this.request('PATCH', path, schema, body)}- private async request<S extends z.ZodType>(- method: Method,- path: string,- schema: S,- body?: unknown- ): Promise<z.output<S>> {+ private async request<T>(method: Method, path: string, schema: Schema<T>, body?: unknown): Promise<T> {const url = this.url(path)const response = await this.session.send((token) => sendRequest(this.fetchFn, url, { method, body, token }),diff --git a/hog/src/posthog/pagination.ts b/hog/src/posthog/pagination.tsindex d7d971a81d0c..006208b259fd 100644--- a/hog/src/posthog/pagination.ts+++ b/hog/src/posthog/pagination.ts@@ -1,7 +1,7 @@import { z } from 'zod'import { HogError } from '@/core/errors'-import { parseOrThrow, type ParseFailure } from '@/core/schema'+import { parseOrThrow, type ParseFailure, type Schema } from '@/core/schema'import type { PostHogApi } from './api'@@ -14,25 +14,37 @@ export interface Page<T> {next?: URL}-export async function getPage<S extends z.ZodType>(api: PostHogApi, path: string, item: S): Promise<Page<z.output<S>>> {- const body = await api.get(- path,- z.object({ results: z.array(item), count: z.number().nullish(), next: z.string().nullish() })- )+// Lets a zod schema hold a hog Schema, so zod adds the place of a failing item, as in `results.3.name`.+function zodOf<T>(schema: Schema<T>): z.ZodType<T> {+ return z.unknown().transform((raw, ctx) => {+ const parsed = schema.safeParse(raw)+ if (parsed.success) {+ return parsed.data+ }+ for (const issue of parsed.error.issues) {+ ctx.addIssue({ code: 'custom', message: issue.message, path: [...issue.path] })+ }+ return z.NEVER+ })+}++export async function getPage<T>(api: PostHogApi, path: string, item: Schema<T>): Promise<Page<T>> {+ const page = z.object({ results: z.array(zodOf(item)), count: z.number().nullish(), next: z.string().nullish() })+ const { results, count, next } = await api.get(path, page)return {- results: body.results,- ...(typeof body.count === 'number' ? { count: body.count } : {}),- ...(body.next ? { next: new URL(body.next) } : {}),+ results,+ ...(typeof count === 'number' ? { count } : {}),+ ...(next ? { next: new URL(next) } : {}),}}// False when it stops at MAX_PAGES first. Keeping only a link's path and query sends every page to// the configured host, whatever host the link names.-async function eachPage<S extends z.ZodType>(+async function eachPage<T>(api: PostHogApi,path: string,- item: S,- visit: (results: z.output<S>[]) => boolean,+ item: Schema<T>,+ visit: (results: T[]) => boolean,maxPages = MAX_PAGES): Promise<boolean> {let next: string | undefined = path@@ -40,7 +52,7 @@ async function eachPage<S extends z.ZodType>(if (read === maxPages) {return false}- const page: Page<z.output<S>> = await getPage(api, next, item)+ const page: Page<T> = await getPage(api, next, item)if (visit(page.results)) {return true}@@ -53,13 +65,8 @@ export function listTooLong(noun: string, hint: string): HogError {return new HogError('list_too_long', `The project has more ${noun} than hog reads in one command.`, hint)}-export async function readAll<S extends z.ZodType>(- api: PostHogApi,- path: string,- item: S,- tooLong: HogError-): Promise<z.output<S>[]> {- const results: z.output<S>[] = []+export async function readAll<T>(api: PostHogApi, path: string, item: Schema<T>, tooLong: HogError): Promise<T[]> {+ const results: T[] = []const complete = await eachPage(api, path, item, (page) => {results.push(...page)return false@@ -72,16 +79,16 @@ export async function readAll<S extends z.ZodType>(// Reads pages until every name has turned up or the list ends. When `complete` is false, hog// stopped at `maxPages` and a name missing from `found` can still be in the list.-export async function findNamed<S extends z.ZodType>(+export async function findNamed<T>(api: PostHogApi,path: string,- item: S,+ item: Schema<T>,names: Iterable<string>,- nameOf: (item: z.output<S>) => string | null,+ nameOf: (item: T) => string | null,maxPages = MAX_PAGES-): Promise<{ found: Map<string, z.output<S>>; complete: boolean }> {+): Promise<{ found: Map<string, T>; complete: boolean }> {const wanted = new Set(names)- const found = new Map<string, z.output<S>>()+ const found = new Map<string, T>()if (wanted.size === 0) {return { found, complete: true }}