Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(hogql): retire the go language service (#128114)

Paweł Szczur7068ed17bf94
diff --git a/.github/workflows/cd-hogql-lang-service-image.yml b/.github/workflows/cd-hogql-lang-service-image.yml
deleted file mode 100644
index 92423d241800..000000000000
--- a/.github/workflows/cd-hogql-lang-service-image.yml
+++ /dev/null
@@ -1,270 +0,0 @@
-name: HogQL Language Service Container Image CD
-
-on:
- push:
- branches:
- - master
- paths:
- - 'services/hogql-language-service/**'
- - '.github/workflows/cd-hogql-lang-service-image.yml'
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.sha }}
- cancel-in-progress: false
-
-jobs:
- build:
- name: Build and push container image
- if: |
- github.repository_owner == 'PostHog' &&
- (
- github.event_name == 'push' ||
- (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/master')
- )
- runs-on: depot-ubuntu-24.04
- timeout-minutes: 30
- permissions:
- contents: read
- packages: write
- id-token: write
-
- outputs:
- sha: ${{ steps.push.outputs.digest }}
-
- steps:
- - name: Check out
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- sparse-checkout: |
- services/hogql-language-service/
- .github/actions/docker-meta/
- sparse-checkout-cone-mode: false
-
- - name: Docker meta and registry login
- id: docker-meta
- uses: ./.github/actions/docker-meta
- with:
- image-name: hogql-lang-service
- aws-role-to-assume: ${{ vars.AWS_ECR_POSTHOG_MASTER_PUBLISH_IAM_ROLE }}
- github-token: ${{ secrets.GITHUB_TOKEN }}
- dockerhub-username: ${{ secrets.DOCKERHUB_USER }}
- dockerhub-password: ${{ secrets.DOCKERHUB_TOKEN }}
- push-to-ghcr: 'false'
- push-to-dockerhub: 'false'
-
- - name: Set up Depot CLI
- uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.7.1
-
- - name: Build and push container image
- id: push
- uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.17.0
- with:
- project: c1f2m5s6zd
- context: ./services/hogql-language-service
- file: ./services/hogql-language-service/Dockerfile
- buildx-fallback: false
- push: ${{ github.ref == 'refs/heads/master' && vars.CD_DEPLOY_ENABLED == 'true' }}
- tags: ${{ steps.docker-meta.outputs.tags }}
- labels: ${{ steps.docker-meta.outputs.labels }}
- annotations: ${{ steps.docker-meta.outputs.annotations }}
- platforms: linux/arm64,linux/amd64
- build-args: COMMIT_HASH=${{ github.sha }}
-
- ordered_image_alias:
- name: Publish ordered image alias
- needs: build
- if: ${{ needs.build.result == 'success' && github.repository_owner == 'PostHog' && vars.CD_DEPLOY_ENABLED == 'true' && vars.ORDERED_IMAGE_ALIAS_PAUSED != 'true' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }}
- runs-on: ubuntu-24.04
- timeout-minutes: 5
- continue-on-error: true
- permissions:
- id-token: write
- contents: read
-
- steps:
- - name: Check out
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
- - name: Compute ordered release tag
- id: ordered-release
- timeout-minutes: 2
- shell: bash
- env:
- ORDERED_RELEASE_TAG_REGEX: ^r[0-9]{12}-[0-9a-f]{6}$
- run: |
- set -uo pipefail
-
- if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then
- fetch_status=0
- timeout --kill-after=10s 90s git fetch \
- --no-tags \
- --prune \
- --no-recurse-submodules \
- --unshallow \
- --filter=tree:0 \
- origin \
- "+${GITHUB_REF}:refs/remotes/origin/${GITHUB_REF_NAME}" || fetch_status=$?
- if [ "$fetch_status" -ne 0 ]; then
- echo "::warning title=Ordered release tag skipped::Source history could not be fetched (status $fetch_status)."
- exit 0
- fi
- fi
-
- if [ "$(git rev-parse --is-shallow-repository 2>/dev/null)" != "false" ]; then
- echo "::warning title=Ordered release tag skipped::Complete source history is unavailable."
- exit 0
- fi
-
- if ! head_revision="$(git rev-parse HEAD)" || [ "$head_revision" != "$GITHUB_SHA" ]; then
- echo "::warning title=Ordered release tag skipped::The checked-out revision does not match GITHUB_SHA."
- exit 0
- fi
-
- if ! position="$(git rev-list --first-parent --count "$GITHUB_SHA")"; then
- echo "::warning title=Ordered release tag skipped::The source position could not be computed."
- exit 0
- fi
-
- if ! [[ "$position" =~ ^[1-9][0-9]{0,11}$ ]]; then
- echo "::warning title=Ordered release tag skipped::The source position is out of range."
- exit 0
- fi
-
- if ! printf -v tag 'r%012d-%.6s' "$position" "$GITHUB_SHA"; then
- echo "::warning title=Ordered release tag skipped::The source position could not be formatted."
- exit 0
- fi
-
- if ! [[ "$tag" =~ $ORDERED_RELEASE_TAG_REGEX ]]; then
- echo "::warning title=Ordered release tag skipped::${tag} does not match the required format."
- exit 0
- fi
-
- echo "tag=${tag}" >> "$GITHUB_OUTPUT"
-
- - name: Warn when ordered image alias tag is unavailable
- if: ${{ steps.ordered-release.outputs.tag == '' }}
- shell: bash
- run: echo "::warning title=Ordered image alias tag unavailable::Ordered image alias publication was skipped because no ordered release tag was generated."
-
- - name: Set up Docker Buildx for ordered image alias
- if: ${{ steps.ordered-release.outputs.tag != '' }}
- uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
-
- - name: Configure AWS credentials for ordered image alias
- if: ${{ steps.ordered-release.outputs.tag != '' }}
- uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
- with:
- role-to-assume: ${{ vars.AWS_ECR_POSTHOG_MASTER_PUBLISH_IAM_ROLE }}
- aws-region: us-east-1
-
- - name: Log in to Amazon ECR for ordered image alias
- if: ${{ steps.ordered-release.outputs.tag != '' }}
- id: ecr-login
- uses: aws-actions/amazon-ecr-login@f2e9fc6c2b355c1890b65e6f6f0e2ac3e6e22f78 # v2.1.2
-
- - name: Publish and read back ordered image alias
- if: ${{ steps.ordered-release.outputs.tag != '' }}
- timeout-minutes: 2
- shell: bash
- env:
- ECR_REGISTRY: ${{ steps.ecr-login.outputs.registry }}
- ECR_REPO: hogql-lang-service
- EXPECTED_DIGEST: ${{ needs.build.outputs.sha }}
- ORDERED_ALIAS: ${{ steps.ordered-release.outputs.tag }}
- run: |
- set -uo pipefail
-
- if [ -z "$EXPECTED_DIGEST" ]; then
- echo "::warning title=Ordered image alias source digest unavailable::Ordered image alias publication was skipped because no image digest was resolved."
- exit 0
- fi
-
- SOURCE_IMAGE="${ECR_REGISTRY}/${ECR_REPO}@${EXPECTED_DIGEST}"
- TARGET_IMAGE="${ECR_REGISTRY}/${ECR_REPO}:${ORDERED_ALIAS}"
-
- publication_status=0
- timeout --kill-after=5s 40s docker buildx imagetools create \
- --prefer-index=false \
- --tag "$TARGET_IMAGE" \
- "$SOURCE_IMAGE" || publication_status=$?
- if [ "$publication_status" -ne 0 ]; then
- echo "::warning::Ordered alias publication exited with status $publication_status"
- fi
-
- resolved_digest="$(timeout --kill-after=5s 40s aws ecr batch-get-image \
- --repository-name "$ECR_REPO" \
- --image-ids imageTag="$ORDERED_ALIAS" \
- --accepted-media-types \
- "application/vnd.oci.image.index.v1+json" \
- "application/vnd.docker.distribution.manifest.list.v2+json" \
- "application/vnd.oci.image.manifest.v1+json" \
- "application/vnd.docker.distribution.manifest.v2+json" \
- --query 'images[0].imageId.imageDigest' \
- --output text)"
-
- if [ "$resolved_digest" != "$EXPECTED_DIGEST" ]; then
- echo "::error title=Ordered image alias integrity mismatch::The ordered image alias does not resolve to the expected image digest."
- exit 1
- fi
-
- echo "::notice title=Ordered image alias present::The ordered image alias resolves to the expected image digest."
-
- deploy:
- name: Trigger HogQL language service deployment
- runs-on: ubuntu-24.04
- timeout-minutes: 5
- needs: build
- if: github.event_name == 'push' && github.repository_owner == 'PostHog' && vars.CD_DEPLOY_ENABLED == 'true' && github.ref == 'refs/heads/master'
- permissions: {}
-
- steps:
- - name: Get deployer token
- id: deployer
- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
- with:
- client-id: ${{ secrets.GH_APP_CHARTS_DEPLOYER_APP_ID }}
- private-key: ${{ secrets.GH_APP_CHARTS_DEPLOYER_PRIVATE_KEY }}
- owner: PostHog
- repositories: charts
-
- - name: Get PR labels
- id: labels
- uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
- with:
- retries: 3
- script: |
- try {
- const { data: pulls } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
- owner: 'PostHog',
- repo: 'posthog',
- commit_sha: context.sha,
- });
- const labels = pulls.flatMap(pr => pr.labels.map(l => l.name));
- return JSON.stringify([...new Set(labels)]);
- } catch (error) {
- if (error.status !== 404 && error.status !== 422) throw error;
- core.notice(`${context.sha} is not a PostHog/posthog commit, so it has no PR labels.`);
- return '[]';
- }
-
- - name: Trigger HogQL language service deployment
- uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
- with:
- token: ${{ steps.deployer.outputs.token }}
- repository: PostHog/charts
- event-type: commit_state_update
- client-payload: |
- {
- "values": {
- "image": {
- "sha": "${{ needs.build.outputs.sha }}"
- }
- },
- "release": "hogql-lang-service",
- "commit": ${{ toJson(github.event.head_commit) }},
- "repository": "PostHog/posthog",
- "labels": ${{ steps.labels.outputs.result }},
- "timestamp": "${{ github.event.head_commit.timestamp }}"
- }
diff --git a/.github/workflows/ci-hogql-language-service.yml b/.github/workflows/ci-hogql-language-service.yml
deleted file mode 100644
index 3e5d3396aa02..000000000000
--- a/.github/workflows/ci-hogql-language-service.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-name: HogQL Language Service
-
-on:
- push:
- branches:
- - master
- paths:
- - '.github/workflows/ci-hogql-language-service.yml'
- - 'services/hogql-language-service/**'
- pull_request:
- paths:
- - '.github/workflows/ci-hogql-language-service.yml'
- - 'services/hogql-language-service/**'
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-permissions:
- contents: read
-
-jobs:
- test:
- name: HogQL language service tests
- runs-on: depot-ubuntu-24.04
- timeout-minutes: 10
-
- steps:
- - name: Check out
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- sparse-checkout: services/hogql-language-service/
- sparse-checkout-cone-mode: false
-
- - name: Set up Go
- uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
- with:
- go-version-file: services/hogql-language-service/go.mod
- cache-dependency-path: services/hogql-language-service/go.sum
-
- - name: Run tests
- working-directory: services/hogql-language-service
- run: go test -race ./...
-
- - name: Enforce autocomplete latency budget
- working-directory: services/hogql-language-service
- env:
- HOGQL_LANGUAGE_SERVICE_ENFORCE_LATENCY_BUDGET: '1'
- run: go test ./internal/completion -run '^TestCompleteContextualCatalogStaysWithinLatencyBudget$' -count=1
diff --git a/.github/workflows/ci-security.yaml b/.github/workflows/ci-security.yaml
index 530e3fe93622..62f0568e5b84 100644
--- a/.github/workflows/ci-security.yaml
+++ b/.github/workflows/ci-security.yaml
@@ -80,7 +80,6 @@ jobs:
- '.github/workflows/ci-security.yaml'
go:
- 'livestream/**'
- - 'services/hogql-language-service/**'
- '.github/workflows/ci-security.yaml'
rust:
- 'cli/**'
@@ -264,7 +263,6 @@ jobs:
--config r/go.lang.security
--exclude-rule go.lang.security.audit.xss.no-fprintf-to-responsewriter.no-fprintf-to-responsewriter
--include /livestream
- --include /services/hogql-language-service
semgrep-rust:
needs: changes
diff --git a/docs/internal/hogql-language-service-demo.md b/docs/internal/hogql-language-service-demo.md
deleted file mode 100644
index 1447f3b1b407..000000000000
--- a/docs/internal/hogql-language-service-demo.md
+++ /dev/null
@@ -1,133 +0,0 @@
-# Local HogQL playground
-
-The playground embeds the real Go language service's HTTP handlers in the demo process.
-It loads a synthetic catalog and serves a small browser editor, without Django, ClickHouse, project data, or credentials.
-It does not execute queries.
-
-## Run
-
-From the repository root:
-
-```bash
-.codex/with-flox env GOTOOLCHAIN=auto go -C services/hogql-language-service run ./cmd/demo
-```
-
-Open <http://127.0.0.1:8092>.
-Use `-port 8093` after `./cmd/demo` if the default port is occupied.
-For port forwarding, bind the demo page to all interfaces:
-
-```bash
-.codex/with-flox env GOTOOLCHAIN=auto go -C services/hogql-language-service run ./cmd/demo -host 0.0.0.0
-```
-
-Forward port `8092` and open the forwarded URL.
-This mode accepts the forwarded hostname and HTTP or HTTPS origins matching that hostname.
-The demo has no authentication; use your development environment's private forwarding controls.
-Only the demo page listens on a port; the embedded service has no separate listener.
-The connection indicator checks the embedded service's health handler through the demo server.
-It reports whether that handler is reachable, not whether a query is valid or supported.
-The page checks periodically while visible and reconnects automatically after a temporary failure.
-Go may download the module's required toolchain on the first run.
-Agents must request elevated execution when starting the demo because it opens a local HTTP listener.
-
-The demo and `cmd/server` share `internal/httpapi`, including request decoding, completion, validation, response encoding, and rate limiting.
-The demo seeds a private in-memory catalog for synthetic team 1 / user 1 and dispatches requests directly to those handlers.
-It does not spawn subprocesses, build another binary, poll readiness, or use an HTTP client.
-Demo settings are explicit and do not read production service environment variables.
-It does not connect to or modify an existing development service.
-The catalog lives in memory with a 24-hour TTL; restart the demo to refresh it.
-Ctrl+C stops the demo server.
-The page defaults to `127.0.0.1` and accepts requests for its own host and origin.
-Binding to `0.0.0.0` or `::` permits other hostnames for forwarding while still rejecting cross-origin browser requests.
-
-The production Dockerfile builds only `cmd/server` and copies only that binary into its final image.
-The demo command, its embedded browser assets, and synthetic catalog are not included in that binary or final image.
-Production authentication and environment configuration remain in `cmd/server`.
-The demo exposes only fixed-scope completion and validation adapters, not the service's catalog mutation routes.
-Do not add the demo to deployment configuration or the production server's routes.
-
-## Manual checks
-
-- Select **Event fields**, click **Complete at cursor**, then click a suggestion to insert it.
-- Select **Valid CTE** and click **Validate**. The service should accept the query and list `events` as its underlying table.
-- Select **Unknown field** or **Unknown table** to inspect diagnostics and typo suggestions.
-- Select **Unicode diagnostic offsets**, validate, and click the diagnostic. It should select `timstamp`, even with the emoji earlier in the query.
-- Select **Property pagination** and use **Load more** to retrieve all 35 matching names.
-- Select **Quoted identifiers** and insert `billing address` to inspect identifier quoting. The service supplies quoted insertion text; the page does not add quotes.
-- Use **CTE completion** and **Subquery completion** to inspect current derived-relation support. Empty suggestions are shown as returned by the service, without a browser fallback.
-- Try `WITH recent AS (SELECT properties FROM events) SELECT recent.pro FROM recent WHERE (` with the cursor after `pro`. Completion should suggest `properties` and retain a parser error; validation should still reject the unfinished query.
-- Open raw responses to inspect parser errors, diagnostics, pagination cursors, physical table names, and catalog revisions.
-- Stop and restart the demo to confirm that the connection indicator changes to disconnected and recovers.
-
-Ctrl/Command+Enter validates; Ctrl/Command+Shift+Enter completes.
-Ctrl+Space is left available for the operating system's input-source shortcut.
-Enable **Analyze as you type** to run both operations after a 300 ms typing pause.
-The checkbox is off by default. Enabling it also analyzes the current query; loading an example or inserting a suggestion schedules analysis too.
-Turning it off cancels scheduled analysis and in-flight requests. Manual buttons and shortcuts remain available.
-Empty queries trigger completion for SELECT and WITH, but not validation.
-Unfinished input-method composition does not trigger automatic requests.
-Completion requests use UTF-16 offsets, matching the textarea selection API.
-Validation requests explicitly select UTF-8 offsets; the page converts those byte ranges to UTF-16 for diagnostic selection.
-Raw responses and copied exchanges keep the service's original offsets.
-The page shows service time separately from browser round-trip time.
-Editing the query clears results and cancels pending requests; moving the cursor clears completion results.
-
-## Share a reproduction
-
-Each result panel has a **Copy request and response** button.
-It copies a JSON record of the most recent completed request for that operation, including the original query, completion cursor position and pagination cursor, completion encoding, HTTP status, and full response with catalog revision.
-The browser path and corresponding service path are included for replay.
-HTTP errors retain their response body, and network errors retain their request and error message.
-Canceled requests do not replace the last captured exchange.
-
-Each panel keeps one captured exchange when you edit the query or move the cursor; the next completed request for that operation replaces it.
-For pagination, the copied exchange represents the latest page, including the cursor used to request it.
-Turn off **Analyze as you type** to keep a reproduction while experimenting further.
-The capture is kept only in page memory and is lost on refresh.
-
-If clipboard access is unavailable, including on some HTTP forwarded URLs, the button opens and selects the captured JSON so you can press Ctrl/Command+C.
-Paste it with a short description of the expected suggestions to report an autocomplete issue.
-The demo does not send feedback anywhere automatically.
-
-## Catalog
-
-`services/hogql-language-service/cmd/demo/catalog.go` defines a hand-maintained subset of the public HogQL schema shapes for `events`, `persons`, `sessions`, and `groups`.
-It also defines invented `postgres.demo.orders` and `demo_customers` warehouse tables, event/person/session/group property namespaces, and 35 `demo_property_*` names for pagination.
-The **CTE join and boolean filters** example joins daily event counts to an invented `demo_rules` table.
-It exercises two CTEs, projected fields, and `TRUE` and `FALSE` filters.
-Load it and validate, then complete after `counts.` or `rules.` to inspect the projected fields.
-The starting cursor is before `day`, so selecting another suggestion replaces that field without making the initial query incomplete.
-To test clause suggestions, add a blank line after the JOIN condition and before ORDER BY; completion there includes WHERE.
-The catalog supplies physical field types only; computed result types may remain unknown, and the Go service does not return Python metadata notices.
-Only names and types are present; there are no event rows, person records, customer examples, or production schema exports.
-The sidebar and `/api/catalog` display the exact catalog published to the service.
-To change it, edit `catalog.go` and restart the command.
-
-## Exclusions and follow-up work
-
-These are deliberate boundaries of the demo or service behavior to investigate later.
-They are not evidence that a query would succeed or fail in the full PostHog compiler.
-
-| Area | Current boundary and reason | Follow-up and acceptance check |
-| -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Full PostHog catalog parity | The fixture is a small, static subset so the demo runs without Django. Virtual joins and all product tables are not modeled completely. | Add an optional catalog generator using a synthetic local project. Compare its published schema with Django's catalog for that project and document required services. |
-| Derived-relation completion | The service resolves CTE and aliased subquery fields through the shared analyzer. Complete CTE definitions can retain outer-query completion through unfinished trailing clauses. | Check fields and property origins both before and inside an unfinished outer predicate. Broken CTE definitions, nested outer-query SELECTs, and incomplete sources still require separate recovery work. |
-| Identifier quoting | The page uses the service's quoted `insertText`. Completion inside an already quoted identifier is not supported by the textarea adapter. | Add quote-aware token replacement before supporting completion inside existing quotes. Check escaped backticks and mid-token edits. |
-| Alias and property semantics | The service preserves direct property origins through derived fields and visible SELECT aliases. Computed or ambiguous origins do not produce property suggestions. | Check event/person separation, renamed containers, and aliases in supported clauses. Computed-expression and nested JSON provenance require separate design and tests. |
-| Other HogQL constructs | Examples do not establish coverage for unions, scalar `WITH` aliases, CTE column-name lists, recursive CTEs, or every dialect. | Decide supported constructs explicitly in the analyzer design; add examples and observable regression coverage for each supported construct. Retain the service's exclusion of recursive CTE support unless that scope changes. |
-| Query execution and compiler parity | No database or Python metadata endpoint is called; validation reflects only this Go service. | Add an opt-in comparison mode against a synthetic local Django project if compiler parity testing is needed. Keep execution separate and explicitly enabled. |
-| Editor integration | A textarea keeps the page dependency-free. Optional debounced validation and completion run after edits. Monaco highlighting, squiggles, and completion inside already quoted identifiers are not provided. Insertion replaces an ordinary identifier around the captured cursor. | Add a locally bundled editor adapter when testing editor behavior, with cursor/range tests for quoted names, escaped characters, and mid-token edits. |
-| Custom catalog editing and permissions | One synthetic team/user catalog is seeded at startup. There is no upload UI, identity picker, catalog refresh button, permission mutation, or JWT workflow. | Add isolated synthetic catalog variants for permission/tenant testing; verify catalog revisions and isolation through authenticated endpoints before adding identity controls. |
-| Performance conclusions | The fixture is small and timings are exploratory. This page is not a load test or latency acceptance gate. | Use the existing large-catalog benchmarks and dedicated latency test; add query-shape benchmarks for the shared analyzer as needed. |
-| Connection status | The indicator checks only whether the embedded service health handler responds successfully. It does not reflect catalog freshness, query correctness, or support for a specific HogQL construct. | Inspect validation and completion responses for query-level behavior. |
-| Hosting and production | Private development port forwarding is supported with `-host 0.0.0.0`. The demo serves HTTP without authentication or built-in TLS and stays excluded from production. | Use authenticated development forwarding. Public hosting would need a separate authentication and TLS design; production service endpoints remain outside this demo. |
-| Persistence | Query text and results live only in the page; refreshing discards them. This avoids silently storing pasted query text. | If useful, add explicit local export/import of synthetic examples with a documented file format. |
-
-## Verification
-
-```bash
-.codex/with-flox env GOTOOLCHAIN=auto go -C services/hogql-language-service test ./cmd/demo
-.codex/with-flox env GOTOOLCHAIN=auto go -C services/hogql-language-service vet ./cmd/demo
-```
-
-Browser checks must exercise this page against the embedded service, including completion insertion, pagination, Unicode diagnostic selection, stale-response handling, and a narrow desktop window.
diff --git a/docs/internal/hogql-language-service.md b/docs/internal/hogql-language-service.md
deleted file mode 100644
index 505cb1e50e01..000000000000
--- a/docs/internal/hogql-language-service.md
+++ /dev/null
@@ -1,588 +0,0 @@
-# HogQL language service
-
-## Goal
-
-The HogQL language service moves latency-sensitive SQL editor operations into a long-lived Go process. Its resident
-catalog avoids rebuilding tables, views, and property metadata for every keystroke.
-
-The service targets:
-
-- low-single-digit-millisecond autocomplete on a warm catalog;
-- a stable internal API for HogQL validation and, later, translation; and
-- low PostHog overhead above ClickHouse execution time.
-
-## Architecture
-
-```text
-Browser
- | authenticated PostHog request
- v
-Django
- | resolves team membership and user permissions
- | publishes visible catalog with a scoped service JWT
- v
-HogQL language service
- | selects the exact team and user catalog
- | parses and resolves entirely in memory
- v
-Autocomplete or validation response
-```
-
-Django remains authoritative for authentication, team membership, entitlements, feature flags, and access-control
-resolution. The Go service does not read PostHog permission tables or accept browser-selected identity without an
-authenticated internal request.
-
-The shared catalog excludes direct-connection table rows because those rows belong only to an explicit `connectionId` database.
-This keeps a direct table's raw dotted name from replacing a synced table that resolves to the same catalog key.
-
-Django adds resolver-confirmed `tableAliases` to the same permission-filtered catalog snapshot.
-For example, a catalog can map `demo_postgres_orders` to `postgres.demo.orders` when both names resolve to the same visible warehouse table.
-Aliases from another project or hidden tables are not published.
-If a candidate is hidden or unresolvable, or a canonical name resolves to different metadata, Django abandons the publication and uses the Python path.
-An alias must resolve to the same table object as exactly one exported canonical name; matching physical IDs alone do not establish equivalence.
-An alias candidate that resolves unambiguously to another visible canonical table follows that effective resolver winner.
-Nonrepresentable resolver collisions require a separate catalog contract before they can use the Go service.
-Built-in `posthog.*` namespaces are outside this rollout.
-
-### Lazy-table traversal catalog
-
-The Python publisher adds traversal metadata alongside the flat catalog using the same permission-filtered HogQL database.
-Deploy the Go traversal consumer before this publisher, because older consumers reject unknown JSON fields.
-Snapshots without traversal metadata keep their existing completion and validation behavior.
-
-A field can carry one of two optional annotations:
-
-- `relation` points to an opaque key in the catalog's `relations` map.
-- `propertyNamespace` points to a key in the catalog's `properties` map.
-
-Each relation definition contains either a `table` reference to an exact canonical key in `tables`, or a `fields` map with the same field format as a top-level table.
-Use `table` when the target has the same permitted fields as an existing catalog table; the consumer reuses that table's prepared field index.
-Use `fields` for virtual targets or targets with a different schema.
-For example, HogQL can resolve `events.person` to a virtual person schema backed by event columns rather than the full `persons` schema.
-The publisher must resolve the effective target instead of inferring it from the field name.
-The relation key is not a SQL table name.
-Traversal definitions never appear in table suggestions, table aliases, or validation's `tableNames` output.
-They are reachable only through fields on a bound source.
-For example, this synthetic catalog exposes a person relation through events without adding another FROM table:
-
-```json
-{
- "tables": {
- "events": {
- "name": "events",
- "type": "posthog",
- "fields": {
- "person": { "name": "person", "type": "field_traverser", "relation": "event-person" }
- }
- }
- },
- "relations": {
- "event-person": {
- "fields": {
- "id": { "name": "id", "type": "string" },
- "properties": { "name": "properties", "type": "json", "propertyNamespace": "person" }
- }
- }
- },
- "properties": {
- "person": [{ "name": "email", "property_type": "String" }]
- }
-}
-```
-
-With that snapshot, completion at `|` supports:
-
-| SQL | Suggestions |
-| -------------------------------------------------------- | ------------------ |
-| `SELECT event.person.\| FROM events AS event` | `id`, `properties` |
-| `SELECT event.person.pro\| FROM events AS event` | `properties` |
-| `SELECT event.person.properties.\| FROM events AS event` | `email` |
-| `SELECT person.\| FROM events` | `id`, `properties` |
-
-The same graph format can describe session fields, group fields, custom lazy joins, and multi-hop relationships.
-For example, when `sessions` and `groups` are published canonical tables, these relation definitions reuse their fields:
-
-```json
-{
- "event-session": { "table": "sessions" },
- "event-group-0": {
- "table": "groups",
- "propertyNamespaces": { "properties": "group:0" }
- },
- "event-group-1": {
- "table": "groups",
- "propertyNamespaces": { "properties": "group:1" }
- }
-}
-```
-
-Attach those relation IDs to the corresponding `events.session`, `events.group_0`, and `events.group_1` fields.
-Completion for `SELECT e.session.| FROM events AS e` then lists fields from the published `sessions` table.
-Completion for `SELECT e.group_0.properties.| FROM events AS e` uses the published `group:0` property list; `group_1` uses `group:1`.
-The optional `propertyNamespaces` map binds direct fields of a referenced table to published property lists.
-Each binding applies only inside that relation, not to the top-level table or another relation that references it.
-The consumer does not infer group indices or carry a binding through a further relation hop.
-
-Completion and validation use the same traversal resolver.
-Explicit traversal metadata takes precedence over built-in property-name heuristics; a missing field on a traversed relation does not inherit a namespace from its spelling.
-Existing source binding, alias visibility, and ambiguity rules still apply.
-
-Relation IDs and property namespaces must refer to published definitions, and a field cannot carry both annotations.
-Table references must use canonical keys, not aliases, and cannot also supply `fields`.
-Property-list bindings require a table reference and an exact target field that has no relation annotation.
-Admission allows at most 4,096 relation definitions and 120,000 inline fields and property-list bindings combined, in addition to existing top-level tables.
-Referenced table fields count once toward cache memory, regardless of how many relations reuse them.
-The cache budget includes the traversal graph and annotations; the catalog request remains bounded by 64 MiB.
-Cycles are allowed without recursive expansion.
-Each query path can follow at most 16 relation hops and also consumes the request's field-lookup work budget.
-
-Traversal works from bound catalog sources, including sources inside CTE bodies.
-Direct top-level property containers retain their explicit namespace through the existing projected-field provenance rules.
-Projecting a relation or nested virtual property container through a CTE, subquery, or SELECT alias does not yet retain traversal provenance.
-Quoted path completion, nested JSON schemas, scalar traverser expression inference, and execution of lazy joins remain unsupported by this graph.
-For deeper JSON paths under an explicit property namespace, validation checks the first property key but does not validate its descendants.
-
-The Python publisher resolves `LazyJoin`, `VirtualTable`, and table-valued `FieldTraverser` targets from the snapshot's database.
-It follows the configured person and session schemas, including person-on-events virtual fields and parent-relative traversers used by custom joins.
-It reuses a canonical table only when the resolved object and permitted field set match; other permitted helper and virtual targets receive deduplicated inline definitions.
-Warehouse tables and views require a matching exported canonical schema; the publisher never copies them into inline definitions.
-Hidden fields and denied targets do not gain traversal metadata.
-An unresolvable edge or publication budget limit omits the affected traversal, while unrelated fields remain available.
-Omissions produce one aggregate warning per catalog build with reason counts, without table names, query text, or exception traces.
-Scalar traversers need no relation annotation.
-No Python resolver chains, join SQL, credentials, or executable expressions belong in the graph.
-
-For example, with the corresponding tables and properties available to the user:
-
-| SQL | Completion behavior |
-| ------------------------------------------------- | ----------------------------------------------------------------------- |
-| `SELECT e.person.\| FROM events AS e` | Fields on the effective person target, not an assumed copy of `persons` |
-| `SELECT e.person.properties.\| FROM events AS e` | Permitted person properties |
-| `SELECT e.session.\| FROM events AS e` | Fields on the configured session table |
-| `SELECT e.group_0.properties.\| FROM events AS e` | Properties for group index 0 only |
-| `SELECT e.pdi.person.\| FROM events AS e` | Fields reached through the person distinct-ID helper |
-
-`CATALOG_VERSION` is the single version for catalog publication and Redis coordination.
-Revisions use `v2:<timestamp>`, and Redis coordination hashes the numeric version with the service target, team, and user.
-Django refreshes snapshots with older revision prefixes on use instead of accepting them until expiry.
-Older Django workers do not accept the new prefix; a mixed rollout can cause extra publications or Python fallback while the service is feature flagged.
-The frontend request and response contracts do not change, and the existing eligibility checks and Python fallback still apply.
-
-## Query analysis
-
-`internal/analysis` owns parsed statements, nested scopes, table and CTE bindings, and projected fields for validation and completion.
-Each document belongs to one request and borrows that request's immutable catalog.
-Statements initialize on demand, while CTE and aliased subquery projections share one budget across the document.
-Validation retains diagnostic formatting, typo suggestions, and position-encoding conversion.
-
-Completion replaces the identifier at the cursor with a placeholder and resolves the containing scope.
-An empty query, whitespace, or the start of a statement offers SELECT and WITH, filtered by the typed prefix.
-Completed comments can precede these starters; completion remains disabled inside strings and unfinished comments.
-These suggestions do not carry a parser error for the unfinished statement.
-After a completed JOIN ON condition, completion includes WHERE, including before an existing ORDER BY clause.
-For example, `SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid |` offers WHERE at `|`.
-Completion does not offer WHERE inside unfinished ON parentheses or between the bounds of BETWEEN.
-Completion also keeps query clauses out of unfinished CASE expressions.
-Before another JOIN at the same query scope, an ON condition offers only AND and OR as clause continuations while retaining applicable comparison operators.
-For example, `SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid | JOIN persons AS p ON 1 = 1` offers AND and OR at `|`, but not WHERE, GROUP BY, ORDER BY, or LIMIT.
-Nested JOINs, comments, strings, and later statements do not suppress a valid WHERE suggestion.
-Other clauses already present after the cursor are not globally filtered from suggestions.
-An incomplete cursor placeholder can still produce a `parseError` alongside these keyword suggestions.
-Unquoted TRUE and FALSE are boolean literals, regardless of case, and validation does not look them up as fields.
-For example, `SELECT uuid FROM events WHERE TRUE AND NOT FALSE` validates without unknown-field errors.
-Expression completion offers TRUE and FALSE, including when typing `tr` or `fa` after a comparison operator.
-Quoted or qualified names such as `events.TRUE` remain field references and require a matching catalog field.
-CTE and aliased `FROM` subquery suggestions contain their projected output names, including aliases and wildcard expansion.
-Direct field projections retain catalog types, and boolean literals project as `boolean`; other expression types remain unknown.
-Qualified CTE completion also works before `FROM`, for example `WITH t AS (SELECT event FROM events) SELECT t.`.
-Inner bindings take precedence, and sibling queries and statements do not contribute suggestions.
-Validation checks aliased subquery output fields and continues to report only underlying catalog tables in `tableNames`.
-Validation reports `duplicate_table` when a FROM or JOIN source reuses a table name or explicit alias in the same query scope.
-Use distinct aliases for self-joins, for example `events AS e JOIN events AS other`.
-This follows HogQL's resolver even when raw ClickHouse accepts an unaliased self-join.
-The duplicate-name check is case-sensitive; separate nested queries and set-operation branches can reuse names.
-Catalog table names, table aliases, and CTE names resolve by exact case, as in the Python resolver for ordinary tables.
-Catalogs can contain both `events` and `Events`; each name retains its own fields, and validation lists both names when both are used.
-With only `events` in the catalog, `SELECT properties FROM Events` reports `unknown_table`.
-The same query is valid when an exact `Events` entry exists and exposes `properties`.
-Table completion still matches prefixes without regard to case, so typing `EV` can suggest both names and inserts the selected name with its original case.
-The optional `tableAliases` catalog map registers exact alternate spellings without duplicating field indexes.
-For example, `"demo_postgres_orders": "postgres.demo.orders"` lets both names resolve to the fields on the canonical `tables` entry.
-Validation retains the spelling from SQL in `tableNames`, and self-joins through two spellings keep separate source bindings.
-Completion shows one spelling per canonical target. It prefers a matching canonical name and otherwise shows one matching alias.
-CTEs shadow only their exact spelling, so a CTE named `demo_postgres_orders` does not hide `postgres.demo.orders`.
-CTE shadowing and table-suggestion deduplication use exact names, so a CTE named `Events` does not hide the catalog table `events`.
-Multi-part names retain their source identity when their parser-safe forms coincide, such as `a.b.c_d` and `a.b_c.d`.
-Duplicate qualifiers do not establish property provenance, including inside CTE projections and qualified wildcards.
-FROM and JOIN completion includes visible table CTEs before catalog tables, with `CTE` in the suggestion detail.
-For an unquoted multi-part prefix, completion returns matching leaf tables with full labels and suffix-only insertion.
-For example, `FROM postgres.` labels the result `postgres.demo.orders` and inserts `demo.orders`; `FROM postgres.demo.or` inserts `orders`.
-Namespace components already present in SQL must match catalog case exactly, so insertion cannot produce an unresolved spelling.
-Completion inside quoted path components and namespace-only suggestions remain follow-up work.
-The service omits dotted candidates when any remaining component would require identifier quotes.
-Monaco replaces the current word only through the cursor, so text after a mid-word cursor remains in the document.
-CTE names follow the same scope, definition-order, and shadowing rules as relation lookup; scalar WITH aliases are not tables.
-A visible CTE hides a catalog table with the same name, and pagination counts that name once.
-CTE insertion quotes the whole name when needed, including names with dots.
-When visible sources expose the same field name, unqualified completion returns one suggestion per source.
-The label remains the field name, the detail includes its type and source, and insertion uses the qualified field, for example `e.uuid`.
-An explicit table alias identifies its source, including separate aliases in a self-join; the table name and its alias do not create duplicate suggestions.
-CTEs and aliased subqueries use the same rules, with identifier quoting for both the source and field.
-Completion leaves `timestamp` unquoted, including qualified references such as `e.timestamp`; other keyword names still use conservative quoting.
-Multi-part physical table paths use HogQL's implicit double-underscore alias, for example `postgres__synced__orders.synced_id`; an explicit alias takes precedence.
-Equal field labels have a deterministic source order across completion pages.
-Unqualified completion uses fixed-width global ranks for `sortText`, so client sorting preserves server order across pages even for labels containing punctuation.
-Unique fields and already-qualified completion retain their existing details and insertion text.
-
-Select aliases follow the resolution order in `posthog/hogql/resolver.py` (`visit_select_query` and `visit_alias`).
-An explicit alias becomes visible after its defining SELECT item, so later items can reference it.
-WHERE, PREWHERE, GROUP BY, HAVING, ORDER BY, named WINDOW definitions, and LIMIT expressions can reference all SELECT aliases in their query.
-FROM and JOIN expressions cannot reference them, and aliases do not cross nested queries, CTE definitions, UNION branches, or statements.
-UNION ALL, UNION DISTINCT, and EXCEPT branches resolve property containers and aliases from their own sources without inheriting another branch's property catalog.
-Alias lookup is case-sensitive, as in the Python resolver; completion prefix matching remains case-insensitive.
-An alias named `UUID` does not hide a source field named `uuid`; they can refer to different values.
-An alias takes precedence over an unqualified field with the same name, while qualified field lookup still uses the relation.
-Direct alias chains retain catalog types, and validation typo suggestions include visible aliases.
-
-Direct property-container projections retain their original catalog namespace through CTEs, aliased subqueries, renamed fields, wildcard expansion, and visible SELECT aliases.
-Completion and property-name validation use that same origin, rather than guessing from the projected name.
-For example, `WITH recent AS (SELECT properties FROM events) SELECT recent.properties.$br FROM recent` suggests `$browser`.
-A CTE named `events` that projects `persons.properties` uses the person property catalog, not the event catalog.
-Renaming a container to `props` preserves its origin; projecting an individual property value does not preserve the whole container's namespace.
-
-Case-variant aliases keep separate property origins:
-
-```sql
-SELECT e.properties.$browser, E.properties.email
-FROM events AS e JOIN persons AS E ON 1 = 1
-```
-
-With those properties in the supplied catalog, both references validate.
-Completion after `e.properties.$br` suggests the event property `$browser`, while `E.properties.em` suggests the person property `email`.
-A reference to `E` does not resolve an alias declared only as `e`.
-CTEs named `t` and `T` also retain separate projected fields and property origins.
-An unrelated custom table named `Events` does not inherit the built-in event property catalog from its spelling.
-
-Physical field completion borrows the catalog prefix index.
-Derived projections have a shared limit of 16,384 fields before deduplication.
-Field resolution also has a request-wide budget of 1,048,576 work units, counting relation visits and identifier bytes used for lookups and derived-field indexes.
-Select-alias indexing, source enumeration, lookup, and field suggestion scans share that work budget.
-Aliases of the same relation share a cached field index and one candidate entry for unqualified type resolution.
-Completion returns HTTP 400 when either limit is exceeded; validation returns a `query_limit` diagnostic.
-Derived qualified suggestions are sorted and deduplicated before pagination.
-
-### Completion recovery
-
-Completion first analyzes the query after replacing the identifier at the cursor.
-If that parse fails, a query with complete CTE definitions can recover its outer scope when the SELECT list and FROM/JOIN sources remain parseable.
-Recovery retains those source slices and discards the outer trailing clause, rather than guessing CTE fields or scanning aliases across scopes.
-The shared analyzer still resolves projected fields, types, exact-case relation names, and known property origins.
-
-With `|` marking the cursor, this incomplete query suggests `properties`:
-
-```sql
-WITH recent AS (SELECT uuid, properties FROM events)
-SELECT recent.pro| FROM recent WHERE (
-```
-
-The cursor can also be inside the unfinished outer predicate:
-
-```sql
-WITH recent AS (SELECT properties AS props FROM events)
-SELECT * FROM recent WHERE (recent.props.$br|
-```
-
-With `$browser` in the event property catalog, completion suggests that property.
-A corresponding projection from `persons` uses the person property catalog instead.
-The response keeps the original cursor-replaced `parseError`; validation still checks the original query and reports its syntax error.
-Recovery never makes invalid SQL executable or changes the validation contract.
-Supported trailing boundaries are WHERE, PREWHERE, GROUP BY, HAVING, ORDER BY, and LIMIT.
-
-Recovery preserves SELECT aliases when their defining SELECT items survive parsing.
-An outer predicate can see those aliases, but an earlier SELECT item cannot see a later alias.
-The recovered cursor position preserves that distinction, including when the original cursor falls inside the discarded clause.
-Recovered analysis uses the existing projection and field-lookup work limits.
-
-For a single SELECT without WITH, the existing fallback recovers only a parseable FROM clause and does not reconstruct discarded SELECT aliases.
-Neither fallback replaces successfully parsed bindings.
-
-An unfinished block comment returns a parser error instead of blocking completion or validation.
-For example, `WITH recent AS (SELECT uuid FROM events) SELECT recent.uuid FROM recent WHERE /* unfinished` produces a `syntax_error` validation diagnostic.
-Completion before that comment returns a `parseError` without recovering CTE field suggestions; completion inside the comment remains disabled.
-Closed block comments and line comments at end of input remain valid.
-
-### Recovery and remaining work
-
-- Multi-part normalization supports unquoted table paths after `FROM` and `JOIN`. Separately quoted path components and multi-part paths in comma-separated sources remain unsupported. Replacing the regular-expression normalization with token-aware handling is follow-up work.
-- Source-specific case-insensitive relation lookup remains unsupported. Python catalog nodes can opt in, for example for Snowflake, but the Go catalog payload does not carry that per-node flag. Supporting it requires publishing the metadata and implementing exact-match-first, opt-in fallback without merging distinct names. The service requires exact relation names until then; autocomplete prefix matching remains case-insensitive.
-- Physical field and property-key case sensitivity remain separate from relation-name resolution. This layer does not claim complete identifier-case parity with the Python resolver.
-- CTE recovery requires complete CTE definitions and a parseable outer SELECT/FROM prefix. Broken CTE bodies, missing CTE-closing parentheses, and incomplete SELECT or JOIN sources remain unsupported because they do not establish a reliable projected schema. Recovering those forms requires separate structural recovery for each damaged scope.
-- Recovery does not target a cursor inside a CTE definition or FROM/JOIN source section. It also rejects queries with any nested SELECT in the outer query, including an intact FROM subquery or predicate subquery. Per-scope recovery must first preserve cursor ownership and alias visibility without importing sibling bindings.
-- Set-operation and multi-statement recovery, scalar WITH declarations, and unterminated quoted tokens or comments remain excluded. These need separate statement/branch selection and lexical recovery rules before the service can infer bindings safely.
-- Property provenance covers direct containers only. Computed JSON expressions, nested JSON schemas, conflicting sources, and duplicate projected names do not establish a namespace. Completion suppresses property suggestions and validation skips property-name checks when the origin is unknown. Expression inference and ambiguous-column diagnostics remain follow-up work; duplicate source names produce `duplicate_table`.
-- Projecting a relation or nested virtual container such as `e.person.properties AS props` does not retain traversal provenance. Extend projected-field provenance before enabling those paths outside their source scope; direct source traversal requires explicit catalog annotations, and existing physical property paths remain available.
-- Single-SELECT recovery without WITH retains only FROM bindings and does not guess discarded aliases. Extending SELECT-alias recovery there requires preserving its SELECT list and visibility positions as the CTE-aware path does.
-- SELECT-alias property provenance follows the existing visibility and case-sensitive precedence rules. An alias without a known container origin suppresses property-owner fallback; it does not inherit a namespace from a name such as `person` or `properties`.
-- Scalar WITH aliases, aliases inside expressions, ARRAY JOIN aliases, QUALIFY, and duplicate SELECT-alias diagnostics remain follow-up work. Model their resolver order and parser support before extending the top-level SELECT alias index. For duplicate declarations, the index retains the first field and type, but property provenance becomes unknown once the duplicate declaration is visible; earlier references retain their original provenance.
-- Validation skips field checks when a query has no known FROM bindings, including SELECT without FROM. Completion can still suggest its aliases. Add explicit empty-source scopes and distinguish unknown relations before enabling strict validation there.
-- JOIN USING output coalescing and ambiguous unqualified-field diagnostics remain follow-up work. Completion offers each source's qualified field; it does not choose a join-wide value or change validation's ambiguity rules.
-- CTE table-name suggestions in malformed FROM/JOIN clauses still require cursor replacement to produce parseable SQL. This recovery slice targets outer expression completion with intact sources; recovering incomplete source declarations needs its own scope and ambiguity checks.
-- Unaliased `FROM` subquery outputs, completion inside quoted identifiers, expression type inference, and complete set-operation semantics remain follow-up work.
-- Recursive CTEs, lateral subqueries, and full HogQL compiler parity are outside this layer. The service does not execute queries or fetch metadata during analysis.
-
-## Isolation boundary
-
-Every protected route requires both `team_id` and `user_id` in its path. Shared middleware converts those values into
-one authorization struct used by JWT verification, rate limiting, catalog lookup, and request handlers.
-
-The service keys its current catalog cache by that pair because the published snapshot contains user-filtered tables
-and properties.
-
-The transport JWT also binds the request to:
-
-- one positive team ID;
-- one positive user ID;
-- one service audience;
-- an explicit operation; and
-- a short validity window.
-
-An absent, invalid, expired, or evicted catalog fails closed. The service never falls back to another team or user's
-catalog.
-
-Before reading a request body, a bounded token bucket limits the direct peer address. After JWT verification, another
-bounded bucket limits the authenticated team and user pair. The service ignores forwarded-IP headers because only
-deployment infrastructure can define a trustworthy proxy chain.
-
-A later implementation may store structural schema once per team and apply smaller user authorization overlays. The
-team remains the primary isolation boundary in that model.
-
-## Resident state
-
-Catalog publication replaces the catalog and revision atomically. Readers observe either the previous complete
-revision or the next complete revision.
-
-The publication contract accepts this alias form:
-
-```json
-{
- "tables": {
- "postgres.demo.orders": { "type": "data_warehouse", "fields": {} }
- },
- "tableAliases": {
- "demo_postgres_orders": "postgres.demo.orders"
- },
- "properties": {}
-}
-```
-
-`tableAliases` is optional, so older publishers remain compatible.
-Each alias target must be a direct key in `tables`; aliases cannot target another alias.
-An alias that equals a canonical key is valid only when it targets itself, which makes the entry a no-op.
-The service rejects empty names, dangling targets, chains, cycles, and aliases that contradict canonical keys before cache admission.
-Alias strings and lookup entries count toward the catalog memory limit.
-
-The in-memory registry has two bounds:
-
-- a publication TTL expires entries after their catalog was published; and
-- least-recently-used eviction caps both the number of entries and their estimated resident bytes.
-
-The generated scale fixture verifies the HTTP publication path, completion, pagination, and validation with 4,096 canonical tables, 25 fields per table, and 120,000 event properties in one property namespace.
-The table fields are columns, not part of the property count.
-This profile fits the `64 MiB` request limit and the default `8 GiB` shared cache budget, but it is not an unlimited count guarantee.
-Longer names and richer metadata increase both serialized and resident sizes.
-
-Warm autocomplete and validation reuse a published catalog without synchronous metadata requests.
-A missing or incompatible catalog triggers synchronous catalog construction and publication in Django before the service retry.
-
-## API direction
-
-The initial internal API supports:
-
-- publishing and deleting a catalog for one team and user;
-- contextual autocomplete with at most 25 results and cursor pagination; and
-- syntax and catalog-backed semantic validation.
-
-Responses return the catalog revision used for computation. Callers can discard results produced from a stale
-revision.
-
-HogQL translation is the next service capability. Its stable response should contain parameterized ClickHouse SQL,
-bound values, referenced resources, output schema, warnings, and catalog revisions. The internal parser AST is not an
-API contract.
-
-## Deployment and rollout
-
-The production container builds a static binary with Go 1.27.3 and runs as a non-root user. A non-loopback listener
-fails startup unless dedicated signing keys are configured.
-
-Local and debug environments may use the service directly. Production integration remains behind a server-side
-feature flag and should progress through shadow comparison before serving editor results.
-The Go consumer accepts alias metadata, and Django always publishes resolver-confirmed warehouse aliases.
-Django refreshes cached catalogs without the current version prefix before it uses their responses.
-Each request attempts at most one publication and one post-publication retry; marker and lease paths add only bounded Go rechecks.
-The retry must return the current catalog version, but a concurrent publication for the same team and user can supersede the requested revision.
-If publication fails or a catalog cannot represent the resolver result, Django uses the Python autocomplete or validation path.
-Malformed HTTP payloads, incompatible revisions after refresh, and malformed autocomplete or validation mappings also use the Python path.
-Malformed service responses produce a sanitized Error Tracking event without the SQL text, response body, user context, or original exception.
-
-Full-query HogQL autocomplete and metadata can use the language service when `sourceQuery` is absent or is a `HogQLQuery`.
-Only the current editor SQL is sent, together with the cursor position for autocomplete.
-The editor can retain an older or incomplete `sourceQuery` while the current SQL changes; full-query metadata does not use that source text.
-For example, metadata for `SELECT distinct_id FROM events` can use Go even if `sourceQuery.query` is `SELECT event FROM events WHERE` and `indexUsage` is true.
-Both operations continue to use Python when `connectionId`, `globals`, `filters`, or `modifiers` is not null.
-Metadata also uses Python when `variables` is not null or `debug` is true.
-Expression languages and non-`HogQLQuery` source contexts remain on Python because they can require surrounding query resolution.
-Service failures preserve the original request, including its source context, for Python fallback.
-
-Go metadata returns diagnostics, logical table names, and source-positioned notices for resolved table and field references, not the full Python compiler metadata.
-`indexUsage: true` does not force Python fallback or enable index analysis in Go.
-Go responses leave `index_usage`, `isUsingIndices`, and `ch_table_names` unset.
-The Django adapter maps Go notices into the existing metadata `notices` list, which the editor displays as hints.
-Older service responses without a `notices` field remain valid and produce an empty list.
-Malformed notices use the same Python fallback and sanitized error reporting as malformed diagnostics.
-Python-only heuristic warnings and actionable index warnings are not added to a successful Go response.
-Index analysis and compiler metadata parity remain separate follow-up work; this routing change does not add a second Python validation pass.
-
-### Table and field notices
-
-The Go validation response includes a separate `notices` array with `message`, `start`, and `end` fields.
-These notices identify resolved table references and report known types for fields written in the query.
-They use the published, permission-filtered catalog and the same scope resolution as validation.
-They do not execute a query, expand the Python compiler's internal expressions, or change whether a query is valid.
-Property definitions published as `Numeric` produce `Float` notices, matching Python's HogQL property conversion.
-
-```sql
-SELECT timestamp FROM events
-```
-
-This query produces a table notice for `events` and a `DateTime` field notice for `timestamp` when that type is published in the catalog.
-Warehouse alias notices identify the canonical catalog table, and CTE source notices identify the common table expression without adding it to `table_names`.
-Table notices mark named `FROM` and `JOIN` sources; alias declarations, qualifiers in field paths, and derived-subquery aliases do not receive separate table notices.
-
-```sql
-SELECT e.person.id FROM events AS e
-```
-
-The field notice uses the lazy relation's target schema when the catalog includes the `person` traversal.
-An older flat catalog cannot provide that traversal's type information.
-
-```sql
-SELECT * FROM events
-```
-
-This query produces a table notice, but no field notices for the columns represented by `*`.
-Generated field expressions have no matching token in the editor and must not produce hints at unrelated source positions.
-
-Notices remain separate from error and warning diagnostics, with at most 128 notices per response.
-Their source ranges follow the requested position encoding, including UTF-16 for the editor.
-Unknown or ambiguous fields do not receive a type hint.
-SELECT alias hints follow declaration order:
-
-```sql
-SELECT event AS v, v, timestamp AS v FROM events ORDER BY v
-```
-
-The middle `v` retains the `String` hint from `event`; the later duplicate declaration makes `ORDER BY v` ambiguous, so it receives no hint.
-Notice collection reuses the parsed query after validation and stops when its output or lookup budget is exhausted; it does not turn a valid query into an error.
-Expression type inference, physical ClickHouse table metadata, and property materialization details remain follow-up work.
-Unrecognized catalog type strings are omitted rather than converted into a guessed type.
-
-### Routing metrics
-
-For authenticated requests that have the service configured and the feature flag enabled, the Prometheus counter `hogql_editor_assist_responses_total` counts the backend that produced the final successful editor response.
-Its bounded attributes are the operation, backend, and routing reason.
-The operation is `autocomplete` or `metadata`, the backend is `language_service` or `python`, and the reason is `served`, `ineligible`, `service_error`, or `invalid_response`.
-The denominator includes enabled requests that are ineligible for the Go service and use Python.
-It excludes disabled requests, requests without a user, and requests that fail before either backend constructs a response.
-The existing Django Prometheus scrape exports the counter for Grafana without another setting.
-It aggregates enabled teams and users because it has no tenant labels.
-Use this query to compare response rates by backend:
-
-```promql
-sum by (operation, backend) (rate(hogql_editor_assist_responses_total[5m]))
-```
-
-Use this query for the Python share of successful enabled responses in each operation:
-
-```promql
-(
- sum by (operation) (rate(hogql_editor_assist_responses_total{backend="python"}[5m]))
- or on (operation)
- 0 * sum by (operation) (rate(hogql_editor_assist_responses_total[5m]))
-)
-/
-sum by (operation) (rate(hogql_editor_assist_responses_total[5m]))
-```
-
-An absent series can mean no observations or a missing scrape.
-The share is undefined when an operation has no successful enabled responses in the selected interval.
-
-After a missing or legacy catalog response, Django coordinates publication in Redis by language-service target, catalog contract, team, and user.
-A publisher holds a 10-second token-owned lease while it rechecks Go, builds the permission-filtered catalog, and publishes it.
-Contenders wait for the lease for at most 250 milliseconds, then recheck Go and use the Python path if the catalog is still unavailable.
-Redis socket operations and Go requests have their own bounds; the 250-millisecond contention budget is not a total refresh deadline.
-A five-second success marker lets a request recheck Go before acquiring a newly released lease.
-The marker is advisory: a missing or legacy Go response overrides it, and neither schemas nor authorization results are stored in Redis.
-Redis outages use the existing direct publication path.
-If catalog construction outlives the lease, a second publisher can duplicate the Go catalog build and publication.
-
-### Autocomplete response timings
-
-Autocomplete responses include timings for the Django editor-assist handler, including requests that fall back to Python.
-The existing `language_service_http` value measures only the final successful service request's HTTP duration.
-The existing `language_service_go` value measures the Go computation reported by that response.
-Neither value includes earlier catalog misses, catalog construction, publication, or Redis coordination.
-
-The `./editor_assist` timing measures the whole autocomplete handler.
-Nested stages use the following suffixes:
-
-| Stage | Work measured |
-| --------------------------------------------- | ------------------------------------------------------------------------------ |
-| `routing` | Feature-flag evaluation and eligibility checks |
-| `language_service_initial` | Initial service call, including a catalog miss or failure |
-| `catalog_coordination` | Full catalog recovery, including its nested stages |
-| `catalog_coordination/redis_marker_lookup` | Redis client setup and success-marker lookup |
-| `catalog_coordination/redis_lock_acquire` | Lease acquisition, including contention waiting |
-| `catalog_coordination/language_service_check` | Accumulated service rechecks and retries |
-| `catalog_coordination/catalog_schema` | Permission-filtered database schema construction |
-| `catalog_coordination/catalog_build` | Catalog payload construction, including properties and aliases |
-| `catalog_coordination/catalog_publish` | Publication, including signing, payload serialization, and the service request |
-| `catalog_coordination/redis_marker_write` | Success-marker write |
-| `catalog_coordination/redis_lock_release` | Lease release |
-| `response_mapping` | Conversion of the service response into autocomplete suggestions |
-| `fallback_error_tracking` | Sanitized error reporting before Python fallback |
-| `fallback_database` | Database preparation for Python autocomplete |
-| `fallback_python_autocomplete` | Python autocomplete computation |
-
-Timings use seconds and fixed stage names, without SQL, table names, or user identifiers.
-Only attempted stages appear; a failed stage retains its duration when the request recovers through Python.
-Repeated stages accumulate their durations within that request.
-Parent timings include their children, so do not sum every entry to calculate total latency.
-The final HTTP/Go timings and Python parser timings also overlap the handler stages.
-
-The handler total excludes request queuing, authentication, API request preparation, final API serialization, middleware, and browser/network latency.
-If the handler total is small but the browser request remains slow, inspect the enclosing query API traces and browser network timings.
-Metadata responses do not expose a timing list; this change does not extend their response schema.
-
-The initial rollout keeps ClickHouse execution in Django:
-
-```text
-Django -> language service translation -> Django -> ClickHouse
-```
-
-Measurements must separate catalog lookup, parsing, translation, network transit, ClickHouse duration, result
-decoding, and serialization. A combined translation and execution path is justified only if the extra service hop
-prevents the overhead target.
-
-## Local playground
-
-For manual development testing, the separate [local playground](hogql-language-service-demo.md) embeds the shared HTTP handlers with an isolated synthetic catalog and a browser editor.
-It is excluded from the production binary and container image.
-Its guide records catalog approximations, unsupported workflows, and follow-up checks.
-
-## Non-goals
-
-- Reimplementing PostHog membership or permission resolution in Go.
-- Exposing the language service directly to browsers.
-- Treating feature flags as an authorization boundary.
-- Moving ClickHouse execution before latency measurements justify it.
-- Making the parser's internal AST a compatibility contract.
diff --git a/hogli.yaml b/hogli.yaml
index c97764f88d19..17a4ae51953a 100644
--- a/hogli.yaml
+++ b/hogli.yaml
@@ -394,9 +394,6 @@ start:
services:
- redis
hidden: true
- start:hogql-lang-service:
- cmd: env GOTOOLCHAIN=auto HOGQL_LANGUAGE_SERVICE_ALLOW_INSECURE=1 MAX_CATALOGS=2 CATALOG_CACHE_MAX_BYTES=268435456 go -C services/hogql-language-service run ./cmd/server
- description: Run the HogQL language service locally with a two-catalog cache
start:celery:
bin_script: start-celery
description: Start Celery worker or beat scheduler with auto-reload
diff --git a/posthog/api/services/query.py b/posthog/api/services/query.py
index fd7743aff437..de45d35fea06 100644
--- a/posthog/api/services/query.py
+++ b/posthog/api/services/query.py
@@ -1,13 +1,10 @@
from __future__ import annotations
-import time
-from contextlib import nullcontext
from dataclasses import dataclass
from typing import TYPE_CHECKING, Literal, Optional, overload
import structlog
import pydantic_core
-import posthoganalytics
from pydantic import BaseModel
from rest_framework.exceptions import APIException, ValidationError
@@ -18,16 +15,11 @@ from posthog.schema import (
DataWarehouseViewLink,
HogLanguage,
HogQLAutocomplete,
- HogQLAutocompleteResponse,
HogQLMetadata,
- HogQLMetadataResponse,
- HogQLNotice,
- HogQLQuery,
HogQLVariable,
HogQuery,
HogQueryResponse,
QuerySchemaRoot,
- QueryTiming,
)
from posthog.hogql.autocomplete import get_hogql_autocomplete
@@ -35,26 +27,14 @@ from posthog.hogql.compiler.bytecode import execute_hog
from posthog.hogql.constants import LimitContext
from posthog.hogql.context import HogQLContext
from posthog.hogql.direct_connection import resolve_database_for_connection
-from posthog.hogql.editor_assist_metrics import EDITOR_ASSIST_DURATION_SECONDS, EDITOR_ASSIST_RESPONSES_TOTAL
+from posthog.hogql.editor_assist_metrics import EDITOR_ASSIST_DURATION_SECONDS
from posthog.hogql.errors import ExposedHogQLError, ResolutionError
-from posthog.hogql.language_service import (
- CATALOG_REVISION_PREFIX,
- CatalogMissing,
- LanguageServiceClient,
- LanguageServiceError,
- LanguageServiceResult,
- MalformedLanguageServiceResponse,
- build_catalog,
- coordinate_catalog_publication,
- is_language_service_enabled,
-)
from posthog.hogql.metadata import get_hogql_metadata
from posthog.hogql.modifiers import create_default_modifiers_for_team
from posthog.hogql.timings import HogQLTimings
from posthog.clickhouse.query_tagging import tag_queries
from posthog.cloud_utils import is_cloud
-from posthog.dataclasses import frozen
from posthog.event_usage import AnalyticsProps
from posthog.exceptions import DatabaseSchemaUnavailable
from posthog.exceptions_capture import capture_exception
@@ -81,256 +61,6 @@ if TYPE_CHECKING:
logger = structlog.get_logger(__name__)
-@frozen
-class _DatabaseSchemaCatalog:
- response: DatabaseSchemaQueryResponse
- database: Database
-
-
-type _EditorAssistReason = Literal["served", "ineligible", "service_error", "invalid_response"]
-type _EditorAssistOperation = Literal["autocomplete", "metadata"]
-type _MalformedResponseStage = Literal["http_response", "response_mapping"]
-
-
-@frozen
-class _EditorAssistRoute:
- enabled: bool
- result: LanguageServiceResult | None
- reason: _EditorAssistReason
- malformed_stage: _MalformedResponseStage | None = None
-
-
-def _is_current_catalog_revision(revision: object) -> bool:
- return isinstance(revision, str) and revision.startswith(CATALOG_REVISION_PREFIX)
-
-
-def _language_service_eligible(query: HogQLAutocomplete | HogQLMetadata) -> bool:
- common = (
- query.language.value == "hogQL"
- and query.connectionId is None
- and query.globals is None
- and query.filters is None
- and query.modifiers is None
- )
- if isinstance(query, HogQLMetadata):
- return (
- common
- and (query.sourceQuery is None or isinstance(query.sourceQuery, HogQLQuery))
- and query.variables is None
- and not query.debug
- and not query.includeOutputTypes
- )
- return common and (query.sourceQuery is None or isinstance(query.sourceQuery, HogQLQuery))
-
-
-def _language_service_call(
- team: Team,
- user: User,
- query: HogQLAutocomplete | HogQLMetadata,
- timings: HogQLTimings | None = None,
-) -> _EditorAssistRoute:
- try:
- client = LanguageServiceClient()
-
- def call() -> LanguageServiceResult:
- if isinstance(query, HogQLAutocomplete):
- return client.autocomplete(team.pk, user.pk, query.query, query.endPosition)
- return client.validate(team.pk, user.pk, query.query)
-
- with timings.measure("language_service_initial") if timings is not None else nullcontext():
- result = call()
- except CatalogMissing:
- result = None
- except MalformedLanguageServiceResponse:
- return _EditorAssistRoute(enabled=True, result=None, reason="invalid_response", malformed_stage="http_response")
- except LanguageServiceError:
- return _EditorAssistRoute(enabled=True, result=None, reason="service_error")
-
- if result is not None:
- if _is_current_catalog_revision(result.body.get("catalogRevision")):
- return _EditorAssistRoute(enabled=True, result=result, reason="served")
-
- publication_succeeded = False
-
- def publish_catalog() -> None:
- nonlocal publication_succeeded
- with timings.measure("catalog_schema") if timings is not None else nullcontext():
- schema_catalog = _build_database_schema_query(team, DatabaseSchemaQuery(), user=user)
- revision = f"{CATALOG_REVISION_PREFIX}{time.time_ns()}"
- with timings.measure("catalog_build") if timings is not None else nullcontext():
- catalog = build_catalog(
- team,
- user,
- schema_catalog.response,
- database=schema_catalog.database,
- )
- with timings.measure("catalog_publish") if timings is not None else nullcontext():
- client.publish(team.pk, user.pk, revision, catalog)
- publication_succeeded = True
-
- def check_catalog() -> LanguageServiceResult | None:
- try:
- current = call()
- except CatalogMissing:
- return None
- if not _is_current_catalog_revision(current.body.get("catalogRevision")):
- if publication_succeeded:
- raise MalformedLanguageServiceResponse("language service returned an incompatible catalog revision")
- return None
- return current
-
- try:
- with timings.measure("catalog_coordination") if timings is not None else nullcontext():
- result = coordinate_catalog_publication(
- team.pk,
- user.pk,
- client.base_url,
- check_catalog,
- publish_catalog,
- timings=timings,
- )
- except MalformedLanguageServiceResponse:
- return _EditorAssistRoute(enabled=True, result=None, reason="invalid_response", malformed_stage="http_response")
- except (DatabaseSchemaUnavailable, LanguageServiceError):
- return _EditorAssistRoute(enabled=True, result=None, reason="service_error")
- if result is None:
- return _EditorAssistRoute(enabled=True, result=None, reason="service_error")
- return _EditorAssistRoute(enabled=True, result=result, reason="served")
-
-
-def _route_editor_assist(
- team: Team,
- user: User | None,
- query: HogQLAutocomplete | HogQLMetadata,
- timings: HogQLTimings | None = None,
-) -> _EditorAssistRoute:
- with timings.measure("routing") if timings is not None else nullcontext():
- if user is None or not is_language_service_enabled(team, user):
- return _EditorAssistRoute(enabled=False, result=None, reason="ineligible")
- if not _language_service_eligible(query):
- return _EditorAssistRoute(enabled=True, result=None, reason="ineligible")
- return _language_service_call(team, user, query, timings=timings)
-
-
-def _capture_malformed_language_service_response(
- operation: _EditorAssistOperation, stage: _MalformedResponseStage
-) -> None:
- try:
- with posthoganalytics.new_context(fresh=True, capture_exceptions=False):
- posthoganalytics.set_capture_exception_code_variables_context(False)
- posthoganalytics.capture_exception(
- RuntimeError("HogQL language service returned a malformed response"),
- properties={
- "component": "hogql_language_service",
- "operation": operation,
- "stage": stage,
- },
- )
- except Exception:
- logger.warning("hogql_language_service_error_tracking_failed")
-
-
-def _record_editor_assist_backend(
- route: _EditorAssistRoute,
- operation: _EditorAssistOperation,
- backend: Literal["language_service", "python"],
- reason: _EditorAssistReason,
-) -> None:
- if not route.enabled:
- return
- try:
- EDITOR_ASSIST_RESPONSES_TOTAL.labels(operation=operation, backend=backend, reason=reason).inc()
- except Exception:
- logger.warning("hogql_editor_assist_metric_failed")
-
-
-def _autocomplete_response_from_language_service(
- language_result: LanguageServiceResult,
-) -> HogQLAutocompleteResponse:
- body = language_result.body
- if not isinstance(body.get("suggestions"), list):
- raise TypeError("suggestions must be a list")
- kind_map = {
- "field": "Field",
- "function": "Function",
- "keyword": "Keyword",
- "operator": "Operator",
- "property": "Property",
- "table": "Class",
- }
- return HogQLAutocompleteResponse(
- suggestions=[
- {
- "label": suggestion["label"],
- "insertText": suggestion.get("insertText", suggestion["label"]),
- "kind": kind_map.get(suggestion["kind"], "Text"),
- "detail": suggestion.get("detail"),
- "sortText": suggestion.get("sortText"),
- }
- for suggestion in body["suggestions"]
- ],
- incomplete_list=bool(body.get("nextCursor")),
- timings=[
- QueryTiming(k="language_service_http", t=language_result.duration_seconds),
- QueryTiming(k="language_service_go", t=body["durationMicros"] / 1_000_000),
- ],
- )
-
-
-def _metadata_response_from_language_service(
- query: HogQLMetadata, language_result: LanguageServiceResult
-) -> HogQLMetadataResponse:
- body = language_result.body
- if not isinstance(body.get("diagnostics"), list):
- raise TypeError("diagnostics must be a list")
- raw_notices = body.get("notices", [])
- if not isinstance(raw_notices, list):
- raise TypeError("notices must be a list")
- query_length_utf16 = len(query.query.encode("utf-16-le", errors="surrogatepass")) // 2
- errors: list[HogQLNotice] = []
- warnings: list[HogQLNotice] = []
- for diagnostic in body["diagnostics"]:
- notice = HogQLNotice(
- message=diagnostic["message"],
- start=diagnostic["start"],
- end=diagnostic["end"],
- fix=diagnostic["suggestions"][0]["label"] if diagnostic.get("suggestions") else None,
- )
- if diagnostic["code"] == "unknown_property":
- warnings.append(notice)
- else:
- errors.append(notice)
- notices: list[HogQLNotice] = []
- for raw_notice in raw_notices:
- if not isinstance(raw_notice, dict):
- raise TypeError("notice must be an object")
- message = raw_notice["message"]
- start = raw_notice["start"]
- end = raw_notice["end"]
- fix = raw_notice.get("fix")
- if (
- not isinstance(message, str)
- or not isinstance(start, int)
- or isinstance(start, bool)
- or not isinstance(end, int)
- or isinstance(end, bool)
- or start < 0
- or end <= start
- or end > query_length_utf16
- or (fix is not None and not isinstance(fix, str))
- ):
- raise TypeError("invalid notice")
- notices.append(HogQLNotice(message=message, start=start, end=end, fix=fix))
- return HogQLMetadataResponse(
- isValid=not errors,
- query=query.query,
- errors=errors,
- warnings=warnings,
- notices=notices,
- table_names=body.get("tableNames", []),
- )
-
-
@dataclass(frozen=True)
class RawCachedQueryResponse:
"""A cached query response whose `results` field is carried as raw JSON bytes.
@@ -476,12 +206,12 @@ def process_database_schema_query(
except Exception as error:
capture_exception(error, {"team_id": team.pk, "query_kind": "DatabaseSchemaQuery"})
raise DatabaseSchemaUnavailable() from error
- return _build_database_schema_query(team, query, user=user).response
+ return _build_database_schema_query(team, query, user=user)
def _build_database_schema_query(
team: Team, query: DatabaseSchemaQuery, *, user: Optional[User] = None
-) -> _DatabaseSchemaCatalog:
+) -> DatabaseSchemaQueryResponse:
try:
_, database = resolve_database_for_connection(
team,
@@ -530,9 +260,7 @@ def _build_database_schema_query(
)
)
- return _DatabaseSchemaCatalog(
- response=DatabaseSchemaQueryResponse(tables=serialized_tables, joins=join_models), database=database
- )
+ return DatabaseSchemaQueryResponse(tables=serialized_tables, joins=join_models)
@overload
@@ -602,43 +330,23 @@ def process_query_model(
timings = HogQLTimings()
with EDITOR_ASSIST_DURATION_SECONDS.labels(kind="autocomplete").time():
with timings.measure("editor_assist"):
- route = _route_editor_assist(team, user, query, timings=timings)
- python_reason = route.reason
- autocomplete_response: HogQLAutocompleteResponse | None = None
- if (language_result := route.result) is not None:
- try:
- with timings.measure("response_mapping"):
- autocomplete_response = _autocomplete_response_from_language_service(language_result)
- except (AttributeError, KeyError, TypeError, ValueError):
- logger.warning("hogql_language_service_invalid_autocomplete_response")
- python_reason = "invalid_response"
- else:
- _record_editor_assist_backend(route, "autocomplete", "language_service", "served")
- if autocomplete_response is None:
- with timings.measure("fallback_error_tracking"):
- _capture_malformed_language_service_response("autocomplete", "response_mapping")
- elif route.malformed_stage is not None:
- with timings.measure("fallback_error_tracking"):
- _capture_malformed_language_service_response("autocomplete", route.malformed_stage)
- if autocomplete_response is None:
- database: Database | None = None
- # Hog and template languages answer from globals, so building the schema is wasted work.
- if query.language in (HogLanguage.HOG_QL, HogLanguage.HOG_QL_EXPR):
- with timings.measure("fallback_database"):
- _, database = resolve_database_for_connection(
- team,
- query.connectionId,
- user=user,
- error_factory=ValidationError,
- modifiers=create_default_modifiers_for_team(team),
- # Editor-assist only: query execution never reads cached sources.
- use_cached_sources=True,
- )
- with timings.measure("fallback_python_autocomplete"):
- autocomplete_response = get_hogql_autocomplete(
- query=query, team=team, database_arg=database, user=user
+ database: Database | None = None
+ # Hog and template languages answer from globals, so building the schema is wasted work.
+ if query.language in (HogLanguage.HOG_QL, HogLanguage.HOG_QL_EXPR):
+ with timings.measure("database"):
+ _, database = resolve_database_for_connection(
+ team,
+ query.connectionId,
+ user=user,
+ error_factory=ValidationError,
+ modifiers=create_default_modifiers_for_team(team),
+ # Editor-assist only: query execution never reads cached sources.
+ use_cached_sources=True,
)
- _record_editor_assist_backend(route, "autocomplete", "python", python_reason)
+ with timings.measure("autocomplete"):
+ autocomplete_response = get_hogql_autocomplete(
+ query=query, team=team, database_arg=database, user=user
+ )
autocomplete_response.timings = [
*(autocomplete_response.timings or []),
*timings.to_list(back_out_stack=False),
@@ -647,24 +355,8 @@ def process_query_model(
if isinstance(query, HogQLMetadata):
with EDITOR_ASSIST_DURATION_SECONDS.labels(kind="metadata").time():
- route = _route_editor_assist(team, user, query)
- python_reason = route.reason
- if (language_result := route.result) is not None:
- try:
- metadata_response = _metadata_response_from_language_service(query, language_result)
- except (AttributeError, KeyError, TypeError, ValueError):
- logger.warning("hogql_language_service_invalid_metadata_response")
- python_reason = "invalid_response"
- else:
- _record_editor_assist_backend(route, "metadata", "language_service", "served")
- return metadata_response
- _capture_malformed_language_service_response("metadata", "response_mapping")
- elif route.malformed_stage is not None:
- _capture_malformed_language_service_response("metadata", route.malformed_stage)
metadata_query = HogQLMetadata.model_validate(query)
- metadata_response = get_hogql_metadata(query=metadata_query, team=team, user=user)
- _record_editor_assist_backend(route, "metadata", "python", python_reason)
- return metadata_response
+ return get_hogql_metadata(query=metadata_query, team=team, user=user)
if isinstance(query, DatabaseSchemaQuery):
return process_database_schema_query(team, query, user=user)
diff --git a/posthog/api/test/test_query_service.py b/posthog/api/test/test_query_service.py
index 1508ec982b51..6b3c37022ecb 100644
--- a/posthog/api/test/test_query_service.py
+++ b/posthog/api/test/test_query_service.py
@@ -1,21 +1,15 @@
-from collections.abc import Callable
from types import SimpleNamespace
from typing import cast
-from uuid import uuid4
from posthog.test.base import APIBaseTest
from unittest.mock import MagicMock, patch
from django.test import SimpleTestCase
-import posthoganalytics
from parameterized import parameterized
-from posthoganalytics.client import Client
-from prometheus_client import REGISTRY, CollectorRegistry, generate_latest
from rest_framework.exceptions import ValidationError
from posthog.schema import (
- AutocompleteCompletionItemKind,
DatabaseSchemaDataWarehouseTable,
DatabaseSchemaField,
DatabaseSchemaPostHogTable,
@@ -28,10 +22,7 @@ from posthog.schema import (
HogLanguage,
HogQLAutocomplete,
HogQLAutocompleteResponse,
- HogQLMetadata,
- HogQLMetadataResponse,
HogQLQuery,
- QueryTiming,
)
from posthog.hogql.database.database import Database
@@ -39,29 +30,11 @@ from posthog.hogql.database.models import TableNode
from posthog.hogql.database.postgres_table import PostgresTable
from posthog.hogql.direct_connection import INVALID_CONNECTION_ID_ERROR
from posthog.hogql.errors import ResolutionError
-from posthog.hogql.language_service import (
- CatalogMissing,
- LanguageServiceError,
- LanguageServiceResult,
- MalformedLanguageServiceResponse,
- build_catalog,
-)
-from posthog.api.services.query import (
- _build_database_schema_query,
- _capture_malformed_language_service_response,
- _DatabaseSchemaCatalog,
- _EditorAssistRoute,
- _language_service_call,
- _metadata_response_from_language_service,
- _record_editor_assist_backend,
- process_query_model,
-)
-from posthog.constants import AvailableFeature
+from posthog.api.services.query import process_query_model
from posthog.exceptions import DatabaseSchemaUnavailable
-from posthog.models import OrganizationMembership, Team, User
+from posthog.models import Team, User
-from products.access_control.backend.models.access_control import AccessControl
from products.data_tools.backend.models.expression import DataWarehouseExpression
from products.data_tools.backend.models.join import DataWarehouseJoin
from products.warehouse_sources.backend.facade.models import (
@@ -72,655 +45,25 @@ from products.warehouse_sources.backend.facade.models import (
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
-class TestLanguageServiceRouting(SimpleTestCase):
- @parameterized.expand(
- [
- (
- "new_payload",
- "SELECT '😀', event FROM events",
- [
- {"message": "Field 'event' is of type 'String'", "start": 13, "end": 18},
- {"message": "Table 'events'", "start": 24, "end": 30},
- ],
- (13, 18),
- (24, 30),
- ),
- (
- "lone_surrogate",
- "SELECT '\ud800', event FROM events",
- [
- {"message": "Field 'event' is of type 'String'", "start": 12, "end": 17},
- {"message": "Table 'events'", "start": 23, "end": 29},
- ],
- (12, 17),
- (23, 29),
- ),
- ("old_payload", "SELECT '😀', event FROM events", None, None, None),
- ]
- )
- def test_metadata_maps_optional_notices_with_utf16_positions(
- self,
- _name: str,
- query_text: str,
- raw_notices: list[dict[str, object]] | None,
- expected_field_span: tuple[int, int] | None,
- expected_table_span: tuple[int, int] | None,
- ) -> None:
- query = HogQLMetadata(query=query_text, language=HogLanguage.HOG_QL)
- body: dict[str, object] = {"valid": True, "diagnostics": [], "tableNames": ["events"]}
- if raw_notices is not None:
- body["notices"] = raw_notices
- response = _metadata_response_from_language_service(
- query, LanguageServiceResult(body=body, duration_seconds=0, response_size_bytes=0)
- )
-
- assert response.isValid is True
- assert len(response.notices or []) == len(raw_notices or [])
- if expected_field_span is not None:
- assert response.notices[0].message == "Field 'event' is of type 'String'"
- assert (response.notices[0].start, response.notices[0].end) == expected_field_span
- assert response.notices[0].fix is None
- assert response.notices[1].message == "Table 'events'"
- assert (response.notices[1].start, response.notices[1].end) == expected_table_span
-
- def test_served_backend_counter_is_exported_for_prometheus(self) -> None:
- labels = {"operation": "metadata", "backend": "python", "reason": "service_error"}
- before = REGISTRY.get_sample_value("hogql_editor_assist_responses_total", labels) or 0
-
- _record_editor_assist_backend(
- _EditorAssistRoute(enabled=True, result=None, reason="service_error"),
- "metadata",
- "python",
- "service_error",
- )
-
- assert REGISTRY.get_sample_value("hogql_editor_assist_responses_total", labels) == before + 1
- assert (
- b'hogql_editor_assist_responses_total{backend="python",operation="metadata",reason="service_error"}'
- in generate_latest(
- cast(CollectorRegistry, REGISTRY.restricted_registry(["hogql_editor_assist_responses_total"]))
- )
- )
-
- @patch("posthog.api.services.query._build_database_schema_query", side_effect=DatabaseSchemaUnavailable())
- @patch("posthog.hogql.language_service.get_client")
- @patch("posthog.api.services.query.LanguageServiceClient")
- def test_schema_refresh_failure_falls_back(
- self, client_class: MagicMock, get_redis_client: MagicMock, _build_schema: MagicMock
- ) -> None:
- client = client_class.return_value
- client.base_url = "http://language-service:8091"
- client.validate.side_effect = CatalogMissing("missing")
- get_redis_client.return_value.get.return_value = None
- get_redis_client.return_value.lock.return_value.acquire.return_value = True
-
- route = _language_service_call(
- cast(Team, SimpleNamespace(pk=12)),
- cast(User, SimpleNamespace(pk=34)),
- HogQLMetadata(query="SELECT event FROM events", language=HogLanguage.HOG_QL),
- )
-
- assert route.result is None
- assert route.reason == "service_error"
-
- def test_malformed_response_tracking_uses_sanitized_context(self) -> None:
- sentinel = uuid4().hex
- client = Client("test", send=False, log_captured_exceptions=True, capture_exception_code_variables=True)
- with (
- patch.object(client, "_enqueue", return_value=True) as enqueue,
- patch.object(client.log, "disabled", False),
- patch.object(posthoganalytics, "capture_exception", side_effect=client.capture_exception),
- ):
- with patch.object(client.log, "handle", wraps=client.log.handle) as handle_log:
- with posthoganalytics.new_context(fresh=True):
- posthoganalytics.tag("query", sentinel)
- try:
- raise ValueError(sentinel)
- except ValueError:
- pass
- _capture_malformed_language_service_response("metadata", "response_mapping")
-
- event = enqueue.call_args.args[0]
- properties = event["properties"]
- exception_values = properties["$exception_list"]
- assert exception_values[0]["value"] == "HogQL language service returned a malformed response"
- assert sentinel not in repr(event)
- records = [call.args[0] for call in handle_log.call_args_list]
- assert records
- assert sentinel not in repr([(record.getMessage(), record.exc_info) for record in records])
- assert all(record.exc_info == (None, None, None) for record in records)
- for exception in exception_values:
- for frame in exception["stacktrace"]["frames"]:
- assert "vars" not in frame
- assert "code_variables" not in frame
- assert {key: properties[key] for key in ("component", "operation", "stage")} == {
- "component": "hogql_language_service",
- "operation": "metadata",
- "stage": "response_mapping",
- }
-
- @parameterized.expand(
- [
- ("absent", None),
- ("matching", HogQLQuery(query="SELECT * FROM ev")),
- ("stale", HogQLQuery(query="SELECT distinct_id FROM persons LIMIT 5")),
- ("substring", HogQLQuery(query="SELECT * FROM events; SELECT 42")),
- ]
- )
- @patch("posthog.api.services.query.EDITOR_ASSIST_RESPONSES_TOTAL")
- @patch("posthog.api.services.query.is_language_service_enabled", return_value=True)
- @patch("posthog.api.services.query.LanguageServiceClient")
- @patch("posthog.hogql.timings.perf_counter")
- def test_hogql_autocomplete_uses_language_service_response(
- self,
- _name: str,
- source_query: HogQLQuery | None,
- perf_counter: MagicMock,
- client_class: MagicMock,
- enabled: MagicMock,
- analytics_client: MagicMock,
- ) -> None:
- now = [0.0]
- perf_counter.side_effect = lambda: now[0]
-
- def enable_language_service(*_args: object, **_kwargs: object) -> bool:
- now[0] += 0.05
- return True
-
- enabled.side_effect = enable_language_service
- language_result = LanguageServiceResult(
- body={
- "catalogRevision": "v2:cached",
- "suggestions": [
- {"label": "events", "kind": "table", "detail": "posthog"},
- {"label": "count", "kind": "function", "insertText": "count()", "sortText": "2-count"},
- {"label": "=", "kind": "operator", "insertText": "="},
- ],
- "durationMicros": 250,
- "nextCursor": "next",
- },
- duration_seconds=0.001,
- response_size_bytes=128,
- )
-
- def autocomplete(*_args: object) -> LanguageServiceResult:
- now[0] += 0.4
- return language_result
-
- client_class.return_value.autocomplete.side_effect = autocomplete
-
- query = HogQLAutocomplete(
- query="SELECT * FROM ev",
- language=HogLanguage.HOG_QL,
- startPosition=14,
- endPosition=16,
- sourceQuery=source_query,
- )
- original_query = query.model_dump()
- response = process_query_model(
- cast(Team, SimpleNamespace(pk=12, modifiers={})),
- query,
- user=cast(User, SimpleNamespace(pk=34)),
- )
-
- client_class.return_value.autocomplete.assert_called_once_with(12, 34, query.query, query.endPosition)
- assert query.model_dump() == original_query
- assert isinstance(response, HogQLAutocompleteResponse)
- assert response.suggestions[0].label == "events"
- assert response.suggestions[1].kind == AutocompleteCompletionItemKind.FUNCTION
- assert response.suggestions[1].insertText == "count()"
- assert response.suggestions[1].sortText == "2-count"
- assert response.suggestions[2].kind == AutocompleteCompletionItemKind.OPERATOR
- assert response.incomplete_list is True
- timing_values = {timing.k: timing.t for timing in response.timings or []}
- assert timing_values == {
- "language_service_http": 0.001,
- "language_service_go": 0.00025,
- "./editor_assist/routing": 0.05,
- "./editor_assist/language_service_initial": 0.4,
- "./editor_assist/response_mapping": 0.0,
- "./editor_assist": 0.45,
- }
- analytics_client.labels.assert_called_once_with(
- operation="autocomplete", backend="language_service", reason="served"
- )
- analytics_client.labels.return_value.inc.assert_called_once_with()
-
- @parameterized.expand([("published", False), ("publication_error", True)])
- def test_cold_autocomplete_reports_catalog_publication_stages(self, _name: str, publication_fails: bool) -> None:
- now = [0.0]
- language_result = LanguageServiceResult(
- body={
- "catalogRevision": "v2:published",
- "suggestions": [],
- "durationMicros": 10_000,
- },
- duration_seconds=0.05,
- response_size_bytes=128,
- )
- client = MagicMock()
- client.base_url = "http://language-service:8091"
-
- def autocomplete(*_args: object) -> LanguageServiceResult:
- now[0] += 0.1
- if client.autocomplete.call_count < 3:
- raise CatalogMissing("missing")
- return language_result
-
- def advance(duration: float, result: object = None) -> Callable[..., object]:
- def callback(*_args: object, **_kwargs: object) -> object:
- now[0] += duration
- return result
-
- return callback
-
- client.autocomplete.side_effect = autocomplete
-
- def publish(*_args: object, **_kwargs: object) -> None:
- now[0] += 0.6
- if publication_fails:
- raise LanguageServiceError("unavailable")
-
- client.publish.side_effect = publish
- redis_client = MagicMock()
- redis_client.get.side_effect = advance(0.02)
- redis_client.lock.return_value.acquire.side_effect = advance(0.3, True)
- redis_client.set.side_effect = advance(0.02)
- redis_client.lock.return_value.release.side_effect = advance(0.01)
- schema_catalog = _DatabaseSchemaCatalog(response=MagicMock(), database=MagicMock())
-
- python_response = HogQLAutocompleteResponse(suggestions=[], incomplete_list=False)
-
- def resolve_database(*_args: object, **_kwargs: object) -> tuple[None, MagicMock]:
- now[0] += 0.3
- return None, MagicMock()
-
- def autocomplete_in_python(*_args: object, **_kwargs: object) -> HogQLAutocompleteResponse:
- now[0] += 0.4
- return python_response
-
- with (
- patch("posthog.hogql.timings.perf_counter", side_effect=lambda: now[0]),
- patch("posthog.api.services.query.is_language_service_enabled", return_value=True),
- patch("posthog.api.services.query.LanguageServiceClient", return_value=client),
- patch("posthog.hogql.language_service.get_client", return_value=redis_client),
- patch(
- "posthog.api.services.query._build_database_schema_query",
- side_effect=advance(0.4, schema_catalog),
- ),
- patch("posthog.api.services.query.build_catalog", side_effect=advance(0.5, {"tables": {}})),
- patch("posthog.api.services.query.resolve_database_for_connection", side_effect=resolve_database),
- patch("posthog.api.services.query.get_hogql_autocomplete", side_effect=autocomplete_in_python),
- patch("posthog.api.services.query.create_default_modifiers_for_team", return_value=MagicMock()),
- patch("posthog.api.services.query.EDITOR_ASSIST_RESPONSES_TOTAL"),
- ):
- response = process_query_model(
- cast(Team, SimpleNamespace(pk=12, modifiers={})),
- HogQLAutocomplete(
- query="SELECT * FROM ev",
- language=HogLanguage.HOG_QL,
- startPosition=14,
- endPosition=16,
- ),
- user=cast(User, SimpleNamespace(pk=34)),
- )
-
- assert isinstance(response, HogQLAutocompleteResponse)
- timing_values = {timing.k: timing.t for timing in response.timings or []}
- if publication_fails:
- assert "language_service_http" not in timing_values
- assert "language_service_go" not in timing_values
- else:
- assert timing_values["language_service_http"] == 0.05
- assert timing_values["language_service_go"] == 0.01
- assert timing_values["./editor_assist/language_service_initial"] == 0.1
- assert timing_values["./editor_assist/catalog_coordination/redis_marker_lookup"] == 0.02
- assert timing_values["./editor_assist/catalog_coordination/redis_lock_acquire"] == 0.3
- assert timing_values["./editor_assist/catalog_coordination/language_service_check"] == (
- 0.1 if publication_fails else 0.2
- )
- assert timing_values["./editor_assist/catalog_coordination/catalog_schema"] == 0.4
- assert timing_values["./editor_assist/catalog_coordination/catalog_build"] == 0.5
- assert timing_values["./editor_assist/catalog_coordination/catalog_publish"] == 0.6
- if publication_fails:
- assert "./editor_assist/catalog_coordination/redis_marker_write" not in timing_values
- assert timing_values["./editor_assist/fallback_database"] == 0.3
- assert timing_values["./editor_assist/fallback_python_autocomplete"] == 0.4
- else:
- assert timing_values["./editor_assist/catalog_coordination/redis_marker_write"] == 0.02
- assert timing_values["./editor_assist/catalog_coordination/redis_lock_release"] == 0.01
- assert timing_values["./editor_assist/catalog_coordination"] == (1.93 if publication_fails else 2.05)
- assert timing_values["./editor_assist"] == (2.73 if publication_fails else 2.15)
-
+class TestEditorAssistDatabase(SimpleTestCase):
@parameterized.expand(
[
- ("without_source", None, False),
- ("with_stale_source", HogQLQuery(query="SELECT event FROM events WHERE"), False),
- ("with_stale_source_and_indexes", HogQLQuery(query="SELECT event FROM events WHERE"), True),
- ]
- )
- @patch("posthog.api.services.query.EDITOR_ASSIST_RESPONSES_TOTAL")
- @patch("posthog.api.services.query.is_language_service_enabled", return_value=True)
- @patch("posthog.api.services.query.LanguageServiceClient")
- def test_hogql_metadata_with_source_context_uses_language_service_result(
- self,
- _name: str,
- source_query: HogQLQuery | None,
- index_usage: bool,
- client_class: MagicMock,
- _enabled: MagicMock,
- responses_total: MagicMock,
- ) -> None:
- client_class.return_value.validate.return_value = LanguageServiceResult(
- body={
- "catalogRevision": "v2:cached",
- "valid": False,
- "diagnostics": [
- {
- "code": "unknown_table",
- "message": 'Unknown table "evnts"',
- "start": 30,
- "end": 35,
- "suggestions": [{"label": "events", "distance": 1}],
- }
- ],
- "notices": [
- {"message": "Field 'event' is of type 'String'", "start": 7, "end": 12},
- {"message": "Table 'events'", "start": 18, "end": 24},
- ],
- "tableNames": ["events", "evnts"],
- },
- duration_seconds=0.001,
- response_size_bytes=128,
- )
-
- query = HogQLMetadata(
- query="SELECT event FROM events JOIN evnts ON 1 = 1",
- language=HogLanguage.HOG_QL,
- sourceQuery=source_query,
- indexUsage=index_usage,
- )
- original_query = query.model_dump()
- response = process_query_model(
- cast(Team, SimpleNamespace(pk=12)),
- query,
- user=cast(User, SimpleNamespace(pk=34)),
- )
-
- client_class.return_value.validate.assert_called_once_with(12, 34, query.query)
- assert query.model_dump() == original_query
- assert isinstance(response, HogQLMetadataResponse)
- assert response.isValid is False
- assert response.errors[0].message == 'Unknown table "evnts"'
- assert response.errors[0].fix == "events"
- assert [(notice.message, notice.start, notice.end) for notice in response.notices] == [
- ("Field 'event' is of type 'String'", 7, 12),
- ("Table 'events'", 18, 24),
- ]
- assert response.query == query.query
- assert response.table_names == ["events", "evnts"]
- assert response.index_usage is None
- assert response.isUsingIndices is None
- assert "timings" not in response.model_dump()
- responses_total.labels.assert_called_once_with(
- operation="metadata", backend="language_service", reason="served"
- )
- responses_total.labels.return_value.inc.assert_called_once_with()
-
- @parameterized.expand(
- [
- ("service_error", LanguageServiceError("unavailable"), "service_error", None),
- (
- "invalid_http",
- MalformedLanguageServiceResponse("malformed"),
- "invalid_response",
- "http_response",
- ),
- ]
- )
- @patch("posthog.api.services.query._capture_malformed_language_service_response")
- @patch("posthog.api.services.query.EDITOR_ASSIST_RESPONSES_TOTAL")
- @patch("posthog.api.services.query.get_hogql_metadata")
- @patch("posthog.api.services.query.is_language_service_enabled", return_value=True)
- @patch("posthog.api.services.query.LanguageServiceClient")
- def test_metadata_fallback_records_the_serving_backend_once(
- self,
- _name: str,
- failure: Exception,
- reason: str,
- malformed_stage: str | None,
- client_class: MagicMock,
- enabled: MagicMock,
- python_metadata: MagicMock,
- analytics_client: MagicMock,
- capture_malformed: MagicMock,
- ) -> None:
- client_class.return_value.validate.side_effect = failure
- python_metadata.return_value = HogQLMetadataResponse(
- isValid=True,
- query="SELECT event FROM events",
- errors=[],
- warnings=[],
- notices=[],
- table_names=["events"],
- )
-
- query = HogQLMetadata(
- query="SELECT event FROM events",
- language=HogLanguage.HOG_QL,
- sourceQuery=HogQLQuery(query="SELECT distinct_id FROM events"),
- indexUsage=True,
- )
- original_query = query.model_dump()
- response = process_query_model(
- cast(Team, SimpleNamespace(pk=12)),
- query,
- user=cast(User, SimpleNamespace(pk=34)),
- )
-
- assert response is python_metadata.return_value
- assert python_metadata.call_args.kwargs["query"] is query
- assert query.model_dump() == original_query
- client_class.return_value.validate.assert_called_once_with(12, 34, query.query)
- analytics_client.labels.assert_called_once_with(operation="metadata", backend="python", reason=reason)
- analytics_client.labels.return_value.inc.assert_called_once_with()
- enabled.assert_called_once()
- if malformed_stage is None:
- capture_malformed.assert_not_called()
- else:
- capture_malformed.assert_called_once_with("metadata", malformed_stage)
-
- @parameterized.expand(
- [
- ("diagnostics_collection", {"diagnostics": {}}, "SELECT event FROM events"),
- ("diagnostics_nested", {"diagnostics": [None]}, "SELECT event FROM events"),
- ("notices_collection", {"notices": {}}, "SELECT event FROM events"),
- ("notices_nested", {"notices": [None]}, "SELECT event FROM events"),
- ("notices_missing_message", {"notices": [{"start": 7, "end": 12}]}, "SELECT event FROM events"),
- ("notices_missing_start", {"notices": [{"message": "field", "end": 12}]}, "SELECT event FROM events"),
- ("notices_missing_end", {"notices": [{"message": "field", "start": 7}]}, "SELECT event FROM events"),
- (
- "notices_out_of_range",
- {"notices": [{"message": "field", "start": 7, "end": 999}]},
- "SELECT event FROM events",
- ),
- ]
- )
- @patch("posthog.api.services.query._capture_malformed_language_service_response")
- @patch("posthog.api.services.query.EDITOR_ASSIST_RESPONSES_TOTAL")
- @patch("posthog.api.services.query.get_hogql_metadata")
- @patch("posthog.api.services.query.is_language_service_enabled", return_value=True)
- @patch("posthog.api.services.query.LanguageServiceClient")
- def test_malformed_metadata_mapping_falls_back(
- self,
- _name: str,
- payload: dict[str, object],
- query_text: str,
- client_class: MagicMock,
- enabled: MagicMock,
- python_metadata: MagicMock,
- analytics_client: MagicMock,
- capture_malformed: MagicMock,
- ) -> None:
- client_class.return_value.validate.return_value = LanguageServiceResult(
- body={"catalogRevision": "v2:cached", "diagnostics": [], **payload},
- duration_seconds=0,
- response_size_bytes=0,
- )
- python_metadata.return_value = HogQLMetadataResponse(
- isValid=True,
- query=query_text,
- errors=[],
- warnings=[],
- notices=[],
- table_names=["events"],
- )
-
- response = process_query_model(
- cast(Team, SimpleNamespace(pk=12)),
- HogQLMetadata(query=query_text, language=HogLanguage.HOG_QL),
- user=cast(User, SimpleNamespace(pk=34)),
- )
-
- assert response is python_metadata.return_value
- capture_malformed.assert_called_once_with("metadata", "response_mapping")
- analytics_client.labels.assert_called_once_with(
- operation="metadata", backend="python", reason="invalid_response"
- )
- analytics_client.labels.return_value.inc.assert_called_once_with()
- enabled.assert_called_once()
-
- @parameterized.expand(
- [
- ("collection", {}, "invalid_response"),
- ("nested", [None], "invalid_response"),
- ("unavailable", [], "service_error"),
- ]
- )
- @patch("posthog.api.services.query._capture_malformed_language_service_response")
- @patch("posthog.api.services.query.EDITOR_ASSIST_RESPONSES_TOTAL")
- @patch("posthog.api.services.query.create_default_modifiers_for_team", return_value=MagicMock())
- @patch("posthog.api.services.query.resolve_database_for_connection", return_value=(None, MagicMock()))
- @patch("posthog.api.services.query.get_hogql_autocomplete")
- @patch("posthog.api.services.query.is_language_service_enabled", return_value=True)
- @patch("posthog.api.services.query.LanguageServiceClient")
- @patch("posthog.hogql.timings.perf_counter")
- def test_autocomplete_failure_preserves_source_context_on_fallback(
- self,
- _name: str,
- suggestions: object,
- reason: str,
- perf_counter: MagicMock,
- client_class: MagicMock,
- _enabled: MagicMock,
- python_autocomplete: MagicMock,
- _resolve_database: MagicMock,
- _modifiers: MagicMock,
- analytics_client: MagicMock,
- capture_malformed: MagicMock,
- ) -> None:
- now = [0.0]
- perf_counter.side_effect = lambda: now[0]
- language_result = LanguageServiceResult(
- body={
- "catalogRevision": "v2:cached",
- "suggestions": suggestions,
- "durationMicros": 1,
- },
- duration_seconds=0,
- response_size_bytes=0,
- )
-
- def autocomplete(*_args: object) -> LanguageServiceResult:
- now[0] += 0.2
- if reason == "service_error":
- raise LanguageServiceError("unavailable")
- return language_result
-
- def resolve_database(*_args: object, **_kwargs: object) -> tuple[None, MagicMock]:
- now[0] += 0.3
- return None, MagicMock()
-
- python_response = HogQLAutocompleteResponse(
- suggestions=[],
- incomplete_list=False,
- timings=[QueryTiming(k="./parse_select", t=0.15)],
- )
-
- def autocomplete_in_python(*_args: object, **_kwargs: object) -> HogQLAutocompleteResponse:
- now[0] += 0.4
- return python_response
-
- def capture_error(*_args: object, **_kwargs: object) -> None:
- now[0] += 0.1
-
- client_class.return_value.autocomplete.side_effect = autocomplete
- _resolve_database.side_effect = resolve_database
- python_autocomplete.side_effect = autocomplete_in_python
- capture_malformed.side_effect = capture_error
-
- query = HogQLAutocomplete(
- query="SELECT event FROM events",
- language=HogLanguage.HOG_QL,
- startPosition=6,
- endPosition=6,
- sourceQuery=HogQLQuery(query="SELECT distinct_id FROM persons LIMIT 5"),
- )
- original_query = query.model_dump()
- response = process_query_model(
- cast(Team, SimpleNamespace(pk=12, modifiers={})),
- query,
- user=cast(User, SimpleNamespace(pk=34)),
- )
-
- assert response is python_response
- assert python_autocomplete.call_args.kwargs["query"] is query
- assert query.model_dump() == original_query
- client_class.return_value.autocomplete.assert_called_once_with(12, 34, query.query, query.endPosition)
- if reason == "invalid_response":
- capture_malformed.assert_called_once_with("autocomplete", "response_mapping")
- else:
- capture_malformed.assert_not_called()
- analytics_client.labels.assert_called_once_with(operation="autocomplete", backend="python", reason=reason)
- analytics_client.labels.return_value.inc.assert_called_once_with()
- timing_values = {timing.k: timing.t for timing in response.timings or []}
- assert timing_values["./parse_select"] == 0.15
- assert timing_values["./editor_assist/language_service_initial"] == 0.2
- assert timing_values["./editor_assist/fallback_database"] == 0.3
- assert timing_values["./editor_assist/fallback_python_autocomplete"] == 0.4
- if reason == "invalid_response":
- assert timing_values["./editor_assist/fallback_error_tracking"] == 0.1
- assert timing_values["./editor_assist"] == 1.0
- else:
- assert "./editor_assist/fallback_error_tracking" not in timing_values
- assert timing_values["./editor_assist"] == 0.9
-
- @parameterized.expand(
- [
- ("expression", {"language": "hogQLExpr"}, True),
- ("non_sql_source", {"sourceQuery": {"kind": "EventsNode"}}, True),
- ("connection", {"connectionId": "example-connection"}, True),
- ("globals", {"globals": {}}, True),
- ("filters", {"filters": {}}, True),
- ("modifiers", {"modifiers": {}}, True),
- ("hog", {"language": "hog", "globals": {"event": {}}}, False),
- ("hog_template", {"language": "hogTemplate", "globals": {"event": {}}}, False),
- ("liquid", {"language": "liquid", "globals": {"event": {}}}, False),
- ("hog_json", {"language": "hogJson", "globals": {"event": {}}}, False),
+ ("hogql", "hogQL", True),
+ ("expression", "hogQLExpr", True),
+ ("hog", "hog", False),
+ ("hog_template", "hogTemplate", False),
+ ("liquid", "liquid", False),
+ ("hog_json", "hogJson", False),
]
)
@patch("posthog.api.services.query.create_default_modifiers_for_team")
@patch("posthog.api.services.query.resolve_database_for_connection", return_value=(None, MagicMock()))
@patch("posthog.api.services.query.get_hogql_autocomplete")
- @patch("posthog.api.services.query.is_language_service_enabled", return_value=True)
- @patch("posthog.api.services.query.LanguageServiceClient")
- def test_context_dependent_autocomplete_uses_python(
+ def test_autocomplete_builds_the_database_only_for_sql_languages(
self,
_name: str,
- context: dict[str, object],
+ language: str,
builds_database: bool,
- client_class: MagicMock,
- _enabled: MagicMock,
python_autocomplete: MagicMock,
resolve_database: MagicMock,
_modifiers: MagicMock,
@@ -728,11 +71,10 @@ class TestLanguageServiceRouting(SimpleTestCase):
query = HogQLAutocomplete.model_validate(
{
"query": "SELECT event FROM events",
- "language": "hogQL",
+ "language": language,
"startPosition": 6,
"endPosition": 6,
- "sourceQuery": {"kind": "HogQLQuery", "query": "SELECT distinct_id FROM persons"},
- **context,
+ "globals": None if builds_database else {"event": {}},
}
)
python_autocomplete.return_value = HogQLAutocompleteResponse(suggestions=[], incomplete_list=False)
@@ -744,439 +86,14 @@ class TestLanguageServiceRouting(SimpleTestCase):
)
assert response is python_autocomplete.return_value
- assert python_autocomplete.call_args.kwargs["query"] is query
- client_class.assert_not_called()
- timing_values = {timing.k: timing.t for timing in response.timings or []}
- assert "./editor_assist/routing" in timing_values
+ timing_values = {timing.k for timing in response.timings or []}
assert resolve_database.called is builds_database
assert (python_autocomplete.call_args.kwargs["database_arg"] is None) is not builds_database
- assert ("./editor_assist/fallback_database" in timing_values) is builds_database
- assert "./editor_assist/fallback_python_autocomplete" in timing_values
- assert "./editor_assist" in timing_values
-
- @patch("posthog.api.services.query.posthoganalytics.capture_exception", side_effect=RuntimeError("tracking failed"))
- @patch("posthog.api.services.query.EDITOR_ASSIST_RESPONSES_TOTAL")
- @patch("posthog.api.services.query.get_hogql_metadata")
- @patch("posthog.api.services.query._route_editor_assist")
- def test_telemetry_failures_do_not_prevent_python_fallback(
- self,
- route: MagicMock,
- python_metadata: MagicMock,
- analytics_client: MagicMock,
- _capture: MagicMock,
- ) -> None:
- route.return_value = _EditorAssistRoute(
- enabled=True,
- result=LanguageServiceResult(body={"diagnostics": {}}, duration_seconds=0, response_size_bytes=0),
- reason="served",
- )
- python_metadata.return_value = HogQLMetadataResponse(
- isValid=True,
- query="SELECT event FROM events",
- errors=[],
- warnings=[],
- notices=[],
- table_names=["events"],
- )
- analytics_client.labels.return_value.inc.side_effect = RuntimeError("metrics failed")
-
- response = process_query_model(
- cast(Team, SimpleNamespace(pk=12)),
- HogQLMetadata(query="SELECT event FROM events", language=HogLanguage.HOG_QL),
- user=cast(User, SimpleNamespace(pk=34)),
- )
-
- assert response is python_metadata.return_value
-
- @parameterized.expand([("disabled", False, object()), ("no_user", True, None)])
- @patch("posthog.api.services.query.EDITOR_ASSIST_RESPONSES_TOTAL")
- @patch("posthog.api.services.query.get_hogql_metadata")
- @patch("posthog.api.services.query.is_language_service_enabled")
- def test_metadata_metric_excludes_requests_outside_the_enabled_denominator(
- self,
- _name: str,
- flag_enabled: bool,
- user: object | None,
- enabled: MagicMock,
- python_metadata: MagicMock,
- analytics_client: MagicMock,
- ) -> None:
- enabled.return_value = flag_enabled
- python_metadata.return_value = HogQLMetadataResponse(
- isValid=True,
- query="SELECT event FROM events",
- errors=[],
- warnings=[],
- notices=[],
- table_names=["events"],
- )
-
- process_query_model(
- cast(Team, SimpleNamespace()),
- HogQLMetadata(query="SELECT event FROM events", language=HogLanguage.HOG_QL),
- user=cast(User | None, user),
- )
-
- analytics_client.labels.assert_not_called()
- if user is None:
- enabled.assert_not_called()
- else:
- enabled.assert_called_once()
-
- @parameterized.expand(
- [
- ("debug", {"debug": True}),
- ("output_types", {"includeOutputTypes": True}),
- (
- "expression",
- {
- "language": "hogQLExpr",
- "sourceQuery": {"kind": "HogQLQuery", "query": "SELECT event FROM events"},
- },
- ),
- ("non_sql_source", {"sourceQuery": {"kind": "EventsNode"}}),
- ("variables", {"variables": {}}),
- ]
- )
- @patch("posthog.api.services.query.LanguageServiceClient")
- @patch("posthog.api.services.query.EDITOR_ASSIST_RESPONSES_TOTAL")
- @patch("posthog.api.services.query.get_hogql_metadata")
- @patch("posthog.api.services.query.is_language_service_enabled", return_value=True)
- def test_ineligible_metadata_records_python_backend(
- self,
- _name: str,
- context: dict[str, object],
- enabled: MagicMock,
- python_metadata: MagicMock,
- analytics_client: MagicMock,
- client_class: MagicMock,
- ) -> None:
- python_metadata.return_value = HogQLMetadataResponse(
- isValid=True,
- query="SELECT event FROM events",
- errors=[],
- warnings=[],
- notices=[],
- table_names=["events"],
- )
-
- process_query_model(
- cast(Team, SimpleNamespace()),
- HogQLMetadata.model_validate({"query": "SELECT event FROM events", "language": "hogQL", **context}),
- user=cast(User, SimpleNamespace()),
- )
-
- analytics_client.labels.assert_called_once_with(operation="metadata", backend="python", reason="ineligible")
- client_class.assert_not_called()
- analytics_client.labels.return_value.inc.assert_called_once_with()
- enabled.assert_called_once()
-
- @patch("posthog.api.services.query.EDITOR_ASSIST_RESPONSES_TOTAL")
- @patch("posthog.api.services.query.get_hogql_metadata", side_effect=RuntimeError("failed"))
- @patch("posthog.api.services.query.is_language_service_enabled", return_value=True)
- def test_failed_python_fallback_does_not_record_a_serving_backend(
- self, _enabled: MagicMock, _python_metadata: MagicMock, analytics_client: MagicMock
- ) -> None:
- with self.assertRaises(RuntimeError):
- process_query_model(
- cast(Team, SimpleNamespace()),
- HogQLMetadata(query="SELECT event FROM events", language=HogLanguage.HOG_QL, debug=True),
- user=cast(User, SimpleNamespace()),
- )
-
- analytics_client.labels.assert_not_called()
-
- @patch("posthog.api.services.query._route_editor_assist")
- def test_unknown_properties_remain_warnings(self, mock_language_service_call: MagicMock) -> None:
- mock_language_service_call.return_value = _EditorAssistRoute(
- enabled=True,
- reason="served",
- result=LanguageServiceResult(
- body={
- "valid": False,
- "diagnostics": [
- {
- "code": "unknown_property",
- "message": 'Unknown property "missing"',
- "start": 7,
- "end": 14,
- }
- ],
- "tableNames": ["events"],
- },
- duration_seconds=0.001,
- response_size_bytes=128,
- ),
- )
-
- response = process_query_model(
- cast(Team, SimpleNamespace()),
- HogQLMetadata(query="SELECT missing FROM events", language=HogLanguage.HOG_QL),
- user=cast(User, SimpleNamespace()),
- )
-
- assert isinstance(response, HogQLMetadataResponse)
- assert response.isValid is True
- assert response.errors == []
- assert response.warnings[0].message == 'Unknown property "missing"'
-
- @patch("posthog.api.services.query.is_language_service_enabled", return_value=True)
- @patch("posthog.hogql.language_service.get_client")
- @patch("posthog.api.services.query.LanguageServiceClient")
- def test_accepts_a_cached_traversal_catalog(
- self,
- client_class: MagicMock,
- get_redis_client: MagicMock,
- _enabled: MagicMock,
- ) -> None:
- client_class.return_value.validate.return_value = LanguageServiceResult(
- body={"valid": True, "catalogRevision": "v2:cached"},
- duration_seconds=0,
- response_size_bytes=0,
- )
-
- result = _language_service_call(
- cast(Team, SimpleNamespace(pk=12)),
- cast(User, SimpleNamespace(pk=34)),
- HogQLMetadata(query="SELECT 1", language=HogLanguage.HOG_QL),
- )
-
- assert result.result is not None
- get_redis_client.assert_not_called()
- client_class.return_value.publish.assert_not_called()
-
- @parameterized.expand(
- [
- ("legacy-v1:cached",),
- ("warehouse-aliases-v1:cached",),
- ("warehouse-aliases-v1:traversals-v1:cached",),
- ("v1:cached",),
- ("1789766573113832612",),
- (None,),
- ]
- )
- @patch("posthog.api.services.query.build_catalog", return_value={"tables": {}, "properties": {}})
- @patch("posthog.api.services.query._build_database_schema_query")
- @patch("posthog.api.services.query.is_language_service_enabled", return_value=True)
- @patch("posthog.hogql.language_service.get_client")
- @patch("posthog.api.services.query.LanguageServiceClient")
- def test_refreshes_a_missing_or_legacy_catalog_once(
- self,
- cached_revision: str | None,
- client_class: MagicMock,
- get_redis_client: MagicMock,
- _enabled: MagicMock,
- build_schema: MagicMock,
- build_catalog_mock: MagicMock,
- ) -> None:
- client = client_class.return_value
- client.base_url = "http://language-service:8091"
- get_redis_client.return_value.get.return_value = None
- get_redis_client.return_value.lock.return_value.acquire.return_value = True
- first = (
- CatalogMissing("missing")
- if cached_revision is None
- else LanguageServiceResult(
- body={"valid": True, "catalogRevision": cached_revision}, duration_seconds=0, response_size_bytes=0
- )
- )
- published_revision: list[str] = []
-
- def publish(_team_id: int, _user_id: int, revision: str, _catalog: dict[str, object]) -> None:
- published_revision.append(revision)
-
- client.publish.side_effect = publish
-
- def validate(*_args: object) -> LanguageServiceResult:
- if client.validate.call_count <= 2:
- if isinstance(first, Exception):
- raise first
- return first
- return LanguageServiceResult(
- body={"valid": True, "catalogRevision": published_revision[0]},
- duration_seconds=0,
- response_size_bytes=0,
- )
-
- client.validate.side_effect = validate
- build_schema.return_value = _DatabaseSchemaCatalog(response=MagicMock(), database=MagicMock())
-
- result = _language_service_call(
- cast(Team, SimpleNamespace(pk=12)),
- cast(User, SimpleNamespace(pk=34)),
- HogQLMetadata(query="SELECT 1", language=HogLanguage.HOG_QL),
- )
-
- assert result.result is not None
- assert published_revision[0].startswith("v2:")
- assert client.validate.call_count == 3
- assert build_catalog_mock.call_args.kwargs["database"] is build_schema.return_value.database
-
- @patch("posthog.api.services.query.build_catalog", return_value={"tableAliases": {"alias": "canonical"}})
- @patch("posthog.api.services.query._build_database_schema_query")
- @patch("posthog.api.services.query.is_language_service_enabled", return_value=True)
- @patch("posthog.hogql.language_service.get_client")
- @patch("posthog.api.services.query.LanguageServiceClient")
- def test_falls_back_when_alias_publication_is_not_supported(
- self,
- client_class: MagicMock,
- get_redis_client: MagicMock,
- _enabled: MagicMock,
- build_schema: MagicMock,
- _build_catalog: MagicMock,
- ) -> None:
- client = client_class.return_value
- client.base_url = "http://language-service:8091"
- client.validate.return_value = LanguageServiceResult(
- body={"valid": True, "catalogRevision": "legacy-v1:cached"},
- duration_seconds=0,
- response_size_bytes=0,
- )
- client.publish.side_effect = LanguageServiceError("language service returned 400")
- get_redis_client.return_value.get.return_value = None
- get_redis_client.return_value.lock.return_value.acquire.return_value = True
- build_schema.return_value = _DatabaseSchemaCatalog(response=MagicMock(), database=MagicMock())
-
- result = _language_service_call(
- cast(Team, SimpleNamespace(pk=12)),
- cast(User, SimpleNamespace(pk=34)),
- HogQLMetadata(query="SELECT 1", language=HogLanguage.HOG_QL),
- )
-
- assert result.result is None
- client_class.return_value.publish.assert_called_once()
-
- @parameterized.expand(
- [
- ("v2:concurrent", True, "served", None),
- ("warehouse-aliases-v1:traversals-v1:concurrent", False, "invalid_response", "http_response"),
- ("legacy-v1:other", False, "invalid_response", "http_response"),
- (None, False, "invalid_response", "http_response"),
- (123, False, "invalid_response", "http_response"),
- ]
- )
- @patch("posthog.api.services.query.build_catalog", return_value={"tables": {}, "properties": {}})
- @patch("posthog.api.services.query._build_database_schema_query")
- @patch("posthog.api.services.query.is_language_service_enabled", return_value=True)
- @patch("posthog.hogql.language_service.get_client")
- @patch("posthog.api.services.query.LanguageServiceClient")
- def test_classifies_the_retry_catalog_revision(
- self,
- retry_revision: object,
- expected_success: bool,
- expected_reason: str,
- expected_malformed_stage: str | None,
- client_class: MagicMock,
- get_redis_client: MagicMock,
- _enabled: MagicMock,
- build_schema: MagicMock,
- _build_catalog: MagicMock,
- ) -> None:
- client = client_class.return_value
- client.base_url = "http://language-service:8091"
- client.validate.side_effect = [
- LanguageServiceResult(
- body={"valid": True, "catalogRevision": "legacy-v1:cached"},
- duration_seconds=0,
- response_size_bytes=0,
- ),
- LanguageServiceResult(
- body={"valid": True, "catalogRevision": "legacy-v1:cached"},
- duration_seconds=0,
- response_size_bytes=0,
- ),
- LanguageServiceResult(
- body={"valid": True, "catalogRevision": retry_revision},
- duration_seconds=0,
- response_size_bytes=0,
- ),
- ]
- get_redis_client.return_value.get.return_value = None
- get_redis_client.return_value.lock.return_value.acquire.return_value = True
- build_schema.return_value = _DatabaseSchemaCatalog(response=MagicMock(), database=MagicMock())
-
- result = _language_service_call(
- cast(Team, SimpleNamespace(pk=12)),
- cast(User, SimpleNamespace(pk=34)),
- HogQLMetadata(query="SELECT 1", language=HogLanguage.HOG_QL),
- )
-
- assert (result.result is not None) is expected_success
- assert result.reason == expected_reason
- assert result.malformed_stage == expected_malformed_stage
- client_class.return_value.publish.assert_called_once()
- assert client_class.return_value.validate.call_count == 3
+ assert ("./editor_assist/database" in timing_values) is builds_database
+ assert "./editor_assist/autocomplete" in timing_values
class TestQueryService(APIBaseTest):
- @patch("posthog.hogql.language_service._properties_for_namespace", return_value=[])
- @patch("posthog.hogql.database.database.feature_enabled_or_false", return_value=True)
- def test_alias_publication_uses_the_permission_filtered_database(
- self, _feature_enabled: MagicMock, _properties: MagicMock
- ) -> None:
- self.organization.available_product_features = [
- {"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}
- ]
- self.organization.save()
- membership = OrganizationMembership.objects.get(user=self.user, organization=self.organization)
- membership.level = OrganizationMembership.Level.MEMBER
- membership.save()
-
- credential = DataWarehouseCredential.objects.create(access_key="key", access_secret="secret", team=self.team)
- source = ExternalDataSource.objects.create(
- team=self.team, source_type=ExternalDataSourceType.POSTGRES, prefix="demo"
- )
- allowed = DataWarehouseTable.objects.create(
- team=self.team,
- name="demo_postgres_orders",
- external_data_source=source,
- credential=credential,
- format="Parquet",
- url_pattern="https://example.com/orders/*",
- columns={"id": {"hogql": "StringDatabaseField", "clickhouse": "String"}},
- )
- denied = DataWarehouseTable.objects.create(
- team=self.team,
- name="demo_postgres_private_orders",
- external_data_source=source,
- credential=credential,
- format="Parquet",
- url_pattern="https://example.com/private-orders/*",
- columns={"id": {"hogql": "StringDatabaseField", "clickhouse": "String"}},
- )
- AccessControl.objects.create(
- team=self.team, resource="warehouse_table", resource_id=str(denied.id), access_level="none"
- )
-
- other_team = Team.objects.create(organization=self.organization, name="Other project")
- other_credential = DataWarehouseCredential.objects.create(
- access_key="key", access_secret="secret", team=other_team
- )
- other_source = ExternalDataSource.objects.create(
- team=other_team, source_type=ExternalDataSourceType.POSTGRES, prefix="other"
- )
- DataWarehouseTable.objects.create(
- team=other_team,
- name="other_postgres_orders",
- external_data_source=other_source,
- credential=other_credential,
- format="Parquet",
- url_pattern="https://example.com/other-orders/*",
- columns={"id": {"hogql": "StringDatabaseField", "clickhouse": "String"}},
- )
-
- schema_catalog = _build_database_schema_query(self.team, DatabaseSchemaQuery(), user=self.user)
- catalog = build_catalog(
- self.team,
- self.user,
- schema_catalog.response,
- database=schema_catalog.database,
- )
-
- assert catalog["tableAliases"][allowed.name] == "postgres.demo.orders"
- assert denied.name not in catalog["tableAliases"]
- assert "other_postgres_orders" not in catalog["tableAliases"]
-
@patch("posthog.api.services.query.get_query_runner_or_none")
def test_data_visualization_node_surfaces_hogql_resolution_error_without_value_error_context(
self, mock_get_query_runner_or_none: MagicMock
diff --git a/posthog/hogql/catalog_traversal.py b/posthog/hogql/catalog_traversal.py
deleted file mode 100644
index 815b8389a2e1..000000000000
--- a/posthog/hogql/catalog_traversal.py
+++ /dev/null
@@ -1,408 +0,0 @@
-from __future__ import annotations
-
-from collections import Counter
-from dataclasses import field
-from typing import TYPE_CHECKING, Literal, TypedDict
-
-import structlog
-
-from posthog.schema import DatabaseSchemaQueryResponse
-
-from posthog.hogql.context import HogQLContext
-from posthog.hogql.database.database import _schema_field_input, serialize_fields
-from posthog.hogql.database.direct_sql_table import DirectSQLTable
-from posthog.hogql.database.lazy_join_tags import GROUP_N
-from posthog.hogql.database.models import (
- FieldOrTable,
- FieldTraverser,
- LazyJoin,
- SavedQuery,
- StringJSONDatabaseField,
- Table,
-)
-from posthog.hogql.database.s3_table import S3Table
-from posthog.hogql.database.schema.event_sessions import EventsSessionSubTable
-from posthog.hogql.database.schema.events import EventsGroupSubTable, EventsPersonSubTable, EventsTable
-from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsPersonSubTable
-from posthog.hogql.database.schema.persons import PersonsTable
-from posthog.hogql.database.schema.sessions_v1 import SessionsTableV1
-from posthog.hogql.database.schema.sessions_v2 import SessionsTableV2
-from posthog.hogql.database.schema.sessions_v3 import SessionsTableV3
-from posthog.hogql.errors import QueryError, ResolutionError
-
-from posthog.dataclasses import frozen
-
-if TYPE_CHECKING:
- from posthog.hogql.database.database import Database
-
- from posthog.models import Team, User
-
-logger = structlog.get_logger(__name__)
-
-MAX_RELATION_DEFINITIONS = 4096
-MAX_RELATION_FIELDS = 120_000
-MAX_RESOLUTION_ATTEMPTS = 240_000
-MAX_RESOLUTION_DEPTH = 16
-
-
-@frozen
-class CatalogTraversalResult:
- field_annotations: dict[str, dict[str, dict[str, str]]] = field(repr=False)
- relations: dict[str, CatalogRelationDefinition] = field(repr=False)
-
-
-class CatalogRelationDefinition(TypedDict, total=False):
- fields: dict[str, dict[str, str]]
- table: str
- propertyNamespaces: dict[str, str]
-
-
-@frozen
-class _ResolvedTarget:
- target: FieldOrTable = field(repr=False)
- property_namespace: str | None
- parents: tuple[Table, ...] = field(repr=False)
-
-
-@frozen
-class _CanonicalTable:
- canonical_name: str = field(repr=False)
- expected_fields: set[str] = field(repr=False)
- table_type: Literal["posthog", "external"]
-
-
-class _TraversalCatalogBuilder:
- def __init__(
- self,
- team: Team,
- user: User,
- schema: DatabaseSchemaQueryResponse,
- database: Database,
- property_namespaces: set[str],
- ) -> None:
- self.schema = schema
- self.database = database
- self.context = HogQLContext(team_id=team.pk, team=team, user=user, database=database)
- self.property_namespaces = property_namespaces
- self.annotations: dict[str, dict[str, dict[str, str]]] = {}
- self.relations: dict[str, CatalogRelationDefinition] = {}
- self.relation_keys: dict[tuple[object, ...], str] = {}
- self.omissions: Counter[str] = Counter()
- self.resolution_attempts = 0
- self.owned_fields = 0
- self.canonical_by_object = self._canonical_tables()
- self.canonical_name_cache: dict[int, str | None] = {}
-
- def build(self) -> CatalogTraversalResult:
- for canonical_name, schema_table in self.schema.tables.items():
- table = self._resolve_canonical(canonical_name)
- if table is None:
- continue
- table_annotations = self.annotations.setdefault(canonical_name, {})
- namespace = self._canonical_property_namespace(table)
- if (
- namespace is not None
- and "properties" in schema_table.fields
- and isinstance(table.fields.get("properties"), StringJSONDatabaseField)
- ):
- table_annotations["properties"] = {"propertyNamespace": namespace}
- for field_name in schema_table.fields:
- if "propertyNamespace" in table_annotations.get(field_name, {}):
- continue
- field_value = table.fields.get(field_name)
- if field_value is None or not isinstance(field_value, (LazyJoin, FieldTraverser, Table)):
- continue
- relation = self._relation_for_field(table, field_value, (), depth=0)
- if relation is not None:
- table_annotations.setdefault(field_name, {})["relation"] = relation
- if not table_annotations:
- self.annotations.pop(canonical_name, None)
-
- if self.omissions:
- logger.warning("hogql_catalog_traversal_edges_omitted", reasons=dict(sorted(self.omissions.items())))
- return CatalogTraversalResult(field_annotations=self.annotations, relations=self.relations)
-
- def _canonical_tables(self) -> dict[int, _CanonicalTable]:
- candidates: dict[int, list[_CanonicalTable]] = {}
- visible_names = set(self.database.tables.resolve_visible_table_names())
- for canonical_name, schema_table in self.schema.tables.items():
- if canonical_name not in visible_names:
- continue
- table = self._resolve_canonical(canonical_name)
- if table is None:
- continue
- table_type: Literal["posthog", "external"] = (
- "posthog" if schema_table.type in ("posthog", "system") else "external"
- )
- candidates.setdefault(id(table), []).append(
- _CanonicalTable(
- canonical_name=canonical_name,
- expected_fields=set(schema_table.fields),
- table_type=table_type,
- )
- )
- return {object_id: entries[0] for object_id, entries in candidates.items() if len(entries) == 1}
-
- def _resolve_canonical(self, name: str) -> Table | None:
- if not self._attempt_resolution():
- return None
- try:
- table = self.database.get_table(name)
- except (QueryError, ResolutionError):
- self.omissions["canonical_unresolvable"] += 1
- return None
- return table
-
- def _relation_for_field(
- self,
- owner: Table,
- field_value: FieldOrTable,
- parents: tuple[Table, ...],
- *,
- depth: int,
- ) -> str | None:
- if depth >= MAX_RESOLUTION_DEPTH or not self._attempt_resolution():
- self.omissions["resolution_limit"] += 1
- return None
- try:
- resolved = self._resolve_field_target(owner, field_value, parents, depth=depth)
- except Exception:
- self.omissions["target_unresolvable"] += 1
- return None
- target = resolved.target
- if not isinstance(target, Table) or target.hidden:
- return None
-
- overrides = self._property_overrides(resolved.property_namespace, target)
- canonical_name = self._canonical_name(target)
- if canonical_name is not None:
- canonical_fields = self.canonical_by_object[id(target)].expected_fields
- overrides = {name: namespace for name, namespace in overrides.items() if name in canonical_fields}
- return self._table_relation(target, canonical_name, overrides)
- if isinstance(target, (S3Table, SavedQuery, DirectSQLTable)):
- self.omissions["target_not_exported"] += 1
- return None
- return self._virtual_relation(target, resolved.parents, overrides, depth=depth + 1)
-
- def _canonical_name(self, target: Table) -> str | None:
- if id(target) in self.canonical_name_cache:
- return self.canonical_name_cache[id(target)]
- canonical = self.canonical_by_object.get(id(target))
- if canonical is None:
- return None
- try:
- serialized = serialize_fields(
- _schema_field_input(target),
- self.context,
- [canonical.canonical_name],
- table_type=canonical.table_type,
- )
- except Exception:
- self.omissions["canonical_unserializable"] += 1
- return None
- result = canonical.canonical_name if {field.name for field in serialized} == canonical.expected_fields else None
- self.canonical_name_cache[id(target)] = result
- return result
-
- def _resolve_field_target(
- self, owner: Table, field_value: FieldOrTable, parents: tuple[Table, ...], *, depth: int
- ) -> _ResolvedTarget:
- if isinstance(field_value, LazyJoin):
- return _ResolvedTarget(
- target=field_value.resolve_table(self.context),
- property_namespace=self._lazy_join_property_namespace(field_value),
- parents=(*parents, owner),
- )
- if isinstance(field_value, FieldTraverser):
- return self._resolve_chain(owner, field_value.chain, parents, depth=depth + 1)
- return _ResolvedTarget(target=field_value, property_namespace=None, parents=(*parents, owner))
-
- def _resolve_chain(
- self, owner: Table, chain: list[str | int], parents: tuple[Table, ...], *, depth: int
- ) -> _ResolvedTarget:
- if depth >= MAX_RESOLUTION_DEPTH:
- raise ResolutionError("catalog traversal chain is too deep")
- current: FieldOrTable = owner
- ancestors = list(parents)
- edge_namespace: str | None = None
- for component in chain:
- if not self._attempt_resolution():
- raise ResolutionError("catalog traversal resolution budget exceeded")
- if component == "..":
- if not ancestors:
- raise ResolutionError("catalog traversal has no parent")
- current = ancestors.pop()
- continue
- if isinstance(current, LazyJoin):
- edge_namespace = self._lazy_join_property_namespace(current)
- current = current.resolve_table(self.context)
- if isinstance(current, FieldTraverser):
- resolved = self._resolve_chain(owner, current.chain, tuple(ancestors), depth=depth + 1)
- current = resolved.target
- edge_namespace = resolved.property_namespace
- ancestors = list(resolved.parents)
- if not isinstance(current, Table):
- raise ResolutionError("catalog traversal reached a scalar field")
- if current.hidden:
- raise ResolutionError("catalog traversal reached a hidden table")
- if isinstance(current, (S3Table, SavedQuery, DirectSQLTable)) and self._canonical_name(current) is None:
- raise ResolutionError("catalog traversal reached an unpublished table")
- next_value = current.fields[str(component)]
- if next_value.hidden:
- raise ResolutionError("catalog traversal reached a hidden field")
- was_traverser = False
- was_lazy_join = isinstance(next_value, LazyJoin)
- if isinstance(next_value, FieldTraverser):
- was_traverser = True
- resolved = self._resolve_chain(current, next_value.chain, tuple(ancestors), depth=depth + 1)
- next_value = resolved.target
- edge_namespace = resolved.property_namespace
- ancestors = list(resolved.parents)
- if isinstance(next_value, LazyJoin):
- edge_namespace = self._lazy_join_property_namespace(next_value)
- next_value = next_value.resolve_table(self.context)
- if isinstance(next_value, Table):
- if next_value.hidden:
- raise ResolutionError("catalog traversal reached a hidden table")
- if not was_traverser:
- ancestors.append(current)
- if not was_traverser and not was_lazy_join:
- edge_namespace = None
- current = next_value
- if isinstance(current, LazyJoin):
- return _ResolvedTarget(
- target=current.resolve_table(self.context),
- property_namespace=self._lazy_join_property_namespace(current),
- parents=tuple(ancestors),
- )
- if isinstance(current, FieldTraverser):
- return self._resolve_chain(owner, current.chain, tuple(ancestors), depth=depth + 1)
- return _ResolvedTarget(target=current, property_namespace=edge_namespace, parents=tuple(ancestors))
-
- def _table_relation(self, target: Table, canonical_name: str, overrides: dict[str, str]) -> str | None:
- key = ("table", canonical_name, tuple(sorted(overrides.items())))
- if cached := self.relation_keys.get(key):
- return cached
- if not self._reserve_definition(len(overrides)):
- return None
- relation_name = self._new_relation_name()
- definition: CatalogRelationDefinition = {"table": canonical_name}
- if overrides:
- definition["propertyNamespaces"] = overrides
- self.relations[relation_name] = definition
- self.relation_keys[key] = relation_name
- return relation_name
-
- def _virtual_relation(
- self, target: Table, parents: tuple[Table, ...], overrides: dict[str, str], *, depth: int
- ) -> str | None:
- parent_context = id(parents[-1]) if parents and self._uses_parent(target) else 0
- key = ("virtual", id(target), parent_context, tuple(sorted(overrides.items())))
- if cached := self.relation_keys.get(key):
- return cached
- fields = self._serialize_table_fields(target)
- if fields is None or not self._reserve_definition(len(fields) + len(overrides)):
- return None
- relation_name = self._new_relation_name()
- definition: CatalogRelationDefinition = {"fields": fields}
- self.relations[relation_name] = definition
- self.relation_keys[key] = relation_name
-
- namespace = self._target_property_namespace(target)
- if namespace is not None and "properties" in fields:
- fields["properties"]["propertyNamespace"] = namespace
- for field_name, namespace_override in overrides.items():
- if field_name in fields:
- fields[field_name]["propertyNamespace"] = namespace_override
- for field_name, field_value in target.fields.items():
- if field_name not in fields or "propertyNamespace" in fields[field_name]:
- continue
- if not isinstance(field_value, (LazyJoin, FieldTraverser, Table)):
- continue
- nested_relation = self._relation_for_field(target, field_value, parents, depth=depth)
- if nested_relation is not None:
- fields[field_name]["relation"] = nested_relation
- return relation_name
-
- def _serialize_table_fields(self, target: Table) -> dict[str, dict[str, str]] | None:
- fields: dict[str, dict[str, str]] = {}
- for field_name, field_value in _schema_field_input(target).items():
- if not self._attempt_resolution():
- self.omissions["resolution_limit"] += 1
- return None
- try:
- serialized = serialize_fields({field_name: field_value}, self.context, ["relation"])
- except Exception:
- self.omissions["field_unserializable"] += 1
- continue
- if len(serialized) != 1:
- continue
- schema_field = serialized[0]
- fields[field_name] = {"name": schema_field.name or field_name, "type": schema_field.type}
- return fields
-
- def _uses_parent(self, target: Table) -> bool:
- return any(isinstance(value, FieldTraverser) and ".." in value.chain for value in target.fields.values())
-
- def _property_overrides(self, edge_namespace: str | None, target: Table) -> dict[str, str]:
- namespace = edge_namespace
- if namespace is None:
- namespace = self._target_property_namespace(target)
- properties_field = target.fields.get("properties")
- if namespace is None or not isinstance(properties_field, StringJSONDatabaseField) or properties_field.hidden:
- return {}
- return {"properties": namespace}
-
- def _lazy_join_property_namespace(self, field_value: LazyJoin) -> str | None:
- if field_value.resolver != GROUP_N:
- return None
- group_index = field_value.resolver_params.get("group_index")
- namespace = f"group:{group_index}" if isinstance(group_index, int) else None
- return namespace if namespace in self.property_namespaces else None
-
- def _target_property_namespace(self, target: Table) -> str | None:
- namespace: str | None = None
- if isinstance(target, EventsPersonSubTable | FlagEvaluationsPersonSubTable):
- namespace = "person"
- elif isinstance(target, EventsSessionSubTable):
- namespace = "session"
- elif isinstance(target, EventsGroupSubTable):
- namespace = f"group:{target.group_index}"
- return namespace if namespace in self.property_namespaces else None
-
- def _canonical_property_namespace(self, target: Table) -> str | None:
- namespace: str | None = None
- if isinstance(target, EventsTable):
- namespace = "event"
- elif isinstance(target, PersonsTable):
- namespace = "person"
- elif isinstance(target, (SessionsTableV1, SessionsTableV2, SessionsTableV3)):
- namespace = "session"
- return namespace if namespace in self.property_namespaces else None
-
- def _reserve_definition(self, field_count: int) -> bool:
- if len(self.relations) >= MAX_RELATION_DEFINITIONS:
- self.omissions["definition_limit"] += 1
- return False
- if self.owned_fields + field_count > MAX_RELATION_FIELDS:
- self.omissions["field_limit"] += 1
- return False
- self.owned_fields += field_count
- return True
-
- def _new_relation_name(self) -> str:
- return f"relation_{len(self.relations)}"
-
- def _attempt_resolution(self) -> bool:
- self.resolution_attempts += 1
- return self.resolution_attempts <= MAX_RESOLUTION_ATTEMPTS
-
-
-def build_catalog_traversals(
- team: Team,
- user: User,
- schema: DatabaseSchemaQueryResponse,
- database: Database,
- property_namespaces: set[str],
-) -> CatalogTraversalResult:
- return _TraversalCatalogBuilder(team, user, schema, database, property_namespaces).build()
diff --git a/posthog/hogql/editor_assist_metrics.py b/posthog/hogql/editor_assist_metrics.py
index fb82ed4f6d24..cace5cfee42f 100644
--- a/posthog/hogql/editor_assist_metrics.py
+++ b/posthog/hogql/editor_assist_metrics.py
@@ -7,26 +7,6 @@ EDITOR_ASSIST_DURATION_SECONDS = Histogram(
buckets=(0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0),
)
-EDITOR_ASSIST_RESPONSES_TOTAL = Counter(
- "hogql_editor_assist_responses",
- "Successful HogQL editor-assist responses by operation, serving backend, and routing reason",
- labelnames=["operation", "backend", "reason"],
-)
-
-LANGUAGE_SERVICE_HTTP_DURATION_SECONDS = Histogram(
- "hogql_language_service_http_duration_seconds",
- "Duration of HTTP requests from Django to the HogQL language service",
- labelnames=["operation"],
- buckets=(0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0),
-)
-
-LANGUAGE_SERVICE_RESPONSE_SIZE_BYTES = Histogram(
- "hogql_language_service_response_size_bytes",
- "Size of responses from the HogQL language service",
- labelnames=["operation"],
- buckets=(128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144),
-)
-
HOGQL_DATABASE_BUILD_DURATION_SECONDS = Histogram(
"hogql_database_build_duration_seconds",
"Duration of HogQL database construction phases",
diff --git a/posthog/hogql/functions/generate_language_service_functions.py b/posthog/hogql/functions/generate_language_service_functions.py
deleted file mode 100644
index 6ece17e2c38f..000000000000
--- a/posthog/hogql/functions/generate_language_service_functions.py
+++ /dev/null
@@ -1,33 +0,0 @@
-import sys
-from pathlib import Path
-
-REPO_ROOT = Path(__file__).resolve().parents[3]
-sys.path.insert(0, str(REPO_ROOT))
-
-from posthog.hogql.functions.mapping import ALL_EXPOSED_FUNCTION_NAMES # noqa: E402
-
-OUTPUT_PATH = REPO_ROOT / "services/hogql-language-service/internal/completion/functions.go"
-
-
-def render() -> str:
- names = sorted(set(ALL_EXPOSED_FUNCTION_NAMES))
- lines = [" ".join(names[index : index + 20]) for index in range(0, len(names), 20)]
- body = "\n".join(lines)
- return f"""// Code generated by posthog/hogql/functions/generate_language_service_functions.py; DO NOT EDIT.
-
-package completion
-
-import "strings"
-
-var hogQLFunctions = strings.Fields(`
-{body}
-`)
-"""
-
-
-def main() -> None:
- OUTPUT_PATH.write_text(render(), encoding="utf-8")
-
-
-if __name__ == "__main__":
- main()
diff --git a/posthog/hogql/language_service.py b/posthog/hogql/language_service.py
deleted file mode 100644
index 1b7771b3d7dd..000000000000
--- a/posthog/hogql/language_service.py
+++ /dev/null
@@ -1,348 +0,0 @@
-from __future__ import annotations
-
-from collections.abc import Callable
-from contextlib import AbstractContextManager, nullcontext
-from dataclasses import dataclass
-from datetime import timedelta
-from hashlib import sha256
-from time import perf_counter
-from typing import TYPE_CHECKING, Any
-
-from django.conf import settings
-from django.core.exceptions import ImproperlyConfigured
-from django.db.models import QuerySet
-
-import redis
-import requests
-import structlog
-import posthoganalytics
-
-from posthog.schema import DatabaseSchemaDataWarehouseTable, DatabaseSchemaQueryResponse
-
-from posthog.hogql.catalog_traversal import build_catalog_traversals
-from posthog.hogql.database.s3_table import S3Table
-from posthog.hogql.editor_assist_metrics import (
- LANGUAGE_SERVICE_HTTP_DURATION_SECONDS,
- LANGUAGE_SERVICE_RESPONSE_SIZE_BYTES,
-)
-from posthog.hogql.errors import QueryError, ResolutionError
-from posthog.hogql.timings import HogQLTimings
-
-from posthog.jwt import PosthogJwtAudience, encode_jwt
-from posthog.models import PropertyDefinition, Team, User
-from posthog.redis import get_client
-from posthog.security.outbound_proxy import internal_requests
-from posthog.taxonomy.property_access import restricted_property_names
-
-from products.event_definitions.backend.models.property_definition import effective_project_id_expr
-
-if TYPE_CHECKING:
- from posthog.hogql.database.database import Database
-
-FEATURE_FLAG = "hogql-language-service"
-AFFINITY_HEADER = "X-HogQL-Affinity-Key"
-CATALOG_VERSION = 2
-CATALOG_REVISION_PREFIX = f"v{CATALOG_VERSION}:"
-
-_CATALOG_PUBLICATION_MARKER_TTL_SECONDS = 5
-_CATALOG_PUBLICATION_LOCK_TTL_SECONDS = 10
-_CATALOG_PUBLICATION_LOCK_WAIT_SECONDS = 0.25
-_CATALOG_PUBLICATION_REDIS_TIMEOUT_SECONDS = 0.1
-
-logger = structlog.get_logger(__name__)
-
-
-def _measure(timings: HogQLTimings | None, key: str) -> AbstractContextManager[None]:
- return timings.measure(key) if timings is not None else nullcontext()
-
-
-class LanguageServiceError(Exception):
- pass
-
-
-class CatalogMissing(LanguageServiceError):
- pass
-
-
-class MalformedLanguageServiceResponse(LanguageServiceError):
- pass
-
-
-@dataclass(frozen=True)
-class LanguageServiceResult:
- body: dict[str, Any]
- duration_seconds: float
- response_size_bytes: int
-
-
-def coordinate_catalog_publication(
- team_id: int,
- user_id: int,
- service_target: str,
- check_catalog: Callable[[], LanguageServiceResult | None],
- publish_catalog: Callable[[], None],
- timings: HogQLTimings | None = None,
-) -> LanguageServiceResult | None:
- scope = sha256(f"{service_target}:{team_id}:{user_id}:{CATALOG_VERSION}".encode()).hexdigest()
- key_prefix = f"hogql-language-service:catalog-publication:{{{scope}}}"
- marker_key = f"{key_prefix}:success"
- lock_key = f"{key_prefix}:lock"
-
- def measured_check_catalog() -> LanguageServiceResult | None:
- with _measure(timings, "language_service_check"):
- return check_catalog()
-
- try:
- with _measure(timings, "redis_marker_lookup"):
- redis_client = get_client(
- socket_timeout=_CATALOG_PUBLICATION_REDIS_TIMEOUT_SECONDS,
- socket_connect_timeout=_CATALOG_PUBLICATION_REDIS_TIMEOUT_SECONDS,
- )
- marker_exists = bool(redis_client.get(marker_key))
- except (redis.exceptions.RedisError, ImproperlyConfigured):
- logger.warning("hogql_catalog_publication_redis_unavailable", exc_info=True)
- publish_catalog()
- return measured_check_catalog()
-
- if marker_exists:
- result = measured_check_catalog()
- if result is not None:
- return result
-
- try:
- with _measure(timings, "redis_lock_acquire"):
- lock = redis_client.lock(
- lock_key,
- timeout=_CATALOG_PUBLICATION_LOCK_TTL_SECONDS,
- blocking_timeout=_CATALOG_PUBLICATION_LOCK_WAIT_SECONDS,
- )
- acquired = lock.acquire()
- except redis.exceptions.RedisError:
- logger.warning("hogql_catalog_publication_lock_unavailable", exc_info=True)
- publish_catalog()
- return measured_check_catalog()
-
- if not acquired:
- return measured_check_catalog()
-
- try:
- result = measured_check_catalog()
- if result is not None:
- return result
-
- publish_catalog()
- try:
- with _measure(timings, "redis_marker_write"):
- redis_client.set(marker_key, "1", ex=_CATALOG_PUBLICATION_MARKER_TTL_SECONDS)
- except redis.exceptions.RedisError:
- logger.warning("hogql_catalog_publication_marker_failed", exc_info=True)
- return measured_check_catalog()
- finally:
- try:
- with _measure(timings, "redis_lock_release"):
- lock.release()
- except redis.exceptions.LockNotOwnedError:
- logger.warning("hogql_catalog_publication_lock_expired")
- except redis.exceptions.RedisError:
- logger.warning("hogql_catalog_publication_lock_release_failed", exc_info=True)
-
-
-def is_language_service_enabled(team: Team, user: User) -> bool:
- if not settings.HOGQL_LANGUAGE_SERVICE_URL or not settings.HOGQL_LANGUAGE_SERVICE_SIGNING_KEYS:
- return False
- if settings.DEBUG:
- return True
- return bool(
- posthoganalytics.feature_enabled(
- FEATURE_FLAG,
- str(user.distinct_id),
- person_properties={"email": user.email},
- groups={"organization": str(team.organization_id), "project": str(team.id)},
- group_properties={
- "organization": {"id": str(team.organization_id)},
- "project": {"id": str(team.id)},
- },
- only_evaluate_locally=True,
- send_feature_flag_events=False,
- )
- )
-
-
-def build_catalog(
- team: Team,
- user: User,
- schema: DatabaseSchemaQueryResponse,
- *,
- database: Database,
-) -> dict[str, Any]:
- tables: dict[str, Any] = {}
- for name, table in schema.tables.items():
- fields = {
- field_name: {"name": field.name or field_name, "type": field.type}
- for field_name, field in table.fields.items()
- }
- tables[name] = {"id": table.id, "name": table.name, "type": table.type, "fields": fields}
-
- properties: dict[str, list[dict[str, str]]] = {}
- for property_type, namespace in (
- (PropertyDefinition.Type.EVENT, "event"),
- (PropertyDefinition.Type.PERSON, "person"),
- (PropertyDefinition.Type.SESSION, "session"),
- ):
- properties[namespace] = _properties_for_namespace(team, user, property_type)
- for group_type_index in range(5):
- properties[f"group:{group_type_index}"] = _properties_for_namespace(
- team, user, PropertyDefinition.Type.GROUP, group_type_index
- )
- traversals = build_catalog_traversals(team, user, schema, database, set(properties))
- for table_name, field_annotations in traversals.field_annotations.items():
- for field_name, annotation in field_annotations.items():
- tables[table_name]["fields"][field_name].update(annotation)
- return {
- "tables": tables,
- "properties": properties,
- "tableAliases": _warehouse_table_aliases(schema, database),
- "relations": traversals.relations,
- }
-
-
-def _warehouse_table_aliases(schema: DatabaseSchemaQueryResponse, database: Database) -> dict[str, str]:
- visible_names = set(database.tables.resolve_visible_table_names())
- canonical_by_resolved_object: dict[int, list[str]] = {}
- warehouse_tables: list[DatabaseSchemaDataWarehouseTable] = []
-
- for canonical_name, schema_table in schema.tables.items():
- if not isinstance(schema_table, DatabaseSchemaDataWarehouseTable):
- continue
- warehouse_tables.append(schema_table)
- resolved = _visible_warehouse_table(database, visible_names, canonical_name)
- if resolved.table_id != schema_table.id:
- raise LanguageServiceError(f"catalog table {canonical_name!r} does not match the HogQL resolver")
- canonical_by_resolved_object.setdefault(id(resolved), []).append(canonical_name)
-
- aliases: dict[str, str] = {}
- canonical_names = set(schema.tables)
- for schema_table in warehouse_tables:
- for alias in schema_table.search_aliases or []:
- resolved = _visible_warehouse_table(database, visible_names, alias)
- targets = canonical_by_resolved_object.get(id(resolved), [])
- if len(targets) != 1:
- raise LanguageServiceError(f"catalog alias {alias!r} has no unique visible target")
- target = targets[0]
- if alias == target:
- continue
- if alias in canonical_names:
- raise LanguageServiceError(f"catalog alias {alias!r} collides with a canonical table")
- previous = aliases.setdefault(alias, target)
- if previous != target:
- raise LanguageServiceError(f"catalog alias {alias!r} resolves to conflicting tables")
- return aliases
-
-
-def _visible_warehouse_table(database: Database, visible_names: set[str], name: str) -> S3Table:
- if name not in visible_names:
- raise LanguageServiceError(f"catalog table {name!r} is not visible")
- try:
- table = database.get_table(name)
- except (QueryError, ResolutionError) as error:
- raise LanguageServiceError(f"catalog table {name!r} cannot be resolved") from error
- if not isinstance(table, S3Table) or table.table_id is None:
- raise LanguageServiceError(f"catalog table {name!r} is not a warehouse table")
- return table
-
-
-def _properties_for_namespace(
- team: Team, user: User, property_type: PropertyDefinition.Type, group_type_index: int | None = None
-) -> list[dict[str, str]]:
- restricted = restricted_property_names(team, user, property_type)
- queryset: QuerySet[PropertyDefinition] = PropertyDefinition.objects.alias(
- effective_project_id=effective_project_id_expr()
- ).filter(effective_project_id=team.pk, type=property_type)
- if property_type == PropertyDefinition.Type.GROUP:
- queryset = queryset.filter(group_type_index=group_type_index)
- if restricted:
- queryset = queryset.exclude(name__in=restricted)
- return [
- {"name": name, "property_type": value_type or ""}
- for name, value_type in queryset.order_by("name").values_list("name", "property_type")
- ]
-
-
-class LanguageServiceClient:
- def __init__(self) -> None:
- self.base_url = settings.HOGQL_LANGUAGE_SERVICE_URL.rstrip("/")
- keys: list[str] = settings.HOGQL_LANGUAGE_SERVICE_SIGNING_KEYS
- if not self.base_url or not keys:
- raise LanguageServiceError("HogQL language service is not configured")
- self.signing_key = keys[0]
-
- def publish(self, team_id: int, user_id: int, revision: str, catalog: dict[str, Any]) -> LanguageServiceResult:
- return self._request(
- "PUT", team_id, user_id, "catalog", "publish", {"revision": revision, "catalog": catalog}, 10
- )
-
- def autocomplete(self, team_id: int, user_id: int, query: str, position: int) -> LanguageServiceResult:
- return self._request(
- "POST",
- team_id,
- user_id,
- "autocomplete",
- "complete",
- {"query": query, "position": position, "positionEncoding": "utf-16"},
- 1,
- )
-
- def validate(self, team_id: int, user_id: int, query: str) -> LanguageServiceResult:
- return self._request(
- "POST",
- team_id,
- user_id,
- "validate",
- "validate",
- {"query": query, "positionEncoding": "utf-16"},
- 1,
- )
-
- def _request(
- self,
- method: str,
- team_id: int,
- user_id: int,
- endpoint: str,
- operation: str,
- payload: dict[str, Any],
- timeout_seconds: float,
- ) -> LanguageServiceResult:
- token = encode_jwt(
- {"team_id": team_id, "user_id": user_id, "operations": [operation]},
- timedelta(minutes=1),
- PosthogJwtAudience.HOGQL_LANGUAGE_SERVICE,
- signing_key=self.signing_key,
- )
- started = perf_counter()
- affinity_key = sha256(f"{team_id}:{user_id}".encode()).hexdigest()
- try:
- response = internal_requests.request(
- method,
- f"{self.base_url}/teams/{team_id}/users/{user_id}/{endpoint}",
- json=payload,
- headers={"Authorization": f"Bearer {token}", AFFINITY_HEADER: affinity_key},
- timeout=(0.25, timeout_seconds),
- )
- except requests.RequestException as error:
- raise LanguageServiceError(str(error)) from error
- finally:
- duration = perf_counter() - started
- LANGUAGE_SERVICE_HTTP_DURATION_SECONDS.labels(operation=operation).observe(duration)
- LANGUAGE_SERVICE_RESPONSE_SIZE_BYTES.labels(operation=operation).observe(len(response.content))
- if response.status_code == 404:
- raise CatalogMissing("catalog not found")
- if not response.ok:
- raise LanguageServiceError(f"language service returned {response.status_code}: {response.text[:256]}")
- try:
- body: object = response.json()
- except requests.JSONDecodeError as error:
- raise MalformedLanguageServiceResponse("language service returned invalid JSON") from error
- if not isinstance(body, dict):
- raise MalformedLanguageServiceResponse("language service returned a non-object JSON response")
- return LanguageServiceResult(body=body, duration_seconds=duration, response_size_bytes=len(response.content))
diff --git a/posthog/hogql/test/test_generated_language_service_functions.py b/posthog/hogql/test/test_generated_language_service_functions.py
deleted file mode 100644
index 710d68a6e6e3..000000000000
--- a/posthog/hogql/test/test_generated_language_service_functions.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from posthog.hogql.functions.generate_language_service_functions import OUTPUT_PATH, render
-
-
-def test_language_service_functions_match_generated_output() -> None:
- assert OUTPUT_PATH.read_text(encoding="utf-8") == render(), (
- "HogQL language service functions are out of sync with ALL_EXPOSED_FUNCTION_NAMES. "
- "Run: python posthog/hogql/functions/generate_language_service_functions.py"
- )
diff --git a/posthog/hogql/test/test_language_service.py b/posthog/hogql/test/test_language_service.py
deleted file mode 100644
index 4a0272ba897d..000000000000
--- a/posthog/hogql/test/test_language_service.py
+++ /dev/null
@@ -1,669 +0,0 @@
-from unittest.mock import MagicMock, patch
-
-from django.test import SimpleTestCase, override_settings
-
-import redis
-import requests
-from parameterized import parameterized
-
-from posthog.schema import (
- DatabaseSchemaDataWarehouseTable,
- DatabaseSchemaField,
- DatabaseSchemaPostHogTable,
- DatabaseSchemaQueryResponse,
- DatabaseSerializedFieldType,
-)
-
-from posthog.hogql.context import HogQLContext
-from posthog.hogql.database.database import Database, _schema_field_input, serialize_fields
-from posthog.hogql.database.lazy_join_tags import EVENTS_TO_SESSIONS_V2, GROUP_N
-from posthog.hogql.database.models import (
- FieldTraverser,
- LazyJoin,
- StringDatabaseField,
- StringJSONDatabaseField,
- Table,
- TableNode,
- VirtualTable,
-)
-from posthog.hogql.database.s3_table import S3Table
-from posthog.hogql.database.schema.events import EventsPersonSubTable, EventsTable
-from posthog.hogql.database.schema.groups import GroupsTable
-from posthog.hogql.database.schema.persons import PersonsTable
-from posthog.hogql.database.schema.sessions_v2 import SessionsTableV2
-from posthog.hogql.language_service import (
- CatalogMissing,
- LanguageServiceClient,
- LanguageServiceError,
- LanguageServiceResult,
- MalformedLanguageServiceResponse,
- build_catalog,
- coordinate_catalog_publication,
- is_language_service_enabled,
-)
-from posthog.hogql.timings import HogQLTimings
-
-from posthog.jwt import PosthogJwtAudience, decode_jwt
-
-
-@override_settings(
- HOGQL_LANGUAGE_SERVICE_URL="http://language-service:8091",
- HOGQL_LANGUAGE_SERVICE_SIGNING_KEYS=["test-language-service-signing-key"],
-)
-class TestLanguageServiceClient(SimpleTestCase):
- @patch("posthog.hogql.language_service.internal_requests.request")
- def test_routes_request_and_scopes_token_to_principal_and_operation(self, request: MagicMock) -> None:
- response = MagicMock()
- response.ok = True
- response.status_code = 200
- response.content = b'{"valid":true,"diagnostics":[],"durationMicros":42}'
- response.json.return_value = {"valid": True, "diagnostics": [], "durationMicros": 42}
- request.return_value = response
-
- result = LanguageServiceClient().validate(12, 34, "SELECT 1")
-
- request.assert_called_once()
- call = request.call_args
- assert call.args == ("POST", "http://language-service:8091/teams/12/users/34/validate")
- assert call.kwargs["json"] == {"query": "SELECT 1", "positionEncoding": "utf-16"}
- assert call.kwargs["timeout"] == (0.25, 1)
- assert call.kwargs["headers"]["X-HogQL-Affinity-Key"] == (
- "a5c8d54c25064f11498a937f38591eba85a3e67cccc102e6c4f76bbf5377cc37"
- )
- token = call.kwargs["headers"]["Authorization"].removeprefix("Bearer ")
- claims = decode_jwt(
- token,
- PosthogJwtAudience.HOGQL_LANGUAGE_SERVICE,
- verification_keys=["test-language-service-signing-key"],
- )
- assert claims["team_id"] == 12
- assert claims["user_id"] == 34
- assert claims["operations"] == ["validate"]
- assert result.body["valid"] is True
- assert result.response_size_bytes == len(response.content)
-
- @patch("posthog.hogql.language_service.internal_requests.request")
- def test_catalog_miss_has_a_distinct_error(self, request: MagicMock) -> None:
- response = MagicMock()
- response.ok = False
- response.status_code = 404
- response.content = b"catalog not found"
- request.return_value = response
-
- with self.assertRaises(CatalogMissing):
- LanguageServiceClient().autocomplete(12, 34, "SELECT ", 7)
-
- assert request.call_args.kwargs["json"] == {
- "query": "SELECT ",
- "position": 7,
- "positionEncoding": "utf-16",
- }
-
- @parameterized.expand(
- [
- ("invalid_json", requests.JSONDecodeError("invalid", "x", 0)),
- ("list", []),
- ("scalar", "unexpected"),
- ("null", None),
- ]
- )
- @patch("posthog.hogql.language_service.internal_requests.request")
- def test_rejects_malformed_responses(self, _name: str, body: object, request: MagicMock) -> None:
- response = MagicMock(ok=True, content=b"malformed")
- response.json.side_effect = body if isinstance(body, Exception) else None
- if not isinstance(body, Exception):
- response.json.return_value = body
- request.return_value = response
-
- with self.assertRaises(MalformedLanguageServiceResponse):
- LanguageServiceClient().validate(12, 34, "SELECT event FROM events")
-
- @patch("posthog.hogql.language_service.LANGUAGE_SERVICE_HTTP_DURATION_SECONDS")
- @patch("posthog.hogql.language_service.internal_requests.request", side_effect=requests.Timeout("timed out"))
- def test_records_latency_for_failed_requests(self, _request: MagicMock, duration: MagicMock) -> None:
- with self.assertRaises(LanguageServiceError):
- LanguageServiceClient().validate(12, 34, "SELECT 1")
-
- duration.labels.assert_called_once_with(operation="validate")
- duration.labels.return_value.observe.assert_called_once()
-
-
-class TestLanguageServiceFeatureFlag(SimpleTestCase):
- @override_settings(DEBUG=True, HOGQL_LANGUAGE_SERVICE_URL="", HOGQL_LANGUAGE_SERVICE_SIGNING_KEYS=[])
- def test_disabled_without_service_configuration(self) -> None:
- assert not is_language_service_enabled(MagicMock(), MagicMock())
-
- @override_settings(
- DEBUG=True,
- HOGQL_LANGUAGE_SERVICE_URL="http://language-service:8091",
- HOGQL_LANGUAGE_SERVICE_SIGNING_KEYS=["key"],
- )
- def test_enabled_in_debug_with_service_configuration(self) -> None:
- assert is_language_service_enabled(MagicMock(), MagicMock())
-
- @override_settings(
- DEBUG=False,
- HOGQL_LANGUAGE_SERVICE_URL="http://language-service:8091",
- HOGQL_LANGUAGE_SERVICE_SIGNING_KEYS=["key"],
- )
- @patch("posthog.hogql.language_service.posthoganalytics.feature_enabled", return_value=True)
- def test_production_uses_local_feature_flag_evaluation(self, feature_enabled: MagicMock) -> None:
- team = MagicMock(id=12, organization_id=56)
- user = MagicMock(distinct_id="user-distinct-id", email="[email protected]")
-
- assert is_language_service_enabled(team, user)
- feature_enabled.assert_called_once_with(
- "hogql-language-service",
- "user-distinct-id",
- person_properties={"email": "[email protected]"},
- groups={"organization": "56", "project": "12"},
- group_properties={"organization": {"id": "56"}, "project": {"id": "12"}},
- only_evaluate_locally=True,
- send_feature_flag_events=False,
- )
-
-
-class TestCatalogPublicationCoordination(SimpleTestCase):
- result = LanguageServiceResult(
- body={"valid": True, "catalogRevision": "v2:ready"},
- duration_seconds=0,
- response_size_bytes=0,
- )
-
- @patch("posthog.hogql.language_service.get_client")
- def test_success_marker_reuses_compatible_catalog(self, get_client: MagicMock) -> None:
- redis_client = get_client.return_value
- redis_client.get.return_value = b"1"
- check_catalog = MagicMock(return_value=self.result)
- publish_catalog = MagicMock()
-
- result = coordinate_catalog_publication(12, 34, "http://language-service:8091", check_catalog, publish_catalog)
-
- assert result is self.result
- version_two_marker = redis_client.get.call_args.args[0]
- with patch("posthog.hogql.language_service.CATALOG_VERSION", 3):
- coordinate_catalog_publication(12, 34, "http://language-service:8091", check_catalog, publish_catalog)
- assert redis_client.get.call_args.args[0] != version_two_marker
- redis_client.lock.assert_not_called()
- publish_catalog.assert_not_called()
-
- @patch("posthog.hogql.language_service.get_client")
- def test_missing_catalog_overrides_success_marker(self, get_client: MagicMock) -> None:
- redis_client = get_client.return_value
- redis_client.get.return_value = b"1"
- redis_client.lock.return_value.acquire.return_value = True
- check_catalog = MagicMock(side_effect=[None, None, self.result])
- publish_catalog = MagicMock()
-
- result = coordinate_catalog_publication(12, 34, "http://language-service:8091", check_catalog, publish_catalog)
-
- assert result is self.result
- publish_catalog.assert_called_once()
- redis_client.lock.assert_called_once_with(
- redis_client.lock.call_args.args[0], timeout=10, blocking_timeout=0.25
- )
- redis_client.set.assert_called_once_with(redis_client.set.call_args.args[0], "1", ex=5)
- redis_client.lock.return_value.release.assert_called_once()
-
- @patch("posthog.hogql.language_service.get_client")
- def test_winner_rechecks_before_publishing(self, get_client: MagicMock) -> None:
- redis_client = get_client.return_value
- redis_client.get.return_value = None
- redis_client.lock.return_value.acquire.return_value = True
- publish_catalog = MagicMock()
-
- result = coordinate_catalog_publication(
- 12, 34, "http://language-service:8091", MagicMock(return_value=self.result), publish_catalog
- )
-
- assert result is self.result
- publish_catalog.assert_not_called()
- redis_client.set.assert_not_called()
- redis_client.lock.return_value.release.assert_called_once()
-
- @parameterized.expand([(True,), (False,)])
- @patch("posthog.hogql.language_service.get_client")
- @patch("posthog.hogql.timings.perf_counter")
- def test_contender_rechecks_without_publishing(
- self, expected_success: bool, perf_counter: MagicMock, get_client: MagicMock
- ) -> None:
- now = [0.0]
- perf_counter.side_effect = lambda: now[0]
- redis_client = get_client.return_value
- redis_client.get.return_value = None
-
- def fail_to_acquire() -> bool:
- now[0] += 0.25
- return False
-
- redis_client.lock.return_value.acquire.side_effect = fail_to_acquire
- publish_catalog = MagicMock()
- timings = HogQLTimings()
-
- def check_catalog() -> LanguageServiceResult | None:
- now[0] += 0.1
- return self.result if expected_success else None
-
- result = coordinate_catalog_publication(
- 12,
- 34,
- "http://language-service:8091",
- check_catalog,
- publish_catalog,
- timings=timings,
- )
-
- assert (result is not None) is expected_success
- publish_catalog.assert_not_called()
- redis_client.lock.return_value.release.assert_not_called()
- timing_values = {timing.k: timing.t for timing in timings.to_list(back_out_stack=False)}
- assert timing_values["./redis_lock_acquire"] == 0.25
- assert "./redis_lock_release" not in timing_values
-
- @patch("posthog.hogql.language_service.get_client")
- def test_next_request_recovers_after_unavailable_lease(self, get_client: MagicMock) -> None:
- first_lock = MagicMock()
- first_lock.acquire.return_value = False
- second_lock = MagicMock()
- second_lock.acquire.return_value = True
- redis_client = get_client.return_value
- redis_client.get.return_value = None
- redis_client.lock.side_effect = [first_lock, second_lock]
- check_catalog = MagicMock(side_effect=[None, None, self.result])
- publish_catalog = MagicMock()
-
- first = coordinate_catalog_publication(12, 34, "http://language-service:8091", check_catalog, publish_catalog)
- second = coordinate_catalog_publication(12, 34, "http://language-service:8091", check_catalog, publish_catalog)
-
- assert first is None
- assert second is self.result
- publish_catalog.assert_called_once()
- second_lock.release.assert_called_once()
-
- @patch("posthog.hogql.language_service.get_client")
- def test_publication_failure_does_not_write_marker(self, get_client: MagicMock) -> None:
- redis_client = get_client.return_value
- redis_client.get.return_value = None
- redis_client.lock.return_value.acquire.return_value = True
- publish_catalog = MagicMock(side_effect=LanguageServiceError("publish failed"))
-
- with self.assertRaises(LanguageServiceError):
- coordinate_catalog_publication(
- 12, 34, "http://language-service:8091", MagicMock(return_value=None), publish_catalog
- )
-
- redis_client.set.assert_not_called()
- redis_client.lock.return_value.release.assert_called_once()
-
- @parameterized.expand([("get",), ("acquire",)])
- @patch("posthog.hogql.language_service.get_client")
- def test_redis_outage_uses_direct_publication(self, failure_stage: str, get_client: MagicMock) -> None:
- if failure_stage == "get":
- get_client.side_effect = redis.exceptions.ConnectionError("unavailable")
- else:
- get_client.return_value.get.return_value = None
- get_client.return_value.lock.return_value.acquire.side_effect = redis.exceptions.ConnectionError(
- "unavailable"
- )
- publish_catalog = MagicMock()
-
- result = coordinate_catalog_publication(
- 12, 34, "http://language-service:8091", MagicMock(return_value=self.result), publish_catalog
- )
-
- assert result is self.result
- publish_catalog.assert_called_once()
-
- @patch("posthog.hogql.language_service.get_client")
- def test_late_redis_errors_preserve_successful_publication(self, get_client: MagicMock) -> None:
- redis_client = get_client.return_value
- redis_client.get.return_value = None
- lock = redis_client.lock.return_value
- lock.acquire.return_value = True
- lock.release.side_effect = redis.exceptions.LockNotOwnedError("expired")
- redis_client.set.side_effect = redis.exceptions.ConnectionError("unavailable")
- publish_catalog = MagicMock()
-
- result = coordinate_catalog_publication(
- 12,
- 34,
- "http://language-service:8091",
- MagicMock(side_effect=[None, self.result]),
- publish_catalog,
- )
-
- assert result is self.result
- publish_catalog.assert_called_once()
-
- @patch("posthog.hogql.language_service.get_client")
- def test_coordination_keys_isolate_principals_and_service_targets(self, get_client: MagicMock) -> None:
- redis_client = get_client.return_value
- redis_client.get.return_value = None
- redis_client.lock.return_value.acquire.return_value = False
-
- for team_id, user_id, target in (
- (12, 34, "http://language-service-a:8091"),
- (13, 34, "http://language-service-a:8091"),
- (12, 35, "http://language-service-a:8091"),
- (12, 34, "http://language-service-b:8091"),
- ):
- coordinate_catalog_publication(team_id, user_id, target, MagicMock(return_value=None), MagicMock())
-
- lock_keys = {call.args[0] for call in redis_client.lock.call_args_list}
- assert len(lock_keys) == 4
-
-
-class TestLanguageServiceCatalog(SimpleTestCase):
- def _database(self, paths: dict[str, S3Table], *, hidden: set[str] | None = None) -> Database:
- database = Database(include_posthog_tables=False)
- for path, table in paths.items():
- node = TableNode.create_nested_for_chain(path.split("."), table)
- if hidden and path in hidden:
- node.hidden = True
- database.tables.add_child(node)
- return database
-
- def _warehouse_schema(self, tables: dict[str, tuple[str, list[str] | None]]) -> DatabaseSchemaQueryResponse:
- return DatabaseSchemaQueryResponse(
- tables={
- name: DatabaseSchemaDataWarehouseTable(fields={}, id=table_id, name=name, search_aliases=aliases)
- for name, (table_id, aliases) in tables.items()
- },
- joins=[],
- )
-
- def _posthog_schema(self, database: Database, names: list[str]) -> DatabaseSchemaQueryResponse:
- context = HogQLContext(team_id=12, database=database)
- tables = {}
- for name in names:
- table = database.get_table(name)
- fields = serialize_fields(_schema_field_input(table), context, [name])
- tables[name] = DatabaseSchemaPostHogTable(
- fields={field.name: field for field in fields}, id=name, name=name
- )
- return DatabaseSchemaQueryResponse(tables=tables, joins=[])
-
- def _posthog_table(self, name: str, field_names: list[str]) -> DatabaseSchemaPostHogTable:
- return DatabaseSchemaPostHogTable(
- fields={
- field_name: DatabaseSchemaField(
- name=field_name,
- hogql_value=field_name,
- schema_valid=True,
- type=DatabaseSerializedFieldType.UNKNOWN,
- )
- for field_name in field_names
- },
- id=name,
- name=name,
- )
-
- def _warehouse_table(self, name: str, table_id: str, table: S3Table) -> DatabaseSchemaDataWarehouseTable:
- fields = serialize_fields(
- table.fields,
- HogQLContext(team_id=12),
- name.split("."),
- table_type="external",
- )
- return DatabaseSchemaDataWarehouseTable(
- fields={field.name: field for field in fields},
- id=table_id,
- name=name,
- )
-
- @patch("posthog.hogql.language_service._properties_for_namespace", return_value=[])
- def test_publishes_resolved_person_session_group_and_virtual_traversals(self, _properties: MagicMock) -> None:
- database = Database(include_posthog_tables=False)
- events = EventsTable()
- persons = PersonsTable()
- sessions = SessionsTableV2()
- groups = GroupsTable()
- groups.fields["member"] = EventsPersonSubTable()
- hidden_groups = GroupsTable()
- hidden_groups.fields["properties"].hidden = True
- subscriptions = S3Table(
- name="subscriptions",
- fields={
- "plan": StringDatabaseField(name="plan"),
- "team_id": StringDatabaseField(name="team_id"),
- },
- table_id="subscriptions-id",
- url="",
- )
- events.fields["session"] = LazyJoin(
- from_field=["$session_id"], join_table=sessions, resolver=EVENTS_TO_SESSIONS_V2
- )
- events.fields["group_0"] = LazyJoin(
- from_field=["$group_0"], join_table=groups, resolver=GROUP_N, resolver_params={"group_index": 0}
- )
- events.fields["organization"] = FieldTraverser(chain=["group_0"])
- events.fields["group_member"] = FieldTraverser(chain=["group_0", "member"])
- events.fields["hidden_group"] = LazyJoin(
- from_field=["$group_1"], join_table=hidden_groups, resolver=GROUP_N, resolver_params={"group_index": 1}
- )
- events.fields["subscription"] = LazyJoin(
- from_field=["subscription_id"], join_table=subscriptions, resolver="foreign_key"
- )
- for name, table in (
- ("events", events),
- ("persons", persons),
- ("sessions", sessions),
- ("groups", groups),
- ("hidden_groups", hidden_groups),
- ):
- database.tables.add_child(TableNode(name=name, table=table))
- database.tables.add_child(TableNode.create_nested_for_chain(["warehouse", "subscriptions"], subscriptions))
- schema = self._posthog_schema(database, ["events", "persons", "sessions", "groups", "hidden_groups"])
- schema.tables["warehouse.subscriptions"] = self._warehouse_table(
- "warehouse.subscriptions", "subscriptions-id", subscriptions
- )
-
- catalog = build_catalog(MagicMock(pk=12), MagicMock(), schema, database=database)
-
- relations = catalog["relations"]
- person = relations[catalog["tables"]["events"]["fields"]["person"]["relation"]]
- assert person == {"table": "persons"}
- session = relations[catalog["tables"]["events"]["fields"]["session"]["relation"]]
- assert session == {"table": "sessions"}
- group_relation_name = catalog["tables"]["events"]["fields"]["group_0"]["relation"]
- assert catalog["tables"]["events"]["fields"]["organization"]["relation"] == group_relation_name
- assert relations[group_relation_name] == {
- "table": "groups",
- "propertyNamespaces": {"properties": "group:0"},
- }
- hidden_group = relations[catalog["tables"]["events"]["fields"]["hidden_group"]["relation"]]
- assert hidden_group == {"table": "hidden_groups"}
- group_member = relations[catalog["tables"]["events"]["fields"]["group_member"]["relation"]]
- assert group_member["fields"]["properties"]["propertyNamespace"] == "person"
- subscription = relations[catalog["tables"]["events"]["fields"]["subscription"]["relation"]]
- assert subscription == {"table": "warehouse.subscriptions"}
- pdi = relations[catalog["tables"]["events"]["fields"]["pdi"]["relation"]]
- assert relations[pdi["fields"]["person"]["relation"]] == {"table": "persons"}
-
- @patch("posthog.hogql.language_service._properties_for_namespace", return_value=[])
- def test_virtual_person_schema_keeps_parent_traversal_and_does_not_reuse_persons(
- self, _properties: MagicMock
- ) -> None:
- database = Database(include_posthog_tables=False)
- events = EventsTable()
- persons = PersonsTable()
- poe = EventsPersonSubTable()
- poe.fields["id"] = FieldTraverser(chain=["..", "pdi", "person_id"])
- poe.fields["again"] = LazyJoin(from_field=["id"], join_table=poe, resolver="foreign_key")
- events.fields["poe"] = poe
- events.fields["person"] = FieldTraverser(chain=["poe"])
- database.tables.add_child(TableNode(name="events", table=events))
- database.tables.add_child(TableNode(name="persons", table=persons))
- schema = self._posthog_schema(database, ["events", "persons"])
-
- catalog = build_catalog(MagicMock(pk=12), MagicMock(), schema, database=database)
-
- relation = catalog["relations"][catalog["tables"]["events"]["fields"]["person"]["relation"]]
- assert "table" not in relation
- assert set(relation["fields"]) == {"again", "created_at", "id", "properties", "revenue_analytics"}
- nested_relation_name = relation["fields"]["again"]["relation"]
- assert catalog["relations"][nested_relation_name]["fields"]["again"]["relation"] == nested_relation_name
- assert relation["fields"]["properties"]["propertyNamespace"] == "person"
- assert "is_identified" not in relation["fields"]
-
- @patch("posthog.hogql.catalog_traversal.logger.warning")
- @patch("posthog.hogql.catalog_traversal.MAX_RELATION_DEFINITIONS", 1)
- @patch("posthog.hogql.language_service._properties_for_namespace", return_value=[])
- def test_omits_denied_hidden_and_broken_edges_without_failing_catalog(
- self, _properties: MagicMock, warning: MagicMock
- ) -> None:
- database = Database(include_posthog_tables=False)
- denied = S3Table(
- name="private",
- fields={
- "nested": VirtualTable(fields={"secret": StringDatabaseField(name="secret")}),
- "secret": StringDatabaseField(name="secret"),
- },
- url="",
- )
- virtual = VirtualTable(
- fields={
- "visible": StringDatabaseField(name="visible"),
- "hidden": StringDatabaseField(name="hidden", hidden=True),
- "broken": LazyJoin(from_field=["id"], join_table="missing", resolver="foreign_key"),
- }
- )
- overflow = VirtualTable(fields={"other": StringDatabaseField(name="other")})
- events = Table(
- fields={
- "denied": LazyJoin(from_field=["id"], join_table=denied, resolver="foreign_key"),
- "denied_nested": FieldTraverser(chain=["denied", "nested"]),
- "allowed": virtual,
- "overflow": overflow,
- "id": StringDatabaseField(name="id"),
- "properties": StringJSONDatabaseField(name="properties"),
- }
- )
- database.tables.add_child(TableNode(name="events", table=events))
- denied_node = TableNode(name="private", table=denied)
- denied_node.hidden = True
- database.tables.add_child(denied_node)
- schema = DatabaseSchemaQueryResponse(
- tables={
- "events": self._posthog_table(
- "events", ["denied", "denied_nested", "allowed", "overflow", "id", "properties"]
- )
- },
- joins=[],
- )
-
- catalog = build_catalog(MagicMock(pk=12), MagicMock(), schema, database=database)
-
- fields = catalog["tables"]["events"]["fields"]
- assert "relation" not in fields["denied"]
- assert "relation" not in fields["denied_nested"]
- assert "relation" not in fields["overflow"]
- allowed = catalog["relations"][fields["allowed"]["relation"]]["fields"]
- assert set(allowed) == {"visible"}
- warning.assert_called_once()
- assert warning.call_args.kwargs["reasons"]["definition_limit"] == 1
- assert "missing" not in str(warning.call_args)
-
- @patch("posthog.hogql.language_service._properties_for_namespace", return_value=[])
- def test_publishes_only_resolver_confirmed_aliases(self, _properties: MagicMock) -> None:
- orders = S3Table(name="orders", fields={}, url="", table_id="orders-id")
- database = self._database(
- {
- "postgres.demo.orders": orders,
- "demo_postgres_orders": orders,
- "DEMO_POSTGRES_ORDERS": orders,
- }
- )
- schema = self._warehouse_schema(
- {"postgres.demo.orders": ("orders-id", ["demo_postgres_orders", "DEMO_POSTGRES_ORDERS"])}
- )
-
- catalog = build_catalog(MagicMock(), MagicMock(), schema, database=database)
-
- assert catalog["tableAliases"] == {
- "demo_postgres_orders": "postgres.demo.orders",
- "DEMO_POSTGRES_ORDERS": "postgres.demo.orders",
- }
-
- @patch("posthog.hogql.language_service._properties_for_namespace", return_value=[])
- def test_uses_the_resolver_winner_for_colliding_alias_candidates(self, _properties: MagicMock) -> None:
- orders = S3Table(name="orders", fields={}, url="", table_id="orders-id")
- customers = S3Table(name="customers", fields={}, url="", table_id="customers-id")
- database = self._database(
- {
- "postgres.demo.orders": orders,
- "postgres.demo.customers": customers,
- "demo_shared": customers,
- }
- )
- schema = self._warehouse_schema(
- {
- "postgres.demo.orders": ("orders-id", ["demo_shared"]),
- "postgres.demo.customers": ("customers-id", None),
- }
- )
-
- catalog = build_catalog(MagicMock(), MagicMock(), schema, database=database)
-
- assert catalog["tableAliases"] == {"demo_shared": "postgres.demo.customers"}
-
- @patch("posthog.hogql.language_service._properties_for_namespace", return_value=[])
- def test_rejects_mismatched_canonical_and_alias_identities(self, _properties: MagicMock) -> None:
- canonical = S3Table(name="orders", fields={}, url="", table_id="resolver-id")
- schema = self._warehouse_schema({"postgres.demo.orders": ("serialized-id", None)})
-
- with self.assertRaisesRegex(LanguageServiceError, "does not match the HogQL resolver"):
- build_catalog(
- MagicMock(),
- MagicMock(),
- schema,
- database=self._database({"postgres.demo.orders": canonical}),
- )
-
- alias = S3Table(name="orders_alias", fields={}, url="", table_id="resolver-id")
- schema = self._warehouse_schema({"postgres.demo.orders": ("resolver-id", ["demo_postgres_orders"])})
- with self.assertRaisesRegex(LanguageServiceError, "has no unique visible target"):
- build_catalog(
- MagicMock(),
- MagicMock(),
- schema,
- database=self._database({"postgres.demo.orders": canonical, "demo_postgres_orders": alias}),
- )
-
- @patch("posthog.hogql.language_service._properties_for_namespace", return_value=[])
- def test_rejects_hidden_or_ambiguous_alias_targets(self, _properties: MagicMock) -> None:
- orders = S3Table(name="orders", fields={}, url="", table_id="orders-id")
- hidden_database = self._database(
- {"postgres.demo.orders": orders, "demo_postgres_orders": orders}, hidden={"demo_postgres_orders"}
- )
- schema = self._warehouse_schema({"postgres.demo.orders": ("orders-id", ["demo_postgres_orders"])})
-
- with self.assertRaisesRegex(LanguageServiceError, "is not visible"):
- build_catalog(MagicMock(), MagicMock(), schema, database=hidden_database)
-
- ambiguous_schema = self._warehouse_schema(
- {
- "postgres.demo.orders": ("orders-id", ["demo_postgres_orders"]),
- "postgres.archive.orders": ("orders-id", None),
- }
- )
- ambiguous_database = self._database(
- {
- "postgres.demo.orders": orders,
- "postgres.archive.orders": orders,
- "demo_postgres_orders": orders,
- }
- )
- with self.assertRaisesRegex(LanguageServiceError, "has no unique visible target"):
- build_catalog(MagicMock(), MagicMock(), ambiguous_schema, database=ambiguous_database)
-
- @patch("posthog.hogql.language_service._properties_for_namespace", return_value=[])
- def test_ignores_builtin_search_metadata_and_identity_aliases(self, _properties: MagicMock) -> None:
- orders = S3Table(name="orders", fields={}, url="", table_id="orders-id")
- database = self._database({"postgres.demo.orders": orders})
- schema = self._warehouse_schema({"postgres.demo.orders": ("orders-id", ["postgres.demo.orders"])})
- schema.tables["events"] = DatabaseSchemaPostHogTable(fields={}, id="events", name="events")
-
- catalog = build_catalog(MagicMock(), MagicMock(), schema, database=database)
-
- assert catalog["tableAliases"] == {}
diff --git a/posthog/jwt.py b/posthog/jwt.py
index 058bbd8b9eee..f7b81e15b2d8 100644
--- a/posthog/jwt.py
+++ b/posthog/jwt.py
@@ -23,7 +23,6 @@ class PosthogJwtAudience(Enum):
DELEGATED_USER = "posthog:delegated_user"
EXPORT_RENDERER = "posthog:export_renderer"
LIVESTREAM = "posthog:livestream"
- HOGQL_LANGUAGE_SERVICE = "hogql-language-service"
SHARING_PASSWORD_PROTECTED = "posthog:sharing_password_protected"
RECORDING_API = "posthog:recording_api"
REPLAY_PROXY = "posthog:replay_proxy"
diff --git a/posthog/settings/data_stores.py b/posthog/settings/data_stores.py
index a68c5925c82a..4980e6bc27fb 100644
--- a/posthog/settings/data_stores.py
+++ b/posthog/settings/data_stores.py
@@ -701,12 +701,6 @@ TASKS_REDIS_URL = os.getenv("TASKS_REDIS_URL", None)
# Rust feature flags service URL
# This is used to proxy flag evaluation requests to the Rust feature flags service
FEATURE_FLAGS_SERVICE_URL = os.getenv("FEATURE_FLAGS_SERVICE_URL", "http://localhost:3001")
-HOGQL_LANGUAGE_SERVICE_URL = get_from_env(
- "HOGQL_LANGUAGE_SERVICE_URL", "http://localhost:8091" if DEBUG and not TEST else ""
-)
-HOGQL_LANGUAGE_SERVICE_SIGNING_KEYS = get_list(
- get_from_env("HOGQL_LANGUAGE_SERVICE_SIGNING_KEYS", "local-development-key" if DEBUG and not TEST else "")
-)
MCP_SERVER_URL: str = get_from_env("MCP_SERVER_URL", "http://localhost:8787/mcp" if DEBUG and not TEST else "")
diff --git a/services/hogql-language-service/.dockerignore b/services/hogql-language-service/.dockerignore
deleted file mode 100644
index 99c0979fea15..000000000000
--- a/services/hogql-language-service/.dockerignore
+++ /dev/null
@@ -1,4 +0,0 @@
-.git
-Dockerfile
-README.md
-**/*_test.go
diff --git a/services/hogql-language-service/Dockerfile b/services/hogql-language-service/Dockerfile
deleted file mode 100644
index 23bf9092fc32..000000000000
--- a/services/hogql-language-service/Dockerfile
+++ /dev/null
@@ -1,36 +0,0 @@
-FROM golang:1.27.3-bookworm@sha256:972b4110ba0f47539a975c31e52c64765e1ff19f2d74469ae4a595c6676e80a5 AS build
-
-WORKDIR /src
-
-COPY go.mod go.sum ./
-RUN --mount=type=cache,target=/go/pkg/mod go mod download
-
-COPY . ./
-
-ARG COMMIT_HASH=unknown
-ARG TARGETOS=linux
-ARG TARGETARCH
-RUN --mount=type=cache,target=/go/pkg/mod \
- --mount=type=cache,target=/root/.cache/go-build \
- CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build \
- -trimpath \
- -ldflags="-s -w" \
- -o /out/hogql-language-service \
- ./cmd/server && \
- printf '%s' "${COMMIT_HASH}" > /out/commit.txt
-
-FROM scratch
-
-WORKDIR /app
-
-COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-COPY --from=build --chown=65532:65532 /out/hogql-language-service /app/hogql-language-service
-COPY --from=build --chown=65532:65532 /out/commit.txt /app/commit.txt
-
-ENV LISTEN_ADDR=0.0.0.0:8091
-
-EXPOSE 8091
-
-USER 65532:65532
-
-ENTRYPOINT ["/app/hogql-language-service"]
diff --git a/services/hogql-language-service/README.md b/services/hogql-language-service/README.md
deleted file mode 100644
index 3950a36efba3..000000000000
--- a/services/hogql-language-service/README.md
+++ /dev/null
@@ -1,222 +0,0 @@
-# HogQL language service prototype
-
-This prototype keeps multiple immutable, permission-filtered catalogs in memory and provides local SQL completion.
-Completion uses the cursor context to suggest fields, functions, comparison operators, and predicate continuations such as `AND` and `OR`.
-The service embeds the global HogQL function list, while Django supplies permission-filtered tables and properties.
-It uses `github.com/orian/clickhouse-sql-parser` to recover table and alias context from the query. Django remains the
-authority for deciding which schema and properties belong in each catalog.
-
-For local development, start the service on its loopback listener with Hogli:
-
-```bash
-hogli start:hogql-lang-service
-```
-
-Publish a permission-filtered catalog through the multitenant endpoint below before making language requests. The Go
-service does not hold a personal API key or fetch schema from PostHog directly.
-
-```bash
-curl -sS http://localhost:8091/health
-curl -sS -X POST http://localhost:8091/teams/2/users/1/autocomplete \
- -H 'Content-Type: application/json' \
- -d '{"query":"SELECT o. FROM orders AS o","position":9,"positionEncoding":"utf-16"}'
-```
-
-Validate syntax and catalog-backed table and field references:
-
-```bash
-curl -sS -X POST http://localhost:8091/teams/2/users/1/validate \
- -H 'Content-Type: application/json' \
- -d '{"query":"SELECT amuont FROM warehouse_0420"}'
-```
-
-Completion and validation share scope analysis for table CTEs and aliased `FROM` subqueries.
-Completion suggests projected fields, including aliases and wildcard outputs, with catalog types for direct field projections.
-FROM and JOIN completion suggests visible CTE names before catalog tables and respects CTE shadowing.
-It reads unquoted multi-part table prefixes from the catalog: `FROM postgres.` shows the full `postgres.demo.orders` label and inserts only `demo.orders`.
-Already-typed namespace components must use the catalog's exact case, while the final partial component remains case-insensitive.
-Completion inside quoted path components remains unsupported.
-Completion omits dotted candidates when a remaining path component would require quotes.
-The editor replaces text from the start of the current word to the cursor and leaves text after the cursor unchanged.
-Empty queries offer SELECT and WITH; typed prefixes filter those starting keywords.
-Joined fields with the same name show their source and insert a qualified reference, including separate aliases in self-joins.
-Unique fields and already-qualified completion keep their existing insertion behavior.
-For example, `WITH t AS (SELECT event AS kind FROM events) SELECT t.` suggests `kind`, even before typing `FROM t`.
-Validation checks those output fields and reports only underlying catalog tables in `tableNames`.
-Each request can expand up to 16,384 projected fields before deduplication.
-Larger projections return HTTP 400 for completion or a `query_limit` validation diagnostic.
-Select fewer fields to stay within the limit.
-Field lookup work has a separate request-wide budget.
-Queries that exceed it return HTTP 400 for completion or a `query_limit` validation diagnostic; reduce the number of sources or qualify field names.
-Joining a CTE or subquery without a `properties` output does not suppress the physical table's property suggestions or validation.
-Direct property containers retain their catalog namespace through CTEs, aliased subqueries, renamed projections, wildcards, and visible SELECT aliases.
-For example, `WITH t AS (SELECT properties AS props FROM events) SELECT t.props.$br FROM t` suggests `$browser`.
-Computed or ambiguous property origins remain unknown; the service does not guess a namespace from a projected name.
-Validation reports `duplicate_table` for repeated table names or explicit aliases in one query scope and asks for distinct aliases.
-Table names, table aliases, and CTE names resolve by exact case; catalogs can contain distinct `events` and `Events` tables.
-Multi-part names also retain distinct identities when their parser-safe forms coincide, such as `a.b.c_d` and `a.b_c.d`.
-Autocomplete prefix matching remains case-insensitive and preserves the selected identifier's case.
-Duplicate qualifiers do not supply property provenance, even when raw ClickHouse accepts the corresponding unaliased self-join.
-
-Validation diagnostic offsets use `positionEncoding`, which defaults to UTF-16. Diagnostics include up to five visible
-typo suggestions ranked by case-insensitive Levenshtein distance. Dynamic properties use the same cached namespaces as
-autocomplete.
-
-```bash
-curl -sS -X POST http://localhost:8091/teams/2/users/1/autocomplete \
- -H 'Content-Type: application/json' \
- -d '{"query":"SELECT events.properties.$geo"}'
-
-curl -sS -X POST http://localhost:8091/teams/2/users/1/validate \
- -H 'Content-Type: application/json' \
- -d '{"query":"SELECT events.properties.$geo_cty FROM events"}'
-```
-
-Autocomplete `position` is optional and defaults to the end of the query. Set `positionEncoding` to `utf-8` (the
-default) or `utf-16`; editor clients such as Monaco should send `utf-16`. Validation accepts the same setting, defaults
-to `utf-16`, and uses it for diagnostic positions. Both responses echo the selected encoding. Suggestion labels
-preserve catalog names, while `insertText` quotes identifiers that contain spaces or special characters.
-Suggestions omit identifiers containing `%` because HogQL does not support them.
-`durationMicros` covers only the
-in-memory completion path; network and JSON decoding are intentionally excluded. Responses contain at most 25
-suggestions, the total match count, and an opaque `nextCursor` when another page exists. Send the same query and
-position with `"cursor":"<nextCursor>"` to retrieve it. The HTTP `Content-Length` is the encoded response size.
-
-The parser currently accepts ClickHouse's `database.table` identifiers but not HogQL's three-part synced-table names.
-Shared analysis normalizes those table references before parsing while preserving byte offsets.
-For incomplete SQL, completion can recover a single query's `FROM` clause and keeps the parser error in `parseError`.
-Queries with complete CTE definitions can also retain their outer SELECT/FROM scope through unfinished trailing clauses, including completion inside an unfinished outer predicate.
-This preserves derived fields, known property origins, and retained SELECT aliases.
-Recovery excludes malformed CTE bodies, nested SELECTs in the outer query, FROM/JOIN cursor positions, and incomplete JOIN sources.
-Validation still reports the original query's syntax errors.
-Completion and validation recognize explicit SELECT aliases in later SELECT items and clauses resolved after SELECT, including WHERE, GROUP BY, HAVING, and ORDER BY.
-Aliases stay within their defining query and do not appear in JOIN conditions.
-Computed and nested property provenance, additional alias forms, parser recovery, and other exclusions are tracked in [query analysis and remaining work](../../docs/internal/hogql-language-service.md#recovery-and-remaining-work).
-
-## Multitenant catalogs
-
-Django can publish a complete catalog for one team and user without restarting the service:
-
-```bash
-curl -sS -X PUT http://localhost:8091/teams/2/users/17/catalog \
- -H 'Content-Type: application/json' \
- -d '{
- "revision": "schema-42:permissions-9",
- "catalog": {
- "tables": {
- "events": {"name": "events", "type": "posthog", "fields": {}},
- "postgres.demo.orders": {"name": "postgres.demo.orders", "type": "data_warehouse", "fields": {}}
- },
- "tableAliases": {"demo_postgres_orders": "postgres.demo.orders"},
- "properties": {"event": [{"name": "$geo_city", "property_type": "String"}]}
- }
- }'
-```
-
-`tableAliases` is optional. Each key is an accepted alternate table spelling, and each value must name a canonical key in `tables`.
-An alias and its canonical table share prepared fields, but validation retains the spelling used in SQL in `tableNames`.
-An identity entry such as `"events": "events"` is a no-op.
-Publication rejects empty names, aliases that replace another canonical key, and targets that create a dangling reference, chain, or cycle.
-Catalogs without `tableAliases` keep the previous behavior.
-
-Every protected route requires positive `teamId` and `userId` path parameters. The response includes
-`catalogRevision`, allowing Django and the editor to detect a stale response. An unknown, expired, or evicted pair
-returns `404`; the service never falls back to another team or user.
-
-`DELETE /teams/{teamId}/users/{userId}/catalog` removes that entry.
-
-Catalogs expire `CATALOG_TTL` (default `30m`) after publication so active projects periodically refresh their schema.
-When `MAX_CATALOGS` (default `1024`) or `CATALOG_CACHE_MAX_BYTES` (default `8 GiB`) is reached, the least recently used
-catalog is evicted. A catalog request is limited to `64 MiB`. Publishing a new revision replaces the old immutable
-catalog atomically.
-
-The generated scale fixture verifies publication, completion, pagination, and validation with 4,096 canonical tables, 25 fields per table, and 120,000 event properties for one user catalog.
-Table fields and event properties are separate catalog entries.
-This is a tested profile, not a count limit: admission still depends on the serialized request size and the prepared catalog's share of the cache byte budget.
-
-Authentication is required unless `HOGQL_LANGUAGE_SERVICE_ALLOW_INSECURE=1` explicitly disables it on a loopback
-listener for local development. Insecure mode logs a startup warning and is rejected on non-loopback listeners.
-`HOGQL_LANGUAGE_SERVICE_SIGNING_KEYS` contains one or more comma-separated HMAC keys. Django sends a short-lived
-HS256 JWT as `Authorization: Bearer …` with these claims:
-
-```json
-{
- "aud": "hogql-language-service",
- "team_id": 2,
- "user_id": 17,
- "operations": ["publish", "complete", "validate", "delete"],
- "nbf": 1788750000,
- "exp": 1788750060
-}
-```
-
-Tokens are valid only for the exact team, user, and operation. List the current signing key first and old keys
-afterward during rotation. Do not expose the service directly to browsers; Django should mint tokens and proxy
-requests after resolving the user's membership and permissions for that team.
-
-## Django integration
-
-Django proxies eligible `HogQLAutocomplete` and `HogQLMetadata` query nodes to the service. Debug builds enable the
-proxy by default. Production requires the `hogql-language-service` feature flag and both settings below:
-
-- `HOGQL_LANGUAGE_SERVICE_URL` points to the service's internal URL.
-- `HOGQL_LANGUAGE_SERVICE_SIGNING_KEYS` lists the current signing key first, followed by keys being rotated out.
-
-On a catalog miss, Django builds the schema visible to that exact team and user, adds their visible event, person,
-session, and group properties, publishes it, and retries once. Unsupported query options and service failures use
-the existing in-process implementation. Prometheus records Django-to-service latency and response size by operation.
-Django also sends `X-HogQL-Affinity-Key`, a stable SHA-256 digest of the team and user IDs. Load balancers may hash on
-this header to route a user's catalog and language requests to the same replica; authorization still comes only from
-the signed JWT and matching path parameters.
-
-## Rate limiting
-
-Protected requests pass through two bounded in-memory token buckets before the handler reads JSON:
-
-1. The pre-authentication bucket keys requests by the direct peer IP address.
-2. The principal bucket keys authenticated requests by `teamId:userId`.
-
-The JWT must match the path before a request consumes principal capacity. The service does not trust forwarded-IP
-headers. Deployments should configure the pre-authentication allowance for the expected number of Django callers.
-
-| Setting | Default |
-| ---------------------------------------- | ------- |
-| `PRE_AUTH_RATE_LIMIT_CAPACITY` | `300` |
-| `PRE_AUTH_RATE_LIMIT_REFILL_PER_SECOND` | `100` |
-| `PRINCIPAL_RATE_LIMIT_CAPACITY` | `120` |
-| `PRINCIPAL_RATE_LIMIT_REFILL_PER_SECOND` | `60` |
-| `RATE_LIMIT_MAX_KEYS` | `10000` |
-| `RATE_LIMIT_IDLE_TTL` | `10m` |
-
-Limited requests return `429` and a `Retry-After` header. The entry bound prevents attacker-controlled path values
-from growing limiter memory without limit.
-
-## Container image
-
-Build the image from the service directory:
-
-```bash
-docker build \
- --build-arg COMMIT_HASH="$(git rev-parse HEAD)" \
- --tag hogql-lang-service:local \
- services/hogql-language-service
-```
-
-The image runs as a non-root user and listens on port `8091`. Because the container binds to all interfaces, it
-requires `HOGQL_LANGUAGE_SERVICE_SIGNING_KEYS`:
-
-```bash
-docker run --rm \
- --publish 127.0.0.1:8091:8091 \
- --env HOGQL_LANGUAGE_SERVICE_SIGNING_KEYS=local-development-key \
- --env MAX_CATALOGS=2 \
- --env CATALOG_CACHE_MAX_BYTES=268435456 \
- hogql-lang-service:local
-```
-
-The production binary is compiled with Go 1.27.3 and `go build -trimpath`. The runtime image contains only the static
-service binary, the commit identifier, and CA certificates. BuildKit's `TARGETOS` and `TARGETARCH` arguments allow
-native `linux/amd64` and `linux/arm64` builds.
-
-Merges that change this service build and publish the `hogql-lang-service` image once, then send its digest to the
-matching Charts release. Pull requests rely on the service tests and repository Dockerfile lint checks.
diff --git a/services/hogql-language-service/benchmarks_test.go b/services/hogql-language-service/benchmarks_test.go
deleted file mode 100644
index 95f30936e04b..000000000000
--- a/services/hogql-language-service/benchmarks_test.go
+++ /dev/null
@@ -1,472 +0,0 @@
-package hogqllanguageservice_test
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io"
- "log/slog"
- "net/http"
- "net/http/httptest"
- "strings"
- "testing"
- "time"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/completion"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/httpapi"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/ratelimit"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/serviceauth"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/validation"
-)
-
-const (
- syntheticTableCount = 4096
- syntheticFieldsPerTable = 25
- syntheticPropertyCount = 120000
- cursorMarker = "§"
-)
-
-const sessionActorQuery = `
-SELECT
- s.session_id,
- s.$start_timestamp,
- s.$end_timestamp,
- s.$session_duration,
- s.$channel_type,
- s.$entry_` + cursorMarker + `pathname,
- s.$entry_referring_domain,
- s.$entry_utm_source,
- s.$entry_utm_medium,
- s.$entry_utm_campaign,
- s.$entry_utm_term,
- s.$entry_utm_content,
- s.$num_uniq_urls,
- s.$autocapture_count,
- s.$exit_pathname,
- s.$last_external_click_url,
- s.$pageview_count
-FROM sessions AS s
-WHERE s.session_id IN (
- SELECT session_id
- FROM events
- WHERE event = '$pageview'
- AND timestamp >= now() - INTERVAL 30 DAY
-)
-ORDER BY s.$start_timestamp DESC
-LIMIT 100`
-
-const traceTreeQuery = `
-WITH matched_traces AS (
- SELECT DISTINCT trace_id
- FROM posthog.trace_spans
- WHERE name = 'database.query'
- AND service_name = 'analytics-api'
- AND timestamp >= now() - INTERVAL 7 DAY
-),
-spans AS (
- SELECT
- s.span_id,
- s.parent_span_id,
- s.trace_id,
- s.service_name,
- s.name,
- s.duration_n` + cursorMarker + `ano,
- s.status_code,
- s.timestamp
- FROM posthog.trace_spans AS s
- WHERE s.trace_id IN (SELECT trace_id FROM matched_traces)
- AND s.service_name = 'analytics-api'
- AND s.timestamp >= now() - INTERVAL 7 DAY
-)
-SELECT
- coalesce(p.service_name, '') AS parent_service,
- if(empty(s.parent_span_id), '<ROOT>', coalesce(p.name, '<ROOT>')) AS parent_name,
- s.service_name,
- s.name,
- count() AS span_count,
- sum(s.duration_nano) AS total_duration_nano,
- avg(s.duration_nano) AS avg_duration_nano,
- quantiles(0.5, 0.95, 0.99, 0.999)(s.duration_nano) AS duration_quantiles,
- countIf(s.status_code = 2) AS error_count,
- avg(
- if(
- empty(s.parent_span_id) OR isNull(p.timestamp),
- toFloat(0),
- toFloat(dateDiff('microsecond', p.timestamp, s.timestamp) * 1000)
- )
- ) AS avg_start_offset_nano
-FROM spans AS s
-LEFT JOIN spans AS p
- ON p.trace_id = s.trace_id AND p.span_id = s.parent_span_id
-GROUP BY parent_service, parent_name, s.service_name, s.name
-ORDER BY total_duration_nano DESC
-LIMIT 1000`
-
-const eventJourneyQuery = `
-WITH first_touch AS (
- SELECT
- e.person_id,
- min(e.timestamp) AS first_seen,
- argMin(e.properties.$event_property_249` + cursorMarker + `99, e.timestamp) AS entry_value
- FROM events AS e
- WHERE e.event = '$pageview'
- AND e.timestamp >= now() - INTERVAL 90 DAY
- AND e.properties.$event_property_00001 != ''
- GROUP BY e.person_id
-),
-returning AS (
- SELECT
- e.person_id,
- countIf(e.event = '$pageview') AS pageviews,
- countDistinct(toDate(e.timestamp)) AS active_days,
- max(e.timestamp) AS last_seen
- FROM events AS e
- WHERE e.timestamp >= now() - INTERVAL 90 DAY
- GROUP BY e.person_id
-)
-SELECT
- f.entry_value,
- count() AS people,
- avg(r.pageviews) AS average_pageviews,
- avg(r.active_days) AS average_active_days,
- quantiles(0.5, 0.9, 0.99)(dateDiff('second', f.first_seen, r.last_seen)) AS retention_seconds
-FROM first_touch AS f
-LEFT JOIN returning AS r ON r.person_id = f.person_id
-WHERE r.pageviews > 1
-GROUP BY f.entry_value
-HAVING people >= 10
-ORDER BY people DESC, f.entry_value ASC
-LIMIT 100`
-
-type catalogUpdate struct {
- Revision string `json:"revision"`
- Catalog catalog.Catalog `json:"catalog"`
-}
-
-type completionCase struct {
- name string
- query string
- position int
-}
-
-func BenchmarkCompleteLargeCatalog(b *testing.B) {
- schema := catalog.Prepare(largeSyntheticCatalog())
- sessionQuery, sessionPosition := queryAndPosition(sessionActorQuery)
- traceQuery, tracePosition := queryAndPosition(traceTreeQuery)
- eventQuery, eventPosition := queryAndPosition(eventJourneyQuery)
- cases := []completionCase{
- {name: "table broad prefix", query: "SELECT * FROM warehouse_table_", position: len("SELECT * FROM warehouse_table_")},
- {name: "alias broad prefix", query: "SELECT * FROM legacy_warehouse_table_", position: len("SELECT * FROM legacy_warehouse_table_")},
- {name: "dotted alias broad prefix", query: "SELECT * FROM legacy.warehouse.table_", position: len("SELECT * FROM legacy.warehouse.table_")},
- {name: "field broad prefix", query: "SELECT w. FROM warehouse_table_2048 AS w", position: len("SELECT w.")},
- {name: "event property broad prefix", query: "SELECT properties.$event_property_ FROM events", position: len("SELECT properties.$event_property_")},
- {name: "event property selective prefix", query: "SELECT properties.$event_property_249 FROM events", position: len("SELECT properties.$event_property_249")},
- {name: "large session query", query: sessionQuery, position: sessionPosition},
- {name: "large trace query", query: traceQuery, position: tracePosition},
- {name: "large event query", query: eventQuery, position: eventPosition},
- }
-
- for _, benchmark := range cases {
- b.Run(benchmark.name, func(b *testing.B) {
- benchmarkCompletion(b, schema, benchmark.query, benchmark.position)
- })
- }
-}
-
-func BenchmarkValidateLargeCatalog(b *testing.B) {
- schema := catalog.Prepare(largeSyntheticCatalog())
- sessionQuery, _ := queryAndPosition(sessionActorQuery)
- eventQuery, _ := queryAndPosition(eventJourneyQuery)
- traceQuery, _ := queryAndPosition(traceTreeQuery)
- for _, benchmark := range []struct {
- name string
- query string
- }{
- {name: "known event query", query: "SELECT event, count() FROM events WHERE timestamp > now() - INTERVAL 30 DAY GROUP BY event ORDER BY count() DESC LIMIT 100"},
- {name: "large session query", query: sessionQuery},
- {name: "large event query", query: eventQuery},
- {name: "large trace query", query: traceQuery},
- {name: "unknown table", query: "SELECT column_00 FROM warehouse_tabel_2048"},
- {name: "unknown event property", query: "SELECT properties.$event_property_120000 FROM events"},
- } {
- b.Run(benchmark.name, func(b *testing.B) {
- benchmarkValidation(b, schema, benchmark.query)
- })
- }
-}
-
-func TestLargeCatalogSupportedScale(t *testing.T) {
- schema := largeSyntheticCatalog()
- if len(schema.Tables) != syntheticTableCount {
- t.Fatalf("tables = %d, want %d", len(schema.Tables), syntheticTableCount)
- }
- for name, table := range schema.Tables {
- if len(table.Fields) != syntheticFieldsPerTable {
- t.Fatalf("table %q fields = %d, want %d", name, len(table.Fields), syntheticFieldsPerTable)
- }
- }
- if len(schema.Properties) != 1 || len(schema.Properties["event"]) != syntheticPropertyCount {
- t.Fatalf("property namespaces = %d, event properties = %d", len(schema.Properties), len(schema.Properties["event"]))
- }
-
- update := catalogUpdate{Revision: "supported-scale", Catalog: *schema}
- payload, err := json.Marshal(update)
- if err != nil {
- t.Fatal(err)
- }
- registry := catalog.NewRegistry(1024, 8<<30, time.Hour)
- limiterConfig := ratelimit.Config{Capacity: 10, RefillPerSec: 10, MaxEntries: 10, IdleTTL: time.Hour}
- preAuthLimiter, err := ratelimit.New(limiterConfig)
- if err != nil {
- t.Fatal(err)
- }
- principalLimiter, err := ratelimit.New(limiterConfig)
- if err != nil {
- t.Fatal(err)
- }
- handler := httpapi.NewHandler(httpapi.Config{
- Catalogs: registry,
- Auth: serviceauth.New(nil, true),
- PreAuthLimiter: preAuthLimiter,
- PrincipalLimiter: principalLimiter,
- Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
- })
- request := httptest.NewRequest(http.MethodPut, "/teams/1/users/1/catalog", bytes.NewReader(payload))
- request.Header.Set("Content-Type", "application/json")
- response := httptest.NewRecorder()
- handler.ServeHTTP(response, request)
- if response.Code != http.StatusOK {
- t.Fatalf("catalog publication returned %d: %s", response.Code, response.Body.String())
- }
- prepared, revision, ok := registry.Get(serviceauth.Authorization{TeamID: 1, UserID: 1})
- if !ok {
- t.Fatal("published catalog was not found")
- }
- if revision != update.Revision || prepared.TableCount() != syntheticTableCount || prepared.PropertyCount() != syntheticPropertyCount {
- t.Fatalf("published catalog = revision %q, found %t, tables %d, properties %d", revision, ok, prepared.TableCount(), prepared.PropertyCount())
- }
-
- for _, test := range []struct {
- query string
- position int
- total int
- }{
- {query: "SELECT * FROM warehouse_table_", total: syntheticTableCount - 3},
- {query: "SELECT * FROM legacy_warehouse_table_", total: syntheticTableCount - 3},
- {query: "SELECT * FROM legacy.warehouse.table_", total: syntheticTableCount - 3},
- {query: "SELECT w. FROM warehouse_table_2048 AS w", position: len("SELECT w."), total: syntheticFieldsPerTable},
- {query: "SELECT properties.$event_property_ FROM events", position: len("SELECT properties.$event_property_"), total: syntheticPropertyCount},
- } {
- position := test.position
- if position == 0 {
- position = len(test.query)
- }
- result, err := completion.Complete(prepared, test.query, position, completion.PositionEncodingUTF8, "")
- if err != nil {
- t.Fatalf("complete %q: %v", test.query, err)
- }
- if result.Total != test.total || len(result.Suggestions) != completion.PageSize {
- t.Fatalf("complete %q returned total %d, page %d, cursor %q", test.query, result.Total, len(result.Suggestions), result.NextCursor)
- }
- if test.total == completion.PageSize {
- if result.NextCursor != "" {
- t.Fatalf("complete %q returned unexpected cursor %q", test.query, result.NextCursor)
- }
- continue
- }
- if result.NextCursor == "" {
- t.Fatalf("complete %q did not return a second-page cursor", test.query)
- }
- second, err := completion.Complete(prepared, test.query, position, completion.PositionEncodingUTF8, result.NextCursor)
- if err != nil {
- t.Fatalf("complete second page %q: %v", test.query, err)
- }
- if second.Total != test.total || len(second.Suggestions) == 0 || second.Suggestions[0].Label == result.Suggestions[0].Label {
- t.Fatalf("complete second page %q returned %#v", test.query, second)
- }
- }
-
- for _, query := range []string{
- "SELECT column_24 FROM legacy.warehouse.table_2048",
- "SELECT properties.$event_property_119999 FROM events",
- } {
- if result := validation.Validate(prepared, query); !result.Valid {
- t.Fatalf("validate %q returned %#v", query, result)
- }
- }
-}
-
-func BenchmarkCatalogPublication(b *testing.B) {
- schema := largeSyntheticCatalog()
- update := catalogUpdate{Revision: "synthetic-revision", Catalog: *schema}
- payload, err := json.Marshal(update)
- if err != nil {
- b.Fatal(err)
- }
- authorization := serviceauth.Authorization{TeamID: 1, UserID: 1}
- prepared := catalog.Prepare(schema)
- b.Run("decode", func(b *testing.B) {
- b.ReportAllocs()
- b.SetBytes(int64(len(payload)))
- b.ResetTimer()
- reportCatalogMetrics(b, schema, len(payload))
- for range b.N {
- var decoded catalogUpdate
- if err := json.Unmarshal(payload, &decoded); err != nil {
- b.Fatal(err)
- }
- }
- })
-
- b.Run("prepare", func(b *testing.B) {
- var result *catalog.PreparedCatalog
- b.ReportAllocs()
- b.ResetTimer()
- reportCatalogMetrics(b, schema, len(payload))
- b.ReportMetric(float64(prepared.EstimatedBytes()), "prepared-B")
- for range b.N {
- result = catalog.Prepare(schema)
- }
- if result == nil {
- b.Fatal("catalog preparation returned nil")
- }
- })
-
- b.Run("cache replacement", func(b *testing.B) {
- registry := catalog.NewRegistry(2, 1<<30, time.Hour)
- b.ReportAllocs()
- b.ResetTimer()
- reportCatalogMetrics(b, schema, len(payload))
- b.ReportMetric(float64(prepared.EstimatedBytes()), "prepared-B")
- for range b.N {
- if err := registry.Put(authorization, update.Revision, prepared); err != nil {
- b.Fatal(err)
- }
- }
- })
-
- b.Run("decode prepare and replace", func(b *testing.B) {
- registry := catalog.NewRegistry(2, 1<<30, time.Hour)
- b.ReportAllocs()
- b.SetBytes(int64(len(payload)))
- b.ResetTimer()
- reportCatalogMetrics(b, schema, len(payload))
- b.ReportMetric(float64(prepared.EstimatedBytes()), "prepared-B")
- for range b.N {
- var decoded catalogUpdate
- if err := json.Unmarshal(payload, &decoded); err != nil {
- b.Fatal(err)
- }
- if err := registry.Put(authorization, decoded.Revision, catalog.Prepare(&decoded.Catalog)); err != nil {
- b.Fatal(err)
- }
- }
- })
-}
-
-func benchmarkCompletion(b *testing.B, schema *catalog.PreparedCatalog, query string, position int) {
- result, err := completion.Complete(schema, query, position, completion.PositionEncodingUTF8, "")
- if err != nil {
- b.Fatal(err)
- }
- payload, err := json.Marshal(result)
- if err != nil {
- b.Fatal(err)
- }
- b.ReportAllocs()
- b.SetBytes(int64(len(query)))
- b.ResetTimer()
- b.ReportMetric(float64(len(query)), "query-B")
- b.ReportMetric(float64(len(payload)), "response-B")
- if result.ParseError != "" {
- b.ReportMetric(1, "parse-errors")
- }
- for range b.N {
- if _, err := completion.Complete(schema, query, position, completion.PositionEncodingUTF8, ""); err != nil {
- b.Fatal(err)
- }
- }
-}
-
-func benchmarkValidation(b *testing.B, schema *catalog.PreparedCatalog, query string) {
- result := validation.Validate(schema, query)
- payload, err := json.Marshal(result)
- if err != nil {
- b.Fatal(err)
- }
- b.ReportAllocs()
- b.SetBytes(int64(len(query)))
- b.ResetTimer()
- b.ReportMetric(float64(len(result.Diagnostics)), "diagnostics")
- b.ReportMetric(float64(len(query)), "query-B")
- b.ReportMetric(float64(len(payload)), "response-B")
- for range b.N {
- validation.Validate(schema, query)
- }
-}
-
-func reportCatalogMetrics(b *testing.B, schema *catalog.Catalog, requestBytes int) {
- b.ReportMetric(float64(len(schema.Tables)), "tables")
- b.ReportMetric(syntheticPropertyCount, "properties")
- b.ReportMetric(float64(requestBytes), "request-B")
-}
-
-func queryAndPosition(marked string) (string, int) {
- position := strings.Index(marked, cursorMarker)
- if position < 0 {
- panic("query does not contain a cursor marker")
- }
- return strings.Replace(marked, cursorMarker, "", 1), position
-}
-
-func largeSyntheticCatalog() *catalog.Catalog {
- tables := make(map[string]catalog.Table, syntheticTableCount)
- tableAliases := make(map[string]string, 2*(syntheticTableCount-3))
- for tableIndex := 0; tableIndex < syntheticTableCount-3; tableIndex++ {
- fields := make(map[string]catalog.Field, syntheticFieldsPerTable)
- for fieldIndex := 0; fieldIndex < syntheticFieldsPerTable; fieldIndex++ {
- name := fmt.Sprintf("column_%02d", fieldIndex)
- fields[name] = catalog.Field{Name: name, Type: "String"}
- }
- name := fmt.Sprintf("warehouse_table_%04d", tableIndex)
- tables[name] = catalog.Table{ID: fmt.Sprintf("table-%04d", tableIndex), Name: name, Type: "data_warehouse", Fields: fields}
- tableAliases[fmt.Sprintf("legacy_warehouse_table_%04d", tableIndex)] = name
- tableAliases[fmt.Sprintf("legacy.warehouse.table_%04d", tableIndex)] = name
- }
- tables["events"] = catalog.Table{Name: "events", Type: "posthog", Fields: scaleFields(
- "distinct_id", "event", "person_id", "properties", "session_id", "timestamp", "uuid",
- )}
- tables["sessions"] = catalog.Table{Name: "sessions", Type: "posthog", Fields: scaleFields(
- "$autocapture_count", "$channel_type", "$end_timestamp", "$entry_pathname", "$entry_referring_domain",
- "$entry_utm_campaign", "$entry_utm_content", "$entry_utm_medium", "$entry_utm_source", "$entry_utm_term",
- "$exit_pathname", "$last_external_click_url", "$num_uniq_urls", "$pageview_count", "$session_duration",
- "$start_timestamp", "session_id",
- )}
- tables["posthog.trace_spans"] = catalog.Table{Name: "posthog.trace_spans", Type: "posthog", Fields: scaleFields(
- "duration_nano", "name", "parent_span_id", "service_name", "span_id", "status_code", "timestamp", "trace_id",
- )}
-
- properties := make([]catalog.Property, syntheticPropertyCount)
- for propertyIndex := range properties {
- properties[propertyIndex] = catalog.Property{Name: fmt.Sprintf("$event_property_%05d", propertyIndex), ValueType: "String"}
- }
- return &catalog.Catalog{Tables: tables, TableAliases: tableAliases, Properties: map[string][]catalog.Property{"event": properties}}
-}
-
-func scaleFields(names ...string) map[string]catalog.Field {
- result := fields(names...)
- for fieldIndex := len(result); fieldIndex < syntheticFieldsPerTable; fieldIndex++ {
- name := fmt.Sprintf("scale_column_%02d", fieldIndex)
- result[name] = catalog.Field{Name: name, Type: "String"}
- }
- return result
-}
-
-func fields(names ...string) map[string]catalog.Field {
- result := make(map[string]catalog.Field, len(names))
- for _, name := range names {
- result[name] = catalog.Field{Name: name, Type: "String"}
- }
- return result
-}
diff --git a/services/hogql-language-service/cmd/demo/assets/app.js b/services/hogql-language-service/cmd/demo/assets/app.js
deleted file mode 100644
index 62dbc980511a..000000000000
--- a/services/hogql-language-service/cmd/demo/assets/app.js
+++ /dev/null
@@ -1,543 +0,0 @@
-const byId = (id) => document.getElementById(id)
-const editor = byId('query')
-const examples = [
- {
- name: 'Event fields',
- query: 'SELECT e.§\nFROM events AS e\nLIMIT 100',
- note: 'Complete after e. to explore event fields.',
- },
- {
- name: 'Valid CTE',
- query: "WITH recent AS (\n SELECT uuid, event, timestamp\n FROM events\n WHERE event = '$pageview'\n)\nSELECT uuid FROM recent§",
- note: 'Validate a CTE with an unqualified projected field.',
- },
- {
- name: 'CTE completion',
- query: 'WITH recent AS (\n SELECT uuid, event AS event_name FROM events\n)\nSELECT recent.§ FROM recent',
- note: 'Probe CTE completion. Missing projected suggestions are a service limitation.',
- },
- {
- name: 'CTE join and boolean filters',
- query: `WITH counts AS (
- SELECT
- toDate(timestamp) AS day,
- count() AS event_count
- FROM events
- GROUP BY day
-), rules AS (
- SELECT
- active_day,
- display_name
- FROM demo_rules
- WHERE is_enabled = TRUE
- AND is_archived = FALSE
-)
-SELECT
- counts.§day,
- counts.event_count,
- rules.display_name
-FROM counts
-LEFT JOIN rules ON counts.day = rules.active_day
-ORDER BY counts.day`,
- note: 'Complete after counts. or rules. to inspect projected fields. Validate TRUE and FALSE boolean filters.',
- },
- {
- name: 'Subquery completion',
- query: 'SELECT nested.§\nFROM (SELECT person_id, count() AS event_count FROM events GROUP BY person_id) AS nested',
- note: 'Probe derived fields. The demo shows exactly what the service supports.',
- },
- {
- name: 'Event properties',
- query: 'SELECT e.properties.$§\nFROM events AS e',
- note: 'Complete event property names from the synthetic catalog.',
- },
- {
- name: 'Person properties',
- query: 'SELECT p.properties.§\nFROM persons AS p',
- note: 'Complete person properties through a table alias.',
- },
- {
- name: 'Property pagination',
- query: 'SELECT properties.demo_property_§ FROM events',
- note: 'There are 35 matching synthetic properties. Use Load more to test pagination.',
- },
- {
- name: 'Unknown field',
- query: 'SELECT timstamp§ FROM events',
- note: 'Validate to see a typo suggestion. Click a diagnostic to select its range.',
- },
- {
- name: 'Unknown table',
- query: 'SELECT uuid FROM evnts§',
- note: 'Validate to see the unknown-table diagnostic and suggested match.',
- },
- {
- name: 'Warehouse table',
- query: 'SELECT o.§ FROM postgres.demo.orders AS o',
- note: 'Test a three-part synced-table name.',
- },
- {
- name: 'Quoted identifiers',
- query: 'SELECT c.§ FROM demo_customers AS c',
- note: 'Insert billing address or café to check identifier quoting.',
- },
- {
- name: 'Unicode diagnostic offsets',
- query: "SELECT '😀', timstamp§ FROM events",
- note: 'Validate, then click the diagnostic. The selected range should be timstamp.',
- },
-]
-
-let catalog = null
-let selection = { query: '', position: 0 }
-let completionSnapshot = null
-let nextCursor = ''
-let analysisTimer = null
-let composing = false
-const requests = { validation: null, completion: null }
-const feedback = { validation: '', completion: '' }
-const copying = { validation: false, completion: false }
-const healthCheckInterval = 5000
-const healthCheckTimeout = 2000
-let healthTimer = null
-let healthController = null
-let healthCheckGeneration = 0
-
-function setServerStatus(connected) {
- const status = byId('server-status')
- status.classList.toggle('connected', connected)
- status.classList.toggle('disconnected', !connected)
- const statusText = byId('server-status-text')
- const nextStatusText = connected ? 'Server connected' : 'Server disconnected'
- if (statusText.textContent !== nextStatusText) {
- statusText.textContent = nextStatusText
- }
-}
-
-function stopHealthChecks() {
- clearTimeout(healthTimer)
- healthTimer = null
- healthCheckGeneration += 1
- healthController?.abort()
- healthController = null
-}
-
-function resumeHealthChecks() {
- clearTimeout(healthTimer)
- healthTimer = null
- checkServerHealth()
-}
-
-async function checkServerHealth() {
- if (healthController || document.hidden) {
- return
- }
- const generation = ++healthCheckGeneration
- const controller = new AbortController()
- healthController = controller
- const timeout = setTimeout(() => controller.abort(), healthCheckTimeout)
- try {
- const response = await fetch('/api/health', { signal: controller.signal, cache: 'no-store' })
- if (generation === healthCheckGeneration) {
- setServerStatus(response.ok)
- }
- } catch {
- if (generation === healthCheckGeneration) {
- setServerStatus(false)
- }
- } finally {
- clearTimeout(timeout)
- if (generation === healthCheckGeneration) {
- healthController = null
- healthTimer = setTimeout(checkServerHealth, healthCheckInterval)
- }
- }
-}
-
-function captureFeedback(kind, exchange) {
- feedback[kind] = JSON.stringify(exchange, null, 2)
- byId(`${kind}-feedback`).value = feedback[kind]
- byId(`${kind}-copy`).disabled = copying[kind]
- byId(`${kind}-copy-status`).textContent = 'Last completed request, retained when the editor changes.'
-}
-
-async function copyFeedback(kind) {
- const text = feedback[kind]
- if (!text || copying[kind]) {
- return
- }
- copying[kind] = true
- byId(`${kind}-copy`).disabled = true
- try {
- await navigator.clipboard.writeText(text)
- byId(`${kind}-copy-status`).textContent = 'Copied request and response.'
- } catch {
- byId(`${kind}-feedback-details`).open = true
- const field = byId(`${kind}-feedback`)
- field.value = text
- field.focus()
- field.select()
- byId(`${kind}-copy-status`).textContent =
- 'Clipboard access is unavailable. Press Ctrl/⌘ C to copy the selected text.'
- } finally {
- copying[kind] = false
- byId(`${kind}-copy`).disabled = false
- }
-}
-
-function node(tag, text, className = '') {
- const element = document.createElement(tag)
- element.textContent = text
- element.className = className
- return element
-}
-
-function setBusy(kind, busy) {
- if (kind === 'validation') {
- byId('validate').disabled = busy
- byId('validate').textContent = busy ? 'Validating…' : 'Validate'
- } else {
- byId('complete').disabled = busy
- byId('more').disabled = busy
- byId('complete').textContent = busy ? 'Completing…' : 'Complete at cursor'
- }
-}
-
-function invalidate(kind) {
- requests[kind]?.abort()
- requests[kind] = null
- setBusy(kind, false)
- byId(`${kind}-time`).textContent = ''
- byId(`${kind}-json`).textContent = 'No request for this editor state.'
- if (kind === 'validation') {
- byId('validation').replaceChildren(node('p', 'Validate the query to see diagnostics.', 'muted'))
- } else {
- completionSnapshot = null
- nextCursor = ''
- byId('suggestions').replaceChildren()
- byId('more').hidden = true
- byId('completion-status').textContent = 'Place the cursor where you want suggestions.'
- }
-}
-
-function cancelScheduledAnalysis() {
- clearTimeout(analysisTimer)
- analysisTimer = null
-}
-
-function scheduleAnalysis() {
- cancelScheduledAnalysis()
- if (!byId('auto-analyze').checked || composing) {
- return
- }
- analysisTimer = setTimeout(() => {
- analysisTimer = null
- if (editor.value.trim()) {
- validate()
- }
- complete()
- }, 300)
-}
-
-function syncSelection() {
- const queryChanged = selection.query !== editor.value
- if (queryChanged) {
- invalidate('validation')
- }
- if (queryChanged || selection.position !== editor.selectionStart) {
- invalidate('completion')
- }
- selection = { query: editor.value, position: editor.selectionStart }
- const before = editor.value.slice(0, editor.selectionStart)
- const lines = before.split('\n')
- byId('position').textContent =
- `Line ${lines.length}, column ${lines.at(-1).length + 1} · UTF-16 offset ${editor.selectionStart}`
- if (queryChanged) {
- scheduleAnalysis()
- }
-}
-
-async function request(kind, payload, onResult) {
- if (requests[kind]) {
- return
- }
- const controller = new AbortController()
- requests[kind] = controller
- setBusy(kind, true)
- const started = performance.now()
- const endpoint = kind === 'completion' ? 'autocomplete' : 'validate'
- const requestBody = kind === 'completion' ? { ...payload, positionEncoding: 'utf-16' } : payload
- const exchange = {
- operation: endpoint,
- recordedAt: new Date().toISOString(),
- catalogSource: 'services/hogql-language-service/cmd/demo/catalog.go',
- request: {
- method: 'POST',
- path: `/api/${endpoint}`,
- servicePath: `/teams/1/users/1/${endpoint}`,
- body: requestBody,
- },
- response: null,
- }
- try {
- const response = await fetch(`/api/${endpoint}`, {
- method: 'POST',
- headers: { 'Content-Type': 'application/json' },
- body: JSON.stringify(requestBody),
- signal: controller.signal,
- })
- const body = await response.text()
- if (requests[kind] !== controller) {
- return
- }
- let responseBody = body
- try {
- responseBody = JSON.parse(body)
- } catch {}
- exchange.response = {
- status: response.status,
- contentType: response.headers.get('Content-Type'),
- body: responseBody,
- }
- captureFeedback(kind, exchange)
- if (!response.ok) {
- throw new Error(`HTTP ${response.status}: ${body}`)
- }
- const data = JSON.parse(body)
- if (requests[kind] !== controller) {
- return
- }
- byId(`${kind}-json`).textContent = JSON.stringify(data, null, 2)
- byId(`${kind}-time`).textContent =
- `${(data.durationMicros / 1000).toFixed(2)} ms service · ${(performance.now() - started).toFixed(0)} ms round trip`
- onResult(data)
- } catch (error) {
- if (controller.signal.aborted) {
- return
- }
- if (!exchange.response) {
- captureFeedback(kind, { ...exchange, error: error.message })
- }
- const message = `${error.message} Retry the request, or restart the demo if the service stopped.`
- byId(`${kind}-json`).textContent = message
- if (kind === 'validation') {
- byId('validation').replaceChildren(node('p', message, 'bad'))
- } else {
- byId('completion-status').textContent = message
- }
- } finally {
- if (requests[kind] === controller) {
- requests[kind] = null
- setBusy(kind, false)
- }
- }
-}
-
-function validate() {
- syncSelection()
- const snapshot = editor.value
- return request('validation', { query: snapshot, positionEncoding: 'utf-8' }, (data) => {
- const queryBytes = new TextEncoder().encode(snapshot)
- const editorOffset = (byteOffset) => new TextDecoder().decode(queryBytes.subarray(0, byteOffset)).length
- const result = byId('validation')
- result.replaceChildren(
- node(
- 'p',
- data.valid ? 'Valid according to the Go service.' : `${data.diagnostics.length} diagnostic(s)`,
- data.valid ? 'good' : 'bad'
- )
- )
- result.append(node('p', `Referenced tables: ${data.tableNames?.join(', ') || 'none'}`, 'hint'))
- for (const diagnostic of data.diagnostics) {
- const button = node('button', diagnostic.message, 'diagnostic')
- button.type = 'button'
- button.append(node('small', `${diagnostic.code} · UTF-8 bytes ${diagnostic.start}–${diagnostic.end}`))
- if (diagnostic.suggestions?.length) {
- button.append(node('small', `Suggestions: ${diagnostic.suggestions.map((s) => s.label).join(', ')}`))
- }
- button.addEventListener('click', () => {
- if (editor.value !== snapshot) {
- return
- }
- editor.focus()
- editor.setSelectionRange(editorOffset(diagnostic.start), editorOffset(diagnostic.end))
- syncSelection()
- })
- result.append(button)
- }
- })
-}
-
-function insertSuggestion(suggestion, snapshot) {
- if (editor.value !== snapshot.query || editor.selectionStart !== snapshot.position) {
- return
- }
- const before = snapshot.query.slice(0, snapshot.position)
- const prefix = before.match(/[\p{L}\p{N}_$]*$/u)[0]
- const suffix = snapshot.query.slice(snapshot.position).match(/^[\p{L}\p{N}_$]*/u)[0]
- editor.focus()
- editor.setRangeText(
- suggestion.insertText || suggestion.label,
- snapshot.position - prefix.length,
- snapshot.position + suffix.length,
- 'end'
- )
- syncSelection()
-}
-
-function complete(more = false) {
- syncSelection()
- if (requests.completion) {
- return
- }
- const snapshot = { ...selection }
- const cursor = more ? nextCursor : ''
- if (more && (!completionSnapshot || !cursor)) {
- return
- }
- if (!more) {
- byId('suggestions').replaceChildren()
- byId('more').hidden = true
- }
- return request('completion', { ...snapshot, cursor }, (data) => {
- completionSnapshot = snapshot
- nextCursor = data.nextCursor || ''
- for (const suggestion of data.suggestions || []) {
- const button = node('button', '', 'suggestion')
- button.type = 'button'
- button.append(node('span', suggestion.label), node('small', suggestion.detail || suggestion.kind))
- button.title = `Insert ${suggestion.insertText || suggestion.label}`
- button.addEventListener('click', () => insertSuggestion(suggestion, snapshot))
- byId('suggestions').append(button)
- }
- const shown = byId('suggestions').childElementCount
- byId('completion-status').textContent = data.total
- ? `${shown} of ${data.total} suggestions. Click to insert.`
- : 'No suggestions at this cursor. Check the catalog or try another example.'
- if (data.parseError) {
- byId('completion-status').textContent += ` Parser: ${data.parseError}`
- }
- byId('more').hidden = !nextCursor
- })
-}
-
-function renderCatalog() {
- if (!catalog) {
- return
- }
- const filter = byId('catalog-filter').value.toLowerCase()
- const root = byId('catalog')
- root.replaceChildren()
- const groups = Object.entries(catalog.tables).map(([name, table]) => [name, Object.values(table.fields)])
- for (const [name, properties] of Object.entries(catalog.properties)) {
- groups.push([`${name} properties`, properties.map((p) => ({ name: p.name, type: p.property_type }))])
- }
- for (const [name, fields] of groups) {
- const visible = fields.filter((field) => `${name} ${field.name}`.toLowerCase().includes(filter))
- if (!visible.length) {
- continue
- }
- const details = node('details', '')
- details.open = !!filter || name === 'events'
- details.append(node('summary', `${name} (${visible.length})`))
- for (const field of visible.sort((a, b) => a.name.localeCompare(b.name))) {
- const row = node('div', '', 'catalog-field')
- row.append(node('span', field.name), node('span', field.type))
- details.append(row)
- }
- root.append(details)
- }
- if (!root.childElementCount) {
- root.append(node('p', 'No matching fields. Try another search.', 'muted'))
- }
-}
-
-function loadExample() {
- const example = examples[Number(byId('example').value)]
- const position = example.query.indexOf('§')
- editor.value = example.query.replace('§', '')
- editor.focus()
- editor.setSelectionRange(position, position)
- byId('example-note').textContent = example.note
- syncSelection()
-}
-
-for (const [index, example] of examples.entries()) {
- const option = node('option', example.name)
- option.value = String(index)
- byId('example').append(option)
-}
-byId('example').addEventListener('change', loadExample)
-byId('validate').addEventListener('click', validate)
-byId('complete').addEventListener('click', () => complete())
-byId('more').addEventListener('click', () => complete(true))
-byId('catalog-filter').addEventListener('input', renderCatalog)
-for (const kind of Object.keys(feedback)) {
- byId(`${kind}-copy`).addEventListener('click', () => copyFeedback(kind))
-}
-byId('auto-analyze').addEventListener('change', () => {
- scheduleAnalysis()
- if (!byId('auto-analyze').checked) {
- for (const kind of Object.keys(requests)) {
- if (requests[kind]) {
- invalidate(kind)
- }
- }
- }
-})
-editor.addEventListener('compositionstart', () => {
- composing = true
- cancelScheduledAnalysis()
-})
-editor.addEventListener('compositionend', () => {
- composing = false
- syncSelection()
- scheduleAnalysis()
-})
-window.addEventListener('pagehide', () => {
- cancelScheduledAnalysis()
- stopHealthChecks()
- for (const kind of Object.keys(requests)) {
- if (requests[kind]) {
- invalidate(kind)
- }
- }
-})
-document.addEventListener('visibilitychange', () => {
- if (document.hidden) {
- stopHealthChecks()
- } else {
- resumeHealthChecks()
- }
-})
-window.addEventListener('pageshow', resumeHealthChecks)
-for (const event of ['input', 'keyup', 'click', 'select']) {
- editor.addEventListener(event, syncSelection)
-}
-editor.addEventListener('keydown', (event) => {
- if ((event.ctrlKey || event.metaKey) && event.key === 'Enter') {
- event.preventDefault()
- if (event.shiftKey) {
- complete()
- } else {
- validate()
- }
- }
-})
-loadExample()
-resumeHealthChecks()
-fetch('/api/catalog')
- .then(async (response) => {
- if (!response.ok) {
- throw new Error(`HTTP ${response.status}`)
- }
- const publication = await response.json()
- catalog = publication.catalog
- renderCatalog()
- byId('connection').textContent =
- `${Object.keys(catalog.tables).length} synthetic tables loaded · ${publication.revision}`
- })
- .catch((error) => {
- byId('connection').textContent =
- `Could not load the catalog: ${error.message}. Refresh the page or restart the demo.`
- })
diff --git a/services/hogql-language-service/cmd/demo/assets/index.html b/services/hogql-language-service/cmd/demo/assets/index.html
deleted file mode 100644
index 83a61b74a5ce..000000000000
--- a/services/hogql-language-service/cmd/demo/assets/index.html
+++ /dev/null
@@ -1,152 +0,0 @@
-<!doctype html>
-<html lang="en">
- <head>
- <meta charset="utf-8" />
- <meta name="viewport" content="width=device-width, initial-scale=1" />
- <title>HogQL local playground</title>
- <link rel="stylesheet" href="/style.css" />
- <script src="/app.js" defer></script>
- </head>
- <body>
- <header>
- <div>
- <span class="eyebrow">Developer tools</span>
- <div class="title-row">
- <h1>HogQL playground</h1>
- <span id="server-status" class="server-status" role="status" aria-live="polite">
- <span id="server-status-dot" class="server-status-dot" aria-hidden="true"></span>
- <span id="server-status-text">Checking server</span>
- </span>
- </div>
- </div>
- <span class="badge">Local only · synthetic catalog</span>
- </header>
- <p class="intro">
- Try autocomplete and validation against a separate instance of the Go language service. Queries are
- analyzed, never executed.
- </p>
- <main>
- <section class="workspace">
- <div class="editor panel">
- <div class="section-heading">
- <h2>Query</h2>
- <span id="position" class="muted"></span>
- </div>
- <label for="example">Load an example</label>
- <select id="example"></select>
- <p id="example-note" class="hint"></p>
- <label for="query" class="sr-only">HogQL query</label>
- <textarea
- id="query"
- spellcheck="false"
- autocapitalize="off"
- autocomplete="off"
- aria-describedby="editor-help"
- ></textarea>
- <div class="toolbar">
- <button id="validate" class="primary" type="button">Validate</button>
- <button id="complete" type="button">Complete at cursor</button>
- <label class="checkbox-label">
- <input id="auto-analyze" type="checkbox" />
- Analyze as you type
- </label>
- <span id="editor-help" class="hint">Ctrl/⌘ Enter: validate · Ctrl/⌘ Shift Enter: complete</span>
- </div>
- <p id="connection" class="hint" role="status">Loading the synthetic catalog…</p>
- </div>
- <div class="results">
- <section class="panel" aria-labelledby="validation-title">
- <div class="section-heading">
- <h2 id="validation-title">Validation</h2>
- <span id="validation-time" class="muted"></span>
- </div>
- <div id="validation" aria-live="polite">
- <p class="muted">Validate the query to see diagnostics.</p>
- </div>
- <button id="validation-copy" type="button" disabled>Copy request and response</button>
- <p id="validation-copy-status" class="hint" role="status"></p>
- <details id="validation-feedback-details">
- <summary>Last validation request and response</summary>
- <textarea
- id="validation-feedback"
- class="feedback-text"
- readonly
- spellcheck="false"
- aria-label="Validation request and response"
- ></textarea>
- </details>
- <details>
- <summary>Raw validation response</summary>
- <pre id="validation-json">No request yet.</pre>
- </details>
- </section>
- <section class="panel" aria-labelledby="completion-title">
- <div class="section-heading">
- <h2 id="completion-title">Completion</h2>
- <span id="completion-time" class="muted"></span>
- </div>
- <p id="completion-status" class="hint" aria-live="polite">
- Place the cursor where you want suggestions.
- </p>
- <div id="suggestions"></div>
- <button id="more" type="button" hidden>Load more</button>
- <button id="completion-copy" type="button" disabled>Copy request and response</button>
- <p id="completion-copy-status" class="hint" role="status"></p>
- <details id="completion-feedback-details">
- <summary>Last completion request and response</summary>
- <textarea
- id="completion-feedback"
- class="feedback-text"
- readonly
- spellcheck="false"
- aria-label="Completion request and response"
- ></textarea>
- </details>
- <details>
- <summary>Raw completion response</summary>
- <pre id="completion-json">No request yet.</pre>
- </details>
- </section>
- </div>
- </section>
- <aside class="panel" aria-labelledby="catalog-title">
- <div class="section-heading">
- <h2 id="catalog-title">Synthetic catalog</h2>
- <a href="/api/catalog" target="_blank" rel="noopener">View JSON</a>
- </div>
- <p class="hint">
- A small PostHog-like schema with invented warehouse tables and property names. No project data is
- loaded.
- </p>
- <label for="catalog-filter">Filter fields and properties</label>
- <input id="catalog-filter" type="search" placeholder="For example, timestamp" />
- <div id="catalog"></div>
- <details class="limitations">
- <summary>Scope and limitations</summary>
- <p>
- This page calls the real Go autocomplete and validate endpoints. It does not run SQL or compare
- results with Django.
- </p>
- <p>
- CTE and subquery examples help expose gaps in the service. The page displays its responses
- without filling in suggestions itself.
- </p>
- <p>
- The catalog is a hand-maintained subset. Virtual joins, all product tables, and project-specific
- schemas are not modeled completely.
- </p>
- <p>
- Enable Analyze as you type to validate and complete after a short typing pause, or use the
- buttons and shortcuts. Click a suggestion to insert it. Results clear when you edit the query or
- move the completion cursor.
- </p>
- <p>
- Follow-up work and exclusions are tracked in
- <code>docs/internal/hogql-language-service-demo.md</code>.
- </p>
- </details>
- </aside>
- </main>
- <footer>UTF-16 editor positions · In-memory catalog · No Django or ClickHouse required</footer>
- </body>
-</html>
diff --git a/services/hogql-language-service/cmd/demo/assets/style.css b/services/hogql-language-service/cmd/demo/assets/style.css
deleted file mode 100644
index 9ad78ec34ada..000000000000
--- a/services/hogql-language-service/cmd/demo/assets/style.css
+++ /dev/null
@@ -1,380 +0,0 @@
-:root {
- font-family: system-ui, sans-serif;
- color: #242424;
- color-scheme: light;
- background: #f5f4f0;
-}
-
-* {
- box-sizing: border-box;
-}
-
-body {
- max-width: 1600px;
- padding: 32px;
- margin: 0 auto;
-}
-
-header,
-.section-heading,
-.toolbar {
- display: flex;
- flex-wrap: wrap;
- gap: 12px;
- align-items: center;
- justify-content: space-between;
-}
-
-.checkbox-label {
- display: inline-flex;
- gap: 7px;
- align-items: center;
- margin: 0;
- font-weight: 400;
- cursor: pointer;
-}
-
-.checkbox-label input {
- width: auto;
- margin: 0;
- accent-color: #242424;
-}
-
-h1 {
- margin: 5px 0 0;
- font-size: 28px;
- letter-spacing: -0.7px;
-}
-
-.title-row,
-.server-status {
- display: flex;
- gap: 8px;
- align-items: center;
-}
-
-.title-row {
- flex-wrap: wrap;
-}
-
-.server-status {
- margin-top: 5px;
- font-size: 12px;
- color: #706c65;
-}
-
-.server-status-dot {
- width: 8px;
- height: 8px;
- background: #9a968d;
- border-radius: 50%;
-}
-
-.server-status.connected .server-status-dot {
- background: #267442;
-}
-
-.server-status.disconnected .server-status-dot {
- background: #ae3329;
-}
-
-h2 {
- margin: 0;
- font-size: 16px;
-}
-
-.eyebrow {
- font-size: 12px;
- color: #666;
-}
-
-.badge {
- padding: 6px 10px;
- font-size: 12px;
- background: #ffefc6;
- border: 1px solid #e6d19d;
- border-radius: 6px;
-}
-
-.intro {
- margin: 20px 0 24px;
- font-size: 14px;
- line-height: 1.6;
- color: #65625d;
-}
-
-main {
- display: grid;
- grid-template-columns: minmax(0, 1fr) 310px;
- gap: 20px;
- align-items: start;
-}
-
-.workspace {
- min-width: 0;
- container-type: inline-size;
-}
-
-.panel {
- min-width: 0;
- padding: 20px;
- background: #fffefa;
- border: 1px solid #dfdcd4;
- border-radius: 10px;
-}
-
-.section-heading {
- margin-bottom: 18px;
-}
-
-.results {
- display: grid;
- grid-template-columns: repeat(2, minmax(0, 1fr));
- gap: 16px;
- margin-top: 16px;
-}
-
-label {
- display: block;
- margin-bottom: 7px;
- font-size: 12px;
- font-weight: 600;
-}
-
-select,
-input,
-textarea,
-button {
- font: inherit;
-}
-
-select,
-input {
- width: 100%;
- padding: 8px 10px;
- font-size: 13px;
- color: inherit;
- background: white;
- border: 1px solid #c9c6be;
- border-radius: 5px;
-}
-
-textarea {
- width: 100%;
- min-height: 260px;
- padding: 16px;
- font:
- 13px/1.8 ui-monospace,
- SFMono-Regular,
- Consolas,
- monospace;
- color: #242424;
- tab-size: 4;
- white-space: pre;
- resize: vertical;
- background: #fff;
- border: 1px solid #c9c6be;
- border-radius: 6px;
-}
-
-.feedback-text {
- min-height: 180px;
- margin-top: 10px;
- font-size: 11px;
- overflow-wrap: anywhere;
- white-space: pre-wrap;
-}
-
-button {
- padding: 8px 12px;
- font-size: 13px;
- color: inherit;
- cursor: pointer;
- background: #fff;
- border: 1px solid #c9c6be;
- border-radius: 5px;
-}
-
-button:hover {
- background: #f2f0eb;
- border-color: #a7a297;
-}
-
-button:disabled {
- cursor: wait;
- opacity: 0.55;
-}
-
-.primary {
- color: #fff;
- background: #242424;
- border-color: #242424;
-}
-
-.primary:hover {
- background: #444;
-}
-
-.toolbar {
- justify-content: flex-start;
- margin-top: 12px;
-}
-
-:focus-visible {
- outline: 2px solid #b86220;
- outline-offset: 3px;
-}
-
-.hint,
-.muted {
- font-size: 12px;
- line-height: 1.6;
- color: #706c65;
-}
-
-.hint {
- margin: 10px 0;
-}
-
-.good {
- color: #267442;
-}
-
-.bad {
- color: #ae3329;
-}
-
-.diagnostic,
-.suggestion {
- display: block;
- width: 100%;
- margin: 7px 0;
- text-align: left;
- overflow-wrap: anywhere;
-}
-
-.diagnostic {
- background: #fff1ee;
- border-color: #eed0c9;
-}
-
-.diagnostic small,
-.suggestion small {
- display: block;
- margin-top: 4px;
- color: #706c65;
-}
-
-.suggestion {
- display: flex;
- gap: 12px;
- align-items: baseline;
- justify-content: space-between;
-}
-
-.suggestion small {
- flex-shrink: 0;
-}
-
-details {
- padding-top: 12px;
- margin-top: 16px;
- font-size: 12px;
- border-top: 1px solid #e8e5de;
-}
-
-summary {
- overflow-wrap: anywhere;
- cursor: pointer;
-}
-
-pre {
- max-height: 260px;
- padding: 12px;
- overflow: auto;
- font:
- 11px/1.6 ui-monospace,
- monospace;
- overflow-wrap: anywhere;
- white-space: pre-wrap;
- background: #f3f1eb;
- border-radius: 5px;
-}
-
-#suggestions {
- max-height: 340px;
- overflow: auto;
-}
-
-#catalog {
- max-height: 620px;
- margin-top: 16px;
- overflow: auto;
-}
-
-.catalog-field {
- display: flex;
- gap: 8px;
- justify-content: space-between;
- padding: 6px 0;
- font:
- 11px/1.5 ui-monospace,
- monospace;
- overflow-wrap: anywhere;
- border-bottom: 1px solid #f0eee8;
-}
-
-.catalog-field span:last-child {
- color: #777;
-}
-
-a {
- font-size: 12px;
- color: #805025;
-}
-
-.limitations {
- line-height: 1.6;
- color: #706c65;
-}
-
-code {
- overflow-wrap: anywhere;
-}
-
-footer {
- margin-top: 24px;
- font-size: 12px;
- color: #807b72;
-}
-
-.sr-only {
- position: absolute;
- width: 1px;
- height: 1px;
- padding: 0;
- margin: -1px;
- overflow: hidden;
- clip: rect(0, 0, 0, 0);
- border: 0;
-}
-
-[hidden] {
- display: none !important;
-}
-
-@container (max-width: 650px) {
- .results {
- grid-template-columns: 1fr;
- }
-}
-
-@media (max-width: 950px) {
- main {
- grid-template-columns: minmax(0, 1fr);
- }
-
- body {
- padding: 20px;
- }
-}
diff --git a/services/hogql-language-service/cmd/demo/catalog.go b/services/hogql-language-service/cmd/demo/catalog.go
deleted file mode 100644
index 8e2d6efe4606..000000000000
--- a/services/hogql-language-service/cmd/demo/catalog.go
+++ /dev/null
@@ -1,79 +0,0 @@
-package main
-
-import (
- "fmt"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
-)
-
-type catalogPublication struct {
- Revision string `json:"revision"`
- Catalog catalog.Catalog `json:"catalog"`
-}
-
-func demoTable(name, kind string, fields map[string]string) catalog.Table {
- table := catalog.Table{Name: name, Type: kind, Fields: make(map[string]catalog.Field, len(fields))}
- for name, fieldType := range fields {
- table.Fields[name] = catalog.Field{Name: name, Type: fieldType}
- }
- return table
-}
-
-func syntheticCatalog() catalogPublication {
- tables := map[string]catalog.Table{}
- for _, table := range []catalog.Table{
- demoTable("events", "posthog", map[string]string{
- "uuid": "uuid", "event": "string", "timestamp": "datetime", "created_at": "datetime",
- "distinct_id": "string", "person_id": "uuid", "properties": "json", "elements_chain": "string",
- "$session_id": "string", "$window_id": "string", "person": "virtual_table",
- "session": "virtual_table", "group_0": "virtual_table", "group_1": "virtual_table",
- }),
- demoTable("persons", "posthog", map[string]string{
- "id": "uuid", "created_at": "datetime", "properties": "json", "is_identified": "boolean", "last_seen_at": "datetime",
- }),
- demoTable("sessions", "posthog", map[string]string{
- "session_id": "string", "distinct_id": "string", "$start_timestamp": "datetime", "$end_timestamp": "datetime",
- "$session_duration": "float", "$pageview_count": "integer", "$autocapture_count": "integer",
- "$entry_current_url": "string", "$exit_current_url": "string", "$entry_pathname": "string", "$channel_type": "string",
- }),
- demoTable("groups", "posthog", map[string]string{
- "key": "string", "index": "integer", "created_at": "datetime", "updated_at": "datetime", "properties": "json",
- }),
- demoTable("postgres.demo.orders", "data_warehouse", map[string]string{
- "id": "integer", "person_id": "uuid", "amount": "float", "currency": "string", "status": "string", "created_at": "datetime",
- }),
- demoTable("demo_customers", "data_warehouse", map[string]string{
- "id": "integer", "email": "string", "plan": "string", "billing address": "string", "café": "string",
- }),
- demoTable("demo_rules", "data_warehouse", map[string]string{
- "active_day": "date", "display_name": "string", "is_enabled": "boolean", "is_archived": "boolean",
- }),
- } {
- tables[table.Name] = table
- }
- properties := map[string][]catalog.Property{
- "event": {
- {Name: "$current_url", ValueType: "String"}, {Name: "$pathname", ValueType: "String"},
- {Name: "$browser", ValueType: "String"}, {Name: "$os", ValueType: "String"},
- {Name: "$device_type", ValueType: "String"}, {Name: "$geoip_country_name", ValueType: "String"},
- {Name: "$geoip_city_name", ValueType: "String"}, {Name: "$referrer", ValueType: "String"},
- {Name: "$utm_source", ValueType: "String"}, {Name: "$session_id", ValueType: "String"},
- {Name: "order_total", ValueType: "Numeric"}, {Name: "button_text", ValueType: "String"},
- },
- "person": {
- {Name: "email", ValueType: "String"}, {Name: "name", ValueType: "String"},
- {Name: "plan", ValueType: "String"}, {Name: "company", ValueType: "String"}, {Name: "$initial_referrer", ValueType: "String"},
- },
- "session": {{Name: "$entry_current_url", ValueType: "String"}, {Name: "$exit_current_url", ValueType: "String"}},
- "group:0": {{Name: "name", ValueType: "String"}, {Name: "industry", ValueType: "String"}, {Name: "employee_count", ValueType: "Numeric"}},
- "group:1": {{Name: "name", ValueType: "String"}, {Name: "region", ValueType: "String"}},
- }
- for index := range 35 {
- properties["event"] = append(properties["event"], catalog.Property{Name: fmt.Sprintf("demo_property_%02d", index), ValueType: "String"})
- }
- return catalogPublication{Revision: "synthetic-demo-v3", Catalog: catalog.Catalog{
- Tables: tables,
- TableAliases: map[string]string{"demo_postgres_orders": "postgres.demo.orders"},
- Properties: properties,
- }}
-}
diff --git a/services/hogql-language-service/cmd/demo/main.go b/services/hogql-language-service/cmd/demo/main.go
deleted file mode 100644
index 48a12894b2bf..000000000000
--- a/services/hogql-language-service/cmd/demo/main.go
+++ /dev/null
@@ -1,157 +0,0 @@
-package main
-
-import (
- "bytes"
- "context"
- "embed"
- "encoding/json"
- "errors"
- "flag"
- "fmt"
- "io"
- "io/fs"
- "log/slog"
- "net"
- "net/http"
- "os"
- "os/signal"
- "strings"
- "syscall"
- "time"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/httpapi"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/ratelimit"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/serviceauth"
-)
-
-// Embedding only in this command keeps the demo out of the production server binary.
-//
-//go:embed assets/*
-var assets embed.FS
-
-func newDemoHandler(host string) (http.Handler, error) {
- publication := syntheticCatalog()
- payload, err := json.Marshal(publication)
- if err != nil {
- return nil, err
- }
- catalogs := catalog.NewRegistry(1, 16<<20, 24*time.Hour)
- if err := catalogs.Put(serviceauth.Authorization{TeamID: 1, UserID: 1}, publication.Revision, catalog.Prepare(&publication.Catalog)); err != nil {
- return nil, err
- }
- preAuthLimiter, err := ratelimit.New(ratelimit.Config{Capacity: 300, RefillPerSec: 100, MaxEntries: 10000, IdleTTL: 10 * time.Minute})
- if err != nil {
- return nil, err
- }
- principalLimiter, err := ratelimit.New(ratelimit.Config{Capacity: 120, RefillPerSec: 60, MaxEntries: 10000, IdleTTL: 10 * time.Minute})
- if err != nil {
- return nil, err
- }
- backend := httpapi.NewHandler(httpapi.Config{
- Catalogs: catalogs,
- Auth: serviceauth.New(nil, true),
- PreAuthLimiter: preAuthLimiter,
- PrincipalLimiter: principalLimiter,
- Logger: slog.Default(),
- })
- return demoHandler(backend, host, payload), nil
-}
-
-func demoHandler(backend http.Handler, host string, payload []byte) http.Handler {
- static, _ := fs.Sub(assets, "assets")
- mux := http.NewServeMux()
- mux.Handle("GET /", http.FileServer(http.FS(static)))
- mux.HandleFunc("GET /api/catalog", func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Content-Type", "application/json")
- // The payload is the JSON-encoded synthetic catalog; application/json and nosniff prevent HTML interpretation.
- // nosemgrep: go.lang.security.audit.xss.no-direct-write-to-responsewriter.no-direct-write-to-responsewriter
- _, _ = w.Write(payload)
- })
- mux.HandleFunc("GET /api/health", func(w http.ResponseWriter, r *http.Request) {
- request, err := http.NewRequestWithContext(r.Context(), http.MethodGet, "/health", nil)
- if err != nil {
- http.Error(w, "Could not check the service. Restart the demo.", http.StatusInternalServerError)
- return
- }
- request.RemoteAddr = r.RemoteAddr
- backend.ServeHTTP(w, request)
- })
- for _, operation := range []string{"autocomplete", "validate"} {
- mux.HandleFunc("POST /api/"+operation, func(w http.ResponseWriter, r *http.Request) {
- body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 128<<10))
- if err != nil {
- http.Error(w, "Request too large. Use a shorter query.", http.StatusRequestEntityTooLarge)
- return
- }
- request, err := http.NewRequestWithContext(r.Context(), http.MethodPost, "/teams/1/users/1/"+operation, bytes.NewReader(body))
- if err != nil {
- http.Error(w, "Could not create request. Restart the demo.", http.StatusInternalServerError)
- return
- }
- request.Header.Set("Content-Type", "application/json")
- request.RemoteAddr = r.RemoteAddr
- backend.ServeHTTP(w, request)
- })
- }
- return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Content-Security-Policy", "default-src 'self'; script-src 'self'; style-src 'self'; connect-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'")
- w.Header().Set("X-Content-Type-Options", "nosniff")
- w.Header().Set("Cache-Control", "no-store")
- origin := r.Header.Get("Origin")
- if (host != "" && r.Host != host) || (origin != "" && origin != "http://"+r.Host && origin != "https://"+r.Host) {
- http.Error(w, "Send requests from the demo page's own address.", http.StatusForbidden)
- return
- }
- if r.Method == http.MethodPost && !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
- http.Error(w, "Send JSON requests from the demo page.", http.StatusUnsupportedMediaType)
- return
- }
- mux.ServeHTTP(w, r)
- })
-}
-
-func run(ctx context.Context, host string, port int) error {
- listener, err := net.Listen("tcp", net.JoinHostPort(host, fmt.Sprint(port)))
- if err != nil {
- return err
- }
- defer listener.Close()
- allowedHost := listener.Addr().String()
- if ip := net.ParseIP(host); ip != nil && ip.IsUnspecified() {
- allowedHost = ""
- }
- handler, err := newDemoHandler(allowedHost)
- if err != nil {
- return err
- }
- server := &http.Server{
- Handler: handler,
- ReadHeaderTimeout: 5 * time.Second,
- ReadTimeout: 10 * time.Second,
- WriteTimeout: 10 * time.Second,
- IdleTimeout: 60 * time.Second,
- }
- go func() {
- <-ctx.Done()
- _ = server.Close()
- }()
- fmt.Printf("\nHogQL demo: http://%s\nSynthetic catalog only. Queries are not executed. Press Ctrl+C to stop.\n\n", listener.Addr())
- err = server.Serve(listener)
- if errors.Is(err, http.ErrServerClosed) {
- return nil
- }
- return err
-}
-
-func main() {
- host := flag.String("host", "127.0.0.1", "bind address; use 0.0.0.0 for port forwarding")
- port := flag.Int("port", 8092, "port for the demo page")
- flag.Parse()
- ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
- defer cancel()
- if err := run(ctx, *host, *port); err != nil && !errors.Is(err, context.Canceled) {
- fmt.Fprintln(os.Stderr, err)
- os.Exit(1)
- }
-}
diff --git a/services/hogql-language-service/cmd/demo/main_test.go b/services/hogql-language-service/cmd/demo/main_test.go
deleted file mode 100644
index 992f2492a9ef..000000000000
--- a/services/hogql-language-service/cmd/demo/main_test.go
+++ /dev/null
@@ -1,146 +0,0 @@
-package main
-
-import (
- "encoding/json"
- "io"
- "net/http"
- "net/http/httptest"
- "strings"
- "testing"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/completion"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/validation"
-)
-
-func TestDemoRejectsForeignBrowserRequests(t *testing.T) {
- for _, test := range []struct {
- name, host, origin, contentType string
- forwarded bool
- status int
- }{
- {name: "foreign host", host: "example.com", contentType: "application/json", status: http.StatusForbidden},
- {name: "foreign origin", host: "127.0.0.1:8092", origin: "https://example.com", contentType: "application/json", status: http.StatusForbidden},
- {name: "opaque origin", host: "127.0.0.1:8092", origin: "null", contentType: "application/json", status: http.StatusForbidden},
- {name: "form post", host: "127.0.0.1:8092", contentType: "application/x-www-form-urlencoded", status: http.StatusUnsupportedMediaType},
- {name: "forwarded foreign origin", host: "demo.example.com", origin: "https://other.example.com", contentType: "application/json", forwarded: true, status: http.StatusForbidden},
- {name: "forwarded opaque origin", host: "demo.example.com", origin: "null", contentType: "application/json", forwarded: true, status: http.StatusForbidden},
- } {
- t.Run(test.name, func(t *testing.T) {
- request := httptest.NewRequest(http.MethodPost, "http://"+test.host+"/api/validate", strings.NewReader(`{"query":"SELECT 1"}`))
- request.Header.Set("Origin", test.origin)
- request.Header.Set("Content-Type", test.contentType)
- response := httptest.NewRecorder()
- allowedHost := "127.0.0.1:8092"
- if test.forwarded {
- allowedHost = ""
- }
- demoHandler(nil, allowedHost, nil).ServeHTTP(response, request)
- if response.Code != test.status {
- t.Fatalf("status = %d, want %d", response.Code, test.status)
- }
- })
- }
-}
-
-func TestDemoEmbeddedService(t *testing.T) {
- handler, err := newDemoHandler("127.0.0.1:8092")
- if err != nil {
- t.Fatal(err)
- }
- for _, test := range []struct {
- method, path, body string
- status int
- }{
- {http.MethodGet, "/api/health", "", http.StatusOK},
- {http.MethodPost, "/api/validate", `{"query":"WITH t AS (SELECT uuid FROM events) SELECT uuid FROM t"}`, http.StatusOK},
- {http.MethodPost, "/api/autocomplete", `{"query":"SELECT events.tim FROM events","position":17}`, http.StatusOK},
- {http.MethodPost, "/api/validate", `{"query":"SELECT 1","unknown":true}`, http.StatusBadRequest},
- {http.MethodPost, "/api/validate", strings.Repeat(" ", (128<<10)+1), http.StatusRequestEntityTooLarge},
- {http.MethodPut, "/teams/1/users/1/catalog", `{}`, http.StatusMethodNotAllowed},
- {http.MethodPost, "/teams/2/users/2/validate", `{"query":"SELECT 1"}`, http.StatusMethodNotAllowed},
- } {
- t.Run(test.method+test.path+"/"+http.StatusText(test.status), func(t *testing.T) {
- request := httptest.NewRequest(test.method, "http://127.0.0.1:8092"+test.path, strings.NewReader(test.body))
- request.Header.Set("Content-Type", "application/json")
- response := httptest.NewRecorder()
- handler.ServeHTTP(response, request)
- if response.Code != test.status {
- t.Fatalf("response = %d %s, want %d", response.Code, response.Body.String(), test.status)
- }
- if test.status != http.StatusOK || test.path == "/api/health" {
- return
- }
- var revision struct {
- CatalogRevision string `json:"catalogRevision"`
- }
- if err := json.Unmarshal(response.Body.Bytes(), &revision); err != nil || revision.CatalogRevision != syntheticCatalog().Revision {
- t.Fatalf("catalog revision = %q (%v)", revision.CatalogRevision, err)
- }
- if test.path == "/api/validate" {
- var result validation.Result
- if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil || !result.Valid || len(result.Diagnostics) != 0 || len(result.TableNames) != 1 || result.TableNames[0] != "events" {
- t.Fatalf("validation = %s (%v)", response.Body.String(), err)
- }
- } else {
- var result completion.Result
- if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil || len(result.Suggestions) != 1 || result.Suggestions[0].Label != "timestamp" {
- t.Fatalf("completion = %s (%v)", response.Body.String(), err)
- }
- }
- })
- }
-}
-
-func TestDemoHealthForwardsServiceFailure(t *testing.T) {
- backend := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.Method != http.MethodGet || r.URL.Path != "/health" {
- t.Fatalf("forwarded request = %s %s", r.Method, r.URL.Path)
- }
- http.Error(w, "unhealthy", http.StatusServiceUnavailable)
- })
- request := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:8092/api/health", nil)
- response := httptest.NewRecorder()
-
- demoHandler(backend, "127.0.0.1:8092", nil).ServeHTTP(response, request)
-
- if response.Code != http.StatusServiceUnavailable {
- t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable)
- }
-}
-
-func TestDemoForwardsLanguageRequests(t *testing.T) {
- for _, test := range []struct {
- operation, host, origin, allowedHost string
- }{
- {"validate", "127.0.0.1:8092", "http://127.0.0.1:8092", "127.0.0.1:8092"},
- {"autocomplete", "127.0.0.1:8092", "http://127.0.0.1:8092", "127.0.0.1:8092"},
- {"validate", "demo.example.com", "https://demo.example.com", ""},
- {"autocomplete", "localhost:9000", "http://localhost:9000", ""},
- } {
- t.Run(test.operation+"/"+test.host, func(t *testing.T) {
- payload := `{"query":"SELECT '😀', e. FROM events AS e","position":16,"positionEncoding":"utf-16","cursor":"MjU="}`
- backend := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- body, err := io.ReadAll(r.Body)
- if err != nil || string(body) != payload || r.URL.Path != "/teams/1/users/1/"+test.operation || r.Method != http.MethodPost {
- t.Errorf("forwarded request = %s %s %s (%v)", r.Method, r.URL.Path, body, err)
- }
- if r.Header.Get("Authorization") != "" || r.Header.Get("Cookie") != "" {
- t.Error("browser credentials forwarded to demo backend")
- }
- w.Header().Set("Content-Type", "application/json")
- w.WriteHeader(http.StatusBadRequest)
- _, _ = w.Write([]byte(`{"error":"demo response"}`))
- })
- request := httptest.NewRequest(http.MethodPost, "http://"+test.host+"/api/"+test.operation, strings.NewReader(payload))
- request.Header.Set("Content-Type", "application/json")
- request.Header.Set("Origin", test.origin)
- request.Header.Set("Authorization", "Bearer fake-demo-token")
- request.Header.Set("Cookie", "session=fake-demo-session")
- response := httptest.NewRecorder()
- demoHandler(backend, test.allowedHost, nil).ServeHTTP(response, request)
- if response.Code != http.StatusBadRequest || response.Body.String() != `{"error":"demo response"}` {
- t.Fatalf("response = %d %s", response.Code, response.Body.String())
- }
- })
- }
-}
diff --git a/services/hogql-language-service/cmd/server/main.go b/services/hogql-language-service/cmd/server/main.go
deleted file mode 100644
index 097a27c80208..000000000000
--- a/services/hogql-language-service/cmd/server/main.go
+++ /dev/null
@@ -1,171 +0,0 @@
-package main
-
-import (
- "errors"
- "fmt"
- "log/slog"
- "math"
- "net"
- "net/http"
- "os"
- "strconv"
- "strings"
- "time"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/httpapi"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/ratelimit"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/serviceauth"
-)
-
-func main() {
- logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
- slog.SetDefault(logger)
- listenAddress := env("LISTEN_ADDR", "127.0.0.1:8091")
- maxCatalogs, err := positiveIntEnv("MAX_CATALOGS", 1024)
- if err != nil {
- fatalConfiguration(err)
- }
- catalogTTL, err := positiveDurationEnv("CATALOG_TTL", 30*time.Minute)
- if err != nil {
- fatalConfiguration(err)
- }
- maxCatalogBytes, err := positiveIntEnv("CATALOG_CACHE_MAX_BYTES", 8<<30)
- if err != nil {
- fatalConfiguration(err)
- }
- keys := splitNonEmpty(os.Getenv("HOGQL_LANGUAGE_SERVICE_SIGNING_KEYS"))
- allowInsecure, err := allowInsecureAuthentication(listenAddress, os.Getenv("HOGQL_LANGUAGE_SERVICE_ALLOW_INSECURE"))
- if err != nil {
- fatalConfiguration(err)
- }
- if len(keys) == 0 && !allowInsecure {
- fatalConfiguration(errors.New("HOGQL_LANGUAGE_SERVICE_SIGNING_KEYS is required"))
- }
- if allowInsecure {
- slog.Warn("authentication disabled for local development", "address", listenAddress)
- }
- maxRateLimitKeys, err := positiveIntEnv("RATE_LIMIT_MAX_KEYS", 10000)
- if err != nil {
- fatalConfiguration(err)
- }
- rateLimitIdleTTL, err := positiveDurationEnv("RATE_LIMIT_IDLE_TTL", 10*time.Minute)
- if err != nil {
- fatalConfiguration(err)
- }
-
- catalogs := catalog.NewRegistry(maxCatalogs, int64(maxCatalogBytes), catalogTTL)
- handler := httpapi.NewHandler(httpapi.Config{
- Catalogs: catalogs,
- Auth: serviceauth.New(keys, allowInsecure),
- PreAuthLimiter: configuredLimiter("PRE_AUTH_RATE_LIMIT", 300, 100, maxRateLimitKeys, rateLimitIdleTTL),
- PrincipalLimiter: configuredLimiter("PRINCIPAL_RATE_LIMIT", 120, 60, maxRateLimitKeys, rateLimitIdleTTL),
- Logger: logger,
- })
- httpServer := &http.Server{
- Addr: listenAddress,
- Handler: handler,
- ReadHeaderTimeout: 5 * time.Second,
- ReadTimeout: 30 * time.Second,
- WriteTimeout: 30 * time.Second,
- IdleTimeout: 60 * time.Second,
- }
- stats := catalogs.Stats()
- slog.Info("HogQL language service listening", "address", listenAddress, "catalogs", stats.Catalogs, "tables", stats.Tables, "properties", stats.Properties)
- if err := httpServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
- slog.Error("server stopped", "error", err)
- os.Exit(1)
- }
-}
-
-func isLoopbackAddress(address string) bool {
- host, _, err := net.SplitHostPort(address)
- if err != nil {
- return false
- }
- return host == "localhost" || net.ParseIP(host).IsLoopback()
-}
-
-func splitNonEmpty(value string) []string {
- var values []string
- for _, item := range strings.Split(value, ",") {
- if item = strings.TrimSpace(item); item != "" {
- values = append(values, item)
- }
- }
- return values
-}
-
-func positiveIntEnv(name string, fallback int) (int, error) {
- value := os.Getenv(name)
- if value == "" {
- return fallback, nil
- }
- parsed, err := strconv.Atoi(value)
- if err != nil || parsed <= 0 {
- return 0, fmt.Errorf("%s must be a positive integer", name)
- }
- return parsed, nil
-}
-
-func positiveDurationEnv(name string, fallback time.Duration) (time.Duration, error) {
- value := os.Getenv(name)
- if value == "" {
- return fallback, nil
- }
- parsed, err := time.ParseDuration(value)
- if err != nil || parsed <= 0 {
- return 0, fmt.Errorf("%s must be a positive duration", name)
- }
- return parsed, nil
-}
-
-func configuredLimiter(prefix string, defaultCapacity, defaultRefill float64, maxEntries int, idleTTL time.Duration) *ratelimit.Limiter {
- capacity, err := positiveFloatEnv(prefix+"_CAPACITY", defaultCapacity)
- if err != nil {
- fatalConfiguration(err)
- }
- refill, err := positiveFloatEnv(prefix+"_REFILL_PER_SECOND", defaultRefill)
- if err != nil {
- fatalConfiguration(err)
- }
- limiter, err := ratelimit.New(ratelimit.Config{Capacity: capacity, RefillPerSec: refill, MaxEntries: maxEntries, IdleTTL: idleTTL})
- if err != nil {
- fatalConfiguration(err)
- }
- return limiter
-}
-
-func positiveFloatEnv(name string, fallback float64) (float64, error) {
- value := os.Getenv(name)
- if value == "" {
- return fallback, nil
- }
- parsed, err := strconv.ParseFloat(value, 64)
- if err != nil || parsed <= 0 || math.IsNaN(parsed) || math.IsInf(parsed, 0) {
- return 0, fmt.Errorf("%s must be a positive number", name)
- }
- return parsed, nil
-}
-
-func allowInsecureAuthentication(listenAddress, configured string) (bool, error) {
- if configured != "1" {
- return false, nil
- }
- if !isLoopbackAddress(listenAddress) {
- return false, errors.New("HOGQL_LANGUAGE_SERVICE_ALLOW_INSECURE requires a loopback LISTEN_ADDR")
- }
- return true, nil
-}
-
-func fatalConfiguration(err error) {
- slog.Error("invalid configuration", "error", err)
- os.Exit(1)
-}
-
-func env(name, fallback string) string {
- if value := os.Getenv(name); value != "" {
- return value
- }
- return fallback
-}
diff --git a/services/hogql-language-service/cmd/server/main_test.go b/services/hogql-language-service/cmd/server/main_test.go
deleted file mode 100644
index dd739754708e..000000000000
--- a/services/hogql-language-service/cmd/server/main_test.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package main
-
-import "testing"
-
-func TestInsecureAuthenticationRequiresExplicitLoopbackOptIn(t *testing.T) {
- for _, test := range []struct {
- address string
- configured string
- allowed bool
- wantError bool
- }{
- {address: "127.0.0.1:8091", configured: "", allowed: false},
- {address: "127.0.0.1:8091", configured: "1", allowed: true},
- {address: "0.0.0.0:8091", configured: "1", allowed: false, wantError: true},
- } {
- allowed, err := allowInsecureAuthentication(test.address, test.configured)
- if allowed != test.allowed || (err != nil) != test.wantError {
- t.Fatalf("address %q configured %q returned allowed=%t error=%v", test.address, test.configured, allowed, err)
- }
- }
-}
diff --git a/services/hogql-language-service/go.mod b/services/hogql-language-service/go.mod
deleted file mode 100644
index d7dbf9ac36f8..000000000000
--- a/services/hogql-language-service/go.mod
+++ /dev/null
@@ -1,5 +0,0 @@
-module github.com/PostHog/posthog/services/hogql-language-service
-
-go 1.27.3
-
-require github.com/orian/clickhouse-sql-parser v1.0.4
diff --git a/services/hogql-language-service/go.sum b/services/hogql-language-service/go.sum
deleted file mode 100644
index 5481d995f0e5..000000000000
--- a/services/hogql-language-service/go.sum
+++ /dev/null
@@ -1,14 +0,0 @@
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/orian/clickhouse-sql-parser v1.0.4 h1:/DKpB9h4Ucw1ZEdERDaR/UceYF78MmEBparYV2UG4ao=
-github.com/orian/clickhouse-sql-parser v1.0.4/go.mod h1:22D/qoWV/Ipbn5Gcub+0ttta1wgzSDaJMprQRZWghoE=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=
-github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
-github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
-github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/services/hogql-language-service/internal/analysis/aliases.go b/services/hogql-language-service/internal/analysis/aliases.go
deleted file mode 100644
index 4b4385191ca9..000000000000
--- a/services/hogql-language-service/internal/analysis/aliases.go
+++ /dev/null
@@ -1,113 +0,0 @@
-package analysis
-
-import (
- "iter"
- "strings"
-
- clickhouse "github.com/orian/clickhouse-sql-parser/parser"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
-)
-
-type selectAlias struct {
- field catalog.Entry
- propertyNamespace string
- end int
- ambiguousAt int
-}
-
-func (s *queryScope) selectAliases() map[string]selectAlias {
- if s.aliases != nil {
- return s.aliases
- }
- s.aliases = map[string]selectAlias{}
- for _, item := range s.query.SelectItems {
- if !s.budget.lookup(1) {
- break
- }
- if item.Alias == nil {
- continue
- }
- name := item.Alias.Name
- if !s.budget.lookup(len(name) + 1) {
- break
- }
- if existing, exists := s.aliases[name]; exists {
- if existing.ambiguousAt == 0 {
- existing.ambiguousAt = int(item.End())
- s.aliases[name] = existing
- }
- continue
- }
- // HogQL resolves each expression before registering its alias (Resolver.visit_alias).
- field := catalog.Entry{Name: name, Type: projectedType(s, item.Expr)}
- namespace, _ := projectedPropertyNamespace(s, item.Expr, int(item.Expr.Pos()))
- s.aliases[name] = selectAlias{field: field, propertyNamespace: namespace, end: int(item.End())}
- }
- return s.aliases
-}
-
-func (b Bindings) aliasCutoff() int {
- if b.scope == nil {
- return -1
- }
- q := b.scope.query
- items := q.SelectItems
- if len(items) > 0 && int(items[0].Pos()) <= b.position && b.position <= int(items[len(items)-1].End()) {
- return b.position
- }
- containsPosition := func(expr clickhouse.Expr) bool {
- return int(expr.Pos()) <= b.position && b.position <= int(expr.End())
- }
- // Resolver.visit_select_query resolves FROM/JOIN before SELECT, then these clauses.
- if q.Where != nil && containsPosition(q.Where) ||
- q.Prewhere != nil && containsPosition(q.Prewhere) ||
- q.GroupBy != nil && containsPosition(q.GroupBy) ||
- q.Having != nil && containsPosition(q.Having) ||
- q.OrderBy != nil && containsPosition(q.OrderBy) ||
- q.Window != nil && containsPosition(q.Window) ||
- q.LimitBy != nil && containsPosition(q.LimitBy) ||
- q.Limit != nil && containsPosition(q.Limit) {
- return int(q.End())
- }
- return -1
-}
-
-func (b Bindings) SelectAlias(name string) (catalog.Entry, bool) {
- alias, ok := b.selectAlias(name)
- return alias.field, ok
-}
-
-func (b Bindings) ResolvedSelectAlias(name string) (catalog.Entry, bool) {
- alias, ok := b.selectAlias(name)
- return alias.field, ok && (alias.ambiguousAt == 0 || alias.ambiguousAt > b.aliasCutoff())
-}
-
-func (b Bindings) selectAlias(name string) (selectAlias, bool) {
- cutoff := b.aliasCutoff()
- if cutoff < 0 || !b.scope.budget.lookup(len(name)+1) {
- return selectAlias{}, false
- }
- alias, ok := b.scope.selectAliases()[name]
- if ok && alias.ambiguousAt != 0 && alias.ambiguousAt <= cutoff {
- alias.propertyNamespace = ""
- }
- return alias, ok && alias.end <= cutoff
-}
-
-func (b Bindings) SelectAliases(prefix string) iter.Seq[catalog.Entry] {
- return func(yield func(catalog.Entry) bool) {
- cutoff := b.aliasCutoff()
- if cutoff < 0 {
- return
- }
- for _, alias := range b.scope.selectAliases() {
- if !b.scope.budget.lookup(len(alias.field.Name) + 1) {
- return
- }
- if alias.end <= cutoff && strings.HasPrefix(strings.ToLower(alias.field.Name), prefix) && !yield(alias.field) {
- return
- }
- }
- }
-}
diff --git a/services/hogql-language-service/internal/analysis/document.go b/services/hogql-language-service/internal/analysis/document.go
deleted file mode 100644
index 1b31dafe1591..000000000000
--- a/services/hogql-language-service/internal/analysis/document.go
+++ /dev/null
@@ -1,484 +0,0 @@
-package analysis
-
-import (
- "iter"
- "slices"
- "sort"
- "strings"
-
- clickhouse "github.com/orian/clickhouse-sql-parser/parser"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/propertyresolver"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/querylimits"
-)
-
-// A Document belongs to one request. Lazy projections share its budget and must not run concurrently.
-type Document struct {
- statements []*Statement
- budget projectionBudget
-}
-
-type Statement struct {
- expr clickhouse.Expr
- schema *catalog.PreparedCatalog
- originalTableNames map[int]string
- budget *projectionBudget
- scopes []*queryScope
- tables []TableReference
- resolvedTables []TableReference
- analyzed bool
-}
-
-type TableReference struct {
- Name string
- Canonical string
- CTE bool
- Start, End int
- Known bool
-}
-
-type Bindings struct {
- relations map[string]Relation
- scope *queryScope
- position int
-}
-
-func Analyze(schema *catalog.PreparedCatalog, query string) (*Document, error) {
- if err := querylimits.Validate(query); err != nil {
- return nil, err
- }
- parserQuery, originalTableNames := normalizeHogQLTableReferences(query)
- statements, err := clickhouse.NewParser(parserQuery).ParseStmts()
- if err != nil {
- return nil, err
- }
- document := &Document{budget: projectionBudget{
- remaining: querylimits.MaxCTEProjectedFields, lookupRemaining: querylimits.MaxFieldLookupWork,
- }}
- for _, expr := range statements {
- document.statements = append(document.statements, &Statement{
- expr: expr, schema: schema, originalTableNames: originalTableNames, budget: &document.budget,
- })
- }
- return document, nil
-}
-
-func (d *Document) Statements() iter.Seq[*Statement] {
- return func(yield func(*Statement) bool) {
- for _, statement := range d.statements {
- // Validation stops between statements when projection work exhausts the request budget.
- statement.analyze()
- if !yield(statement) {
- return
- }
- }
- }
-}
-
-func (d *Document) LimitError() error {
- if d.budget.exceeded {
- return querylimits.ErrCTEProjectionTooLarge
- }
- if d.budget.lookupExceeded {
- return querylimits.ErrFieldLookupTooLarge
- }
- if d.budget.relationExceeded {
- return querylimits.ErrRelationTraversalTooDeep
- }
- return nil
-}
-
-func (s *Statement) analyze() {
- if s.analyzed {
- return
- }
- s.analyzed = true
- s.scopes = queryScopes(s.expr, s.budget)
- bindTables := func(node clickhouse.Expr) bool {
- expr, ok := node.(*clickhouse.TableExpr)
- if !ok {
- return true
- }
- if bindSubquery(expr, s.scopes, s.budget) {
- return true
- }
- name, alias, implicitAlias, start, end, ok := tableReference(expr)
- if !ok {
- return true
- }
- scope := innermostScope(s.scopes, start, end)
- if scope == nil {
- return true
- }
- if original, exists := s.originalTableNames[start]; exists {
- name = original
- implicitAlias = strings.ReplaceAll(original, ".", "__")
- }
- if cte := resolveCTE(scope, name, start); cte != nil {
- addBinding(scope, name, alias, Relation{name: cte.name, cte: cte}, start, end)
- s.resolvedTables = append(s.resolvedTables, TableReference{Name: name, Canonical: cte.name, CTE: true, Start: start, End: end, Known: true})
- return true
- }
- table, exists := s.schema.Table(name)
- s.tables = append(s.tables, TableReference{Name: name, Start: start, End: end, Known: exists})
- if exists {
- s.resolvedTables = append(s.resolvedTables, TableReference{Name: name, Canonical: table.Name, Start: start, End: end, Known: true})
- if alias == "" && implicitAlias != name {
- // HogQL registers multi-part table paths under a double-underscore alias.
- alias = implicitAlias
- }
- addBinding(scope, name, alias, Relation{name: name, table: table}, start, end)
- }
- return true
- }
- walkIncludingExcept(s.expr, bindTables)
-}
-
-// Walk borrows parser nodes for validation; callers must not mutate them or retain them across requests.
-func (s *Statement) Walk(visit func(clickhouse.Expr) bool) {
- walkIncludingExcept(s.expr, visit)
-}
-
-func (s *Statement) Tables() iter.Seq[TableReference] {
- return slices.Values(s.tables)
-}
-
-func (s *Statement) ResolvedTables() iter.Seq[TableReference] {
- return slices.Values(s.resolvedTables)
-}
-
-func (s *Statement) DuplicateSources() iter.Seq[Source] {
- return func(yield func(Source) bool) {
- for _, scope := range s.scopes {
- for _, source := range scope.duplicateSources {
- if !yield(source) {
- return
- }
- }
- }
- }
-}
-
-func (s *Statement) ContainsPosition(position int) bool {
- if int(s.expr.Pos()) <= position && position <= int(s.expr.End()) {
- return true
- }
- for _, scope := range s.scopes {
- if contains(scope.query, position, position) {
- return true
- }
- }
- return false
-}
-
-func (s *Statement) BindingsAt(start, end int) Bindings {
- scope := innermostScope(s.scopes, start, end)
- if scope == nil {
- return Bindings{}
- }
- if scope.visible == nil {
- scope.visible = visibleBindings(scope)
- }
- return Bindings{relations: scope.visible, scope: scope, position: start}
-}
-
-// Qualified completion can refer to a visible CTE before the user has typed FROM.
-func (s *Statement) RelationAt(name string, position int) (Relation, bool) {
- if relation, ok := s.BindingsAt(position, position).Relation(name); ok {
- return relation, true
- }
- if cte := resolveCTE(innermostScope(s.scopes, position, position), name, position); cte != nil {
- return Relation{name: cte.name, cte: cte}, true
- }
- return Relation{}, false
-}
-
-func (b Bindings) Len() int {
- return len(b.relations)
-}
-
-func (b Bindings) CTENames(prefix string) iter.Seq[catalog.Entry] {
- return func(yield func(catalog.Entry) bool) {
- seen := map[string]bool{}
- prefix = foldedFieldName(prefix)
- for scope := b.scope; scope != nil; scope = scope.parent {
- ctes := scope.visibleCTEs(b.position)
- for index := len(ctes) - 1; index >= 0; index-- {
- name := ctes[index].name
- if !scope.budget.lookup(len(name) + 1) {
- return
- }
- if seen[name] {
- continue
- }
- seen[name] = true
- if strings.HasPrefix(foldedFieldName(name), prefix) && !yield(catalog.Entry{Name: name, Type: "CTE"}) {
- return
- }
- }
- }
- }
-}
-
-func (b Bindings) Relation(name string) (Relation, bool) {
- relation, ok := b.relations[name]
- return relation, ok
-}
-
-func (b Bindings) UnambiguousRelation(name string) (Relation, bool) {
- relation, ok := b.Relation(name)
- return relation, ok && b.scope != nil && !b.scope.hasDuplicateSource(name)
-}
-
-func (b Bindings) ResolvedField(name string) (catalog.Entry, bool) {
- if b.scope == nil {
- return catalog.Entry{}, false
- }
- var found catalog.Entry
- matches := 0
- for source := range b.sources() {
- if b.scope.hasDuplicateSource(source.name) {
- return catalog.Entry{}, false
- }
- if _, ok := source.relation.Field(name); ok {
- matches++
- field, resolved := source.relation.ResolvedField(name)
- if !resolved {
- return catalog.Entry{}, false
- }
- found = field
- }
- }
- return found, matches == 1 && !b.scope.budget.lookupExceeded
-}
-
-func (b Bindings) All() iter.Seq2[string, Relation] {
- return func(yield func(string, Relation) bool) {
- for name, relation := range b.relations {
- if !yield(name, relation) {
- return
- }
- }
- }
-}
-
-func (b Bindings) UniqueRelations() iter.Seq[Relation] {
- if b.scope == nil {
- return slices.Values([]Relation(nil))
- }
- return slices.Values(b.scope.uniqueBindings())
-}
-
-func (b Bindings) PropertyNamespace(parts []string) (string, bool) {
- if len(parts) < 2 {
- return "", false
- }
- if target := b.Traversal(parts[:len(parts)-1]); target.Explicit {
- return target.PropertyNamespace, target.Valid && target.PropertyNamespace != "" && !target.HasProperty
- }
- if len(parts) > 2 && b.scope.hasDuplicateSource(parts[0]) {
- return "", false
- }
- if len(parts) >= 2 {
- ownerParts := parts[:len(parts)-1]
- if len(ownerParts) == 1 {
- if alias, ok := b.selectAlias(ownerParts[0]); ok {
- return alias.propertyNamespace, alias.propertyNamespace != ""
- }
- _, qualified := b.Relation(ownerParts[0])
- if !qualified && resolveCTE(b.scope, ownerParts[0], b.position) == nil {
- namespace, ok, matched := b.scope.unqualifiedPropertyNamespace(ownerParts[0])
- if ok {
- return namespace, true
- }
- if matched {
- return "", false
- }
- }
- }
- if len(ownerParts) == 2 {
- if relation, ok := b.Relation(ownerParts[0]); ok {
- return bindingPropertyNamespace(relation, ownerParts[1])
- }
- }
- }
- if len(parts) >= 2 {
- _, bound := b.Relation(parts[0])
- if _, shadowed := b.SelectAlias(parts[0]); shadowed {
- if len(parts) == 2 || !bound {
- return "", false
- }
- }
- }
- if len(parts) > 2 {
- if _, bound := b.Relation(parts[0]); !bound {
- if resolveCTE(b.scope, parts[0], b.position) != nil || len(parts) > 3 {
- return "", false
- }
- }
- }
- names := make(map[string]string, len(b.relations))
- for name, relation := range b.relations {
- if relation.cte != nil {
- if len(parts) > 2 && parts[0] == name {
- return "", false
- }
- if len(parts) == 2 {
- if _, hasProperties := relation.Field("properties"); hasProperties || relation.cte.budget.exceeded || relation.cte.budget.lookupExceeded {
- return "", false
- }
- }
- continue
- }
- names[name] = relation.table.Name
- }
- return propertyresolver.Resolve(parts, names)
-}
-
-type TraversalTarget struct {
- Fields *catalog.PreparedFields
- PropertyNamespace string
- Explicit bool
- Valid bool
- Failed bool
- FailureAt int
- HasProperty bool
- PropertyAt int
-}
-
-func (b Bindings) Traversal(parts []string) TraversalTarget {
- if len(parts) == 0 || b.scope == nil {
- return TraversalTarget{}
- }
- var fields *catalog.PreparedFields
- var relationView *catalog.PreparedRelation
- index := 0
- if relation, ok := b.Relation(parts[0]); ok {
- if b.scope.hasDuplicateSource(parts[0]) {
- return TraversalTarget{Explicit: true}
- }
- if relation.table == nil {
- return TraversalTarget{}
- }
- fields = &relation.table.Fields
- index = 1
- } else {
- matches := 0
- annotated := 0
- for source := range b.sources() {
- if _, ok := source.relation.Field(parts[0]); !ok {
- continue
- }
- matches++
- if b.scope.hasDuplicateSource(source.name) {
- return TraversalTarget{Explicit: true}
- }
- if source.relation.table != nil {
- if _, ok := source.relation.table.Fields.Traversal(parts[0]); ok {
- fields = &source.relation.table.Fields
- annotated++
- }
- }
- }
- if annotated == 0 {
- return TraversalTarget{}
- }
- if alias, ok := b.selectAlias(parts[0]); ok {
- if len(parts) == 1 && alias.propertyNamespace != "" {
- return TraversalTarget{PropertyNamespace: alias.propertyNamespace, Explicit: true, Valid: true}
- }
- return TraversalTarget{Explicit: true}
- }
- if matches != 1 || annotated != 1 {
- return TraversalTarget{Explicit: true}
- }
- }
- hops := 0
- explicit := false
- for ; index < len(parts); index++ {
- if !b.scope.budget.lookup(len(parts[index]) + 1) {
- return TraversalTarget{Explicit: explicit}
- }
- entry, ok := fields.Exact(parts[index])
- if !ok {
- return TraversalTarget{Fields: fields, Explicit: explicit, Failed: explicit, FailureAt: index}
- }
- var traversal catalog.FieldTraversal
- if relationView != nil {
- traversal, ok = relationView.Traversal(parts[index])
- } else {
- traversal, ok = fields.Traversal(parts[index])
- }
- if !ok {
- if explicit && strings.EqualFold(entry.Type, "JSON") {
- return TraversalTarget{Explicit: true}
- }
- return TraversalTarget{Fields: fields, Explicit: explicit, Failed: explicit, FailureAt: index + 1}
- }
- explicit = true
- if traversal.PropertyNamespace != "" {
- if index != len(parts)-1 {
- return TraversalTarget{
- PropertyNamespace: traversal.PropertyNamespace,
- Explicit: true,
- Valid: true,
- HasProperty: true,
- PropertyAt: index + 1,
- }
- }
- return TraversalTarget{PropertyNamespace: traversal.PropertyNamespace, Explicit: true, Valid: true}
- }
- hops++
- if hops > querylimits.MaxRelationTraversalHops {
- b.scope.budget.relationExceeded = true
- return TraversalTarget{Explicit: true}
- }
- if traversal.Relation == nil {
- return TraversalTarget{Explicit: true}
- }
- relationView = traversal.Relation
- fields = traversal.Relation.Fields
- }
- return TraversalTarget{Fields: fields, Explicit: explicit, Valid: fields != nil}
-}
-
-func (r Relation) Name() string {
- return r.name
-}
-
-func (r Relation) Field(name string) (catalog.Entry, bool) {
- return bindingField(r, name)
-}
-
-func (r Relation) ResolvedField(name string) (catalog.Entry, bool) {
- field, ok := bindingField(r, name)
- if !ok || r.cte == nil {
- return field, ok
- }
- projected, exists := r.cte.fieldIndex[foldedFieldName(name)]
- return field, exists && !projected.ambiguous
-}
-
-// Fields yields values without copying catalog indexes or exposing their backing slices.
-func (r Relation) Fields() iter.Seq[catalog.Entry] {
- return slices.Values(bindingFields(r))
-}
-
-// Physical prefixes borrow the catalog index; derived projections have a request-wide size bound.
-func (r Relation) Prefix(prefix string) iter.Seq[catalog.Entry] {
- if r.table != nil {
- return slices.Values(r.table.Fields.Prefix(prefix))
- }
- var fields []catalog.Entry
- seen := map[string]bool{}
- for field := range r.Fields() {
- name := strings.ToLower(field.Name)
- if strings.HasPrefix(name, prefix) && !seen[name] {
- fields = append(fields, field)
- seen[name] = true
- }
- }
- sort.Slice(fields, func(i, j int) bool { return strings.ToLower(fields[i].Name) < strings.ToLower(fields[j].Name) })
- return slices.Values(fields)
-}
diff --git a/services/hogql-language-service/internal/analysis/literals.go b/services/hogql-language-service/internal/analysis/literals.go
deleted file mode 100644
index 71608d2170cc..000000000000
--- a/services/hogql-language-service/internal/analysis/literals.go
+++ /dev/null
@@ -1,11 +0,0 @@
-package analysis
-
-import (
- "strings"
-
- clickhouse "github.com/orian/clickhouse-sql-parser/parser"
-)
-
-func IsBooleanLiteral(ident *clickhouse.Ident) bool {
- return ident.QuoteType == clickhouse.Unquoted && (strings.EqualFold(ident.Name, "TRUE") || strings.EqualFold(ident.Name, "FALSE"))
-}
diff --git a/services/hogql-language-service/internal/analysis/scopes.go b/services/hogql-language-service/internal/analysis/scopes.go
deleted file mode 100644
index a09275eefffd..000000000000
--- a/services/hogql-language-service/internal/analysis/scopes.go
+++ /dev/null
@@ -1,665 +0,0 @@
-package analysis
-
-import (
- "regexp"
- "strings"
- "unicode"
-
- clickhouse "github.com/orian/clickhouse-sql-parser/parser"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/propertyresolver"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/querylimits"
-)
-
-type Relation struct {
- name string
- table *catalog.PreparedTable
- cte *cteBinding
-}
-
-type cteBinding struct {
- name string
- query *clickhouse.SelectQuery
- scope *queryScope
- budget *projectionBudget
- fields []projectedField
- entries []catalog.Entry
- fieldIndex map[string]projectedField
- fieldsDone bool
- resolving bool
-}
-
-type projectedField struct {
- entry catalog.Entry
- propertyNamespace string
- ambiguous bool
-}
-
-type projectionBudget struct {
- remaining int
- exceeded bool
- lookupRemaining int
- lookupExceeded bool
- relationExceeded bool
-}
-
-type queryScope struct {
- query *clickhouse.SelectQuery
- parent *queryScope
- bindings map[string]Relation
- sources []Source
- visible map[string]Relation
- unique []Relation
- budget *projectionBudget
- ctes []*cteBinding
- cteRoot bool
- aliases map[string]selectAlias
- propertyNamespaces map[string]propertyNamespace
- sourceNames map[string]int
- duplicateSources []Source
-}
-
-type propertyNamespace struct {
- name string
- ok bool
- matched bool
-}
-
-var tableReferencePattern = regexp.MustCompile(`(?i)\b(?:FROM|JOIN)\s+([A-Za-z_][A-Za-z0-9_.$]*)`)
-
-func queryScopes(statement clickhouse.Expr, budget *projectionBudget) []*queryScope {
- var scopes []*queryScope
- byQuery := map[*clickhouse.SelectQuery]*queryScope{}
- walkIncludingExcept(statement, func(node clickhouse.Expr) bool {
- if query, ok := node.(*clickhouse.SelectQuery); ok {
- scope := &queryScope{query: query, bindings: map[string]Relation{}, budget: budget}
- scopes = append(scopes, scope)
- byQuery[query] = scope
- }
- return true
- })
- for _, scope := range scopes {
- for _, candidate := range scopes {
- if scope == candidate || span(candidate.query) <= span(scope.query) || !contains(candidate.query, int(scope.query.Pos()), int(scope.query.End())) {
- continue
- }
- if scope.parent == nil || span(candidate.query) < span(scope.parent.query) {
- scope.parent = candidate
- }
- }
- if scope.parent != nil && (scope.parent.query.UnionAll == scope.query || scope.parent.query.UnionDistinct == scope.query || scope.parent.query.Except == scope.query) {
- scope.cteRoot = true
- }
- }
- for _, scope := range scopes {
- if scope.query.With == nil {
- continue
- }
- for _, statement := range scope.query.With.CTEs {
- name, nameOK := statement.Expr.(*clickhouse.Ident)
- query, queryOK := statement.Alias.(*clickhouse.SelectQuery)
- if !nameOK || !queryOK {
- continue
- }
- cte := &cteBinding{name: name.Name, query: query, scope: byQuery[query], budget: budget}
- if cte.scope != nil {
- cte.scope.cteRoot = true
- }
- scope.ctes = append(scope.ctes, cte)
- }
- }
- return scopes
-}
-
-func walkIncludingExcept(expr clickhouse.Expr, visit func(clickhouse.Expr) bool) {
- var exceptBranches []*clickhouse.SelectQuery
- seenExcept := map[*clickhouse.SelectQuery]bool{}
- wrapper := func(node clickhouse.Expr) bool {
- walkChildren := visit(node)
- if query, ok := node.(*clickhouse.SelectQuery); ok && walkChildren && query.Except != nil && !seenExcept[query.Except] {
- seenExcept[query.Except] = true
- exceptBranches = append(exceptBranches, query.Except)
- }
- return walkChildren
- }
- clickhouse.Walk(expr, wrapper)
- for len(exceptBranches) > 0 {
- branch := exceptBranches[0]
- exceptBranches = exceptBranches[1:]
- clickhouse.Walk(branch, wrapper)
- }
-}
-
-func addBinding(scope *queryScope, name, alias string, binding Relation, start, end int) {
- scope.bindings[name] = binding
- source := Source{name: name, relation: binding, start: start, end: end}
- if alias != "" {
- scope.bindings[alias] = binding
- source.name = alias
- }
- scope.sources = append(scope.sources, source)
- if scope.sourceNames == nil {
- scope.sourceNames = map[string]int{}
- }
- scope.sourceNames[source.name]++
- if scope.sourceNames[source.name] == 2 && len(scope.duplicateSources) < querylimits.MaxDiagnostics {
- scope.duplicateSources = append(scope.duplicateSources, source)
- }
-}
-
-func (s *queryScope) hasDuplicateSource(name string) bool {
- for current := s; current != nil; current = current.parent {
- if !s.budget.lookup(len(name) + 1) {
- return true
- }
- if count, visible := current.sourceNames[name]; visible {
- return count > 1
- }
- if current.cteRoot {
- break
- }
- }
- return false
-}
-
-func (s *queryScope) visibleCTEs(position int) []*cteBinding {
- for index, cte := range s.ctes {
- if contains(cte.query, position, position) {
- return s.ctes[:index]
- }
- }
- return s.ctes
-}
-
-func resolveCTE(scope *queryScope, name string, position int) *cteBinding {
- for current := scope; current != nil; current = current.parent {
- ctes := current.visibleCTEs(position)
- for index := len(ctes) - 1; index >= 0; index-- {
- if ctes[index].name == name {
- return ctes[index]
- }
- }
- }
- return nil
-}
-
-func innermostScope(scopes []*queryScope, start, end int) *queryScope {
- var found *queryScope
- for _, scope := range scopes {
- if contains(scope.query, start, end) && (found == nil || span(scope.query) < span(found.query)) {
- found = scope
- }
- }
- return found
-}
-
-func contains(query *clickhouse.SelectQuery, start, end int) bool {
- return int(query.Pos()) <= start && end <= int(query.End())
-}
-
-func span(query *clickhouse.SelectQuery) int {
- return int(query.End() - query.Pos())
-}
-
-func visibleBindings(scope *queryScope) map[string]Relation {
- if scope.visible != nil {
- return scope.visible
- }
- bindings := map[string]Relation{}
- for current := scope; current != nil; current = current.parent {
- for name, binding := range current.bindings {
- if _, exists := bindings[name]; !exists {
- bindings[name] = binding
- }
- }
- if current.cteRoot {
- break
- }
- }
- scope.visible = bindings
- return bindings
-}
-
-func (s *queryScope) uniqueBindings() []Relation {
- if s.unique != nil {
- return s.unique
- }
- s.unique = make([]Relation, 0)
- seen := map[Relation]bool{}
- for _, relation := range visibleBindings(s) {
- if !s.budget.lookup(1) {
- return nil
- }
- if !seen[relation] {
- seen[relation] = true
- s.unique = append(s.unique, relation)
- }
- }
- return s.unique
-}
-
-func (s *queryScope) provenanceSources(name string) []Relation {
- if s == nil {
- return nil
- }
- var sources []Relation
- seen := map[string]bool{}
- for current := s; current != nil; current = current.parent {
- currentNames := map[string]bool{}
- for _, source := range current.sources {
- if !s.budget.lookup(len(name) + 1) {
- return nil
- }
- key := source.name
- if seen[key] {
- continue
- }
- currentNames[key] = true
- sources = append(sources, source.relation)
- }
- for key := range currentNames {
- seen[key] = true
- }
- if current.cteRoot {
- break
- }
- }
- return sources
-}
-
-func (s *queryScope) unqualifiedPropertyNamespace(name string) (string, bool, bool) {
- if s == nil {
- return "", false, false
- }
- key := foldedFieldName(name)
- if namespace, ok := s.propertyNamespaces[key]; ok {
- return namespace.name, namespace.ok, namespace.matched
- }
- if s.propertyNamespaces == nil {
- s.propertyNamespaces = map[string]propertyNamespace{}
- }
- resolved := propertyNamespace{}
- matches := 0
- for _, binding := range s.provenanceSources(name) {
- if _, ok := bindingField(binding, name); !ok {
- continue
- }
- matches++
- resolved.matched = true
- if matches > 1 {
- resolved = propertyNamespace{matched: true}
- break
- }
- resolved.name, resolved.ok = bindingPropertyNamespace(binding, name)
- }
- if matches != 1 {
- resolved = propertyNamespace{matched: matches > 0}
- }
- s.propertyNamespaces[key] = resolved
- return resolved.name, resolved.ok, resolved.matched
-}
-
-func normalizeHogQLTableReferences(query string) (string, map[int]string) {
- normalized := []byte(query)
- originalNames := map[int]string{}
- for _, indexes := range tableReferencePattern.FindAllStringSubmatchIndex(query, -1) {
- start, end := indexes[2], indexes[3]
- name := query[start:end]
- firstDot := strings.IndexByte(name, '.')
- if firstDot == -1 || !strings.Contains(name[firstDot+1:], ".") {
- continue
- }
- for index := start + firstDot + 1; index < end; index++ {
- if normalized[index] == '.' {
- normalized[index] = '_'
- }
- }
- originalNames[start] = name
- }
- return string(normalized), originalNames
-}
-
-func tableReference(expr *clickhouse.TableExpr) (name, alias, implicitAlias string, start, end int, ok bool) {
- node := expr.Expr
- if aliased, isAlias := node.(*clickhouse.AliasExpr); isAlias {
- node = aliased.Expr
- if ident, isIdent := aliased.Alias.(*clickhouse.Ident); isIdent {
- alias = ident.Name
- }
- }
- identifier, isTable := node.(*clickhouse.TableIdentifier)
- if !isTable || identifier.Table == nil {
- return "", "", "", 0, 0, false
- }
- name = identifier.Table.Name
- implicitAlias = name
- if identifier.Database != nil {
- name = identifier.Database.Name + "." + name
- implicitAlias = identifier.Database.Name + "__" + identifier.Table.Name
- }
- return name, alias, implicitAlias, int(identifier.Pos()), int(identifier.End()), true
-}
-
-func bindSubquery(expr *clickhouse.TableExpr, scopes []*queryScope, budget *projectionBudget) bool {
- node := expr.Expr
- var alias string
- if aliased, ok := node.(*clickhouse.AliasExpr); ok {
- node = aliased.Expr
- if ident, ok := aliased.Alias.(*clickhouse.Ident); ok {
- alias = ident.Name
- }
- }
- subquery, ok := node.(*clickhouse.SubQuery)
- if !ok {
- return false
- }
- inner := innermostScope(scopes, int(subquery.Select.Pos()), int(subquery.Select.End()))
- if inner != nil && inner.parent != nil {
- // FROM subqueries do not inherit the containing query's table bindings.
- inner.cteRoot = true
- if alias != "" {
- derived := &cteBinding{name: alias, query: subquery.Select, scope: inner, budget: budget}
- addBinding(inner.parent, alias, "", Relation{name: alias, cte: derived}, int(expr.Pos()), int(expr.End()))
- }
- }
- return true
-}
-
-func foldedFieldName(name string) string {
- return strings.Map(func(r rune) rune {
- first := r
- for next := unicode.SimpleFold(r); next != r; next = unicode.SimpleFold(next) {
- first = min(first, next)
- }
- return first
- }, name)
-}
-
-func bindingField(binding Relation, name string) (catalog.Entry, bool) {
- if binding.table != nil {
- return binding.table.Fields.Exact(name)
- }
- if binding.cte == nil {
- return catalog.Entry{}, false
- }
- c := binding.cte
- fields := c.projectedFields()
- if !c.fieldsDone || !c.budget.lookup(len(name)+1) {
- return catalog.Entry{}, false
- }
- if c.fieldIndex == nil {
- c.fieldIndex = make(map[string]projectedField, len(fields))
- for _, field := range fields {
- if !c.budget.lookup(len(field.entry.Name) + 1) {
- return catalog.Entry{}, false
- }
- key := foldedFieldName(field.entry.Name)
- if existing, exists := c.fieldIndex[key]; !exists {
- c.fieldIndex[key] = field
- } else {
- existing.propertyNamespace = ""
- existing.ambiguous = true
- c.fieldIndex[key] = existing
- }
- }
- }
- field, ok := c.fieldIndex[foldedFieldName(name)]
- return field.entry, ok
-}
-
-func bindingPropertyNamespace(binding Relation, name string) (string, bool) {
- if binding.table != nil {
- if _, ok := binding.table.Fields.Exact(name); !ok {
- return "", false
- }
- if traversal, ok := binding.table.Fields.Traversal(name); ok {
- return traversal.PropertyNamespace, traversal.PropertyNamespace != ""
- }
- return propertyresolver.Resolve([]string{binding.table.Name, name, "property"}, nil)
- }
- if binding.cte == nil {
- return "", false
- }
- c := binding.cte
- _ = c.projectedFields()
- if !c.fieldsDone || !c.budget.lookup(len(name)+1) {
- return "", false
- }
- if c.fieldIndex == nil {
- _, _ = bindingField(binding, name)
- }
- if c.budget.exceeded || c.budget.lookupExceeded {
- return "", false
- }
- field, ok := c.fieldIndex[foldedFieldName(name)]
- return field.propertyNamespace, ok && !field.ambiguous && field.propertyNamespace != ""
-}
-
-func bindingFields(binding Relation) []catalog.Entry {
- if binding.table != nil {
- return binding.table.Fields.Entries()
- }
- if binding.cte == nil {
- return nil
- }
- projected := binding.cte.projectedFields()
- if binding.cte.entries == nil && len(projected) > 0 {
- binding.cte.entries = make([]catalog.Entry, 0, len(projected))
- for _, field := range projected {
- binding.cte.entries = append(binding.cte.entries, field.entry)
- }
- }
- return binding.cte.entries
-}
-
-func (c *cteBinding) projectedFields() []projectedField {
- if c.fieldsDone || c.resolving || c.scope == nil || c.budget.exceeded || c.budget.lookupExceeded {
- return c.fields
- }
- c.resolving = true
- for _, item := range c.query.SelectItems {
- if item.Alias != nil {
- namespace, _ := projectedPropertyNamespace(c.scope, item.Expr, int(item.Expr.Pos()))
- c.appendField(projectedField{entry: catalog.Entry{Name: item.Alias.Name, Type: projectedType(c.scope, item.Expr)}, propertyNamespace: namespace})
- if c.budget.exceeded || c.budget.lookupExceeded {
- break
- }
- continue
- }
- switch expr := item.Expr.(type) {
- case *clickhouse.Ident:
- if expr.Name == "*" {
- c.appendWildcardFields(c.scope, "")
- } else {
- namespace, _ := projectedPropertyNamespace(c.scope, expr, int(expr.Pos()))
- c.appendField(projectedField{entry: catalog.Entry{Name: expr.Name, Type: projectedType(c.scope, expr)}, propertyNamespace: namespace})
- }
- case *clickhouse.Path:
- if len(expr.Fields) > 0 {
- namespace, _ := projectedPropertyNamespace(c.scope, expr, int(expr.Pos()))
- c.appendField(projectedField{entry: catalog.Entry{Name: expr.Fields[len(expr.Fields)-1].Name, Type: projectedType(c.scope, expr)}, propertyNamespace: namespace})
- }
- case *clickhouse.NestedIdentifier:
- if expr.DotIdent != nil && expr.DotIdent.Name == "*" {
- c.appendWildcardFields(c.scope, expr.Ident.Name)
- } else if expr.DotIdent != nil {
- namespace, _ := projectedPropertyNamespace(c.scope, expr, int(expr.Pos()))
- c.appendField(projectedField{entry: catalog.Entry{Name: expr.DotIdent.Name, Type: projectedType(c.scope, expr)}, propertyNamespace: namespace})
- } else {
- namespace, _ := projectedPropertyNamespace(c.scope, expr, int(expr.Pos()))
- c.appendField(projectedField{entry: catalog.Entry{Name: expr.Ident.Name, Type: projectedType(c.scope, expr)}, propertyNamespace: namespace})
- }
- default:
- c.appendField(projectedField{entry: catalog.Entry{Name: item.Expr.String()}})
- }
- if c.budget.exceeded || c.budget.lookupExceeded {
- break
- }
- }
- c.resolving = false
- c.fieldsDone = true
- return c.fields
-}
-
-func (c *cteBinding) appendField(field projectedField) {
- if c.budget.take(1) == 1 {
- c.fields = append(c.fields, field)
- }
-}
-
-func (c *cteBinding) appendFields(fields []projectedField) {
- count := c.budget.take(len(fields))
- c.fields = append(c.fields, fields[:count]...)
-}
-
-func (c *cteBinding) appendWildcardFields(scope *queryScope, qualifier string) {
- bindings := visibleBindings(scope)
- if qualifier != "" {
- c.appendBindingFields(bindings[qualifier], scope.hasDuplicateSource(qualifier))
- return
- }
- seen := map[string]bool{}
- for current := scope; current != nil; current = current.parent {
- for _, source := range current.sources {
- key := source.name
- if current != scope && seen[key] {
- continue
- }
- seen[key] = true
- c.appendBindingFields(source.relation, false)
- if c.budget.exceeded {
- return
- }
- }
- if current.cteRoot {
- break
- }
- }
-}
-
-func (c *cteBinding) appendBindingFields(binding Relation, suppressProvenance bool) {
- if binding.table != nil {
- fields := binding.table.Fields.Entries()
- count := c.budget.take(len(fields))
- for _, field := range fields[:count] {
- namespace, _ := bindingPropertyNamespace(binding, field.Name)
- if suppressProvenance {
- namespace = ""
- }
- c.fields = append(c.fields, projectedField{entry: field, propertyNamespace: namespace})
- }
- return
- }
- if binding.cte != nil {
- fields := binding.cte.projectedFields()
- if !suppressProvenance {
- c.appendFields(fields)
- return
- }
- count := c.budget.take(len(fields))
- for _, field := range fields[:count] {
- field.propertyNamespace = ""
- c.fields = append(c.fields, field)
- }
- }
-}
-
-func projectedPropertyNamespace(scope *queryScope, expr clickhouse.Expr, position int) (string, bool) {
- bindings := visibleBindings(scope)
- switch typed := expr.(type) {
- case *clickhouse.Ident:
- if IsBooleanLiteral(typed) {
- return "", false
- }
- if alias, ok := (Bindings{scope: scope, position: position}).selectAlias(typed.Name); ok {
- return alias.propertyNamespace, alias.propertyNamespace != ""
- }
- namespace, ok, _ := scope.unqualifiedPropertyNamespace(typed.Name)
- return namespace, ok
- case *clickhouse.Path:
- if len(typed.Fields) == 2 {
- if scope.hasDuplicateSource(typed.Fields[0].Name) {
- return "", false
- }
- if binding, ok := bindings[typed.Fields[0].Name]; ok {
- return bindingPropertyNamespace(binding, typed.Fields[1].Name)
- }
- }
- case *clickhouse.NestedIdentifier:
- if typed.DotIdent != nil {
- if scope.hasDuplicateSource(typed.Ident.Name) {
- return "", false
- }
- if binding, ok := bindings[typed.Ident.Name]; ok {
- return bindingPropertyNamespace(binding, typed.DotIdent.Name)
- }
- }
- }
- return "", false
-}
-
-func (b *projectionBudget) take(count int) int {
- if b.lookupExceeded || b.exceeded {
- return 0
- }
- if count <= b.remaining {
- b.remaining -= count
- return count
- }
- taken := b.remaining
- b.remaining = 0
- b.exceeded = true
- return taken
-}
-
-// Count names as bytes so long identifiers cannot hide expensive work behind one lookup.
-func (b *projectionBudget) lookup(work int) bool {
- if b.exceeded || b.lookupExceeded {
- return false
- }
- if work > b.lookupRemaining {
- b.lookupExceeded = true
- return false
- }
- b.lookupRemaining -= work
- return true
-}
-
-func projectedType(scope *queryScope, expr clickhouse.Expr) string {
- bindings := visibleBindings(scope)
- switch typed := expr.(type) {
- case *clickhouse.Ident:
- if IsBooleanLiteral(typed) {
- return "boolean"
- }
- if field, ok := (Bindings{scope: scope, position: int(expr.Pos())}).SelectAlias(typed.Name); ok {
- return field.Type
- }
- for _, binding := range scope.uniqueBindings() {
- if !scope.budget.lookup(len(typed.Name) + 1) {
- return ""
- }
- if field, ok := bindingField(binding, typed.Name); ok {
- return field.Type
- }
- }
- case *clickhouse.Path:
- if len(typed.Fields) >= 2 {
- if binding, ok := bindings[typed.Fields[0].Name]; ok {
- if field, exists := bindingField(binding, typed.Fields[1].Name); exists {
- return field.Type
- }
- }
- }
- case *clickhouse.NestedIdentifier:
- if typed.DotIdent != nil {
- if binding, ok := bindings[typed.Ident.Name]; ok {
- if field, exists := bindingField(binding, typed.DotIdent.Name); exists {
- return field.Type
- }
- }
- }
- }
- return ""
-}
diff --git a/services/hogql-language-service/internal/analysis/sources.go b/services/hogql-language-service/internal/analysis/sources.go
deleted file mode 100644
index e74aaaff931e..000000000000
--- a/services/hogql-language-service/internal/analysis/sources.go
+++ /dev/null
@@ -1,66 +0,0 @@
-package analysis
-
-import (
- "iter"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
-)
-
-type Source struct {
- name string
- relation Relation
- start int
- end int
-}
-
-func (s Source) Qualifier() string {
- return s.name
-}
-
-func (s Source) Start() int { return s.start }
-
-func (s Source) End() int { return s.end }
-
-func (b Bindings) sources() iter.Seq[Source] {
- return func(yield func(Source) bool) {
- seen := map[string]bool{}
- for scope := b.scope; scope != nil; scope = scope.parent {
- for index := len(scope.sources) - 1; index >= 0; index-- {
- source := scope.sources[index]
- if !scope.budget.lookup(len(source.name) + 1) {
- return
- }
- name := source.name
- if seen[name] {
- continue
- }
- seen[name] = true
- if relation, ok := b.Relation(name); ok && relation == source.relation && !yield(source) {
- return
- }
- }
- if scope.cteRoot {
- break
- }
- }
- }
-}
-
-func (b Bindings) Fields(prefix string) iter.Seq2[Source, catalog.Entry] {
- return func(yield func(Source, catalog.Entry) bool) {
- // Self-joins share field indexes but need a suggestion for each visible source.
- prefixes := map[Relation]iter.Seq[catalog.Entry]{}
- for source := range b.sources() {
- fields, ok := prefixes[source.relation]
- if !ok {
- fields = source.relation.Prefix(prefix)
- prefixes[source.relation] = fields
- }
- for field := range fields {
- if !b.scope.budget.lookup(len(field.Name)+len(source.name)+1) || !yield(source, field) {
- return
- }
- }
- }
- }
-}
diff --git a/services/hogql-language-service/internal/catalog/catalog.go b/services/hogql-language-service/internal/catalog/catalog.go
deleted file mode 100644
index 61657a7b9a9f..000000000000
--- a/services/hogql-language-service/internal/catalog/catalog.go
+++ /dev/null
@@ -1,490 +0,0 @@
-package catalog
-
-import (
- "slices"
- "sort"
- "strings"
- "unicode"
- "unicode/utf8"
-)
-
-type Field struct {
- Name string `json:"name"`
- Type string `json:"type"`
- Relation string `json:"relation,omitempty"`
- PropertyNamespace string `json:"propertyNamespace,omitempty"`
-}
-
-type RelationDefinition struct {
- Fields map[string]Field `json:"fields"`
- Table string `json:"table,omitempty"`
- PropertyNamespaces map[string]string `json:"propertyNamespaces,omitempty"`
-}
-
-type Table struct {
- ID string `json:"id"`
- Name string `json:"name"`
- Type string `json:"type"`
- Fields map[string]Field `json:"fields"`
-}
-
-type Property struct {
- Name string `json:"name"`
- ValueType string `json:"property_type"`
-}
-
-type Catalog struct {
- Tables map[string]Table `json:"tables"`
- TableAliases map[string]string `json:"tableAliases,omitempty"`
- Properties map[string][]Property `json:"properties"`
- Relations map[string]RelationDefinition `json:"relations,omitempty"`
-}
-
-type Entry struct {
- Name string
- Type string
-}
-
-type Index struct {
- entries []Entry
-}
-
-type PreparedTable struct {
- Name string
- Type string
- Fields PreparedFields
-}
-
-type PreparedRelation struct {
- Fields *PreparedFields
- propertyNamespaces map[string]string
- ownedFields bool
-}
-
-type FieldTraversal struct {
- Relation *PreparedRelation
- PropertyNamespace string
-}
-
-type PreparedFields struct {
- Index
- traversals map[string]FieldTraversal
-}
-
-type PreparedCatalog struct {
- tables Index
- tableSpellings Index
- tablesByName map[string]int
- tableValues []PreparedTable
- properties map[string]*Index
- relations map[string]*PreparedRelation
- valid bool
- tableCount int
- propertyCount int
- hasAliases bool
- estimatedBytes int64
-}
-
-func Prepare(value *Catalog) *PreparedCatalog {
- if value == nil {
- return nil
- }
- prepared := &PreparedCatalog{
- tablesByName: make(map[string]int, len(value.Tables)),
- tableValues: make([]PreparedTable, 0, len(value.Tables)),
- properties: make(map[string]*Index, len(value.Properties)),
- valid: ValidateCatalog(value) == nil,
- tableCount: len(value.Tables),
- }
- fieldCount := 0
- for _, table := range value.Tables {
- fieldCount += len(table.Fields)
- }
- fieldEntries := make([]Entry, 0, fieldCount)
- tableEntries := make([]Entry, 0, len(value.Tables))
- types := map[string]string{}
- for name, table := range value.Tables {
- fieldStart := len(fieldEntries)
- for fieldName, field := range table.Fields {
- fieldEntries = append(fieldEntries, newEntry(fieldName, intern(types, field.Type)))
- }
- tableType := intern(types, table.Type)
- preparedTable := PreparedTable{Name: name, Type: tableType, Fields: PreparedFields{Index: newIndex(fieldEntries[fieldStart:])}}
- prepared.tablesByName[name] = len(prepared.tableValues)
- prepared.tableValues = append(prepared.tableValues, preparedTable)
- tableEntries = append(tableEntries, newEntry(name, tableType))
- }
- prepared.tables = newIndex(tableEntries)
- if len(value.Relations) > 0 {
- prepared.relations = make(map[string]*PreparedRelation, len(value.Relations))
- }
- for name, relation := range value.Relations {
- preparedRelation := &PreparedRelation{}
- if relation.Fields != nil {
- entries := make([]Entry, 0, len(relation.Fields))
- for fieldName, field := range relation.Fields {
- entries = append(entries, newEntry(fieldName, intern(types, field.Type)))
- }
- fields := PreparedFields{Index: newIndex(entries)}
- preparedRelation.Fields = &fields
- preparedRelation.ownedFields = true
- } else if tableIndex, ok := prepared.tablesByName[relation.Table]; ok {
- preparedRelation.Fields = &prepared.tableValues[tableIndex].Fields
- }
- if len(relation.PropertyNamespaces) > 0 {
- preparedRelation.propertyNamespaces = make(map[string]string, len(relation.PropertyNamespaces))
- for fieldName, namespace := range relation.PropertyNamespaces {
- preparedRelation.propertyNamespaces[fieldName] = namespace
- }
- }
- prepared.relations[name] = preparedRelation
- }
- prepareTraversals := func(fields map[string]Field, target *PreparedFields) {
- for fieldName, field := range fields {
- if field.Relation == "" && field.PropertyNamespace == "" {
- continue
- }
- if target.traversals == nil {
- target.traversals = make(map[string]FieldTraversal)
- }
- target.traversals[fieldName] = FieldTraversal{
- Relation: prepared.relations[field.Relation], PropertyNamespace: field.PropertyNamespace,
- }
- }
- }
- for name, table := range value.Tables {
- prepareTraversals(table.Fields, &prepared.tableValues[prepared.tablesByName[name]].Fields)
- }
- for name, relation := range value.Relations {
- if relation.Fields != nil {
- prepareTraversals(relation.Fields, prepared.relations[name].Fields)
- }
- }
- prepared.tableSpellings = prepared.tables
- if prepared.valid {
- spellingEntries := slices.Clone(tableEntries)
- for alias, target := range value.TableAliases {
- if alias == target {
- continue
- }
- prepared.hasAliases = true
- prepared.tablesByName[alias] = prepared.tablesByName[target]
- spellingEntries = append(spellingEntries, newEntry(alias, target))
- }
- if prepared.hasAliases {
- prepared.tableSpellings = newIndex(spellingEntries)
- }
- }
- for namespace, properties := range value.Properties {
- entries := make([]Entry, len(properties))
- for index, property := range properties {
- entries[index] = newEntry(property.Name, intern(types, property.ValueType))
- }
- index := newIndex(entries)
- prepared.properties[namespace] = &index
- prepared.propertyCount += len(entries)
- }
- prepared.estimatedBytes = prepared.estimateSize()
- return prepared
-}
-
-func (f *PreparedFields) Traversal(name string) (FieldTraversal, bool) {
- entry, ok := f.Exact(name)
- if !ok || f.traversals == nil {
- return FieldTraversal{}, false
- }
- traversal, ok := f.traversals[entry.Name]
- return traversal, ok
-}
-
-func (r *PreparedRelation) Traversal(name string) (FieldTraversal, bool) {
- if r == nil || r.Fields == nil {
- return FieldTraversal{}, false
- }
- entry, ok := r.Fields.Exact(name)
- if !ok {
- return FieldTraversal{}, false
- }
- if namespace, ok := r.propertyNamespaces[entry.Name]; ok {
- return FieldTraversal{PropertyNamespace: namespace}, true
- }
- return r.Fields.Traversal(entry.Name)
-}
-
-func (c *PreparedCatalog) Table(name string) (*PreparedTable, bool) {
- index, ok := c.tablesByName[name]
- if !ok {
- return nil, false
- }
- return &c.tableValues[index], true
-}
-
-func (c *PreparedCatalog) Tables() *Index {
- return &c.tables
-}
-
-func (c *PreparedCatalog) TableSpellings() *Index {
- return &c.tableSpellings
-}
-
-func (c *PreparedCatalog) TableSuggestions(prefix string, excluded map[string]bool) []Entry {
- return c.tableSuggestions(prefix, excluded, nil)
-}
-
-func (c *PreparedCatalog) TableSuggestionsMatching(prefix string, excluded map[string]bool, matches func(Entry) bool) []Entry {
- return c.tableSuggestions(prefix, excluded, matches)
-}
-
-func (c *PreparedCatalog) tableSuggestions(prefix string, excluded map[string]bool, matches func(Entry) bool) []Entry {
- accepted := func(entry Entry) bool {
- return !excluded[entry.Name] && (matches == nil || matches(entry))
- }
- if !c.hasAliases {
- candidates := c.tables.Prefix(prefix)
- if len(excluded) == 0 && matches == nil {
- return candidates
- }
- result := make([]Entry, 0, len(candidates))
- for _, candidate := range candidates {
- if accepted(candidate) {
- result = append(result, candidate)
- }
- }
- return result
- }
- candidates := c.tableSpellings.Prefix(prefix)
- canonicalMatches := make(map[int]bool, len(candidates))
- for _, candidate := range candidates {
- if !accepted(candidate) {
- continue
- }
- index := c.tablesByName[candidate.Name]
- if c.tableValues[index].Name == candidate.Name {
- canonicalMatches[index] = true
- }
- }
- seen := make(map[int]bool, len(candidates))
- result := make([]Entry, 0, len(candidates))
- for _, candidate := range candidates {
- if !accepted(candidate) {
- continue
- }
- index := c.tablesByName[candidate.Name]
- canonical := c.tableValues[index].Name == candidate.Name
- if seen[index] || (!canonical && canonicalMatches[index]) {
- continue
- }
- seen[index] = true
- result = append(result, candidate)
- }
- return result
-}
-
-func (c *PreparedCatalog) CanonicalTableName(name string) (string, bool) {
- table, ok := c.Table(name)
- if !ok {
- return "", false
- }
- return table.Name, true
-}
-
-func (c *PreparedCatalog) Properties(namespace string) *Index {
- return c.properties[namespace]
-}
-
-func (c *PreparedCatalog) TableCount() int {
- return c.tableCount
-}
-
-func (c *PreparedCatalog) PropertyCount() int {
- return c.propertyCount
-}
-
-func (c *PreparedCatalog) EstimatedBytes() int64 {
- return c.estimatedBytes
-}
-
-func (i *Index) Exact(name string) (Entry, bool) {
- if i == nil {
- return Entry{}, false
- }
- position := sort.Search(len(i.entries), func(index int) bool {
- return compareFold(i.entries[index].Name, name) >= 0
- })
- if position == len(i.entries) || compareFold(i.entries[position].Name, name) != 0 {
- return Entry{}, false
- }
- return i.entries[position], true
-}
-
-func (i *Index) Prefix(prefix string) []Entry {
- if i == nil {
- return nil
- }
- foldedPrefix := foldName(prefix)
- start := sort.Search(len(i.entries), func(index int) bool {
- return compareFold(i.entries[index].Name, foldedPrefix) >= 0
- })
- end := len(i.entries)
- if upperBound, ok := prefixUpperBound(foldedPrefix); ok {
- end = sort.Search(len(i.entries), func(index int) bool {
- return compareFold(i.entries[index].Name, upperBound) >= 0
- })
- }
- return i.entries[start:end]
-}
-
-func (i *Index) Entries() []Entry {
- if i == nil {
- return nil
- }
- return i.entries
-}
-
-func newEntry(name, valueType string) Entry {
- return Entry{Name: name, Type: valueType}
-}
-
-func intern(values map[string]string, value string) string {
- if existing, ok := values[value]; ok {
- return existing
- }
- values[value] = value
- return value
-}
-
-func newIndex(entries []Entry) Index {
- if !slices.IsSortedFunc(entries, compareEntries) {
- slices.SortFunc(entries, compareEntries)
- }
- return Index{entries: entries}
-}
-
-func compareEntries(left, right Entry) int {
- comparison := compareFold(left.Name, right.Name)
- if comparison == 0 {
- return strings.Compare(left.Name, right.Name)
- }
- return comparison
-}
-
-func (c *PreparedCatalog) estimateSize() int64 {
- var size int64
- for _, entry := range c.tables.entries {
- size += entrySize(entry) + 64
- table := &c.tableValues[c.tablesByName[entry.Name]]
- for _, field := range table.Fields.entries {
- size += entrySize(field)
- }
- }
- if c.hasAliases {
- for _, entry := range c.tableSpellings.entries {
- if c.tableValues[c.tablesByName[entry.Name]].Name == entry.Name {
- size += 32
- continue
- }
- size += entrySize(entry) + 32
- }
- }
- for namespace, properties := range c.properties {
- size += int64(len(namespace) + 64)
- for _, property := range properties.entries {
- size += entrySize(property)
- }
- }
- for name, relation := range c.relations {
- size += int64(len(name) + 64)
- if relation.ownedFields && relation.Fields != nil {
- for _, field := range relation.Fields.entries {
- size += entrySize(field)
- if traversal, ok := relation.Fields.traversals[field.Name]; ok {
- size += int64(len(traversal.PropertyNamespace) + 48)
- }
- }
- if relation.Fields.traversals != nil {
- size += 64
- }
- }
- if relation.propertyNamespaces != nil {
- size += 64
- for fieldName, namespace := range relation.propertyNamespaces {
- size += int64(len(fieldName) + len(namespace) + 48)
- }
- }
- }
- for _, table := range c.tableValues {
- if table.Fields.traversals != nil {
- size += 64
- }
- for _, traversal := range table.Fields.traversals {
- size += int64(len(traversal.PropertyNamespace) + 48)
- }
- }
- return size
-}
-
-func entrySize(entry Entry) int64 {
- return int64(len(entry.Name) + len(entry.Type) + 32)
-}
-
-func compareFold(left, right string) int {
- for len(left) > 0 && len(right) > 0 {
- leftRune, leftSize := utf8.DecodeRuneInString(left)
- rightRune, rightSize := utf8.DecodeRuneInString(right)
- leftRune = foldRune(leftRune)
- rightRune = foldRune(rightRune)
- if leftRune < rightRune {
- return -1
- }
- if leftRune > rightRune {
- return 1
- }
- left = left[leftSize:]
- right = right[rightSize:]
- }
- if len(left) > 0 {
- return 1
- }
- if len(right) > 0 {
- return -1
- }
- return 0
-}
-
-func foldName(value string) string {
- return strings.Map(foldRune, value)
-}
-
-func foldRune(value rune) rune {
- if value >= 'A' && value <= 'Z' {
- return value + 'a' - 'A'
- }
- if value >= 'a' && value <= 'z' {
- return value
- }
- canonical := value
- for candidate := unicode.SimpleFold(value); candidate != value; candidate = unicode.SimpleFold(candidate) {
- if candidate >= 'a' && candidate <= 'z' {
- return candidate
- }
- if candidate < canonical {
- canonical = candidate
- }
- }
- return canonical
-}
-
-func prefixUpperBound(prefix string) (string, bool) {
- characters := []rune(prefix)
- for index := len(characters) - 1; index >= 0; index-- {
- if characters[index] == utf8.MaxRune {
- continue
- }
- characters[index]++
- if characters[index] >= 0xD800 && characters[index] <= 0xDFFF {
- characters[index] = 0xE000
- }
- return string(characters[:index+1]), true
- }
- return "", false
-}
diff --git a/services/hogql-language-service/internal/catalog/catalog_test.go b/services/hogql-language-service/internal/catalog/catalog_test.go
deleted file mode 100644
index 0fb80b3f23a3..000000000000
--- a/services/hogql-language-service/internal/catalog/catalog_test.go
+++ /dev/null
@@ -1,256 +0,0 @@
-package catalog
-
-import (
- "encoding/json"
- "errors"
- "fmt"
- "testing"
-)
-
-func TestIndexExactUsesUnicodeCaseFolding(t *testing.T) {
- index := newIndex([]Entry{newEntry("t", "String"), newEntry("ſ", "String")})
-
- entry, ok := index.Exact("s")
- if !ok || entry.Name != "ſ" {
- t.Fatalf("Exact(\"s\") = %#v, %t", entry, ok)
- }
-}
-
-func TestPreparedCatalogUsesExactTableNamesAndFoldedFieldPrefixes(t *testing.T) {
- prepared := Prepare(&Catalog{
- Tables: map[string]Table{
- "ς": {
- Fields: map[string]Field{
- "ςuffix": {Name: "ςuffix", Type: "String"},
- },
- },
- },
- Properties: map[string][]Property{},
- })
-
- if _, ok := prepared.Table("Σ"); ok {
- t.Fatal("Table(\"Σ\") matched ς")
- }
- table, ok := prepared.Table("ς")
- if !ok {
- t.Fatal("Table(\"ς\") did not find ς")
- }
- entry, ok := table.Fields.Exact("Σuffix")
- if !ok || entry.Name != "ςuffix" {
- t.Fatalf("Exact(\"Σuffix\") = %#v, %t", entry, ok)
- }
- prefix := table.Fields.Prefix("Σ")
- if len(prefix) != 1 || prefix[0].Name != "ςuffix" {
- t.Fatalf("Prefix(\"Σ\") = %#v", prefix)
- }
-}
-
-func TestPreparedCatalogRetainsCaseVariantTables(t *testing.T) {
- prepared := Prepare(&Catalog{
- Tables: map[string]Table{
- "Events": {Fields: map[string]Field{"upper": {Name: "upper", Type: "String"}}},
- "events": {Fields: map[string]Field{"lower": {Name: "lower", Type: "String"}}},
- },
- Properties: map[string][]Property{},
- })
-
- upper, upperOK := prepared.Table("Events")
- lower, lowerOK := prepared.Table("events")
- if !prepared.valid || !upperOK || !lowerOK || upper.Name != "Events" || lower.Name != "events" || prepared.TableCount() != 2 {
- t.Fatalf("prepared catalog = %#v", prepared)
- }
- prefix := prepared.Tables().Prefix("EV")
- if len(prefix) != 2 || prefix[0].Name != "Events" || prefix[1].Name != "events" {
- t.Fatalf("Prefix(\"EV\") = %#v", prefix)
- }
-}
-
-func TestPreparedCatalogResolvesAliasesToCanonicalTables(t *testing.T) {
- prepared := Prepare(&Catalog{
- Tables: map[string]Table{
- "postgres.demo.orders": {Type: "data_warehouse", Fields: map[string]Field{"id": {Type: "integer"}}},
- },
- TableAliases: map[string]string{"demo_postgres_orders": "postgres.demo.orders"},
- Properties: map[string][]Property{},
- })
-
- canonical, canonicalOK := prepared.Table("postgres.demo.orders")
- alias, aliasOK := prepared.Table("demo_postgres_orders")
- if !canonicalOK || !aliasOK || canonical != alias {
- t.Fatalf("alias and canonical lookup did not share a prepared table: canonical=%p alias=%p", canonical, alias)
- }
- if _, ok := prepared.Table("Demo_postgres_orders"); ok {
- t.Fatal("alias lookup ignored exact case")
- }
- if field, ok := alias.Fields.Exact("id"); !ok || field.Type != "integer" {
- t.Fatalf("alias fields = %#v, %t", field, ok)
- }
-}
-
-func TestValidateCatalogRejectsInvalidAliases(t *testing.T) {
- base := func(aliases map[string]string) *Catalog {
- return &Catalog{
- Tables: map[string]Table{
- "orders": {Fields: map[string]Field{}},
- "events": {Fields: map[string]Field{}},
- },
- TableAliases: aliases,
- Properties: map[string][]Property{},
- }
- }
-
- for name, aliases := range map[string]map[string]string{
- "empty alias": {"": "orders"},
- "empty target": {"legacy_orders": ""},
- "dangling target": {"legacy_orders": "missing"},
- "chain": {"legacy_orders": "older_orders", "older_orders": "orders"},
- "cycle": {"legacy_orders": "older_orders", "older_orders": "legacy_orders"},
- "canonical collision": {"orders": "events"},
- } {
- t.Run(name, func(t *testing.T) {
- value := base(aliases)
- if err := ValidateCatalog(value); err == nil {
- t.Fatal("invalid alias map was accepted")
- }
- if prepared := Prepare(value); prepared == nil || prepared.valid {
- t.Fatalf("prepared invalid catalog = %#v", prepared)
- }
- })
- }
-
- if err := ValidateCatalog(base(map[string]string{"orders": "orders"})); err != nil {
- t.Fatalf("canonical identity alias was rejected: %v", err)
- }
-}
-
-func TestPrepareCopiesAndValidatesTraversalRelations(t *testing.T) {
- value := &Catalog{
- Tables: map[string]Table{"events": {Fields: map[string]Field{"person": {Type: "lazy", Relation: "person"}}}},
- Relations: map[string]RelationDefinition{"person": {Fields: map[string]Field{
- "email": {Type: "String"}, "properties": {Type: "JSON", PropertyNamespace: "person"},
- }}},
- Properties: map[string][]Property{"person": {{Name: "plan", ValueType: "String"}}},
- }
- prepared := Prepare(value)
- value.Relations["person"] = RelationDefinition{Fields: map[string]Field{"changed": {Type: "String"}}}
- table, ok := prepared.Table("events")
- if !ok {
- t.Fatal("events table was not prepared")
- }
- traversal, ok := table.Fields.Traversal("person")
- if !ok || traversal.Relation == nil {
- t.Fatal("person traversal was not prepared")
- }
- if _, ok := traversal.Relation.Fields.Exact("email"); !ok {
- t.Fatal("prepared relation changed with source catalog")
- }
- if _, ok := traversal.Relation.Fields.Exact("changed"); ok {
- t.Fatal("prepared relation retained the source map")
- }
-
- for name, invalid := range map[string]*Catalog{
- "dangling relation": {Tables: map[string]Table{"events": {Fields: map[string]Field{"person": {Relation: "missing"}}}}, Properties: map[string][]Property{}},
- "dangling namespace": {Tables: map[string]Table{"events": {Fields: map[string]Field{"properties": {PropertyNamespace: "missing"}}}}, Properties: map[string][]Property{}},
- "two targets": {Tables: map[string]Table{"events": {Fields: map[string]Field{"person": {Relation: "person", PropertyNamespace: "person"}}}}, Relations: map[string]RelationDefinition{"person": {Fields: map[string]Field{}}}, Properties: map[string][]Property{"person": {}}},
- "relation union": {Tables: map[string]Table{"events": {Fields: map[string]Field{}}}, Relations: map[string]RelationDefinition{"person": {Fields: map[string]Field{}, Table: "events"}}, Properties: map[string][]Property{}},
- "missing table": {Tables: map[string]Table{"events": {Fields: map[string]Field{}}}, Relations: map[string]RelationDefinition{"person": {Table: "missing"}}, Properties: map[string][]Property{}},
- "table alias": {Tables: map[string]Table{"events": {Fields: map[string]Field{}}}, TableAliases: map[string]string{"legacy_events": "events"}, Relations: map[string]RelationDefinition{"person": {Table: "legacy_events"}}, Properties: map[string][]Property{}},
- "bad override field": {Tables: map[string]Table{"events": {Fields: map[string]Field{}}}, Relations: map[string]RelationDefinition{"person": {Table: "events", PropertyNamespaces: map[string]string{"missing": "person"}}}, Properties: map[string][]Property{"person": {}}},
- "neither target": {Tables: map[string]Table{"events": {Fields: map[string]Field{}}}, Relations: map[string]RelationDefinition{"person": {}}, Properties: map[string][]Property{}},
- "virtual override": {Tables: map[string]Table{"events": {Fields: map[string]Field{}}}, Relations: map[string]RelationDefinition{"person": {Fields: map[string]Field{}, PropertyNamespaces: map[string]string{"properties": "person"}}}, Properties: map[string][]Property{"person": {}}},
- "unknown override": {Tables: map[string]Table{"events": {Fields: map[string]Field{"properties": {Type: "JSON"}}}}, Relations: map[string]RelationDefinition{"person": {Table: "events", PropertyNamespaces: map[string]string{"properties": "missing"}}}, Properties: map[string][]Property{}},
- "relation override": {Tables: map[string]Table{"events": {Fields: map[string]Field{"owner": {Relation: "person"}}}}, Relations: map[string]RelationDefinition{
- "events": {Table: "events", PropertyNamespaces: map[string]string{"owner": "person"}}, "person": {Fields: map[string]Field{}},
- }, Properties: map[string][]Property{"person": {}}},
- } {
- t.Run(name, func(t *testing.T) {
- if ValidateCatalog(invalid) == nil {
- t.Fatal("invalid traversal catalog was accepted")
- }
- })
- }
-}
-
-func TestPrepareSharesTableRelationFieldsAndCopiesOverrides(t *testing.T) {
- value := &Catalog{
- Tables: map[string]Table{
- "events": {Fields: map[string]Field{"person": {Type: "lazy", Relation: "person"}}},
- "persons": {Fields: map[string]Field{"email": {Type: "String"}, "properties": {Type: "JSON"}}},
- },
- Relations: map[string]RelationDefinition{
- "person": {Table: "persons", PropertyNamespaces: map[string]string{"properties": "person"}},
- },
- Properties: map[string][]Property{"person": {{Name: "plan", ValueType: "String"}}},
- }
- prepared := Prepare(value)
- persons, _ := prepared.Table("persons")
- events, _ := prepared.Table("events")
- personTraversal, ok := events.Fields.Traversal("person")
- if !ok || personTraversal.Relation.Fields != &persons.Fields {
- t.Fatal("table-backed relation did not share canonical prepared fields")
- }
- value.Relations["person"].PropertyNamespaces["properties"] = "changed"
- propertiesTraversal, ok := personTraversal.Relation.Traversal("PROPERTIES")
- if !ok || propertiesTraversal.PropertyNamespace != "person" {
- t.Fatalf("prepared override = %#v, %t", propertiesTraversal, ok)
- }
- if _, ok := persons.Fields.Traversal("properties"); ok {
- t.Fatal("relation override mutated canonical table metadata")
- }
-
- invalid := &Catalog{
- Tables: map[string]Table{"events": {Fields: map[string]Field{}}},
- Relations: map[string]RelationDefinition{"missing": {Table: "missing"}},
- Properties: map[string][]Property{},
- }
- if preparedInvalid := Prepare(invalid); preparedInvalid == nil || preparedInvalid.valid {
- t.Fatalf("invalid table-backed relation prepared as valid: %#v", preparedInvalid)
- }
-}
-
-func TestCatalogJSONPreservesEmptyVirtualRelation(t *testing.T) {
- value := Catalog{
- Tables: map[string]Table{"events": {Fields: map[string]Field{}}},
- Relations: map[string]RelationDefinition{"empty": {Fields: map[string]Field{}}},
- Properties: map[string][]Property{},
- }
- encoded, err := json.Marshal(value)
- if err != nil {
- t.Fatal(err)
- }
- var decoded Catalog
- if err := json.Unmarshal(encoded, &decoded); err != nil {
- t.Fatal(err)
- }
- if decoded.Relations["empty"].Fields == nil || ValidateCatalog(&decoded) != nil {
- t.Fatalf("empty virtual relation did not round trip: %s", encoded)
- }
-}
-
-func TestValidateCatalogRejectsTraversalLimits(t *testing.T) {
- relations := make(map[string]RelationDefinition, MaxRelationDefinitions+1)
- for index := 0; index <= MaxRelationDefinitions; index++ {
- relations[fmt.Sprintf("relation_%d", index)] = RelationDefinition{Fields: map[string]Field{}}
- }
- if err := ValidateCatalog(&Catalog{Tables: map[string]Table{"events": {Fields: map[string]Field{}}}, Properties: map[string][]Property{}, Relations: relations}); !errors.Is(err, ErrInvalidRelations) {
- t.Fatalf("relation definition limit error = %v", err)
- }
-
- fields := make(map[string]Field, MaxRelationFields/2+1)
- for index := 0; index <= MaxRelationFields/2; index++ {
- fields[fmt.Sprintf("field_%d", index)] = Field{Type: "String"}
- }
- overrides := make(map[string]string, len(fields))
- for fieldName := range fields {
- overrides[fieldName] = "shared"
- }
- if err := ValidateCatalog(&Catalog{
- Tables: map[string]Table{"events": {Fields: fields}},
- Properties: map[string][]Property{"shared": {}},
- Relations: map[string]RelationDefinition{
- "left": {Fields: fields}, "right": {Table: "events", PropertyNamespaces: overrides},
- },
- }); !errors.Is(err, ErrInvalidRelations) {
- t.Fatalf("relation field limit error = %v", err)
- }
-}
diff --git a/services/hogql-language-service/internal/catalog/registry.go b/services/hogql-language-service/internal/catalog/registry.go
deleted file mode 100644
index d1d492f6b7d2..000000000000
--- a/services/hogql-language-service/internal/catalog/registry.go
+++ /dev/null
@@ -1,225 +0,0 @@
-package catalog
-
-import (
- "errors"
- "sync"
- "time"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/serviceauth"
-)
-
-var (
- ErrInvalidScope = errors.New("team ID and user ID must be positive")
- ErrInvalidRevision = errors.New("invalid catalog revision")
- ErrInvalidCatalog = errors.New("catalog must contain tables and properties")
- ErrInvalidAliases = errors.New("table aliases must map non-empty alternate names directly to canonical table keys")
- ErrCatalogTooLarge = errors.New("catalog exceeds cache capacity")
- ErrInvalidRelations = errors.New("catalog contains invalid traversal relations")
-)
-
-const MaxRelationDefinitions = 4096
-const MaxRelationFields = 120000
-
-type Registry struct {
- mu sync.Mutex
- entries map[serviceauth.Authorization]registryEntry
- maxEntries int
- maxBytes int64
- totalBytes int64
- ttl time.Duration
- now func() time.Time
-}
-
-type registryEntry struct {
- catalog *PreparedCatalog
- revision string
- createdAt time.Time
- lastAccess time.Time
- sizeBytes int64
-}
-
-type RegistryStats struct {
- Catalogs int `json:"catalogs"`
- Tables int `json:"tables"`
- Properties int `json:"properties"`
-}
-
-func NewRegistry(maxEntries int, maxBytes int64, ttl time.Duration) *Registry {
- return newRegistry(maxEntries, maxBytes, ttl, time.Now)
-}
-
-func newRegistry(maxEntries int, maxBytes int64, ttl time.Duration, now func() time.Time) *Registry {
- return &Registry{entries: map[serviceauth.Authorization]registryEntry{}, maxEntries: maxEntries, maxBytes: maxBytes, ttl: ttl, now: now}
-}
-
-func ValidateRevision(revision string) error {
- if revision == "" || len(revision) > 128 {
- return ErrInvalidRevision
- }
- return nil
-}
-
-func ValidateCatalog(value *Catalog) error {
- if value == nil || value.Tables == nil || value.Properties == nil {
- return ErrInvalidCatalog
- }
- for alias, target := range value.TableAliases {
- if alias == "" || target == "" {
- return ErrInvalidAliases
- }
- if _, targetExists := value.Tables[target]; !targetExists {
- return ErrInvalidAliases
- }
- if _, aliasIsCanonical := value.Tables[alias]; aliasIsCanonical && alias != target {
- return ErrInvalidAliases
- }
- }
- if len(value.Relations) > MaxRelationDefinitions {
- return ErrInvalidRelations
- }
- relationFields := 0
- validateFields := func(fields map[string]Field) error {
- for _, field := range fields {
- if field.Relation != "" && field.PropertyNamespace != "" {
- return ErrInvalidRelations
- }
- if field.Relation != "" {
- if _, ok := value.Relations[field.Relation]; !ok {
- return ErrInvalidRelations
- }
- }
- if field.PropertyNamespace != "" {
- if _, ok := value.Properties[field.PropertyNamespace]; !ok {
- return ErrInvalidRelations
- }
- }
- }
- return nil
- }
- for _, table := range value.Tables {
- if err := validateFields(table.Fields); err != nil {
- return err
- }
- }
- for name, relation := range value.Relations {
- ownsFields := relation.Fields != nil
- referencesTable := relation.Table != ""
- if name == "" || ownsFields == referencesTable || (!referencesTable && len(relation.PropertyNamespaces) > 0) {
- return ErrInvalidRelations
- }
- relationFields += len(relation.Fields) + len(relation.PropertyNamespaces)
- if relationFields > MaxRelationFields {
- return ErrInvalidRelations
- }
- if ownsFields {
- if err := validateFields(relation.Fields); err != nil {
- return err
- }
- continue
- }
- table, ok := value.Tables[relation.Table]
- if !ok {
- return ErrInvalidRelations
- }
- for fieldName, namespace := range relation.PropertyNamespaces {
- field, fieldExists := table.Fields[fieldName]
- _, namespaceExists := value.Properties[namespace]
- if fieldName == "" || !fieldExists || field.Relation != "" || namespace == "" || !namespaceExists {
- return ErrInvalidRelations
- }
- }
- }
- return nil
-}
-
-func (r *Registry) Put(authorization serviceauth.Authorization, revision string, value *PreparedCatalog) error {
- if !authorization.Valid() {
- return ErrInvalidScope
- }
- if err := ValidateRevision(revision); err != nil {
- return err
- }
- if value == nil || !value.valid {
- return ErrInvalidCatalog
- }
- sizeBytes := value.EstimatedBytes()
- if sizeBytes > r.maxBytes {
- return ErrCatalogTooLarge
- }
-
- r.mu.Lock()
- defer r.mu.Unlock()
- now := r.now()
- r.removeExpired(now)
- if existing, exists := r.entries[authorization]; exists {
- r.totalBytes -= existing.sizeBytes
- delete(r.entries, authorization)
- }
- for len(r.entries) >= r.maxEntries || r.totalBytes+sizeBytes > r.maxBytes {
- r.removeLeastRecentlyUsed()
- }
- r.entries[authorization] = registryEntry{catalog: value, revision: revision, createdAt: now, lastAccess: now, sizeBytes: sizeBytes}
- r.totalBytes += sizeBytes
- return nil
-}
-
-func (r *Registry) Get(authorization serviceauth.Authorization) (*PreparedCatalog, string, bool) {
- r.mu.Lock()
- defer r.mu.Unlock()
- now := r.now()
- r.removeExpired(now)
- entry, ok := r.entries[authorization]
- if !ok {
- return nil, "", false
- }
- entry.lastAccess = now
- r.entries[authorization] = entry
- return entry.catalog, entry.revision, true
-}
-
-func (r *Registry) Delete(authorization serviceauth.Authorization) bool {
- r.mu.Lock()
- defer r.mu.Unlock()
- if _, exists := r.entries[authorization]; !exists {
- return false
- }
- r.totalBytes -= r.entries[authorization].sizeBytes
- delete(r.entries, authorization)
- return true
-}
-
-func (r *Registry) Stats() RegistryStats {
- r.mu.Lock()
- defer r.mu.Unlock()
- r.removeExpired(r.now())
- stats := RegistryStats{Catalogs: len(r.entries)}
- for _, entry := range r.entries {
- stats.Tables += entry.catalog.TableCount()
- stats.Properties += entry.catalog.PropertyCount()
- }
- return stats
-}
-
-func (r *Registry) removeExpired(now time.Time) {
- for authorization, entry := range r.entries {
- if now.Sub(entry.createdAt) >= r.ttl {
- r.totalBytes -= entry.sizeBytes
- delete(r.entries, authorization)
- }
- }
-}
-
-func (r *Registry) removeLeastRecentlyUsed() {
- var oldestAuthorization serviceauth.Authorization
- found := false
- var oldest time.Time
- for authorization, entry := range r.entries {
- if !found || entry.lastAccess.Before(oldest) {
- oldestAuthorization = authorization
- oldest = entry.lastAccess
- found = true
- }
- }
- r.totalBytes -= r.entries[oldestAuthorization].sizeBytes
- delete(r.entries, oldestAuthorization)
-}
diff --git a/services/hogql-language-service/internal/catalog/registry_test.go b/services/hogql-language-service/internal/catalog/registry_test.go
deleted file mode 100644
index 071f25b08dd3..000000000000
--- a/services/hogql-language-service/internal/catalog/registry_test.go
+++ /dev/null
@@ -1,144 +0,0 @@
-package catalog
-
-import (
- "strings"
- "testing"
- "time"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/serviceauth"
-)
-
-func TestRegistryAcceptsCaseVariantTables(t *testing.T) {
- registry := NewRegistry(1, 1<<20, time.Hour)
- value := Prepare(&Catalog{
- Tables: map[string]Table{
- "Events": {Name: "Events"},
- "events": {Name: "events"},
- },
- Properties: map[string][]Property{},
- })
-
- if err := registry.Put(serviceauth.Authorization{TeamID: 1, UserID: 10}, "1", value); err != nil {
- t.Fatalf("Put() error = %v", err)
- }
-}
-
-func TestRegistryIsolatesCatalogsAndReplacesRevisionAtomically(t *testing.T) {
- now := time.Unix(100, 0)
- registry := newRegistry(2, 1<<20, time.Hour, func() time.Time { return now })
- first := Prepare(&Catalog{Tables: map[string]Table{"events": {Name: "events"}}, Properties: map[string][]Property{}})
- second := Prepare(&Catalog{Tables: map[string]Table{"persons": {Name: "persons"}}, Properties: map[string][]Property{}})
- if err := registry.Put(serviceauth.Authorization{TeamID: 1, UserID: 10}, "1", first); err != nil {
- t.Fatal(err)
- }
- if err := registry.Put(serviceauth.Authorization{TeamID: 2, UserID: 20}, "7", second); err != nil {
- t.Fatal(err)
- }
- loaded, revision, ok := registry.Get(serviceauth.Authorization{TeamID: 1, UserID: 10})
- events, eventsExist := loaded.Table("events")
- if !ok || revision != "1" || !eventsExist || events.Name != "events" {
- t.Fatalf("unexpected first catalog: %#v, %q, %t", loaded, revision, ok)
- }
- if _, exists := loaded.Table("persons"); exists {
- t.Fatal("one team and user scope received another scope's table")
- }
- if err := registry.Put(serviceauth.Authorization{TeamID: 1, UserID: 10}, "2", second); err != nil {
- t.Fatal(err)
- }
- loaded, revision, ok = registry.Get(serviceauth.Authorization{TeamID: 1, UserID: 10})
- persons, personsExist := loaded.Table("persons")
- if !ok || revision != "2" || !personsExist || persons.Name != "persons" {
- t.Fatalf("replacement was not visible: %#v, %q, %t", loaded, revision, ok)
- }
-}
-
-func TestRegistryExpiresAndEvictsLeastRecentlyUsedCatalogs(t *testing.T) {
- now := time.Unix(100, 0)
- registry := newRegistry(2, 1<<20, time.Minute, func() time.Time { return now })
- value := Prepare(&Catalog{Tables: map[string]Table{}, Properties: map[string][]Property{}})
- for _, scope := range []serviceauth.Authorization{{TeamID: 1, UserID: 10}, {TeamID: 2, UserID: 20}} {
- if err := registry.Put(scope, "1", value); err != nil {
- t.Fatal(err)
- }
- now = now.Add(time.Second)
- }
- if _, _, ok := registry.Get(serviceauth.Authorization{TeamID: 1, UserID: 10}); !ok {
- t.Fatal("first team and user scope unexpectedly missing")
- }
- now = now.Add(time.Second)
- if err := registry.Put(serviceauth.Authorization{TeamID: 3, UserID: 30}, "1", value); err != nil {
- t.Fatal(err)
- }
- if _, _, ok := registry.Get(serviceauth.Authorization{TeamID: 2, UserID: 20}); ok {
- t.Fatal("least recently used catalog was not evicted")
- }
- now = now.Add(time.Minute)
- if stats := registry.Stats(); stats.Catalogs != 0 {
- t.Fatalf("expired catalogs remain: %#v", stats)
- }
-}
-
-func TestRegistryExpiresActiveCatalogFromPublicationTime(t *testing.T) {
- now := time.Unix(100, 0)
- registry := newRegistry(1, 1<<20, time.Minute, func() time.Time { return now })
- scope := serviceauth.Authorization{TeamID: 1, UserID: 10}
- value := Prepare(&Catalog{Tables: map[string]Table{}, Properties: map[string][]Property{}})
- if err := registry.Put(scope, "1", value); err != nil {
- t.Fatal(err)
- }
- now = now.Add(45 * time.Second)
- if _, _, ok := registry.Get(scope); !ok {
- t.Fatal("catalog expired before its publication TTL")
- }
- now = now.Add(16 * time.Second)
- if _, _, ok := registry.Get(scope); ok {
- t.Fatal("catalog access extended its publication TTL")
- }
-}
-
-func TestRegistryEvictsCatalogsToStayWithinMemoryBudget(t *testing.T) {
- now := time.Unix(100, 0)
- value := &Catalog{Tables: map[string]Table{"events": {Name: "events", Fields: map[string]Field{"long_field_name": {Name: "long_field_name", Type: "String"}}}}, Properties: map[string][]Property{}}
- prepared := Prepare(value)
- size := prepared.EstimatedBytes()
- registry := newRegistry(10, size, time.Hour, func() time.Time { return now })
- first := serviceauth.Authorization{TeamID: 1, UserID: 10}
- second := serviceauth.Authorization{TeamID: 2, UserID: 20}
- if err := registry.Put(first, "1", prepared); err != nil {
- t.Fatal(err)
- }
- if err := registry.Put(second, "1", prepared); err != nil {
- t.Fatal(err)
- }
- if _, _, ok := registry.Get(first); ok {
- t.Fatal("oldest catalog was not evicted to meet the memory budget")
- }
- if _, _, ok := registry.Get(second); !ok {
- t.Fatal("new catalog was not retained")
- }
-}
-
-func TestRegistryCountsAliasesAgainstMemoryBudget(t *testing.T) {
- canonicalName := "orders." + strings.Repeat("canonical", 32)
- aliasName := "demo_" + strings.Repeat("alias", 32)
- base := &Catalog{Tables: map[string]Table{canonicalName: {Fields: map[string]Field{"id": {Type: "integer"}}}}, Properties: map[string][]Property{}}
- withAlias := &Catalog{
- Tables: base.Tables,
- TableAliases: map[string]string{aliasName: strings.Clone(canonicalName)},
- Properties: base.Properties,
- }
- baseSize := Prepare(base).EstimatedBytes()
- aliased := Prepare(withAlias)
- expectedAliasBytes := int64(len(aliasName) + len(canonicalName) + 96)
- if actual := aliased.EstimatedBytes() - baseSize; actual != expectedAliasBytes {
- t.Fatalf("alias bytes = %d, want %d", actual, expectedAliasBytes)
- }
- rightSized := NewRegistry(1, baseSize+expectedAliasBytes, time.Hour)
- if err := rightSized.Put(serviceauth.Authorization{TeamID: 1, UserID: 10}, "1", aliased); err != nil {
- t.Fatalf("Put() error = %v for exact derived budget", err)
- }
- registry := NewRegistry(1, baseSize, time.Hour)
- if err := registry.Put(serviceauth.Authorization{TeamID: 1, UserID: 10}, "1", aliased); err != ErrCatalogTooLarge {
- t.Fatalf("Put() error = %v, want %v", err, ErrCatalogTooLarge)
- }
-}
diff --git a/services/hogql-language-service/internal/completion/bindings.go b/services/hogql-language-service/internal/completion/bindings.go
deleted file mode 100644
index cfe9a999aa97..000000000000
--- a/services/hogql-language-service/internal/completion/bindings.go
+++ /dev/null
@@ -1,181 +0,0 @@
-package completion
-
-import (
- "fmt"
- "strings"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/analysis"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
-)
-
-func cursorBindings(schema *catalog.PreparedCatalog, query string, position int, qualifier string) (*analysis.Document, analysis.Bindings, analysis.Relation, error) {
- document, parseErr := analysis.Analyze(schema, query)
- if parseErr != nil {
- parseErr = fmt.Errorf("parse incomplete SQL: %w", parseErr)
- if recovered, recoveredPosition, ok := recoverCTEOuterSelect(query, position); ok {
- document, _ = analysis.Analyze(schema, recovered)
- position = recoveredPosition
- } else if recovered := recoverSingleSelect(query); recovered != "" {
- document, _ = analysis.Analyze(schema, recovered)
- position = len("SELECT ")
- }
- }
- if document != nil {
- for statement := range document.Statements() {
- if statement.ContainsPosition(position) {
- bindings := statement.BindingsAt(position, position)
- relation, _ := statement.RelationAt(qualifier, position)
- return document, bindings, relation, parseErr
- }
- }
- }
- return document, analysis.Bindings{}, analysis.Relation{}, parseErr
-}
-
-func recoverCTEOuterSelect(query string, position int) (string, int, bool) {
- tokens, _, incomplete := scanSQLTokens(query)
- if incomplete || len(tokens) == 0 || tokens[0].text != "WITH" || tokens[0].depth != 0 {
- return "", 0, false
- }
- outerSelect := -1
- from := -1
- boundary := -1
- for index, token := range tokens {
- if token.depth == 0 && (token.text == ";" || token.text == "UNION" || token.text == "EXCEPT" || token.text == "INTERSECT") {
- return "", 0, false
- }
- if token.kind != sqlTokenWord {
- continue
- }
- if token.text == "SELECT" {
- if token.depth == 0 {
- if outerSelect >= 0 {
- return "", 0, false
- }
- outerSelect = index
- } else if outerSelect >= 0 {
- return "", 0, false
- }
- }
- if outerSelect < 0 || token.depth != 0 {
- continue
- }
- if token.text == "FROM" && from < 0 {
- from = index
- continue
- }
- if from >= 0 && boundary < 0 {
- switch token.text {
- case "WHERE", "PREWHERE", "GROUP", "HAVING", "ORDER", "LIMIT":
- boundary = index
- }
- }
- }
- if outerSelect < 0 || from < 0 || boundary < 0 || position < tokens[outerSelect].start {
- return "", 0, false
- }
- if !hasCompleteTableCTEs(tokens[:outerSelect]) {
- return "", 0, false
- }
-
- retainedEnd := tokens[boundary].start
- if position >= tokens[from].start && position < retainedEnd {
- return "", 0, false
- }
- if position < retainedEnd {
- return query[:retainedEnd], position, true
- }
- recovered := query[:retainedEnd]
- if retainedEnd == 0 || !isSQLSpace(query[retainedEnd-1]) {
- recovered += " "
- }
- // Map discarded expressions into the outer predicate so SELECT aliases keep their normal visibility.
- recovered += "WHERE 0"
- return recovered, len(recovered) - 1, true
-}
-
-func isSQLSpace(value byte) bool {
- switch value {
- case ' ', '\t', '\r', '\n':
- return true
- }
- return false
-}
-
-func hasCompleteTableCTEs(tokens []sqlToken) bool {
- for index := 1; index < len(tokens); {
- if tokens[index].kind != sqlTokenWord || tokens[index].depth != 0 {
- return false
- }
- index++
- if index >= len(tokens) || tokens[index].text != "AS" || tokens[index].depth != 0 {
- return false
- }
- index++
- if index >= len(tokens) || tokens[index].kind != sqlTokenLeftParen || tokens[index].depth != 0 {
- return false
- }
- index++
- for index < len(tokens) && !(tokens[index].kind == sqlTokenRightParen && tokens[index].depth == 0) {
- index++
- }
- if index >= len(tokens) {
- return false
- }
- index++
- if index == len(tokens) {
- return true
- }
- if tokens[index].kind != sqlTokenComma || tokens[index].depth != 0 {
- return false
- }
- index++
- }
- return false
-}
-
-// Incomplete predicates can retain a parsed FROM clause only when no other query scope exists.
-func recoverSingleSelect(query string) string {
- tokens, _, incomplete := scanSQLTokens(query)
- if incomplete {
- return ""
- }
- selects := 0
- from := -1
- end := len(tokens)
- for index, token := range tokens {
- if token.text == ";" || token.text == "WITH" {
- return ""
- }
- if token.kind != sqlTokenWord {
- continue
- }
- if token.text == "SELECT" {
- selects++
- if token.depth != 0 || selects > 1 {
- return ""
- }
- }
- if token.depth == 0 {
- if token.text == "FROM" && from == -1 {
- from = index
- }
- switch token.text {
- case "WHERE", "PREWHERE", "GROUP", "ORDER", "HAVING", "LIMIT", "SETTINGS":
- if from >= 0 && end == len(tokens) {
- end = index
- }
- }
- }
- }
- if selects != 1 || from < 0 {
- return ""
- }
- var recovered strings.Builder
- recovered.WriteString("SELECT * ")
- for _, token := range tokens[from:end] {
- recovered.WriteString(token.raw)
- recovered.WriteByte(' ')
- }
- return recovered.String()
-}
diff --git a/services/hogql-language-service/internal/completion/completion.go b/services/hogql-language-service/internal/completion/completion.go
deleted file mode 100644
index 21427256f6ec..000000000000
--- a/services/hogql-language-service/internal/completion/completion.go
+++ /dev/null
@@ -1,477 +0,0 @@
-package completion
-
-import (
- "encoding/base64"
- "fmt"
- "iter"
- "regexp"
- "slices"
- "sort"
- "strconv"
- "strings"
- "unicode"
- "unicode/utf8"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/analysis"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/querylimits"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/textposition"
-)
-
-type Suggestion struct {
- Label string `json:"label"`
- Kind string `json:"kind"`
- Detail string `json:"detail,omitempty"`
- InsertText string `json:"insertText,omitempty"`
- SortText string `json:"sortText,omitempty"`
-}
-
-type Result struct {
- Suggestions []Suggestion `json:"suggestions"`
- Total int `json:"total"`
- NextCursor string `json:"nextCursor,omitempty"`
- ParseError string `json:"parseError,omitempty"`
-}
-
-const PageSize = 25
-
-type PositionEncoding = textposition.Encoding
-
-const (
- PositionEncodingUTF8 = textposition.UTF8
- PositionEncodingUTF16 = textposition.UTF16
-)
-
-var keywords = []string{"SELECT", "FROM", "WHERE", "GROUP BY", "ORDER BY", "LIMIT", "JOIN", "AS", "CASE", "NULL", "TRUE", "FALSE", "NOT"}
-var queryStarters = []catalog.Entry{{Name: "SELECT"}, {Name: "WITH"}}
-var betweenSeparator = []string{"AND"}
-var predicateContinuations = []string{"AND", "OR", "GROUP BY", "ORDER BY", "LIMIT"}
-var joinPredicateContinuations = []string{"AND", "OR", "WHERE", "GROUP BY", "ORDER BY", "LIMIT"}
-var joinPredicateContinuationsBeforeJoin = []string{"AND", "OR"}
-var caseExpressionKeywords = []string{"CASE", "WHEN", "THEN", "ELSE", "END", "NULL", "TRUE", "FALSE", "NOT"}
-var comparisonOperators = []string{"=", "!=", "<", "<=", ">", ">=", "LIKE", "ILIKE", "IN", "NOT IN", "IS NULL", "IS NOT NULL", "BETWEEN", "NOT BETWEEN"}
-var commonFunctions = []string{"avg", "coalesce", "count", "countDistinct", "countIf", "if", "max", "min", "now", "sum", "sumIf", "toDate", "toDateTime", "uniq", "uniqExact"}
-var simpleHogQLIdentifier = regexp.MustCompile(`^[A-Za-z_$][A-Za-z0-9_$]*$`)
-var quotedHogQLKeywords = map[string]struct{}{
- "ALL": {}, "AND": {}, "ANTI": {}, "ANY": {}, "ARRAY": {}, "AS": {}, "ASC": {}, "ASCENDING": {}, "ASOF": {},
- "BETWEEN": {}, "BOTH": {}, "BY": {}, "CASE": {}, "CAST": {}, "CATCH": {}, "COHORT": {}, "COLLATE": {}, "COLUMNS": {},
- "CROSS": {}, "CUBE": {}, "CURRENT": {}, "DATE": {}, "DAY": {}, "DESC": {}, "DESCENDING": {}, "DISTINCT": {},
- "ELSE": {}, "END": {}, "EXCEPT": {}, "EXCLUDE": {}, "EXTRACT": {}, "FILL": {}, "FILTER": {}, "FINAL": {},
- "FINALLY": {}, "FIRST": {}, "FN": {}, "FOLLOWING": {}, "FOR": {}, "FROM": {}, "FULL": {}, "FUN": {},
- "GROUP": {}, "GROUPING": {}, "HAVING": {}, "HOUR": {}, "ID": {}, "IF": {}, "INF": {}, "INFINITY": {},
- "IGNORE": {}, "ILIKE": {}, "IN": {}, "INCLUDE": {}, "INNER": {}, "INTERPOLATE": {}, "INTERVAL": {}, "IS": {},
- "INTERSECT": {}, "JOIN": {}, "KEY": {}, "LAMBDA": {}, "LAST": {}, "LEADING": {}, "LEFT": {}, "LET": {},
- "LIKE": {}, "LIMIT": {}, "LOCAL": {}, "MATERIALIZED": {}, "MINUTE": {}, "MONTH": {}, "NAME": {}, "NAN": {},
- "NATURAL": {}, "NOT": {}, "NULL": {}, "NULLS": {}, "OFFSET": {}, "ON": {}, "OR": {},
- "ORDER": {}, "OUTER": {}, "OVER": {}, "PARTITION": {}, "PIVOT": {}, "POSITIONAL": {}, "PRECEDING": {},
- "PREWHERE": {}, "QUALIFY": {}, "QUARTER": {}, "RANGE": {}, "RECURSIVE": {}, "REPLACE": {}, "RETURN": {}, "RIGHT": {},
- "ROLLUP": {}, "ROW": {}, "ROWS": {}, "SAMPLE": {}, "SELECT": {}, "SEMI": {}, "SETS": {}, "SETTINGS": {},
- "SECOND": {}, "STEP": {}, "SUBSTRING": {}, "THEN": {}, "THROW": {}, "TIES": {}, "TIME": {},
- "TO": {}, "TOP": {}, "TOTALS": {}, "TRAILING": {}, "TRIM": {}, "TRUNCATE": {}, "TRY": {}, "TRY_CAST": {},
- "UNBOUNDED": {}, "UNION": {}, "UNPIVOT": {}, "USING": {}, "VALUES": {}, "WEEK": {}, "WHEN": {},
- "WHERE": {}, "WHILE": {}, "WINDOW": {}, "WITH": {}, "WITHIN": {}, "YEAR": {}, "YYYY": {}, "ZONE": {},
-}
-var quotedHogQLKeywordsFolded = func() map[string]struct{} {
- result := make(map[string]struct{}, len(quotedHogQLKeywords))
- for keyword := range quotedHogQLKeywords {
- result[strings.ToLower(keyword)] = struct{}{}
- }
- return result
-}()
-var hogQLIdentifierEscaper = strings.NewReplacer(
- "\\", "\\\\",
- "`", "``",
- "\b", "\\b",
- "\f", "\\f",
- "\r", "\\r",
- "\n", "\\n",
- "\t", "\\t",
- "\x00", "\\0",
- "\a", "\\a",
- "\v", "\\v",
-)
-
-func Complete(schema *catalog.PreparedCatalog, query string, position int, positionEncoding PositionEncoding, cursor string) (Result, error) {
- if err := querylimits.Validate(query); err != nil {
- return Result{}, err
- }
- offset, err := decodeCursor(cursor)
- if err != nil {
- return Result{}, err
- }
- position, err = textposition.ToByteOffset(query, position, positionEncoding)
- if err != nil {
- return Result{}, err
- }
- prefix, qualifier, start := cursorWord(query[:position])
- mode := analyzeCursorContext(query[:start])
- tableNamespace := ""
- if path, ok := dottedTablePathAtCursor(query[:position]); ok {
- prefix = path.prefix
- qualifier = ""
- start = path.start
- tableNamespace = path.namespace
- mode = completionModeTable
- }
- if len(prefix) > querylimits.MaxSuggestionInputBytes {
- return Result{Suggestions: []Suggestion{}}, nil
- }
- lowerPrefix := strings.ToLower(prefix)
- if mode == completionModeNone {
- return Result{Suggestions: []Suggestion{}}, nil
- }
- if mode == completionModeStatementStart {
- entries := func(yield func(catalog.Entry) bool) {
- for _, entry := range queryStarters {
- if hasLowerPrefix(entry.Name, lowerPrefix) && !yield(entry) {
- return
- }
- }
- }
- return indexedResult(entries, "keyword", offset, nil), nil
- }
- repaired := query[:start] + "__posthog_cursor__" + query[position:]
- document, bindings, qualified, parseErr := cursorBindings(schema, repaired, start, qualifier)
-
- var suggestions []Suggestion
- if document != nil && document.LimitError() != nil {
- return Result{}, document.LimitError()
- }
- if mode == completionModeTable {
- result := tableResult(schema, bindings, lowerPrefix, tableNamespace, offset, parseErr)
- if document != nil && document.LimitError() != nil {
- return Result{}, document.LimitError()
- }
- return result, nil
- }
- namespace, propertyPrefix, propertyOK := propertyContext(query[:position], bindings)
- if propertyOK {
- if document != nil && document.LimitError() != nil {
- return Result{}, document.LimitError()
- }
- return indexedResult(slices.Values(schema.Properties(namespace).Prefix(propertyPrefix)), "property", offset, parseErr), nil
- } else if target, fieldPrefix := traversalContext(query[:position], bindings); target.Explicit {
- if document != nil && document.LimitError() != nil {
- return Result{}, document.LimitError()
- }
- if !target.Valid || target.Fields == nil {
- return Result{Suggestions: []Suggestion{}, ParseError: errorString(parseErr)}, nil
- }
- return indexedResult(slices.Values(target.Fields.Prefix(fieldPrefix)), "field", offset, parseErr), nil
- } else if qualifier != "" {
- entries := qualified.Prefix(lowerPrefix)
- if document != nil && document.LimitError() != nil {
- return Result{}, document.LimitError()
- }
- return indexedResult(entries, "field", offset, parseErr), nil
- } else if mode == completionModeComparison {
- suggestions = appendNamed(suggestions, comparisonOperators, lowerPrefix, "operator", "")
- } else if mode == completionModeBetweenSeparator {
- suggestions = appendNamed(suggestions, betweenSeparator, lowerPrefix, "keyword", "")
- } else if mode == completionModePredicateContinuation {
- suggestions = appendNamed(suggestions, predicateContinuations, lowerPrefix, "keyword", "")
- } else if mode == completionModeJoinPredicateContinuation {
- continuations := joinPredicateContinuations
- if laterJoinAtCursor(query, position) {
- continuations = joinPredicateContinuationsBeforeJoin
- }
- suggestions = appendNamed(suggestions, continuations, lowerPrefix, "keyword", "")
- } else if mode == completionModePostExpression {
- suggestions = appendNamed(suggestions, comparisonOperators, lowerPrefix, "operator", "")
- suggestions = appendNamed(suggestions, predicateContinuations, lowerPrefix, "keyword", "")
- } else if mode == completionModeJoinPostExpression {
- suggestions = appendNamed(suggestions, comparisonOperators, lowerPrefix, "operator", "")
- continuations := joinPredicateContinuations
- if laterJoinAtCursor(query, position) {
- continuations = joinPredicateContinuationsBeforeJoin
- }
- suggestions = appendNamed(suggestions, continuations, lowerPrefix, "keyword", "")
- } else {
- suggestions = fieldSuggestions(bindings, lowerPrefix)
- if document != nil && document.LimitError() != nil {
- return Result{}, document.LimitError()
- }
- if mode == completionModeExpression || mode == completionModeCaseExpression {
- suggestions = appendFunctions(suggestions, lowerPrefix)
- }
- availableKeywords := keywords
- if mode == completionModeCaseExpression {
- availableKeywords = caseExpressionKeywords
- }
- for _, keyword := range availableKeywords {
- if hasLowerPrefix(keyword, lowerPrefix) {
- suggestions = append(suggestions, Suggestion{Label: keyword, Kind: "keyword"})
- }
- }
- }
- sort.Slice(suggestions, func(i, j int) bool {
- leftRank := suggestionRank(suggestions[i].Kind)
- rightRank := suggestionRank(suggestions[j].Kind)
- if leftRank != rightRank {
- return leftRank < rightRank
- }
- left, right := strings.ToLower(suggestions[i].Label), strings.ToLower(suggestions[j].Label)
- if left == right {
- if suggestions[i].Label == suggestions[j].Label {
- return suggestions[i].InsertText < suggestions[j].InsertText
- }
- return suggestions[i].Label < suggestions[j].Label
- }
- return left < right
- })
- result := Result{Suggestions: suggestions, Total: len(suggestions)}
- if offset > len(suggestions) {
- offset = len(suggestions)
- }
- end := min(offset+PageSize, len(suggestions))
- result.Suggestions = suggestions[offset:end]
- for index := range result.Suggestions {
- // Global ranks preserve client-side page order even when labels contain punctuation.
- result.Suggestions[index].SortText = fmt.Sprintf("%020d", offset+index)
- }
- if end < len(suggestions) {
- result.NextCursor = encodeCursor(end)
- }
- if parseErr != nil {
- result.ParseError = parseErr.Error()
- }
- return result, nil
-}
-
-func errorString(err error) string {
- if err == nil {
- return ""
- }
- return err.Error()
-}
-
-func laterJoinAtCursor(query string, position int) bool {
- _, cursorDepth, _ := scanSQLTokens(query[:position])
- return hasLaterJoinAtDepth(query[position:], cursorDepth)
-}
-
-func indexedResult(entries iter.Seq[catalog.Entry], kind string, offset int, parseErr error) Result {
- result := Result{Suggestions: make([]Suggestion, 0, PageSize)}
- rank := strconv.Itoa(suggestionRank(kind)) + "-"
- for entry := range entries {
- if !supportedHogQLIdentifier(entry.Name) {
- continue
- }
- if result.Total >= offset && len(result.Suggestions) < PageSize {
- result.Suggestions = append(result.Suggestions, Suggestion{
- Label: entry.Name, Kind: kind, Detail: entry.Type, InsertText: suggestionInsertText(kind, entry.Name), SortText: rank + strings.ToLower(entry.Name),
- })
- }
- result.Total++
- }
- nextOffset := offset + len(result.Suggestions)
- if nextOffset < result.Total {
- result.NextCursor = encodeCursor(nextOffset)
- }
- if parseErr != nil {
- result.ParseError = parseErr.Error()
- }
- return result
-}
-
-func utf16OffsetToByteOffset(value string, offset int) int {
- byteOffset, _ := textposition.ToByteOffset(value, offset, PositionEncodingUTF16)
- return byteOffset
-}
-
-func propertyContext(input string, bindings analysis.Bindings) (string, string, bool) {
- start := len(input)
- for start > 0 {
- character, size := utf8.DecodeLastRuneInString(input[:start])
- if character != '.' && !isIdentifier(character) {
- break
- }
- start -= size
- }
- parts := strings.Split(input[start:], ".")
- if len(parts) < 2 {
- return "", "", false
- }
- namespace, ok := bindings.PropertyNamespace(parts)
- return namespace, parts[len(parts)-1], ok
-}
-
-func traversalContext(input string, bindings analysis.Bindings) (analysis.TraversalTarget, string) {
- start := len(input)
- for start > 0 {
- character, size := utf8.DecodeLastRuneInString(input[:start])
- if character != '.' && !isIdentifier(character) {
- break
- }
- start -= size
- }
- parts := strings.Split(input[start:], ".")
- if len(parts) < 2 {
- return analysis.TraversalTarget{}, ""
- }
- return bindings.Traversal(parts[:len(parts)-1]), parts[len(parts)-1]
-}
-
-func decodeCursor(cursor string) (int, error) {
- if cursor == "" {
- return 0, nil
- }
- decoded, err := base64.RawURLEncoding.DecodeString(cursor)
- if err != nil {
- return 0, fmt.Errorf("invalid cursor")
- }
- offset, err := strconv.Atoi(string(decoded))
- if err != nil || offset < 0 {
- return 0, fmt.Errorf("invalid cursor")
- }
- return offset, nil
-}
-
-func encodeCursor(offset int) string {
- return base64.RawURLEncoding.EncodeToString([]byte(strconv.Itoa(offset)))
-}
-
-func suggestionInsertText(kind, name string) string {
- insertText := name
- switch kind {
- case "field", "property":
- insertText = quoteHogQLFieldIdentifier(name)
- case "table":
- insertText = quoteHogQLTablePath(name)
- }
- if insertText == name {
- return ""
- }
- return insertText
-}
-
-func quoteHogQLTablePath(name string) string {
- parts := strings.Split(name, ".")
- for index := range parts {
- parts[index] = quoteHogQLFieldIdentifier(parts[index])
- }
- return strings.Join(parts, ".")
-}
-
-func supportedHogQLIdentifier(name string) bool {
- return !strings.Contains(name, "%")
-}
-
-func quoteHogQLFieldIdentifier(name string) string {
- if _, keyword := quotedHogQLKeywords[strings.ToUpper(name)]; keyword {
- return "`" + hogQLIdentifierEscaper.Replace(name) + "`"
- }
- return quoteHogQLIdentifier(name)
-}
-
-func quoteHogQLIdentifier(name string) string {
- if simpleHogQLIdentifier.MatchString(name) {
- return name
- }
- return "`" + hogQLIdentifierEscaper.Replace(name) + "`"
-}
-
-func appendFunctions(out []Suggestion, lowerPrefix string) []Suggestion {
- if lowerPrefix == "" {
- for _, name := range commonFunctions {
- out = append(out, Suggestion{Label: name, Kind: "function", Detail: "HogQL function", InsertText: name + "()"})
- }
- return out
- }
- for _, name := range hogQLFunctions {
- if hasLowerPrefix(name, lowerPrefix) {
- out = append(out, Suggestion{Label: name, Kind: "function", Detail: "HogQL function", InsertText: name + "()"})
- }
- }
- return out
-}
-
-func appendNamed(out []Suggestion, values []string, lowerPrefix, kind, detail string) []Suggestion {
- for _, value := range values {
- if hasLowerPrefix(value, lowerPrefix) {
- out = append(out, Suggestion{Label: value, Kind: kind, Detail: detail, InsertText: value})
- }
- }
- return out
-}
-
-func suggestionRank(kind string) int {
- switch kind {
- case "field", "property", "table":
- return 1
- case "function":
- return 2
- default:
- return 3
- }
-}
-
-func cursorWord(input string) (prefix, qualifier string, start int) {
- start = len(input)
- for start > 0 {
- character, size := utf8.DecodeLastRuneInString(input[:start])
- if !isIdentifier(character) {
- break
- }
- start -= size
- }
- prefix = input[start:]
- if start > 0 && input[start-1] == '.' {
- qualifierEnd := start - 1
- qualifierStart := qualifierEnd
- for qualifierStart > 0 {
- character, size := utf8.DecodeLastRuneInString(input[:qualifierStart])
- if !isIdentifier(character) {
- break
- }
- qualifierStart -= size
- }
- qualifier = input[qualifierStart:qualifierEnd]
- }
- return prefix, qualifier, start
-}
-
-type dottedTablePath struct {
- prefix string
- namespace string
- start int
-}
-
-func dottedTablePathAtCursor(input string) (dottedTablePath, bool) {
- start := len(input)
- for start > 0 {
- character, size := utf8.DecodeLastRuneInString(input[:start])
- if character != '.' && !isIdentifier(character) {
- break
- }
- start -= size
- }
- prefix := input[start:]
- lastDot := strings.LastIndexByte(prefix, '.')
- if lastDot < 1 || !directTablePathContext(input[:start]) {
- return dottedTablePath{}, false
- }
- for _, component := range strings.Split(prefix[:lastDot], ".") {
- if component == "" {
- return dottedTablePath{}, false
- }
- }
- return dottedTablePath{prefix: prefix, namespace: prefix[:lastDot+1], start: start}, true
-}
-
-func directTablePathContext(input string) bool {
- tokens, depth, incomplete := scanSQLTokens(input)
- if incomplete {
- return false
- }
- last := lastTokenAtDepth(tokens, depth)
- return last.kind == sqlTokenWord && (last.text == "FROM" || last.text == "JOIN")
-}
-
-func isIdentifier(r rune) bool {
- return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' || r == '$'
-}
-
-func hasLowerPrefix(value, lowerPrefix string) bool {
- return strings.HasPrefix(strings.ToLower(value), lowerPrefix)
-}
diff --git a/services/hogql-language-service/internal/completion/completion_test.go b/services/hogql-language-service/internal/completion/completion_test.go
deleted file mode 100644
index 8b51716ab0c6..000000000000
--- a/services/hogql-language-service/internal/completion/completion_test.go
+++ /dev/null
@@ -1,1551 +0,0 @@
-package completion
-
-import (
- "encoding/json"
- "errors"
- "fmt"
- "math"
- "os"
- "slices"
- "strings"
- "testing"
- "time"
- "unicode/utf16"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/querylimits"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/validation"
-)
-
-func testCatalog() *catalog.PreparedCatalog {
- return catalog.Prepare(&catalog.Catalog{Tables: map[string]catalog.Table{
- "events": {Name: "events", Type: "posthog", Fields: map[string]catalog.Field{
- "uuid": {Name: "uuid", Type: "string"}, "event": {Name: "event", Type: "string"},
- "properties": {Name: "properties", Type: "json"}, "timestamp": {Name: "timestamp", Type: "datetime"},
- }},
- "Events": {Name: "Events", Type: "data_warehouse", Fields: map[string]catalog.Field{
- "custom_field": {Name: "custom_field", Type: "string"}, "properties": {Name: "properties", Type: "json"},
- }},
- "persons": {Name: "persons", Type: "posthog", Fields: map[string]catalog.Field{
- "id": {Name: "id", Type: "string"}, "properties": {Name: "properties", Type: "json"},
- }},
- "orders": {Name: "orders", Type: "data_warehouse", Fields: map[string]catalog.Field{
- "order_id": {Name: "order_id", Type: "string"},
- "amount": {Name: "amount", Type: "float"},
- }},
- "organizations": {Name: "organizations", Type: "posthog", Fields: map[string]catalog.Field{}},
- "postgres.synced.orders": {Name: "postgres.synced.orders", Type: "data_warehouse", Fields: map[string]catalog.Field{
- "synced_id": {Name: "synced_id", Type: "string"},
- }},
- }, Properties: map[string][]catalog.Property{
- "event": {{Name: "$browser", ValueType: "String"}, {Name: "$geo_city", ValueType: "String"}, {Name: "$geo_country", ValueType: "String"}, {Name: "$Geo_Region", ValueType: "String"}},
- "person": {{Name: "$geo_city", ValueType: "String"}, {Name: "email", ValueType: "String"}},
- "session": {{Name: "$entry_current_url", ValueType: "String"}},
- "group:0": {{Name: "industry", ValueType: "String"}},
- }})
-}
-
-func traversalCatalog() *catalog.PreparedCatalog {
- return catalog.Prepare(&catalog.Catalog{
- Tables: map[string]catalog.Table{
- "events": {Type: "posthog", Fields: map[string]catalog.Field{
- "event": {Type: "String"}, "person": {Type: "lazy", Relation: "person"},
- "properties": {Type: "JSON", PropertyNamespace: "person"},
- }},
- },
- Relations: map[string]catalog.RelationDefinition{
- "person": {Fields: map[string]catalog.Field{
- "email": {Type: "String"}, "properties": {Type: "JSON", PropertyNamespace: "person"},
- "manager": {Type: "lazy", Relation: "person"}, "payload": {Type: "JSON"},
- }},
- },
- Properties: map[string][]catalog.Property{"person": {{Name: "plan", ValueType: "String"}}},
- })
-}
-
-func TestCompletesTraversalRelationsWithoutGlobalLeakage(t *testing.T) {
- for _, test := range []struct {
- query, expected, kind string
- }{
- {"SELECT e.person.em FROM events AS e", "email", "field"},
- {"SELECT person.em FROM events", "email", "field"},
- {"SELECT e.person.properties.pl FROM events AS e", "plan", "property"},
- {"SELECT e.person.manager.em FROM events AS e", "email", "field"},
- {"SELECT properties AS properties, properties.pl FROM events", "plan", "property"},
- } {
- result, err := Complete(traversalCatalog(), test.query, strings.Index(test.query, " FROM"), PositionEncodingUTF8, "")
- if err != nil || len(result.Suggestions) != 1 || result.Suggestions[0].Label != test.expected || result.Suggestions[0].Kind != test.kind {
- t.Fatalf("query %q returned %#v, %v", test.query, result, err)
- }
- }
- for _, query := range []string{
- "SELECT e.person. FROM events AS e",
- "SELECT person. FROM events",
- } {
- result, err := Complete(traversalCatalog(), query, strings.Index(query, " FROM"), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatalf("query %q returned %v", query, err)
- }
- labels := make([]string, len(result.Suggestions))
- for index, suggestion := range result.Suggestions {
- labels[index] = suggestion.Label
- }
- if got, want := strings.Join(labels, ","), "email,manager,payload,properties"; got != want {
- t.Fatalf("query %q labels = %q, want %q", query, got, want)
- }
- }
- result, err := Complete(traversalCatalog(), "SELECT * FROM per", len("SELECT * FROM per"), PositionEncodingUTF8, "")
- if err != nil || len(result.Suggestions) != 0 {
- t.Fatalf("traversal relation leaked into tables: %#v, %v", result, err)
- }
- query := "SELECT e.person.properties.plan. FROM events AS e"
- result, err = Complete(traversalCatalog(), query, strings.Index(query, " FROM"), PositionEncodingUTF8, "")
- if err != nil || len(result.Suggestions) != 0 {
- t.Fatalf("nested property returned top-level suggestions: %#v, %v", result, err)
- }
-}
-
-func TestExplicitTraversalDoesNotFallBackToLegacyPropertyNamespace(t *testing.T) {
- schema := catalog.Prepare(&catalog.Catalog{
- Tables: map[string]catalog.Table{"events": {Fields: map[string]catalog.Field{
- "person": {Type: "lazy", Relation: "person"},
- }}},
- Relations: map[string]catalog.RelationDefinition{"person": {Fields: map[string]catalog.Field{
- "email": {Type: "String"},
- }}},
- Properties: map[string][]catalog.Property{"person": {{Name: "plan", ValueType: "String"}}},
- })
- query := "SELECT e.person.properties. FROM events AS e"
- result, err := Complete(schema, query, strings.Index(query, " FROM"), PositionEncodingUTF8, "")
- if err != nil || len(result.Suggestions) != 0 {
- t.Fatalf("explicit relation returned legacy properties: %#v, %v", result, err)
- }
-}
-
-func TestTableBackedTraversalReusesFieldsAndScopesPropertyOverrides(t *testing.T) {
- schema := catalog.Prepare(&catalog.Catalog{
- Tables: map[string]catalog.Table{
- "events": {Fields: map[string]catalog.Field{
- "group": {Type: "lazy", Relation: "group_0"}, "other_group": {Type: "lazy", Relation: "group_1"},
- }},
- "groups": {Fields: map[string]catalog.Field{
- "name": {Type: "String"}, "properties": {Type: "JSON", PropertyNamespace: "group"},
- "owner": {Type: "lazy", Relation: "person"},
- }},
- },
- Relations: map[string]catalog.RelationDefinition{
- "group_0": {Table: "groups", PropertyNamespaces: map[string]string{"properties": "group:0"}},
- "group_1": {Table: "groups", PropertyNamespaces: map[string]string{"properties": "group:1"}},
- "person": {Fields: map[string]catalog.Field{
- "properties": {Type: "JSON", PropertyNamespace: "person"}, "group": {Type: "lazy", Relation: "group_1"},
- }},
- },
- Properties: map[string][]catalog.Property{
- "group": {{Name: "base", ValueType: "String"}}, "group:0": {{Name: "zero", ValueType: "String"}},
- "group:1": {{Name: "one", ValueType: "String"}},
- "person": {{Name: "email", ValueType: "String"}},
- },
- })
- for _, test := range []struct {
- query string
- expected []string
- }{
- {"SELECT e.group.na FROM events AS e", []string{"name"}},
- {"SELECT e.group.properties. FROM events AS e", []string{"zero"}},
- {"SELECT e.other_group.properties. FROM events AS e", []string{"one"}},
- {"SELECT g.properties. FROM groups AS g", []string{"base"}},
- {"SELECT e.group.owner.properties. FROM events AS e", []string{"email"}},
- {"SELECT e.group.owner.group.properties. FROM events AS e", []string{"one"}},
- } {
- result, err := Complete(schema, test.query, strings.Index(test.query, " FROM"), PositionEncodingUTF8, "")
- labels := make([]string, len(result.Suggestions))
- for index, suggestion := range result.Suggestions {
- labels[index] = suggestion.Label
- }
- if err != nil || !slices.Equal(labels, test.expected) {
- t.Fatalf("query %q returned %#v, %v", test.query, result, err)
- }
- }
- result := validation.Validate(schema, "SELECT e.group.properties.one FROM events AS e")
- if result.Valid || len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != "unknown_property" {
- t.Fatalf("group namespace isolation validation = %#v", result)
- }
- path := "e.group." + strings.Repeat("owner.group.", querylimits.MaxRelationTraversalHops/2)
- query := "SELECT " + path + "properties. FROM events AS e"
- completion, err := Complete(schema, query, strings.Index(query, " FROM"), PositionEncodingUTF8, "")
- if !errors.Is(err, querylimits.ErrRelationTraversalTooDeep) || len(completion.Suggestions) != 0 {
- t.Fatalf("overlong mixed traversal returned %#v, %v", completion, err)
- }
-}
-
-func TestTraversalCompletionRespectsAmbiguityAndShadowing(t *testing.T) {
- queries := []string{
- "SELECT person.em FROM events AS left JOIN events AS right ON left.event = right.event",
- "SELECT 1 AS person, person.em FROM events",
- "SELECT E.person.em FROM events AS e",
- }
- for _, query := range queries {
- result, err := Complete(traversalCatalog(), query, strings.Index(query, " FROM"), PositionEncodingUTF8, "")
- if err != nil || len(result.Suggestions) != 0 {
- t.Fatalf("query %q returned %#v, %v", query, result, err)
- }
- }
-}
-
-func TestTraversalCompletionRejectsDuplicateQualifiedSources(t *testing.T) {
- schema := catalog.Prepare(&catalog.Catalog{
- Tables: map[string]catalog.Table{
- "events": {Fields: map[string]catalog.Field{"owner": {Type: "lazy", Relation: "person"}}},
- "orders": {Fields: map[string]catalog.Field{"owner": {Type: "lazy", Relation: "account"}}},
- },
- Relations: map[string]catalog.RelationDefinition{
- "person": {Fields: map[string]catalog.Field{"email": {Type: "String"}}},
- "account": {Fields: map[string]catalog.Field{"external_id": {Type: "String"}}},
- },
- Properties: map[string][]catalog.Property{},
- })
- for _, query := range []string{
- "SELECT source.owner.e FROM events AS source JOIN orders AS source ON 1 = 1",
- "SELECT owner.e FROM events AS source JOIN orders AS source ON 1 = 1",
- } {
- result, err := Complete(schema, query, strings.Index(query, " FROM"), PositionEncodingUTF8, "")
- if err != nil || len(result.Suggestions) != 0 {
- t.Fatalf("duplicate source query %q returned %#v, %v", query, result, err)
- }
- }
-}
-
-func TestTraversalCompletionReportsHopLimit(t *testing.T) {
- path := "e.person." + strings.Repeat("manager.", querylimits.MaxRelationTraversalHops)
- query := "SELECT " + path + "em FROM events AS e"
- result, err := Complete(traversalCatalog(), query, strings.Index(query, " FROM"), PositionEncodingUTF8, "")
- if !errors.Is(err, querylimits.ErrRelationTraversalTooDeep) || len(result.Suggestions) != 0 {
- t.Fatalf("overlong traversal returned %#v, %v", result, err)
- }
-}
-
-func TestCompletionRejectsQueriesOutsideResourceLimits(t *testing.T) {
- _, err := Complete(testCatalog(), strings.Repeat("x", 64<<10+1), 0, PositionEncodingUTF8, "")
- if err == nil {
- t.Fatal("oversized query was accepted")
- }
-}
-
-func TestUTF16OffsetToByteOffset(t *testing.T) {
- tests := []struct {
- value string
- offset int
- expect int
- }{
- {value: "SELECT ", offset: 7, expect: 7},
- {value: "SELECT '😀' FROM ", offset: 17, expect: 19},
- {value: "😀", offset: 1, expect: 0},
- {value: "😀", offset: 3, expect: 4},
- {value: "SELECT ", offset: -1, expect: 7},
- }
- for _, test := range tests {
- if actual := utf16OffsetToByteOffset(test.value, test.offset); actual != test.expect {
- t.Errorf("utf16OffsetToByteOffset(%q, %d) = %d, want %d", test.value, test.offset, actual, test.expect)
- }
- }
-}
-
-func TestCompletesPropertiesForGenericNamespaces(t *testing.T) {
- tests := []struct {
- query string
- position int
- expect string
- }{
- {query: "SELECT events.properties.$geo FROM events", position: len("SELECT events.properties.$geo"), expect: "$geo_city"},
- {query: "SELECT e.properties.$geo FROM events AS e", position: len("SELECT e.properties.$geo"), expect: "$geo_city"},
- {query: "SELECT properties.$geo FROM persons", position: len("SELECT properties.$geo"), expect: "$geo_city"},
- {query: "SELECT session.properties.$entry FROM events", position: len("SELECT session.properties.$entry"), expect: "$entry_current_url"},
- {query: "SELECT group_0.properties.ind FROM events", position: len("SELECT group_0.properties.ind"), expect: "industry"},
- {query: "SELECT properties.$geo_r FROM events", position: len("SELECT properties.$geo_r"), expect: "$Geo_Region"},
- }
- for _, test := range tests {
- result, err := Complete(testCatalog(), test.query, test.position, PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- if len(result.Suggestions) == 0 || result.Suggestions[0].Label != test.expect || result.Suggestions[0].Kind != "property" {
- t.Fatalf("query %q returned %#v", test.query, result)
- }
- }
-}
-
-func TestTableAliasesUseCanonicalPropertyOrigins(t *testing.T) {
- properties := map[string][]catalog.Property{
- "event": {{Name: "$browser", ValueType: "String"}},
- "person": {{Name: "email", ValueType: "String"}},
- }
- for _, test := range []struct {
- name string
- tables map[string]catalog.Table
- aliases map[string]string
- query string
- expected string
- excluded string
- }{
- {
- name: "ordinary alias",
- tables: map[string]catalog.Table{"events": {Fields: map[string]catalog.Field{"properties": {Type: "json"}}}},
- aliases: map[string]string{"legacy_events": "events"},
- query: "SELECT legacy_events.properties.$br FROM legacy_events",
- expected: "$browser",
- },
- {
- name: "misleading person alias",
- tables: map[string]catalog.Table{"events": {Fields: map[string]catalog.Field{"properties": {Type: "json"}}}},
- aliases: map[string]string{"persons": "events"},
- query: "SELECT persons.properties.$br FROM persons",
- expected: "$browser",
- excluded: "email",
- },
- {
- name: "event-like warehouse alias",
- tables: map[string]catalog.Table{"warehouse_records": {Fields: map[string]catalog.Field{"properties": {Type: "json"}}}},
- aliases: map[string]string{"events": "warehouse_records"},
- query: "SELECT events.properties.$br FROM events",
- excluded: "$browser",
- },
- } {
- t.Run(test.name, func(t *testing.T) {
- prepared := catalog.Prepare(&catalog.Catalog{Tables: test.tables, TableAliases: test.aliases, Properties: properties})
- result, err := Complete(prepared, test.query, strings.Index(test.query, " FROM "), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- if test.expected != "" && !hasSuggestion(result.Suggestions, test.expected) {
- t.Fatalf("query %q returned %#v", test.query, result)
- }
- if test.excluded != "" && hasSuggestion(result.Suggestions, test.excluded) {
- t.Fatalf("query %q returned excluded property %#v", test.query, result)
- }
- })
- }
-}
-
-func TestCompletesFieldsForHogQLQualifiedTable(t *testing.T) {
- query := "SELECT s. FROM postgres.synced.orders AS s"
- result, err := Complete(testCatalog(), query, len("SELECT s."), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- if len(result.Suggestions) != 1 || result.Suggestions[0].Label != "synced_id" {
- t.Fatalf("suggestions = %#v; parse error = %q", result.Suggestions, result.ParseError)
- }
-}
-
-func TestCompletesFieldsForCollidingNormalizedTableReferences(t *testing.T) {
- fieldTable := func(name, field string) catalog.Table {
- return catalog.Table{Name: name, Type: "data_warehouse", Fields: map[string]catalog.Field{field: {Name: field, Type: "string"}}}
- }
- for _, test := range []struct {
- name string
- catalog *catalog.Catalog
- }{
- {
- name: "canonical tables",
- catalog: &catalog.Catalog{Tables: map[string]catalog.Table{
- "a.b.c_d": fieldTable("a.b.c_d", "left_field"),
- "a.b_c.d": fieldTable("a.b_c.d", "right_field"),
- }},
- },
- {
- name: "alias and canonical table",
- catalog: &catalog.Catalog{
- Tables: map[string]catalog.Table{
- "left_target": fieldTable("left_target", "left_field"),
- "a.b_c.d": fieldTable("a.b_c.d", "right_field"),
- },
- TableAliases: map[string]string{"a.b.c_d": "left_target"},
- },
- },
- } {
- t.Run(test.name, func(t *testing.T) {
- test.catalog.Properties = map[string][]catalog.Property{}
- prepared := catalog.Prepare(test.catalog)
- query := "SELECT l. FROM a.b.c_d AS l JOIN a.b_c.d AS r ON 1 = 1"
- left, err := Complete(prepared, query, len("SELECT l."), PositionEncodingUTF8, "")
- if err != nil || len(left.Suggestions) != 1 || left.Suggestions[0].Label != "left_field" {
- t.Fatalf("left completion = %#v, error = %v", left, err)
- }
- query = "SELECT r. FROM a.b.c_d AS l JOIN a.b_c.d AS r ON 1 = 1"
- right, err := Complete(prepared, query, len("SELECT r."), PositionEncodingUTF8, "")
- if err != nil || len(right.Suggestions) != 1 || right.Suggestions[0].Label != "right_field" {
- t.Fatalf("right completion = %#v, error = %v", right, err)
- }
- })
- }
-}
-
-func TestCompletesTablesAfterFrom(t *testing.T) {
- for _, test := range []struct {
- name, query string
- tables map[string]string
- }{
- {"catalog", "SELECT * FROM ord|", map[string]string{"orders": "data_warehouse"}},
- {"cte from", "WITH recent AS (SELECT event FROM events) SELECT * FROM rec|", map[string]string{"recent": "CTE"}},
- {"cte join", "WITH recent AS (SELECT event FROM events) SELECT * FROM events JOIN rec| ON 1 = 1", map[string]string{"recent": "CTE"}},
- {"cte comma", "WITH recent AS (SELECT event FROM events) SELECT * FROM events, rec|", map[string]string{"recent": "CTE"}},
- {"catalog and cte", "WITH order_summary AS (SELECT event FROM events) SELECT * FROM ord|", map[string]string{"orders": "data_warehouse", "order_summary": "CTE"}},
- {"case-variant catalog tables", "SELECT * FROM EV|", map[string]string{"Events": "data_warehouse", "events": "posthog"}},
- {"case-variant catalog and CTE", "WITH Orders AS (SELECT event FROM events) SELECT * FROM ord|", map[string]string{"Orders": "CTE", "orders": "data_warehouse"}},
- {"unicode prefix", "WITH `Σ` AS (SELECT event FROM events) SELECT * FROM ς|", map[string]string{"Σ": "CTE"}},
- {"case-variant nested CTEs", "WITH recent AS (SELECT event FROM events) SELECT * FROM (WITH Recent AS (SELECT uuid FROM events) SELECT * FROM rec|) AS s", map[string]string{"Recent": "CTE", "recent": "CTE"}},
- {"outer visible", "WITH recent AS (SELECT event FROM events) SELECT * FROM (SELECT * FROM rec|) AS s", map[string]string{"recent": "CTE"}},
- {"previous cte", "WITH recent AS (SELECT event FROM events), recent_next AS (SELECT * FROM rec|) SELECT * FROM recent_next", map[string]string{"recent": "CTE"}},
- {"no self or later cte", "WITH recent AS (SELECT * FROM rec|), recent_next AS (SELECT event FROM events) SELECT * FROM recent", nil},
- {"no sibling cte", "SELECT * FROM (WITH recent AS (SELECT event FROM events) SELECT * FROM recent) AS a JOIN (SELECT * FROM rec|) AS b ON 1 = 1", nil},
- {"no previous statement", "WITH recent AS (SELECT event FROM events) SELECT * FROM recent; SELECT * FROM rec|", nil},
- {"no scalar alias", "WITH 1 AS recent SELECT * FROM rec|", nil},
- {"malformed cte", "WITH recent AS (SELECT event FROM events SELECT * FROM rec|", nil},
- } {
- t.Run(test.name, func(t *testing.T) {
- position := strings.IndexByte(test.query, '|')
- query := strings.Replace(test.query, "|", "", 1)
- result, err := Complete(testCatalog(), query, position, PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- if len(result.Suggestions) != len(test.tables) || result.Total != len(test.tables) {
- t.Fatalf("result = %#v, want tables %#v", result, test.tables)
- }
- seen := map[string]bool{}
- for _, suggestion := range result.Suggestions {
- detail, exists := test.tables[suggestion.Label]
- if !exists || seen[suggestion.Label] || suggestion.Kind != "table" || suggestion.Detail != detail {
- t.Fatalf("unexpected suggestion %#v, want tables %#v", suggestion, test.tables)
- }
- seen[suggestion.Label] = true
- }
- })
- }
-}
-
-func TestCompletesDottedTablePaths(t *testing.T) {
- prepared := catalog.Prepare(&catalog.Catalog{
- Tables: map[string]catalog.Table{
- "postgres.demo.orders": {Type: "data_warehouse", Fields: map[string]catalog.Field{}},
- "postgres.demo.order items": {Type: "data_warehouse", Fields: map[string]catalog.Field{}},
- "events.properties.archive": {Type: "data_warehouse", Fields: map[string]catalog.Field{}},
- "persons.properties.archive": {Type: "data_warehouse", Fields: map[string]catalog.Field{}},
- "models.marts.revenue": {Type: "view", Fields: map[string]catalog.Field{}},
- },
- TableAliases: map[string]string{"POSTGRES.demo.orders": "postgres.demo.orders"},
- Properties: map[string][]catalog.Property{
- "event": {{Name: "$browser", ValueType: "String"}},
- "person": {{Name: "email", ValueType: "String"}},
- },
- })
- for _, test := range []struct {
- name string
- source string
- encoding PositionEncoding
- expected *Suggestion
- }{
- {name: "namespace", source: "SELECT * FROM postgres.|", expected: &Suggestion{Label: "postgres.demo.orders", Detail: "data_warehouse", InsertText: "demo.orders"}},
- {name: "leaf", source: "SELECT * FROM postgres.demo.or|", expected: &Suggestion{Label: "postgres.demo.orders", Detail: "data_warehouse", InsertText: "orders"}},
- {name: "join", source: "SELECT * FROM events JOIN postgres.demo.or| ON 1 = 1", expected: &Suggestion{Label: "postgres.demo.orders", Detail: "data_warehouse", InsertText: "orders"}},
- {name: "alias exact namespace case", source: "SELECT * FROM POSTGRES.demo.or|", expected: &Suggestion{Label: "POSTGRES.demo.orders", Detail: "postgres.demo.orders", InsertText: "orders"}},
- {name: "leaf prefix case is replaced", source: "SELECT * FROM postgres.demo.OR|", expected: &Suggestion{Label: "postgres.demo.orders", Detail: "data_warehouse", InsertText: "orders"}},
- {name: "wrong namespace case", source: "SELECT * FROM postgres.DEMO.or|"},
- {name: "comma source remains unsupported", source: "SELECT * FROM events, postgres.demo.or|"},
- {name: "event property spelling remains table context", source: "SELECT * FROM events.properties.|", expected: &Suggestion{Label: "events.properties.archive", Detail: "data_warehouse", InsertText: "archive"}},
- {name: "person property spelling remains table context", source: "SELECT * FROM persons.properties.|", expected: &Suggestion{Label: "persons.properties.archive", Detail: "data_warehouse", InsertText: "archive"}},
- {name: "unicode utf16", source: "SELECT '😀'; SELECT * FROM postgres.demo.or|", encoding: PositionEncodingUTF16, expected: &Suggestion{Label: "postgres.demo.orders", Detail: "data_warehouse", InsertText: "orders"}},
- {name: "midword server cursor", source: "SELECT * FROM postgres.demo.or|suffix", expected: &Suggestion{Label: "postgres.demo.orders", Detail: "data_warehouse", InsertText: "orders"}},
- {name: "model partial namespace", source: "SELECT * FROM models.mar|", expected: &Suggestion{Label: "models.marts.revenue", Detail: "view", InsertText: "marts.revenue"}},
- } {
- t.Run(test.name, func(t *testing.T) {
- bytePosition := strings.IndexByte(test.source, '|')
- query := strings.Replace(test.source, "|", "", 1)
- position := bytePosition
- if test.encoding == "" {
- test.encoding = PositionEncodingUTF8
- }
- if test.encoding == PositionEncodingUTF16 {
- position = len(utf16.Encode([]rune(query[:bytePosition])))
- }
- result, err := Complete(prepared, query, position, test.encoding, "")
- if err != nil {
- t.Fatal(err)
- }
- if test.expected == nil {
- if len(result.Suggestions) != 0 {
- t.Fatalf("result = %#v", result)
- }
- return
- }
- if len(result.Suggestions) != 1 {
- t.Fatalf("result = %#v", result)
- }
- actual := result.Suggestions[0]
- if actual.Label != test.expected.Label || actual.Detail != test.expected.Detail || actual.InsertText != test.expected.InsertText {
- t.Fatalf("suggestion = %#v, want %#v", actual, *test.expected)
- }
- })
- }
-
- quotedCanonical := catalog.Prepare(&catalog.Catalog{
- Tables: map[string]catalog.Table{
- "postgres.demo.order items": {Type: "data_warehouse", Fields: map[string]catalog.Field{}},
- },
- TableAliases: map[string]string{"postgres.demo.order_items": "postgres.demo.order items"},
- Properties: map[string][]catalog.Property{},
- })
- query := "SELECT * FROM postgres.demo.order"
- result, err := Complete(quotedCanonical, query, len(query), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- if len(result.Suggestions) != 1 || result.Suggestions[0].Label != "postgres.demo.order_items" || result.Suggestions[0].InsertText != "order_items" {
- t.Fatalf("quoted canonical alias result = %#v", result)
- }
-
- cteCatalog := catalog.Prepare(&catalog.Catalog{Tables: map[string]catalog.Table{
- "postgres.demo.orders": {Type: "data_warehouse", Fields: map[string]catalog.Field{}},
- "postgres.demo.catalog_table": {Type: "data_warehouse", Fields: map[string]catalog.Field{}},
- }})
- cteQuery := "WITH `postgres.demo.cte` AS (SELECT 1), `postgres.demo.orders` AS (SELECT 2) SELECT * FROM postgres.demo."
- cteResult, err := Complete(cteCatalog, cteQuery, len(cteQuery), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- if len(cteResult.Suggestions) != 1 || cteResult.Suggestions[0].Label != "postgres.demo.catalog_table" || cteResult.Suggestions[0].InsertText != "catalog_table" {
- t.Fatalf("dotted CTE result = %#v", cteResult)
- }
-}
-
-func TestCompletesOneSpellingPerAliasedTableAndHonorsExactCTEShadowing(t *testing.T) {
- prepared := catalog.Prepare(&catalog.Catalog{
- Tables: map[string]catalog.Table{
- "postgres.demo.orders": {Type: "data_warehouse", Fields: map[string]catalog.Field{"id": {Type: "integer"}}},
- "events": {Type: "posthog", Fields: map[string]catalog.Field{"uuid": {Type: "uuid"}}},
- },
- TableAliases: map[string]string{"demo_postgres_orders": "postgres.demo.orders"},
- Properties: map[string][]catalog.Property{},
- })
- for _, test := range []struct {
- query string
- expected []string
- }{
- {query: "SELECT * FROM ", expected: []string{"events", "postgres.demo.orders"}},
- {query: "SELECT * FROM demo_", expected: []string{"demo_postgres_orders"}},
- {query: "WITH demo_postgres_orders AS (SELECT uuid FROM events) SELECT * FROM ", expected: []string{"demo_postgres_orders", "events", "postgres.demo.orders"}},
- {query: "WITH `postgres.demo.orders` AS (SELECT uuid FROM events) SELECT * FROM ", expected: []string{"postgres.demo.orders", "demo_postgres_orders", "events"}},
- } {
- result, err := Complete(prepared, test.query, len(test.query), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- labels := make([]string, len(result.Suggestions))
- for index, suggestion := range result.Suggestions {
- labels[index] = suggestion.Label
- }
- if strings.Join(labels, ",") != strings.Join(test.expected, ",") {
- t.Fatalf("query %q returned %#v", test.query, result)
- }
- }
-}
-
-func TestCompletesFieldsForAlias(t *testing.T) {
- type testCase struct {
- name, query string
- fields []Suggestion
- }
- tests := []testCase{
- {"qualified", "SELECT o.| FROM orders AS o", []Suggestion{{Label: "amount", Detail: "float"}, {Label: "order_id", Detail: "string"}}},
- {"alias is not another source", "SELECT uu| FROM events AS e", []Suggestion{{Label: "uuid", Detail: "string"}}},
- {"self join", "SELECT uu| FROM events AS e JOIN events AS other ON e.uuid = other.uuid", []Suggestion{
- {Label: "uuid", Detail: "string from e", InsertText: "e.uuid"}, {Label: "uuid", Detail: "string from other", InsertText: "other.uuid"},
- }},
- {"physical join", "SELECT prop| FROM events JOIN persons ON 1 = 1", []Suggestion{
- {Label: "properties", Detail: "json from events", InsertText: "events.properties"}, {Label: "properties", Detail: "json from persons", InsertText: "persons.properties"},
- }},
- {"cte and subquery", "WITH recent AS (SELECT uuid FROM events) SELECT uu| FROM recent AS r JOIN (SELECT uuid FROM events) AS s ON 1 = 1", []Suggestion{
- {Label: "uuid", Detail: "string from r", InsertText: "r.uuid"}, {Label: "uuid", Detail: "string from s", InsertText: "s.uuid"},
- }},
- {"cte self join", "WITH recent AS (SELECT uuid FROM events) SELECT uu| FROM recent AS r JOIN recent AS s ON 1 = 1", []Suggestion{
- {Label: "uuid", Detail: "string from r", InsertText: "r.uuid"}, {Label: "uuid", Detail: "string from s", InsertText: "s.uuid"},
- }},
- {"quoted qualifier", "SELECT uu| FROM events AS `recent.items` JOIN events AS `FROM` ON 1 = 1", []Suggestion{
- {Label: "uuid", Detail: "string from `FROM`", InsertText: "`FROM`.uuid"}, {Label: "uuid", Detail: "string from `recent.items`", InsertText: "`recent.items`.uuid"},
- }},
- {"dotted cte", "WITH `recent.items` AS (SELECT uuid FROM events) SELECT uu| FROM `recent.items` JOIN events AS e ON 1 = 1", []Suggestion{
- {Label: "uuid", Detail: "string from `recent.items`", InsertText: "`recent.items`.uuid"}, {Label: "uuid", Detail: "string from e", InsertText: "e.uuid"},
- }},
- {"warehouse qualifier", "WITH recent AS (SELECT uuid AS synced_id FROM events) SELECT synced_| FROM postgres.synced.orders JOIN recent ON 1 = 1", []Suggestion{
- {Label: "synced_id", Detail: "string from postgres__synced__orders", InsertText: "postgres__synced__orders.synced_id"}, {Label: "synced_id", Detail: "string from recent", InsertText: "recent.synced_id"},
- }},
- {"quoted warehouse segment", "WITH recent AS (SELECT uuid AS synced_id FROM events) SELECT synced_| FROM `postgres.synced`.orders JOIN recent ON 1 = 1", []Suggestion{
- {Label: "synced_id", Detail: "string from `postgres.synced__orders`", InsertText: "`postgres.synced__orders`.synced_id"}, {Label: "synced_id", Detail: "string from recent", InsertText: "recent.synced_id"},
- }},
- {"quoted field", "WITH t AS (SELECT uuid AS `user id` FROM events) SELECT us| FROM t AS a JOIN t AS b ON 1 = 1", []Suggestion{
- {Label: "user id", Detail: "string from a", InsertText: "a.`user id`"}, {Label: "user id", Detail: "string from b", InsertText: "b.`user id`"},
- }},
- {"unknown expression type", "WITH t AS (SELECT count() AS total FROM events) SELECT tot| FROM t AS a JOIN t AS b ON 1 = 1", []Suggestion{
- {Label: "total", Detail: "from a", InsertText: "a.total"}, {Label: "total", Detail: "from b", InsertText: "b.total"},
- }},
- {"case-folded fields", "WITH a AS (SELECT uuid AS shared FROM events), b AS (SELECT uuid AS SHARED FROM events) SELECT sha| FROM a JOIN b ON 1 = 1", []Suggestion{
- {Label: "SHARED", Detail: "string from b", InsertText: "b.SHARED"}, {Label: "shared", Detail: "string from a", InsertText: "a.shared"},
- }},
- {"select alias precedence", "SELECT e.event AS uuid FROM events AS e JOIN events AS other ON 1 = 1 ORDER BY uu|", []Suggestion{{Label: "uuid", Detail: "string"}}},
- {"case-sensitive select alias precedence", "SELECT e.properties AS UUID FROM events AS e JOIN events AS other ON 1 = 1 ORDER BY uu|", []Suggestion{
- {Label: "UUID", Detail: "json"}, {Label: "uuid", Detail: "string from e", InsertText: "e.uuid"}, {Label: "uuid", Detail: "string from other", InsertText: "other.uuid"},
- }},
- {"case-variant relation aliases", "SELECT prop| FROM events AS e JOIN persons AS E ON 1 = 1", []Suggestion{
- {Label: "properties", Detail: "json from E", InsertText: "E.properties"}, {Label: "properties", Detail: "json from e", InsertText: "e.properties"},
- }},
- {"qualified join stays unqualified", "SELECT e.uu| FROM events AS e JOIN events AS other ON 1 = 1", []Suggestion{{Label: "uuid", Detail: "string"}}},
- {"nested alias shadow", "SELECT * FROM events AS e WHERE uuid IN (SELECT uu| FROM events AS e)", []Suggestion{{Label: "uuid", Detail: "string"}}},
- {"cte scope isolation", "WITH t AS (SELECT uu| FROM events AS e) SELECT * FROM t JOIN events AS other ON 1 = 1", []Suggestion{{Label: "uuid", Detail: "string"}}},
- {"subquery scope isolation", "SELECT * FROM events AS e JOIN (SELECT uu| FROM events AS other) AS s ON 1 = 1", []Suggestion{{Label: "uuid", Detail: "string"}}},
- }
- var sources []string
- var fields []Suggestion
- for index := range PageSize + 2 {
- alias := fmt.Sprintf("source_%02d", index)
- sources = append(sources, "events AS "+alias)
- fields = append(fields, Suggestion{Label: "uuid", Detail: "string from " + alias, InsertText: alias + ".uuid"})
- }
- tests = append(tests, testCase{"joined pagination", "SELECT uu| FROM " + strings.Join(sources, " CROSS JOIN "), fields})
- for _, test := range tests {
- t.Run(test.name, func(t *testing.T) {
- position := strings.IndexByte(test.query, '|')
- query := strings.Replace(test.query, "|", "", 1)
- var fields []Suggestion
- cursor := ""
- for {
- result, err := Complete(testCatalog(), query, position, PositionEncodingUTF8, cursor)
- if err != nil || result.ParseError != "" || len(result.Suggestions) > PageSize {
- t.Fatalf("result = %#v, err = %v", result, err)
- }
- for _, suggestion := range result.Suggestions {
- if suggestion.Kind == "field" {
- fields = append(fields, suggestion)
- }
- }
- cursor = result.NextCursor
- if cursor == "" {
- break
- }
- if len(fields) >= len(test.fields) {
- t.Fatalf("unexpected next page: %#v", result)
- }
- }
- if len(fields) != len(test.fields) {
- t.Fatalf("fields = %#v, want %#v", fields, test.fields)
- }
- for index, expected := range test.fields {
- actual := fields[index]
- if actual.Label != expected.Label || actual.Detail != expected.Detail || actual.InsertText != expected.InsertText {
- t.Errorf("field = %#v, want %#v", actual, expected)
- }
- if actual.InsertText != "" {
- start := strings.LastIndexByte(query[:position], ' ') + 1
- completed := query[:start] + actual.InsertText + query[position:]
- if checked := validation.Validate(testCatalog(), completed); !checked.Valid {
- t.Errorf("inserted query %q is invalid: %#v", completed, checked)
- }
- }
- if index > 0 && fields[index-1].SortText >= actual.SortText {
- t.Errorf("sort keys disagree with page order: %#v", fields)
- }
- }
- })
- }
-}
-
-func TestRecoversCTEBindingsForIncompleteOuterClause(t *testing.T) {
- for _, test := range []struct {
- name, source, label, detail string
- encoding PositionEncoding
- }{
- {name: "cursor in select", source: "WITH recent AS (SELECT uuid, properties FROM events) SELECT recent.pro| FROM recent WHERE (", label: "properties", detail: "json"},
- {name: "cursor in predicate", source: "WITH recent AS (SELECT uuid, properties FROM events) SELECT uuid FROM recent WHERE (recent.pro|", label: "properties", detail: "json"},
- {name: "chain alias and renamed field", source: "WITH base AS (SELECT uuid, timestamp FROM events), recent AS (SELECT uuid, timestamp AS happened_at FROM base) SELECT r.hap| FROM recent AS r WHERE (", label: "happened_at", detail: "datetime"},
- {name: "wildcard", source: "WITH recent AS (SELECT * FROM events) SELECT recent.uu| FROM recent WHERE uuid =", label: "uuid", detail: "string"},
- {name: "unqualified chain field", source: "WITH base AS (SELECT uuid FROM events), recent AS (SELECT uuid FROM base) SELECT uu| FROM recent WHERE (", label: "uuid", detail: "string"},
- {name: "quoted exact case", source: "WITH `Recent` AS (SELECT uuid AS `EventID` FROM events) SELECT R.Eve| FROM `Recent` AS R WHERE (", label: "EventID", detail: "string"},
- {name: "prewhere", source: "WITH recent AS (SELECT uuid FROM events) SELECT recent.uu| FROM recent PREWHERE (", label: "uuid", detail: "string"},
- {name: "group by", source: "WITH recent AS (SELECT uuid FROM events) SELECT recent.uu| FROM recent GROUP BY (", label: "uuid", detail: "string"},
- {name: "having", source: "WITH recent AS (SELECT uuid FROM events) SELECT recent.uu| FROM recent HAVING (", label: "uuid", detail: "string"},
- {name: "order by", source: "WITH recent AS (SELECT uuid FROM events) SELECT recent.uu| FROM recent ORDER BY (", label: "uuid", detail: "string"},
- {name: "limit", source: "WITH recent AS (SELECT uuid FROM events) SELECT recent.uu| FROM recent LIMIT (", label: "uuid", detail: "string"},
- {name: "comments and keyword strings", source: "WITH recent AS (SELECT uuid FROM events WHERE event = 'FROM WHERE'), /* SELECT FROM */ next AS (SELECT uuid FROM recent) SELECT next.uu| FROM next WHERE (", label: "uuid", detail: "string"},
- {name: "unicode utf8", source: "WITH `Σ` AS (SELECT uuid FROM events) SELECT '😀', `Σ`.uu| FROM `Σ` WHERE (", label: "uuid", detail: "string", encoding: PositionEncodingUTF8},
- {name: "unicode utf16", source: "WITH `Σ` AS (SELECT uuid FROM events) SELECT '😀', `Σ`.uu| FROM `Σ` WHERE (", label: "uuid", detail: "string", encoding: PositionEncodingUTF16},
- } {
- t.Run(test.name, func(t *testing.T) {
- bytePosition := strings.IndexByte(test.source, '|')
- query := strings.Replace(test.source, "|", "", 1)
- position := bytePosition
- if test.encoding == "" {
- test.encoding = PositionEncodingUTF8
- }
- if test.encoding == PositionEncodingUTF16 {
- position = len(utf16.Encode([]rune(query[:bytePosition])))
- }
- result, err := Complete(testCatalog(), query, position, test.encoding, "")
- if err != nil {
- t.Fatal(err)
- }
- if result.ParseError == "" {
- t.Fatal("expected the original incomplete query to retain its parse error")
- }
- suggestion, ok := findSuggestion(result.Suggestions, test.label)
- if !ok || suggestion.Detail != test.detail {
- t.Fatalf("result = %#v", result)
- }
- })
- }
-}
-
-func TestIncompleteOuterQueryRemainsInvalidForValidation(t *testing.T) {
- for _, suffix := range []string{"(", "/* unfinished"} {
- t.Run(suffix, func(t *testing.T) {
- incomplete := "WITH recent AS (SELECT uuid FROM events) SELECT recent.uuid FROM recent WHERE " + suffix
- checked := validation.Validate(testCatalog(), incomplete)
- if checked.Valid || len(checked.Diagnostics) != 1 || checked.Diagnostics[0].Code != "syntax_error" {
- t.Fatalf("validation accepted the incomplete source: %#v", checked)
- }
- })
- }
-}
-
-func TestIncompleteOuterRecoveryKeepsSelectAliasVisibility(t *testing.T) {
- for _, test := range []struct {
- name, source, expected, excluded string
- }{
- {name: "where sees prior alias", source: "WITH recent AS (SELECT uuid FROM events) SELECT uuid AS event_id FROM recent WHERE (eve|", expected: "event_id"},
- {name: "earlier select item does not see later alias", source: "WITH recent AS (SELECT uuid FROM events) SELECT eve|, uuid AS event_id FROM recent WHERE (", excluded: "event_id"},
- {name: "from does not see select alias", source: "WITH recent AS (SELECT uuid FROM events) SELECT uuid AS event_id FROM eve| WHERE (", excluded: "event_id"},
- } {
- t.Run(test.name, func(t *testing.T) {
- position := strings.IndexByte(test.source, '|')
- query := strings.Replace(test.source, "|", "", 1)
- result, err := Complete(testCatalog(), query, position, PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- if test.expected != "" {
- if _, ok := findSuggestion(result.Suggestions, test.expected); !ok {
- t.Fatalf("result = %#v", result)
- }
- }
- if test.excluded != "" {
- if _, ok := findSuggestion(result.Suggestions, test.excluded); ok {
- t.Fatalf("result leaked %q: %#v", test.excluded, result)
- }
- }
- })
- }
-}
-
-func TestRecoversPropertyOriginsForIncompleteOuterClause(t *testing.T) {
- for _, test := range []struct {
- name, source, expected, excluded string
- }{
- {name: "event property", source: "WITH unused AS (SELECT properties AS props FROM persons), recent AS (SELECT properties AS props FROM events) SELECT recent.props.$geo_c| FROM recent WHERE uuid =", expected: "$geo_country", excluded: "email"},
- {name: "person property", source: "WITH unused AS (SELECT properties AS props FROM events), recent AS (SELECT properties AS props FROM persons) SELECT recent.props.$geo_c| FROM recent WHERE id =", expected: "$geo_city", excluded: "$geo_country"},
- } {
- t.Run(test.name, func(t *testing.T) {
- position := strings.IndexByte(test.source, '|')
- query := strings.Replace(test.source, "|", "", 1)
- result, err := Complete(testCatalog(), query, position, PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- if _, ok := findSuggestion(result.Suggestions, test.expected); result.ParseError == "" || !ok {
- t.Fatalf("result = %#v", result)
- }
- if _, ok := findSuggestion(result.Suggestions, test.excluded); ok {
- t.Fatalf("result leaked %q: %#v", test.excluded, result)
- }
- })
- }
-}
-
-func TestDoesNotRecoverUnsupportedIncompleteCTEScopes(t *testing.T) {
- for _, test := range []struct {
- name, source, excluded string
- }{
- {name: "damaged cte", source: "WITH recent AS (SELECT uuid FROM events WHERE ( SELECT recent.uu| FROM recent WHERE (", excluded: "uuid"},
- {name: "missing cte close", source: "WITH recent AS (SELECT uuid FROM events SELECT recent.uu| FROM recent WHERE (", excluded: "uuid"},
- {name: "cursor in cte", source: "WITH recent AS (SELECT uu| FROM events) SELECT * FROM recent WHERE (", excluded: "uuid"},
- {name: "nested outer select", source: "WITH recent AS (SELECT uuid FROM events) SELECT * FROM (SELECT recent.uu| FROM recent) AS nested WHERE (", excluded: "uuid"},
- {name: "incomplete join source", source: "WITH recent AS (SELECT uuid FROM events) SELECT recent.uu| FROM recent JOIN WHERE (", excluded: "uuid"},
- {name: "incomplete join condition", source: "WITH recent AS (SELECT uuid FROM events) SELECT recent.uu| FROM recent JOIN events ON ( WHERE (", excluded: "uuid"},
- {name: "set operation", source: "WITH recent AS (SELECT uuid FROM events) SELECT recent.uu| FROM recent UNION SELECT uuid FROM events WHERE (", excluded: "uuid"},
- {name: "multiple statements", source: "WITH recent AS (SELECT uuid FROM events) SELECT recent.uu| FROM recent; SELECT * FROM events WHERE (", excluded: "uuid"},
- {name: "scalar with alias", source: "WITH 1 AS recent SELECT events.uu| FROM events WHERE (", excluded: "uuid"},
- {name: "unterminated comment after cursor", source: "WITH recent AS (SELECT uuid FROM events) SELECT recent.uu| FROM recent WHERE /* unfinished", excluded: "uuid"},
- } {
- t.Run(test.name, func(t *testing.T) {
- position := strings.IndexByte(test.source, '|')
- query := strings.Replace(test.source, "|", "", 1)
- result, err := Complete(testCatalog(), query, position, PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- if result.ParseError == "" {
- t.Fatalf("incomplete query lost its parse error: %#v", result)
- }
- if _, ok := findSuggestion(result.Suggestions, test.excluded); ok {
- t.Fatalf("result recovered unsupported scope: %#v", result)
- }
- })
- }
-}
-
-func TestCompletesScopedProjections(t *testing.T) {
- for _, test := range []struct {
- name, query string
- fields map[string]string
- }{
- {"cte", "WITH t AS (SELECT order_id, amount AS total FROM orders) SELECT t.| FROM t", map[string]string{"order_id": "string", "total": "float"}},
- {"boolean literal type", "WITH t AS (SELECT TRUE AS enabled FROM events) SELECT t.| FROM t", map[string]string{"enabled": "boolean"}},
- {"closed comment after cursor", "WITH t AS (SELECT uuid FROM events) SELECT t.uu| FROM t /* finished */", map[string]string{"uuid": "string"}},
- {"line comment at eof after cursor", "WITH t AS (SELECT uuid FROM events) SELECT t.uu| FROM t -- finished", map[string]string{"uuid": "string"}},
- {"before from", "WITH t AS (SELECT amount AS total FROM orders) SELECT t.|", map[string]string{"total": "float"}},
- {"unqualified", "WITH t AS (SELECT amount AS total FROM orders) SELECT tot| FROM t", map[string]string{"total": "float"}},
- {"chained", "WITH a AS (SELECT amount AS total FROM orders), b AS (SELECT * FROM a) SELECT b.| FROM b", map[string]string{"total": "float"}},
- {"shadow catalog", "WITH orders AS (SELECT event FROM events) SELECT orders.| FROM orders", map[string]string{"event": "string"}},
- {"nested shadow", "WITH t AS (SELECT amount FROM orders) SELECT * FROM (WITH t AS (SELECT event FROM events) SELECT t.| FROM t) AS s", map[string]string{"event": "string"}},
- {"subquery", "SELECT s.| FROM (SELECT order_id, amount AS total FROM orders) AS s", map[string]string{"order_id": "string", "total": "float"}},
- {"nested subquery", "SELECT s.| FROM (SELECT x.total FROM (SELECT amount AS total FROM orders) AS x) AS s", map[string]string{"total": "float"}},
- {"warehouse", "WITH t AS (SELECT * FROM postgres.synced.orders) SELECT t.| FROM t", map[string]string{"synced_id": "string"}},
- {"no sibling alias", "SELECT x.| FROM (SELECT amount FROM orders AS x) AS s", nil},
- {"no sibling cte", "WITH t AS (SELECT amount FROM orders), u AS (SELECT x.| FROM events) SELECT * FROM orders AS x", nil},
- {"no later cte", "WITH a AS (SELECT b.| FROM orders), b AS (SELECT event FROM events) SELECT * FROM a", nil},
- {"no self cte", "WITH a AS (SELECT a.| FROM orders) SELECT * FROM a", nil},
- {"no sibling statement", "SELECT * FROM orders AS x; SELECT x.| FROM events", nil},
- {"no sibling union", "SELECT * FROM orders AS x UNION ALL SELECT x.| FROM events", nil},
- {"malformed scopes", "WITH t AS (SELECT amount FROM orders AS x) SELECT x.| FROM (", nil},
- {"malformed literal", "SELECT 'FROM orders AS x' WHERE x.| =", nil},
- {"property shadow", "WITH events AS (SELECT amount AS properties FROM orders) SELECT events.properties.| FROM events", nil},
- {"property shadow before from", "WITH events AS (SELECT amount AS properties FROM orders) SELECT events.properties.|", nil},
- {"unrelated cte preserves properties", "WITH t AS (SELECT 1 AS x) SELECT properties.$geo_ci| FROM events JOIN t ON 1 = 1", map[string]string{"$geo_city": "String"}},
- {"unrelated subquery preserves properties", "SELECT properties.$geo_ci| FROM events JOIN (SELECT 1 AS x) AS t ON 1 = 1", map[string]string{"$geo_city": "String"}},
- {"renamed properties are unrelated", "WITH t AS (SELECT properties AS attrs FROM events) SELECT properties.$geo_ci| FROM events JOIN t ON 1 = 1", map[string]string{"$geo_city": "String"}},
- {"derived properties are ambiguous", "WITH t AS (SELECT properties FROM events) SELECT properties.$geo_ci| FROM events JOIN t ON 1 = 1", nil},
- {"qualified physical properties remain available", "WITH t AS (SELECT properties FROM events) SELECT e.properties.$geo_ci| FROM events AS e JOIN t ON 1 = 1", map[string]string{"$geo_city": "String"}},
- {"lowercase alias keeps event properties", "SELECT e.properties.$geo_co| FROM events AS e JOIN persons AS E ON 1 = 1", map[string]string{"$geo_country": "String"}},
- {"uppercase alias keeps person properties", "SELECT E.properties.$geo| FROM events AS e JOIN persons AS E ON 1 = 1", map[string]string{"$geo_city": "String"}},
- {"wrong-case alias does not recover event properties", "SELECT E.properties.$geo| FROM events AS e", nil},
- {"exact custom table does not inherit event properties", "SELECT Events.properties.$geo| FROM Events", nil},
- {"nested virtual owner requires exact qualifier", "SELECT e.person.properties.$geo_ci| FROM events AS e", map[string]string{"$geo_city": "String"}},
- {"nested virtual owner rejects wrong-case qualifier", "SELECT E.person.properties.$geo| FROM events AS e", nil},
- {"cte property provenance", "WITH recent AS (SELECT properties FROM events) SELECT recent.properties.$geo_co| FROM recent", map[string]string{"$geo_country": "String"}},
- {"unqualified cte property provenance", "WITH recent AS (SELECT properties FROM events) SELECT properties.$geo_co| FROM recent", map[string]string{"$geo_country": "String"}},
- {"subquery property provenance", "SELECT recent.properties.$geo_co| FROM (SELECT properties FROM events) AS recent", map[string]string{"$geo_country": "String"}},
- {"renamed property provenance", "WITH recent AS (SELECT properties AS props FROM events) SELECT recent.props.$geo_co| FROM recent", map[string]string{"$geo_country": "String"}},
- {"unqualified renamed property provenance", "WITH recent AS (SELECT properties AS props FROM events) SELECT props.$geo_co| FROM recent", map[string]string{"$geo_country": "String"}},
- {"chained property provenance", "WITH a AS (SELECT properties FROM events), b AS (SELECT properties FROM a) SELECT b.properties.$geo_co| FROM b", map[string]string{"$geo_country": "String"}},
- {"wildcard property provenance", "WITH a AS (SELECT * FROM events), b AS (SELECT * FROM a) SELECT b.properties.$geo_co| FROM b", map[string]string{"$geo_country": "String"}},
- {"qualified wildcard property provenance", "WITH a AS (SELECT e.* FROM events AS e JOIN events AS other ON 1 = 1) SELECT a.properties.$geo_co| FROM a", map[string]string{"$geo_country": "String"}},
- {"visible property alias", "SELECT properties AS props, props.$geo_co| FROM events", map[string]string{"$geo_country": "String"}},
- {"property alias chain", "SELECT properties AS props, props AS attrs, attrs.$geo_co| FROM events", map[string]string{"$geo_country": "String"}},
- {"property alias before duplicate", "SELECT properties AS props, props.$geo_co|, uuid AS props FROM events", map[string]string{"$geo_country": "String"}},
- {"property alias after duplicate", "SELECT properties AS props, uuid AS props, props.$geo| FROM events", nil},
- {"cte name does not determine provenance", "WITH events AS (SELECT properties FROM persons) SELECT events.properties.$geo| FROM events", map[string]string{"$geo_city": "String"}},
- {"case-variant CTE property provenance", "WITH t AS (SELECT properties FROM events), T AS (SELECT properties FROM persons) SELECT T.properties.$geo| FROM t JOIN T ON 1 = 1", map[string]string{"$geo_city": "String"}},
- {"lowercase case-variant CTE property provenance", "WITH t AS (SELECT properties FROM events), T AS (SELECT properties FROM persons) SELECT t.properties.$geo_co| FROM t JOIN T ON 1 = 1", map[string]string{"$geo_country": "String"}},
- {"wrong-case CTE has no fields", "WITH t AS (SELECT properties FROM events) SELECT T.| FROM T", nil},
- {"case-variant table fields stay isolated", "SELECT Events.| FROM Events", map[string]string{"custom_field": "string", "properties": "json"}},
- {"ambiguous joined properties", "WITH t AS (SELECT properties FROM events JOIN persons ON 1 = 1) SELECT t.properties.$geo| FROM t", nil},
- {"self join properties are ambiguous", "WITH t AS (SELECT properties FROM events AS e JOIN events AS other ON 1 = 1) SELECT t.properties.$geo| FROM t", nil},
- {"unaliased self join properties are ambiguous", "WITH t AS (SELECT properties FROM events JOIN events ON 1 = 1) SELECT t.properties.$geo| FROM t", nil},
- {"qualified duplicate table name suppresses provenance", "WITH t AS (SELECT events.properties FROM events JOIN events ON 1 = 1) SELECT t.properties.$geo| FROM t", nil},
- {"qualified duplicate wildcard suppresses provenance", "WITH t AS (SELECT e.* FROM events AS e JOIN events AS e ON 1 = 1) SELECT t.properties.$geo| FROM t", nil},
- {"duplicate qualifier suppresses virtual properties", "SELECT e.person.properties.$geo| FROM events AS e JOIN events AS e ON 1 = 1", nil},
- {"self join wildcard properties are ambiguous", "WITH t AS (SELECT * FROM events AS e JOIN events AS other ON 1 = 1) SELECT t.properties.$geo| FROM t", nil},
- {"inner alias hides outer property source", "SELECT (SELECT properties.$geo| FROM persons AS e) FROM events AS e", map[string]string{"$geo_city": "String"}},
- {"duplicate projected properties", "WITH t AS (SELECT events.properties, events.properties FROM events) SELECT t.properties.$geo| FROM t", nil},
- {"scalar property leaf has no provenance", "WITH t AS (SELECT properties.$geo_city AS city FROM events) SELECT t.city.$geo| FROM t", nil},
- {"boolean literal does not inherit alias provenance", "WITH t AS (SELECT properties AS TRUE, TRUE AS enabled FROM events) SELECT t.enabled.$geo| FROM t", nil},
- {"derived body isolation", "SELECT * FROM orders AS x JOIN (SELECT x.| FROM events) AS s ON 1 = 1", nil},
- {"joined derived sources", "WITH t AS (SELECT event FROM events) SELECT s.| FROM t JOIN (SELECT amount AS total FROM orders) AS s ON 1 = 1", map[string]string{"total": "float"}},
- {"unicode prefix", "WITH t AS (SELECT amount AS `数額` FROM orders) SELECT t.数| FROM t", map[string]string{"数額": "float"}},
- {"earlier select alias", "SELECT amount AS total, tot| FROM orders", map[string]string{"total": "float"}},
- {"alias chain", "SELECT amount AS total, total AS subtotal, sub| FROM orders", map[string]string{"subtotal": "float"}},
- {"alias in where", "SELECT amount AS total FROM orders WHERE tot| > 0", map[string]string{"total": "float"}},
- {"alias in prewhere", "SELECT amount AS total FROM orders PREWHERE tot| > 0", map[string]string{"total": "float"}},
- {"alias in group by", "SELECT amount AS total FROM orders GROUP BY tot|", map[string]string{"total": "float"}},
- {"alias in having", "SELECT sum(amount) AS total FROM orders HAVING tot| > 0", map[string]string{"total": ""}},
- {"alias in order by", "SELECT amount AS total FROM orders ORDER BY tot|", map[string]string{"total": "float"}},
- {"alias in window", "SELECT amount AS total FROM orders WINDOW w AS (PARTITION BY tot|)", map[string]string{"total": "float"}},
- {"alias in limit", "SELECT amount AS total FROM orders LIMIT tot|", map[string]string{"total": "float"}},
- {"alias without from", "SELECT 1 AS total ORDER BY tot|", map[string]string{"total": ""}},
- {"alias shadows field", "SELECT order_id AS amount FROM orders ORDER BY amo|", map[string]string{"amount": "string"}},
- {"qualified field bypasses alias", "SELECT order_id AS amount FROM orders ORDER BY orders.amo|", map[string]string{"amount": "float"}},
- {"projected alias chain", "SELECT s.sub| FROM (SELECT amount AS total, total AS subtotal FROM orders) AS s", map[string]string{"subtotal": "float"}},
- {"no forward select alias", "SELECT tot|, amount AS total FROM orders", nil},
- {"no self select alias", "SELECT tot| AS total FROM orders", nil},
- {"no alias in join", "SELECT amount AS total FROM orders JOIN events ON tot| = 1", nil},
- {"no outer select alias", "SELECT amount AS total FROM orders WHERE order_id IN (SELECT tot| FROM events)", nil},
- {"no inner select alias", "SELECT tot| FROM orders WHERE order_id IN (SELECT event AS total FROM events)", nil},
- {"no select alias across statements", "SELECT amount AS total FROM orders; SELECT tot| FROM events", nil},
- {"no select alias across union", "SELECT amount AS total FROM orders UNION ALL SELECT tot| FROM orders", nil},
- {"union all property isolation", "SELECT properties FROM events UNION ALL SELECT properties.$geo| FROM persons", map[string]string{"$geo_city": "String"}},
- {"union arm local property alias", "SELECT properties AS props FROM events UNION ALL SELECT properties AS props FROM persons WHERE props.$geo| = 'x'", map[string]string{"$geo_city": "String"}},
- {"union arm does not inherit property alias", "SELECT properties AS props FROM events UNION ALL SELECT props.$geo| FROM persons", nil},
- {"union distinct property isolation", "SELECT properties FROM events UNION DISTINCT SELECT properties.$geo| FROM persons", map[string]string{"$geo_city": "String"}},
- {"except property isolation", "SELECT properties FROM events EXCEPT SELECT properties.$geo| FROM persons", map[string]string{"$geo_city": "String"}},
- {"no select alias in cte", "WITH t AS (SELECT tot| FROM orders) SELECT amount AS total FROM orders", nil},
- {"alias property shadow", "SELECT uuid AS properties FROM events ORDER BY properties.$geo_ci|", nil},
- {"alias virtual property shadow", "SELECT uuid AS session FROM events ORDER BY session.properties.$entry|", nil},
- {"qualified properties bypass alias", "SELECT uuid AS properties FROM events ORDER BY events.properties.$geo_ci|", map[string]string{"$geo_city": "String"}},
- {"no recovered select aliases", "SELECT amount AS total FROM orders WHERE tot| >", nil},
- {"recovery preserves relation case", "SELECT Mixed.cus| FROM Events AS Mixed WHERE custom_field =", map[string]string{"custom_field": "string"}},
- } {
- t.Run(test.name, func(t *testing.T) {
- position := strings.IndexByte(test.query, '|')
- query := strings.Replace(test.query, "|", "", 1)
- result, err := Complete(testCatalog(), query, position, PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- if test.name == "recovery preserves relation case" && !strings.HasPrefix(result.ParseError, "parse incomplete SQL:") {
- t.Fatalf("parse error = %q, want recovered incomplete SQL error", result.ParseError)
- }
- fields := map[string]string{}
- for _, suggestion := range result.Suggestions {
- if suggestion.Kind == "field" || suggestion.Kind == "property" {
- if _, duplicate := fields[suggestion.Label]; duplicate {
- t.Fatalf("duplicate suggestion: %#v", suggestion)
- }
- fields[suggestion.Label] = suggestion.Detail
- }
- }
- if len(fields) != len(test.fields) {
- t.Fatalf("fields = %#v, want %#v; parse error = %s", fields, test.fields, result.ParseError)
- }
- for name, detail := range test.fields {
- if actual, ok := fields[name]; !ok || actual != detail {
- t.Errorf("field %q = %q (%t), want %q", name, actual, ok, detail)
- }
- }
- })
- }
-}
-
-func derivedLookupQuery(sources, aliases, fields, missing int) (string, int) {
- items := make([]string, fields)
- for index := range items {
- items[index] = fmt.Sprintf("amount AS field_%03d", index)
- }
- ctes := []string{"c0 AS (SELECT " + strings.Join(items, ", ") + " FROM orders)"}
- for index := 1; index < sources; index++ {
- ctes = append(ctes, fmt.Sprintf("c%d AS (SELECT * FROM c0)", index))
- }
- from := "c0 AS a0"
- for index := 1; index < aliases; index++ {
- from += fmt.Sprintf(" JOIN c%d AS a%d ON 1 = 1", index%sources, index)
- }
- projections := strings.Repeat("unknown_identifier, ", missing) + "a0.field_000 AS known"
- ctes = append(ctes, "result AS (SELECT "+projections+" FROM "+from+")")
- prefix := "WITH " + strings.Join(ctes, ", ") + " SELECT result."
- return prefix + " FROM result", len(prefix)
-}
-
-func TestDerivedLookupWork(t *testing.T) {
- for _, test := range []struct {
- name string
- sources, aliases, fields, missing int
- limit bool
- }{
- {name: "repeated aliases share one field index", sources: 1, aliases: 512, fields: 256, missing: 256},
- {name: "distinct sources exhaust lookup budget", sources: 128, aliases: 128, fields: 8, missing: 512, limit: true},
- } {
- t.Run(test.name, func(t *testing.T) {
- query, position := derivedLookupQuery(test.sources, test.aliases, test.fields, test.missing)
- if err := querylimits.Validate(query); err != nil {
- t.Fatal(err)
- }
- result, err := Complete(testCatalog(), query, position, PositionEncodingUTF8, "")
- if test.limit {
- if !errors.Is(err, querylimits.ErrFieldLookupTooLarge) || len(result.Suggestions) != 0 {
- t.Fatalf("result = %#v, err = %v", result, err)
- }
- } else if known, ok := findSuggestion(result.Suggestions, "known"); err != nil || !ok || known.Detail != "float" {
- t.Fatalf("result = %#v, err = %v", result, err)
- }
- if test.limit {
- result, err = Complete(testCatalog(), query+" WHERE (", position, PositionEncodingUTF8, "")
- if !errors.Is(err, querylimits.ErrFieldLookupTooLarge) || len(result.Suggestions) != 0 {
- t.Fatalf("incomplete result = %#v, err = %v", result, err)
- }
- }
- })
- }
-}
-
-func BenchmarkCompleteDerivedLookups(b *testing.B) {
- query, position := derivedLookupQuery(1, 512, 256, 256)
- schema := testCatalog()
- b.ReportAllocs()
- b.ResetTimer()
- for range b.N {
- if _, err := Complete(schema, query, position, PositionEncodingUTF8, ""); err != nil {
- b.Fatal(err)
- }
- }
-}
-
-func TestCompletionFieldLookupWorkBudget(t *testing.T) {
- tables := map[string]catalog.Table{}
- var sources []string
- for index := range 128 {
- name := fmt.Sprintf("source_%d", index)
- tables[name] = catalog.Table{Name: name, Fields: map[string]catalog.Field{
- "amount": {Name: "amount", Type: "float"},
- "field_" + strings.Repeat("x", 8192): {Type: "float"},
- }}
- sources = append(sources, name)
- }
- schema := catalog.Prepare(&catalog.Catalog{Tables: tables})
- for _, query := range []string{
- "SELECT " + strings.Repeat("x", 8192) + " AS total FROM " + strings.Join(sources, " CROSS JOIN ") + " ORDER BY tot|",
- "SELECT field_| FROM " + strings.Join(sources, " CROSS JOIN "),
- } {
- position := strings.IndexByte(query, '|')
- query = strings.Replace(query, "|", "", 1)
- if err := querylimits.Validate(query); err != nil {
- t.Fatal(err)
- }
- result, err := Complete(schema, query, position, PositionEncodingUTF8, "")
- if !errors.Is(err, querylimits.ErrFieldLookupTooLarge) || len(result.Suggestions) != 0 {
- t.Fatalf("result = %#v, err = %v", result, err)
- }
- }
-}
-
-func TestProjectionPaginationAndLimits(t *testing.T) {
- var items []string
- var names []string
- for index := 0; index < PageSize+2; index++ {
- name := fmt.Sprintf("field_%02d", index)
- if index == PageSize {
- name = names[index-1] + "$x"
- }
- names = append(names, name)
- items = append(items, "amount AS "+name)
- }
- items = append(items, "amount AS field_00")
- for _, source := range []string{
- "WITH t AS (SELECT " + strings.Join(items, ", ") + " FROM orders) SELECT t.| FROM t",
- "SELECT t.| FROM (SELECT " + strings.Join(items, ", ") + " FROM orders) AS t",
- "SELECT " + strings.Join(items[:PageSize+2], ", ") + " FROM orders ORDER BY field_|",
- } {
- position := strings.IndexByte(source, '|')
- query := strings.Replace(source, "|", "", 1)
- cursor := ""
- var fields []string
- previousSortText := ""
- for {
- result, err := Complete(testCatalog(), query, position, PositionEncodingUTF8, cursor)
- if err != nil || result.Total != PageSize+2 || len(result.Suggestions) > PageSize {
- t.Fatalf("result = %#v, err = %v", result, err)
- }
- for _, suggestion := range result.Suggestions {
- if suggestion.SortText <= previousSortText {
- t.Fatalf("query %q: sort key %q does not follow %q", query, suggestion.SortText, previousSortText)
- }
- previousSortText = suggestion.SortText
- fields = append(fields, suggestion.Label)
- }
- cursor = result.NextCursor
- if cursor == "" {
- break
- }
- if len(fields) > PageSize+2 {
- t.Fatal("pagination did not terminate")
- }
- }
- if len(fields) != PageSize+2 {
- t.Fatalf("fields = %#v", fields)
- }
- for index, name := range fields {
- if name != names[index] {
- t.Fatalf("fields = %#v", fields)
- }
- }
- }
-
- ctes := []string{"c0 AS (SELECT * FROM orders)"}
- for index := 1; index < 15; index++ {
- ctes = append(ctes, fmt.Sprintf("c%d AS (SELECT a.*, b.* FROM c%d AS a JOIN c%d AS b ON 1 = 1)", index, index-1, index-1))
- }
- for _, projection := range []string{"c14.", ""} {
- prefix := "WITH " + strings.Join(ctes, ", ") + " SELECT " + projection
- for _, suffix := range []string{" FROM c14", " FROM c14 WHERE ("} {
- _, err := Complete(testCatalog(), prefix+suffix, len(prefix), PositionEncodingUTF8, "")
- if !errors.Is(err, querylimits.ErrCTEProjectionTooLarge) {
- t.Fatalf("projection %q, suffix %q: err = %v", projection, suffix, err)
- }
- }
- }
-}
-
-func TestCompletionQuotesIdentifierInsertionText(t *testing.T) {
- schema := catalog.Prepare(&catalog.Catalog{Tables: map[string]catalog.Table{
- "from": {Name: "from", Type: "data_warehouse", Fields: map[string]catalog.Field{}},
- "order-items": {Name: "order-items", Type: "data_warehouse", Fields: map[string]catalog.Field{}},
- "percent%table": {Name: "percent%table", Type: "data_warehouse", Fields: map[string]catalog.Field{}},
- "orders": {Name: "orders", Type: "data_warehouse", Fields: map[string]catalog.Field{
- "billing address": {Name: "billing address", Type: "string"},
- "FROM": {Name: "FROM", Type: "string"},
- "order-total": {Name: "order-total", Type: "float"},
- "percent%field": {Name: "percent%field", Type: "string"},
- "tick`value": {Name: "tick`value", Type: "string"},
- "timestamp": {Name: "timestamp", Type: "datetime"},
- }},
- }, Properties: map[string][]catalog.Property{}})
-
- tableResult, err := Complete(schema, "SELECT * FROM order", len("SELECT * FROM order"), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- keywordTableResult, err := Complete(schema, "SELECT * FROM fr", len("SELECT * FROM fr"), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- unsupportedTableResult, err := Complete(schema, "SELECT * FROM percent", len("SELECT * FROM percent"), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- fieldResult, err := Complete(schema, "SELECT o. FROM orders AS o", len("SELECT o."), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- aliasQuery := "SELECT o.\"order-total\" AS \"billing total\" FROM orders AS o ORDER BY bill"
- aliasResult, err := Complete(schema, aliasQuery, len(aliasQuery), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- cteQuery := "WITH `recent.items` AS (SELECT * FROM orders), `recent items` AS (SELECT * FROM orders) SELECT * FROM rec"
- cteResult, err := Complete(schema, cteQuery, len(cteQuery), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
-
- for _, test := range []struct {
- result Result
- label string
- insertText string
- }{
- {result: tableResult, label: "order-items", insertText: "`order-items`"},
- {result: keywordTableResult, label: "from", insertText: "`from`"},
- {result: fieldResult, label: "billing address", insertText: "`billing address`"},
- {result: fieldResult, label: "FROM", insertText: "`FROM`"},
- {result: fieldResult, label: "order-total", insertText: "`order-total`"},
- {result: fieldResult, label: "tick`value", insertText: "`tick``value`"},
- {result: fieldResult, label: "timestamp", insertText: ""},
- {result: aliasResult, label: "billing total", insertText: "`billing total`"},
- {result: cteResult, label: "recent.items", insertText: "`recent.items`"},
- {result: cteResult, label: "recent items", insertText: "`recent items`"},
- } {
- suggestion, ok := findSuggestion(test.result.Suggestions, test.label)
- if !ok || suggestion.InsertText != test.insertText {
- t.Fatalf("suggestion %q = %#v, want insert text %q", test.label, suggestion, test.insertText)
- }
- }
- for _, test := range []struct {
- query, insertText string
- }{
- {"SELECT * FROM orders WHERE 1 = 1 AND tim| > now()", "timestamp"},
- {"SELECT o.tim| FROM orders AS o", "timestamp"},
- {"SELECT tim| FROM orders AS a JOIN orders AS b ON 1 = 1", "a.timestamp"},
- } {
- position := strings.IndexByte(test.query, '|')
- query := strings.Replace(test.query, "|", "", 1)
- result, err := Complete(schema, query, position, PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- suggestion, ok := findSuggestion(result.Suggestions, "timestamp")
- insertText := suggestion.InsertText
- if insertText == "" {
- insertText = suggestion.Label
- }
- if !ok || insertText != test.insertText {
- t.Fatalf("query %q: suggestion = %#v, want insertion %q", query, suggestion, test.insertText)
- }
- completed := query[:position-len("tim")] + insertText + query[position:]
- if checked := validation.Validate(schema, completed); !checked.Valid {
- t.Errorf("inserted query %q is invalid: %#v", completed, checked)
- }
- }
- for _, test := range []struct {
- result Result
- label string
- }{
- {result: fieldResult, label: "percent%field"},
- {result: unsupportedTableResult, label: "percent%table"},
- } {
- if suggestion, ok := findSuggestion(test.result.Suggestions, test.label); ok {
- t.Fatalf("unsupported suggestion %q = %#v", test.label, suggestion)
- }
- }
-}
-
-func TestCompletionPaginationSkipsUnsupportedIdentifiers(t *testing.T) {
- tables := make(map[string]catalog.Table, PageSize+2)
- for index := range PageSize + 1 {
- name := fmt.Sprintf("table_%02d", index)
- tables[name] = catalog.Table{Name: name, Type: "data_warehouse", Fields: map[string]catalog.Field{}}
- }
- tables["table_%"] = catalog.Table{Name: "table_%", Type: "data_warehouse", Fields: map[string]catalog.Field{}}
- schema := catalog.Prepare(&catalog.Catalog{Tables: tables, Properties: map[string][]catalog.Property{}})
- query := "SELECT * FROM table_"
-
- first, err := Complete(schema, query, len(query), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- second, err := Complete(schema, query, len(query), PositionEncodingUTF8, first.NextCursor)
- if err != nil {
- t.Fatal(err)
- }
- if first.Total != PageSize+1 || len(first.Suggestions) != PageSize || first.NextCursor == "" {
- t.Fatalf("first page = %#v", first)
- }
- if second.Total != PageSize+1 || len(second.Suggestions) != 1 || second.NextCursor != "" {
- t.Fatalf("second page = %#v", second)
- }
-}
-
-func TestDoesNotCompleteFieldsForWrongCaseTableReference(t *testing.T) {
- query := "SELECT Orders. FROM Orders"
- result, err := Complete(testCatalog(), query, len("SELECT Orders."), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- if len(result.Suggestions) != 0 {
- t.Fatalf("suggestions = %#v; parse error = %q", result.Suggestions, result.ParseError)
- }
-}
-
-func TestCompletesSQLSyntaxForCursorContext(t *testing.T) {
- tests := []struct {
- name string
- query string
- position int
- label string
- kind string
- insertText string
- required []string
- excluded []string
- total int
- }{
- {name: "empty query select", query: "", position: 0, label: "SELECT", kind: "keyword", total: 2},
- {name: "empty query with", query: "", position: 0, label: "WITH", kind: "keyword", total: 2},
- {name: "whitespace query", query: " \n\t\u2003", position: len(" \n\t\u2003"), label: "SELECT", kind: "keyword", total: 2},
- {name: "select prefix", query: "sel", position: 3, label: "SELECT", kind: "keyword", total: 1},
- {name: "with prefix", query: "wi", position: 2, label: "WITH", kind: "keyword", total: 1},
- {name: "after comment", query: "-- example\n", position: len("-- example\n"), label: "WITH", kind: "keyword", total: 2},
- {name: "after statement", query: "SELECT 1; ", position: len("SELECT 1; "), label: "SELECT", kind: "keyword", total: 2},
- {name: "function in select", query: "SELECT cou FROM orders", position: len("SELECT cou"), label: "count", kind: "function", insertText: "count()"},
- {name: "embedded function in select", query: "SELECT geoD FROM orders", position: len("SELECT geoD"), label: "geoDistance", kind: "function", insertText: "geoDistance()"},
- {name: "function in where", query: "SELECT * FROM orders WHERE coa", position: len("SELECT * FROM orders WHERE coa"), label: "coalesce", kind: "function", insertText: "coalesce()"},
- {name: "true after operator", query: "SELECT * FROM orders WHERE amount = tr", position: len("SELECT * FROM orders WHERE amount = tr"), label: "TRUE", kind: "keyword"},
- {name: "false after operator", query: "SELECT * FROM orders WHERE amount = fa", position: len("SELECT * FROM orders WHERE amount = fa"), label: "FALSE", kind: "keyword"},
- {name: "operator after field", query: "SELECT * FROM orders WHERE amount ", position: len("SELECT * FROM orders WHERE amount "), label: "=", kind: "operator", insertText: "=", excluded: []string{"AND"}},
- {name: "boolean after predicate", query: "SELECT * FROM orders WHERE amount > 0 ", position: len("SELECT * FROM orders WHERE amount > 0 "), label: "AND", kind: "keyword", insertText: "AND", excluded: []string{"=", "WHERE"}},
- {name: "where after join predicate", query: "SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid ", position: len("SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid "), label: "WHERE", kind: "keyword", insertText: "WHERE"},
- {name: "where prefix after join predicate", query: "SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid wh", position: len("SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid wh"), label: "WHERE", kind: "keyword", insertText: "WHERE", total: 1},
- {name: "where after parenthesized join predicate", query: "SELECT * FROM events AS e JOIN events AS other ON (e.uuid = other.uuid) ", position: len("SELECT * FROM events AS e JOIN events AS other ON (e.uuid = other.uuid) "), label: "WHERE", kind: "keyword", insertText: "WHERE"},
- {name: "no where inside join parentheses", query: "SELECT * FROM events AS e JOIN events AS other ON (e.uuid = other.uuid ", position: len("SELECT * FROM events AS e JOIN events AS other ON (e.uuid = other.uuid "), label: "AND", kind: "keyword", insertText: "AND", excluded: []string{"WHERE"}},
- {name: "no where within join between bounds", query: "SELECT * FROM orders AS a JOIN orders AS b ON a.amount BETWEEN 1 ", position: len("SELECT * FROM orders AS a JOIN orders AS b ON a.amount BETWEEN 1 "), label: "AND", kind: "keyword", insertText: "AND", total: 1},
- {name: "where after joined ctes before order by", query: "WITH a AS (SELECT uuid FROM events WHERE event = 'demo'), b AS (SELECT uuid FROM events) SELECT a.uuid FROM a LEFT JOIN b ON a.uuid = b.uuid\n\nORDER BY a.uuid", position: len("WITH a AS (SELECT uuid FROM events WHERE event = 'demo'), b AS (SELECT uuid FROM events) SELECT a.uuid FROM a LEFT JOIN b ON a.uuid = b.uuid\n"), label: "WHERE", kind: "keyword", insertText: "WHERE"},
- {name: "only boolean continuations after join predicate before later join", query: "SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid JOIN persons AS p ON 1 = 1", position: len("SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid "), label: "AND", kind: "keyword", insertText: "AND", required: []string{"OR"}, excluded: []string{"WHERE", "GROUP BY", "ORDER BY", "LIMIT"}, total: 2},
- {name: "comparison and boolean continuations after join expression before later join", query: "SELECT * FROM events AS e JOIN events AS other ON TRUE JOIN persons AS p ON 1 = 1", position: len("SELECT * FROM events AS e JOIN events AS other ON TRUE "), label: "=", kind: "operator", insertText: "=", required: []string{"AND", "OR"}, excluded: []string{"WHERE", "GROUP BY", "ORDER BY", "LIMIT"}},
- {name: "nested later join does not suppress where", query: "SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid ORDER BY (SELECT 1 FROM events JOIN persons ON 1 = 1)", position: len("SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid "), label: "WHERE", kind: "keyword", insertText: "WHERE"},
- {name: "join in comment does not suppress where", query: "SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid /* JOIN persons */ ORDER BY e.uuid", position: len("SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid "), label: "WHERE", kind: "keyword", insertText: "WHERE"},
- {name: "join in string does not suppress where", query: "SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid ORDER BY 'JOIN persons'", position: len("SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid "), label: "WHERE", kind: "keyword", insertText: "WHERE"},
- {name: "join in later statement does not suppress where", query: "SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid ; SELECT * FROM events JOIN persons ON 1 = 1", position: len("SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid "), label: "WHERE", kind: "keyword", insertText: "WHERE"},
- {name: "join in next union branch does not suppress where", query: "SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid UNION ALL SELECT * FROM events JOIN persons ON 1 = 1", position: len("SELECT * FROM events AS e JOIN events AS other ON e.uuid = other.uuid "), label: "WHERE", kind: "keyword", insertText: "WHERE"},
- {name: "no clauses inside unfinished join case", query: "SELECT * FROM orders AS a JOIN orders AS b ON a.amount = CASE WHEN b.amount > 0 ", position: len("SELECT * FROM orders AS a JOIN orders AS b ON a.amount = CASE WHEN b.amount > 0 "), label: "THEN", kind: "keyword", excluded: []string{"WHERE", "GROUP BY", "ORDER BY", "LIMIT"}},
- {name: "function inside unfinished join case", query: "SELECT * FROM orders AS a JOIN orders AS b ON a.amount = CASE WHEN b.amount > 0 THEN cou", position: len("SELECT * FROM orders AS a JOIN orders AS b ON a.amount = CASE WHEN b.amount > 0 THEN cou"), label: "count", kind: "function", insertText: "count()", excluded: []string{"WHERE"}},
- {name: "no clauses inside parenthesized join case condition", query: "SELECT * FROM orders AS a JOIN orders AS b ON CASE WHEN (", position: len("SELECT * FROM orders AS a JOIN orders AS b ON CASE WHEN ("), label: "count", kind: "function", insertText: "count()", excluded: []string{"WHERE", "GROUP BY", "ORDER BY", "LIMIT"}},
- {name: "no clauses inside join case function", query: "SELECT * FROM orders AS a JOIN orders AS b ON a.amount = CASE WHEN b.amount > 0 THEN coalesce(", position: len("SELECT * FROM orders AS a JOIN orders AS b ON a.amount = CASE WHEN b.amount > 0 THEN coalesce("), label: "count", kind: "function", insertText: "count()", excluded: []string{"WHERE", "GROUP BY", "ORDER BY", "LIMIT"}},
- {name: "inner select keeps its own where", query: "SELECT * FROM orders AS a JOIN orders AS b ON CASE WHEN (SELECT amount FROM orders )", position: len("SELECT * FROM orders AS a JOIN orders AS b ON CASE WHEN (SELECT amount FROM orders "), label: "WHERE", kind: "keyword"},
- {name: "else inside unfinished join case", query: "SELECT * FROM orders AS a JOIN orders AS b ON a.amount = CASE WHEN b.amount > 0 THEN 1 el", position: len("SELECT * FROM orders AS a JOIN orders AS b ON a.amount = CASE WHEN b.amount > 0 THEN 1 el"), label: "ELSE", kind: "keyword", excluded: []string{"WHERE"}, total: 1},
- {name: "end inside nested unfinished join case", query: "SELECT * FROM orders AS a JOIN orders AS b ON a.amount = CASE WHEN b.amount > 0 THEN CASE WHEN a.amount > 0 THEN 1 END ELSE 0 en", position: len("SELECT * FROM orders AS a JOIN orders AS b ON a.amount = CASE WHEN b.amount > 0 THEN CASE WHEN a.amount > 0 THEN 1 END ELSE 0 en"), label: "END", kind: "keyword", excluded: []string{"WHERE"}},
- {name: "where after closed join case", query: "SELECT * FROM orders AS a JOIN orders AS b ON a.amount = CASE WHEN b.amount > 0 THEN 1 ELSE 0 END ", position: len("SELECT * FROM orders AS a JOIN orders AS b ON a.amount = CASE WHEN b.amount > 0 THEN 1 ELSE 0 END "), label: "WHERE", kind: "keyword", insertText: "WHERE"},
- {name: "where after standalone closed join case", query: "SELECT * FROM events JOIN persons ON CASE WHEN TRUE THEN 1 ELSE 0 END ", position: len("SELECT * FROM events JOIN persons ON CASE WHEN TRUE THEN 1 ELSE 0 END "), label: "WHERE", kind: "keyword", insertText: "WHERE"},
- {name: "where after true join predicate", query: "SELECT * FROM events JOIN persons ON TRUE ", position: len("SELECT * FROM events JOIN persons ON TRUE "), label: "WHERE", kind: "keyword", insertText: "WHERE"},
- {name: "where after lowercase false join predicate", query: "SELECT * FROM events JOIN persons ON false ", position: len("SELECT * FROM events JOIN persons ON false "), label: "WHERE", kind: "keyword", insertText: "WHERE"},
- {name: "qualified true stays a field", query: "SELECT * FROM events AS e JOIN persons ON e.TRUE ", position: len("SELECT * FROM events AS e JOIN persons ON e.TRUE "), label: "=", kind: "operator", insertText: "=", excluded: []string{"WHERE"}},
- {name: "quoted true stays an identifier", query: `SELECT * FROM events JOIN persons ON "TRUE" `, position: len(`SELECT * FROM events JOIN persons ON "TRUE" `), label: "=", kind: "operator", insertText: "=", excluded: []string{"WHERE"}},
- {name: "between separator", query: "SELECT * FROM orders WHERE amount BETWEEN 1 ", position: len("SELECT * FROM orders WHERE amount BETWEEN 1 "), label: "AND", kind: "keyword", insertText: "AND", excluded: []string{"OR", "GROUP BY", "ORDER BY", "LIMIT"}, total: 1},
- {name: "between unfinished arithmetic expression", query: "SELECT * FROM orders WHERE amount BETWEEN 1 + cou", position: len("SELECT * FROM orders WHERE amount BETWEEN 1 + cou"), label: "count", kind: "function", insertText: "count()", excluded: []string{"AND"}},
- {name: "between unfinished function call", query: "SELECT * FROM orders WHERE amount BETWEEN toDate(amo", position: len("SELECT * FROM orders WHERE amount BETWEEN toDate(amo"), label: "amount", kind: "field", excluded: []string{"AND"}},
- {name: "between unfinished interval", query: "SELECT * FROM orders WHERE amount BETWEEN INTERVAL ", position: len("SELECT * FROM orders WHERE amount BETWEEN INTERVAL "), label: "amount", kind: "field", excluded: []string{"AND"}},
- {name: "between interval missing unit", query: "SELECT * FROM orders WHERE amount BETWEEN INTERVAL 1 ", position: len("SELECT * FROM orders WHERE amount BETWEEN INTERVAL 1 "), label: "amount", kind: "field", excluded: []string{"AND"}},
- {name: "between complete interval", query: "SELECT * FROM orders WHERE amount BETWEEN INTERVAL 1 DAY ", position: len("SELECT * FROM orders WHERE amount BETWEEN INTERVAL 1 DAY "), label: "AND", kind: "keyword", insertText: "AND", total: 1},
- {name: "between nested interval missing unit", query: "SELECT * FROM orders WHERE amount BETWEEN toDate(INTERVAL 1) ", position: len("SELECT * FROM orders WHERE amount BETWEEN toDate(INTERVAL 1) "), label: "amount", kind: "field", excluded: []string{"AND"}},
- {name: "between complete nested interval", query: "SELECT * FROM orders WHERE amount BETWEEN toDate(INTERVAL 1 DAY) ", position: len("SELECT * FROM orders WHERE amount BETWEEN toDate(INTERVAL 1 DAY) "), label: "AND", kind: "keyword", insertText: "AND", total: 1},
- {name: "between unfinished case", query: "SELECT * FROM orders WHERE amount BETWEEN CASE ", position: len("SELECT * FROM orders WHERE amount BETWEEN CASE "), label: "amount", kind: "field", excluded: []string{"AND"}},
- {name: "between unfinished case conjunction", query: "SELECT * FROM orders WHERE amount BETWEEN CASE WHEN amount > 0 AND amo", position: len("SELECT * FROM orders WHERE amount BETWEEN CASE WHEN amount > 0 AND amo"), label: "amount", kind: "field", excluded: []string{"AND"}},
- {name: "between complete case", query: "SELECT * FROM orders WHERE amount BETWEEN CASE WHEN amount > 0 THEN 1 ELSE 0 END ", position: len("SELECT * FROM orders WHERE amount BETWEEN CASE WHEN amount > 0 THEN 1 ELSE 0 END "), label: "AND", kind: "keyword", insertText: "AND", total: 1},
- {name: "field after boolean", query: "SELECT * FROM orders WHERE amount > 0 AND ", position: len("SELECT * FROM orders WHERE amount > 0 AND "), label: "amount", kind: "field"},
- {name: "field after select comma", query: "SELECT amount, FROM orders", position: len("SELECT amount, "), label: "amount", kind: "field", excluded: []string{"orders"}},
- {name: "double-quoted clause identifier", query: `SELECT "FROM" FROM orders`, position: len(`SELECT "FROM" `), label: "count", kind: "function", insertText: "count()", excluded: []string{"orders"}},
- {name: "backtick-quoted clause identifier", query: "SELECT `JOIN` FROM orders", position: len("SELECT `JOIN` "), label: "count", kind: "function", insertText: "count()", excluded: []string{"orders"}},
- }
- for _, test := range tests {
- t.Run(test.name, func(t *testing.T) {
- result, err := Complete(testCatalog(), test.query, test.position, PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- if test.total > 0 && result.Total != test.total {
- t.Fatalf("suggestion total = %d, want %d; suggestions = %#v", result.Total, test.total, result.Suggestions)
- }
- suggestion, ok := findSuggestion(result.Suggestions, test.label)
- if !ok || suggestion.Kind != test.kind || suggestion.InsertText != test.insertText {
- t.Fatalf("suggestion %q = %#v; all suggestions = %#v; parse error = %q", test.label, suggestion, result.Suggestions, result.ParseError)
- }
- for _, required := range test.required {
- if !hasSuggestion(result.Suggestions, required) {
- t.Fatalf("missing suggestion %q in %#v", required, result.Suggestions)
- }
- }
- for _, excluded := range test.excluded {
- if hasSuggestion(result.Suggestions, excluded) {
- t.Fatalf("unexpected suggestion %q in %#v", excluded, result.Suggestions)
- }
- }
- })
- }
-}
-
-func TestCompletionReturnsNoSuggestionsInsideStringOrComment(t *testing.T) {
- for _, query := range []string{
- "-- sel",
- "/* wi",
- "-- FROM postgres.demo.or",
- "/* FROM postgres.demo.or",
- "SELECT 'FROM postgres.demo.or",
- "SELECT * FROM orders WHERE order_id = 'cou",
- "SELECT * FROM orders -- cou",
- "SELECT * FROM orders /* cou",
- } {
- result, err := Complete(testCatalog(), query, len(query), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- if len(result.Suggestions) != 0 {
- t.Fatalf("query %q returned %#v", query, result.Suggestions)
- }
- }
-}
-
-func TestCompletionPagesWithoutSkippingOrRepeatingTables(t *testing.T) {
- schema := &catalog.Catalog{Tables: map[string]catalog.Table{}}
- for index := 0; index < 30; index++ {
- name := fmt.Sprintf("table_%02d", index)
- schema.Tables[name] = catalog.Table{Name: name, Type: "data_warehouse", Fields: map[string]catalog.Field{}}
- }
- prepared := catalog.Prepare(schema)
- query := "SELECT * FROM table_"
- for _, ctePrefix := range []string{"", "WITH table_05 AS (SELECT 1), table_30 AS (SELECT 2) "} {
- t.Run(ctePrefix, func(t *testing.T) {
- input := ctePrefix + query
- var expected []string
- if ctePrefix != "" {
- expected = append(expected, "table_05", "table_30")
- }
- for index := range 30 {
- if ctePrefix == "" || index != 5 {
- expected = append(expected, fmt.Sprintf("table_%02d", index))
- }
- }
- first, err := Complete(prepared, input, len(input), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- if len(first.Suggestions) != PageSize || first.NextCursor == "" || first.Total != len(expected) {
- t.Fatalf("first page = %#v", first)
- }
- second, err := Complete(prepared, input, len(input), PositionEncodingUTF8, first.NextCursor)
- if err != nil {
- t.Fatal(err)
- }
- if len(second.Suggestions) != len(expected)-PageSize || second.NextCursor != "" || second.Total != len(expected) {
- t.Fatalf("second page = %#v", second)
- }
- for index, suggestion := range append(first.Suggestions, second.Suggestions...) {
- if suggestion.Label != expected[index] {
- t.Fatalf("suggestion %d = %#v, want %s", index, suggestion, expected[index])
- }
- }
- })
- }
- if _, err := Complete(prepared, query, len(query), PositionEncodingUTF8, "not-a-cursor"); err == nil {
- t.Fatal("invalid cursor was accepted")
- }
-}
-
-func TestAliasCompletionPagesWithoutRepeatingCanonicalTargets(t *testing.T) {
- value := &catalog.Catalog{Tables: map[string]catalog.Table{}, TableAliases: map[string]string{}, Properties: map[string][]catalog.Property{}}
- for index := range 30 {
- canonical := fmt.Sprintf("postgres.demo.table_%02d", index)
- alias := fmt.Sprintf("legacy.demo.table_%02d", index)
- value.Tables[canonical] = catalog.Table{Type: "data_warehouse", Fields: map[string]catalog.Field{}}
- value.TableAliases[alias] = canonical
- }
- prepared := catalog.Prepare(value)
- query := "SELECT * FROM legacy.demo.table_"
- first, err := Complete(prepared, query, len(query), PositionEncodingUTF8, "")
- if err != nil {
- t.Fatal(err)
- }
- second, err := Complete(prepared, query, len(query), PositionEncodingUTF8, first.NextCursor)
- if err != nil {
- t.Fatal(err)
- }
- all := append(first.Suggestions, second.Suggestions...)
- if first.Total != 30 || second.Total != 30 || len(all) != 30 || second.NextCursor != "" {
- t.Fatalf("pages = %#v, %#v", first, second)
- }
- for index, suggestion := range all {
- expectedAlias := fmt.Sprintf("legacy.demo.table_%02d", index)
- expectedCanonical := fmt.Sprintf("postgres.demo.table_%02d", index)
- expectedInsertText := fmt.Sprintf("table_%02d", index)
- if suggestion.Label != expectedAlias || suggestion.Detail != expectedCanonical || suggestion.InsertText != expectedInsertText {
- t.Fatalf("suggestion %d = %#v", index, suggestion)
- }
- }
-}
-
-func hasSuggestion(suggestions []Suggestion, label string) bool {
- for _, suggestion := range suggestions {
- if suggestion.Label == label {
- return true
- }
- }
- return false
-}
-
-func findSuggestion(suggestions []Suggestion, label string) (Suggestion, bool) {
- for _, suggestion := range suggestions {
- if suggestion.Label == label {
- return suggestion, true
- }
- }
- return Suggestion{}, false
-}
-
-func largeContextualCatalog() *catalog.PreparedCatalog {
- schema := &catalog.Catalog{Tables: make(map[string]catalog.Table, 1024)}
- for tableIndex := 0; tableIndex < 1024; tableIndex++ {
- fields := make(map[string]catalog.Field, 25)
- for fieldIndex := 0; fieldIndex < 25; fieldIndex++ {
- name := fmt.Sprintf("column_%02d", fieldIndex)
- fields[name] = catalog.Field{Name: name, Type: "String"}
- }
- name := fmt.Sprintf("table_%04d", tableIndex)
- schema.Tables[name] = catalog.Table{Name: name, Type: "data_warehouse", Fields: fields}
- }
- return catalog.Prepare(schema)
-}
-
-func TestCompleteContextualCatalogStaysWithinLatencyBudget(t *testing.T) {
- if os.Getenv("HOGQL_LANGUAGE_SERVICE_ENFORCE_LATENCY_BUDGET") != "1" {
- t.Skip("latency budget requires the dedicated non-race test environment")
- }
-
- const sampleCount = 20
- const maxAverage = 5 * time.Millisecond
- const maxStandardDeviation = 5 * time.Millisecond
-
- schema := largeContextualCatalog()
- intervalPrefix := "SELECT * FROM table_0500 WHERE column_10 BETWEEN "
- intervalQuery := intervalPrefix + strings.Repeat("INTERVAL ", ((16<<10)-len(intervalPrefix)-len("1 DAY "))/len("INTERVAL ")) + "1 DAY "
- for _, test := range []struct {
- name string
- query string
- position int
- callsPerSample int
- }{
- {name: "contextual catalog", query: "SELECT countD FROM table_0500", position: len("SELECT countD"), callsPerSample: 100},
- {name: "joined fields", query: "SELECT column_ FROM table_0500 AS a JOIN table_0500 AS b ON 1 = 1", position: len("SELECT column_"), callsPerSample: 100},
- {name: "adversarial interval expression", query: intervalQuery, position: len(intervalQuery), callsPerSample: 20},
- } {
- t.Run(test.name, func(t *testing.T) {
- for range test.callsPerSample {
- if _, err := Complete(schema, test.query, test.position, PositionEncodingUTF8, ""); err != nil {
- t.Fatal(err)
- }
- }
- durations := make([]float64, sampleCount)
- for sample := range sampleCount {
- startedAt := time.Now()
- for range test.callsPerSample {
- if _, err := Complete(schema, test.query, test.position, PositionEncodingUTF8, ""); err != nil {
- t.Fatal(err)
- }
- }
- durations[sample] = float64(time.Since(startedAt)) / float64(test.callsPerSample)
- }
-
- var total float64
- for _, duration := range durations {
- total += duration
- }
- average := total / sampleCount
- var squaredDifferences float64
- for _, duration := range durations {
- difference := duration - average
- squaredDifferences += difference * difference
- }
- standardDeviation := math.Sqrt(squaredDifferences / sampleCount)
- if time.Duration(average) > maxAverage || time.Duration(standardDeviation) > maxStandardDeviation {
- t.Fatalf(
- "completion latency average = %s (max %s), standard deviation = %s (max %s)",
- time.Duration(average),
- maxAverage,
- time.Duration(standardDeviation),
- maxStandardDeviation,
- )
- }
- })
- }
-}
-
-func BenchmarkCompleteContextualCatalog(b *testing.B) {
- schema := largeContextualCatalog()
- for _, benchmark := range []struct {
- name string
- query string
- position int
- }{
- {name: "operator", query: "SELECT * FROM table_0500 WHERE column_10 ", position: len("SELECT * FROM table_0500 WHERE column_10 ")},
- {name: "function prefix", query: "SELECT countD FROM table_0500", position: len("SELECT countD")},
- {name: "joined fields", query: "SELECT column_ FROM table_0500 AS a JOIN table_0500 AS b ON 1 = 1", position: len("SELECT column_")},
- {name: "repeated interval", query: "SELECT * FROM table_0500 WHERE column_10 BETWEEN " + strings.Repeat("INTERVAL ", 1000) + "1 DAY ", position: len("SELECT * FROM table_0500 WHERE column_10 BETWEEN ") + len("INTERVAL ")*1000 + len("1 DAY ")},
- } {
- b.Run(benchmark.name, func(b *testing.B) {
- result, err := Complete(schema, benchmark.query, benchmark.position, PositionEncodingUTF8, "")
- if err != nil {
- b.Fatal(err)
- }
- payload, err := json.Marshal(result)
- if err != nil {
- b.Fatal(err)
- }
- b.ResetTimer()
- b.ReportMetric(float64(len(payload)), "response-B")
- for range b.N {
- if _, err := Complete(schema, benchmark.query, benchmark.position, PositionEncodingUTF8, ""); err != nil {
- b.Fatal(err)
- }
- }
- })
- }
-}
diff --git a/services/hogql-language-service/internal/completion/context.go b/services/hogql-language-service/internal/completion/context.go
deleted file mode 100644
index 3a9f2f2849d0..000000000000
--- a/services/hogql-language-service/internal/completion/context.go
+++ /dev/null
@@ -1,442 +0,0 @@
-package completion
-
-import (
- "strings"
- "unicode"
- "unicode/utf8"
-)
-
-type completionMode uint8
-
-const (
- completionModeGeneral completionMode = iota
- completionModeNone
- completionModeTable
- completionModeExpression
- completionModeComparison
- completionModeBetweenSeparator
- completionModePredicateContinuation
- completionModeJoinPredicateContinuation
- completionModePostExpression
- completionModeJoinPostExpression
- completionModeCaseExpression
- completionModeStatementStart
-)
-
-type sqlTokenKind uint8
-
-const (
- sqlTokenWord sqlTokenKind = iota
- sqlTokenValue
- sqlTokenOperator
- sqlTokenLeftParen
- sqlTokenRightParen
- sqlTokenComma
-)
-
-type sqlToken struct {
- text string
- raw string
- kind sqlTokenKind
- depth int
- start int
-}
-
-func analyzeCursorContext(input string) completionMode {
- tokens, depth, incomplete := scanSQLTokens(input)
- if incomplete {
- return completionModeNone
- }
- if len(tokens) == 0 || tokens[len(tokens)-1].text == ";" && depth == 0 {
- return completionModeStatementStart
- }
- clauseIndex, clause := activeClause(tokens, depth)
- switch clause {
- case "FROM", "JOIN":
- if clauseIndex == len(tokens)-1 || lastTokenAtDepth(tokens[clauseIndex+1:], depth).kind == sqlTokenComma {
- return completionModeTable
- }
- case "SELECT", "GROUP BY", "ORDER BY":
- return completionModeExpression
- case "WHERE", "PREWHERE", "HAVING":
- return predicateMode(tokens[clauseIndex+1:], depth)
- case "ON":
- mode := predicateMode(tokens[clauseIndex+1:], depth)
- if tokens[clauseIndex].depth != depth {
- return mode
- }
- if mode == completionModePredicateContinuation {
- return completionModeJoinPredicateContinuation
- }
- if mode == completionModePostExpression {
- return completionModeJoinPostExpression
- }
- return mode
- }
- return completionModeGeneral
-}
-
-func activeClause(tokens []sqlToken, depth int) (int, string) {
- for index := len(tokens) - 1; index >= 0; index-- {
- token := tokens[index]
- if token.depth > depth || token.kind != sqlTokenWord {
- continue
- }
- switch token.text {
- case "WHERE", "PREWHERE", "HAVING", "ON", "FROM", "JOIN", "SELECT", "LIMIT":
- return index, token.text
- case "BY":
- if index > 0 && tokens[index-1].depth == token.depth && (tokens[index-1].text == "GROUP" || tokens[index-1].text == "ORDER") {
- return index, tokens[index-1].text + " BY"
- }
- }
- }
- return -1, ""
-}
-
-func predicateMode(tokens []sqlToken, depth int) completionMode {
- start := 0
- betweenPending := false
- caseDepth := 0
- openExpressionCases := 0
- closedCase := false
- for index, token := range tokens {
- if token.kind == sqlTokenWord {
- switch token.text {
- case "CASE":
- openExpressionCases++
- case "END":
- if openExpressionCases > 0 {
- openExpressionCases--
- }
- }
- }
- if token.kind == sqlTokenLeftParen && token.depth == depth-1 {
- start = index + 1
- betweenPending = false
- caseDepth = 0
- continue
- }
- if token.depth != depth || token.kind != sqlTokenWord {
- continue
- }
- switch token.text {
- case "CASE":
- caseDepth++
- case "END":
- if caseDepth > 0 {
- caseDepth--
- closedCase = true
- }
- case "BETWEEN":
- if caseDepth == 0 {
- betweenPending = true
- }
- case "AND":
- if caseDepth > 0 {
- continue
- }
- if betweenPending {
- betweenPending = false
- continue
- }
- start = index + 1
- case "OR":
- if caseDepth == 0 {
- start = index + 1
- betweenPending = false
- }
- }
- }
- segment := tokens[start:]
- last := lastTokenAtDepth(segment, depth)
- if openExpressionCases > 0 || caseDepth > 0 {
- return completionModeCaseExpression
- }
- if last.text == "" || last.text == "AND" || last.text == "OR" || last.kind == sqlTokenComma || last.kind == sqlTokenLeftParen {
- return completionModeExpression
- }
- if last.kind == sqlTokenOperator {
- return completionModeExpression
- }
- comparisonIndex := -1
- for index, token := range segment {
- if token.depth == depth && isComparisonToken(token) {
- comparisonIndex = index
- break
- }
- }
- if comparisonIndex >= 0 {
- if betweenPending && segment[comparisonIndex].text == "BETWEEN" {
- if betweenLowerBoundComplete(segment[comparisonIndex+1:], depth) {
- return completionModeBetweenSeparator
- }
- return completionModeExpression
- }
- for _, token := range segment[comparisonIndex+1:] {
- if token.depth <= depth && token.text != "NOT" {
- return completionModePredicateContinuation
- }
- }
- return completionModeExpression
- }
- if last.kind == sqlTokenRightParen || last.kind == sqlTokenValue || closedCase && last.text == "END" || isUnqualifiedBooleanToken(segment, depth) {
- return completionModePostExpression
- }
- return completionModeComparison
-}
-
-func isUnqualifiedBooleanToken(tokens []sqlToken, depth int) bool {
- lastIndex := -1
- for index := len(tokens) - 1; index >= 0; index-- {
- if tokens[index].depth <= depth {
- lastIndex = index
- break
- }
- }
- if lastIndex < 0 {
- return false
- }
- last := tokens[lastIndex]
- if last.kind != sqlTokenWord || !strings.EqualFold(last.raw, "TRUE") && !strings.EqualFold(last.raw, "FALSE") {
- return false
- }
- for index := lastIndex - 1; index >= 0; index-- {
- if tokens[index].depth < depth {
- break
- }
- if tokens[index].depth == depth {
- return tokens[index].text != "."
- }
- }
- return true
-}
-
-func hasLaterJoinAtDepth(input string, depth int) bool {
- tokens, _, _ := scanSQLTokensAtDepth(input, depth)
- for _, token := range tokens {
- if token.depth < depth || token.depth == depth && token.text == ";" {
- return false
- }
- if token.depth != depth || token.kind != sqlTokenWord {
- continue
- }
- switch token.text {
- case "JOIN":
- return true
- case "WHERE", "PREWHERE", "GROUP", "ORDER", "HAVING", "QUALIFY", "LIMIT", "UNION", "EXCEPT", "INTERSECT":
- return false
- }
- }
- return false
-}
-
-func betweenLowerBoundComplete(tokens []sqlToken, depth int) bool {
- hasExpressionToken := false
- openCases := 0
- lastExpressionIndex := -1
- pendingIntervals := make([]int, depth+1)
- for index := range pendingIntervals {
- pendingIntervals[index] = -1
- }
- for index, token := range tokens {
- if token.depth < depth {
- break
- }
- for len(pendingIntervals) > token.depth+1 {
- if pendingIntervals[len(pendingIntervals)-1] >= 0 {
- return false
- }
- pendingIntervals = pendingIntervals[:len(pendingIntervals)-1]
- }
- for len(pendingIntervals) <= token.depth {
- pendingIntervals = append(pendingIntervals, -1)
- }
- if token.kind == sqlTokenWord {
- switch token.text {
- case "CASE":
- openCases++
- case "END":
- if openCases > 0 {
- openCases--
- }
- case "INTERVAL":
- pendingIntervals[token.depth] = index
- default:
- if isIntervalUnit(token.text) && lastExpressionIndex > pendingIntervals[token.depth] {
- pendingIntervals[token.depth] = -1
- }
- }
- }
- if token.text != "NOT" {
- hasExpressionToken = true
- }
- if token.kind == sqlTokenWord || token.kind == sqlTokenValue {
- lastExpressionIndex = index
- }
- }
- for _, intervalIndex := range pendingIntervals {
- if intervalIndex >= 0 {
- return false
- }
- }
- return hasExpressionToken && openCases == 0
-}
-
-func isIntervalUnit(value string) bool {
- switch value {
- case "MILLISECOND", "SECOND", "MINUTE", "HOUR", "DAY", "WEEK", "MONTH", "QUARTER", "YEAR":
- return true
- }
- return false
-}
-
-func lastTokenAtDepth(tokens []sqlToken, depth int) sqlToken {
- for index := len(tokens) - 1; index >= 0; index-- {
- if tokens[index].depth <= depth {
- return tokens[index]
- }
- }
- return sqlToken{}
-}
-
-func isComparisonToken(token sqlToken) bool {
- if token.kind == sqlTokenOperator {
- switch token.text {
- case "=", "==", "!=", "<>", "<", "<=", ">", ">=", "<=>":
- return true
- }
- }
- if token.kind != sqlTokenWord {
- return false
- }
- switch token.text {
- case "LIKE", "ILIKE", "IN", "IS", "BETWEEN", "REGEXP":
- return true
- }
- return false
-}
-
-func scanSQLTokens(input string) ([]sqlToken, int, bool) {
- return scanSQLTokensAtDepth(input, 0)
-}
-
-func scanSQLTokensAtDepth(input string, initialDepth int) ([]sqlToken, int, bool) {
- var tokens []sqlToken
- depth := initialDepth
- for index := 0; index < len(input); {
- character := input[index]
- r, size := utf8.DecodeRuneInString(input[index:])
- if unicode.IsSpace(r) {
- index += size
- continue
- }
- if character == '-' && index+1 < len(input) && input[index+1] == '-' {
- end := strings.IndexByte(input[index+2:], '\n')
- if end < 0 {
- return tokens, depth, true
- }
- index += end + 3
- continue
- }
- if character == '/' && index+1 < len(input) && input[index+1] == '*' {
- end := strings.Index(input[index+2:], "*/")
- if end < 0 {
- return tokens, depth, true
- }
- index += end + 4
- continue
- }
- if character == '\'' || character == '"' || character == '`' {
- end := quotedTokenEnd(input, index, character)
- if end < 0 {
- return tokens, depth, true
- }
- kind := sqlTokenValue
- if character != '\'' {
- kind = sqlTokenWord
- }
- tokens = append(tokens, sqlToken{text: strings.ToUpper(input[index:end]), raw: input[index:end], kind: kind, depth: depth, start: index})
- index = end
- continue
- }
- if isSQLIdentifierStart(input[index:]) {
- end := index
- for end < len(input) {
- r, size := utf8.DecodeRuneInString(input[end:])
- if !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '_' && r != '$' {
- break
- }
- end += size
- }
- tokens = append(tokens, sqlToken{text: strings.ToUpper(input[index:end]), raw: input[index:end], kind: sqlTokenWord, depth: depth, start: index})
- index = end
- continue
- }
- if character >= '0' && character <= '9' {
- end := index + 1
- for end < len(input) && ((input[end] >= '0' && input[end] <= '9') || input[end] == '.') {
- end++
- }
- tokens = append(tokens, sqlToken{text: input[index:end], raw: input[index:end], kind: sqlTokenValue, depth: depth, start: index})
- index = end
- continue
- }
- switch character {
- case '(':
- tokens = append(tokens, sqlToken{text: "(", raw: "(", kind: sqlTokenLeftParen, depth: depth, start: index})
- depth++
- index++
- case ')':
- if depth > 0 {
- depth--
- }
- tokens = append(tokens, sqlToken{text: ")", raw: ")", kind: sqlTokenRightParen, depth: depth, start: index})
- index++
- case ',':
- tokens = append(tokens, sqlToken{text: ",", raw: ",", kind: sqlTokenComma, depth: depth, start: index})
- index++
- default:
- end := operatorEnd(input, index)
- tokens = append(tokens, sqlToken{text: strings.ToUpper(input[index:end]), raw: input[index:end], kind: sqlTokenOperator, depth: depth, start: index})
- index = end
- }
- }
- return tokens, depth, false
-}
-
-func quotedTokenEnd(input string, start int, quote byte) int {
- for index := start + 1; index < len(input); index++ {
- if input[index] == '\\' {
- index++
- continue
- }
- if input[index] != quote {
- continue
- }
- if index+1 < len(input) && input[index+1] == quote {
- index++
- continue
- }
- return index + 1
- }
- return -1
-}
-
-func isSQLIdentifierStart(input string) bool {
- r, _ := utf8.DecodeRuneInString(input)
- return unicode.IsLetter(r) || r == '_' || r == '$'
-}
-
-func operatorEnd(input string, start int) int {
- for _, width := range []int{3, 2} {
- if start+width > len(input) {
- continue
- }
- switch input[start : start+width] {
- case "<=>", "!=", "<=", ">=", "==", "<>", "||", "->", "::":
- return start + width
- }
- }
- return start + 1
-}
diff --git a/services/hogql-language-service/internal/completion/fields.go b/services/hogql-language-service/internal/completion/fields.go
deleted file mode 100644
index ad92fcc99845..000000000000
--- a/services/hogql-language-service/internal/completion/fields.go
+++ /dev/null
@@ -1,62 +0,0 @@
-package completion
-
-import (
- "strings"
- "unicode"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/analysis"
-)
-
-func fieldSuggestions(bindings analysis.Bindings, prefix string) []Suggestion {
- var suggestions []Suggestion
- // HogQL alias precedence is case-sensitive (resolver_utils.lookup_field_by_name).
- aliases := map[string]bool{}
- for alias := range bindings.SelectAliases(prefix) {
- aliases[alias.Name] = true
- if supportedHogQLIdentifier(alias.Name) {
- suggestions = append(suggestions, Suggestion{Label: alias.Name, Kind: "field", Detail: alias.Type, InsertText: suggestionInsertText("field", alias.Name)})
- }
- }
- type candidate struct {
- field Suggestion
- qualifier string
- key string
- }
- var candidates []candidate
- counts := map[string]int{}
- qualifiers := map[analysis.Source]string{}
- for source, field := range bindings.Fields(prefix) {
- if aliases[field.Name] || !supportedHogQLIdentifier(field.Name) {
- continue
- }
- qualifier, ok := qualifiers[source]
- if !ok {
- qualifier = quoteHogQLFieldIdentifier(source.Qualifier())
- qualifiers[source] = qualifier
- }
- key := strings.Map(func(r rune) rune {
- first := r
- for next := unicode.SimpleFold(r); next != r; next = unicode.SimpleFold(next) {
- first = min(first, next)
- }
- return first
- }, field.Name)
- counts[key]++
- candidates = append(candidates, candidate{
- field: Suggestion{Label: field.Name, Kind: "field", Detail: field.Type, InsertText: suggestionInsertText("field", field.Name)},
- qualifier: qualifier, key: key,
- })
- }
- for _, candidate := range candidates {
- field := candidate.field
- if counts[candidate.key] > 1 {
- if !supportedHogQLIdentifier(candidate.qualifier) {
- continue
- }
- field.InsertText = candidate.qualifier + "." + quoteHogQLFieldIdentifier(field.Label)
- field.Detail = strings.TrimSpace(field.Detail + " from " + candidate.qualifier)
- }
- suggestions = append(suggestions, field)
- }
- return suggestions
-}
diff --git a/services/hogql-language-service/internal/completion/functions.go b/services/hogql-language-service/internal/completion/functions.go
deleted file mode 100644
index cfb8490844d9..000000000000
--- a/services/hogql-language-service/internal/completion/functions.go
+++ /dev/null
@@ -1,67 +0,0 @@
-// Code generated by posthog/hogql/functions/generate_language_service_functions.py; DO NOT EDIT.
-
-package completion
-
-import "strings"
-
-var hogQLFunctions = strings.Fields(`
-IPv4CIDRToRange IPv4NumToString IPv4StringToNum IPv4StringToNumOrDefault IPv4StringToNumOrNull IPv4ToIPv6 IPv6CIDRToRange IPv6NumToString IPv6StringToNum IPv6StringToNumOrDefault IPv6StringToNumOrNull JSONArrayLength JSONExtract JSONExtractArrayRaw JSONExtractBool JSONExtractFloat JSONExtractInt JSONExtractKeys JSONExtractKeysAndValues JSONExtractKeysAndValuesRaw
-JSONExtractRaw JSONExtractString JSONExtractUInt JSONHas JSONLength JSONType JSON_VALUE L1Distance L1Norm L1Normalize L2Distance L2Norm L2Normalize LinfDistance LinfNorm LinfNormalize LpDistance LpNorm LpNormalize MD5
-URLHierarchy URLPathHierarchy UUIDv7ToDateTime abs accurateCast accurateCastOrNull acos acosh addDays addHours addMinutes addMonths addQuarters addSeconds addWeeks addYears age aggregate_funnel aggregate_funnel_array aggregate_funnel_array_json
-aggregate_funnel_array_trends aggregate_funnel_array_trends_json aggregate_funnel_array_trends_test aggregate_funnel_cohort aggregate_funnel_cohort_json aggregate_funnel_cohort_trends aggregate_funnel_cohort_trends_json aggregate_funnel_json aggregate_funnel_test aggregate_funnel_trends aggregate_funnel_trends_json alphaTokens and any anyHeavy anyHeavyIf anyIf anyLast anyLastIf appendTrailingCharIfAbsent
-argMax argMaxIf argMaxMerge argMaxState argMin argMinIf argMinMerge argMinState array arrayAUC arrayAll arrayAvg arrayCompact arrayConcat arrayCount arrayCumSum arrayCumSumNonNegative arrayDifference arrayDistinct arrayElement
-arrayEnumerate arrayEnumerateDense arrayEnumerateUniq arrayExists arrayFill arrayFilter arrayFirst arrayFirstIndex arrayFlatten arrayFold arrayIntersect arrayJoin arrayLast arrayLastIndex arrayMap arrayMax arrayMin arrayPopBack arrayPopFront arrayProduct
-arrayPushBack arrayPushFront arrayReduce arrayResize arrayReverse arrayReverseFill arrayReverseSort arrayReverseSplit arrayRotateLeft arrayRotateRight arraySlice arraySort arraySplit arrayStringConcat arraySum arrayUniq arrayWithConstant arrayZip array_agg ascii
-asin asinh assumeNotNull atan atan2 atanh avg avgArgMax avgArgMaxIf avgArgMaxOrDefault avgArgMaxOrDefaultIf avgArgMaxOrNull avgArgMaxOrNullIf avgArgMin avgArgMinIf avgArgMinOrDefault avgArgMinOrDefaultIf avgArgMinOrNull avgArgMinOrNullIf avgArray
-avgArrayIf avgArrayOrDefault avgArrayOrDefaultIf avgArrayOrNull avgArrayOrNullIf avgForEach avgForEachIf avgForEachOrDefault avgForEachOrDefaultIf avgForEachOrNull avgForEachOrNullIf avgIf avgMap avgMapIf avgMapMerge avgMapMergeIf avgMapOrDefault avgMapOrDefaultIf avgMapOrNull avgMapOrNullIf
-avgMapState avgMerge avgMergeIf avgMergeOrDefault avgMergeOrDefaultIf avgMergeOrNull avgMergeOrNullIf avgOrDefault avgOrDefaultIf avgOrNull avgOrNullIf avgState avgStateIf avgStateOrDefault avgStateOrDefaultIf avgStateOrNull avgStateOrNullIf avgWeighted avgWeightedIf bar
-base58Decode base58Encode base64Decode base64Encode bitAnd bitCount bitHammingDistance bitNot bitOr bitRotateLeft bitRotateRight bitShiftLeft bitShiftRight bitSlice bitTest bitTestAll bitTestAny bitXor bitmapAnd bitmapAndCardinality
-bitmapAndnot bitmapAndnotCardinality bitmapBuild bitmapCardinality bitmapContains bitmapHasAll bitmapHasAny bitmapMax bitmapMin bitmapOr bitmapOrCardinality bitmapSubsetInRange bitmapSubsetLimit bitmapToArray bitmapTransform bitmapXor bitmapXorCardinality btrim cbrt ceil
-cityHash64 coalesce concat concatWithSeparator contingency contingencyIf convertCharset convertCurrency corr cos cosh cosineDistance count countArgMax countArgMaxIf countArgMaxOrDefault countArgMaxOrDefaultIf countArgMaxOrNull countArgMaxOrNullIf countArgMin
-countArgMinIf countArgMinOrDefault countArgMinOrDefaultIf countArgMinOrNull countArgMinOrNullIf countArray countArrayIf countArrayOrDefault countArrayOrDefaultIf countArrayOrNull countArrayOrNullIf countDistinct countDistinctArgMax countDistinctArgMaxIf countDistinctArgMaxOrDefault countDistinctArgMaxOrDefaultIf countDistinctArgMaxOrNull countDistinctArgMaxOrNullIf countDistinctArgMin countDistinctArgMinIf
-countDistinctArgMinOrDefault countDistinctArgMinOrDefaultIf countDistinctArgMinOrNull countDistinctArgMinOrNullIf countDistinctArray countDistinctArrayIf countDistinctArrayOrDefault countDistinctArrayOrDefaultIf countDistinctArrayOrNull countDistinctArrayOrNullIf countDistinctForEach countDistinctForEachIf countDistinctForEachOrDefault countDistinctForEachOrDefaultIf countDistinctForEachOrNull countDistinctForEachOrNullIf countDistinctIf countDistinctMap countDistinctMapIf countDistinctMapOrDefault
-countDistinctMapOrDefaultIf countDistinctMapOrNull countDistinctMapOrNullIf countDistinctMerge countDistinctMergeIf countDistinctMergeOrDefault countDistinctMergeOrDefaultIf countDistinctMergeOrNull countDistinctMergeOrNullIf countDistinctOrDefault countDistinctOrDefaultIf countDistinctOrNull countDistinctOrNullIf countDistinctState countDistinctStateIf countDistinctStateOrDefault countDistinctStateOrDefaultIf countDistinctStateOrNull countDistinctStateOrNullIf countEqual
-countForEach countForEachIf countForEachOrDefault countForEachOrDefaultIf countForEachOrNull countForEachOrNullIf countIf countMap countMapIf countMapOrDefault countMapOrDefaultIf countMapOrNull countMapOrNullIf countMatches countMatchesCaseInsensitive countMerge countMergeIf countMergeOrDefault countMergeOrDefaultIf countMergeOrNull
-countMergeOrNullIf countOrDefault countOrDefaultIf countOrNull countOrNullIf countState countStateIf countStateOrDefault countStateOrDefaultIf countStateOrNull countStateOrNullIf countSubstrings countSubstringsCaseInsensitive countSubstringsCaseInsensitiveUTF8 covarPop covarPopIf covarSamp covarSampIf cramersV cramersVBiasCorrected
-cramersVBiasCorrectedIf cramersVIf current_date current_timestamp cutFragment cutIPv6 cutQueryString cutQueryStringAndFragment cutToFirstSignificantSubdomain cutToFirstSignificantSubdomainWithWWW cutURLParameter cutWWW dateAdd dateDiff dateName dateSub dateTrunc date_add date_bin date_diff
-date_part date_subtract date_trunc decide decodeURLComponent decodeURLFormComponent decodeXMLComponent defaultValueOfTypeName degrees deltaSum deltaSumIf deltaSumTimestamp deltaSumTimestampIf dense_rank divide divideDecimal domain domainWithoutWWW dotProduct dynamicType
-e embedText empty encodeURLComponent encodeURLFormComponent encodeXMLComponent endsWith equals erf erfc every exp exp10 exp2 explainCSPReport extract extractAll extractAllGroups extractAllGroupsHorizontal extractAllGroupsVertical
-extractGroups extractIPv4Substrings extractTextFromHTML extractURLParameter extractURLParameterNames extractURLParameters factorial firstSignificantSubdomain first_value floor format formatDateTime formatReadableDecimalSize formatReadableQuantity formatReadableSize formatReadableTimeDelta fragment fromModifiedJulianDay fromUnixTimestamp fromUnixTimestamp64Milli
-gcd generateSeries geoDistance geoToH3 geohashDecode geohashEncode geohashesInBox getBotName getBotOperator getBotType getSurveyResponse getTrafficCategory getTrafficType greatCircleAngle greatCircleDistance greater greaterOrEquals greatest groupArray groupArrayIf
-groupArrayInsertAt groupArrayInsertAtIf groupArrayMovingAvg groupArrayMovingAvgIf groupArrayMovingSum groupArrayMovingSumIf groupArraySample groupArraySampleIf groupBitAnd groupBitAndIf groupBitOr groupBitOrIf groupBitXor groupBitXorIf groupBitmap groupBitmapAnd groupBitmapAndIf groupBitmapAndState groupBitmapIf groupBitmapOr
-groupBitmapOrIf groupBitmapOrState groupBitmapState groupBitmapXor groupBitmapXorIf groupUniqArray groupUniqArrayArray groupUniqArrayArrayIf groupUniqArrayIf grouping h3CellAreaM2 h3CellAreaRads2 h3Distance h3EdgeAngle h3EdgeLengthKm h3EdgeLengthM h3ExactEdgeLengthKm h3ExactEdgeLengthM h3ExactEdgeLengthRads h3GetBaseCell
-h3GetDestinationIndexFromUnidirectionalEdge h3GetFaces h3GetIndexesFromUnidirectionalEdge h3GetOriginIndexFromUnidirectionalEdge h3GetPentagonIndexes h3GetRes0Indexes h3GetResolution h3GetUnidirectionalEdge h3GetUnidirectionalEdgeBoundary h3GetUnidirectionalEdgesFromHexagon h3HexAreaKm2 h3HexAreaM2 h3HexRing h3IndexesAreNeighbors h3IsPentagon h3IsResClassIII h3IsValid h3Line h3NumHexagons h3PointDistKm
-h3PointDistM h3PointDistRads h3ToCenterChild h3ToChildren h3ToGeo h3ToGeoBoundary h3ToParent h3ToString h3UnidirectionalEdgeIsValid h3kRing has hasAll hasAllTokens hasAny hasAnyTokens hasSubsequence hasSubsequenceCaseInsensitive hasSubsequenceCaseInsensitiveUTF8 hasSubsequenceUTF8 hasSubstr
-hasToken hasTokenCaseInsensitive hasTokenCaseInsensitiveOrNull hasTokenOrNull hex hop hopEnd hopStart hypot if ifNotFinite ifnull ilike in indexHint indexOf initcap intDiv intDivOrZero intExp10
-intExp2 isFinite isIPAddressInRange isIPv4String isIPv6String isInfinite isLikelyBot isNaN isNotNull isValidJSON isValidUTF8 isnull jev json_agg kurtPop kurtPopIf kurtSamp kurtSampIf lag lagInFrame
-languageCodeToName last_value lcm lead leadInFrame least left leftPad leftPadUTF8 length lengthUTF8 less lessOrEquals lgamma like ln locate log log10 log1p
-log2 lookupDomainType lookupOrganicMediumType lookupOrganicSourceType lookupPaidMediumType lookupPaidSourceType lower lowerUTF8 lpad ltrim make_date make_interval make_timestamp make_timestamptz map mapAdd mapApply mapContains mapContainsKeyLike mapExists
-mapExtractKeyLike mapFilter mapFromArrays mapKeys mapPopulateSeries mapSubtract mapUpdate mapValues match matchesAction max max2 maxArgMax maxArgMaxIf maxArgMaxOrDefault maxArgMaxOrDefaultIf maxArgMaxOrNull maxArgMaxOrNullIf maxArgMin maxArgMinIf
-maxArgMinOrDefault maxArgMinOrDefaultIf maxArgMinOrNull maxArgMinOrNullIf maxArray maxArrayIf maxArrayOrDefault maxArrayOrDefaultIf maxArrayOrNull maxArrayOrNullIf maxForEach maxForEachIf maxForEachOrDefault maxForEachOrDefaultIf maxForEachOrNull maxForEachOrNullIf maxIf maxIntersections maxIntersectionsIf maxIntersectionsPosition
-maxIntersectionsPositionIf maxMap maxMapIf maxMapOrDefault maxMapOrDefaultIf maxMapOrNull maxMapOrNullIf maxMerge maxMergeIf maxMergeOrDefault maxMergeOrDefaultIf maxMergeOrNull maxMergeOrNullIf maxOrDefault maxOrDefaultIf maxOrNull maxOrNullIf maxState maxStateIf maxStateOrDefault
-maxStateOrDefaultIf maxStateOrNull maxStateOrNullIf md5 median medianArgMax medianArgMaxIf medianArgMaxOrDefault medianArgMaxOrDefaultIf medianArgMaxOrNull medianArgMaxOrNullIf medianArgMin medianArgMinIf medianArgMinOrDefault medianArgMinOrDefaultIf medianArgMinOrNull medianArgMinOrNullIf medianArray medianArrayIf medianArrayOrDefault
-medianArrayOrDefaultIf medianArrayOrNull medianArrayOrNullIf medianBFloat16 medianBFloat16If medianDeterministic medianDeterministicIf medianExact medianExactHigh medianExactHighIf medianExactIf medianExactLow medianExactLowIf medianExactWeighted medianExactWeightedIf medianForEach medianForEachIf medianForEachOrDefault medianForEachOrDefaultIf medianForEachOrNull
-medianForEachOrNullIf medianIf medianMap medianMapIf medianMapOrDefault medianMapOrDefaultIf medianMapOrNull medianMapOrNullIf medianMerge medianMergeIf medianMergeOrDefault medianMergeOrDefaultIf medianMergeOrNull medianMergeOrNullIf medianOrDefault medianOrDefaultIf medianOrNull medianOrNullIf medianState medianStateIf
-medianStateOrDefault medianStateOrDefaultIf medianStateOrNull medianStateOrNullIf medianTDigest medianTDigestIf medianTDigestWeighted medianTDigestWeightedIf medianTiming medianTimingIf medianTimingWeighted medianTimingWeightedIf min min2 minArgMax minArgMaxIf minArgMaxOrDefault minArgMaxOrDefaultIf minArgMaxOrNull minArgMaxOrNullIf
-minArgMin minArgMinIf minArgMinOrDefault minArgMinOrDefaultIf minArgMinOrNull minArgMinOrNullIf minArray minArrayIf minArrayOrDefault minArrayOrDefaultIf minArrayOrNull minArrayOrNullIf minForEach minForEachIf minForEachOrDefault minForEachOrDefaultIf minForEachOrNull minForEachOrNullIf minIf minMap
-minMapIf minMapOrDefault minMapOrDefaultIf minMapOrNull minMapOrNullIf minMerge minMergeIf minMergeOrDefault minMergeOrDefaultIf minMergeOrNull minMergeOrNullIf minOrDefault minOrDefaultIf minOrNull minOrNullIf minState minStateIf minStateOrDefault minStateOrDefaultIf minStateOrNull
-minStateOrNullIf minus modulo moduloOrZero monthName multiFuzzyMatchAllIndices multiFuzzyMatchAny multiFuzzyMatchAnyIndex multiIf multiMatchAllIndices multiMatchAny multiMatchAnyIndex multiSearchAllPositions multiSearchAllPositionsCaseInsensitive multiSearchAllPositionsCaseInsensitiveUTF8 multiSearchAllPositionsUTF8 multiSearchAny multiSearchAnyCaseInsensitive multiSearchAnyCaseInsensitiveUTF8 multiSearchAnyUTF8
-multiSearchFirstIndex multiSearchFirstIndexCaseInsensitive multiSearchFirstIndexCaseInsensitiveUTF8 multiSearchFirstIndexUTF8 multiSearchFirstPosition multiSearchFirstPositionCaseInsensitive multiSearchFirstPositionCaseInsensitiveUTF8 multiSearchFirstPositionUTF8 multiply multiplyDecimal negate netloc ngramDistance ngramDistanceCaseInsensitive ngramDistanceCaseInsensitiveUTF8 ngramDistanceUTF8 ngramSearch ngramSearchCaseInsensitive ngramSearchCaseInsensitiveUTF8 ngramSearchUTF8
-ngrams not notEmpty notEquals notILike notIn notLike now nowInBlock nth_value nullif or parseDateTime parseDateTimeBestEffort path pathFull percentile_cont percentile_disc pi plus
-pointInEllipses pointInPolygon port position positionCaseInsensitive positionCaseInsensitiveUTF8 positionUTF8 positiveModulo pow power protocol quantile quantileExact quantileIf quantiles quantilesIf quantilesMerge quantilesMergeIf quantilesState quantilesStateIf
-queryString queryStringAndFragment radians rand range rank recordingButton regexpExtract regexpQuoteMeta reinterpretAsFloat32 reinterpretAsFloat64 reinterpretAsInt128 reinterpretAsInt16 reinterpretAsInt256 reinterpretAsInt32 reinterpretAsInt64 reinterpretAsInt8 reinterpretAsUInt128 reinterpretAsUInt16 reinterpretAsUInt256
-reinterpretAsUInt32 reinterpretAsUInt64 reinterpretAsUInt8 reinterpretAsUUID repeat replace replaceAll replaceOne replaceRegexpAll replaceRegexpOne reverse reverseUTF8 right rightPad rightPadUTF8 round roundAge roundBankers roundDown roundDuration
-roundToExp2 rowNumberInAllBlocks rowNumberInBlock row_number rpad rtrim sha1 sign simpleLinearRegression simpleLinearRegressionIf sin sinh skewPop skewPopIf skewSamp skewSampIf sortablesemver sparkline splitByChar splitByNonAlpha
-splitByRegexp splitByString splitByWhitespace split_part sqrt startsWith stddevPop stddevPopIf stddevSamp stddevSampIf stringToH3 string_agg subBitmap substring substringUTF8 subtractDays subtractHours subtractMinutes subtractMonths subtractQuarters
-subtractSeconds subtractWeeks subtractYears sum sumArgMax sumArgMaxIf sumArgMaxOrDefault sumArgMaxOrDefaultIf sumArgMaxOrNull sumArgMaxOrNullIf sumArgMin sumArgMinIf sumArgMinOrDefault sumArgMinOrDefaultIf sumArgMinOrNull sumArgMinOrNullIf sumArray sumArrayIf sumArrayOrDefault sumArrayOrDefaultIf
-sumArrayOrNull sumArrayOrNullIf sumForEach sumForEachIf sumForEachMerge sumForEachOrDefault sumForEachOrDefaultIf sumForEachOrNull sumForEachOrNullIf sumForEachState sumIf sumMap sumMapIf sumMapMerge sumMapMergeIf sumMapOrDefault sumMapOrDefaultIf sumMapOrNull sumMapOrNullIf sumMerge
-sumMergeIf sumMergeOrDefault sumMergeOrDefaultIf sumMergeOrNull sumMergeOrNullIf sumOrDefault sumOrDefaultIf sumOrNull sumOrNullIf sumState sumStateIf sumStateOrDefault sumStateOrDefaultIf sumStateOrNull sumStateOrNullIf sumWithOverflow sumWithOverflowIf tan tgamma theilsU
-theilsUIf throwIf timeSlot timeSlots timeStampAdd timeStampSub timeZoneOf timeZoneOffset timezone toBool toDate toDateTime toDateTime64 toDateTimeUS toDayOfMonth toDayOfWeek toDayOfYear toDecimal toFloat toFloat64OrNull
-toFloatOrDefault toFloatOrNull toFloatOrZero toHour toIPv4 toIPv4OrDefault toIPv4OrNull toIPv4OrZero toIPv6 toIPv6OrDefault toIPv6OrNull toIPv6OrZero toISOWeek toISOYear toInt toIntOrDefault toIntOrZero toIntervalDay toIntervalHour toIntervalMinute
-toIntervalMonth toIntervalQuarter toIntervalSecond toIntervalWeek toIntervalYear toJSONString toLastDayOfMonth toLastDayOfWeek toMinute toModifiedJulianDay toMonday toMonth toNullable toNullableString toQuarter toSecond toStartOfDay toStartOfFifteenMinutes toStartOfFiveMinutes toStartOfHour
-toStartOfISOYear toStartOfInterval toStartOfMinute toStartOfMonth toStartOfQuarter toStartOfSecond toStartOfTenMinutes toStartOfWeek toStartOfYear toString toTime toTimeZone toTypeName toUUID toUUIDOrDefault toUnixTimestamp toUnixTimestamp64Milli toValidUTF8 toWeek toYYYYMM
-toYYYYMMDD toYYYYMMDDhhmmss toYear toYearWeek to_char to_date to_timestamp today tokens topK topLevelDomain transform translate translateUTF8 trim trimLeft trimRight trunc tryBase58Decode tryBase64Decode
-tumble tumbleEnd tumbleStart tuple tupleDivide tupleDivideByNumber tupleElement tupleHammingDistance tupleMinus tupleMultiply tupleMultiplyByNumber tupleNegate tuplePlus tupleToNameValuePairs unhex uniq uniqCombined uniqCombined64 uniqExact uniqExactIf
-uniqExactMerge uniqExactState uniqHLL12 uniqHLL12If uniqIf uniqMap uniqMapMerge uniqMapMergeIf uniqMerge uniqMergeIf uniqState uniqStateIf uniqTheta uniqThetaIf uniqUpToMerge uniqueSurveySubmissionsFilter untuple upper upperUTF8 varPop
-varPopIf varSamp varSampIf width_bucket windowFunnel withIcon xor yesterday
-`)
diff --git a/services/hogql-language-service/internal/completion/tables.go b/services/hogql-language-service/internal/completion/tables.go
deleted file mode 100644
index c20b39ca6130..000000000000
--- a/services/hogql-language-service/internal/completion/tables.go
+++ /dev/null
@@ -1,107 +0,0 @@
-package completion
-
-import (
- "iter"
- "slices"
- "strings"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/analysis"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
-)
-
-func tableResult(schema *catalog.PreparedCatalog, bindings analysis.Bindings, prefix, namespace string, offset int, parseErr error) Result {
- ctes := slices.Collect(bindings.CTENames(prefix))
- if len(ctes) == 0 {
- result := indexedResult(tableEntries(schema, prefix, namespace, nil), "table", offset, parseErr)
- setTablePathInsertText(&result, namespace)
- return result
- }
- slices.SortFunc(ctes, func(left, right catalog.Entry) int {
- if comparison := strings.Compare(strings.ToLower(left.Name), strings.ToLower(right.Name)); comparison != 0 {
- return comparison
- }
- return strings.Compare(left.Name, right.Name)
- })
- cteNames := map[string]bool{}
- shadowed := map[string]bool{}
- for _, cte := range ctes {
- cteNames[cte.Name] = true
- shadowed[cte.Name] = true
- }
- entries := func(yield func(catalog.Entry) bool) {
- if namespace == "" {
- for _, cte := range ctes {
- if !yield(cte) {
- return
- }
- }
- }
- for table := range tableEntries(schema, prefix, namespace, shadowed) {
- if !yield(table) {
- return
- }
- }
- }
- result := indexedResult(entries, "table", offset, parseErr)
- setTablePathInsertText(&result, namespace)
- if namespace != "" {
- return result
- }
- for index := range result.Suggestions {
- suggestion := &result.Suggestions[index]
- if cteNames[suggestion.Label] {
- // A dotted CTE name is one identifier, unlike a qualified catalog table name.
- suggestion.InsertText = suggestionInsertText("field", suggestion.Label)
- suggestion.SortText = "0-" + strings.ToLower(suggestion.Label)
- }
- }
- return result
-}
-
-func tableEntries(schema *catalog.PreparedCatalog, prefix, namespace string, excluded map[string]bool) iter.Seq[catalog.Entry] {
- var entries []catalog.Entry
- if namespace != "" {
- entries = schema.TableSuggestionsMatching(prefix, excluded, func(entry catalog.Entry) bool {
- return strings.HasPrefix(entry.Name, namespace) && supportedUnquotedTablePath(strings.TrimPrefix(entry.Name, namespace))
- })
- } else {
- entries = schema.TableSuggestions(prefix, excluded)
- }
- return func(yield func(catalog.Entry) bool) {
- for _, entry := range entries {
- if !yield(entry) {
- return
- }
- }
- }
-}
-
-func supportedUnquotedTablePath(path string) bool {
- for component := range strings.SplitSeq(path, ".") {
- if !simpleHogQLIdentifier.MatchString(component) || isQuotedHogQLKeyword(component) {
- return false
- }
- }
- return true
-}
-
-func isQuotedHogQLKeyword(name string) bool {
- folded := name
- for index := 0; index < len(name); index++ {
- if name[index] >= 'A' && name[index] <= 'Z' {
- folded = strings.ToLower(name)
- break
- }
- }
- _, exists := quotedHogQLKeywordsFolded[folded]
- return exists
-}
-
-func setTablePathInsertText(result *Result, namespace string) {
- if namespace == "" {
- return
- }
- for index := range result.Suggestions {
- result.Suggestions[index].InsertText = quoteHogQLTablePath(strings.TrimPrefix(result.Suggestions[index].Label, namespace))
- }
-}
diff --git a/services/hogql-language-service/internal/httpapi/handler.go b/services/hogql-language-service/internal/httpapi/handler.go
deleted file mode 100644
index d307ec987cc7..000000000000
--- a/services/hogql-language-service/internal/httpapi/handler.go
+++ /dev/null
@@ -1,399 +0,0 @@
-package httpapi
-
-import (
- "context"
- "encoding/json"
- "errors"
- "log/slog"
- "net"
- "net/http"
- "strconv"
- "time"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/completion"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/ratelimit"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/serviceauth"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/textposition"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/validation"
-)
-
-type server struct {
- catalogs *catalog.Registry
- auth *serviceauth.Authenticator
- preAuthLimiter *ratelimit.Limiter
- principalLimiter *ratelimit.Limiter
- logger *slog.Logger
-}
-
-type requestLogDetails struct {
- operation string
- authorization *serviceauth.Authorization
- result string
- catalogTables int
- catalogProperties int
-}
-
-type requestLogDetailsKey struct{}
-
-type loggingResponseWriter struct {
- http.ResponseWriter
- statusCode int
- responseBytes int
-}
-
-type completionRequest struct {
- Query string `json:"query"`
- Position *int `json:"position,omitempty"`
- PositionEncoding completion.PositionEncoding `json:"positionEncoding,omitempty"`
- Cursor string `json:"cursor,omitempty"`
-}
-
-type completionResponse struct {
- completion.Result
- CatalogRevision string `json:"catalogRevision"`
- DurationMicros int64 `json:"durationMicros"`
- PositionEncoding completion.PositionEncoding `json:"positionEncoding"`
-}
-
-type validationRequest struct {
- Query string `json:"query"`
- PositionEncoding textposition.Encoding `json:"positionEncoding,omitempty"`
-}
-
-type validationResponse struct {
- validation.Result
- CatalogRevision string `json:"catalogRevision"`
- PositionEncoding textposition.Encoding `json:"positionEncoding"`
-}
-
-type catalogUpdate struct {
- Revision string `json:"revision"`
- Catalog catalog.Catalog `json:"catalog"`
-}
-
-type Config struct {
- Catalogs *catalog.Registry
- Auth *serviceauth.Authenticator
- PreAuthLimiter *ratelimit.Limiter
- PrincipalLimiter *ratelimit.Limiter
- Logger *slog.Logger
-}
-
-func NewHandler(config Config) http.Handler {
- s := &server{
- catalogs: config.Catalogs,
- auth: config.Auth,
- preAuthLimiter: config.PreAuthLimiter,
- principalLimiter: config.PrincipalLimiter,
- logger: config.Logger,
- }
- return s.handler()
-}
-
-func (s *server) handler() http.Handler {
- mux := http.NewServeMux()
- mux.Handle("GET /health", requestOperation("health", http.HandlerFunc(s.health)))
- mux.Handle("PUT /teams/{teamID}/users/{userID}/catalog", requestOperation("publish", s.authorized(serviceauth.OperationPublish, s.putCatalog)))
- mux.Handle("DELETE /teams/{teamID}/users/{userID}/catalog", requestOperation("delete", s.authorized(serviceauth.OperationDelete, s.deleteCatalog)))
- mux.Handle("POST /teams/{teamID}/users/{userID}/autocomplete", requestOperation("complete", s.authorized(serviceauth.OperationComplete, s.autocomplete)))
- mux.Handle("POST /teams/{teamID}/users/{userID}/validate", requestOperation("validate", s.authorized(serviceauth.OperationValidate, s.validate)))
- return securityHeaders(s.logRequests(mux))
-}
-
-func (s *server) logRequests(next http.Handler) http.Handler {
- return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- started := time.Now()
- details := &requestLogDetails{operation: "unmatched"}
- response := &loggingResponseWriter{ResponseWriter: w}
- next.ServeHTTP(response, r.WithContext(context.WithValue(r.Context(), requestLogDetailsKey{}, details)))
-
- statusCode := response.statusCode
- if statusCode == 0 {
- statusCode = http.StatusOK
- }
- result := details.result
- if result == "" {
- if statusCode < http.StatusBadRequest {
- result = "success"
- } else {
- result = "error"
- }
- }
- attributes := []any{
- "operation", details.operation,
- "method", r.Method,
- "status_code", statusCode,
- "duration_ms", float64(time.Since(started).Microseconds()) / 1000,
- "response_bytes", response.responseBytes,
- "result", result,
- }
- if details.authorization != nil {
- attributes = append(attributes, "team_id", details.authorization.TeamID, "user_id", details.authorization.UserID)
- }
- if details.result == "catalog_published" {
- attributes = append(attributes, "catalog_tables", details.catalogTables, "catalog_properties", details.catalogProperties)
- }
-
- logger := s.logger
- if logger == nil {
- logger = slog.Default()
- }
- switch {
- case details.operation == "health" && statusCode < http.StatusBadRequest:
- logger.Debug("http_request", attributes...)
- case statusCode >= http.StatusInternalServerError:
- logger.Error("http_request", attributes...)
- case statusCode >= http.StatusBadRequest && details.result != "catalog_miss":
- logger.Warn("http_request", attributes...)
- default:
- logger.Info("http_request", attributes...)
- }
- })
-}
-
-func requestOperation(operation string, next http.Handler) http.Handler {
- return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if details := requestDetails(r); details != nil {
- details.operation = operation
- }
- next.ServeHTTP(w, r)
- })
-}
-
-func (w *loggingResponseWriter) WriteHeader(statusCode int) {
- if w.statusCode != 0 {
- return
- }
- w.statusCode = statusCode
- w.ResponseWriter.WriteHeader(statusCode)
-}
-
-func (w *loggingResponseWriter) Write(body []byte) (int, error) {
- if w.statusCode == 0 {
- w.WriteHeader(http.StatusOK)
- }
- written, err := w.ResponseWriter.Write(body)
- w.responseBytes += written
- return written, err
-}
-
-func (w *loggingResponseWriter) Unwrap() http.ResponseWriter {
- return w.ResponseWriter
-}
-
-func requestDetails(r *http.Request) *requestLogDetails {
- details, _ := r.Context().Value(requestLogDetailsKey{}).(*requestLogDetails)
- return details
-}
-
-func setRequestResult(r *http.Request, result string) {
- if details := requestDetails(r); details != nil {
- details.result = result
- }
-}
-
-func securityHeaders(next http.Handler) http.Handler {
- return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("X-Content-Type-Options", "nosniff")
- next.ServeHTTP(w, r)
- })
-}
-
-type authorizedHandler func(http.ResponseWriter, *http.Request, serviceauth.Authorization)
-
-func (s *server) authorized(operation serviceauth.Operation, next authorizedHandler) http.Handler {
- return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- preAuthAllowed, retryAfter := s.preAuthLimiter.Allow(remoteAddress(r))
- authorization, err := authorizationFromPath(r)
- if err != nil {
- if !preAuthAllowed {
- setRequestResult(r, "rate_limited")
- writeRateLimitResponse(w, retryAfter)
- } else {
- setRequestResult(r, "invalid_scope")
- http.Error(w, err.Error(), http.StatusBadRequest)
- }
- return
- }
- if err := s.auth.Verify(r.Header.Get("Authorization"), authorization, operation); err != nil {
- if !preAuthAllowed {
- setRequestResult(r, "rate_limited")
- writeRateLimitResponse(w, retryAfter)
- } else {
- setRequestResult(r, "unauthorized")
- http.Error(w, "unauthorized", http.StatusUnauthorized)
- }
- return
- }
- if details := requestDetails(r); details != nil {
- details.authorization = &authorization
- }
- if allowed, retryAfter := s.principalLimiter.Allow(authorizationKey(authorization)); !allowed {
- setRequestResult(r, "rate_limited")
- writeRateLimitResponse(w, retryAfter)
- return
- }
- next(w, r, authorization)
- })
-}
-
-func (s *server) putCatalog(w http.ResponseWriter, r *http.Request, authorization serviceauth.Authorization) {
- var input catalogUpdate
- if !decodeJSON(w, r, 64<<20, &input) {
- return
- }
- if err := catalog.ValidateRevision(input.Revision); err != nil {
- setRequestResult(r, "catalog_rejected")
- http.Error(w, err.Error(), http.StatusBadRequest)
- return
- }
- if err := catalog.ValidateCatalog(&input.Catalog); err != nil {
- setRequestResult(r, "catalog_rejected")
- http.Error(w, err.Error(), http.StatusBadRequest)
- return
- }
- if err := s.catalogs.Put(authorization, input.Revision, catalog.Prepare(&input.Catalog)); err != nil {
- setRequestResult(r, "catalog_rejected")
- http.Error(w, err.Error(), http.StatusBadRequest)
- return
- }
- if details := requestDetails(r); details != nil {
- details.result = "catalog_published"
- details.catalogTables = len(input.Catalog.Tables)
- for _, properties := range input.Catalog.Properties {
- details.catalogProperties += len(properties)
- }
- }
- writeJSON(w, http.StatusOK, map[string]any{"teamId": authorization.TeamID, "userId": authorization.UserID, "revision": input.Revision})
-}
-
-func (s *server) deleteCatalog(w http.ResponseWriter, r *http.Request, authorization serviceauth.Authorization) {
- if !s.catalogs.Delete(authorization) {
- setRequestResult(r, "catalog_miss")
- http.Error(w, "catalog not found", http.StatusNotFound)
- return
- }
- setRequestResult(r, "catalog_deleted")
- w.WriteHeader(http.StatusNoContent)
-}
-
-func (s *server) autocomplete(w http.ResponseWriter, r *http.Request, authorization serviceauth.Authorization) {
- var input completionRequest
- if !decodeJSON(w, r, 128<<10, &input) {
- return
- }
- current, revision, ok := s.catalogs.Get(authorization)
- if !ok {
- setRequestResult(r, "catalog_miss")
- http.Error(w, "catalog not found", http.StatusNotFound)
- return
- }
- setRequestResult(r, "catalog_hit")
- position := -1
- if input.Position != nil {
- position = *input.Position
- }
- positionEncoding := input.PositionEncoding
- if positionEncoding == "" {
- positionEncoding = completion.PositionEncodingUTF8
- }
- started := time.Now()
- result, err := completion.Complete(current, input.Query, position, positionEncoding, input.Cursor)
- if err != nil {
- setRequestResult(r, "invalid_query")
- http.Error(w, err.Error(), http.StatusBadRequest)
- return
- }
- writeJSON(w, http.StatusOK, completionResponse{
- Result: result,
- CatalogRevision: revision,
- DurationMicros: time.Since(started).Microseconds(),
- PositionEncoding: positionEncoding,
- })
-}
-
-func (s *server) validate(w http.ResponseWriter, r *http.Request, authorization serviceauth.Authorization) {
- var input validationRequest
- if !decodeJSON(w, r, 128<<10, &input) {
- return
- }
- current, revision, ok := s.catalogs.Get(authorization)
- if !ok {
- setRequestResult(r, "catalog_miss")
- http.Error(w, "catalog not found", http.StatusNotFound)
- return
- }
- setRequestResult(r, "catalog_hit")
- positionEncoding := input.PositionEncoding
- if positionEncoding == "" {
- positionEncoding = textposition.UTF16
- }
- result, err := validation.ValidateWithEncoding(current, input.Query, positionEncoding)
- if err != nil {
- setRequestResult(r, "invalid_query")
- http.Error(w, err.Error(), http.StatusBadRequest)
- return
- }
- writeJSON(w, http.StatusOK, validationResponse{Result: result, CatalogRevision: revision, PositionEncoding: positionEncoding})
-}
-
-func (s *server) health(w http.ResponseWriter, _ *http.Request) {
- writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
-}
-
-func decodeJSON(w http.ResponseWriter, r *http.Request, maxBytes int64, target any) bool {
- decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBytes))
- decoder.DisallowUnknownFields()
- if err := decoder.Decode(target); err != nil {
- setRequestResult(r, "invalid_json")
- http.Error(w, "invalid request: "+err.Error(), http.StatusBadRequest)
- return false
- }
- return true
-}
-
-func authorizationFromPath(r *http.Request) (serviceauth.Authorization, error) {
- teamID, err := strconv.ParseInt(r.PathValue("teamID"), 10, 64)
- if err != nil {
- return serviceauth.Authorization{}, errors.New("teamID and userID must be positive integers")
- }
- userID, err := strconv.ParseInt(r.PathValue("userID"), 10, 64)
- if err != nil || teamID <= 0 || userID <= 0 {
- return serviceauth.Authorization{}, errors.New("teamID and userID must be positive integers")
- }
- return serviceauth.Authorization{TeamID: teamID, UserID: userID}, nil
-}
-
-func remoteAddress(r *http.Request) string {
- host, _, err := net.SplitHostPort(r.RemoteAddr)
- if err != nil {
- return r.RemoteAddr
- }
- return host
-}
-
-func authorizationKey(authorization serviceauth.Authorization) string {
- return strconv.FormatInt(authorization.TeamID, 10) + ":" + strconv.FormatInt(authorization.UserID, 10)
-}
-
-func writeRateLimitResponse(w http.ResponseWriter, retryAfter time.Duration) {
- seconds := max(int64(1), int64((retryAfter+time.Second-1)/time.Second))
- w.Header().Set("Retry-After", strconv.FormatInt(seconds, 10))
- http.Error(w, "rate limit exceeded", http.StatusTooManyRequests)
-}
-
-func writeJSON(w http.ResponseWriter, status int, value any) {
- body, err := json.Marshal(value)
- if err != nil {
- http.Error(w, "encode response", http.StatusInternalServerError)
- return
- }
- w.Header().Set("Content-Type", "application/json; charset=utf-8")
- w.Header().Set("Content-Length", strconv.Itoa(len(body)))
- w.WriteHeader(status)
- // nosemgrep: go.lang.security.audit.xss.no-direct-write-to-responsewriter.no-direct-write-to-responsewriter -- json.Marshal escapes strings and this response has an application/json content type.
- if _, err := w.Write(body); err != nil {
- slog.Warn("write response", "error", err)
- }
-}
diff --git a/services/hogql-language-service/internal/httpapi/handler_test.go b/services/hogql-language-service/internal/httpapi/handler_test.go
deleted file mode 100644
index 41f3f7a7cdf6..000000000000
--- a/services/hogql-language-service/internal/httpapi/handler_test.go
+++ /dev/null
@@ -1,416 +0,0 @@
-package httpapi
-
-import (
- "bytes"
- "encoding/json"
- "io"
- "log/slog"
- "net/http"
- "net/http/httptest"
- "strconv"
- "strings"
- "testing"
- "time"
- "unicode/utf16"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/completion"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/ratelimit"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/serviceauth"
-)
-
-func TestAutocompleteUsesOnlyRequestedTeamAndUserCatalog(t *testing.T) {
- s := newTestServer(t)
- handler := s.handler()
- putCatalogForTest(t, handler, 1, 10, "revision-one", "orders")
- putCatalogForTest(t, handler, 1, 20, "revision-two", "accounts")
- putCatalogForTest(t, handler, 2, 10, "revision-three", "invoices")
-
- for _, test := range []struct {
- teamID int64
- userID int64
- revision string
- table string
- }{
- {teamID: 1, userID: 10, revision: "revision-one", table: "orders"},
- {teamID: 1, userID: 20, revision: "revision-two", table: "accounts"},
- {teamID: 2, userID: 10, revision: "revision-three", table: "invoices"},
- } {
- body := `{"query":"SELECT * FROM "}`
- path := scopePath(test.teamID, test.userID) + "/autocomplete"
- request := httptest.NewRequest(http.MethodPost, path, strings.NewReader(body))
- request.Header.Set("Content-Type", "application/json")
- response := httptest.NewRecorder()
- handler.ServeHTTP(response, request)
- if response.Code != http.StatusOK {
- t.Fatalf("autocomplete returned %d: %s", response.Code, response.Body.String())
- }
- if contentType := response.Header().Get("Content-Type"); contentType != "application/json; charset=utf-8" {
- t.Fatalf("unexpected Content-Type: %q", contentType)
- }
- if response.Header().Get("X-Content-Type-Options") != "nosniff" {
- t.Fatal("response is missing X-Content-Type-Options: nosniff")
- }
- if contentLength := response.Header().Get("Content-Length"); contentLength != strconv.Itoa(response.Body.Len()) {
- t.Fatalf("Content-Length = %q, response size = %d", contentLength, response.Body.Len())
- }
- var result completionResponse
- if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
- t.Fatal(err)
- }
- if result.CatalogRevision != test.revision || !hasSuggestion(result.Suggestions, test.table) {
- t.Fatalf("unexpected response for team %d user %d: %#v", test.teamID, test.userID, result)
- }
- if result.PositionEncoding != completion.PositionEncodingUTF8 {
- t.Fatalf("unexpected position encoding: %q", result.PositionEncoding)
- }
- for _, otherTable := range []string{"orders", "accounts", "invoices"} {
- if otherTable != test.table && hasSuggestion(result.Suggestions, otherTable) {
- t.Fatalf("%s leaked into team %d user %d", otherTable, test.teamID, test.userID)
- }
- }
- }
-}
-
-func TestAutocompleteRequiresKnownTeamAndUser(t *testing.T) {
- s := newTestServer(t)
- for _, test := range []struct {
- path string
- body string
- status int
- }{
- {path: "/teams/1/users/invalid/autocomplete", body: `{"query":"SELECT "}`, status: http.StatusBadRequest},
- {path: "/teams/invalid/users/10/validate", body: `{"query":"SELECT 1"}`, status: http.StatusBadRequest},
- {path: scopePath(1, 10) + "/autocomplete", body: `{"query":"SELECT "}`, status: http.StatusNotFound},
- } {
- request := httptest.NewRequest(http.MethodPost, test.path, strings.NewReader(test.body))
- response := httptest.NewRecorder()
- s.handler().ServeHTTP(response, request)
- if response.Code != test.status {
- t.Fatalf("expected %d, got %d: %s", test.status, response.Code, response.Body.String())
- }
- if response.Header().Get("X-Content-Type-Options") != "nosniff" {
- t.Fatal("error response is missing X-Content-Type-Options: nosniff")
- }
- }
-}
-
-func TestCatalogAliasesStayWithinPublishedTeamAndUser(t *testing.T) {
- s := newTestServer(t)
- handler := s.handler()
- body := `{"revision":"aliases","catalog":{"tables":{"postgres.demo.orders":{"type":"data_warehouse","fields":{"id":{"type":"integer"}}}},"tableAliases":{"demo_postgres_orders":"postgres.demo.orders"},"properties":{}}}`
- request := httptest.NewRequest(http.MethodPut, scopePath(1, 10)+"/catalog", strings.NewReader(body))
- response := httptest.NewRecorder()
- handler.ServeHTTP(response, request)
- if response.Code != http.StatusOK {
- t.Fatalf("catalog upload returned %d: %s", response.Code, response.Body.String())
- }
- putCatalogForTest(t, handler, 1, 20, "without-aliases", "postgres.demo.orders")
-
- for _, test := range []struct {
- userID int64
- valid bool
- }{
- {userID: 10, valid: true},
- {userID: 20, valid: false},
- } {
- request = httptest.NewRequest(http.MethodPost, scopePath(1, test.userID)+"/validate", strings.NewReader(`{"query":"SELECT id FROM demo_postgres_orders"}`))
- response = httptest.NewRecorder()
- handler.ServeHTTP(response, request)
- if response.Code != http.StatusOK {
- t.Fatalf("validate returned %d: %s", response.Code, response.Body.String())
- }
- var result validationResponse
- if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
- t.Fatal(err)
- }
- if result.Valid != test.valid {
- t.Fatalf("user %d result = %#v", test.userID, result)
- }
- }
-}
-
-func TestTraversalRelationsStayWithinPublishedTeamAndUser(t *testing.T) {
- s := newTestServer(t)
- handler := s.handler()
- body := `{"revision":"traversal","catalog":{"tables":{"events":{"fields":{"person":{"type":"lazy","relation":"person"},"session":{"type":"lazy","relation":"session"}}},"sessions":{"fields":{"session_id":{"type":"String"}}}},"relations":{"empty":{"fields":{}},"person":{"fields":{"email":{"type":"String"},"properties":{"type":"JSON","propertyNamespace":"person"}}},"session":{"table":"sessions"}},"properties":{"person":[{"name":"plan","property_type":"String"}]}}}`
- request := httptest.NewRequest(http.MethodPut, scopePath(1, 10)+"/catalog", strings.NewReader(body))
- response := httptest.NewRecorder()
- handler.ServeHTTP(response, request)
- if response.Code != http.StatusOK {
- t.Fatalf("catalog upload returned %d: %s", response.Code, response.Body.String())
- }
- putCatalogForTest(t, handler, 1, 20, "flat", "events")
-
- for _, test := range []struct {
- userID int64
- valid bool
- }{
- {userID: 10, valid: true},
- {userID: 20, valid: false},
- } {
- request = httptest.NewRequest(http.MethodPost, scopePath(1, test.userID)+"/validate", strings.NewReader(`{"query":"SELECT e.person.email FROM events AS e"}`))
- response = httptest.NewRecorder()
- handler.ServeHTTP(response, request)
- if response.Code != http.StatusOK {
- t.Fatalf("validate returned %d: %s", response.Code, response.Body.String())
- }
- var result validationResponse
- if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
- t.Fatal(err)
- }
- if result.Valid != test.valid {
- t.Fatalf("user %d result = %#v", test.userID, result)
- }
- }
- request = httptest.NewRequest(http.MethodPost, scopePath(1, 10)+"/validate", strings.NewReader(`{"query":"SELECT e.session.session_id FROM events AS e"}`))
- response = httptest.NewRecorder()
- handler.ServeHTTP(response, request)
- if response.Code != http.StatusOK {
- t.Fatalf("table-backed validate returned %d: %s", response.Code, response.Body.String())
- }
- var result validationResponse
- if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
- t.Fatal(err)
- }
- if !result.Valid {
- t.Fatalf("table-backed traversal result = %#v", result)
- }
-}
-
-func TestCatalogRejectsInvalidAliasMaps(t *testing.T) {
- handler := newTestServer(t).handler()
- for name, aliases := range map[string]string{
- "dangling": `{"legacy_orders":"missing"}`,
- "chain": `{"legacy_orders":"older_orders","older_orders":"orders"}`,
- "cycle": `{"legacy_orders":"older_orders","older_orders":"legacy_orders"}`,
- "collision": `{"orders":"events"}`,
- } {
- t.Run(name, func(t *testing.T) {
- body := `{"revision":"invalid","catalog":{"tables":{"orders":{"fields":{}},"events":{"fields":{}}},"tableAliases":` + aliases + `,"properties":{}}}`
- request := httptest.NewRequest(http.MethodPut, scopePath(1, 10)+"/catalog", strings.NewReader(body))
- response := httptest.NewRecorder()
- handler.ServeHTTP(response, request)
- if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), catalog.ErrInvalidAliases.Error()) {
- t.Fatalf("response = %d: %s", response.Code, response.Body.String())
- }
- })
- }
-}
-
-func TestValidateEncodesDiagnosticPositions(t *testing.T) {
- s := newTestServer(t)
- handler := s.handler()
- putCatalogForTest(t, handler, 1, 10, "revision-one", "events")
- query := "SELECT '😀', missing FROM events"
- byteStart := strings.Index(query, "missing")
- utf16Start := len(utf16.Encode([]rune(query[:byteStart])))
- tableByteStart := strings.Index(query, "events")
- tableUTF16Start := len(utf16.Encode([]rune(query[:tableByteStart])))
-
- for _, test := range []struct {
- encoding string
- responseEncoding string
- start int
- }{
- {encoding: "utf-8", responseEncoding: "utf-8", start: byteStart},
- {encoding: "utf-16", responseEncoding: "utf-16", start: utf16Start},
- {responseEncoding: "utf-16", start: utf16Start},
- } {
- body, err := json.Marshal(map[string]any{"query": query, "positionEncoding": test.encoding})
- if err != nil {
- t.Fatal(err)
- }
- request := httptest.NewRequest(http.MethodPost, scopePath(1, 10)+"/validate", bytes.NewReader(body))
- request.Header.Set("Content-Type", "application/json")
- response := httptest.NewRecorder()
- handler.ServeHTTP(response, request)
- if response.Code != http.StatusOK {
- t.Fatalf("validate returned %d: %s", response.Code, response.Body.String())
- }
- var result validationResponse
- if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
- t.Fatal(err)
- }
- if string(result.PositionEncoding) != test.responseEncoding {
- t.Fatalf("position encoding = %q, want %q", result.PositionEncoding, test.responseEncoding)
- }
- if len(result.Diagnostics) != 1 {
- t.Fatalf("diagnostics = %#v", result.Diagnostics)
- }
- diagnostic := result.Diagnostics[0]
- if diagnostic.Start != test.start || diagnostic.End != test.start+len("missing") {
- t.Fatalf("%s diagnostic span = [%d,%d), want [%d,%d)", test.responseEncoding, diagnostic.Start, diagnostic.End, test.start, test.start+len("missing"))
- }
- if len(result.Notices) != 1 || result.Notices[0].Message != "Table 'events'" {
- t.Fatalf("notices = %#v", result.Notices)
- }
- tableStart := tableByteStart
- if test.responseEncoding == "utf-16" {
- tableStart = tableUTF16Start
- }
- if result.Notices[0].Start != tableStart || result.Notices[0].End != tableStart+len("events") {
- t.Fatalf("%s notice span = [%d,%d), want [%d,%d)", test.responseEncoding, result.Notices[0].Start, result.Notices[0].End, tableStart, tableStart+len("events"))
- }
- }
-}
-
-func TestRequestLogIncludesMetadataWithoutRequestContents(t *testing.T) {
- var logs bytes.Buffer
- s := newTestServer(t)
- s.logger = slog.New(slog.NewJSONHandler(&logs, nil))
- handler := s.handler()
- putCatalogForTest(t, handler, 1, 10, "revision-one", "events")
- logs.Reset()
-
- request := httptest.NewRequest(http.MethodPost, scopePath(1, 10)+"/validate", strings.NewReader(`{"query":"SELECT 'do-not-log-query'"}`))
- request.Header.Set("Authorization", "Bearer do-not-log-token")
- response := httptest.NewRecorder()
- handler.ServeHTTP(response, request)
- if response.Code != http.StatusOK {
- t.Fatalf("validate returned %d: %s", response.Code, response.Body.String())
- }
-
- var entry map[string]any
- if err := json.Unmarshal(bytes.TrimSpace(logs.Bytes()), &entry); err != nil {
- t.Fatalf("decode request log: %v\n%s", err, logs.String())
- }
- for key, expected := range map[string]any{
- "msg": "http_request",
- "operation": "validate",
- "method": http.MethodPost,
- "status_code": float64(http.StatusOK),
- "response_bytes": float64(response.Body.Len()),
- "result": "catalog_hit",
- "team_id": float64(1),
- "user_id": float64(10),
- } {
- if entry[key] != expected {
- t.Errorf("%s = %#v, want %#v", key, entry[key], expected)
- }
- }
- if duration, ok := entry["duration_ms"].(float64); !ok || duration < 0 {
- t.Errorf("duration_ms = %#v", entry["duration_ms"])
- }
- if strings.Contains(logs.String(), "do-not-log-query") || strings.Contains(logs.String(), "do-not-log-token") {
- t.Fatalf("request contents leaked into log: %s", logs.String())
- }
-
- logs.Reset()
- request = httptest.NewRequest(http.MethodGet, "/unknown", nil)
- response = httptest.NewRecorder()
- handler.ServeHTTP(response, request)
- if response.Code != http.StatusNotFound {
- t.Fatalf("unknown route returned %d: %s", response.Code, response.Body.String())
- }
- entry = map[string]any{}
- if err := json.Unmarshal(bytes.TrimSpace(logs.Bytes()), &entry); err != nil {
- t.Fatalf("decode unmatched request log: %v\n%s", err, logs.String())
- }
- for key, expected := range map[string]any{
- "level": "WARN",
- "msg": "http_request",
- "operation": "unmatched",
- "method": http.MethodGet,
- "status_code": float64(http.StatusNotFound),
- "result": "error",
- } {
- if entry[key] != expected {
- t.Errorf("%s = %#v, want %#v", key, entry[key], expected)
- }
- }
-}
-
-func TestPrincipalRateLimitRunsBeforeBodyDecodeAndDoesNotCrossScopes(t *testing.T) {
- preAuthLimiter, err := ratelimit.New(ratelimit.Config{Capacity: 1, RefillPerSec: 0.001, MaxEntries: 10, IdleTTL: time.Hour})
- if err != nil {
- t.Fatal(err)
- }
- principalLimiter, err := ratelimit.New(ratelimit.Config{Capacity: 1, RefillPerSec: 0.001, MaxEntries: 10, IdleTTL: time.Hour})
- if err != nil {
- t.Fatal(err)
- }
- s := &server{
- catalogs: catalog.NewRegistry(10, 1<<20, time.Hour),
- auth: serviceauth.New(nil, true),
- preAuthLimiter: preAuthLimiter,
- principalLimiter: principalLimiter,
- logger: discardLogger(),
- }
- value := &catalog.Catalog{Tables: map[string]catalog.Table{}, Properties: map[string][]catalog.Property{}}
- for _, authorization := range []serviceauth.Authorization{{TeamID: 1, UserID: 10}, {TeamID: 1, UserID: 20}} {
- if err := s.catalogs.Put(authorization, "1", catalog.Prepare(value)); err != nil {
- t.Fatal(err)
- }
- }
-
- request := httptest.NewRequest(http.MethodPost, scopePath(1, 10)+"/autocomplete", strings.NewReader(`{"query":"SELECT "}`))
- response := httptest.NewRecorder()
- s.handler().ServeHTTP(response, request)
- if response.Code != http.StatusOK {
- t.Fatalf("first request returned %d: %s", response.Code, response.Body.String())
- }
-
- request = httptest.NewRequest(http.MethodPost, scopePath(1, 10)+"/autocomplete", strings.NewReader(`{`))
- response = httptest.NewRecorder()
- s.handler().ServeHTTP(response, request)
- if response.Code != http.StatusTooManyRequests || response.Header().Get("Retry-After") == "" {
- t.Fatalf("limited request returned %d without Retry-After: %s", response.Code, response.Body.String())
- }
-
- request = httptest.NewRequest(http.MethodPost, scopePath(1, 20)+"/autocomplete", strings.NewReader(`{"query":"SELECT "}`))
- response = httptest.NewRecorder()
- s.handler().ServeHTTP(response, request)
- if response.Code != http.StatusOK {
- t.Fatalf("another user inherited the rate limit: %d: %s", response.Code, response.Body.String())
- }
-}
-
-func putCatalogForTest(t *testing.T, handler http.Handler, teamID, userID int64, revision, table string) {
- t.Helper()
- body := `{"revision":"` + revision + `","catalog":{"tables":{"` + table + `":{"name":"` + table + `","type":"warehouse","fields":{}}},"properties":{}}}`
- path := scopePath(teamID, userID) + "/catalog"
- request := httptest.NewRequest(http.MethodPut, path, strings.NewReader(body))
- response := httptest.NewRecorder()
- handler.ServeHTTP(response, request)
- if response.Code != http.StatusOK {
- t.Fatalf("catalog upload returned %d: %s", response.Code, response.Body.String())
- }
-}
-
-func newTestServer(t *testing.T) *server {
- t.Helper()
- config := ratelimit.Config{Capacity: 1000, RefillPerSec: 1000, MaxEntries: 100, IdleTTL: time.Hour}
- preAuthLimiter, err := ratelimit.New(config)
- if err != nil {
- t.Fatal(err)
- }
- principalLimiter, err := ratelimit.New(config)
- if err != nil {
- t.Fatal(err)
- }
- return &server{
- catalogs: catalog.NewRegistry(10, 1<<20, time.Hour),
- auth: serviceauth.New(nil, true),
- preAuthLimiter: preAuthLimiter,
- principalLimiter: principalLimiter,
- logger: discardLogger(),
- }
-}
-
-func discardLogger() *slog.Logger {
- return slog.New(slog.NewTextHandler(io.Discard, nil))
-}
-
-func scopePath(teamID, userID int64) string {
- return "/teams/" + strconv.FormatInt(teamID, 10) + "/users/" + strconv.FormatInt(userID, 10)
-}
-
-func hasSuggestion(suggestions []completion.Suggestion, label string) bool {
- for _, suggestion := range suggestions {
- if suggestion.Label == label {
- return true
- }
- }
- return false
-}
diff --git a/services/hogql-language-service/internal/propertyresolver/resolver.go b/services/hogql-language-service/internal/propertyresolver/resolver.go
deleted file mode 100644
index 1faffeec94e4..000000000000
--- a/services/hogql-language-service/internal/propertyresolver/resolver.go
+++ /dev/null
@@ -1,59 +0,0 @@
-package propertyresolver
-
-import (
- "regexp"
- "strings"
-)
-
-var groupOwner = regexp.MustCompile(`^group_?([0-4])$`)
-
-func Resolve(parts []string, bindings map[string]string) (string, bool) {
- propertyIndex := -1
- for index, part := range parts {
- if strings.EqualFold(part, "properties") {
- propertyIndex = index
- }
- }
- if propertyIndex == -1 || propertyIndex != len(parts)-2 {
- return "", false
- }
- if propertyIndex > 0 {
- owner := parts[propertyIndex-1]
- if namespace, ok := namespaceForOwner(owner); ok {
- return namespace, true
- }
- if tableName, ok := bindings[owner]; ok {
- return namespaceForOwner(tableName)
- }
- return "", false
- }
-
- namespaces := map[string]bool{}
- for _, tableName := range bindings {
- if namespace, ok := namespaceForOwner(tableName); ok {
- namespaces[namespace] = true
- }
- }
- if len(namespaces) != 1 {
- return "", false
- }
- for namespace := range namespaces {
- return namespace, true
- }
- return "", false
-}
-
-func namespaceForOwner(owner string) (string, bool) {
- switch owner {
- case "event", "events":
- return "event", true
- case "person", "persons":
- return "person", true
- case "session", "sessions":
- return "session", true
- }
- if match := groupOwner.FindStringSubmatch(owner); match != nil {
- return "group:" + match[1], true
- }
- return "", false
-}
diff --git a/services/hogql-language-service/internal/querylimits/limits.go b/services/hogql-language-service/internal/querylimits/limits.go
deleted file mode 100644
index 6730336ae5ab..000000000000
--- a/services/hogql-language-service/internal/querylimits/limits.go
+++ /dev/null
@@ -1,62 +0,0 @@
-package querylimits
-
-import "errors"
-
-const MaxQueryBytes = 64 << 10
-const MaxNestingDepth = 128
-const MaxSuggestionInputBytes = 128
-const MaxDiagnostics = 25
-const MaxNotices = 128
-const MaxCTEProjectedFields = 16 << 10
-const MaxFieldLookupWork = 1 << 20
-const MaxRelationTraversalHops = 16
-
-var ErrQueryTooLarge = errors.New("query exceeds maximum size")
-var ErrQueryTooDeep = errors.New("query exceeds maximum nesting depth")
-var ErrCTEProjectionTooLarge = errors.New("query expands too many CTE fields; select fewer fields in each CTE")
-var ErrFieldLookupTooLarge = errors.New("query requires too much field lookup work; use fewer sources or qualify field names")
-var ErrRelationTraversalTooDeep = errors.New("query traverses too many lazy relations")
-
-func Validate(query string) error {
- if len(query) > MaxQueryBytes {
- return ErrQueryTooLarge
- }
- depth := 0
- var quote byte
- escaped := false
- for index := 0; index < len(query); index++ {
- character := query[index]
- if quote != 0 {
- if escaped {
- escaped = false
- continue
- }
- if character == '\\' {
- escaped = true
- continue
- }
- if character == quote {
- if index+1 < len(query) && query[index+1] == quote {
- index++
- continue
- }
- quote = 0
- }
- continue
- }
- switch character {
- case '\'', '"', '`':
- quote = character
- case '(':
- depth++
- if depth > MaxNestingDepth {
- return ErrQueryTooDeep
- }
- case ')':
- if depth > 0 {
- depth--
- }
- }
- }
- return nil
-}
diff --git a/services/hogql-language-service/internal/querylimits/limits_test.go b/services/hogql-language-service/internal/querylimits/limits_test.go
deleted file mode 100644
index adbce055be78..000000000000
--- a/services/hogql-language-service/internal/querylimits/limits_test.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package querylimits
-
-import (
- "strings"
- "testing"
-)
-
-func TestValidateBoundsQuerySizeAndNesting(t *testing.T) {
- for _, test := range []struct {
- name string
- query string
- err error
- }{
- {name: "ordinary", query: "SELECT sum(amount) FROM orders"},
- {name: "parentheses in string", query: "SELECT '" + strings.Repeat("(", MaxNestingDepth+1) + "'"},
- {name: "oversized", query: strings.Repeat("a", MaxQueryBytes+1), err: ErrQueryTooLarge},
- {name: "deeply nested", query: "SELECT " + strings.Repeat("(", MaxNestingDepth+1) + "1" + strings.Repeat(")", MaxNestingDepth+1), err: ErrQueryTooDeep},
- } {
- if err := Validate(test.query); err != test.err {
- t.Fatalf("%s: got %v, want %v", test.name, err, test.err)
- }
- }
-}
diff --git a/services/hogql-language-service/internal/ratelimit/limiter.go b/services/hogql-language-service/internal/ratelimit/limiter.go
deleted file mode 100644
index 91a228494364..000000000000
--- a/services/hogql-language-service/internal/ratelimit/limiter.go
+++ /dev/null
@@ -1,85 +0,0 @@
-package ratelimit
-
-import (
- "container/list"
- "errors"
- "math"
- "sync"
- "time"
-)
-
-type Config struct {
- Capacity float64
- RefillPerSec float64
- MaxEntries int
- IdleTTL time.Duration
-}
-
-type Limiter struct {
- mu sync.Mutex
- config Config
- entries map[string]*entry
- recency *list.List
- now func() time.Time
-}
-
-type entry struct {
- tokens float64
- updatedAt time.Time
- lastAccess time.Time
- element *list.Element
-}
-
-func New(config Config) (*Limiter, error) {
- return newLimiter(config, time.Now)
-}
-
-func newLimiter(config Config, now func() time.Time) (*Limiter, error) {
- if config.Capacity <= 0 || math.IsNaN(config.Capacity) || math.IsInf(config.Capacity, 0) ||
- config.RefillPerSec <= 0 || math.IsNaN(config.RefillPerSec) || math.IsInf(config.RefillPerSec, 0) ||
- config.MaxEntries <= 0 || config.IdleTTL <= 0 {
- return nil, errors.New("rate limit values must be positive")
- }
- return &Limiter{config: config, entries: make(map[string]*entry), recency: list.New(), now: now}, nil
-}
-
-func (l *Limiter) Allow(key string) (bool, time.Duration) {
- l.mu.Lock()
- defer l.mu.Unlock()
-
- now := l.now()
- current, exists := l.entries[key]
- if !exists || now.Sub(current.lastAccess) >= l.config.IdleTTL {
- if exists {
- l.recency.Remove(current.element)
- delete(l.entries, key)
- }
- if len(l.entries) >= l.config.MaxEntries {
- l.removeLeastRecentlyUsed()
- }
- current = &entry{tokens: l.config.Capacity, updatedAt: now}
- current.element = l.recency.PushBack(key)
- l.entries[key] = current
- } else {
- l.recency.MoveToBack(current.element)
- }
- elapsed := now.Sub(current.updatedAt).Seconds()
- current.tokens = math.Min(l.config.Capacity, current.tokens+elapsed*l.config.RefillPerSec)
- current.updatedAt = now
- current.lastAccess = now
- if current.tokens >= 1 {
- current.tokens--
- return true, 0
- }
- retryAfter := time.Duration(math.Ceil((1-current.tokens)/l.config.RefillPerSec*float64(time.Second))) * time.Nanosecond
- return false, retryAfter
-}
-
-func (l *Limiter) removeLeastRecentlyUsed() {
- oldest := l.recency.Front()
- if oldest == nil {
- return
- }
- delete(l.entries, oldest.Value.(string))
- l.recency.Remove(oldest)
-}
diff --git a/services/hogql-language-service/internal/ratelimit/limiter_test.go b/services/hogql-language-service/internal/ratelimit/limiter_test.go
deleted file mode 100644
index 78f5946c47ae..000000000000
--- a/services/hogql-language-service/internal/ratelimit/limiter_test.go
+++ /dev/null
@@ -1,60 +0,0 @@
-package ratelimit
-
-import (
- "math"
- "testing"
- "time"
-)
-
-func TestLimiterRefillsTokens(t *testing.T) {
- now := time.Unix(100, 0)
- limiter, err := newLimiter(Config{Capacity: 2, RefillPerSec: 1, MaxEntries: 2, IdleTTL: time.Minute}, func() time.Time { return now })
- if err != nil {
- t.Fatal(err)
- }
- for range 2 {
- if allowed, _ := limiter.Allow("scope"); !allowed {
- t.Fatal("request was unexpectedly limited")
- }
- }
- if allowed, retryAfter := limiter.Allow("scope"); allowed || retryAfter != time.Second {
- t.Fatalf("expected one second retry, got allowed=%t retry=%s", allowed, retryAfter)
- }
- now = now.Add(time.Second)
- if allowed, _ := limiter.Allow("scope"); !allowed {
- t.Fatal("refilled request was limited")
- }
-}
-
-func TestLimiterBoundsKeysAndEvictsLeastRecentlyUsed(t *testing.T) {
- now := time.Unix(100, 0)
- limiter, err := newLimiter(Config{Capacity: 2, RefillPerSec: 1, MaxEntries: 2, IdleTTL: time.Minute}, func() time.Time { return now })
- if err != nil {
- t.Fatal(err)
- }
- limiter.Allow("first")
- now = now.Add(time.Second)
- limiter.Allow("second")
- now = now.Add(time.Second)
- limiter.Allow("first")
- now = now.Add(time.Second)
- limiter.Allow("third")
-
- if len(limiter.entries) != 2 || limiter.entries["first"] == nil || limiter.entries["third"] == nil {
- t.Fatalf("unexpected bounded entries: %#v", limiter.entries)
- }
- if limiter.entries["second"] != nil {
- t.Fatal("least recently used key was not evicted")
- }
-}
-
-func TestLimiterRejectsNonFiniteValues(t *testing.T) {
- for _, config := range []Config{
- {Capacity: math.NaN(), RefillPerSec: 1, MaxEntries: 1, IdleTTL: time.Minute},
- {Capacity: 1, RefillPerSec: math.Inf(1), MaxEntries: 1, IdleTTL: time.Minute},
- } {
- if _, err := New(config); err == nil {
- t.Fatal("non-finite configuration was accepted")
- }
- }
-}
diff --git a/services/hogql-language-service/internal/serviceauth/auth.go b/services/hogql-language-service/internal/serviceauth/auth.go
deleted file mode 100644
index 897797dd5947..000000000000
--- a/services/hogql-language-service/internal/serviceauth/auth.go
+++ /dev/null
@@ -1,121 +0,0 @@
-package serviceauth
-
-import (
- "crypto/hmac"
- "crypto/sha256"
- "encoding/base64"
- "encoding/json"
- "errors"
- "strings"
- "time"
-)
-
-var ErrUnauthorized = errors.New("unauthorized")
-
-type Authorization struct {
- TeamID int64
- UserID int64
-}
-
-func (a Authorization) Valid() bool {
- return a.TeamID > 0 && a.UserID > 0
-}
-
-type Operation string
-
-const (
- OperationPublish Operation = "publish"
- OperationDelete Operation = "delete"
- OperationComplete Operation = "complete"
- OperationValidate Operation = "validate"
-)
-
-type Authenticator struct {
- keys [][]byte
- allowInsecure bool
- now func() time.Time
-}
-
-type header struct {
- Algorithm string `json:"alg"`
-}
-
-type Claims struct {
- Audience string `json:"aud"`
- TeamID int64 `json:"team_id"`
- UserID int64 `json:"user_id"`
- Operations []string `json:"operations"`
- ExpiresAt int64 `json:"exp"`
- NotBefore int64 `json:"nbf,omitempty"`
-}
-
-func New(keys []string, allowInsecure bool) *Authenticator {
- parsed := make([][]byte, 0, len(keys))
- for _, key := range keys {
- if key = strings.TrimSpace(key); key != "" {
- parsed = append(parsed, []byte(key))
- }
- }
- return &Authenticator{keys: parsed, allowInsecure: allowInsecure, now: time.Now}
-}
-
-func (a *Authenticator) Verify(headerValue string, authorization Authorization, operation Operation) error {
- if !authorization.Valid() {
- return ErrUnauthorized
- }
- if len(a.keys) == 0 {
- if a.allowInsecure {
- return nil
- }
- return ErrUnauthorized
- }
- if !strings.HasPrefix(headerValue, "Bearer ") {
- return ErrUnauthorized
- }
- parts := strings.Split(strings.TrimPrefix(headerValue, "Bearer "), ".")
- if len(parts) != 3 {
- return ErrUnauthorized
- }
- signed := parts[0] + "." + parts[1]
- signature, err := base64.RawURLEncoding.DecodeString(parts[2])
- if err != nil || !validSignature([]byte(signed), signature, a.keys) {
- return ErrUnauthorized
- }
-
- var tokenHeader header
- if err := decodePart(parts[0], &tokenHeader); err != nil || tokenHeader.Algorithm != "HS256" {
- return ErrUnauthorized
- }
- var claims Claims
- if err := decodePart(parts[1], &claims); err != nil {
- return ErrUnauthorized
- }
- now := a.now().Unix()
- if claims.Audience != "hogql-language-service" || claims.ExpiresAt <= now || claims.NotBefore > now || claims.TeamID != authorization.TeamID || claims.UserID != authorization.UserID {
- return ErrUnauthorized
- }
- for _, allowed := range claims.Operations {
- if allowed == string(operation) {
- return nil
- }
- }
- return ErrUnauthorized
-}
-
-func decodePart(part string, target any) error {
- decoded, err := base64.RawURLEncoding.DecodeString(part)
- if err != nil {
- return err
- }
- return json.Unmarshal(decoded, target)
-}
-
-func validSignature(message, signature []byte, keys [][]byte) bool {
- valid := false
- for _, key := range keys {
- mac := hmac.New(sha256.New, key)
- _, _ = mac.Write(message)
- valid = hmac.Equal(signature, mac.Sum(nil)) || valid
- }
- return valid
-}
diff --git a/services/hogql-language-service/internal/serviceauth/auth_test.go b/services/hogql-language-service/internal/serviceauth/auth_test.go
deleted file mode 100644
index 8abc51af51fe..000000000000
--- a/services/hogql-language-service/internal/serviceauth/auth_test.go
+++ /dev/null
@@ -1,73 +0,0 @@
-package serviceauth
-
-import (
- "crypto/hmac"
- "crypto/sha256"
- "encoding/base64"
- "encoding/json"
- "testing"
- "time"
-)
-
-func TestVerifyScopesTokenToCatalogAndOperation(t *testing.T) {
- authenticator := New([]string{"current-key", "previous-key"}, false)
- authenticator.now = func() time.Time { return time.Unix(100, 0) }
- token := signToken(t, "previous-key", Claims{
- Audience: "hogql-language-service",
- TeamID: 1,
- UserID: 10,
- Operations: []string{"complete"},
- ExpiresAt: 200,
- })
-
- if err := authenticator.Verify("Bearer "+token, Authorization{TeamID: 1, UserID: 10}, OperationComplete); err != nil {
- t.Fatalf("valid rotated key was rejected: %v", err)
- }
- for _, test := range []struct {
- teamID int64
- userID int64
- operation Operation
- }{
- {teamID: 2, userID: 10, operation: OperationComplete},
- {teamID: 1, userID: 20, operation: OperationComplete},
- {teamID: 1, userID: 10, operation: OperationPublish},
- } {
- authorization := Authorization{TeamID: test.teamID, UserID: test.userID}
- if err := authenticator.Verify("Bearer "+token, authorization, test.operation); err == nil {
- t.Fatalf("token unexpectedly authorized %s for team %d user %d", test.operation, test.teamID, test.userID)
- }
- }
-}
-
-func TestVerifyRejectsExpiredAndUnsignedTokens(t *testing.T) {
- authenticator := New([]string{"key"}, false)
- authenticator.now = func() time.Time { return time.Unix(100, 0) }
- expired := signToken(t, "key", Claims{Audience: "hogql-language-service", TeamID: 1, UserID: 10, Operations: []string{"complete"}, ExpiresAt: 100})
-
- for _, authorization := range []string{"", "Bearer unsigned", "Bearer " + expired} {
- if err := authenticator.Verify(authorization, Authorization{TeamID: 1, UserID: 10}, OperationComplete); err == nil {
- t.Fatalf("invalid token was accepted: %q", authorization)
- }
- }
- if err := New(nil, true).Verify("", Authorization{}, OperationComplete); err == nil {
- t.Fatal("insecure local mode accepted an empty authorization scope")
- }
-}
-
-func signToken(t *testing.T, key string, claims Claims) string {
- t.Helper()
- headerJSON, err := json.Marshal(header{Algorithm: "HS256"})
- if err != nil {
- t.Fatal(err)
- }
- claimsJSON, err := json.Marshal(claims)
- if err != nil {
- t.Fatal(err)
- }
- encodedHeader := base64.RawURLEncoding.EncodeToString(headerJSON)
- encodedClaims := base64.RawURLEncoding.EncodeToString(claimsJSON)
- message := encodedHeader + "." + encodedClaims
- mac := hmac.New(sha256.New, []byte(key))
- _, _ = mac.Write([]byte(message))
- return message + "." + base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
-}
diff --git a/services/hogql-language-service/internal/textposition/position.go b/services/hogql-language-service/internal/textposition/position.go
deleted file mode 100644
index 191db067f033..000000000000
--- a/services/hogql-language-service/internal/textposition/position.go
+++ /dev/null
@@ -1,77 +0,0 @@
-package textposition
-
-import (
- "fmt"
- "unicode/utf8"
-)
-
-type Encoding string
-
-const (
- UTF8 Encoding = "utf-8"
- UTF16 Encoding = "utf-16"
-)
-
-func (e Encoding) Valid() bool {
- return e == UTF8 || e == UTF16
-}
-
-func ToByteOffset(value string, offset int, encoding Encoding) (int, error) {
- if !encoding.Valid() {
- return 0, fmt.Errorf("unsupported position encoding %q", encoding)
- }
- if offset < 0 {
- return len(value), nil
- }
- if encoding == UTF8 {
- if offset > len(value) {
- return len(value), nil
- }
- for offset > 0 && offset < len(value) && !utf8.RuneStart(value[offset]) {
- offset--
- }
- return offset, nil
- }
-
- utf16Offset := 0
- for byteOffset, character := range value {
- if utf16Offset >= offset {
- return byteOffset, nil
- }
- characterWidth := 1
- if character > 0xFFFF {
- characterWidth = 2
- }
- if utf16Offset+characterWidth > offset {
- return byteOffset, nil
- }
- utf16Offset += characterWidth
- }
- return len(value), nil
-}
-
-func FromByteOffset(value string, offset int, encoding Encoding) (int, error) {
- if !encoding.Valid() {
- return 0, fmt.Errorf("unsupported position encoding %q", encoding)
- }
- if offset < 0 {
- offset = 0
- } else if offset > len(value) {
- offset = len(value)
- }
- if encoding == UTF8 {
- return offset, nil
- }
-
- utf16Offset := 0
- for byteOffset, character := range value {
- if byteOffset >= offset {
- break
- }
- utf16Offset++
- if character > 0xFFFF {
- utf16Offset++
- }
- }
- return utf16Offset, nil
-}
diff --git a/services/hogql-language-service/internal/textposition/position_test.go b/services/hogql-language-service/internal/textposition/position_test.go
deleted file mode 100644
index 17f640771214..000000000000
--- a/services/hogql-language-service/internal/textposition/position_test.go
+++ /dev/null
@@ -1,28 +0,0 @@
-package textposition
-
-import "testing"
-
-func TestToByteOffsetNormalizesUTF8CharacterBoundaries(t *testing.T) {
- value := "a😀éb"
- for _, test := range []struct {
- offset int
- expect int
- }{
- {offset: 0, expect: 0},
- {offset: 1, expect: 1},
- {offset: 2, expect: 1},
- {offset: 4, expect: 1},
- {offset: 5, expect: 5},
- {offset: 6, expect: 5},
- {offset: 7, expect: 7},
- {offset: 8, expect: 8},
- } {
- actual, err := ToByteOffset(value, test.offset, UTF8)
- if err != nil {
- t.Fatal(err)
- }
- if actual != test.expect {
- t.Errorf("ToByteOffset(%q, %d, UTF8) = %d, want %d", value, test.offset, actual, test.expect)
- }
- }
-}
diff --git a/services/hogql-language-service/internal/validation/notices.go b/services/hogql-language-service/internal/validation/notices.go
deleted file mode 100644
index bcb9ccbd770e..000000000000
--- a/services/hogql-language-service/internal/validation/notices.go
+++ /dev/null
@@ -1,196 +0,0 @@
-package validation
-
-import (
- "fmt"
- "sort"
- "strings"
-
- clickhouse "github.com/orian/clickhouse-sql-parser/parser"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/analysis"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/querylimits"
-)
-
-type Notice struct {
- Message string `json:"message"`
- Start int `json:"start"`
- End int `json:"end"`
-}
-
-type noticeCollector struct {
- query string
- notices []Notice
- seen map[[2]int]bool
-}
-
-func newNoticeCollector(query string) *noticeCollector {
- return &noticeCollector{query: query, seen: make(map[[2]int]bool)}
-}
-
-func (c *noticeCollector) add(message string, start, end int) {
- if start < 0 || end > len(c.query) || start >= end {
- return
- }
- if start > 0 && end < len(c.query) && (c.query[start-1] == '`' && c.query[end] == '`' || c.query[start-1] == '"' && c.query[end] == '"') {
- start--
- end++
- }
- if len(c.notices) >= querylimits.MaxNotices || c.seen[[2]int{start, end}] {
- return
- }
- c.seen[[2]int{start, end}] = true
- c.notices = append(c.notices, Notice{Message: message, Start: start, End: end})
-}
-
-func displayType(raw string) string {
- switch strings.ToLower(raw) {
- case "string":
- return "String"
- case "integer", "int":
- return "Integer"
- case "float", "numeric":
- return "Float"
- case "boolean", "bool":
- return "Boolean"
- case "datetime":
- return "DateTime"
- case "date":
- return "Date"
- case "uuid":
- return "UUID"
- case "json":
- return "JSON"
- case "array":
- return "Array"
- case "tuple":
- return "Tuple"
- case "decimal":
- return "Decimal"
- default:
- return ""
- }
-}
-
-func (c *noticeCollector) addField(ident *clickhouse.Ident, rawType string) {
- if fieldType := displayType(rawType); fieldType != "" {
- c.add(fmt.Sprintf("Field '%s' is of type '%s'", ident.Name, fieldType), int(ident.Pos()), int(ident.End()))
- }
-}
-
-func (c *noticeCollector) addProperty(ident *clickhouse.Ident, namespace string, properties *catalog.Index) {
- field, ok := properties.Exact(ident.Name)
- if !ok {
- return
- }
- fieldType := displayType(field.Type)
- if fieldType == "" {
- return
- }
- owner := namespace
- if strings.HasPrefix(owner, "group:") {
- owner = "group"
- }
- c.add(fmt.Sprintf("%s property '%s' is of type '%s'", strings.ToUpper(owner[:1])+owner[1:], ident.Name, fieldType), int(ident.Pos()), int(ident.End()))
-}
-
-func (c *noticeCollector) sorted() []Notice {
- sort.Slice(c.notices, func(i, j int) bool {
- if c.notices[i].Start != c.notices[j].Start {
- return c.notices[i].Start < c.notices[j].Start
- }
- return c.notices[i].End < c.notices[j].End
- })
- return c.notices
-}
-
-func collectNotices(schema *catalog.PreparedCatalog, document *analysis.Document, query string) []Notice {
- if document.LimitError() != nil {
- return nil
- }
- collector := newNoticeCollector(query)
- for statement := range document.Statements() {
- for table := range statement.ResolvedTables() {
- message := fmt.Sprintf("Table '%s'", table.Name)
- if table.CTE {
- message = fmt.Sprintf("Table '%s' is a common table expression", table.Name)
- } else if table.Canonical != table.Name {
- message = fmt.Sprintf("Table '%s' refers to '%s'", table.Name, table.Canonical)
- }
- collector.add(message, table.Start, table.End)
- }
- ignoredIdents := ignoredIdentifierNodes(statement)
- statement.Walk(func(node clickhouse.Expr) bool {
- if len(collector.notices) >= querylimits.MaxNotices || document.LimitError() != nil {
- return false
- }
- switch typed := node.(type) {
- case *clickhouse.NestedIdentifier:
- if typed.DotIdent == nil {
- return true
- }
- ignoredIdents[typed.Ident] = true
- ignoredIdents[typed.DotIdent] = true
- bindings := statement.BindingsAt(int(node.Pos()), int(node.End()))
- if resolved, ok := bindings.UnambiguousRelation(typed.Ident.Name); ok {
- if field, ok := resolved.ResolvedField(typed.DotIdent.Name); ok {
- collector.addField(typed.DotIdent, field.Type)
- }
- }
- case *clickhouse.Path:
- if len(typed.Fields) < 2 {
- return true
- }
- for _, field := range typed.Fields {
- ignoredIdents[field] = true
- }
- bindings := statement.BindingsAt(int(node.Pos()), int(node.End()))
- parts := make([]string, len(typed.Fields))
- for index, field := range typed.Fields {
- parts[index] = field.Name
- }
- target := bindings.Traversal(parts[:len(parts)-1])
- if target.Explicit {
- if target.Valid && target.PropertyNamespace != "" {
- propertyAt := len(typed.Fields) - 1
- if target.HasProperty {
- propertyAt = target.PropertyAt
- }
- collector.addProperty(typed.Fields[propertyAt], target.PropertyNamespace, schema.Properties(target.PropertyNamespace))
- } else if target.Valid && target.Fields != nil {
- if field, ok := target.Fields.Exact(typed.Fields[len(typed.Fields)-1].Name); ok {
- collector.addField(typed.Fields[len(typed.Fields)-1], field.Type)
- }
- }
- return true
- }
- if namespace, ok := bindings.PropertyNamespace(parts); ok {
- collector.addProperty(typed.Fields[len(typed.Fields)-1], namespace, schema.Properties(namespace))
- return true
- }
- if resolved, ok := bindings.UnambiguousRelation(typed.Fields[0].Name); ok {
- if field, ok := resolved.ResolvedField(typed.Fields[1].Name); ok {
- collector.addField(typed.Fields[1], field.Type)
- }
- }
- case *clickhouse.Ident:
- if ignoredIdents[typed] || typed.Name == "*" || analysis.IsBooleanLiteral(typed) {
- return true
- }
- bindings := statement.BindingsAt(int(node.Pos()), int(node.End()))
- if field, ok := bindings.ResolvedSelectAlias(typed.Name); ok {
- collector.addField(typed, field.Type)
- } else if _, exists := bindings.SelectAlias(typed.Name); !exists {
- if field, ok := bindings.ResolvedField(typed.Name); ok {
- collector.addField(typed, field.Type)
- }
- }
- }
- return true
- })
- if len(collector.notices) >= querylimits.MaxNotices || document.LimitError() != nil {
- break
- }
- }
- return collector.sorted()
-}
diff --git a/services/hogql-language-service/internal/validation/validation.go b/services/hogql-language-service/internal/validation/validation.go
deleted file mode 100644
index 53473ef9e646..000000000000
--- a/services/hogql-language-service/internal/validation/validation.go
+++ /dev/null
@@ -1,518 +0,0 @@
-package validation
-
-import (
- "errors"
- "fmt"
- "iter"
- "slices"
- "sort"
- "strings"
- "time"
- "unicode/utf8"
-
- clickhouse "github.com/orian/clickhouse-sql-parser/parser"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/analysis"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/querylimits"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/textposition"
-)
-
-type Suggestion struct {
- Label string `json:"label"`
- Distance int `json:"distance"`
-}
-
-type Diagnostic struct {
- Code string `json:"code"`
- Message string `json:"message"`
- Start int `json:"start"`
- End int `json:"end"`
- Suggestions []Suggestion `json:"suggestions,omitempty"`
-}
-
-type Result struct {
- Valid bool `json:"valid"`
- Diagnostics []Diagnostic `json:"diagnostics"`
- Notices []Notice `json:"notices"`
- TableNames []string `json:"tableNames"`
- DurationMicros int64 `json:"durationMicros"`
-}
-
-func Validate(schema *catalog.PreparedCatalog, query string) Result {
- started := time.Now()
- document, err := analysis.Analyze(schema, query)
- if err != nil {
- code := "syntax_error"
- if errors.Is(err, querylimits.ErrQueryTooLarge) || errors.Is(err, querylimits.ErrQueryTooDeep) {
- code = "query_limit"
- }
- return result([]Diagnostic{{Code: code, Message: err.Error(), Start: 0, End: len(query)}}, nil, nil, started)
- }
-
- var diagnostics []Diagnostic
- var referencedTableNames []string
- seenTableNames := map[string]bool{}
- for statement := range document.Statements() {
- for table := range statement.Tables() {
- if !seenTableNames[table.Name] {
- referencedTableNames = append(referencedTableNames, table.Name)
- seenTableNames[table.Name] = true
- }
- if !table.Known && len(diagnostics) < querylimits.MaxDiagnostics {
- diagnostics = append(diagnostics, Diagnostic{
- Code: "unknown_table", Message: fmt.Sprintf("Unknown table %q", table.Name), Start: table.Start, End: table.End,
- Suggestions: closestTables(table.Name, schema, 5),
- })
- }
- }
- for source := range statement.DuplicateSources() {
- if len(diagnostics) >= querylimits.MaxDiagnostics {
- break
- }
- diagnostics = append(diagnostics, Diagnostic{
- Code: "duplicate_table",
- Message: fmt.Sprintf("Table name %q is used more than once. Use a distinct alias for each table.", source.Qualifier()),
- Start: source.Start(),
- End: source.End(),
- })
- }
- ignoredIdents := ignoredIdentifierNodes(statement)
- seen := map[string]bool{}
- statement.Walk(func(node clickhouse.Expr) bool {
- switch typed := node.(type) {
- case *clickhouse.NestedIdentifier:
- if typed.DotIdent == nil {
- return true
- }
- ignoredIdents[typed.DotIdent] = true
- bindings := statement.BindingsAt(int(node.Pos()), int(node.End()))
- if binding, ok := bindings.Relation(typed.Ident.Name); ok {
- ignoredIdents[typed.Ident] = true
- if typed.DotIdent.Name != "*" {
- validateField(&diagnostics, seen, binding, typed.DotIdent, document)
- }
- }
- case *clickhouse.Path:
- if len(typed.Fields) < 2 {
- return true
- }
- bindings := statement.BindingsAt(int(node.Pos()), int(node.End()))
- if bindings.Len() == 0 {
- return true
- }
- parts := make([]string, len(typed.Fields))
- for index, field := range typed.Fields {
- parts[index] = field.Name
- }
- target := bindings.Traversal(parts[:len(parts)-1])
- if target.Explicit {
- for _, field := range typed.Fields {
- ignoredIdents[field] = true
- }
- if target.Failed {
- failedAt := min(target.FailureAt, len(typed.Fields)-1)
- validateUnknownIndexedField(&diagnostics, seen, target.Fields, typed.Fields[failedAt], document)
- } else if target.Valid && target.PropertyNamespace != "" {
- propertyAt := len(typed.Fields) - 1
- if target.HasProperty {
- propertyAt = target.PropertyAt
- }
- validateProperty(&diagnostics, seen, schema.Properties(target.PropertyNamespace), typed.Fields[propertyAt])
- } else if target.Valid && target.Fields != nil {
- validateIndexedField(&diagnostics, seen, target.Fields, typed.Fields[len(typed.Fields)-1], document)
- }
- return true
- }
- if namespace, ok := bindings.PropertyNamespace(parts); ok {
- for _, field := range typed.Fields {
- ignoredIdents[field] = true
- }
- validateProperty(&diagnostics, seen, schema.Properties(namespace), typed.Fields[len(typed.Fields)-1])
- return true
- }
- if binding, ok := bindings.Relation(typed.Fields[0].Name); ok {
- for _, field := range typed.Fields {
- ignoredIdents[field] = true
- }
- validateField(&diagnostics, seen, binding, typed.Fields[1], document)
- } else {
- for _, field := range typed.Fields[1:] {
- ignoredIdents[field] = true
- }
- }
- case *clickhouse.Ident:
- if ignoredIdents[typed] || typed.Name == "*" || analysis.IsBooleanLiteral(typed) {
- return true
- }
- bindings := statement.BindingsAt(int(node.Pos()), int(node.End()))
- if bindings.Len() > 0 {
- validateUnqualifiedField(&diagnostics, seen, bindings, typed, document)
- }
- }
- return true
- })
- if document.LimitError() != nil {
- break
- }
- }
- if err := document.LimitError(); err != nil && len(diagnostics) < querylimits.MaxDiagnostics {
- diagnostics = append(diagnostics, Diagnostic{
- Code: "query_limit", Message: err.Error(), Start: 0, End: len(query),
- })
- }
- return result(diagnostics, collectNotices(schema, document, query), referencedTableNames, started)
-}
-
-func ValidateWithEncoding(schema *catalog.PreparedCatalog, query string, encoding textposition.Encoding) (Result, error) {
- if !encoding.Valid() {
- return Result{}, fmt.Errorf("unsupported position encoding %q", encoding)
- }
- result := Validate(schema, query)
- for index := range result.Diagnostics {
- start, err := textposition.FromByteOffset(query, result.Diagnostics[index].Start, encoding)
- if err != nil {
- return Result{}, err
- }
- end, err := textposition.FromByteOffset(query, result.Diagnostics[index].End, encoding)
- if err != nil {
- return Result{}, err
- }
- result.Diagnostics[index].Start = start
- result.Diagnostics[index].End = end
- }
- for index := range result.Notices {
- start, err := textposition.FromByteOffset(query, result.Notices[index].Start, encoding)
- if err != nil {
- return Result{}, err
- }
- end, err := textposition.FromByteOffset(query, result.Notices[index].End, encoding)
- if err != nil {
- return Result{}, err
- }
- result.Notices[index].Start = start
- result.Notices[index].End = end
- }
- return result, nil
-}
-
-func ignoredIdentifierNodes(statement *analysis.Statement) map[*clickhouse.Ident]bool {
- ignoredIdents := map[*clickhouse.Ident]bool{}
- statement.Walk(func(node clickhouse.Expr) bool {
- switch typed := node.(type) {
- case *clickhouse.TableIdentifier:
- ignoredIdents[typed.Database] = true
- ignoredIdents[typed.Table] = true
- case *clickhouse.CTEStmt:
- if ident, ok := typed.Expr.(*clickhouse.Ident); ok {
- ignoredIdents[ident] = true
- }
- case *clickhouse.FunctionExpr:
- ignoredIdents[typed.Name] = true
- case *clickhouse.IntervalExpr:
- ignoredIdents[typed.Unit] = true
- case *clickhouse.IntervalFrom:
- ignoredIdents[typed.Interval] = true
- case *clickhouse.SelectItem:
- ignoredIdents[typed.Alias] = true
- case *clickhouse.AliasExpr:
- if alias, ok := typed.Alias.(*clickhouse.Ident); ok {
- ignoredIdents[alias] = true
- }
- }
- return true
- })
- return ignoredIdents
-}
-
-func validateProperty(diagnostics *[]Diagnostic, seen map[string]bool, properties *catalog.Index, ident *clickhouse.Ident) {
- if len(*diagnostics) >= querylimits.MaxDiagnostics {
- return
- }
- if _, ok := properties.Exact(ident.Name); ok {
- return
- }
- key := fmt.Sprintf("%d:%d", ident.Pos(), ident.End())
- if seen[key] {
- return
- }
- seen[key] = true
- *diagnostics = append(*diagnostics, Diagnostic{
- Code: "unknown_property", Message: fmt.Sprintf("Unknown property %q", ident.Name), Start: int(ident.Pos()), End: int(ident.End()),
- Suggestions: closest(ident.Name, slices.Values(properties.Entries()), 5),
- })
-}
-
-func validateField(diagnostics *[]Diagnostic, seen map[string]bool, binding analysis.Relation, ident *clickhouse.Ident, document *analysis.Document) {
- if len(*diagnostics) >= querylimits.MaxDiagnostics || document.LimitError() != nil {
- return
- }
- if _, ok := binding.Field(ident.Name); ok {
- return
- }
- if document.LimitError() != nil {
- return
- }
- key := fmt.Sprintf("%d:%d", ident.Pos(), ident.End())
- if seen[key] {
- return
- }
- seen[key] = true
- *diagnostics = append(*diagnostics, Diagnostic{
- Code: "unknown_field", Message: fmt.Sprintf("Unknown field %q", ident.Name), Start: int(ident.Pos()), End: int(ident.End()),
- Suggestions: closest(ident.Name, binding.Fields(), 5),
- })
-}
-
-func validateIndexedField(diagnostics *[]Diagnostic, seen map[string]bool, fields *catalog.PreparedFields, ident *clickhouse.Ident, document *analysis.Document) {
- if _, ok := fields.Exact(ident.Name); ok {
- return
- }
- validateUnknownIndexedField(diagnostics, seen, fields, ident, document)
-}
-
-func validateUnknownIndexedField(diagnostics *[]Diagnostic, seen map[string]bool, fields *catalog.PreparedFields, ident *clickhouse.Ident, document *analysis.Document) {
- if len(*diagnostics) >= querylimits.MaxDiagnostics || document.LimitError() != nil {
- return
- }
- key := fmt.Sprintf("%d:%d", ident.Pos(), ident.End())
- if seen[key] {
- return
- }
- seen[key] = true
- var entries []catalog.Entry
- if fields != nil {
- entries = fields.Entries()
- }
- *diagnostics = append(*diagnostics, Diagnostic{
- Code: "unknown_field", Message: fmt.Sprintf("Unknown field %q", ident.Name), Start: int(ident.Pos()), End: int(ident.End()),
- Suggestions: closest(ident.Name, slices.Values(entries), 5),
- })
-}
-
-func validateUnqualifiedField(diagnostics *[]Diagnostic, seen map[string]bool, bindings analysis.Bindings, ident *clickhouse.Ident, document *analysis.Document) {
- if len(*diagnostics) >= querylimits.MaxDiagnostics || document.LimitError() != nil {
- return
- }
- if _, ok := bindings.SelectAlias(ident.Name); ok {
- return
- }
- uniqueTables := map[string]analysis.Relation{}
- for binding := range bindings.UniqueRelations() {
- uniqueTables[binding.Name()] = binding
- if _, ok := binding.Field(ident.Name); ok {
- return
- }
- if document.LimitError() != nil {
- return
- }
- }
- candidates := slices.Collect(bindings.SelectAliases(""))
- for _, binding := range uniqueTables {
- candidates = slices.AppendSeq(candidates, binding.Fields())
- if document.LimitError() != nil {
- return
- }
- }
- key := fmt.Sprintf("%d:%d", ident.Pos(), ident.End())
- if seen[key] {
- return
- }
- seen[key] = true
- *diagnostics = append(*diagnostics, Diagnostic{
- Code: "unknown_field", Message: fmt.Sprintf("Unknown field %q", ident.Name), Start: int(ident.Pos()), End: int(ident.End()),
- Suggestions: closest(ident.Name, slices.Values(candidates), 5),
- })
-}
-
-func closest(input string, candidates iter.Seq[catalog.Entry], limit int) []Suggestion {
- return closestBy(input, candidates, limit, func(entry catalog.Entry) string { return strings.ToLower(entry.Name) })
-}
-
-func closestTables(input string, schema *catalog.PreparedCatalog, limit int) []Suggestion {
- return closestBy(input, slices.Values(schema.TableSpellings().Entries()), limit, func(entry catalog.Entry) string {
- canonical, _ := schema.CanonicalTableName(entry.Name)
- return canonical
- })
-}
-
-func closestBy(input string, candidates iter.Seq[catalog.Entry], limit int, identity func(catalog.Entry) string) []Suggestion {
- if len(input) > querylimits.MaxSuggestionInputBytes {
- return nil
- }
- lowerInput := strings.ToLower(input)
- leftLength := utf8.RuneCountInString(lowerInput)
- threshold := min(4, max(2, leftLength/3))
- best := make([]Suggestion, 0, limit)
- workspace := levenshteinWorkspace{}
- for candidate := range candidates {
- if len(candidate.Name) > querylimits.MaxSuggestionInputBytes {
- continue
- }
- lowerCandidate := strings.ToLower(candidate.Name)
- rightLength := utf8.RuneCountInString(lowerCandidate)
- lengthDifference := max(leftLength, rightLength) - min(leftLength, rightLength)
- prefix := strings.HasPrefix(lowerCandidate, lowerInput)
- if lengthDifference > threshold && !prefix {
- continue
- }
- distance := lengthDifference
- if !prefix {
- distance = workspace.distance(lowerInput, lowerCandidate, threshold)
- }
- if distance > threshold {
- continue
- }
- suggestion := Suggestion{Label: candidate.Name, Distance: distance}
- duplicate := -1
- candidateIdentity := identity(candidate)
- for index, existing := range best {
- existingIdentity := identity(catalog.Entry{Name: existing.Label})
- if existingIdentity == candidateIdentity {
- duplicate = index
- break
- }
- }
- if duplicate >= 0 {
- if suggestionLess(suggestion, best[duplicate]) {
- best[duplicate] = suggestion
- sortSuggestions(best)
- }
- continue
- }
- if len(best) < limit {
- best = append(best, suggestion)
- sortSuggestions(best)
- } else if suggestionLess(suggestion, best[len(best)-1]) {
- best[len(best)-1] = suggestion
- sortSuggestions(best)
- }
- }
- return best
-}
-
-func sortSuggestions(suggestions []Suggestion) {
- sort.Slice(suggestions, func(left, right int) bool {
- return suggestionLess(suggestions[left], suggestions[right])
- })
-}
-
-func suggestionLess(left, right Suggestion) bool {
- if left.Distance != right.Distance {
- return left.Distance < right.Distance
- }
- return left.Label < right.Label
-}
-
-type levenshteinWorkspace struct {
- previous []int
- current []int
-}
-
-func (w *levenshteinWorkspace) distance(left, right string, limit int) int {
- if isASCII(left) && isASCII(right) {
- for len(left) > 0 && len(right) > 0 && left[0] == right[0] {
- left = left[1:]
- right = right[1:]
- }
- for len(left) > 0 && len(right) > 0 && left[len(left)-1] == right[len(right)-1] {
- left = left[:len(left)-1]
- right = right[:len(right)-1]
- }
- return w.distanceASCII(left, right, limit)
- }
- leftRunes := []rune(left)
- rightRunes := []rune(right)
- for len(leftRunes) > 0 && len(rightRunes) > 0 && leftRunes[0] == rightRunes[0] {
- leftRunes = leftRunes[1:]
- rightRunes = rightRunes[1:]
- }
- for len(leftRunes) > 0 && len(rightRunes) > 0 && leftRunes[len(leftRunes)-1] == rightRunes[len(rightRunes)-1] {
- leftRunes = leftRunes[:len(leftRunes)-1]
- rightRunes = rightRunes[:len(rightRunes)-1]
- }
- return w.distanceRunes(leftRunes, rightRunes, limit)
-}
-
-func (w *levenshteinWorkspace) distanceASCII(left, right string, limit int) int {
- w.resize(len(right) + 1)
- for index := range len(right) + 1 {
- w.previous[index] = index
- }
- for leftIndex := range len(left) {
- w.current[0] = leftIndex + 1
- rowMinimum := w.current[0]
- for rightIndex := range len(right) {
- cost := 1
- if left[leftIndex] == right[rightIndex] {
- cost = 0
- }
- w.current[rightIndex+1] = min(w.current[rightIndex]+1, w.previous[rightIndex+1]+1, w.previous[rightIndex]+cost)
- rowMinimum = min(rowMinimum, w.current[rightIndex+1])
- }
- if rowMinimum > limit {
- return limit + 1
- }
- w.previous, w.current = w.current, w.previous
- }
- return w.previous[len(right)]
-}
-
-func (w *levenshteinWorkspace) distanceRunes(left, right []rune, limit int) int {
- w.resize(len(right) + 1)
- for index := range len(right) + 1 {
- w.previous[index] = index
- }
- for leftIndex, leftRune := range left {
- w.current[0] = leftIndex + 1
- rowMinimum := w.current[0]
- for rightIndex, rightRune := range right {
- cost := 1
- if leftRune == rightRune {
- cost = 0
- }
- w.current[rightIndex+1] = min(w.current[rightIndex]+1, w.previous[rightIndex+1]+1, w.previous[rightIndex]+cost)
- rowMinimum = min(rowMinimum, w.current[rightIndex+1])
- }
- if rowMinimum > limit {
- return limit + 1
- }
- w.previous, w.current = w.current, w.previous
- }
- return w.previous[len(right)]
-}
-
-func (w *levenshteinWorkspace) resize(size int) {
- if cap(w.previous) < size {
- w.previous = make([]int, size)
- w.current = make([]int, size)
- return
- }
- w.previous = w.previous[:size]
- w.current = w.current[:size]
-}
-
-func isASCII(value string) bool {
- for index := range len(value) {
- if value[index] >= utf8.RuneSelf {
- return false
- }
- }
- return true
-}
-
-func result(diagnostics []Diagnostic, notices []Notice, tableNames []string, started time.Time) Result {
- if diagnostics == nil {
- diagnostics = []Diagnostic{}
- }
- if tableNames == nil {
- tableNames = []string{}
- }
- if notices == nil {
- notices = []Notice{}
- }
- return Result{
- Valid: len(diagnostics) == 0, Diagnostics: diagnostics, Notices: notices, TableNames: tableNames,
- DurationMicros: time.Since(started).Microseconds(),
- }
-}
diff --git a/services/hogql-language-service/internal/validation/validation_test.go b/services/hogql-language-service/internal/validation/validation_test.go
deleted file mode 100644
index 9b5eed7113d1..000000000000
--- a/services/hogql-language-service/internal/validation/validation_test.go
+++ /dev/null
@@ -1,784 +0,0 @@
-package validation
-
-import (
- "fmt"
- "strings"
- "testing"
-
- "github.com/PostHog/posthog/services/hogql-language-service/internal/catalog"
- "github.com/PostHog/posthog/services/hogql-language-service/internal/querylimits"
-)
-
-func schema() *catalog.PreparedCatalog {
- return catalog.Prepare(&catalog.Catalog{Tables: map[string]catalog.Table{
- "warehouse_orders": {Name: "warehouse_orders", Type: "data_warehouse", Fields: map[string]catalog.Field{
- "order_id": {Name: "order_id", Type: "string"},
- "amount": {Name: "amount", Type: "float"},
- }},
- "warehouse_people": {Name: "warehouse_people", Type: "data_warehouse", Fields: map[string]catalog.Field{
- "person_id": {Name: "person_id", Type: "string"},
- }},
- "postgres.synced.orders": {Name: "postgres.synced.orders", Type: "data_warehouse", Fields: map[string]catalog.Field{
- "synced_id": {Name: "synced_id", Type: "string"},
- }},
- "events": {Name: "events", Type: "posthog", Fields: map[string]catalog.Field{
- "event": {Name: "event", Type: "string"},
- "properties": {Name: "properties", Type: "json"},
- "timestamp": {Name: "timestamp", Type: "datetime"},
- "uuid": {Name: "uuid", Type: "uuid"},
- }},
- "persons": {Name: "persons", Type: "posthog", Fields: map[string]catalog.Field{"properties": {Name: "properties", Type: "json"}}},
- }, Properties: map[string][]catalog.Property{
- "event": {{Name: "$geo_city", ValueType: "String"}, {Name: "café", ValueType: "String"}, {Name: "amount", ValueType: "Numeric"}},
- "person": {{Name: "$geo_country", ValueType: "String"}},
- "session": {{Name: "$entry_current_url", ValueType: "String"}},
- "group:0": {{Name: "industry", ValueType: "String"}},
- }})
-}
-
-func traversalSchema() *catalog.PreparedCatalog {
- return catalog.Prepare(&catalog.Catalog{
- Tables: map[string]catalog.Table{
- "events": {Fields: map[string]catalog.Field{
- "event": {Type: "String"}, "person": {Type: "lazy", Relation: "person"},
- "session": {Type: "lazy", Relation: "session"},
- }},
- "sessions": {Fields: map[string]catalog.Field{"session_id": {Type: "String"}}},
- },
- Relations: map[string]catalog.RelationDefinition{
- "person": {Fields: map[string]catalog.Field{
- "email": {Type: "String"}, "properties": {Type: "JSON", PropertyNamespace: "person"},
- "manager": {Type: "lazy", Relation: "person"}, "payload": {Type: "JSON"},
- }},
- "session": {Table: "sessions"},
- },
- Properties: map[string][]catalog.Property{"person": {{Name: "plan", ValueType: "String"}}},
- })
-}
-
-func TestValidateTraversalRelations(t *testing.T) {
- for _, query := range []string{
- "SELECT e.person.email FROM events AS e",
- "SELECT person.email FROM events",
- "SELECT e.person.properties.plan FROM events AS e",
- "SELECT e.person.manager.email FROM events AS e",
- "SELECT e.person.properties.plan.nested FROM events AS e",
- "SELECT e.person.payload.unknown FROM events AS e",
- "SELECT e.session.session_id FROM events AS e",
- } {
- result := Validate(traversalSchema(), query)
- if !result.Valid || strings.Join(result.TableNames, ",") != "events" {
- t.Fatalf("query %q returned %#v", query, result)
- }
- }
- for _, test := range []struct {
- query, code, value string
- }{
- {"SELECT e.person.emali FROM events AS e", "unknown_field", "emali"},
- {"SELECT e.person.properties.paln FROM events AS e", "unknown_property", "paln"},
- {"SELECT e.person.properties.paln.nested FROM events AS e", "unknown_property", "paln"},
- {"SELECT e.person.missing.email FROM events AS e", "unknown_field", "missing"},
- } {
- result := Validate(traversalSchema(), test.query)
- if result.Valid || len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != test.code {
- t.Fatalf("query %q returned %#v", test.query, result)
- }
- if got := test.query[result.Diagnostics[0].Start:result.Diagnostics[0].End]; got != test.value {
- t.Fatalf("query %q diagnostic covered %q", test.query, got)
- }
- }
-}
-
-func TestValidateNotices(t *testing.T) {
- tests := []struct {
- name string
- query string
- schema *catalog.PreparedCatalog
- notices []struct{ source, message string }
- }{
- {
- name: "qualified fields and table",
- query: "SELECT e.event, e.timestamp FROM events AS e",
- notices: []struct{ source, message string }{
- {"event", "Field 'event' is of type 'String'"},
- {"timestamp", "Field 'timestamp' is of type 'DateTime'"},
- {"events", "Table 'events'"},
- },
- },
- {
- name: "cte projection",
- query: "WITH x AS (SELECT event AS kind FROM events) SELECT x.kind FROM x",
- notices: []struct{ source, message string }{
- {"event", "Field 'event' is of type 'String'"},
- {"events", "Table 'events'"},
- {"kind", "Field 'kind' is of type 'String'"},
- {"x", "Table 'x' is a common table expression"},
- },
- },
- {
- name: "select alias before duplicate",
- query: "SELECT event AS v, v, timestamp AS v FROM events",
- notices: []struct{ source, message string }{
- {"event", "Field 'event' is of type 'String'"},
- {"v", "Field 'v' is of type 'String'"},
- {"timestamp", "Field 'timestamp' is of type 'DateTime'"},
- {"events", "Table 'events'"},
- },
- },
- {
- name: "property",
- query: "SELECT e.properties.$geo_city FROM events AS e",
- notices: []struct{ source, message string }{
- {"$geo_city", "Event property '$geo_city' is of type 'String'"},
- {"events", "Table 'events'"},
- },
- },
- {
- name: "numeric property",
- query: "SELECT e.properties.amount FROM events AS e",
- notices: []struct{ source, message string }{
- {"amount", "Event property 'amount' is of type 'Float'"},
- {"events", "Table 'events'"},
- },
- },
- {
- name: "traversed property",
- query: "SELECT e.person.properties.plan FROM events AS e",
- schema: traversalSchema(),
- notices: []struct{ source, message string }{
- {"plan", "Person property 'plan' is of type 'String'"},
- {"events", "Table 'events'"},
- },
- },
- {
- name: "quoted field",
- query: "SELECT `event` FROM events",
- notices: []struct{ source, message string }{
- {"`event`", "Field 'event' is of type 'String'"},
- {"events", "Table 'events'"},
- },
- },
- {
- name: "double-quoted field",
- query: "SELECT \"event\" FROM events",
- notices: []struct{ source, message string }{
- {"\"event\"", "Field 'event' is of type 'String'"},
- {"events", "Table 'events'"},
- },
- },
- {
- name: "relation traversal",
- query: "SELECT e.person.email FROM events AS e",
- schema: traversalSchema(),
- notices: []struct{ source, message string }{
- {"email", "Field 'email' is of type 'String'"},
- {"events", "Table 'events'"},
- },
- },
- {
- name: "unicode offsets",
- query: "SELECT '🐱', event FROM events",
- notices: []struct{ source, message string }{
- {"event", "Field 'event' is of type 'String'"},
- {"events", "Table 'events'"},
- },
- },
- {
- name: "wildcard",
- query: "SELECT * FROM events",
- notices: []struct{ source, message string }{
- {"events", "Table 'events'"},
- },
- },
- {
- name: "unknown field",
- query: "SELECT missing FROM events",
- notices: []struct{ source, message string }{
- {"events", "Table 'events'"},
- },
- },
- {
- name: "unresolved qualifier",
- query: "SELECT missing.event FROM events",
- notices: []struct{ source, message string }{
- {"events", "Table 'events'"},
- },
- },
- }
- for _, test := range tests {
- t.Run(test.name, func(t *testing.T) {
- testSchema := test.schema
- if testSchema == nil {
- testSchema = schema()
- }
- result := Validate(testSchema, test.query)
- if len(result.Notices) != len(test.notices) {
- t.Fatalf("notices = %#v", result.Notices)
- }
- for index, expected := range test.notices {
- actual := result.Notices[index]
- if test.query[actual.Start:actual.End] != expected.source || actual.Message != expected.message {
- t.Fatalf("notice %d = %#v, want %q: %q", index, actual, expected.source, expected.message)
- }
- }
- })
- }
-}
-
-func TestValidateNoticesUseCanonicalTableAndSkipAmbiguousFields(t *testing.T) {
- aliased := catalog.Prepare(&catalog.Catalog{
- Tables: map[string]catalog.Table{"warehouse_orders": {
- Name: "warehouse_orders", Fields: map[string]catalog.Field{"order_id": {Type: "integer"}},
- }},
- TableAliases: map[string]string{"orders": "warehouse_orders"},
- Properties: map[string][]catalog.Property{},
- })
- query := "SELECT orders.order_id FROM orders"
- result := Validate(aliased, query)
- if !result.Valid || len(result.Notices) != 2 || result.Notices[0].Message != "Field 'order_id' is of type 'Integer'" || result.Notices[1].Message != "Table 'orders' refers to 'warehouse_orders'" {
- t.Fatalf("alias notices = %#v", result)
- }
- for _, query := range []string{
- "SELECT amount FROM warehouse_orders AS o JOIN warehouse_orders AS p ON o.order_id = p.order_id",
- "WITH x AS (SELECT event AS v, timestamp AS v FROM events) SELECT x.v FROM x",
- "SELECT event AS v, timestamp AS v FROM events ORDER BY v",
- } {
- checked := Validate(schema(), query)
- fieldAt := strings.LastIndex(query, "v")
- if strings.Contains(query, "amount") {
- fieldAt = strings.Index(query, "amount")
- } else if strings.Contains(query, "x.v") {
- fieldAt = strings.Index(query, "x.v") + len("x.")
- }
- for _, notice := range checked.Notices {
- if notice.Start == fieldAt {
- t.Fatalf("ambiguous field has notice in %q: %#v", query, notice)
- }
- }
- }
-}
-
-func TestTraversalRelationHopLimit(t *testing.T) {
- parts := make([]string, querylimits.MaxRelationTraversalHops+1)
- for index := range parts {
- parts[index] = "manager"
- }
- query := "SELECT e.person." + strings.Join(parts, ".") + ".email FROM events AS e"
- result := Validate(traversalSchema(), query)
- if result.Valid || len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != "query_limit" || result.Diagnostics[0].Message != querylimits.ErrRelationTraversalTooDeep.Error() {
- t.Fatalf("result = %#v", result)
- }
-}
-
-func TestValidateDoesNotShareBindingsAcrossStatements(t *testing.T) {
- result := Validate(schema(), "SELECT person_id FROM warehouse_orders; SELECT person_id FROM warehouse_people")
- if result.Valid || len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != "unknown_field" {
- t.Fatalf("result = %#v", result)
- }
-}
-
-func TestValidateDoesNotShareBindingsAcrossNestedQueries(t *testing.T) {
- result := Validate(schema(), "SELECT person_id FROM warehouse_orders WHERE order_id IN (SELECT person_id FROM warehouse_people)")
- if result.Valid || len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != "unknown_field" {
- t.Fatalf("result = %#v", result)
- }
-}
-
-func TestValidateNestedAliasShadowsOuterAlias(t *testing.T) {
- result := Validate(schema(), "SELECT o.person_id FROM warehouse_orders AS o WHERE order_id IN (SELECT o.person_id FROM warehouse_people AS o)")
- if result.Valid || len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != "unknown_field" {
- t.Fatalf("result = %#v", result)
- }
-}
-
-func TestValidateAllJoinedTables(t *testing.T) {
- result := Validate(schema(), "SELECT o.order_id, p.person_id FROM warehouse_orders AS o JOIN warehouse_people AS p ON o.order_id = p.person_id")
- if !result.Valid {
- t.Fatalf("result = %#v", result)
- }
-}
-
-func TestValidateRejectsQueriesOutsideResourceLimits(t *testing.T) {
- result := Validate(schema(), strings.Repeat("x", 64<<10+1))
- if result.Valid || len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != "query_limit" {
- t.Fatalf("result = %#v", result)
- }
-}
-
-func TestValidateCapsDiagnostics(t *testing.T) {
- fields := make([]string, 150)
- for index := range fields {
- fields[index] = "missing_" + strings.Repeat("x", index%10) + string(rune('a'+index%26))
- }
- result := Validate(schema(), "SELECT "+strings.Join(fields, ", ")+" FROM warehouse_orders")
- if len(result.Diagnostics) != 25 {
- t.Fatalf("diagnostics = %d", len(result.Diagnostics))
- }
-}
-
-func TestValidateCapsNoticesWithoutSuppressingDiagnostics(t *testing.T) {
- fields := strings.Repeat("event, ", querylimits.MaxNotices) + "event"
- valid := Validate(schema(), "SELECT "+fields+" FROM events")
- if !valid.Valid || len(valid.Notices) != querylimits.MaxNotices {
- t.Fatalf("valid result = %#v", valid)
- }
-
- invalid := Validate(schema(), "SELECT "+fields+", missing FROM events")
- if invalid.Valid || len(invalid.Diagnostics) != 1 || invalid.Diagnostics[0].Code != "unknown_field" || len(invalid.Notices) > querylimits.MaxNotices {
- t.Fatalf("invalid result = %#v", invalid)
- }
-}
-
-func TestValidateUnknownTableSuggestsVisibleMatch(t *testing.T) {
- result := Validate(schema(), "SELECT order_id FROM warehose_orders")
- if result.Valid || len(result.Diagnostics) != 1 {
- t.Fatalf("result = %#v", result)
- }
- diagnostic := result.Diagnostics[0]
- if diagnostic.Code != "unknown_table" || len(diagnostic.Suggestions) == 0 || diagnostic.Suggestions[0].Label != "warehouse_orders" {
- t.Fatalf("diagnostic = %#v", diagnostic)
- }
-}
-
-func TestValidateRelationNamesAreCaseSensitive(t *testing.T) {
- result := Validate(schema(), "SELECT properties FROM Events")
- if result.Valid || len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != "unknown_table" || result.Diagnostics[0].Start != len("SELECT properties FROM ") {
- t.Fatalf("wrong-case table result = %#v", result)
- }
-
- caseVariants := catalog.Prepare(&catalog.Catalog{Tables: map[string]catalog.Table{
- "events": {Name: "events", Fields: map[string]catalog.Field{
- "properties": {Name: "properties", Type: "json"},
- }},
- "Events": {Name: "Events", Fields: map[string]catalog.Field{
- "custom_field": {Name: "custom_field", Type: "string"},
- }},
- "persons": {Name: "persons", Fields: map[string]catalog.Field{
- "properties": {Name: "properties", Type: "json"},
- }},
- }, Properties: map[string][]catalog.Property{
- "event": {{Name: "$geo_city", ValueType: "String"}},
- "person": {{Name: "$geo_country", ValueType: "String"}},
- }})
-
- result = Validate(caseVariants, "SELECT events.properties, Events.custom_field FROM events JOIN Events ON 1 = 1")
- if !result.Valid || len(result.Diagnostics) != 0 || strings.Join(result.TableNames, ",") != "events,Events" {
- t.Fatalf("case-variant table result = %#v", result)
- }
-
- for _, test := range []struct{ query, suggestion string }{
- {"SELECT e.properties.$geo_cty FROM events AS e JOIN persons AS E ON 1 = 1", "$geo_city"},
- {"SELECT E.properties.$geo_contry FROM events AS e JOIN persons AS E ON 1 = 1", "$geo_country"},
- {"WITH t AS (SELECT properties FROM events), T AS (SELECT properties FROM persons) SELECT t.properties.$geo_cty FROM t JOIN T ON 1 = 1", "$geo_city"},
- {"WITH t AS (SELECT properties FROM events), T AS (SELECT properties FROM persons) SELECT T.properties.$geo_contry FROM t JOIN T ON 1 = 1", "$geo_country"},
- } {
- checked := Validate(caseVariants, test.query)
- if checked.Valid || len(checked.Diagnostics) != 1 || checked.Diagnostics[0].Code != "unknown_property" || len(checked.Diagnostics[0].Suggestions) == 0 || checked.Diagnostics[0].Suggestions[0].Label != test.suggestion {
- t.Fatalf("query %q returned %#v", test.query, checked)
- }
- }
-
- result = Validate(caseVariants, "WITH t AS (SELECT properties FROM events) SELECT properties FROM T")
- if result.Valid || len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != "unknown_table" || result.Diagnostics[0].Start != len("WITH t AS (SELECT properties FROM events) SELECT properties FROM ") {
- t.Fatalf("wrong-case CTE result = %#v", result)
- }
-}
-
-func TestValidateUnknownAliasedFieldSuggestsVisibleMatch(t *testing.T) {
- for _, test := range []struct{ query, suggestion string }{
- {"SELECT o.amuont FROM warehouse_orders AS o", "amount"},
- {"SELECT amount AS total FROM warehouse_orders ORDER BY totla", "total"},
- } {
- result := Validate(schema(), test.query)
- if result.Valid || len(result.Diagnostics) != 1 {
- t.Fatalf("query %q: result = %#v", test.query, result)
- }
- diagnostic := result.Diagnostics[0]
- if diagnostic.Code != "unknown_field" || len(diagnostic.Suggestions) == 0 || diagnostic.Suggestions[0].Label != test.suggestion || diagnostic.Suggestions[0].Distance != 2 {
- t.Fatalf("query %q: diagnostic = %#v", test.query, diagnostic)
- }
- }
-}
-
-func TestValidateAcceptsKnownFieldsAndFunctions(t *testing.T) {
- for _, test := range []struct {
- query string
- tableName string
- }{
- {query: "SELECT sum(o.amount), o.order_id FROM warehouse_orders AS o WHERE o.amount > 0", tableName: "warehouse_orders"},
- {query: "SELECT uuid FROM events WHERE event = '$pageview' AND timestamp > now() - interval 1 month", tableName: "events"},
- {query: "SELECT extract(month FROM timestamp) FROM events", tableName: "events"},
- {query: "SELECT properties.$GEO_CITY FROM events", tableName: "events"},
- {query: "SELECT s.kind FROM (SELECT event AS kind FROM events) AS s", tableName: "events"},
- {query: "WITH t AS (SELECT event AS `Σ` FROM events) SELECT t.`ς` FROM t", tableName: "events"},
- {query: "WITH t AS (SELECT event AS kind FROM events) SELECT s.kind FROM (SELECT * FROM t) AS s", tableName: "events"},
- {query: "SELECT amount AS total, total AS subtotal FROM warehouse_orders PREWHERE subtotal > 0 WHERE total > 0 GROUP BY total, subtotal HAVING total > 1 ORDER BY subtotal", tableName: "warehouse_orders"},
- {query: "SELECT event AS kind FROM events WHERE uuid IN (SELECT uuid AS kind FROM events WHERE kind != '') ORDER BY kind", tableName: "events"},
- {query: "SELECT s.subtotal FROM (SELECT amount AS total, total AS subtotal FROM warehouse_orders) AS s", tableName: "warehouse_orders"},
- {query: "SELECT amount AS amount FROM warehouse_orders ORDER BY amount", tableName: "warehouse_orders"},
- {query: "SELECT TRUE, false FROM events WHERE TRUE AND false = FALSE", tableName: "events"},
- {query: "SELECT CASE WHEN TRUE THEN false ELSE FALSE END AS enabled FROM events", tableName: "events"},
- {query: "WITH flags AS (SELECT TRUE AS enabled FROM events) SELECT enabled FROM flags WHERE enabled = FALSE", tableName: "events"},
- } {
- result := Validate(schema(), test.query)
- if !result.Valid || len(result.Diagnostics) != 0 {
- t.Fatalf("query %q returned %#v", test.query, result)
- }
- if len(result.TableNames) != 1 || result.TableNames[0] != test.tableName {
- t.Fatalf("query %q returned table names %#v", test.query, result.TableNames)
- }
- }
-}
-
-func TestValidateAcceptsHogQLQualifiedTable(t *testing.T) {
- result := Validate(schema(), "SELECT o.synced_id FROM postgres.synced.orders AS o")
- if !result.Valid || len(result.Diagnostics) != 0 {
- t.Fatalf("result = %#v", result)
- }
-}
-
-func TestValidatePreservesCollidingNormalizedTableReferences(t *testing.T) {
- fieldTable := func(name, field string) catalog.Table {
- return catalog.Table{Name: name, Type: "data_warehouse", Fields: map[string]catalog.Field{field: {Name: field, Type: "string"}}}
- }
- for _, test := range []struct {
- name string
- catalog *catalog.Catalog
- query string
- tableNames string
- }{
- {
- name: "distinct canonical tables across statements and Unicode bytes",
- catalog: &catalog.Catalog{Tables: map[string]catalog.Table{
- "a.b.c_d": fieldTable("a.b.c_d", "left_field"),
- "a.b_c.d": fieldTable("a.b_c.d", "right_field"),
- }},
- query: "SELECT 'café'; SELECT l.left_field, r.right_field FROM a.b.c_d AS l JOIN a.b_c.d AS r ON 1 = 1",
- tableNames: "a.b.c_d,a.b_c.d",
- },
- {
- name: "alias and canonical table",
- catalog: &catalog.Catalog{
- Tables: map[string]catalog.Table{
- "left_target": fieldTable("left_target", "left_field"),
- "a.b_c.d": fieldTable("a.b_c.d", "right_field"),
- },
- TableAliases: map[string]string{"a.b.c_d": "left_target"},
- },
- query: "SELECT l.left_field, r.right_field FROM a.b.c_d AS l JOIN a.b_c.d AS r ON 1 = 1",
- tableNames: "a.b.c_d,a.b_c.d",
- },
- {
- name: "same target aliases keep implicit qualifiers",
- catalog: &catalog.Catalog{
- Tables: map[string]catalog.Table{"target": fieldTable("target", "shared_field")},
- TableAliases: map[string]string{
- "a.b.c_d": "target",
- "a.b_c.d": "target",
- },
- },
- query: "SELECT a__b__c_d.shared_field, a__b_c__d.shared_field FROM a.b.c_d JOIN a.b_c.d ON 1 = 1",
- tableNames: "a.b.c_d,a.b_c.d",
- },
- {
- name: "normalized and unchanged names",
- catalog: &catalog.Catalog{Tables: map[string]catalog.Table{
- "a.b.c_d": fieldTable("a.b.c_d", "left_field"),
- "a.b_c_d": fieldTable("a.b_c_d", "right_field"),
- }},
- query: "SELECT l.left_field, r.right_field FROM a.b.c_d AS l JOIN a.b_c_d AS r ON 1 = 1",
- tableNames: "a.b.c_d,a.b_c_d",
- },
- {
- name: "quoted normalized spelling CTE does not shadow",
- catalog: &catalog.Catalog{Tables: map[string]catalog.Table{
- "a.b.c_d": fieldTable("a.b.c_d", "left_field"),
- }},
- query: "WITH `a.b_c_d` AS (SELECT 1 AS cte_field) SELECT a__b__c_d.left_field FROM a.b.c_d",
- tableNames: "a.b.c_d",
- },
- } {
- t.Run(test.name, func(t *testing.T) {
- test.catalog.Properties = map[string][]catalog.Property{}
- result := Validate(catalog.Prepare(test.catalog), test.query)
- if !result.Valid || strings.Join(result.TableNames, ",") != test.tableNames {
- t.Fatalf("result = %#v", result)
- }
- })
- }
-}
-
-func TestValidateCatalogTableAliasesPreserveQuerySpellingAndOccurrences(t *testing.T) {
- value := &catalog.Catalog{
- Tables: map[string]catalog.Table{
- "postgres.demo.orders": {Type: "data_warehouse", Fields: map[string]catalog.Field{"id": {Type: "integer"}}},
- "events": {Type: "posthog", Fields: map[string]catalog.Field{"uuid": {Type: "uuid"}}},
- },
- TableAliases: map[string]string{"demo_postgres_orders": "postgres.demo.orders"},
- Properties: map[string][]catalog.Property{},
- }
- prepared := catalog.Prepare(value)
-
- for _, query := range []string{
- "SELECT id FROM demo_postgres_orders",
- "SELECT a.id, b.id FROM demo_postgres_orders AS a JOIN postgres.demo.orders AS b ON a.id = b.id",
- } {
- result := Validate(prepared, query)
- if !result.Valid {
- t.Fatalf("query %q returned %#v", query, result)
- }
- }
- result := Validate(prepared, "SELECT a.id, b.id FROM demo_postgres_orders AS a JOIN postgres.demo.orders AS b ON a.id = b.id")
- if strings.Join(result.TableNames, ",") != "demo_postgres_orders,postgres.demo.orders" {
- t.Fatalf("table names = %#v", result.TableNames)
- }
-
- cte := Validate(prepared, "WITH demo_postgres_orders AS (SELECT uuid FROM events) SELECT c.uuid, o.id FROM demo_postgres_orders AS c JOIN postgres.demo.orders AS o ON 1 = 1")
- if !cte.Valid || strings.Join(cte.TableNames, ",") != "events,postgres.demo.orders" {
- t.Fatalf("CTE shadow result = %#v", cte)
- }
-}
-
-func TestUnknownTableSuggestionsDeduplicateAliasTargets(t *testing.T) {
- prepared := catalog.Prepare(&catalog.Catalog{
- Tables: map[string]catalog.Table{
- "postgres.demo.orders": {Fields: map[string]catalog.Field{}},
- },
- TableAliases: map[string]string{
- "demo_postgres_orders": "postgres.demo.orders",
- "legacy_postgres_orders": "postgres.demo.orders",
- },
- Properties: map[string][]catalog.Property{},
- })
- result := Validate(prepared, "SELECT * FROM demo_postgres_order")
- if result.Valid || len(result.Diagnostics) != 1 || len(result.Diagnostics[0].Suggestions) != 1 || result.Diagnostics[0].Suggestions[0].Label != "demo_postgres_orders" {
- t.Fatalf("result = %#v", result)
- }
-}
-
-func TestValidateDuplicateTableNames(t *testing.T) {
- for _, test := range []struct {
- name, query, qualifier string
- valid bool
- }{
- {name: "unaliased", query: "SELECT events.properties FROM events JOIN events ON 1 = 1", qualifier: "events"},
- {name: "duplicate alias", query: "SELECT e.properties FROM events AS e JOIN persons AS e ON 1 = 1", qualifier: "e"},
- {name: "derived property", query: "WITH t AS (SELECT events.properties FROM events JOIN events ON 1 = 1) SELECT t.properties.$geo_cty FROM t", qualifier: "events"},
- {name: "distinct aliases", query: "SELECT e.properties FROM events AS e JOIN events AS other ON 1 = 1", valid: true},
- {name: "case-sensitive aliases", query: "SELECT e.properties FROM events AS e JOIN events AS E ON 1 = 1", valid: true},
- {name: "nested reuse", query: "SELECT e.properties FROM events AS e WHERE uuid IN (SELECT properties FROM persons AS e)", valid: true},
- } {
- t.Run(test.name, func(t *testing.T) {
- result := Validate(schema(), test.query)
- if test.valid {
- if !result.Valid || len(result.Diagnostics) != 0 {
- t.Fatalf("result = %#v", result)
- }
- return
- }
- if result.Valid || len(result.Diagnostics) != 1 {
- t.Fatalf("result = %#v", result)
- }
- diagnostic := result.Diagnostics[0]
- if diagnostic.Code != "duplicate_table" || diagnostic.Message != fmt.Sprintf("Table name %q is used more than once. Use a distinct alias for each table.", test.qualifier) {
- t.Fatalf("diagnostic = %#v", diagnostic)
- }
- secondJoin := strings.Index(test.query, " JOIN ") + len(" JOIN ")
- if diagnostic.Start != secondJoin || diagnostic.End <= diagnostic.Start {
- t.Fatalf("diagnostic span = %d:%d, want start %d", diagnostic.Start, diagnostic.End, secondJoin)
- }
- })
- }
-}
-
-func TestValidateCommonTableExpressions(t *testing.T) {
- tests := []struct {
- name string
- query string
- tableNames []string
- }{
- {
- name: "basic",
- query: "WITH x AS (SELECT event FROM events) SELECT * FROM x",
- tableNames: []string{"events"},
- },
- {
- name: "projected alias",
- query: "WITH x AS (SELECT event AS kind FROM events) SELECT x.kind FROM x",
- tableNames: []string{"events"},
- },
- {
- name: "wildcard projection",
- query: "WITH x AS (SELECT * FROM events) SELECT x.uuid FROM x",
- tableNames: []string{"events"},
- },
- {
- name: "qualified wildcard projection",
- query: "WITH x AS (SELECT e.* FROM events AS e) SELECT x.uuid FROM x",
- tableNames: []string{"events"},
- },
- {
- name: "chained",
- query: "WITH x AS (SELECT event AS kind FROM events), y AS (SELECT kind FROM x) SELECT y.kind FROM y",
- tableNames: []string{"events"},
- },
- {
- name: "shadows physical table",
- query: "WITH events AS (SELECT person_id FROM warehouse_people) SELECT events.person_id FROM events",
- tableNames: []string{"warehouse_people"},
- },
- {
- name: "definition does not reference itself",
- query: "WITH events AS (SELECT event FROM events) SELECT events.event FROM events",
- tableNames: []string{"events"},
- },
- {
- name: "nested definition shadows outer definition",
- query: "WITH x AS (SELECT event AS outer_field FROM events), y AS (WITH x AS (SELECT person_id AS inner_field FROM warehouse_people) SELECT inner_field FROM x) SELECT y.inner_field FROM y",
- tableNames: []string{"events", "warehouse_people"},
- },
- }
-
- for _, test := range tests {
- t.Run(test.name, func(t *testing.T) {
- result := Validate(schema(), test.query)
- if !result.Valid || len(result.Diagnostics) != 0 {
- t.Fatalf("result = %#v", result)
- }
- if strings.Join(result.TableNames, ",") != strings.Join(test.tableNames, ",") {
- t.Fatalf("table names = %#v", result.TableNames)
- }
- })
- }
-}
-
-func TestValidateRejectsUnknownCommonTableExpressionField(t *testing.T) {
- for _, query := range []string{
- "WITH x AS (SELECT event AS kind FROM events) SELECT x.timestamp FROM x",
- "SELECT x.timestamp FROM (SELECT event AS kind FROM events) AS x",
- "SELECT timestamp FROM (SELECT event AS kind FROM events) AS x",
- } {
- result := Validate(schema(), query)
- if result.Valid || len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != "unknown_field" {
- t.Fatalf("query %q: result = %#v", query, result)
- }
- if len(result.TableNames) != 1 || result.TableNames[0] != "events" {
- t.Fatalf("table names = %#v", result.TableNames)
- }
- }
-}
-
-func TestValidateRejectsUnknownFields(t *testing.T) {
- for _, query := range []string{
- "SELECT missing.event FROM events",
- "SELECT missing.properties.value FROM events",
- "SELECT missing.* FROM events",
- "SELECT total, amount AS total FROM warehouse_orders",
- "SELECT total AS total FROM warehouse_orders",
- "SELECT amount AS total FROM warehouse_orders JOIN events ON total = 1",
- "SELECT amount AS total FROM warehouse_orders WHERE order_id IN (SELECT total FROM events)",
- "SELECT total FROM warehouse_orders WHERE order_id IN (SELECT event AS total FROM events)",
- "SELECT amount AS total FROM warehouse_orders; SELECT total FROM events",
- "SELECT amount AS total FROM warehouse_orders UNION ALL SELECT total FROM warehouse_orders",
- "WITH t AS (SELECT total FROM warehouse_orders) SELECT amount AS total FROM warehouse_orders",
- "SELECT event AS kind FROM events ORDER BY events.kind",
- "SELECT amount AS Total FROM warehouse_orders ORDER BY total",
- `SELECT "TRUE" FROM events`,
- "SELECT events.FALSE FROM events",
- } {
- result := Validate(schema(), query)
- if result.Valid || len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != "unknown_field" {
- t.Fatalf("query %q returned %#v", query, result)
- }
- }
-}
-
-func TestValidateBoundsCommonTableExpressionProjectionExpansion(t *testing.T) {
- ctes := []string{"c0 AS (SELECT * FROM events)"}
- for index := 1; index < 14; index++ {
- ctes = append(ctes, fmt.Sprintf(
- "c%d AS (SELECT left_side.*, right_side.* FROM c%d AS left_side JOIN c%d AS right_side ON 1 = 1)",
- index, index-1, index-1,
- ))
- }
- withinBudget := "WITH " + strings.Join(ctes[:12], ", ") + " SELECT event FROM c11"
- for _, test := range []struct {
- name, query string
- valid bool
- }{
- {name: "single statement within budget", query: withinBudget, valid: true},
- {name: "single statement exceeds budget", query: "WITH " + strings.Join(ctes, ", ") + " SELECT missing FROM c13"},
- {name: "shared budget stops before later statements", query: withinBudget + "; " + withinBudget + "; SELECT person_id FROM warehouse_people"},
- } {
- t.Run(test.name, func(t *testing.T) {
- result := Validate(schema(), test.query)
- if result.Valid != test.valid {
- t.Fatalf("result = %#v", result)
- }
- if !test.valid && (len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != "query_limit") {
- t.Fatalf("result = %#v", result)
- }
- if strings.Join(result.TableNames, ",") != "events" {
- t.Fatalf("table names = %#v", result.TableNames)
- }
- })
- }
-}
-
-func TestValidateBoundsFieldLookupWork(t *testing.T) {
- ctes := make([]string, 128)
- from := "c0"
- for index := range ctes {
- ctes[index] = fmt.Sprintf("c%d AS (SELECT event FROM events)", index)
- if index > 0 {
- from += fmt.Sprintf(" JOIN c%d ON 1 = 1", index)
- }
- }
- ctes = append(ctes, "result AS (SELECT "+strings.Repeat("unknown", 1500)+", c0.event FROM "+from+")")
- query := "WITH " + strings.Join(ctes, ", ") + " SELECT result.event FROM result"
- result := Validate(schema(), query)
- if result.Valid || len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != "query_limit" || result.Diagnostics[0].Message != querylimits.ErrFieldLookupTooLarge.Error() {
- t.Fatalf("result = %#v", result)
- }
-}
-
-func TestValidatePropertiesAcrossGenericNamespaces(t *testing.T) {
- tests := []struct {
- query string
- suggestion string
- }{
- {query: "SELECT e.properties.$geo_cty FROM events AS e", suggestion: "$geo_city"},
- {query: "SELECT persons.properties.$geo_contry FROM persons", suggestion: "$geo_country"},
- {query: "SELECT session.properties.$entry_curent_url FROM events", suggestion: "$entry_current_url"},
- {query: "SELECT group_0.properties.indstry FROM events", suggestion: "industry"},
- {query: "SELECT properties.cafe FROM events", suggestion: "café"},
- {query: "WITH t AS (SELECT 1 AS x) SELECT properties.$geo_cty FROM events JOIN t ON 1 = 1", suggestion: "$geo_city"},
- {query: "SELECT properties.$geo_cty FROM events JOIN (SELECT 1 AS x) AS t ON 1 = 1", suggestion: "$geo_city"},
- {query: "WITH t AS (SELECT properties AS attrs FROM events) SELECT properties.$geo_cty FROM events JOIN t ON 1 = 1", suggestion: "$geo_city"},
- {query: "WITH recent AS (SELECT properties FROM events) SELECT recent.properties.$geo_cty FROM recent", suggestion: "$geo_city"},
- {query: "WITH recent AS (SELECT properties FROM events) SELECT properties.$geo_cty FROM recent", suggestion: "$geo_city"},
- {query: "SELECT recent.properties.$geo_cty FROM (SELECT properties FROM events) AS recent", suggestion: "$geo_city"},
- {query: "WITH recent AS (SELECT properties AS props FROM events) SELECT recent.props.$geo_cty FROM recent", suggestion: "$geo_city"},
- {query: "WITH recent AS (SELECT properties AS props FROM events) SELECT props.$geo_cty FROM recent", suggestion: "$geo_city"},
- {query: "WITH a AS (SELECT properties FROM persons), events AS (SELECT * FROM a) SELECT events.properties.$geo_contry FROM events", suggestion: "$geo_country"},
- {query: "SELECT properties AS props, props AS attrs, attrs.$geo_cty FROM events", suggestion: "$geo_city"},
- {query: "SELECT properties AS props, uuid AS props, props.$geo_cty FROM events", suggestion: ""},
- {query: "SELECT properties FROM events EXCEPT SELECT properties.$geo_contry FROM persons", suggestion: "$geo_country"},
- }
- for _, test := range tests {
- result := Validate(schema(), test.query)
- if test.suggestion == "" {
- if !result.Valid || len(result.Diagnostics) != 0 {
- t.Fatalf("query %q returned %#v", test.query, result)
- }
- continue
- }
- if result.Valid || len(result.Diagnostics) != 1 {
- t.Fatalf("query %q returned %#v", test.query, result)
- }
- diagnostic := result.Diagnostics[0]
- if diagnostic.Code != "unknown_property" || len(diagnostic.Suggestions) == 0 || diagnostic.Suggestions[0].Label != test.suggestion {
- t.Fatalf("query %q returned %#v", test.query, diagnostic)
- }
- }
-}