Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(ci): run product tests when only their data files change (#128208)

Georges-Antoine Assi6d2fa7c25dac
diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.yml
index daec6e598196..ca5369b1d73c 100644
--- a/.depot/workflows/ci-backend.yml
+++ b/.depot/workflows/ci-backend.yml
@@ -225,7 +225,7 @@ jobs:
# gate reads this output, so the label cannot reach them while the paths
# filter says false. Nothing under .github/ is in the backend filter, so
# a sharding or timing script change was untestable on its own PR.
- backend: ${{ contains(github.event.pull_request.labels.*.name, 'run-ci-backend') && 'true' || (steps.filter.outputs.backend || 'true') }}
+ backend: ${{ contains(github.event.pull_request.labels.*.name, 'run-ci-backend') && 'true' || ((steps.filter.outputs.backend || 'true') == 'true' || steps.filter.outputs.product_skills_read_at_runtime == 'true') && 'true' || 'false' }}
timing_scripts: ${{ fromJSON(toJSON(steps.filter.outputs)).timing_scripts }}
backend_files: ${{ steps.filter.outputs.backend_files }}
migrations: ${{ steps.filter.outputs.migrations || 'true' }}
@@ -397,6 +397,9 @@ jobs:
# files from `backend_files`.
- '!**/*.md'
- '!**/*.mdx'
+ product_skills_read_at_runtime:
+ - 'products/signals/skills/**'
+ - 'products/review_hog/skills/**'
legacy:
# Non-product backend code — when only products/ change,
# turbo-discover uses Turbo query affectedness to detect changed
diff --git a/.github/scripts/turbo-discover-selection.test.js b/.github/scripts/turbo-discover-selection.test.js
index a500ba5d7b21..a328944a367a 100644
--- a/.github/scripts/turbo-discover-selection.test.js
+++ b/.github/scripts/turbo-discover-selection.test.js
@@ -11,6 +11,7 @@ const {
decideSelection,
isDagsterOnlyLegacyEdit,
narrowedProducts,
+ productsWithUndeclaredInputChanges,
selectedShards,
} = require('./turbo-discover')
@@ -175,6 +176,26 @@ test('only Dagster Python that still exists counts as a Dagster-only legacy edit
}
})
+test('a changed non-Python test input selects its own product', () => {
+ const allProducts = new Set(['access-control', 'signals'])
+ const cases = [
+ [['products/access_control/backend/tests/__snapshots__/test_access_control.ambr'], ['access-control']],
+ [['products/signals/skills/authoring-scouts/SKILL.md'], ['signals']],
+ [['products/signals/backend/AGENTS.md'], []],
+ [['products/signals/frontend/Scouts.tsx'], []],
+ [['products/signals/backend/__pycache__/models.cpython-312.pyc'], []],
+ [['products/unregistered/backend/fixtures/rows.json'], []],
+ [['posthog/test/__snapshots__/test_team.ambr'], []],
+ ]
+ for (const [changedFiles, expected] of cases) {
+ assert.deepEqual(
+ productsWithUndeclaredInputChanges(changedFiles, allProducts),
+ expected,
+ JSON.stringify(changedFiles)
+ )
+ }
+})
+
test('applies the full-matrix budget to the selected seconds of each segment', () => {
// Work budget per shard is (target wall - segment overhead), same as the full
// matrix: Core ceil(5000 / (720 - 295)) = 12, Temporal ceil(900 / (720 - 182)) = 2.
diff --git a/.github/scripts/turbo-discover.js b/.github/scripts/turbo-discover.js
index a3fdee673379..30869e26f31f 100644
--- a/.github/scripts/turbo-discover.js
+++ b/.github/scripts/turbo-discover.js
@@ -384,6 +384,33 @@ function productOfFile(file) {
return rest === undefined ? null : product
}
+const UNDECLARED_PRODUCT_INPUT = /^products\/([^/]+)\/(backend|skills)\/(.+)$/
+
+function productsWithUndeclaredInputChanges(changedFiles, allProductSet) {
+ const products = new Set()
+ for (const file of changedFiles) {
+ const match = UNDECLARED_PRODUCT_INPUT.exec(file)
+ if (!match) {
+ continue
+ }
+ const [, directory, root, rest] = match
+ if (rest.includes('__pycache__/') || (root === 'backend' && rest.endsWith('.md'))) {
+ continue
+ }
+ const product = moduleToProduct(directory)
+ if (allProductSet.has(product)) {
+ products.add(product)
+ }
+ }
+ return [...products].sort()
+}
+
+function listChangedFiles(base, head) {
+ return execFileSync('git', ['diff', '--name-only', '--no-renames', base, head], TURBO_EXEC_OPTS)
+ .split('\n')
+ .filter(Boolean)
+}
+
// Collapse tach's file map ({ file: [files that import it] }) into
// product -> [products it imports]. Keys and values are product directory
// names (underscores); callers normalize to/from Turbo's dashed names. Every
@@ -1580,6 +1607,7 @@ module.exports = {
libImportName,
productsImportingModule,
coreFilesImportingModule,
+ productsWithUndeclaredInputChanges,
}
// --- Main ---
@@ -1765,6 +1793,32 @@ if (legacyChanged) {
}
}
+let undeclaredInputProductsAdded = []
+if (process.env.TURBO_SCM_BASE) {
+ let changedFiles
+ try {
+ changedFiles = listChangedFiles(process.env.TURBO_SCM_BASE, process.env.TURBO_SCM_HEAD || 'HEAD')
+ } catch (e) {
+ console.error(`changed-file listing failed: ${e.message}`)
+ process.exit(1)
+ }
+ const undeclaredInputProducts = productsWithUndeclaredInputChanges(changedFiles, allProductSet)
+ const unselected = undeclaredInputProducts.filter((p) => !products.includes(p))
+ if (unselected.length > 0) {
+ console.error(
+ `Products whose non-Python test inputs changed, which Turbo's affected query cannot see: ${JSON.stringify(unselected)}`
+ )
+ products = [...new Set([...products, ...unselected])].sort()
+ undeclaredInputProductsAdded = unselected
+ }
+ if (mustRunProducts !== null) {
+ mustRunProducts = [...new Set([...mustRunProducts, ...undeclaredInputProducts])].sort()
+ }
+ if (diffProducts !== null) {
+ diffProducts = [...new Set([...diffProducts, ...undeclaredInputProducts])].sort()
+ }
+}
+
// Kill switch: products named in the SKIP_PRODUCT_TESTS repo variable (comma-
// separated) are dropped from the matrix without a code change — use it to stop
// running, and blocking on, a product whose tests are temporarily too flaky.
@@ -1892,6 +1946,7 @@ const result = {
product_matrix_narrowed: productMatrixNarrowed,
product_count: products.length,
product_count_full: productCountBeforeNarrowing,
+ undeclared_input_products: undeclaredInputProductsAdded,
...runContext(),
},
}