The open-source agent harness - the runtime layer that turns an LLM into a working agent.
ci: publish Python SDK to PyPI from release.yml (#718)
Bhavesh Patele64be05a4e4d
diff --git a/.github/fern/generators.yml b/.github/fern/generators.ymlindex 63bde8ded..edd459b41 100644--- a/.github/fern/generators.yml+++ b/.github/fern/generators.yml@@ -109,7 +109,7 @@ groups:location: local-file-systempath: ../../python/trueforge_sdkconfig:- # Import name; PyPI install trueforge-sdk resolves via PEP 503.+ # Import + upload name. pip install trueforge-sdk resolves via PEP 503.package_name: trueforge_sdkoptional_auth: truestream_abstraction: truediff --git a/.github/workflows/generate-sdk.yaml b/.github/workflows/generate-sdk.yamlindex 0afde503b..2869b8512 100644--- a/.github/workflows/generate-sdk.yaml+++ b/.github/workflows/generate-sdk.yaml@@ -126,8 +126,11 @@ jobs:# Same entrypoint as local `pnpm sdk:generate` (re-runs openapi:write; idempotent).# App-token success = we can push; forks skip Fern so a flake cannot block the PR.+ # FERN_TOKEN is required so Python list() emits SyncPager (Fern README already documents it).- name: Generate OpenAPI document and SDKsif: steps.decide.outputs.needed == 'true' && steps.app-token.outcome == 'success'+ env:+ FERN_TOKEN: ${{ secrets.FERN_TOKEN }}run: pnpm sdk:generate# App-token success = writable head; forks cannot receive this commit.@@ -154,7 +157,7 @@ jobs:# Bot-authored sdk diffs need a changeset; the PR author never touches this commit.# Skip packages/trueforge-sdk/.fern — Fern metadata only, not a consumer-facing change.# changeset:sdk-regen is a no-op if the sdk is already named in a pending file.- # Python is not Changesets-published yet; do not add it here.+ # Python version mirrors TS on Version Packages; do not add a Python changeset here.if [ -n "$(git status --porcelain -- packages/trueforge-sdk ':!packages/trueforge-sdk/.fern')" ]; thenpnpm changeset:sdk-regenfidiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 6a975693e..e31b55f13 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -1,18 +1,20 @@-# npm publish for the four public packages.+# npm publish for the four public packages, plus PyPI for python/trueforge_sdk.#-# npm trusted publishers are bound to `release.yml`.-# Do not rename this file without updating every package on npmjs.com.+# npm and PyPI trusted publishers are bound to `release.yml`.+# Do not rename this file without updating every package on npmjs.com and PyPI.## On every push to main, changesets/action/select-mode chooses:# - pending `.changeset/*.md` → version job opens/updates the Version Packages PR-# (install only; that PR is gated by CI.yml)-# - none pending + unpublished versions → pack (build/test) + Windows npx smoke, then publish+# (install only; that PR is gated by CI.yml). When @truefoundry/trueforge-sdk+# moves, scripts/version.mjs mirrors that version into python/trueforge_sdk and rebakes Fern.+# - none pending + unpublished versions → pack (build/test) + Windows npx smoke,+# then npm publish and PyPI publish in parallel# Merge the Version Packages PR to publish. There is no separate publish Action# and no `v*` tag trigger.#-# Auth is GitHub OIDC (no NPM_TOKEN). An `_authToken` in user `.npmrc` disables-# pnpm's OIDC exchange, so the publish job must not set NPM_TOKEN / NODE_AUTH_TOKEN.-# id-token is granted only on the publish job (trusted-publishing best practice).+# Auth is GitHub OIDC (no NPM_TOKEN / PYPI_TOKEN). An `_authToken` in user `.npmrc`+# disables pnpm's OIDC exchange, so the npm publish job must not set NPM_TOKEN /+# NODE_AUTH_TOKEN. id-token is granted only on the publish jobs.name: Releaseon:@@ -91,6 +93,8 @@ jobs:- name: Version packagesid: versionuses: changesets/action/version@v2+ env:+ FERN_TOKEN: ${{ secrets.FERN_TOKEN }}with:script: pnpm run versionpush-with-git-cli: true@@ -238,6 +242,119 @@ jobs:echo "@truefoundry/trueforge was not in this publish set"fi+ # Sibling of npm publish (same gates). Skips when pyproject version is already on PyPI.+ publish-python:+ name: Publish Python SDK+ needs: [pack, windows-npx-smoke]+ runs-on: ubuntu-latest+ timeout-minutes: 20+ permissions:+ contents: read+ id-token: write+ steps:+ - name: Check out repo+ uses: actions/checkout@v7+ with:+ persist-credentials: false++ - name: Set up Python+ uses: actions/setup-python@v5+ with:+ python-version: '3.12'++ - name: Install build tools+ run: python -m pip install --upgrade pip build packaging++ - name: Plan PyPI versions+ id: version+ working-directory: python/trueforge_sdk+ run: |+ set -euo pipefail+ pypi_code() {+ curl -sS -L -o /dev/null -w '%{http_code}' "https://pypi.org/pypi/trueforge-sdk/${1}/json"+ }+ VERSION="$(python3 -c '+ import re, pathlib+ from packaging.version import Version+ text = pathlib.Path("pyproject.toml").read_text()+ match = re.search(r"(?m)^version\s*=\s*\"([^\"]+)\"", text)+ if match is None:+ raise SystemExit("missing version in pyproject.toml")+ print(Version(match.group(1)))+ ')"+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"++ code="$(pypi_code "$VERSION")"+ if [[ "$code" == "200" ]]; then+ echo "already=true" >> "$GITHUB_OUTPUT"+ echo "trueforge-sdk==$VERSION already on PyPI; skipping"+ elif [[ "$code" == "404" ]]; then+ echo "already=false" >> "$GITHUB_OUTPUT"+ else+ echo "Unexpected PyPI response HTTP $code for trueforge-sdk==$VERSION" >&2+ exit 1+ fi++ # Remove this 0.1.3 bootstrap block in a follow-up PR.+ # HTTP 200 means 0.1.3 is already on PyPI, so later rc publishes skip it.+ # HTTP 404 when the current version is not 0.1.3 means this is the first+ # upload: we publish 0.1.3 so pip does not install the 0.0.0 placeholder.+ # HTTP 404 when the current version is already 0.1.3 means the normal+ # publish step will upload it, so we must not build it twice.+ bootstrap_code="$(pypi_code "0.1.3")"+ if [[ "$bootstrap_code" == "200" ]]; then+ echo "bootstrap=false" >> "$GITHUB_OUTPUT"+ elif [[ "$bootstrap_code" == "404" && "$VERSION" != "0.1.3" ]]; then+ echo "bootstrap=true" >> "$GITHUB_OUTPUT"+ echo "trueforge-sdk==0.1.3 missing; will upload before $VERSION"+ elif [[ "$bootstrap_code" == "404" ]]; then+ echo "bootstrap=false" >> "$GITHUB_OUTPUT"+ else+ echo "Unexpected PyPI response HTTP $bootstrap_code for trueforge-sdk==0.1.3" >&2+ exit 1+ fi++ # Remove this 0.1.3 bootstrap block in a follow-up PR.+ # Rewrite pyproject.toml to version 0.1.3, build that wheel from today's+ # tree, then restore pyproject.toml so the next step can publish the real+ # rc (for example 0.2.0rc5).+ - name: Build 0.1.3 bootstrap+ if: steps.version.outputs.bootstrap == 'true'+ working-directory: python/trueforge_sdk+ run: |+ set -euo pipefail+ python3 -c '+ import re+ from pathlib import Path+ path = Path("pyproject.toml")+ text = path.read_text()+ next_text, count = re.subn(r"(?m)^version\s*=\s*\"[^\"]+\"", "version = \"0.1.3\"", text, count=1)+ if count != 1:+ raise SystemExit("python/trueforge_sdk/pyproject.toml is missing version = \"…\"")+ path.write_text(next_text)+ '+ python -m build --outdir dist-bootstrap+ git checkout -- pyproject.toml++ - name: Publish 0.1.3 bootstrap+ if: steps.version.outputs.bootstrap == 'true'+ uses: pypa/gh-action-pypi-publish@release/v1+ with:+ packages-dir: python/trueforge_sdk/dist-bootstrap++ - name: Build distributions+ if: steps.version.outputs.already == 'false'+ working-directory: python/trueforge_sdk+ run: |+ set -euo pipefail+ python -m build++ - name: Publish to PyPI+ if: steps.version.outputs.already == 'false'+ uses: pypa/gh-action-pypi-publish@release/v1+ with:+ packages-dir: python/trueforge_sdk/dist+# Same commit as this publish run (GITHUB_SHA). A workflow_dispatch --ref SHA# is rejected: the dispatch API only accepts a branch or tag name (HTTP 422).build-and-prepare-chart-release:diff --git a/RELEASING.md b/RELEASING.mdindex d35fb1ad0..abb35adf5 100644--- a/RELEASING.md+++ b/RELEASING.md@@ -6,6 +6,7 @@ sandbox image, and optional from-source **dev** images.| What | Trigger | Workflow || ----------------------------------- | ---------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- || npm packages | Push to `main` (Changesets) | [`release.yml`](.github/workflows/release.yml) |+| PyPI `trueforge-sdk` | Same `mode=publish` run as npm (parallel OIDC job) | [`release.yml`](.github/workflows/release.yml) || Prod image + chart-release PR | After `@truefoundry/trueforge` npm publish (reusable workflow), or manual dispatch | [`build-and-prepare-chart-release.yml`](.github/workflows/build-and-prepare-chart-release.yml) || Chart tag, GitHub Release, OCI push | Merge of `release-chart/trueforge`, or push/dispatch of `charts/trueforge@*` | [`release-chart.yml`](.github/workflows/release-chart.yml) || Sandbox image + pin PR | Push to `main` when `scripts/sandbox/**` changes, or dispatch | [`push-sandbox-image.yml`](.github/workflows/push-sandbox-image.yml) |@@ -16,6 +17,7 @@ sandbox image, and optional from-source **dev** images.| Artifact | Identity || ---------------------------- | --------------------------------------------------------------------------------------------------------- || npm `@truefoundry/trueforge` | SemVer `X.Y.Z` — source of truth for app bits |+| PyPI `trueforge-sdk` | Same SemVer as `@truefoundry/trueforge-sdk` (mirrored into `pyproject.toml` on Version Packages) || Chart `appVersion` | A **published** npm version || Prod image | Root [`Dockerfile`](Dockerfile): `npm install @truefoundry/trueforge@$APP_VERSION` || Prod image tag | `{appVersion}-{shortSha}` (shortSha of the build commit) |@@ -52,10 +54,13 @@ No `v*` tag publish. [`release.yml`](.github/workflows/release.yml) does both ve`pnpm change --bump patch --summary "…" <pkg>`). SDK regen already adds`@truefoundry/trueforge-sdk` via `pnpm changeset:sdk-regen`.2. Merge to `main`. Pending changesets → **Version Packages** PR- (`pnpm run version`). Review and merge.+ (`pnpm run version`). When `@truefoundry/trueforge-sdk` moves,+ `scripts/version.mjs` mirrors that version into `python/trueforge_sdk` and+ regenerates both SDKs. Review and merge.3. With no pending changesets, **pack** (build/test) and **Windows npx smoke**- run in parallel, then **publish** via npm trusted publishing (OIDC; no- `NPM_TOKEN`).+ run in parallel, then **npm publish** and **PyPI publish** run in parallel+ via trusted publishing (OIDC; no `NPM_TOKEN` / `PYPI_TOKEN`). PyPI skips when+ that `pyproject.toml` version is already published.4. If `@truefoundry/trueforge` was published, **Release** calls **Build andprepare chart release** as a reusable workflow on the same commit (so anewer `main` push cannot change the Dockerfile / shortSha). GitHub's@@ -78,19 +83,30 @@ Repo-wide via `.changeset/pre.json` (absent = publish to `latest`):## Trusted publishing-Each public package must list this repo + workflow as a trusted publisher on npmjs.com:+Each public **npm** package must list this repo + workflow as a trusted publisher on npmjs.com:- Repository: `truefoundry/trueforge`- Workflow: `release.yml` (exact filename)- No GitHub Environment name-Do not set `NPM_TOKEN` / `_authToken` on the publish job — that disables OIDC.-Only the **publish** job uses npm OIDC (`id-token: write`).+Do not set `NPM_TOKEN` / `_authToken` on the npm publish job — that disables OIDC.+Only the **publish** / **publish-python** jobs use OIDC (`id-token: write`).Publish attaches npm provenance (`NPM_CONFIG_PROVENANCE` on the publish job, and`publishConfig.provenance: true` on every public package). That publicly atteststhe source repo and commit on npmjs.com.+**PyPI** `trueforge-sdk` uses the same workflow file via a trusted publisher:++- Repository: `truefoundry/trueforge`+- Workflow: `release.yml` (exact filename)+- No Environment name (unless you add one to the job and mirror it on PyPI)+- Create the project once on PyPI (or publish the first version), then add the+ pending/trusted publisher before the first OIDC upload succeeds.+- Import and `pyproject.toml` name stay `trueforge_sdk` (what we upload); install with `pip install trueforge-sdk`.++> Note: if PyPI has no `0.1.3`, the first publish also uploads `0.1.3` (so default pip is not the `0.0.0` placeholder), then the current `pyproject.toml` version. Delete that bootstrap code in a follow-up PR.+## Local without publishing```bash@@ -106,7 +122,9 @@ pnpm clean && pnpm build && pnpm standalone:start- **OIDC fail** — pnpm >= 11.0.7; remove registry `_authToken`.- **Missing `dist/_frontend/index.html`** — root `pnpm build` must build `frontend` first.- **SDK not regenerated on Version PR** — only when `@truefoundry/trueforge-sdk` version moved- (`scripts/version.mjs`; needs Docker).+ (`scripts/version.mjs`; needs Docker). That path also mirrors the version into `python/trueforge_sdk`.+- **PyPI 403 / invalid-publisher** — register a trusted publisher for `trueforge-sdk` bound to+ `release.yml` (and create the project if it does not exist yet).- **Prod image missing after npm publish** — dispatch the chart workflow on a**branch or tag** (not a SHA): `gh workflow run build-and-prepare-chart-release.yml --ref main -f app_version=X.Y.Z -f update_app_version=true`.diff --git a/python/trueforge_sdk/.fern/metadata.json b/python/trueforge_sdk/.fern/metadata.jsonindex 474bf54ab..a75cfb4e4 100644--- a/python/trueforge_sdk/.fern/metadata.json+++ b/python/trueforge_sdk/.fern/metadata.json@@ -21,7 +21,7 @@},"pyproject_python_version": ">=3.10"},- "originGitCommit": "824204676ae4dba2bc9ffa6ba2af1ba587edaf49",+ "originGitCommit": "239af061164b797e062d890b9ef85bd1159e91c8","originGitCommitIsDirty": false,"invokedBy": "ci","requestedVersion": "0.2.0-rc.5",diff --git a/scripts/generate-sdk.sh b/scripts/generate-sdk.shindex fac7967a2..017fcb10d 100755--- a/scripts/generate-sdk.sh+++ b/scripts/generate-sdk.sh@@ -14,21 +14,23 @@ fern() {}# Fern --version stamps package metadata and baked-in SDK version / User-Agent.-# Pass the version already in the TS package.json so a regen never stomps a-# version Changesets (or a human) already set. TS and Python stay on the same-# SemVer (both groups get --version "$current_version"); PyPI publish wiring is-# separate.-current_version="$(node -p "require('./packages/trueforge-sdk/package.json').version")"+# Pass the version already in each package's metadata so a regen never stomps a+# version a human (or Changesets, for TS) already set.+ts_version="$(node -p "require('./packages/trueforge-sdk/package.json').version")"+# Prefer pyproject.toml (lockstep with TS on Version Packages); fallback for a fresh tree.+py_version="0.1.0-rc.1"+if [[ -f python/trueforge_sdk/pyproject.toml ]]; then+ py_version="$(node -p 'require("fs").readFileSync("python/trueforge_sdk/pyproject.toml","utf8").match(/^version\s*=\s*"([^"]+)"/m)[1]')"+fipnpm --filter @truefoundry/trueforge-core buildpnpm openapi:writefern check# --force skips the overwrite prompt when the SDK dirs already exist (needed non-interactively / in CI).-fern generate --group ts-sdk --version "$current_version" --local --generate-tests --force --log-level debug-fern generate --group python-sdk --version "$current_version" --local --generate-tests --force --log-level debug+fern generate --group ts-sdk --version "$ts_version" --local --generate-tests --force --log-level debug+fern generate --group python-sdk --version "$py_version" --local --generate-tests --force --log-level debugtest -f python/trueforge_sdk/src/trueforge_sdk/client.py-# Dist name is Fern package_name (trueforge_sdk); pip install trueforge-sdk works via PEP 503.# Fern's generated verify.sh runs `pnpm install` from packages/trueforge-sdk, which now# resolves to this workspace. CI sets frozen-lockfile, so refresh the root# lockfile first or that install fails when the generator added/removed deps.diff --git a/scripts/version.mjs b/scripts/version.mjsindex e8aef8d69..46e1a9e8f 100644--- a/scripts/version.mjs+++ b/scripts/version.mjs@@ -2,16 +2,18 @@* Root `pnpm version` entry for changesets/action.** `changeset version` only patches package.json (+ CHANGELOG). Fern also bakes the- * SDK version into generated TS, so if `@truefoundry/trueforge-sdk` actually moved,- * re-run `pnpm sdk:generate` and stage that output.+ * SDK version into generated TS/Python, so if `@truefoundry/trueforge-sdk` actually+ * moved, mirror that version into `python/trueforge_sdk/pyproject.toml`, re-run+ * `pnpm sdk:generate`, and stage that output.*/import { spawn } from 'node:child_process';-import { readFile } from 'node:fs/promises';+import { readFile, writeFile } from 'node:fs/promises';import path from 'node:path';import { fileURLToPath } from 'node:url';const rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');const sdkPackageJsonPath = path.join(rootDir, 'packages/trueforge-sdk/package.json');+const pythonPyprojectPath = path.join(rootDir, 'python/trueforge_sdk/pyproject.toml');function run(command, args) {return new Promise((resolve, reject) => {@@ -57,14 +59,59 @@ async function readSdkVersion() {return parsed.version;}+/** Lockstep: Poetry version tracks the Changesets-bumped TS SDK version. */+async function setPythonSdkVersion(version) {+ let toml;+ try {+ toml = await readFile(pythonPyprojectPath, 'utf8');+ } catch (error) {+ throw new Error('Failed to read python/trueforge_sdk/pyproject.toml', { cause: error });+ }++ const next = toml.replace(/^version\s*=\s*"[^"]+"/m, `version = "${version}"`);+ if (next === toml) {+ throw new Error('python/trueforge_sdk/pyproject.toml is missing version = "…"');+ }+ await writeFile(pythonPyprojectPath, next, 'utf8');+ console.log(`Python SDK version → ${version}`);+}++function readPoetryVersion(toml) {+ const match = /^version\s*=\s*"([^"]+)"/m.exec(toml);+ if (match === null) {+ throw new Error('python/trueforge_sdk/pyproject.toml is missing version = "…"');+ }+ return match[1];+}+const before = await readSdkVersion();+let pythonToml;+try {+ pythonToml = await readFile(pythonPyprojectPath, 'utf8');+} catch (error) {+ throw new Error('Failed to read python/trueforge_sdk/pyproject.toml', { cause: error });+}+const pythonBefore = readPoetryVersion(pythonToml);+await run('pnpm', ['changeset', 'version']);const after = await readSdkVersion();-if (before === after) {+if (before === after && pythonBefore === after) {console.log(`SDK version unchanged (${before}); skipping sdk:generate`);} else {- console.log(`SDK version ${before} → ${after}; regenerating SDK so Fern rebakes version literals`);+ if (before !== after) {+ console.log(`SDK version ${before} → ${after}; regenerating SDK so Fern rebakes version literals`);+ } else {+ console.log(`Python SDK ${pythonBefore} diverged from TS ${after}; syncing and regenerating`);+ }+ await setPythonSdkVersion(after);await run('pnpm', ['sdk:generate']);- await run('git', ['add', '.github/fern/openapi', 'docs/openapi.json', 'packages/trueforge-sdk', 'pnpm-lock.yaml']);+ await run('git', [+ 'add',+ '.github/fern/openapi',+ 'docs/openapi.json',+ 'packages/trueforge-sdk',+ 'python/trueforge_sdk',+ 'pnpm-lock.yaml',+ ]);}