Skip to content

The open-source agent harness - the runtime layer that turns an LLM into a working agent.

ci: publish Python SDK to PyPI from release.yml (#718)

Bhavesh Patele64be05a4e4d
diff --git a/.github/fern/generators.yml b/.github/fern/generators.yml
index 63bde8ded..edd459b41 100644
--- a/.github/fern/generators.yml
+++ b/.github/fern/generators.yml
@@ -109,7 +109,7 @@ groups:
location: local-file-system
path: ../../python/trueforge_sdk
config:
- # Import name; PyPI install trueforge-sdk resolves via PEP 503.
+ # Import + upload name. pip install trueforge-sdk resolves via PEP 503.
package_name: trueforge_sdk
optional_auth: true
stream_abstraction: true
diff --git a/.github/workflows/generate-sdk.yaml b/.github/workflows/generate-sdk.yaml
index 0afde503b..2869b8512 100644
--- a/.github/workflows/generate-sdk.yaml
+++ b/.github/workflows/generate-sdk.yaml
@@ -126,8 +126,11 @@ jobs:
# Same entrypoint as local `pnpm sdk:generate` (re-runs openapi:write; idempotent).
# App-token success = we can push; forks skip Fern so a flake cannot block the PR.
+ # FERN_TOKEN is required so Python list() emits SyncPager (Fern README already documents it).
- name: Generate OpenAPI document and SDKs
if: steps.decide.outputs.needed == 'true' && steps.app-token.outcome == 'success'
+ env:
+ FERN_TOKEN: ${{ secrets.FERN_TOKEN }}
run: pnpm sdk:generate
# App-token success = writable head; forks cannot receive this commit.
@@ -154,7 +157,7 @@ jobs:
# Bot-authored sdk diffs need a changeset; the PR author never touches this commit.
# Skip packages/trueforge-sdk/.fern — Fern metadata only, not a consumer-facing change.
# changeset:sdk-regen is a no-op if the sdk is already named in a pending file.
- # Python is not Changesets-published yet; do not add it here.
+ # Python version mirrors TS on Version Packages; do not add a Python changeset here.
if [ -n "$(git status --porcelain -- packages/trueforge-sdk ':!packages/trueforge-sdk/.fern')" ]; then
pnpm changeset:sdk-regen
fi
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 6a975693e..e31b55f13 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,18 +1,20 @@
-# npm publish for the four public packages.
+# npm publish for the four public packages, plus PyPI for python/trueforge_sdk.
#
-# npm trusted publishers are bound to `release.yml`.
-# Do not rename this file without updating every package on npmjs.com.
+# npm and PyPI trusted publishers are bound to `release.yml`.
+# Do not rename this file without updating every package on npmjs.com and PyPI.
#
# On every push to main, changesets/action/select-mode chooses:
# - pending `.changeset/*.md` → version job opens/updates the Version Packages PR
-# (install only; that PR is gated by CI.yml)
-# - none pending + unpublished versions → pack (build/test) + Windows npx smoke, then publish
+# (install only; that PR is gated by CI.yml). When @truefoundry/trueforge-sdk
+# moves, scripts/version.mjs mirrors that version into python/trueforge_sdk and rebakes Fern.
+# - none pending + unpublished versions → pack (build/test) + Windows npx smoke,
+# then npm publish and PyPI publish in parallel
# Merge the Version Packages PR to publish. There is no separate publish Action
# and no `v*` tag trigger.
#
-# Auth is GitHub OIDC (no NPM_TOKEN). An `_authToken` in user `.npmrc` disables
-# pnpm's OIDC exchange, so the publish job must not set NPM_TOKEN / NODE_AUTH_TOKEN.
-# id-token is granted only on the publish job (trusted-publishing best practice).
+# Auth is GitHub OIDC (no NPM_TOKEN / PYPI_TOKEN). An `_authToken` in user `.npmrc`
+# disables pnpm's OIDC exchange, so the npm publish job must not set NPM_TOKEN /
+# NODE_AUTH_TOKEN. id-token is granted only on the publish jobs.
name: Release
on:
@@ -91,6 +93,8 @@ jobs:
- name: Version packages
id: version
uses: changesets/action/version@v2
+ env:
+ FERN_TOKEN: ${{ secrets.FERN_TOKEN }}
with:
script: pnpm run version
push-with-git-cli: true
@@ -238,6 +242,119 @@ jobs:
echo "@truefoundry/trueforge was not in this publish set"
fi
+ # Sibling of npm publish (same gates). Skips when pyproject version is already on PyPI.
+ publish-python:
+ name: Publish Python SDK
+ needs: [pack, windows-npx-smoke]
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ permissions:
+ contents: read
+ id-token: write
+ steps:
+ - name: Check out repo
+ uses: actions/checkout@v7
+ with:
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+
+ - name: Install build tools
+ run: python -m pip install --upgrade pip build packaging
+
+ - name: Plan PyPI versions
+ id: version
+ working-directory: python/trueforge_sdk
+ run: |
+ set -euo pipefail
+ pypi_code() {
+ curl -sS -L -o /dev/null -w '%{http_code}' "https://pypi.org/pypi/trueforge-sdk/${1}/json"
+ }
+ VERSION="$(python3 -c '
+ import re, pathlib
+ from packaging.version import Version
+ text = pathlib.Path("pyproject.toml").read_text()
+ match = re.search(r"(?m)^version\s*=\s*\"([^\"]+)\"", text)
+ if match is None:
+ raise SystemExit("missing version in pyproject.toml")
+ print(Version(match.group(1)))
+ ')"
+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
+ code="$(pypi_code "$VERSION")"
+ if [[ "$code" == "200" ]]; then
+ echo "already=true" >> "$GITHUB_OUTPUT"
+ echo "trueforge-sdk==$VERSION already on PyPI; skipping"
+ elif [[ "$code" == "404" ]]; then
+ echo "already=false" >> "$GITHUB_OUTPUT"
+ else
+ echo "Unexpected PyPI response HTTP $code for trueforge-sdk==$VERSION" >&2
+ exit 1
+ fi
+
+ # Remove this 0.1.3 bootstrap block in a follow-up PR.
+ # HTTP 200 means 0.1.3 is already on PyPI, so later rc publishes skip it.
+ # HTTP 404 when the current version is not 0.1.3 means this is the first
+ # upload: we publish 0.1.3 so pip does not install the 0.0.0 placeholder.
+ # HTTP 404 when the current version is already 0.1.3 means the normal
+ # publish step will upload it, so we must not build it twice.
+ bootstrap_code="$(pypi_code "0.1.3")"
+ if [[ "$bootstrap_code" == "200" ]]; then
+ echo "bootstrap=false" >> "$GITHUB_OUTPUT"
+ elif [[ "$bootstrap_code" == "404" && "$VERSION" != "0.1.3" ]]; then
+ echo "bootstrap=true" >> "$GITHUB_OUTPUT"
+ echo "trueforge-sdk==0.1.3 missing; will upload before $VERSION"
+ elif [[ "$bootstrap_code" == "404" ]]; then
+ echo "bootstrap=false" >> "$GITHUB_OUTPUT"
+ else
+ echo "Unexpected PyPI response HTTP $bootstrap_code for trueforge-sdk==0.1.3" >&2
+ exit 1
+ fi
+
+ # Remove this 0.1.3 bootstrap block in a follow-up PR.
+ # Rewrite pyproject.toml to version 0.1.3, build that wheel from today's
+ # tree, then restore pyproject.toml so the next step can publish the real
+ # rc (for example 0.2.0rc5).
+ - name: Build 0.1.3 bootstrap
+ if: steps.version.outputs.bootstrap == 'true'
+ working-directory: python/trueforge_sdk
+ run: |
+ set -euo pipefail
+ python3 -c '
+ import re
+ from pathlib import Path
+ path = Path("pyproject.toml")
+ text = path.read_text()
+ next_text, count = re.subn(r"(?m)^version\s*=\s*\"[^\"]+\"", "version = \"0.1.3\"", text, count=1)
+ if count != 1:
+ raise SystemExit("python/trueforge_sdk/pyproject.toml is missing version = \"…\"")
+ path.write_text(next_text)
+ '
+ python -m build --outdir dist-bootstrap
+ git checkout -- pyproject.toml
+
+ - name: Publish 0.1.3 bootstrap
+ if: steps.version.outputs.bootstrap == 'true'
+ uses: pypa/gh-action-pypi-publish@release/v1
+ with:
+ packages-dir: python/trueforge_sdk/dist-bootstrap
+
+ - name: Build distributions
+ if: steps.version.outputs.already == 'false'
+ working-directory: python/trueforge_sdk
+ run: |
+ set -euo pipefail
+ python -m build
+
+ - name: Publish to PyPI
+ if: steps.version.outputs.already == 'false'
+ uses: pypa/gh-action-pypi-publish@release/v1
+ with:
+ packages-dir: python/trueforge_sdk/dist
+
# Same commit as this publish run (GITHUB_SHA). A workflow_dispatch --ref SHA
# is rejected: the dispatch API only accepts a branch or tag name (HTTP 422).
build-and-prepare-chart-release:
diff --git a/RELEASING.md b/RELEASING.md
index d35fb1ad0..abb35adf5 100644
--- a/RELEASING.md
+++ b/RELEASING.md
@@ -6,6 +6,7 @@ sandbox image, and optional from-source **dev** images.
| What | Trigger | Workflow |
| ----------------------------------- | ---------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- |
| npm packages | Push to `main` (Changesets) | [`release.yml`](.github/workflows/release.yml) |
+| PyPI `trueforge-sdk` | Same `mode=publish` run as npm (parallel OIDC job) | [`release.yml`](.github/workflows/release.yml) |
| Prod image + chart-release PR | After `@truefoundry/trueforge` npm publish (reusable workflow), or manual dispatch | [`build-and-prepare-chart-release.yml`](.github/workflows/build-and-prepare-chart-release.yml) |
| Chart tag, GitHub Release, OCI push | Merge of `release-chart/trueforge`, or push/dispatch of `charts/trueforge@*` | [`release-chart.yml`](.github/workflows/release-chart.yml) |
| Sandbox image + pin PR | Push to `main` when `scripts/sandbox/**` changes, or dispatch | [`push-sandbox-image.yml`](.github/workflows/push-sandbox-image.yml) |
@@ -16,6 +17,7 @@ sandbox image, and optional from-source **dev** images.
| Artifact | Identity |
| ---------------------------- | --------------------------------------------------------------------------------------------------------- |
| npm `@truefoundry/trueforge` | SemVer `X.Y.Z` — source of truth for app bits |
+| PyPI `trueforge-sdk` | Same SemVer as `@truefoundry/trueforge-sdk` (mirrored into `pyproject.toml` on Version Packages) |
| Chart `appVersion` | A **published** npm version |
| Prod image | Root [`Dockerfile`](Dockerfile): `npm install @truefoundry/trueforge@$APP_VERSION` |
| Prod image tag | `{appVersion}-{shortSha}` (shortSha of the build commit) |
@@ -52,10 +54,13 @@ No `v*` tag publish. [`release.yml`](.github/workflows/release.yml) does both ve
`pnpm change --bump patch --summary "…" <pkg>`). SDK regen already adds
`@truefoundry/trueforge-sdk` via `pnpm changeset:sdk-regen`.
2. Merge to `main`. Pending changesets → **Version Packages** PR
- (`pnpm run version`). Review and merge.
+ (`pnpm run version`). When `@truefoundry/trueforge-sdk` moves,
+ `scripts/version.mjs` mirrors that version into `python/trueforge_sdk` and
+ regenerates both SDKs. Review and merge.
3. With no pending changesets, **pack** (build/test) and **Windows npx smoke**
- run in parallel, then **publish** via npm trusted publishing (OIDC; no
- `NPM_TOKEN`).
+ run in parallel, then **npm publish** and **PyPI publish** run in parallel
+ via trusted publishing (OIDC; no `NPM_TOKEN` / `PYPI_TOKEN`). PyPI skips when
+ that `pyproject.toml` version is already published.
4. If `@truefoundry/trueforge` was published, **Release** calls **Build and
prepare chart release** as a reusable workflow on the same commit (so a
newer `main` push cannot change the Dockerfile / shortSha). GitHub's
@@ -78,19 +83,30 @@ Repo-wide via `.changeset/pre.json` (absent = publish to `latest`):
## Trusted publishing
-Each public package must list this repo + workflow as a trusted publisher on npmjs.com:
+Each public **npm** package must list this repo + workflow as a trusted publisher on npmjs.com:
- Repository: `truefoundry/trueforge`
- Workflow: `release.yml` (exact filename)
- No GitHub Environment name
-Do not set `NPM_TOKEN` / `_authToken` on the publish job — that disables OIDC.
-Only the **publish** job uses npm OIDC (`id-token: write`).
+Do not set `NPM_TOKEN` / `_authToken` on the npm publish job — that disables OIDC.
+Only the **publish** / **publish-python** jobs use OIDC (`id-token: write`).
Publish attaches npm provenance (`NPM_CONFIG_PROVENANCE` on the publish job, and
`publishConfig.provenance: true` on every public package). That publicly attests
the source repo and commit on npmjs.com.
+**PyPI** `trueforge-sdk` uses the same workflow file via a trusted publisher:
+
+- Repository: `truefoundry/trueforge`
+- Workflow: `release.yml` (exact filename)
+- No Environment name (unless you add one to the job and mirror it on PyPI)
+- Create the project once on PyPI (or publish the first version), then add the
+ pending/trusted publisher before the first OIDC upload succeeds.
+- Import and `pyproject.toml` name stay `trueforge_sdk` (what we upload); install with `pip install trueforge-sdk`.
+
+> Note: if PyPI has no `0.1.3`, the first publish also uploads `0.1.3` (so default pip is not the `0.0.0` placeholder), then the current `pyproject.toml` version. Delete that bootstrap code in a follow-up PR.
+
## Local without publishing
```bash
@@ -106,7 +122,9 @@ pnpm clean && pnpm build && pnpm standalone:start
- **OIDC fail** — pnpm >= 11.0.7; remove registry `_authToken`.
- **Missing `dist/_frontend/index.html`** — root `pnpm build` must build `frontend` first.
- **SDK not regenerated on Version PR** — only when `@truefoundry/trueforge-sdk` version moved
- (`scripts/version.mjs`; needs Docker).
+ (`scripts/version.mjs`; needs Docker). That path also mirrors the version into `python/trueforge_sdk`.
+- **PyPI 403 / invalid-publisher** — register a trusted publisher for `trueforge-sdk` bound to
+ `release.yml` (and create the project if it does not exist yet).
- **Prod image missing after npm publish** — dispatch the chart workflow on a
**branch or tag** (not a SHA): `gh workflow run build-and-prepare-chart-release.yml --ref main -f app_version=X.Y.Z -f update_app_version=true`.
diff --git a/python/trueforge_sdk/.fern/metadata.json b/python/trueforge_sdk/.fern/metadata.json
index 474bf54ab..a75cfb4e4 100644
--- a/python/trueforge_sdk/.fern/metadata.json
+++ b/python/trueforge_sdk/.fern/metadata.json
@@ -21,7 +21,7 @@
},
"pyproject_python_version": ">=3.10"
},
- "originGitCommit": "824204676ae4dba2bc9ffa6ba2af1ba587edaf49",
+ "originGitCommit": "239af061164b797e062d890b9ef85bd1159e91c8",
"originGitCommitIsDirty": false,
"invokedBy": "ci",
"requestedVersion": "0.2.0-rc.5",
diff --git a/scripts/generate-sdk.sh b/scripts/generate-sdk.sh
index fac7967a2..017fcb10d 100755
--- a/scripts/generate-sdk.sh
+++ b/scripts/generate-sdk.sh
@@ -14,21 +14,23 @@ fern() {
}
# Fern --version stamps package metadata and baked-in SDK version / User-Agent.
-# Pass the version already in the TS package.json so a regen never stomps a
-# version Changesets (or a human) already set. TS and Python stay on the same
-# SemVer (both groups get --version "$current_version"); PyPI publish wiring is
-# separate.
-current_version="$(node -p "require('./packages/trueforge-sdk/package.json').version")"
+# Pass the version already in each package's metadata so a regen never stomps a
+# version a human (or Changesets, for TS) already set.
+ts_version="$(node -p "require('./packages/trueforge-sdk/package.json').version")"
+# Prefer pyproject.toml (lockstep with TS on Version Packages); fallback for a fresh tree.
+py_version="0.1.0-rc.1"
+if [[ -f python/trueforge_sdk/pyproject.toml ]]; then
+ py_version="$(node -p 'require("fs").readFileSync("python/trueforge_sdk/pyproject.toml","utf8").match(/^version\s*=\s*"([^"]+)"/m)[1]')"
+fi
pnpm --filter @truefoundry/trueforge-core build
pnpm openapi:write
fern check
# --force skips the overwrite prompt when the SDK dirs already exist (needed non-interactively / in CI).
-fern generate --group ts-sdk --version "$current_version" --local --generate-tests --force --log-level debug
-fern generate --group python-sdk --version "$current_version" --local --generate-tests --force --log-level debug
+fern generate --group ts-sdk --version "$ts_version" --local --generate-tests --force --log-level debug
+fern generate --group python-sdk --version "$py_version" --local --generate-tests --force --log-level debug
test -f python/trueforge_sdk/src/trueforge_sdk/client.py
-# Dist name is Fern package_name (trueforge_sdk); pip install trueforge-sdk works via PEP 503.
# Fern's generated verify.sh runs `pnpm install` from packages/trueforge-sdk, which now
# resolves to this workspace. CI sets frozen-lockfile, so refresh the root
# lockfile first or that install fails when the generator added/removed deps.
diff --git a/scripts/version.mjs b/scripts/version.mjs
index e8aef8d69..46e1a9e8f 100644
--- a/scripts/version.mjs
+++ b/scripts/version.mjs
@@ -2,16 +2,18 @@
* Root `pnpm version` entry for changesets/action.
*
* `changeset version` only patches package.json (+ CHANGELOG). Fern also bakes the
- * SDK version into generated TS, so if `@truefoundry/trueforge-sdk` actually moved,
- * re-run `pnpm sdk:generate` and stage that output.
+ * SDK version into generated TS/Python, so if `@truefoundry/trueforge-sdk` actually
+ * moved, mirror that version into `python/trueforge_sdk/pyproject.toml`, re-run
+ * `pnpm sdk:generate`, and stage that output.
*/
import { spawn } from 'node:child_process';
-import { readFile } from 'node:fs/promises';
+import { readFile, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
const rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const sdkPackageJsonPath = path.join(rootDir, 'packages/trueforge-sdk/package.json');
+const pythonPyprojectPath = path.join(rootDir, 'python/trueforge_sdk/pyproject.toml');
function run(command, args) {
return new Promise((resolve, reject) => {
@@ -57,14 +59,59 @@ async function readSdkVersion() {
return parsed.version;
}
+/** Lockstep: Poetry version tracks the Changesets-bumped TS SDK version. */
+async function setPythonSdkVersion(version) {
+ let toml;
+ try {
+ toml = await readFile(pythonPyprojectPath, 'utf8');
+ } catch (error) {
+ throw new Error('Failed to read python/trueforge_sdk/pyproject.toml', { cause: error });
+ }
+
+ const next = toml.replace(/^version\s*=\s*"[^"]+"/m, `version = "${version}"`);
+ if (next === toml) {
+ throw new Error('python/trueforge_sdk/pyproject.toml is missing version = "…"');
+ }
+ await writeFile(pythonPyprojectPath, next, 'utf8');
+ console.log(`Python SDK version → ${version}`);
+}
+
+function readPoetryVersion(toml) {
+ const match = /^version\s*=\s*"([^"]+)"/m.exec(toml);
+ if (match === null) {
+ throw new Error('python/trueforge_sdk/pyproject.toml is missing version = "…"');
+ }
+ return match[1];
+}
+
const before = await readSdkVersion();
+let pythonToml;
+try {
+ pythonToml = await readFile(pythonPyprojectPath, 'utf8');
+} catch (error) {
+ throw new Error('Failed to read python/trueforge_sdk/pyproject.toml', { cause: error });
+}
+const pythonBefore = readPoetryVersion(pythonToml);
+
await run('pnpm', ['changeset', 'version']);
const after = await readSdkVersion();
-if (before === after) {
+if (before === after && pythonBefore === after) {
console.log(`SDK version unchanged (${before}); skipping sdk:generate`);
} else {
- console.log(`SDK version ${before} → ${after}; regenerating SDK so Fern rebakes version literals`);
+ if (before !== after) {
+ console.log(`SDK version ${before} → ${after}; regenerating SDK so Fern rebakes version literals`);
+ } else {
+ console.log(`Python SDK ${pythonBefore} diverged from TS ${after}; syncing and regenerating`);
+ }
+ await setPythonSdkVersion(after);
await run('pnpm', ['sdk:generate']);
- await run('git', ['add', '.github/fern/openapi', 'docs/openapi.json', 'packages/trueforge-sdk', 'pnpm-lock.yaml']);
+ await run('git', [
+ 'add',
+ '.github/fern/openapi',
+ 'docs/openapi.json',
+ 'packages/trueforge-sdk',
+ 'python/trueforge_sdk',
+ 'pnpm-lock.yaml',
+ ]);
}