Skip to content

The open-source agent harness - the runtime layer that turns an LLM into a working agent.

feat: accept request-scoped model providers via x-tfg-models (#893)

mitanshudodiab4409691474c
diff --git a/.changeset/inline-model-providers.md b/.changeset/inline-model-providers.md
new file mode 100644
index 000000000..5cd7f7cd2
--- /dev/null
+++ b/.changeset/inline-model-providers.md
@@ -0,0 +1,5 @@
+---
+"@truefoundry/trueforge": patch
+---
+
+Accept `x-tfg-models` in TrueFoundry mode alongside `x-tfg-mcp` and `x-tfg-skills`: model providers a request defines by name, each an OpenAI-compatible endpoint with its own `base_url`, optional `auth.api_key`, and `models`. A spec's `provider/model` resolves against them for validation and turn execution, taking precedence over the tenant registry for that request only; unfiltered provider and model lists still show only configured resources.
diff --git a/packages/trueforge/src/main.ts b/packages/trueforge/src/main.ts
index 8c8847fd4..fe9924aa2 100644
--- a/packages/trueforge/src/main.ts
+++ b/packages/trueforge/src/main.ts
@@ -96,7 +96,15 @@ import type { ConnectedRedis } from './runtime/redis';
import { initSentry } from './sentry';
import { printStandaloneStartupBanner } from './startupBanner';
import { InlineMcpServerStore } from './truefoundry/InlineMcpServerStore';
-import { parseInlineMcpServers, parseInlineSkills, X_TFG_MCP, X_TFG_SKILLS } from './truefoundry/inlineResources';
+import { InlineModelProviderStore } from './truefoundry/InlineModelProviderStore';
+import {
+ parseInlineMcpServers,
+ parseInlineModelProviders,
+ parseInlineSkills,
+ X_TFG_MCP,
+ X_TFG_MODELS,
+ X_TFG_SKILLS,
+} from './truefoundry/inlineResources';
import { InlineSkillStore } from './truefoundry/InlineSkillStore';
import {
parsePerServerMcpHeaders,
@@ -599,8 +607,17 @@ async function createServerRuntime<TTransaction>(persistence: ServerPersistence<
: undefined;
// Hono handlers get Context; persistence resolvers take RequestContext.
- const resolveModelProviderStore = (c: Context, runAsAgent?: AgentRecord) =>
- persistence.resolveModelProviderStore(resolveRequestContext(c), runAsAgent);
+ const resolveModelProviderStore = (c: Context, runAsAgent?: AgentRecord) => {
+ const store = persistence.resolveModelProviderStore(resolveRequestContext(c), runAsAgent);
+ if (!isTrueFoundryModeEnabled(configuration)) {
+ return store;
+ }
+ const rawInline = c.req.header(X_TFG_MODELS);
+ if (rawInline === undefined) {
+ return store;
+ }
+ return new InlineModelProviderStore({ inner: store, inline: parseInlineModelProviders(rawInline) });
+ };
const resolveMcpServerStore = (c?: Context, runAsAgent?: AgentRecord) => {
if (c === undefined) {
return mcpOAuthStore;
diff --git a/packages/trueforge/src/truefoundry/InlineModelProviderStore.ts b/packages/trueforge/src/truefoundry/InlineModelProviderStore.ts
new file mode 100644
index 000000000..d36c6e33d
--- /dev/null
+++ b/packages/trueforge/src/truefoundry/InlineModelProviderStore.ts
@@ -0,0 +1,91 @@
+import type {
+ CreateModelProviderInput,
+ GetModelProviderForUpdateInput,
+ GetModelProviderInput,
+ IModelProviderStore,
+ ListModelProvidersInput,
+ ModelProviderRecord,
+ UpsertModelProviderInput,
+} from '../db/modelProviderStore';
+import type { TurnMetadata } from '../db/turnMetadata';
+import type { AvailableModel } from '../schemas/modelProvider';
+import { gatewayMetadataHeadersForTurn } from './gatewayMetadata';
+import type { InlineModelProviders } from './inlineResources';
+
+/**
+ * Serves the model providers a request brought with it, and delegates everything else.
+ *
+ * Only the by-name lookup a `provider/model` reference goes through is overlaid, plus its invoke
+ * headers. Unfiltered lists pass straight through, so a request-scoped provider never shows up in
+ * the tenant's settings or model picker. Writes delegate: there is no row to write.
+ */
+export class InlineModelProviderStore<TTransaction = never> implements IModelProviderStore<TTransaction> {
+ readonly #inner: IModelProviderStore<TTransaction>;
+ readonly #inline: InlineModelProviders;
+
+ constructor(input: { inner: IModelProviderStore<TTransaction>; inline: InlineModelProviders }) {
+ this.#inner = input.inner;
+ this.#inline = input.inline;
+ }
+
+ listProviders(input: ListModelProvidersInput, transaction?: TTransaction): Promise<ModelProviderRecord[]> {
+ return this.#inner.listProviders(input, transaction);
+ }
+
+ async getProvider(
+ input: GetModelProviderInput,
+ transaction?: TTransaction,
+ ): Promise<ModelProviderRecord | undefined> {
+ const record = this.#toRecord({ tenant_id: input.tenant_id, name: input.name });
+ return record ?? (await this.#inner.getProvider(input, transaction));
+ }
+
+ getProviderForUpdate(
+ input: GetModelProviderForUpdateInput,
+ transaction: TTransaction,
+ ): Promise<ModelProviderRecord | undefined> {
+ return this.#inner.getProviderForUpdate(input, transaction);
+ }
+
+ createProvider(input: CreateModelProviderInput, transaction?: TTransaction): Promise<ModelProviderRecord> {
+ return this.#inner.createProvider(input, transaction);
+ }
+
+ upsertProvider(input: UpsertModelProviderInput, transaction?: TTransaction): Promise<ModelProviderRecord> {
+ return this.#inner.upsertProvider(input, transaction);
+ }
+
+ listModels(input: ListModelProvidersInput, transaction?: TTransaction): Promise<AvailableModel[]> {
+ return this.#inner.listModels(input, transaction);
+ }
+
+ /**
+ * The manifest carries its own `api_key`, which the adapter sends as the bearer, so no caller
+ * token is resolved here. Mid-turn, still stamp `x-tfy-metadata` the same way registry
+ * TrueFoundry invokes do.
+ */
+ resolveInvokeHeaders(input: {
+ record: ModelProviderRecord;
+ turnMetadata?: TurnMetadata;
+ }): Promise<Record<string, string>> {
+ if (this.#inline[input.record.name] === undefined) {
+ return this.#inner.resolveInvokeHeaders(input);
+ }
+ return Promise.resolve(gatewayMetadataHeadersForTurn(input.turnMetadata));
+ }
+
+ #toRecord(input: { tenant_id: string; name: string }): ModelProviderRecord | undefined {
+ const manifest = this.#inline[input.name];
+ if (manifest === undefined) {
+ return undefined;
+ }
+ const now = new Date().toISOString();
+ return {
+ tenant_id: input.tenant_id,
+ name: input.name,
+ manifest,
+ created_at: now,
+ updated_at: now,
+ };
+ }
+}
diff --git a/packages/trueforge/src/truefoundry/inlineResources.ts b/packages/trueforge/src/truefoundry/inlineResources.ts
index 681899b42..e8d0d523d 100644
--- a/packages/trueforge/src/truefoundry/inlineResources.ts
+++ b/packages/trueforge/src/truefoundry/inlineResources.ts
@@ -1,6 +1,6 @@
/**
- * MCP servers and skills supplied per request instead of configured in the tenant's registry,
- * each keyed by name.
+ * MCP servers, skills, and model providers supplied per request instead of configured in the
+ * tenant's registry, each keyed by name.
*
* A caller that owns its own agent definition brings resources the tenant never registered and
* should not see in its settings or its gateway metrics. Sending them per request keeps them out
@@ -9,14 +9,18 @@
*/
import { HTTPException } from 'hono/http-exception';
import { McpServerManifestSchema, type McpServerManifest } from '../schemas/mcpServer';
+import { ModelProviderManifestSchema, type ModelProviderManifest } from '../schemas/modelProvider';
import { SkillManifestSchema, type SkillManifest } from '../schemas/skill';
export const X_TFG_MCP = 'x-tfg-mcp';
export const X_TFG_SKILLS = 'x-tfg-skills';
+export const X_TFG_MODELS = 'x-tfg-models';
/** Manifests by name. `type` and `name` are implied by the header and the key, so callers omit them. */
export type InlineMcpServers = Readonly<Record<string, McpServerManifest>>;
export type InlineSkills = Readonly<Record<string, SkillManifest>>;
+/** Keyed by provider name: a spec addresses a model as `provider/model`, so the provider is the unit. */
+export type InlineModelProviders = Readonly<Record<string, ModelProviderManifest>>;
export function parseInlineMcpServers(raw: string): InlineMcpServers {
return parseByName({
@@ -48,6 +52,23 @@ export function parseInlineSkills(raw: string): InlineSkills {
});
}
+/**
+ * `custom` is the only provider type that carries its own name and endpoint, and it speaks the
+ * OpenAI-compatible API every gateway exposes, so it is the one an inline definition maps onto.
+ */
+export function parseInlineModelProviders(raw: string): InlineModelProviders {
+ return parseByName({
+ raw,
+ header: X_TFG_MODELS,
+ parseEntry: (name, definition) => {
+ const parsed = ModelProviderManifestSchema.safeParse({ ...definition, type: 'custom', name });
+ return parsed.success
+ ? { ok: true, manifest: parsed.data }
+ : { ok: false, reason: 'is not a valid model provider definition' };
+ },
+ });
+}
+
type EntryResult<TManifest> = { ok: true; manifest: TManifest } | { ok: false; reason: string };
/**
diff --git a/packages/trueforge/tests/unit/truefoundry/inlineResources.test.ts b/packages/trueforge/tests/unit/truefoundry/inlineResources.test.ts
index ea3bbb786..20d997b3a 100644
--- a/packages/trueforge/tests/unit/truefoundry/inlineResources.test.ts
+++ b/packages/trueforge/tests/unit/truefoundry/inlineResources.test.ts
@@ -1,9 +1,15 @@
import { HTTPException } from 'hono/http-exception';
import type { IMcpServerWithAuthStore, McpServerRecord } from '../../../src/db/mcpServerStore';
+import type { IModelProviderStore, ModelProviderRecord } from '../../../src/db/modelProviderStore';
import type { ISkillStore, SkillRecord } from '../../../src/db/skillStore';
import { TFG_METADATA_PREFIX, X_TFY_METADATA } from '../../../src/truefoundry/gatewayMetadata';
import { InlineMcpServerStore } from '../../../src/truefoundry/InlineMcpServerStore';
-import { parseInlineMcpServers, parseInlineSkills } from '../../../src/truefoundry/inlineResources';
+import { InlineModelProviderStore } from '../../../src/truefoundry/InlineModelProviderStore';
+import {
+ parseInlineMcpServers,
+ parseInlineModelProviders,
+ parseInlineSkills,
+} from '../../../src/truefoundry/inlineResources';
import { InlineSkillStore } from '../../../src/truefoundry/InlineSkillStore';
const DOCS_MCP = {
@@ -19,6 +25,12 @@ const ASK_AI_SKILL = {
description: 'How to answer questions about the platform.',
};
+const GATEWAY_PROVIDER = {
+ base_url: 'https://gateway.example/api/inference/openai',
+ auth: { api_key: 'rotating-token' },
+ models: [{ model_id: 'openai-main/gpt-5', name: 'gpt-5', properties: { context_length: 200000 } }],
+};
+
const registryServer: McpServerRecord = {
id: '01JREGISTRY',
tenant_id: 'default',
@@ -42,6 +54,33 @@ const registrySkill: SkillRecord = {
updated_at: '2026-01-15T12:00:00.000Z',
};
+const registryProvider: ModelProviderRecord = {
+ tenant_id: 'default',
+ name: 'openai',
+ manifest: {
+ type: 'openai',
+ base_url: 'https://api.openai.com/v1',
+ auth: { api_key: 'stored-key' },
+ models: [{ model_id: 'gpt-5', name: 'gpt-5', properties: {} }],
+ },
+ created_at: '2026-01-15T12:00:00.000Z',
+ updated_at: '2026-01-15T12:00:00.000Z',
+};
+
+function modelProviderStoreWith(inlineRaw: object) {
+ const inner = {
+ listProviders: jest.fn().mockResolvedValue([registryProvider]),
+ getProvider: jest.fn().mockResolvedValue(registryProvider),
+ listModels: jest.fn().mockResolvedValue([]),
+ resolveInvokeHeaders: jest.fn().mockResolvedValue({ Authorization: 'Bearer caller-token' }),
+ } as unknown as IModelProviderStore;
+ const store = new InlineModelProviderStore({
+ inner,
+ inline: parseInlineModelProviders(JSON.stringify(inlineRaw)),
+ });
+ return { store, inner };
+}
+
function mcpStoreWith(inlineRaw: object) {
const inner = {
getServer: jest.fn().mockResolvedValue(registryServer),
@@ -116,6 +155,118 @@ describe('parseInlineSkills', () => {
});
});
+describe('parseInlineModelProviders', () => {
+ it('maps each entry onto a custom provider named by its key', () => {
+ expect(parseInlineModelProviders(JSON.stringify({ 'tfy-gateway': GATEWAY_PROVIDER }))).toEqual({
+ 'tfy-gateway': { ...GATEWAY_PROVIDER, type: 'custom', name: 'tfy-gateway' },
+ });
+ });
+
+ it('accepts a provider that carries no credentials', () => {
+ const { auth: _auth, ...withoutAuth } = GATEWAY_PROVIDER;
+
+ expect(parseInlineModelProviders(JSON.stringify({ 'tfy-gateway': withoutAuth }))).toEqual({
+ 'tfy-gateway': { ...withoutAuth, type: 'custom', name: 'tfy-gateway' },
+ });
+ });
+
+ it.each([
+ ['no base_url', { auth: GATEWAY_PROVIDER.auth, models: GATEWAY_PROVIDER.models }],
+ ['no models', { ...GATEWAY_PROVIDER, models: [] }],
+ [
+ 'a model name the FQN parser would split',
+ { ...GATEWAY_PROVIDER, models: [{ model_id: 'x', name: 'a/b', properties: {} }] },
+ ],
+ [
+ 'two models sharing a name',
+ {
+ ...GATEWAY_PROVIDER,
+ models: [...GATEWAY_PROVIDER.models, { ...GATEWAY_PROVIDER.models[0], model_id: 'other' }],
+ },
+ ],
+ ])(
+ 'rejects a provider with %s rather than falling back to a registry that has no such provider',
+ (_case, definition) => {
+ expect(() => parseInlineModelProviders(JSON.stringify({ 'tfy-gateway': definition }))).toThrow(HTTPException);
+ },
+ );
+});
+
+describe('InlineModelProviderStore', () => {
+ it('resolves an inline provider by name without asking the registry', async () => {
+ const { store, inner } = modelProviderStoreWith({ 'tfy-gateway': GATEWAY_PROVIDER });
+
+ const record = await store.getProvider({ tenant_id: 'default', name: 'tfy-gateway', model_name: 'gpt-5' });
+
+ expect(record?.name).toBe('tfy-gateway');
+ expect(record?.manifest).toEqual({ ...GATEWAY_PROVIDER, type: 'custom', name: 'tfy-gateway' });
+ expect(inner.getProvider).not.toHaveBeenCalled();
+ });
+
+ it('falls through to the registry for a provider the request did not bring', async () => {
+ const { store } = modelProviderStoreWith({ 'tfy-gateway': GATEWAY_PROVIDER });
+
+ expect(await store.getProvider({ tenant_id: 'default', name: 'openai', model_name: 'gpt-5' })).toEqual(
+ registryProvider,
+ );
+ });
+
+ it('does not treat Object.prototype keys as inline resources', async () => {
+ const { store, inner } = modelProviderStoreWith({ 'tfy-gateway': GATEWAY_PROVIDER });
+
+ await store.getProvider({ tenant_id: 'default', name: 'constructor', model_name: 'gpt-5' });
+
+ expect(inner.getProvider).toHaveBeenCalledWith(
+ { tenant_id: 'default', name: 'constructor', model_name: 'gpt-5' },
+ undefined,
+ );
+ });
+
+ it('keeps request-scoped providers out of settings and the model picker', async () => {
+ const { store } = modelProviderStoreWith({ 'tfy-gateway': GATEWAY_PROVIDER });
+
+ const providers = await store.listProviders({ tenant_id: 'default' });
+ const models = await store.listModels({ tenant_id: 'default' });
+
+ expect(providers.map(record => record.name)).toEqual(['openai']);
+ expect(models).toEqual([]);
+ });
+
+ it('adds only x-tfy-metadata on inline invokes, since the manifest already carries the api_key', async () => {
+ const { store, inner } = modelProviderStoreWith({ 'tfy-gateway': GATEWAY_PROVIDER });
+ const record = { ...registryProvider, name: 'tfy-gateway' };
+
+ const headers = await store.resolveInvokeHeaders({
+ record,
+ turnMetadata: {
+ sessionId: 'sess-1',
+ turnId: 'turn-1',
+ agent: { id: 'agent-1', name: 'named' },
+ requestHeaders: { 'x-tfy-metadata': JSON.stringify({ env: 'prod' }) },
+ },
+ });
+
+ expect(Object.keys(headers)).toEqual([X_TFY_METADATA]);
+ expect(JSON.parse(headers[X_TFY_METADATA] ?? '')).toMatchObject({
+ env: 'prod',
+ [`${TFG_METADATA_PREFIX}.session_id`]: 'sess-1',
+ [`${TFG_METADATA_PREFIX}.turn_id`]: 'turn-1',
+ [`${TFG_METADATA_PREFIX}.agent_id`]: 'agent-1',
+ [`${TFG_METADATA_PREFIX}.agent_name`]: 'named',
+ });
+ expect(inner.resolveInvokeHeaders).not.toHaveBeenCalled();
+ });
+
+ it('leaves a registry provider to the store that knows how to authenticate it', async () => {
+ const { store, inner } = modelProviderStoreWith({ 'tfy-gateway': GATEWAY_PROVIDER });
+
+ expect(await store.resolveInvokeHeaders({ record: registryProvider })).toEqual({
+ Authorization: 'Bearer caller-token',
+ });
+ expect(inner.resolveInvokeHeaders).toHaveBeenCalled();
+ });
+});
+
describe('InlineMcpServerStore', () => {
it('sends the credentials the manifest carries, with no caller Bearer added over them', () => {
const { store } = mcpStoreWith({ 'docs-mcp': DOCS_MCP });