Skip to content

The open-source agent harness - the runtime layer that turns an LLM into a working agent.

feat: add warning field for missing model properties in API response (#1143)

Sujai Kumar Gupta288d76ee3744
diff --git a/.changeset/model-warning-missing-properties.md b/.changeset/model-warning-missing-properties.md
new file mode 100644
index 000000000..8891f044e
--- /dev/null
+++ b/.changeset/model-warning-missing-properties.md
@@ -0,0 +1,5 @@
+---
+"@truefoundry/trueforge": patch
+---
+
+Add a `warning` field to models returned by `GET /api/v1/models` when reasoning effort or context length is not configured, noting it may affect TrueForge performance.
diff --git a/.github/fern/openapi/openapi.json b/.github/fern/openapi/openapi.json
index 3ba7bba95..f56a160a2 100644
--- a/.github/fern/openapi/openapi.json
+++ b/.github/fern/openapi/openapi.json
@@ -397,6 +397,11 @@
},
"provider": {
"$ref": "#/components/schemas/AvailableModelProvider"
+ },
+ "warning": {
+ "description": "Set when reasoning effort or context length is not configured; may affect TrueForge performance.",
+ "minLength": 1,
+ "type": "string"
}
},
"required": [
diff --git a/docs/openapi.json b/docs/openapi.json
index 3ba7bba95..f56a160a2 100644
--- a/docs/openapi.json
+++ b/docs/openapi.json
@@ -397,6 +397,11 @@
},
"provider": {
"$ref": "#/components/schemas/AvailableModelProvider"
+ },
+ "warning": {
+ "description": "Set when reasoning effort or context length is not configured; may affect TrueForge performance.",
+ "minLength": 1,
+ "type": "string"
}
},
"required": [
diff --git a/packages/trueforge-sdk/src/api/types/AvailableModel.ts b/packages/trueforge-sdk/src/api/types/AvailableModel.ts
index 66e14db24..5703b5f0e 100644
--- a/packages/trueforge-sdk/src/api/types/AvailableModel.ts
+++ b/packages/trueforge-sdk/src/api/types/AvailableModel.ts
@@ -9,4 +9,6 @@ export interface AvailableModel {
name: string;
properties: TrueForge.ModelProperties;
provider: TrueForge.AvailableModelProvider;
+ /** Warning for not so capable models. */
+ warning?: string;
}
diff --git a/packages/trueforge-sdk/src/serialization/types/AvailableModel.ts b/packages/trueforge-sdk/src/serialization/types/AvailableModel.ts
index affcd2ede..9df18dd52 100644
--- a/packages/trueforge-sdk/src/serialization/types/AvailableModel.ts
+++ b/packages/trueforge-sdk/src/serialization/types/AvailableModel.ts
@@ -12,6 +12,7 @@ export const AvailableModel: core.serialization.ObjectSchema<serializers.Availab
name: core.serialization.string(),
properties: ModelProperties,
provider: AvailableModelProvider,
+ warning: core.serialization.string().optional(),
});
export declare namespace AvailableModel {
@@ -20,5 +21,6 @@ export declare namespace AvailableModel {
name: string;
properties: ModelProperties.Raw;
provider: AvailableModelProvider.Raw;
+ warning?: string | null;
}
}
diff --git a/packages/trueforge-sdk/tests/wire/models.test.ts b/packages/trueforge-sdk/tests/wire/models.test.ts
index eb42a6f36..212dfa397 100644
--- a/packages/trueforge-sdk/tests/wire/models.test.ts
+++ b/packages/trueforge-sdk/tests/wire/models.test.ts
@@ -10,7 +10,9 @@ describe("ModelsClient", () => {
const client = new TrueForge({ maxRetries: 0, token: "test", baseUrl: server.baseUrl });
const rawResponseBody = {
- data: [{ model_id: "model_id", name: "name", properties: {}, provider: { name: "name" } }],
+ data: [
+ { model_id: "model_id", name: "name", properties: {}, provider: { name: "name" }, warning: "warning" },
+ ],
};
server.mockEndpoint().get("/api/v1/models").respondWith().statusCode(200).jsonBody(rawResponseBody).build();
@@ -25,6 +27,7 @@ describe("ModelsClient", () => {
provider: {
name: "name",
},
+ warning: "warning",
},
],
});
diff --git a/packages/trueforge/src/db/modelProviderStore.ts b/packages/trueforge/src/db/modelProviderStore.ts
index 3190b09f1..314f99319 100644
--- a/packages/trueforge/src/db/modelProviderStore.ts
+++ b/packages/trueforge/src/db/modelProviderStore.ts
@@ -4,7 +4,7 @@
* TrueFoundry (read-only ServiceFoundry server listing).
*/
import type { ResourceName } from '../schemas/common';
-import type { AvailableModel, ModelProviderManifest } from '../schemas/modelProvider';
+import type { AvailableModel, ModelProperties, ModelProviderManifest } from '../schemas/modelProvider';
import type { TurnMetadata } from './turnMetadata';
export interface ModelProviderRecord {
@@ -87,14 +87,33 @@ export interface IModelProviderStore<TTransaction = never> {
}): Promise<Record<string, string>>;
}
+export function modelPropertiesWarning(input: { properties: ModelProperties }): string | undefined {
+ const missing: string[] = [];
+ if (input.properties.reasoning_efforts === undefined || input.properties.reasoning_efforts.length === 0) {
+ missing.push('reasoning effort');
+ }
+ if (input.properties.context_length === undefined) {
+ missing.push('context length');
+ }
+ if (missing.length === 0) {
+ return undefined;
+ }
+ const missingText = missing.join(' and ');
+ return `${missingText.charAt(0).toUpperCase()}${missingText.slice(1)} ${missing.length > 1 ? 'are' : 'is'} not configured for this model, which may affect TrueForge's performance.`;
+}
+
/** Application-side flatten shared by both store implementations. */
export function flattenProviderModels(records: ModelProviderRecord[]): AvailableModel[] {
return records.flatMap(record =>
- record.manifest.models.map(model => ({
- name: `${record.name}/${model.name}`,
- model_id: model.model_id,
- provider: { name: record.name },
- properties: model.properties,
- })),
+ record.manifest.models.map(model => {
+ const warning = modelPropertiesWarning({ properties: model.properties });
+ return {
+ name: `${record.name}/${model.name}`,
+ model_id: model.model_id,
+ provider: { name: record.name },
+ properties: model.properties,
+ ...(warning === undefined ? {} : { warning }),
+ };
+ }),
);
}
diff --git a/packages/trueforge/src/schemas/modelProvider.ts b/packages/trueforge/src/schemas/modelProvider.ts
index a2e8d77b2..8dd128f1d 100644
--- a/packages/trueforge/src/schemas/modelProvider.ts
+++ b/packages/trueforge/src/schemas/modelProvider.ts
@@ -245,6 +245,11 @@ export const AvailableModelSchema = z
model_id: z.string().describe('Upstream, provider-specific identifier sent to the provider API.'),
provider: AvailableModelProviderSchema,
properties: ModelPropertiesSchema,
+ warning: z
+ .string()
+ .min(1)
+ .optional()
+ .describe('Set when reasoning effort or context length is not configured; may affect TrueForge performance.'),
})
.strict()
.openapi('AvailableModel');
diff --git a/packages/trueforge/tests/db/modelProviderStoreContractSuite.ts b/packages/trueforge/tests/db/modelProviderStoreContractSuite.ts
index 7e0f266cc..084d715e5 100644
--- a/packages/trueforge/tests/db/modelProviderStoreContractSuite.ts
+++ b/packages/trueforge/tests/db/modelProviderStoreContractSuite.ts
@@ -139,7 +139,7 @@ export function runModelProviderStoreContractSuite(getStore: () => IModelProvide
await store.upsertProvider({ tenant_id: 'other-tenant', name: 'openai', manifest: openai });
const models = await store.listModels({ tenant_id: TENANT });
- expect(models).toEqual([
+ expect(models).toMatchObject([
{
name: 'anthropic/claude-sonnet-4-6',
model_id: 'claude-sonnet-4-6',
diff --git a/python/trueforge_sdk/src/trueforge_sdk/types/available_model.py b/python/trueforge_sdk/src/trueforge_sdk/types/available_model.py
index c683de00d..2f9024d0f 100644
--- a/python/trueforge_sdk/src/trueforge_sdk/types/available_model.py
+++ b/python/trueforge_sdk/src/trueforge_sdk/types/available_model.py
@@ -22,6 +22,10 @@ class AvailableModel(UncheckedBaseModel):
properties: ModelProperties
provider: AvailableModelProvider
+ warning: typing.Optional[str] = pydantic.Field(default=None)
+ """
+ Warning for not so capable models.
+ """
if IS_PYDANTIC_V2:
model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow") # type: ignore # Pydantic v2