Python
test: qualify production partner packages with a local model provider
Rohit P0bf27a9c303e
diff --git a/harness/partner/build-sdk.integration.mjs b/harness/partner/build-sdk.integration.mjsindex 83aa1988..68b0f400 100644--- a/harness/partner/build-sdk.integration.mjs+++ b/harness/partner/build-sdk.integration.mjs@@ -1,7 +1,9 @@-// Explicit real build/runtime qualification; run on Linux, no provider secrets.+// Real production-package qualification with a local HTTP/SSE model, no live keys.import { test } from "node:test";import assert from "node:assert/strict";-import { execFileSync } from "node:child_process";+import { execFile } from "node:child_process";+import { promisify } from "node:util";+import { createServer } from "node:http";import {mkdtemp,mkdir,@@ -10,6 +12,7 @@ import {copyFile,rename,rm,+ realpath,} from "node:fs/promises";import { tmpdir } from "node:os";import { join, resolve, dirname } from "node:path";@@ -21,17 +24,45 @@ import { Harness } from "../typescript-sdk/dist/index.js";const here = dirname(fileURLToPath(import.meta.url));const sdkRoot = resolve(here, "..");const toml = (value) => JSON.stringify(value.replaceAll("\\", "/"));+const runFile = promisify(execFile);test("external native tool crate produces SDK and executable usable in Rust and TypeScript",{ timeout: 900_000 },async (t) => {- assert.equal(- process.platform,- "linux",- "This acceptance profile is Linux-only",- );- const root = await mkdtemp(join(tmpdir(), "ow-partner-build-"));+ const root = await realpath(await mkdtemp(join(tmpdir(), "ow-partner-build-")));t.after(() => rm(root, { recursive: true, force: true }));+ let requests = 0;+ let providerError;+ const server = createServer(async (req, res) => {+ try {+ let raw = "";+ for await (const chunk of req) {+ raw += chunk;+ assert.ok(raw.length < 1_000_000);+ }+ const body = JSON.parse(raw);+ assert.equal(req.url, "/v1/responses");+ assert.equal(body.stream, true);+ assert.ok(body.tools.some(tool => tool.name === "partner__note"));+ assert.ok(++requests <= 6, "Unexpected model retry or extra request");+ const done = body.input.some(item => item.type === "function_call_output");+ const output = done+ ? [{type: "message", role: "assistant", content: [{type: "output_text", text: "Partner proof written."}]}]+ : [{type: "function_call", call_id: "partner-note", name: "partner__note", arguments: "{}"}];+ res.writeHead(200, {"Content-Type": "text/event-stream"});+ res.end(`data: ${JSON.stringify({type: "response.completed", response: {status: "completed", output, usage: {input_tokens: 10, output_tokens: 4}}})}\n\n`);+ } catch (error) {+ providerError = error;+ res.writeHead(500);+ res.end();+ }+ });+ await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));+ const baseUrl = `http://127.0.0.1:${server.address().port}/v1`;+ t.after(async () => {+ server.closeAllConnections();+ await new Promise(resolve => server.close(resolve));+ });const tools = join(root, "custom tools");await mkdir(join(tools, "src"), { recursive: true });const toolsManifest = `[package]\nname = "independent-partner-tools"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nanyhow = "1"\nserde_json = "1"\nopenworker-tools = { path = ${toml(join(sdkRoot, "tools-sdk"))} }\n`;@@ -79,7 +110,8 @@ test(join(here, "test-fixtures/sdk-consumer.rs"),join(consumer, "src/main.rs"),);- execFileSync(+ // Keep Node's event loop free to serve the model while the Rust consumer runs.+ const consumerRun = await runFile("cargo",["run",@@ -89,13 +121,16 @@ test("--",info.directory,join(root, "rust-tests"),+ baseUrl,],{cwd: sdkRoot,- stdio: "inherit",+ maxBuffer: 10 * 1024 * 1024,timeout: 600_000,},);+ process.stdout.write(consumerRun.stdout);+ process.stderr.write(consumerRun.stderr);// Runtime deployment needs the binary, not the custom tool's source directory.const install = join(root, "relocated install");@@ -105,24 +140,33 @@ test(await rename(tools, join(root, "source unavailable at build path"));const work = join(root, "node-work");await mkdir(work);+ // Qualification must not enable test-support in a distributable package.+ await assert.rejects(Harness.start({+ executable: relocated,+ stateDir: join(root, "scripted-rejected"),+ sandbox: "off",+ scripted: true,+ }));const h = await Harness.start({executable: relocated,stateDir: join(root, "node-state"),sandbox: "off",- scripted: true,agents: [{id: "assistant",instructions: "Use the test tool.",+ model: { connection: "Fixture", model: "fixture" },tools: ["partner.note"],},],});try {+ await h.setModelProvider({+ name: "Fixture", api: "responses", baseUrl,+ credentials: { apiKey: "synthetic-partner-key", storage: "memory" },+ });const s = await h.session("assistant", { workspace: work });- const pending = await s.run(- 'TEST_RESOURCE_GRANTS [{"name":"partner__note","arguments":{}}]',- );+ const pending = await s.run("Write the partner proof using your tool.");assert.equal(pending.status, "interrupted");await assert.rejects(readFile(join(work, "partner-proof.txt")), {code: "ENOENT",@@ -139,6 +183,8 @@ test(console.log("verified relocated binary from TypeScript with native tool source absent",);+ assert.equal(providerError, undefined);+ assert.equal(requests, 6, "Embedded Rust, subprocess Rust and TypeScript each used the production provider path");// A bad native registration contract must fail the actual build. Never publish// a misleading successful wrapper or overwrite partner source to make it work.diff --git a/harness/partner/test-fixtures/sdk-consumer.rs b/harness/partner/test-fixtures/sdk-consumer.rsindex 9660607b..e13f234a 100644--- a/harness/partner/test-fixtures/sdk-consumer.rs+++ b/harness/partner/test-fixtures/sdk-consumer.rs@@ -14,14 +14,21 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {std::fs::create_dir_all(&workspace)?;let mut config = fixture_sdk::config(root.join(format!("state-{name}")), &package);config.sandbox = Sandbox::Off;- config.scripted = true;config.agents = vec![Agent::new("assistant").instructions("Use the test tool.")+ .model("Fixture", "fixture").tool("partner.note")];let harness = fixture_sdk::start(config, mode).await?;+ harness+ .set_model_provider(+ ModelProvider::responses("Fixture")+ .base_url(args[3].clone())+ .credentials(Credentials::secure("synthetic-partner-key").memory_only()),+ )+ .await?;let session = harness.session("assistant", &workspace).await?;let result = session- .run("TEST_RESOURCE_GRANTS [{\"name\":\"partner__note\",\"arguments\":{}}]")+ .run("Write the partner proof using your tool.").await?;assert_eq!(result.status, RunStatus::Interrupted);assert!(!workspace.join("partner-proof.txt").exists());