Skip to content
Python

test: qualify production partner packages with a local model provider

Rohit P0bf27a9c303e
diff --git a/harness/partner/build-sdk.integration.mjs b/harness/partner/build-sdk.integration.mjs
index 83aa1988..68b0f400 100644
--- a/harness/partner/build-sdk.integration.mjs
+++ b/harness/partner/build-sdk.integration.mjs
@@ -1,7 +1,9 @@
-// Explicit real build/runtime qualification; run on Linux, no provider secrets.
+// Real production-package qualification with a local HTTP/SSE model, no live keys.
import { test } from "node:test";
import assert from "node:assert/strict";
-import { execFileSync } from "node:child_process";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { createServer } from "node:http";
import {
mkdtemp,
mkdir,
@@ -10,6 +12,7 @@ import {
copyFile,
rename,
rm,
+ realpath,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve, dirname } from "node:path";
@@ -21,17 +24,45 @@ import { Harness } from "../typescript-sdk/dist/index.js";
const here = dirname(fileURLToPath(import.meta.url));
const sdkRoot = resolve(here, "..");
const toml = (value) => JSON.stringify(value.replaceAll("\\", "/"));
+const runFile = promisify(execFile);
test(
"external native tool crate produces SDK and executable usable in Rust and TypeScript",
{ timeout: 900_000 },
async (t) => {
- assert.equal(
- process.platform,
- "linux",
- "This acceptance profile is Linux-only",
- );
- const root = await mkdtemp(join(tmpdir(), "ow-partner-build-"));
+ const root = await realpath(await mkdtemp(join(tmpdir(), "ow-partner-build-")));
t.after(() => rm(root, { recursive: true, force: true }));
+ let requests = 0;
+ let providerError;
+ const server = createServer(async (req, res) => {
+ try {
+ let raw = "";
+ for await (const chunk of req) {
+ raw += chunk;
+ assert.ok(raw.length < 1_000_000);
+ }
+ const body = JSON.parse(raw);
+ assert.equal(req.url, "/v1/responses");
+ assert.equal(body.stream, true);
+ assert.ok(body.tools.some(tool => tool.name === "partner__note"));
+ assert.ok(++requests <= 6, "Unexpected model retry or extra request");
+ const done = body.input.some(item => item.type === "function_call_output");
+ const output = done
+ ? [{type: "message", role: "assistant", content: [{type: "output_text", text: "Partner proof written."}]}]
+ : [{type: "function_call", call_id: "partner-note", name: "partner__note", arguments: "{}"}];
+ res.writeHead(200, {"Content-Type": "text/event-stream"});
+ res.end(`data: ${JSON.stringify({type: "response.completed", response: {status: "completed", output, usage: {input_tokens: 10, output_tokens: 4}}})}\n\n`);
+ } catch (error) {
+ providerError = error;
+ res.writeHead(500);
+ res.end();
+ }
+ });
+ await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
+ const baseUrl = `http://127.0.0.1:${server.address().port}/v1`;
+ t.after(async () => {
+ server.closeAllConnections();
+ await new Promise(resolve => server.close(resolve));
+ });
const tools = join(root, "custom tools");
await mkdir(join(tools, "src"), { recursive: true });
const toolsManifest = `[package]\nname = "independent-partner-tools"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nanyhow = "1"\nserde_json = "1"\nopenworker-tools = { path = ${toml(join(sdkRoot, "tools-sdk"))} }\n`;
@@ -79,7 +110,8 @@ test(
join(here, "test-fixtures/sdk-consumer.rs"),
join(consumer, "src/main.rs"),
);
- execFileSync(
+ // Keep Node's event loop free to serve the model while the Rust consumer runs.
+ const consumerRun = await runFile(
"cargo",
[
"run",
@@ -89,13 +121,16 @@ test(
"--",
info.directory,
join(root, "rust-tests"),
+ baseUrl,
],
{
cwd: sdkRoot,
- stdio: "inherit",
+ maxBuffer: 10 * 1024 * 1024,
timeout: 600_000,
},
);
+ process.stdout.write(consumerRun.stdout);
+ process.stderr.write(consumerRun.stderr);
// Runtime deployment needs the binary, not the custom tool's source directory.
const install = join(root, "relocated install");
@@ -105,24 +140,33 @@ test(
await rename(tools, join(root, "source unavailable at build path"));
const work = join(root, "node-work");
await mkdir(work);
+ // Qualification must not enable test-support in a distributable package.
+ await assert.rejects(Harness.start({
+ executable: relocated,
+ stateDir: join(root, "scripted-rejected"),
+ sandbox: "off",
+ scripted: true,
+ }));
const h = await Harness.start({
executable: relocated,
stateDir: join(root, "node-state"),
sandbox: "off",
- scripted: true,
agents: [
{
id: "assistant",
instructions: "Use the test tool.",
+ model: { connection: "Fixture", model: "fixture" },
tools: ["partner.note"],
},
],
});
try {
+ await h.setModelProvider({
+ name: "Fixture", api: "responses", baseUrl,
+ credentials: { apiKey: "synthetic-partner-key", storage: "memory" },
+ });
const s = await h.session("assistant", { workspace: work });
- const pending = await s.run(
- 'TEST_RESOURCE_GRANTS [{"name":"partner__note","arguments":{}}]',
- );
+ const pending = await s.run("Write the partner proof using your tool.");
assert.equal(pending.status, "interrupted");
await assert.rejects(readFile(join(work, "partner-proof.txt")), {
code: "ENOENT",
@@ -139,6 +183,8 @@ test(
console.log(
"verified relocated binary from TypeScript with native tool source absent",
);
+ assert.equal(providerError, undefined);
+ assert.equal(requests, 6, "Embedded Rust, subprocess Rust and TypeScript each used the production provider path");
// A bad native registration contract must fail the actual build. Never publish
// a misleading successful wrapper or overwrite partner source to make it work.
diff --git a/harness/partner/test-fixtures/sdk-consumer.rs b/harness/partner/test-fixtures/sdk-consumer.rs
index 9660607b..e13f234a 100644
--- a/harness/partner/test-fixtures/sdk-consumer.rs
+++ b/harness/partner/test-fixtures/sdk-consumer.rs
@@ -14,14 +14,21 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
std::fs::create_dir_all(&workspace)?;
let mut config = fixture_sdk::config(root.join(format!("state-{name}")), &package);
config.sandbox = Sandbox::Off;
- config.scripted = true;
config.agents = vec![Agent::new("assistant")
.instructions("Use the test tool.")
+ .model("Fixture", "fixture")
.tool("partner.note")];
let harness = fixture_sdk::start(config, mode).await?;
+ harness
+ .set_model_provider(
+ ModelProvider::responses("Fixture")
+ .base_url(args[3].clone())
+ .credentials(Credentials::secure("synthetic-partner-key").memory_only()),
+ )
+ .await?;
let session = harness.session("assistant", &workspace).await?;
let result = session
- .run("TEST_RESOURCE_GRANTS [{\"name\":\"partner__note\",\"arguments\":{}}]")
+ .run("Write the partner proof using your tool.")
.await?;
assert_eq!(result.status, RunStatus::Interrupted);
assert!(!workspace.join("partner-proof.txt").exists());