Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(intake): log each router's own model call (#74)

Elyas Mehtabuddin91e77dfdd48d
diff --git a/crates/switchyard-components/src/intake.rs b/crates/switchyard-components/src/intake.rs
index 94f6b423c..24ae31c11 100644
--- a/crates/switchyard-components/src/intake.rs
+++ b/crates/switchyard-components/src/intake.rs
@@ -9,14 +9,14 @@ pub mod context;
pub mod payload;
pub use client::{HttpIntakeSink, IntakeSink};
-pub use config::{IntakeQueueFullPolicy, IntakeSinkConfig};
+pub use config::{IntakeFormat, IntakeQueueFullPolicy, IntakeSinkConfig, IntakeTarget};
pub use context::{
- IntakeRequestMetadata, IntakeRequestState, RequestMetadata, INTAKE_APP_HEADER,
- INTAKE_ENABLED_HEADER, INTAKE_TASK_HEADER, PROXY_SESSION_ID_HEADER,
+ IntakeRequestMetadata, IntakeRequestState, RequestMetadata, SubModelCall, SubModelCalls,
+ INTAKE_APP_HEADER, INTAKE_ENABLED_HEADER, INTAKE_TASK_HEADER, PROXY_SESSION_ID_HEADER,
};
pub use payload::{
anthropic_response_from_stream, now_millis, openai_chat_response_from_stream,
- request_type_value, responses_response_from_stream, to_nvdataflow_document,
- IntakePayloadBuilder, IntakePayloadContext, IntakeStreamCapture, IntakeStreamFormat,
- SYNTHETIC_STREAM_RESPONSE_IDS, UNKNOWN_MODEL,
+ request_type_value, responses_response_from_stream, to_flat_document, IntakePayloadBuilder,
+ IntakePayloadContext, IntakeStreamCapture, IntakeStreamFormat, SYNTHETIC_STREAM_RESPONSE_IDS,
+ UNKNOWN_MODEL,
};
diff --git a/crates/switchyard-components/src/intake/client.rs b/crates/switchyard-components/src/intake/client.rs
index 3de1edef0..4cc18ec60 100644
--- a/crates/switchyard-components/src/intake/client.rs
+++ b/crates/switchyard-components/src/intake/client.rs
@@ -231,18 +231,18 @@ async fn post_once(
client: &reqwest::Client,
payload: &Value,
) -> Result<()> {
- // NVDataflow posting is unauthenticated; chat-completions ingest needs the bearer.
- let request = if let Some(url) = config.nvdataflow_posting_url() {
- client.post(url).json(payload)
- } else {
- let mut request = client
- .post(chat_completions_ingest_url(config))
- .json(payload);
+ // A configured target posts to its own URL with its own auth mode; otherwise
+ // fall back to the authenticated nemo-platform chat-completions ingest.
+ let (url, authenticated) = match &config.target {
+ Some(target) => (target.url.clone(), target.authenticated),
+ None => (chat_completions_ingest_url(config), true),
+ };
+ let mut request = client.post(url).json(payload);
+ if authenticated {
if let Some(api_key) = config.api_key.as_deref() {
request = request.bearer_auth(api_key);
}
- request
- };
+ }
let response = request.send().await.map_err(|error| {
SwitchyardError::Upstream(format!("intake payload POST failed: {error}"))
})?;
@@ -261,9 +261,9 @@ async fn post_once(
/// Validates sink configuration before any background worker is started.
fn validate_config(config: &IntakeSinkConfig) -> Result<()> {
- // NVDataflow mode defaults its own host, so intake_base_url is only
- // required for chat-completions ingest.
- if config.nvdataflow_project.is_none()
+ // A configured target carries its own URL; only the nemo-platform fallback
+ // needs intake_base_url to build the ingest URL.
+ if config.target.is_none()
&& !matches!(config.intake_base_url.as_deref(), Some(base_url) if !base_url.is_empty())
{
return Err(SwitchyardError::InvalidConfig(
diff --git a/crates/switchyard-components/src/intake/config.rs b/crates/switchyard-components/src/intake/config.rs
index e3a55e6c7..ed8ae2839 100644
--- a/crates/switchyard-components/src/intake/config.rs
+++ b/crates/switchyard-components/src/intake/config.rs
@@ -18,6 +18,31 @@ pub enum IntakeQueueFullPolicy {
Block,
}
+/// Payload shape a configured intake target expects.
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum IntakeFormat {
+ /// Nested OpenAI chat-completions JSON (the nemo-platform ingest shape).
+ ChatCompletions,
+ /// Flat, top-level type-prefixed telemetry document (data-lake posting shape).
+ FlatDocument,
+}
+
+/// A configurable intake destination.
+///
+/// When a sink has no target it posts nested chat-completions documents to the
+/// authenticated nemo-platform ingest URL built from `intake_base_url` +
+/// `workspace`. Set a target to post a chosen payload shape to an explicit URL.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct IntakeTarget {
+ /// Full posting URL. The caller builds it; the sink POSTs to it verbatim.
+ pub url: String,
+ /// Payload shape to emit.
+ pub format: IntakeFormat,
+ /// Send the sink's `api_key` as a bearer token with each POST.
+ pub authenticated: bool,
+}
+
/// Runtime configuration for the HTTP intake sink.
#[derive(Clone, PartialEq, Serialize, Deserialize)]
pub struct IntakeSinkConfig {
@@ -29,9 +54,9 @@ pub struct IntakeSinkConfig {
pub user_id: String,
/// Bearer token used when posting intake payloads.
pub api_key: Option<String>,
- /// When set, post a flat NVDataflow telemetry document to this project's
- /// posting endpoint instead of the nemo-platform chat-completions ingest.
- pub nvdataflow_project: Option<String>,
+ /// Optional explicit destination. Unset posts nested chat-completions
+ /// documents to the authenticated nemo-platform ingest.
+ pub target: Option<IntakeTarget>,
/// Maximum buffered payloads before applying `on_queue_full`.
pub max_queue_size: usize,
/// Per-request HTTP timeout in seconds.
@@ -53,7 +78,7 @@ impl fmt::Debug for IntakeSinkConfig {
.field("workspace", &self.workspace)
.field("user_id", &self.user_id)
.field("api_key", &self.api_key.as_ref().map(|_| "<redacted>"))
- .field("nvdataflow_project", &self.nvdataflow_project)
+ .field("target", &self.target)
.field("max_queue_size", &self.max_queue_size)
.field("request_timeout_s", &self.request_timeout_s)
.field("max_retries", &self.max_retries)
@@ -70,7 +95,7 @@ impl Default for IntakeSinkConfig {
workspace: None,
user_id: "switchyard".to_string(),
api_key: None,
- nvdataflow_project: None,
+ target: None,
max_queue_size: 1000,
request_timeout_s: 10.0,
max_retries: 2,
@@ -80,25 +105,9 @@ impl Default for IntakeSinkConfig {
}
}
-/// Default NVDataflow host used when `intake_base_url` is unset.
-pub const DEFAULT_NVDATAFLOW_BASE_URL: &str = "https://nvdataflow.nvidia.com";
-
impl IntakeSinkConfig {
/// Returns the configured workspace or the Python-compatible default.
pub fn workspace_or_default(&self) -> &str {
self.workspace.as_deref().unwrap_or("default")
}
-
- /// Posting URL for the configured NVDataflow project, or `None` when the
- /// sink runs in chat-completions ingest mode.
- pub fn nvdataflow_posting_url(&self) -> Option<String> {
- self.nvdataflow_project.as_deref().map(|project| {
- let base = self
- .intake_base_url
- .as_deref()
- .unwrap_or(DEFAULT_NVDATAFLOW_BASE_URL)
- .trim_end_matches('/');
- format!("{base}/dataflow/{project}/posting")
- })
- }
}
diff --git a/crates/switchyard-components/src/intake/context.rs b/crates/switchyard-components/src/intake/context.rs
index ea13af048..9d633494a 100644
--- a/crates/switchyard-components/src/intake/context.rs
+++ b/crates/switchyard-components/src/intake/context.rs
@@ -77,6 +77,30 @@ impl IntakeRequestState {
}
}
+/// One routing-strategy sub-model call captured for intake: the model a router
+/// called to pick a route, plus that call's token usage. Never message content.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct SubModelCall {
+ /// Model the router called to make its routing decision.
+ pub model: String,
+ /// Prompt tokens the routing call spent.
+ pub prompt_tokens: i64,
+ /// Completion tokens the routing call spent.
+ pub completion_tokens: i64,
+ /// Cached prompt tokens the routing call reused.
+ pub cached_tokens: i64,
+ /// Router family that made the call (e.g. `deterministic`, `stage_router`).
+ pub router_type: String,
+ /// Route label the router produced (e.g. `classifier`, `planner`).
+ pub routed_to: String,
+}
+
+/// Routing-strategy sub-model calls made while serving one request. Routers
+/// append one entry per call; the response-side intake processor emits each as
+/// its own anonymous record.
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct SubModelCalls(pub Vec<SubModelCall>);
+
// Header extraction treats empty strings as absent to match Python behavior.
fn header_value(headers: &BTreeMap<String, &str>, name: &str) -> Option<String> {
headers
diff --git a/crates/switchyard-components/src/intake/payload.rs b/crates/switchyard-components/src/intake/payload.rs
index 93664d716..f6bac0534 100644
--- a/crates/switchyard-components/src/intake/payload.rs
+++ b/crates/switchyard-components/src/intake/payload.rs
@@ -15,8 +15,8 @@ use switchyard_core::{
use switchyard_translation::{TranslationEngine, TranslationPolicy, WireFormat};
use crate::backends::BackendSelection;
-use crate::intake::config::IntakeSinkConfig;
-use crate::intake::context::{IntakeRequestState, RequestMetadata};
+use crate::intake::config::{IntakeFormat, IntakeSinkConfig};
+use crate::intake::context::{IntakeRequestState, RequestMetadata, SubModelCall};
use crate::request_processors::RandomRoutingDecision;
use crate::stats::{estimate_model_cost, usage_from_body, StatsRouteLabel};
use crate::telemetry::switchyard_version;
@@ -124,7 +124,7 @@ impl IntakePayloadBuilder {
) -> Result<Value> {
let openai_request = self.translate_request_to_openai(request_snapshot)?;
let openai_response = self.translate_response_to_openai(response)?;
- self.build_from_openai(ctx, openai_request, openai_response, stream)
+ self.build_from_openai(ctx, openai_request, openai_response, stream, None)
}
/// Builds a payload from an already-normalized OpenAI Chat response body.
@@ -136,16 +136,65 @@ impl IntakePayloadBuilder {
stream: bool,
) -> Result<Value> {
let openai_request = self.translate_request_to_openai(request_snapshot)?;
- self.build_from_openai(ctx, openai_request, openai_response, stream)
+ self.build_from_openai(ctx, openai_request, openai_response, stream, None)
+ }
+
+ /// Builds an anonymous intake record for one routing-strategy sub-model call.
+ ///
+ /// The record carries only the router's own model, token usage, and route
+ /// label — never message content. It reuses the turn's session, task, and
+ /// timestamps but overrides the routing label with this call's own. `index`
+ /// yields a per-call id suffix so several calls in one turn stay distinct as
+ /// separate flat documents instead of overwriting one another.
+ pub fn build_submodel_record(
+ &self,
+ ctx: &IntakePayloadContext,
+ call: &SubModelCall,
+ index: usize,
+ ) -> Result<Value> {
+ let openai_request = json!({ "model": call.model });
+ let openai_response = json!({
+ "model": call.model,
+ "object": "chat.completion",
+ "choices": [],
+ "usage": {
+ "prompt_tokens": call.prompt_tokens,
+ "completion_tokens": call.completion_tokens,
+ "total_tokens": call.prompt_tokens.saturating_add(call.completion_tokens),
+ "prompt_tokens_details": { "cached_tokens": call.cached_tokens },
+ },
+ });
+ let mut call_ctx = ctx.clone();
+ call_ctx.routing = Some(IntakeRoutingMetadata {
+ router_type: call.router_type.clone(),
+ routed_to: call.routed_to.clone(),
+ });
+ // A routing call's latency isn't the turn's, so drop the start stamp:
+ // no misleading latency_ms is derived and created_at falls back to the
+ // turn end (≈ now), matching the Python record this replaces.
+ call_ctx.started_at_ms = None;
+ let discriminator = format!("{}-{index}", call.routed_to);
+ self.build_from_openai(
+ &call_ctx,
+ openai_request,
+ openai_response,
+ false,
+ Some(&discriminator),
+ )
}
/// Assembles the final intake API JSON payload.
+ ///
+ /// `id_discriminator` makes a flat document's `_id` unique among
+ /// records built for the same turn (same session + timestamp); the primary
+ /// per-turn record passes `None`.
fn build_from_openai(
&self,
ctx: &IntakePayloadContext,
openai_request: Value,
openai_response: Value,
stream: bool,
+ id_discriminator: Option<&str>,
) -> Result<Value> {
let mut request_entry = object_from_value(openai_request, "OpenAI intake request")?;
normalize_logged_stream_request(&mut request_entry, stream);
@@ -184,8 +233,17 @@ impl IntakePayloadBuilder {
Value::String("switchyard".to_string()),
);
let payload = Value::Object(payload);
- if self.config().nvdataflow_project.is_some() {
- return Ok(to_nvdataflow_document(&payload, ctx.ended_at_ms));
+ let flat = self
+ .config()
+ .target
+ .as_ref()
+ .is_some_and(|target| target.format == IntakeFormat::FlatDocument);
+ if flat {
+ return Ok(to_flat_document(
+ &payload,
+ ctx.ended_at_ms,
+ id_discriminator,
+ ));
}
Ok(payload)
}
@@ -859,10 +917,14 @@ fn anthropic_stop_reason_to_openai(reason: Value) -> Value {
}
/// Flattens a chat-completions intake payload into a top-level, type-prefixed
-/// (`s_`/`l_`/`f_`/`b_`/`text_`) NVDataflow document, since NVDataflow only
-/// indexes top-level fields. The raw record is kept in
+/// (`s_`/`l_`/`f_`/`b_`/`text_`) flat document, since flat posting endpoints
+/// only index top-level fields. The raw record is kept in
/// `text_switchyard_record_json`.
-pub fn to_nvdataflow_document(payload: &Value, ts_created_ms: Option<i64>) -> Value {
+pub fn to_flat_document(
+ payload: &Value,
+ ts_created_ms: Option<i64>,
+ id_discriminator: Option<&str>,
+) -> Value {
let request = payload.get("request");
let response = payload.get("response");
let switchyard = request.and_then(|value| value.get("switchyard"));
@@ -876,6 +938,12 @@ pub fn to_nvdataflow_document(payload: &Value, ts_created_ms: Option<i64>) -> Va
Some(session_id) => format!("{session_id}-{ts}"),
None => format!("switchyard-{ts}"),
};
+ // Several records can share one session + timestamp (the primary turn plus
+ // each routing sub-model call); the discriminator keeps their `_id`s unique.
+ let id = match id_discriminator {
+ Some(discriminator) => format!("{id}-{discriminator}"),
+ None => id,
+ };
doc.insert("_id".to_string(), Value::String(id));
doc.insert("ts_created".to_string(), Value::from(ts));
doc.insert(
diff --git a/crates/switchyard-components/src/lib.rs b/crates/switchyard-components/src/lib.rs
index 5041b63c0..5bd2e7180 100644
--- a/crates/switchyard-components/src/lib.rs
+++ b/crates/switchyard-components/src/lib.rs
@@ -24,8 +24,9 @@ pub use dimension_collector::{
ScoringConfig, ToolResultSignal,
};
pub use intake::{
- HttpIntakeSink, IntakePayloadBuilder, IntakeQueueFullPolicy, IntakeRequestMetadata,
- IntakeRequestState, IntakeSink, IntakeSinkConfig, RequestMetadata,
+ HttpIntakeSink, IntakeFormat, IntakePayloadBuilder, IntakeQueueFullPolicy,
+ IntakeRequestMetadata, IntakeRequestState, IntakeSink, IntakeSinkConfig, IntakeTarget,
+ RequestMetadata, SubModelCall, SubModelCalls,
};
pub use request_processors::{
DimensionCollector, IntakeRequestProcessor, RandomRoutingDecision, RandomRoutingEngine,
diff --git a/crates/switchyard-components/src/response_processors/intake.rs b/crates/switchyard-components/src/response_processors/intake.rs
index a212af21c..8a6c09cd9 100644
--- a/crates/switchyard-components/src/response_processors/intake.rs
+++ b/crates/switchyard-components/src/response_processors/intake.rs
@@ -11,7 +11,7 @@ use switchyard_core::{BoxResponseStream, ChatResponse, ProxyContext, Result};
use crate::intake::{
now_millis, HttpIntakeSink, IntakePayloadBuilder, IntakePayloadContext, IntakeRequestState,
- IntakeSink, IntakeSinkConfig, IntakeStreamCapture, IntakeStreamFormat,
+ IntakeSink, IntakeSinkConfig, IntakeStreamCapture, IntakeStreamFormat, SubModelCalls,
};
/// Response processor that converts completed responses into intake payloads.
@@ -57,6 +57,13 @@ impl IntakeResponseProcessor {
return Ok(response);
}
+ // Routing sub-model records don't depend on the response body — the
+ // router already made its call at request time — so emit them here,
+ // gated by the same opt-in, instead of threading through the
+ // buffered/streaming split below.
+ self.emit_submodel_records(ctx, Arc::clone(&self.sink))
+ .await;
+
match response {
ChatResponse::OpenAiCompletion(_)
| ChatResponse::OpenAiResponsesCompletion(_)
@@ -100,6 +107,23 @@ impl IntakeResponseProcessor {
}
impl IntakeResponseProcessor {
+ /// Emits one anonymous record per routing sub-model call stashed on context.
+ async fn emit_submodel_records(&self, ctx: &ProxyContext, sink: Arc<dyn IntakeSink>) {
+ // Clone the calls out before the await so no context borrow is held
+ // across it.
+ let calls = match ctx.get::<SubModelCalls>() {
+ Some(calls) if !calls.0.is_empty() => calls.0.clone(),
+ _ => return,
+ };
+ let payload_ctx = IntakePayloadContext::from_proxy_context(ctx, Some(now_millis()));
+ for (index, call) in calls.iter().enumerate() {
+ let payload = self
+ .builder
+ .build_submodel_record(&payload_ctx, call, index);
+ enqueue_payload(Arc::clone(&sink), payload).await;
+ }
+ }
+
/// Builds a payload for a buffered response using request state from context.
fn buffered_payload(
&self,
diff --git a/crates/switchyard-components/tests/nvdataflow_live.rs b/crates/switchyard-components/tests/flat_target_live.rs
similarity index 64%
rename from crates/switchyard-components/tests/nvdataflow_live.rs
rename to crates/switchyard-components/tests/flat_target_live.rs
index c72469cff..7e129cc61 100644
--- a/crates/switchyard-components/tests/nvdataflow_live.rs
+++ b/crates/switchyard-components/tests/flat_target_live.rs
@@ -1,24 +1,26 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Live NVDataflow sandbox test. Ignored by default (needs the NVIDIA internal
-//! network / VPN). Run explicitly:
+//! Live flat-document posting test. Ignored by default (needs a reachable
+//! posting endpoint, typically on the NVIDIA internal network / VPN). Run:
//!
-//! cargo test -p switchyard-components --test nvdataflow_live -- --ignored
+//! SWITCHYARD_INTAKE_TARGET_URL=<full posting url> \
+//! cargo test -p switchyard-components --test flat_target_live -- --ignored
//!
-//! Override the project with SWITCHYARD_NVDATAFLOW_PROJECT. Posting is
-//! fail-open, so this drives records through the real sink; verify they landed
-//! by querying df-<project>-<YYYYMM>.
+//! Posting is fail-open, so this drives records through the real sink; verify
+//! they landed by querying the target store. No-ops when the env var is unset.
use std::time::{SystemTime, UNIX_EPOCH};
use serde_json::{json, Value};
-use switchyard_components::intake::to_nvdataflow_document;
-use switchyard_components::{HttpIntakeSink, IntakeSink, IntakeSinkConfig};
+use switchyard_components::intake::to_flat_document;
+use switchyard_components::{
+ HttpIntakeSink, IntakeFormat, IntakeSink, IntakeSinkConfig, IntakeTarget,
+};
use switchyard_core::Result;
// A representative chat-completions intake payload, the input the production
-// builder produces before NVDataflow flattening.
+// builder produces before flattening.
fn sample_chat_payload(
session: &str,
served: &str,
@@ -54,12 +56,18 @@ fn sample_chat_payload(
}
#[tokio::test]
-#[ignore = "live: posts to the NVDataflow sandbox; requires VPN. run with --ignored"]
-async fn posts_flat_documents_to_nvdataflow_sandbox() -> Result<()> {
- let project = std::env::var("SWITCHYARD_NVDATAFLOW_PROJECT")
- .unwrap_or_else(|_| "sandbox-switchyard".to_string());
+#[ignore = "live: posts to a real flat-document endpoint; requires VPN. run with --ignored"]
+async fn posts_flat_documents_to_live_target() -> Result<()> {
+ // No endpoint configured means nothing to exercise; keep the test a no-op.
+ let Ok(url) = std::env::var("SWITCHYARD_INTAKE_TARGET_URL") else {
+ return Ok(());
+ };
let sink = HttpIntakeSink::new(IntakeSinkConfig {
- nvdataflow_project: Some(project),
+ target: Some(IntakeTarget {
+ url,
+ format: IntakeFormat::FlatDocument,
+ authenticated: false,
+ }),
..IntakeSinkConfig::default()
})?;
@@ -89,7 +97,8 @@ async fn posts_flat_documents_to_nvdataflow_sandbox() -> Result<()> {
0.0061,
),
];
- // NVDataflow rejects backdated documents per retention policy, so stamp now.
+ // The target store may reject backdated documents per retention policy, so
+ // stamp now.
let base_ts = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|elapsed| elapsed.as_millis() as i64)
@@ -98,7 +107,7 @@ async fn posts_flat_documents_to_nvdataflow_sandbox() -> Result<()> {
samples.iter().enumerate()
{
let chat = sample_chat_payload(session, served, routed_to, *prompt, *completion, *cost);
- let doc = to_nvdataflow_document(&chat, Some(base_ts + index as i64));
+ let doc = to_flat_document(&chat, Some(base_ts + index as i64), None);
sink.enqueue(doc).await?;
}
sink.shutdown().await?;
diff --git a/crates/switchyard-components/tests/intake_http_sink.rs b/crates/switchyard-components/tests/intake_http_sink.rs
index ae218fec9..de08bb3f1 100644
--- a/crates/switchyard-components/tests/intake_http_sink.rs
+++ b/crates/switchyard-components/tests/intake_http_sink.rs
@@ -4,7 +4,9 @@
mod support;
use serde_json::json;
-use switchyard_components::{HttpIntakeSink, IntakeSink, IntakeSinkConfig};
+use switchyard_components::{
+ HttpIntakeSink, IntakeFormat, IntakeSink, IntakeSinkConfig, IntakeTarget,
+};
use switchyard_core::{Result, SwitchyardError};
use support::{OneShotServer, SequenceServer};
@@ -156,15 +158,18 @@ fn http_intake_sink_rejects_zero_queue_size() -> Result<()> {
Ok(())
}
-// NVDataflow mode posts the flat document to the project posting endpoint and
-// must not send the bearer token (NVDataflow posting is unauthenticated).
+// An unauthenticated flat-document target posts to its full URL and must not
+// send the bearer token even when an api_key is configured.
#[tokio::test]
-async fn http_intake_sink_posts_to_nvdataflow_project_path_without_auth() -> Result<()> {
+async fn http_intake_sink_posts_to_flat_target_url_without_auth() -> Result<()> {
let server = OneShotServer::json(201, json!({"status": "Created"}))?;
let sink = HttpIntakeSink::new(IntakeSinkConfig {
- intake_base_url: Some(server.base_url().to_string()),
- nvdataflow_project: Some("sandbox-switchyard".to_string()),
- // Set but must not be sent in NVDataflow mode.
+ target: Some(IntakeTarget {
+ url: format!("{}/lake/switchyard/posting", server.base_url()),
+ format: IntakeFormat::FlatDocument,
+ authenticated: false,
+ }),
+ // Set but must not be sent for an unauthenticated target.
api_key: Some("secret-token".to_string()),
..IntakeSinkConfig::default()
})?;
@@ -175,7 +180,7 @@ async fn http_intake_sink_posts_to_nvdataflow_project_path_without_auth() -> Res
let request = server.captured()?;
assert_eq!(request.method, "POST");
- assert_eq!(request.path, "/dataflow/sandbox-switchyard/posting");
+ assert_eq!(request.path, "/lake/switchyard/posting");
assert_eq!(request.header("authorization"), None);
assert_eq!(request.body["s_source"], "switchyard");
assert_eq!(request.body["l_switchyard_input_tokens"], 19);
diff --git a/crates/switchyard-components/tests/intake_payload.rs b/crates/switchyard-components/tests/intake_payload.rs
index d5835b688..460cd3a09 100644
--- a/crates/switchyard-components/tests/intake_payload.rs
+++ b/crates/switchyard-components/tests/intake_payload.rs
@@ -7,8 +7,9 @@ mod support;
use serde_json::{json, Value};
use switchyard_components::{
- IntakePayloadBuilder, IntakeRequestMetadata, IntakeRequestState, IntakeSinkConfig,
- RandomRoutingDecision, RandomRoutingTier, RequestMetadata, StatsRouteLabel,
+ IntakeFormat, IntakePayloadBuilder, IntakeRequestMetadata, IntakeRequestState,
+ IntakeSinkConfig, IntakeTarget, RandomRoutingDecision, RandomRoutingTier, RequestMetadata,
+ StatsRouteLabel, SubModelCall,
};
use switchyard_core::{ChatRequest, ChatRequestType, LlmTargetId, ModelId, ProxyContext, Result};
@@ -340,12 +341,16 @@ fn payload_usage_uses_null_model_when_served_model_is_missing() -> Result<()> {
Ok(())
}
-// NVDataflow mode flattens the record into top-level, type-prefixed fields.
+// A flat-document target flattens the record into top-level, type-prefixed fields.
#[test]
-fn payload_builds_flat_nvdataflow_document_when_project_set() -> Result<()> {
+fn payload_builds_flat_document_when_target_is_flat() -> Result<()> {
let builder = IntakePayloadBuilder::new(IntakeSinkConfig {
user_id: "0badf00d".to_string(),
- nvdataflow_project: Some("sandbox-switchyard".to_string()),
+ target: Some(IntakeTarget {
+ url: "https://example.invalid/posting".to_string(),
+ format: IntakeFormat::FlatDocument,
+ authenticated: false,
+ }),
..IntakeSinkConfig::default()
});
let request = openai_chat_request("openai/openai/gpt-5.2");
@@ -410,11 +415,90 @@ fn payload_builds_flat_nvdataflow_document_when_project_set() -> Result<()> {
Ok(())
}
+// Sub-model routing records reuse the turn's context but carry the router's own
+// model/usage/route, and each gets a distinct flat-document _id so the primary
+// record and every routing call coexist instead of overwriting one another.
+#[test]
+fn submodel_records_get_distinct_flat_ids_and_router_fields() -> Result<()> {
+ let builder = IntakePayloadBuilder::new(IntakeSinkConfig {
+ user_id: "0badf00d".to_string(),
+ target: Some(IntakeTarget {
+ url: "https://example.invalid/posting".to_string(),
+ format: IntakeFormat::FlatDocument,
+ authenticated: false,
+ }),
+ ..IntakeSinkConfig::default()
+ });
+ let request = openai_chat_request("openai/openai/gpt-5.2");
+ let mut ctx = ProxyContext::new();
+ record_backend_selection(&mut ctx, ModelId::from_static("openai/openai/gpt-5.2"));
+ ctx.insert(IntakeRequestState {
+ started_at_ms: 1_700_000_000_000,
+ inbound_format: ChatRequestType::OpenAiChat,
+ session_id: Some("sess-1".to_string()),
+ skip: false,
+ request_snapshot: Some(request.clone()),
+ });
+ let payload_ctx = switchyard_components::intake::IntakePayloadContext::from_proxy_context(
+ &ctx,
+ Some(1_700_000_001_840),
+ );
+
+ // Primary turn record and two routing calls share one session + timestamp.
+ let primary = builder.build(
+ &payload_ctx,
+ &request,
+ &completion_with_usage(
+ "chatcmpl-test",
+ "openai/openai/gpt-5.2",
+ "hello",
+ Some(json!({"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15})),
+ ),
+ false,
+ )?;
+ let call = SubModelCall {
+ model: "gcp/google/gemini-3.5-flash".to_string(),
+ prompt_tokens: 1200,
+ completion_tokens: 80,
+ cached_tokens: 0,
+ router_type: "deterministic".to_string(),
+ routed_to: "classifier".to_string(),
+ };
+ let first = builder.build_submodel_record(&payload_ctx, &call, 0)?;
+ let second = builder.build_submodel_record(&payload_ctx, &call, 1)?;
+
+ // Distinct _id per record so nothing overwrites another in the data lake.
+ assert_eq!(primary["_id"], "sess-1-1700000001840");
+ assert_eq!(first["_id"], "sess-1-1700000001840-classifier-0");
+ assert_eq!(second["_id"], "sess-1-1700000001840-classifier-1");
+
+ // The routing record carries the router's own model, usage, and route label.
+ assert_eq!(
+ first["s_switchyard_requested_model"],
+ "gcp/google/gemini-3.5-flash"
+ );
+ assert_eq!(
+ first["s_switchyard_served_model"],
+ "gcp/google/gemini-3.5-flash"
+ );
+ assert_eq!(first["s_switchyard_router_type"], "deterministic");
+ assert_eq!(first["s_switchyard_routed_to"], "classifier");
+ assert_eq!(first["l_switchyard_input_tokens"], 1200);
+ assert_eq!(first["l_switchyard_output_tokens"], 80);
+ // Start stamp is dropped for routing calls, so no turn-wide latency leaks in.
+ assert!(first.get("l_switchyard_latency_ms").is_none());
+ Ok(())
+}
+
// Default is metadata-only: no prompt/response text in the document, metrics kept.
#[test]
-fn nvdataflow_document_is_metadata_only_by_default() -> Result<()> {
+fn flat_document_is_metadata_only_by_default() -> Result<()> {
let builder = IntakePayloadBuilder::new(IntakeSinkConfig {
- nvdataflow_project: Some("sandbox-switchyard".to_string()),
+ target: Some(IntakeTarget {
+ url: "https://example.invalid/posting".to_string(),
+ format: IntakeFormat::FlatDocument,
+ authenticated: false,
+ }),
..IntakeSinkConfig::default()
});
let request = ChatRequest::openai_chat(json!({
diff --git a/crates/switchyard-components/tests/intake_response_processor.rs b/crates/switchyard-components/tests/intake_response_processor.rs
index 4f967e17d..2a00561f7 100644
--- a/crates/switchyard-components/tests/intake_response_processor.rs
+++ b/crates/switchyard-components/tests/intake_response_processor.rs
@@ -8,7 +8,9 @@ mod support;
use std::sync::Arc;
use serde_json::json;
-use switchyard_components::{IntakeRequestState, IntakeResponseProcessor, IntakeSinkConfig};
+use switchyard_components::{
+ IntakeRequestState, IntakeResponseProcessor, IntakeSinkConfig, SubModelCall, SubModelCalls,
+};
use switchyard_core::{
ChatRequestType, ChatResponse, ModelId, ProxyContext, Result, StreamEvent, SwitchyardError,
};
@@ -71,6 +73,85 @@ async fn response_processor_enqueues_non_streaming_payload() -> Result<()> {
Ok(())
}
+// Routing sub-model calls stashed on context each emit their own record next to
+// the primary one, all through the same sink.
+#[tokio::test]
+async fn response_processor_emits_a_record_per_submodel_call() -> Result<()> {
+ let sink = Arc::new(RecordingSink::default());
+ let processor = IntakeResponseProcessor::new(
+ IntakeSinkConfig {
+ workspace: Some("default".to_string()),
+ ..IntakeSinkConfig::default()
+ },
+ sink.clone(),
+ );
+ let mut ctx = opted_in_context();
+ ctx.insert(SubModelCalls(vec![
+ SubModelCall {
+ model: "router-clf".to_string(),
+ prompt_tokens: 100,
+ completion_tokens: 10,
+ cached_tokens: 0,
+ router_type: "deterministic".to_string(),
+ routed_to: "classifier".to_string(),
+ },
+ SubModelCall {
+ model: "planner".to_string(),
+ prompt_tokens: 50,
+ completion_tokens: 5,
+ cached_tokens: 0,
+ router_type: "plan_execute".to_string(),
+ routed_to: "planner".to_string(),
+ },
+ ]));
+
+ processor
+ .process(&mut ctx, completion("chatcmpl-test", "world"))
+ .await?;
+
+ // One primary record plus one per routing call.
+ let payloads = sink.payloads()?;
+ assert_eq!(payloads.len(), 3);
+ let routed: Vec<&str> = payloads
+ .iter()
+ .filter_map(|payload| payload["request"]["switchyard"]["routing"]["routed_to"].as_str())
+ .collect();
+ assert!(routed.contains(&"classifier"));
+ assert!(routed.contains(&"planner"));
+ Ok(())
+}
+
+// The opt-in gate covers sub-model records too: a skipped turn emits nothing,
+// even with routing calls stashed on context.
+#[tokio::test]
+async fn response_processor_skips_submodel_records_when_opted_out() -> Result<()> {
+ let sink = Arc::new(RecordingSink::default());
+ let processor = IntakeResponseProcessor::new(IntakeSinkConfig::default(), sink.clone());
+ let mut ctx = ProxyContext::new();
+ ctx.insert(IntakeRequestState {
+ started_at_ms: 1,
+ inbound_format: ChatRequestType::OpenAiChat,
+ session_id: None,
+ skip: true,
+ request_snapshot: None,
+ });
+ ctx.insert(SubModelCalls(vec![SubModelCall {
+ model: "router-clf".to_string(),
+ prompt_tokens: 100,
+ completion_tokens: 10,
+ cached_tokens: 0,
+ router_type: "deterministic".to_string(),
+ routed_to: "classifier".to_string(),
+ }]));
+
+ processor
+ .process(&mut ctx, completion("chatcmpl-test", "world"))
+ .await?;
+
+ assert!(sink.payloads()?.is_empty());
+ Ok(())
+}
+
// Intake failures must be fail-open so user responses are not replaced.
#[tokio::test]
async fn response_processor_fail_open_when_payload_build_or_sink_enqueue_fails() -> Result<()> {
diff --git a/crates/switchyard-py/src/component_bindings/config.rs b/crates/switchyard-py/src/component_bindings/config.rs
index ed328aae5..dd576aed4 100644
--- a/crates/switchyard-py/src/component_bindings/config.rs
+++ b/crates/switchyard-py/src/component_bindings/config.rs
@@ -9,7 +9,8 @@ use pyo3::prelude::*;
use pyo3::types::PyBool;
use serde::Serialize;
use switchyard_components::{
- IntakeQueueFullPolicy, IntakeSinkConfig, RandomRoutingProcessorConfig,
+ IntakeFormat, IntakeQueueFullPolicy, IntakeSinkConfig, IntakeTarget,
+ RandomRoutingProcessorConfig,
};
use switchyard_core::{BackendFormat, EndpointConfig, LlmTarget, LlmTargetId, ModelId};
@@ -530,7 +531,9 @@ impl PyIntakeSinkConfig {
workspace=None,
user_id=None,
api_key=None,
- nvdataflow_project=None,
+ target_url=None,
+ target_format=None,
+ target_authenticated=None,
max_queue_size=None,
request_timeout_s=None,
max_retries=None,
@@ -542,7 +545,9 @@ impl PyIntakeSinkConfig {
workspace: Option<String>,
user_id: Option<String>,
api_key: Option<String>,
- nvdataflow_project: Option<String>,
+ target_url: Option<String>,
+ target_format: Option<String>,
+ target_authenticated: Option<bool>,
max_queue_size: Option<usize>,
request_timeout_s: Option<f64>,
max_retries: Option<u32>,
@@ -562,8 +567,12 @@ impl PyIntakeSinkConfig {
if api_key.is_some() {
config.api_key = api_key;
}
- if nvdataflow_project.is_some() {
- config.nvdataflow_project = nvdataflow_project;
+ if let Some(url) = target_url {
+ config.target = Some(IntakeTarget {
+ url,
+ format: intake_format_from_str(target_format.as_deref())?,
+ authenticated: target_authenticated.unwrap_or(false),
+ });
}
if let Some(max_queue_size) = max_queue_size {
config.max_queue_size = max_queue_size;
@@ -597,8 +606,24 @@ impl PyIntakeSinkConfig {
}
#[getter]
- fn nvdataflow_project(&self) -> Option<String> {
- self.inner.nvdataflow_project.clone()
+ fn target_url(&self) -> Option<String> {
+ self.inner.target.as_ref().map(|target| target.url.clone())
+ }
+
+ #[getter]
+ fn target_format(&self) -> Option<String> {
+ self.inner
+ .target
+ .as_ref()
+ .map(|target| intake_format_name(target.format).to_string())
+ }
+
+ #[getter]
+ fn target_authenticated(&self) -> Option<bool> {
+ self.inner
+ .target
+ .as_ref()
+ .map(|target| target.authenticated)
}
#[getter]
@@ -637,7 +662,7 @@ impl PyIntakeSinkConfig {
fn __repr__(&self) -> String {
format!(
- "IntakeSinkConfig(intake_base_url={:?}, workspace={:?}, user_id={:?}, api_key={}, nvdataflow_project={:?}, max_queue_size={}, request_timeout_s={}, max_retries={}, on_queue_full='{}', capture_content={})",
+ "IntakeSinkConfig(intake_base_url={:?}, workspace={:?}, user_id={:?}, api_key={}, target={:?}, max_queue_size={}, request_timeout_s={}, max_retries={}, on_queue_full='{}', capture_content={})",
self.inner.intake_base_url,
self.inner.workspace,
self.inner.user_id,
@@ -646,7 +671,7 @@ impl PyIntakeSinkConfig {
} else {
"None"
},
- self.inner.nvdataflow_project,
+ self.inner.target,
self.inner.max_queue_size,
self.inner.request_timeout_s,
self.inner.max_retries,
@@ -769,6 +794,27 @@ fn intake_queue_policy_name(policy: IntakeQueueFullPolicy) -> &'static str {
}
}
+/// Parses the Python `target_format` kwarg into an `IntakeFormat`.
+///
+/// Defaults to the flat document shape, since an explicit target is a data-lake
+/// posting endpoint for every current caller.
+fn intake_format_from_str(value: Option<&str>) -> PyResult<IntakeFormat> {
+ match value.unwrap_or("flat_document") {
+ "flat_document" => Ok(IntakeFormat::FlatDocument),
+ "chat_completions" => Ok(IntakeFormat::ChatCompletions),
+ other => Err(PyValueError::new_err(format!(
+ "invalid target_format {other:?}; expected 'flat_document' or 'chat_completions'"
+ ))),
+ }
+}
+
+fn intake_format_name(format: IntakeFormat) -> &'static str {
+ match format {
+ IntakeFormat::FlatDocument => "flat_document",
+ IntakeFormat::ChatCompletions => "chat_completions",
+ }
+}
+
fn intake_queue_policy_variant_name(policy: IntakeQueueFullPolicy) -> &'static str {
match policy {
IntakeQueueFullPolicy::Drop => "DROP",
diff --git a/crates/switchyard-py/src/core_bindings/context.rs b/crates/switchyard-py/src/core_bindings/context.rs
index 5f2a97e03..e21271cef 100644
--- a/crates/switchyard-py/src/core_bindings/context.rs
+++ b/crates/switchyard-py/src/core_bindings/context.rs
@@ -12,7 +12,9 @@ use pyo3::exceptions::{PyKeyError, PyRuntimeError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::{PyBool, PyDict};
use std::time::Duration;
-use switchyard_components::{BackendSelection, BackendSelectionReason, StatsBackendLatency};
+use switchyard_components::{
+ BackendSelection, BackendSelectionReason, StatsBackendLatency, SubModelCall, SubModelCalls,
+};
use switchyard_core::{EvictedTargets, LlmTargetId, ModelId, ProxyContext, RequestId};
use crate::component_bindings::intake::request_metadata_from_mapping;
@@ -439,6 +441,67 @@ impl PyProxyContext {
self.metadata.clone_ref(py)
}
+ /// Records one routing-strategy sub-model call for intake capture.
+ ///
+ /// Routers that call a model to pick a route (the LLM classifier, the
+ /// stage-router tier classifier, the plan/execute planner) call this after a
+ /// successful routing call so the response-side intake processor can emit
+ /// each as its own anonymous record — model and token counts only, never the
+ /// messages. Appends to a Rust-typed slot the native intake sink reads, so
+ /// the routing record shares the primary record's transport and opt-in.
+ #[pyo3(signature = (model, prompt_tokens, completion_tokens, cached_tokens, router_type, routed_to))]
+ fn record_submodel_call(
+ &self,
+ model: String,
+ prompt_tokens: i64,
+ completion_tokens: i64,
+ cached_tokens: i64,
+ router_type: String,
+ routed_to: String,
+ ) -> PyResult<()> {
+ let call = SubModelCall {
+ model,
+ prompt_tokens,
+ completion_tokens,
+ cached_tokens,
+ router_type,
+ routed_to,
+ };
+ let mut guard = self.lock()?;
+ match guard.get_mut::<SubModelCalls>() {
+ Some(calls) => calls.0.push(call),
+ None => {
+ guard.insert(SubModelCalls(vec![call]));
+ }
+ }
+ Ok(())
+ }
+
+ /// Returns the routing sub-model calls recorded on this context.
+ ///
+ /// Each entry is a dict with `model`, `prompt_tokens`, `completion_tokens`,
+ /// `cached_tokens`, `router_type`, and `routed_to`; the list is empty when no
+ /// routing call was recorded. Lets Python observe what
+ /// [`record_submodel_call`] stashed for the intake sink.
+ #[getter]
+ fn submodel_calls(&self, py: Python<'_>) -> PyResult<Vec<Py<PyDict>>> {
+ let calls = self.get_cloned::<SubModelCalls>()?.unwrap_or_default();
+ calls
+ .0
+ .into_iter()
+ .map(|call| {
+ let dict = PyDict::new(py);
+ dict.set_item("model", call.model)?;
+ dict.set_item("prompt_tokens", call.prompt_tokens)?;
+ dict.set_item("completion_tokens", call.completion_tokens)?;
+ dict.set_item("cached_tokens", call.cached_tokens)?;
+ dict.set_item("router_type", call.router_type)?;
+ dict.set_item("routed_to", call.routed_to)?;
+ Ok(dict.unbind())
+ })
+ .collect()
+ }
+
/// Returns the optional request ID.
#[getter]
fn request_id(&self) -> PyResult<Option<String>> {
diff --git a/docs/cli_reference.md b/docs/cli_reference.md
index 840125623..c9b47cda1 100644
--- a/docs/cli_reference.md
+++ b/docs/cli_reference.md
@@ -99,7 +99,7 @@ Benchmark runs started through `benchmark/run-baseline.sh --routing-profiles` re
| `--intake-base-url URL` | Override intake base URL. |
| `--intake-workspace NAME` | Override workspace for Intake records. |
| `--intake-api-key VALUE` | Override bearer token. Disables the SDK's transparent refresh. |
-| `--intake-nvdataflow-project PROJECT` | Post flat per-request telemetry to this NVDataflow project instead of chat-completions ingest. Defaults to `$SWITCHYARD_NVDATAFLOW_PROJECT`. |
+| `--intake-target-url URL` | Post flat per-request telemetry to this full URL (a data-lake posting endpoint) instead of chat-completions ingest. Defaults to `$SWITCHYARD_INTAKE_TARGET_URL`. |
Launchers additionally accept context fields stamped into each ingested request:
@@ -121,6 +121,12 @@ Switchyard also maps the Intake app/task labels into top-level
`evaluation_context.dataset_*` and `evaluation_context.test_case_id` for span
queries while keeping the original labels under `request.switchyard`.
+Set `--intake-target-url` to a full posting URL to send captures to a different
+store instead. Switchyard then posts a flat, top-level, type-prefixed document
+(`s_` string, `l_` long, `f_` float, `b_` bool, `text_` debug — one indexable
+field per metric) to that URL, unauthenticated, rather than the nested
+nemo-platform payload. Unset, it uses the nemo-platform ingest above.
+
### RL trace logging
`--enable-rl-logging` attaches a response-side logger to the proxy chain of any
@@ -221,7 +227,7 @@ switchyard [--routing-profiles PATH] serve [--config PATH]
| `--host`, `--port`/`-p`, `--reload` | [Transport](#transport-server-verbs). |
| `--inbound FORMAT` | Valid only for legacy route-bundle serve (`--routing-profiles`); `serve --config` actively rejects it with an error. For legacy serve, the flag is a no-op — all request APIs are always registered regardless of the value (accepted for backwards compat only). |
| `--workers` / `-w` | uvicorn worker count. |
-| `--intake-enabled` / `--enable-intake`, `--intake-base-url`, `--intake-workspace`, `--intake-api-key`, `--intake-nvdataflow-project` | [Intake sink](#intake-sink-serve-and-launchers). |
+| `--intake-enabled` / `--enable-intake`, `--intake-base-url`, `--intake-workspace`, `--intake-api-key`, `--intake-target-url` | [Intake sink](#intake-sink-serve-and-launchers). |
**Notes**
@@ -566,7 +572,7 @@ switchyard verify --api-key "$OPENROUTER_API_KEY" --base-url https://openrouter.
| `OPENROUTER_BASE_URL`, `NVIDIA_BASE_URL`, `OPENAI_BASE_URL` | Backend base URL overrides paired with the selected provider credential. |
| `OPENAI_API_BASE` | Legacy alias for `OPENAI_BASE_URL`. Consulted as a last fallback when neither `--base-url` nor `OPENAI_BASE_URL` is set. Prefer `OPENAI_BASE_URL` for new configurations. |
| `SWITCHYARD_INTAKE_ENABLED` | Boolean equivalent of `--intake-enabled` / `--enable-intake`. Set to `1` or `true` to enable the intake sink without a CLI flag. Precedence: CLI flag first, then this env var. |
-| `SWITCHYARD_NVDATAFLOW_PROJECT` | NVDataflow project name for the alternate intake sink (paired with `--intake-nvdataflow-project`). Precedence: CLI flag first, then this env var. |
+| `SWITCHYARD_INTAKE_TARGET_URL` | Full posting URL for the alternate flat-document intake sink (paired with `--intake-target-url`). Precedence: CLI flag first, then this env var. |
| `SWITCHYARD_WORKERS` | Default uvicorn worker count for `serve`. |
| `SWITCHYARD_TELEMETRY_OPT_OUT` | Disable the `X-Switchyard-Version` telemetry header on outbound calls. `NEMO_SWITCHYARD_TELEMETRY_OPT_OUT` is honored for backwards compatibility. |
| `SWITCHYARD_INTAKE_BASE_URL`, `SWITCHYARD_INTAKE_WORKSPACE`, `SWITCHYARD_INTAKE_API_KEY`, `SWITCHYARD_INTAKE_APP`, `SWITCHYARD_INTAKE_TASK`, `SWITCHYARD_SESSION_ID`, `SWITCHYARD_USER_ID` | Intake-sink overrides for CI / headless runs. Precedence: the matching CLI flag (e.g. `--intake-user-id`) first, then the env var, then any persisted / SDK default (e.g. `~/.switchyard/user_id`). |
diff --git a/switchyard/cli/intake_cli_config.py b/switchyard/cli/intake_cli_config.py
index 149100b0a..d37d0b19a 100644
--- a/switchyard/cli/intake_cli_config.py
+++ b/switchyard/cli/intake_cli_config.py
@@ -3,8 +3,6 @@
"""Shared CLI/env resolution for intake options."""
-from __future__ import annotations
-
import argparse
import os
from collections.abc import Mapping
@@ -23,7 +21,7 @@ class IntakeCliConfig:
task: str | None = None
session_id: str | None = None
user_id: str | None = None
- nvdataflow_project: str | None = None
+ target_url: str | None = None
@classmethod
def from_server_args(
@@ -31,7 +29,7 @@ class IntakeCliConfig:
args: argparse.Namespace,
*,
env: Mapping[str, str] | None = None,
- ) -> IntakeCliConfig:
+ ) -> "IntakeCliConfig":
resolved_env = os.environ if env is None else env
base_url, workspace, api_key = _resolve_sink_connection(args, resolved_env)
return cls(
@@ -40,8 +38,8 @@ class IntakeCliConfig:
base_url=base_url,
workspace=workspace,
api_key=api_key,
- nvdataflow_project=_arg_or_env(
- args, "intake_nvdataflow_project", resolved_env, "SWITCHYARD_NVDATAFLOW_PROJECT",
+ target_url=_arg_or_env(
+ args, "intake_target_url", resolved_env, "SWITCHYARD_INTAKE_TARGET_URL",
),
)
@@ -51,7 +49,7 @@ class IntakeCliConfig:
args: argparse.Namespace,
*,
env: Mapping[str, str] | None = None,
- ) -> IntakeCliConfig:
+ ) -> "IntakeCliConfig":
resolved_env = os.environ if env is None else env
base_url, workspace, api_key = _resolve_sink_connection(args, resolved_env)
return cls(
@@ -68,8 +66,8 @@ class IntakeCliConfig:
user_id=_arg_or_env(
args, "intake_user_id", resolved_env, "SWITCHYARD_USER_ID",
),
- nvdataflow_project=_arg_or_env(
- args, "intake_nvdataflow_project", resolved_env, "SWITCHYARD_NVDATAFLOW_PROJECT",
+ target_url=_arg_or_env(
+ args, "intake_target_url", resolved_env, "SWITCHYARD_INTAKE_TARGET_URL",
),
)
diff --git a/switchyard/cli/launch_command.py b/switchyard/cli/launch_command.py
index c44e12fa3..5db565de5 100644
--- a/switchyard/cli/launch_command.py
+++ b/switchyard/cli/launch_command.py
@@ -440,7 +440,7 @@ def resolve_launch_intake_config(
task=intake.task or "developer-session",
session_id=intake.session_id,
user_id=intake.user_id,
- nvdataflow_project=intake.nvdataflow_project,
+ target_url=intake.target_url,
target=target,
)
diff --git a/switchyard/cli/launchers/launch_intake_config.py b/switchyard/cli/launchers/launch_intake_config.py
index dbefbff94..95f9cf54a 100644
--- a/switchyard/cli/launchers/launch_intake_config.py
+++ b/switchyard/cli/launchers/launch_intake_config.py
@@ -19,8 +19,6 @@ env vars through both seams. CLI parsing lives in
``switchyard.cli.launch_command``.
"""
-from __future__ import annotations
-
import re
import secrets
import sys
@@ -74,7 +72,7 @@ class LaunchIntakeConfig:
task: str
session_id: str
user_id: str
- nvdataflow_project: str | None = None
+ target_url: str | None = None
@classmethod
def from_resolved(
@@ -87,9 +85,9 @@ class LaunchIntakeConfig:
task: str,
session_id: str | None,
user_id: str | None = None,
- nvdataflow_project: str | None = None,
+ target_url: str | None = None,
target: str,
- ) -> LaunchIntakeConfig:
+ ) -> "LaunchIntakeConfig":
"""Resolve any blank fields and return the immutable config.
``session_id`` defaults to ``<target>-<unix-ms>-<uuid4-short>`` so each
@@ -108,7 +106,7 @@ class LaunchIntakeConfig:
task=task,
session_id=session_id or _default_session_id(target),
user_id=user_id or resolve_machine_user_id(),
- nvdataflow_project=nvdataflow_project,
+ target_url=target_url,
)
def opt_in_headers(self) -> dict[str, str]:
@@ -126,7 +124,7 @@ class LaunchIntakeConfig:
"proxy_x_session_id": self.session_id,
}
- def to_sink_config(self) -> IntakeSinkConfig:
+ def to_sink_config(self) -> "IntakeSinkConfig":
"""Build the server-side intake sink config for this launch."""
from switchyard.lib.config import IntakeSinkConfig
@@ -135,7 +133,7 @@ class LaunchIntakeConfig:
workspace=self.workspace,
api_key=self.api_key,
user_id=self.user_id,
- nvdataflow_project=self.nvdataflow_project,
+ target_url=self.target_url,
)
@@ -150,7 +148,10 @@ def build_intake_processors(
from switchyard.lib.processors.intake_response_processor import IntakeResponseProcessor
config = intake.to_sink_config()
- return [IntakeRequestProcessor()], [IntakeResponseProcessor(config)]
+ return (
+ [IntakeRequestProcessor()],
+ [IntakeResponseProcessor(config)],
+ )
def build_launch_capture_processors(
diff --git a/switchyard/cli/switchyard_cli.py b/switchyard/cli/switchyard_cli.py
index a2c41fd5d..cac0ff3d3 100644
--- a/switchyard/cli/switchyard_cli.py
+++ b/switchyard/cli/switchyard_cli.py
@@ -224,7 +224,7 @@ def _resolve_intake_config(args: argparse.Namespace) -> IntakeSinkConfig | None:
intake_base_url=intake.base_url,
workspace=intake.workspace,
api_key=intake.api_key,
- nvdataflow_project=intake.nvdataflow_project,
+ target_url=intake.target_url,
)
@@ -234,7 +234,10 @@ def _resolve_intake_processors(
intake = _resolve_intake_config(args)
if intake is None:
return [], []
- return [IntakeRequestProcessor()], [IntakeResponseProcessor(intake)]
+ return (
+ [IntakeRequestProcessor()],
+ [IntakeResponseProcessor(intake)],
+ )
def _add_intake_args(parser: argparse.ArgumentParser) -> None:
@@ -273,11 +276,11 @@ def _add_common_intake_args(parser: argparse.ArgumentParser) -> None:
),
)
parser.add_argument(
- "--intake-nvdataflow-project", type=str, default=None,
+ "--intake-target-url", type=str, default=None,
help=(
- "Post flat per-request telemetry to this NVDataflow project's "
- "posting endpoint instead of nemo-platform chat-completions "
- "ingest. Defaults to $SWITCHYARD_NVDATAFLOW_PROJECT."
+ "Post flat per-request telemetry to this full URL (a data-lake "
+ "posting endpoint) instead of nemo-platform chat-completions "
+ "ingest. Defaults to $SWITCHYARD_INTAKE_TARGET_URL."
),
)
@@ -485,7 +488,7 @@ def _cmd_serve_profile_config(args: argparse.Namespace) -> None:
args.intake_base_url,
args.intake_workspace,
args.intake_api_key,
- args.intake_nvdataflow_project,
+ args.intake_target_url,
))
if unsupported_intake:
raise SystemExit("serve --config does not support Intake options yet.")
diff --git a/switchyard/lib/processors/intake_payload_builder.py b/switchyard/lib/processors/intake_payload_builder.py
index cc9edc814..a279687d3 100644
--- a/switchyard/lib/processors/intake_payload_builder.py
+++ b/switchyard/lib/processors/intake_payload_builder.py
@@ -3,8 +3,6 @@
"""Build intake payloads from request/response primitives."""
-from __future__ import annotations
-
from datetime import UTC, datetime
from functools import cache
from importlib.metadata import PackageNotFoundError, version
diff --git a/switchyard/lib/processors/llm_classifier/request_processor.py b/switchyard/lib/processors/llm_classifier/request_processor.py
index 8c6f73d57..c233d5690 100644
--- a/switchyard/lib/processors/llm_classifier/request_processor.py
+++ b/switchyard/lib/processors/llm_classifier/request_processor.py
@@ -3,8 +3,6 @@
"""Request processor that asks an LLM to extract routing signals."""
-from __future__ import annotations
-
import json
import logging
import sys
@@ -384,9 +382,24 @@ class LLMClassifierRequestProcessor:
_fail_open_exc: Exception | None = exc
else:
_fail_open_exc = None
+ prompt_tokens, completion_tokens, cached_tokens = _classifier_token_counts(
+ completion.usage,
+ )
+ # Record usage so the intake sink emits the classifier sub-call as its
+ # own record; the routed-turn record never sees this call.
+ ctx.record_submodel_call(
+ model=self._config.model,
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ cached_tokens=cached_tokens,
+ router_type="deterministic",
+ routed_to="classifier",
+ )
if self._stats_accumulator is not None:
await self._record_classifier_call(
- usage=completion.usage,
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ cached_tokens=cached_tokens,
latency_ms=(time.perf_counter() - started_at) * 1000,
)
@@ -417,38 +430,40 @@ class LLMClassifierRequestProcessor:
async def _record_classifier_call(
self,
- *,
- usage: Any,
+ prompt_tokens: int,
+ completion_tokens: int,
+ cached_tokens: int,
latency_ms: float,
) -> None:
- """Extract token counts from the SDK ``usage`` object and record them.
+ """Record classifier token spend into the ``StatsAccumulator``.
- Records into the classifier bucket on :class:`StatsAccumulator`
- so the classifier model's token spend doesn't merge with the
- same-named backend tier (default TB-lite config has classifier
- and weak both pointing at Nemotron-3-Super-v3). Latency is
- recorded too so the per-request classifier-tax line shows up
+ Records into the classifier bucket so the classifier model's token
+ spend doesn't merge with the same-named backend tier (default TB-lite
+ config has classifier and weak both pointing at Nemotron-3-Super-v3).
+ Latency is recorded too so the per-request classifier-tax line shows up
on the snapshot's classifier-models block.
"""
assert self._stats_accumulator is not None
- prompt = 0
- completion = 0
- cached = 0
- if usage is not None:
- prompt = getattr(usage, "prompt_tokens", 0) or 0
- completion = getattr(usage, "completion_tokens", 0) or 0
- ptd = getattr(usage, "prompt_tokens_details", None)
- if ptd is not None:
- cached = getattr(ptd, "cached_tokens", 0) or 0
await self._stats_accumulator.record_classifier_usage(
model=self._config.model,
- prompt_tokens=prompt,
- completion_tokens=completion,
- cached_tokens=cached,
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ cached_tokens=cached_tokens,
latency_ms=latency_ms,
)
+def _classifier_token_counts(usage: Any) -> tuple[int, int, int]:
+ """Return ``(prompt, completion, cached)`` tokens from an SDK usage object."""
+ if usage is None:
+ return 0, 0, 0
+ prompt = getattr(usage, "prompt_tokens", 0) or 0
+ completion = getattr(usage, "completion_tokens", 0) or 0
+ ptd = getattr(usage, "prompt_tokens_details", None)
+ cached = (getattr(ptd, "cached_tokens", 0) or 0) if ptd is not None else 0
+ return prompt, completion, cached
+
+
def parse_route_decision(
raw: str,
schema: type[RouteDecision] = RouteSignals,
diff --git a/switchyard/lib/processors/plan_execute/request_processor.py b/switchyard/lib/processors/plan_execute/request_processor.py
index 25e63c232..2122ed804 100644
--- a/switchyard/lib/processors/plan_execute/request_processor.py
+++ b/switchyard/lib/processors/plan_execute/request_processor.py
@@ -3,8 +3,6 @@
"""LLM-backed planning request processor."""
-from __future__ import annotations
-
import json
import logging
import sys
@@ -619,6 +617,20 @@ class PlanningRequestProcessor:
# the planner declined to plan when ``plan_needed=False``.
ctx.metadata[CTX_PLANNER_DECISION] = decision
+ prompt_tokens, completion_tokens, cached_tokens = _planner_token_counts(
+ completion.usage,
+ )
+ # Record usage so the intake sink emits the planner call as its own
+ # intake record; the routed-turn record never sees it.
+ ctx.record_submodel_call(
+ model=self._config.model,
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ cached_tokens=cached_tokens,
+ router_type="plan_execute",
+ routed_to="planner",
+ )
+
# Record planner-side spend into the stats accumulator's
# ``planner`` bucket (if wired). Mirrors the classifier's
# ``record_classifier_usage`` call. Latency captures the full
@@ -628,7 +640,9 @@ class PlanningRequestProcessor:
# audit line below remains the only observability path.
if self._stats_accumulator is not None:
await self._record_planner_call(
- usage=completion.usage,
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ cached_tokens=cached_tokens,
latency_ms=(time.perf_counter() - started_at) * 1000,
)
@@ -673,39 +687,41 @@ class PlanningRequestProcessor:
async def _record_planner_call(
self,
- *,
- usage: Any,
+ prompt_tokens: int,
+ completion_tokens: int,
+ cached_tokens: int,
latency_ms: float,
) -> None:
- """Extract token counts from the SDK ``usage`` object and record them.
-
- Records into the planner bucket on :class:`StatsAccumulator` so
- the planner model's token spend doesn't merge with the
- same-named executor backend (e.g. when running V4-Pro-as-planner
- + V4-Pro-as-executor for self-planning sweeps). Latency is
- recorded too so the planner-side latency histogram shows up on
- the snapshot's ``planner.models`` block. Mirrors
+ """Record planner token spend into the ``StatsAccumulator``.
+
+ Records into the planner bucket so the planner model's token spend
+ doesn't merge with the same-named executor backend (e.g. when running
+ V4-Pro-as-planner + V4-Pro-as-executor for self-planning sweeps).
+ Latency is recorded too so the planner-side latency histogram shows up
+ on the snapshot's ``planner.models`` block. Mirrors
:meth:`switchyard.lib.processors.llm_classifier.LLMClassifierRequestProcessor._record_classifier_call`.
"""
assert self._stats_accumulator is not None
- prompt = 0
- completion_tokens = 0
- cached = 0
- if usage is not None:
- prompt = getattr(usage, "prompt_tokens", 0) or 0
- completion_tokens = getattr(usage, "completion_tokens", 0) or 0
- ptd = getattr(usage, "prompt_tokens_details", None)
- if ptd is not None:
- cached = getattr(ptd, "cached_tokens", 0) or 0
await self._stats_accumulator.record_planner_usage(
model=self._config.model,
- prompt_tokens=prompt,
+ prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
- cached_tokens=cached,
+ cached_tokens=cached_tokens,
latency_ms=latency_ms,
)
+def _planner_token_counts(usage: Any | None) -> tuple[int, int, int]:
+ """Return ``(prompt, completion, cached)`` tokens from an SDK usage object."""
+ if usage is None:
+ return 0, 0, 0
+ prompt = getattr(usage, "prompt_tokens", 0) or 0
+ completion = getattr(usage, "completion_tokens", 0) or 0
+ ptd = getattr(usage, "prompt_tokens_details", None)
+ cached = (getattr(ptd, "cached_tokens", 0) or 0) if ptd is not None else 0
+ return prompt, completion, cached
+
+
def _build_audit_payload(
decision: PlannerDecision,
usage: Any | None,
diff --git a/switchyard/lib/processors/stage_router/classifier.py b/switchyard/lib/processors/stage_router/classifier.py
index 0e4fc0951..aec9e2569 100644
--- a/switchyard/lib/processors/stage_router/classifier.py
+++ b/switchyard/lib/processors/stage_router/classifier.py
@@ -3,8 +3,6 @@
"""LLM tier classifier — invoked on low-confidence turns, fails open to ``None``."""
-from __future__ import annotations
-
import json
import logging
import time
@@ -44,7 +42,7 @@ _SYSTEM_PROMPT: str = _load_system_prompt()
def _summarise(
- signal: ToolResultSignal,
+ signal: "ToolResultSignal",
*,
recent_messages: list[Any] | None = None,
) -> str:
@@ -158,7 +156,7 @@ class TierClassifier:
recent_turn_window: int = 3,
disable_reasoning: bool = True,
client: _LLMClient | None = None,
- stats_accumulator: StatsAccumulator | None = None,
+ stats_accumulator: "StatsAccumulator | None" = None,
) -> None:
if recent_turn_window < 0:
raise ValueError(f"recent_turn_window must be >= 0, got {recent_turn_window}")
@@ -183,11 +181,11 @@ class TierClassifier:
))
self._client = client
- def attach_stats_accumulator(self, stats_accumulator: StatsAccumulator) -> None:
+ def attach_stats_accumulator(self, stats_accumulator: "StatsAccumulator") -> None:
"""Attach the serving-level accumulator used for classifier overhead."""
self._stats = stats_accumulator
- async def classify(self, ctx: ProxyContext, signal: ToolResultSignal) -> str | None:
+ async def classify(self, ctx: "ProxyContext", signal: "ToolResultSignal") -> str | None:
"""Return ``"capable"``, ``"efficient"``, or ``None`` (fall-open)."""
recent_messages: list[Any] = []
if self._recent_turn_window > 0:
@@ -230,15 +228,27 @@ class TierClassifier:
pass
return None
latency_ms = (time.perf_counter() - started_at) * 1000.0
+ prompt_tokens, completion_tokens, cached_tokens = _tier_token_counts(
+ getattr(response, "usage", None),
+ )
+ # Record usage so the intake sink emits the tier-classifier call as its
+ # own intake record; the routed-turn record never sees it.
+ if hasattr(ctx, "record_submodel_call"):
+ ctx.record_submodel_call(
+ model=self._model,
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ cached_tokens=cached_tokens,
+ router_type="stage_router",
+ routed_to="tier_classifier",
+ )
if self._stats is not None:
try:
- usage = getattr(response, "usage", None)
- details = getattr(usage, "prompt_tokens_details", None)
await self._stats.record_classifier_usage(
self._model,
- prompt_tokens=getattr(usage, "prompt_tokens", 0) or 0,
- completion_tokens=getattr(usage, "completion_tokens", 0) or 0,
- cached_tokens=getattr(details, "cached_tokens", None) or 0,
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ cached_tokens=cached_tokens,
latency_ms=latency_ms,
)
except Exception:
@@ -246,4 +256,15 @@ class TierClassifier:
return _parse_tier(response)
+def _tier_token_counts(usage: Any) -> tuple[int, int, int]:
+ """Return ``(prompt, completion, cached)`` tokens from an SDK usage object."""
+ if usage is None:
+ return 0, 0, 0
+ prompt = getattr(usage, "prompt_tokens", 0) or 0
+ completion = getattr(usage, "completion_tokens", 0) or 0
+ details = getattr(usage, "prompt_tokens_details", None)
+ cached = (getattr(details, "cached_tokens", 0) or 0) if details is not None else 0
+ return prompt, completion, cached
+
+
__all__ = ["CAPABLE_TIER", "TierClassifier", "EFFICIENT_TIER"]
diff --git a/switchyard_rust/components.pyi b/switchyard_rust/components.pyi
index 330f85160..01621be5c 100644
--- a/switchyard_rust/components.pyi
+++ b/switchyard_rust/components.pyi
@@ -97,7 +97,9 @@ class IntakeSinkConfig:
workspace: str | None
user_id: str
api_key: str | None
- nvdataflow_project: str | None
+ target_url: str | None
+ target_format: str | None
+ target_authenticated: bool | None
max_queue_size: int
request_timeout_s: float
max_retries: int
@@ -110,7 +112,9 @@ class IntakeSinkConfig:
workspace: str | None = None,
user_id: str | None = None,
api_key: str | None = None,
- nvdataflow_project: str | None = None,
+ target_url: str | None = None,
+ target_format: str | None = None,
+ target_authenticated: bool | None = None,
max_queue_size: int | None = None,
request_timeout_s: float | None = None,
max_retries: int | None = None,
diff --git a/switchyard_rust/core.py b/switchyard_rust/core.py
index 678ba8fd5..3633cfc2c 100644
--- a/switchyard_rust/core.py
+++ b/switchyard_rust/core.py
@@ -130,6 +130,17 @@ class _ProxyContext(Protocol):
selected_target: str | None
backend_call_latency_ms: float | None
evicted_targets: list[str] | None
+ submodel_calls: list[dict[str, Any]]
+
+ def record_submodel_call(
+ self,
+ model: str,
+ prompt_tokens: int,
+ completion_tokens: int,
+ cached_tokens: int,
+ router_type: str,
+ routed_to: str,
+ ) -> None: ...
class _LLMBackend(Protocol):
@@ -551,6 +562,7 @@ if TYPE_CHECKING:
selected_target: str | None
backend_call_latency_ms: float | None
evicted_targets: list[str] | None
+ submodel_calls: list[dict[str, Any]]
def __init__(
self,
@@ -558,6 +570,16 @@ if TYPE_CHECKING:
request_id: str | None = None,
) -> None: ...
+ def record_submodel_call(
+ self,
+ model: str,
+ prompt_tokens: int,
+ completion_tokens: int,
+ cached_tokens: int,
+ router_type: str,
+ routed_to: str,
+ ) -> None: ...
+
class LLMBackend:
"""Static view of the Rust-owned LLMBackend role class."""
diff --git a/tests/test_cli_reference_docs.py b/tests/test_cli_reference_docs.py
index 5f6e39753..46eebdeb7 100644
--- a/tests/test_cli_reference_docs.py
+++ b/tests/test_cli_reference_docs.py
@@ -131,7 +131,7 @@ def test_cli_reference_documents_shared_intake_flags_once() -> None:
assert doc.count("`--intake-base-url URL`") == 1
assert doc.count("`--intake-workspace NAME`") == 1
assert doc.count("`--intake-api-key VALUE`") == 1
- assert doc.count("`--intake-nvdataflow-project PROJECT`") == 1
+ assert doc.count("`--intake-target-url URL`") == 1
assert "`--intake-app NAME`" in doc
diff --git a/tests/test_intake_cli_flags.py b/tests/test_intake_cli_flags.py
index 9520cab29..58e656cc2 100644
--- a/tests/test_intake_cli_flags.py
+++ b/tests/test_intake_cli_flags.py
@@ -68,7 +68,7 @@ COMMON_INTAKE_ARGS = [
"--intake-base-url", "https://nmp.example",
"--intake-workspace", "workspace-a",
"--intake-api-key", "ci-token",
- "--intake-nvdataflow-project", "project-a",
+ "--intake-target-url", "https://data-lake.example/posting",
]
@@ -76,7 +76,7 @@ def _assert_common_intake_args(args: argparse.Namespace) -> None:
assert args.intake_base_url == "https://nmp.example"
assert args.intake_workspace == "workspace-a"
assert args.intake_api_key == "ci-token"
- assert args.intake_nvdataflow_project == "project-a"
+ assert args.intake_target_url == "https://data-lake.example/posting"
def test_serve_accepts_common_intake_args() -> None:
diff --git a/tests/test_intake_nvdataflow.py b/tests/test_intake_nvdataflow.py
deleted file mode 100644
index 18960d4c2..000000000
--- a/tests/test_intake_nvdataflow.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for NVDataflow project CLI/env wiring into the intake sink config."""
-
-from __future__ import annotations
-
-import argparse
-
-from switchyard.cli.intake_cli_config import IntakeCliConfig
-from switchyard.cli.launchers.launch_intake_config import LaunchIntakeConfig
-
-
-def test_launch_args_resolve_project_from_flag() -> None:
- args = argparse.Namespace(intake_enabled=True, intake_nvdataflow_project="ipp-nova-tokenomics")
- resolved = IntakeCliConfig.from_launch_args(args, env={})
- assert resolved.nvdataflow_project == "ipp-nova-tokenomics"
-
-
-def test_launch_args_resolve_project_from_env() -> None:
- args = argparse.Namespace(intake_enabled=True, intake_nvdataflow_project=None)
- resolved = IntakeCliConfig.from_launch_args(
- args, env={"SWITCHYARD_NVDATAFLOW_PROJECT": "from-env"}
- )
- assert resolved.nvdataflow_project == "from-env"
-
-
-def test_flag_wins_over_env() -> None:
- args = argparse.Namespace(intake_enabled=True, intake_nvdataflow_project="from-flag")
- resolved = IntakeCliConfig.from_launch_args(
- args, env={"SWITCHYARD_NVDATAFLOW_PROJECT": "from-env"}
- )
- assert resolved.nvdataflow_project == "from-flag"
-
-
-def test_server_args_resolve_project() -> None:
- args = argparse.Namespace(intake_enabled=True, intake_nvdataflow_project="ipp-nova-tokenomics")
- resolved = IntakeCliConfig.from_server_args(args, env={})
- assert resolved.nvdataflow_project == "ipp-nova-tokenomics"
-
-
-def test_project_absent_defaults_to_none() -> None:
- args = argparse.Namespace(intake_enabled=True, intake_nvdataflow_project=None)
- resolved = IntakeCliConfig.from_launch_args(args, env={})
- assert resolved.nvdataflow_project is None
-
-
-def test_to_sink_config_passes_project_through_binding() -> None:
- config = LaunchIntakeConfig(
- base_url=None,
- workspace=None,
- api_key=None,
- app="claude-code",
- task="developer-session",
- session_id="sess-1",
- user_id="0badf00d",
- nvdataflow_project="ipp-nova-tokenomics",
- )
- assert config.to_sink_config().nvdataflow_project == "ipp-nova-tokenomics"
-
-
-def test_launch_env_var_enables_intake() -> None:
- # SWITCHYARD_INTAKE_ENABLED turns intake on for launch (parity with serve).
- args = argparse.Namespace(intake_enabled=False, intake_nvdataflow_project=None)
- resolved = IntakeCliConfig.from_launch_args(args, env={"SWITCHYARD_INTAKE_ENABLED": "1"})
- assert resolved.enabled is True
-
-
-def test_launch_disabled_without_flag_or_env() -> None:
- args = argparse.Namespace(intake_enabled=False, intake_nvdataflow_project=None)
- resolved = IntakeCliConfig.from_launch_args(args, env={})
- assert resolved.enabled is False
-
-
-def test_server_env_var_enables_intake() -> None:
- args = argparse.Namespace(intake_enabled=False, intake_nvdataflow_project=None)
- resolved = IntakeCliConfig.from_server_args(args, env={"SWITCHYARD_INTAKE_ENABLED": "1"})
- assert resolved.enabled is True
diff --git a/tests/test_intake_sink.py b/tests/test_intake_sink.py
index 1070fc9ff..df6e9d8a2 100644
--- a/tests/test_intake_sink.py
+++ b/tests/test_intake_sink.py
@@ -619,6 +619,55 @@ class TestIntakeRequestProcessor:
assert "task" not in payload["request"]["switchyard"]
assert payload["session_id"] == "session-123"
+ async def test_store_true_posts_routed_turn_and_submodel_records_to_flat_target(self):
+ """`store: true` alone (no header) opts in, so the routed turn and every
+ recorded routing call post to the flat-document target as distinct
+ documents. This is the opt-in the response-side Python processor could
+ never see."""
+ request_processor = IntakeRequestProcessor()
+ request = ChatRequest.openai_chat({
+ "model": "gpt-4o",
+ "messages": [{"role": "user", "content": "hi"}],
+ "store": True,
+ })
+ ctx = ProxyContext()
+ attach_request_metadata(
+ ctx,
+ RequestMetadata.from_headers({"proxy_x_session_id": "sess-xyz"}),
+ )
+ # A router records its own model call, exactly as the LLM classifier does.
+ ctx.record_submodel_call(
+ model="router-clf",
+ prompt_tokens=120,
+ completion_tokens=8,
+ cached_tokens=0,
+ router_type="deterministic",
+ routed_to="classifier",
+ )
+
+ with _IntakeHttpStub() as server:
+ response_processor = IntakeResponseProcessor(
+ IntakeSinkConfig(
+ target_url=f"{server.base_url}/lake/switchyard/posting",
+ max_retries=0,
+ )
+ )
+ await request_processor.process(ctx, request)
+ await response_processor.process(
+ ctx, ChatResponse.openai_completion(_completion())
+ )
+ await response_processor.shutdown()
+
+ # One routed-turn document plus one per routing call, all flat documents.
+ posted = server.requests
+ assert len(posted) == 2
+ for entry in posted:
+ assert entry["path"] == "/lake/switchyard/posting"
+ # Distinct _id so the routing record never overwrites the routed turn.
+ assert len({entry["body"]["_id"] for entry in posted}) == 2
+ routed = {entry["body"].get("s_switchyard_routed_to") for entry in posted}
+ assert "classifier" in routed
+
async def test_header_opt_out_overrides_store_true_in_rust_context(self):
request_processor = IntakeRequestProcessor()
request = ChatRequest.openai_chat({
diff --git a/tests/test_intake_target.py b/tests/test_intake_target.py
new file mode 100644
index 000000000..6b47f309b
--- /dev/null
+++ b/tests/test_intake_target.py
@@ -0,0 +1,84 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Tests for intake target-URL CLI/env wiring into the intake sink config."""
+
+import argparse
+
+from switchyard.cli.intake_cli_config import IntakeCliConfig
+from switchyard.cli.launchers.launch_intake_config import LaunchIntakeConfig
+
+_URL = "https://data-lake.example/lake/example-project/posting"
+
+
+def test_launch_args_resolve_target_from_flag() -> None:
+ args = argparse.Namespace(intake_enabled=True, intake_target_url=_URL)
+ resolved = IntakeCliConfig.from_launch_args(args, env={})
+ assert resolved.target_url == _URL
+
+
+def test_launch_args_resolve_target_from_env() -> None:
+ args = argparse.Namespace(intake_enabled=True, intake_target_url=None)
+ resolved = IntakeCliConfig.from_launch_args(
+ args, env={"SWITCHYARD_INTAKE_TARGET_URL": "https://from-env.example/posting"}
+ )
+ assert resolved.target_url == "https://from-env.example/posting"
+
+
+def test_flag_wins_over_env() -> None:
+ args = argparse.Namespace(
+ intake_enabled=True, intake_target_url="https://from-flag.example/posting"
+ )
+ resolved = IntakeCliConfig.from_launch_args(
+ args, env={"SWITCHYARD_INTAKE_TARGET_URL": "https://from-env.example/posting"}
+ )
+ assert resolved.target_url == "https://from-flag.example/posting"
+
+
+def test_server_args_resolve_target() -> None:
+ args = argparse.Namespace(intake_enabled=True, intake_target_url=_URL)
+ resolved = IntakeCliConfig.from_server_args(args, env={})
+ assert resolved.target_url == _URL
+
+
+def test_target_absent_defaults_to_none() -> None:
+ args = argparse.Namespace(intake_enabled=True, intake_target_url=None)
+ resolved = IntakeCliConfig.from_launch_args(args, env={})
+ assert resolved.target_url is None
+
+
+def test_to_sink_config_passes_target_through_binding() -> None:
+ config = LaunchIntakeConfig(
+ base_url=None,
+ workspace=None,
+ api_key=None,
+ app="claude-code",
+ task="developer-session",
+ session_id="sess-1",
+ user_id="0badf00d",
+ target_url=_URL,
+ )
+ sink = config.to_sink_config()
+ assert sink.target_url == _URL
+ # A target URL alone defaults to the flat, unauthenticated data-lake shape.
+ assert sink.target_format == "flat_document"
+ assert sink.target_authenticated is False
+
+
+def test_launch_env_var_enables_intake() -> None:
+ # SWITCHYARD_INTAKE_ENABLED turns intake on for launch (parity with serve).
+ args = argparse.Namespace(intake_enabled=False, intake_target_url=None)
+ resolved = IntakeCliConfig.from_launch_args(args, env={"SWITCHYARD_INTAKE_ENABLED": "1"})
+ assert resolved.enabled is True
+
+
+def test_launch_disabled_without_flag_or_env() -> None:
+ args = argparse.Namespace(intake_enabled=False, intake_target_url=None)
+ resolved = IntakeCliConfig.from_launch_args(args, env={})
+ assert resolved.enabled is False
+
+
+def test_server_env_var_enables_intake() -> None:
+ args = argparse.Namespace(intake_enabled=False, intake_target_url=None)
+ resolved = IntakeCliConfig.from_server_args(args, env={"SWITCHYARD_INTAKE_ENABLED": "1"})
+ assert resolved.enabled is True
diff --git a/tests/test_llm_classifier_request_processor.py b/tests/test_llm_classifier_request_processor.py
index 57e82bb88..5337c56d6 100644
--- a/tests/test_llm_classifier_request_processor.py
+++ b/tests/test_llm_classifier_request_processor.py
@@ -3,8 +3,6 @@
"""Tests for the experimental LLM classifier request processor."""
-from __future__ import annotations
-
import json
from typing import Any, cast
@@ -125,6 +123,46 @@ async def test_request_processor_stamps_classifier_signals() -> None:
assert "debug this traceback" in fake.calls[0]["request_summary"]
+async def test_request_processor_stashes_submodel_call_on_success() -> None:
+ """Success records the classifier's usage for the intake sink to emit."""
+ fake = _FakeClassifierClient(
+ _signals_json(),
+ usage=_FakeUsage(prompt_tokens=420, completion_tokens=80, cached_tokens=10),
+ )
+ processor = LLMClassifierRequestProcessor(
+ LLMClassifierConfig(model="router-classifier"),
+ client=fake,
+ )
+ ctx = ProxyContext()
+
+ await processor.process(ctx, _request())
+
+ assert ctx.submodel_calls == [
+ {
+ "model": "router-classifier",
+ "prompt_tokens": 420,
+ "completion_tokens": 80,
+ "cached_tokens": 10,
+ "router_type": "deterministic",
+ "routed_to": "classifier",
+ }
+ ]
+
+
+async def test_request_processor_stashes_no_submodel_call_on_fail_open() -> None:
+ """Fail-open (classifier errored) has no usage, so nothing is recorded."""
+ fake = _FakeClassifierClient(RuntimeError("upstream down"))
+ processor = LLMClassifierRequestProcessor(
+ LLMClassifierConfig(model="router-classifier", fail_open=True),
+ client=fake,
+ )
+ ctx = ProxyContext()
+
+ await processor.process(ctx, _request())
+
+ assert ctx.submodel_calls == []
+
+
async def test_classifier_skips_llm_call_when_session_pinned() -> None:
"""Once a conversation is pinned, the classifier runs once per task, not per turn."""
fake = _FakeClassifierClient(_signals_json())
diff --git a/tests/test_planning_request_processor.py b/tests/test_planning_request_processor.py
index ebaea6935..856796960 100644
--- a/tests/test_planning_request_processor.py
+++ b/tests/test_planning_request_processor.py
@@ -3,8 +3,6 @@
"""Tests for the planning request processor (slim two-field schema)."""
-from __future__ import annotations
-
import json
from types import SimpleNamespace
from typing import Any, cast
@@ -381,3 +379,50 @@ async def test_planning_processor_prefills_responses_input_string() -> None:
assert isinstance(input_value, list)
assert input_value[0] == {"role": "user", "content": "refactor auth middleware"}
assert input_value[-1] == {"role": "assistant", "content": _WRAPPED_PLAN}
+
+
+class _UsagePlannerClient:
+ """Planner double that returns a token-usage block, so the producer has
+ counts to record for the intake sink."""
+
+ def __init__(self, content: str, usage: object) -> None:
+ self.content = content
+ self.usage = usage
+
+ async def plan(
+ self,
+ model: str,
+ system_prompt: str,
+ request_summary: str,
+ ) -> PlannerCompletion:
+ return PlannerCompletion(content=self.content, usage=self.usage)
+
+
+async def test_process_stashes_planner_submodel_call_for_intake() -> None:
+ """A successful planner call is recorded for the intake sink to emit."""
+ fake = _UsagePlannerClient(
+ _decision_with_plan_json(),
+ SimpleNamespace(
+ prompt_tokens=512,
+ completion_tokens=64,
+ prompt_tokens_details=SimpleNamespace(cached_tokens=8),
+ ),
+ )
+ processor = PlanningRequestProcessor(
+ PlanningConfig(model="planner-model", cadence_n=1),
+ client=fake,
+ )
+ ctx = ProxyContext()
+
+ await processor.process(ctx, _openai_request_first_turn())
+
+ assert ctx.submodel_calls == [
+ {
+ "model": "planner-model",
+ "prompt_tokens": 512,
+ "completion_tokens": 64,
+ "cached_tokens": 8,
+ "router_type": "plan_execute",
+ "routed_to": "planner",
+ }
+ ]
diff --git a/tests/test_rl_logging.py b/tests/test_rl_logging.py
index c85350d74..bb93ecdc6 100644
--- a/tests/test_rl_logging.py
+++ b/tests/test_rl_logging.py
@@ -342,7 +342,7 @@ def test_serve_attaches_rl_logging_processors(monkeypatch, tmp_path: Path) -> No
config=None, routing_profiles="profiles.yaml",
enable_rl_logging=True, rl_log_dir=str(tmp_path),
intake_enabled=False, intake_base_url=None, intake_workspace=None,
- intake_api_key=None, intake_nvdataflow_project=None,
+ intake_api_key=None, intake_target_url=None,
)
cli._cmd_serve(args)
@@ -358,7 +358,7 @@ def test_serve_config_rejects_rl_logging() -> None:
config="profiles.yaml", routing_profiles=None, inbound=None,
reload=False, workers=1,
intake_enabled=False, intake_base_url=None, intake_workspace=None,
- intake_api_key=None, intake_nvdataflow_project=None,
+ intake_api_key=None, intake_target_url=None,
enable_rl_logging=True, rl_log_dir="./rl_data",
)
with pytest.raises(SystemExit, match="enable-rl-logging"):
diff --git a/tests/test_serve_profile_config.py b/tests/test_serve_profile_config.py
index ba5d5a997..29809fe34 100644
--- a/tests/test_serve_profile_config.py
+++ b/tests/test_serve_profile_config.py
@@ -314,7 +314,7 @@ def _serve_args(path: Path) -> argparse.Namespace:
intake_base_url=None,
intake_workspace=None,
intake_api_key=None,
- intake_nvdataflow_project=None,
+ intake_target_url=None,
host="127.0.0.1",
port=4000,
)
diff --git a/tests/test_stage_router_classifier.py b/tests/test_stage_router_classifier.py
index 92518f8cf..4670f1d1a 100644
--- a/tests/test_stage_router_classifier.py
+++ b/tests/test_stage_router_classifier.py
@@ -8,14 +8,11 @@ coverage matters more than success — any real-world flakiness must fall open
to ``None`` so the picker keeps moving.
"""
-from __future__ import annotations
-
import json
from dataclasses import dataclass
+from types import SimpleNamespace
from typing import Any
-import pytest
-
from switchyard.lib.processors.stage_router.classifier import (
CAPABLE_TIER,
EFFICIENT_TIER,
@@ -65,7 +62,6 @@ async def _build_signal() -> Any:
return ctx, signal
async def test_returns_capable_on_valid_capable_response():
ctx, signal = await _build_signal()
classifier = TierClassifier(
@@ -75,7 +71,6 @@ async def test_returns_capable_on_valid_capable_response():
assert await classifier.classify(ctx, signal) == CAPABLE_TIER
async def test_returns_efficient_on_valid_efficient_response():
ctx, signal = await _build_signal()
classifier = TierClassifier(
@@ -85,7 +80,6 @@ async def test_returns_efficient_on_valid_efficient_response():
assert await classifier.classify(ctx, signal) == EFFICIENT_TIER
async def test_falls_open_on_malformed_json():
ctx, signal = await _build_signal()
classifier = TierClassifier(
@@ -95,7 +89,6 @@ async def test_falls_open_on_malformed_json():
assert await classifier.classify(ctx, signal) is None
async def test_falls_open_on_unexpected_tier_value():
ctx, signal = await _build_signal()
classifier = TierClassifier(
@@ -105,7 +98,6 @@ async def test_falls_open_on_unexpected_tier_value():
assert await classifier.classify(ctx, signal) is None
async def test_falls_open_on_network_error():
ctx, signal = await _build_signal()
classifier = TierClassifier(
@@ -115,7 +107,6 @@ async def test_falls_open_on_network_error():
assert await classifier.classify(ctx, signal) is None
async def test_falls_open_on_empty_choices():
ctx, signal = await _build_signal()
classifier = TierClassifier(
@@ -125,7 +116,6 @@ async def test_falls_open_on_empty_choices():
assert await classifier.classify(ctx, signal) is None
async def test_default_window_omits_recent_messages():
"""recent_turn_window=0 keeps the prompt to the aggregate state only."""
ctx, signal = await _build_signal()
@@ -152,7 +142,6 @@ async def test_default_window_omits_recent_messages():
assert "Recent turns" not in user_prompt
async def test_disable_reasoning_passes_enable_thinking_false_extra_body():
"""Reasoning models on the NVIDIA Inference Hub misroute JSON output into
`reasoning_content` unless `enable_thinking=False` is hinted via vLLM's
@@ -174,7 +163,6 @@ async def test_disable_reasoning_passes_enable_thinking_false_extra_body():
assert captured["temperature"] == 0
async def test_disable_reasoning_false_omits_extra_body():
"""Opt-out path: `disable_reasoning=False` sends no extra_body."""
ctx, signal = await _build_signal()
@@ -192,7 +180,6 @@ async def test_disable_reasoning_false_omits_extra_body():
assert captured["extra_body"] is None
async def test_window_4_appends_last_four_messages():
"""recent_turn_window=4 includes the last four messages, in order."""
ctx, signal = await _build_signal()
@@ -224,3 +211,29 @@ async def test_window_4_appends_last_four_messages():
assert "msg-5" in user_prompt
assert "msg-0" not in user_prompt
assert "msg-1" not in user_prompt
+
+
+async def test_classify_stashes_submodel_call_for_intake() -> None:
+ """A successful tier-classifier call is recorded for the intake sink to emit."""
+ ctx, signal = await _build_signal()
+ resp = _Resp(content=json.dumps({"tier": "capable"}))
+ # The producer reads token usage off the response; _Resp carries no usage
+ # field, so attach one shaped like the OpenAI SDK usage object.
+ resp.usage = SimpleNamespace( # type: ignore[attr-defined]
+ prompt_tokens=333,
+ completion_tokens=12,
+ prompt_tokens_details=SimpleNamespace(cached_tokens=5),
+ )
+ classifier = TierClassifier(model="tier-clf", api_key="k", client=_StubClient(resp))
+
+ assert await classifier.classify(ctx, signal) == CAPABLE_TIER
+ assert ctx.submodel_calls == [
+ {
+ "model": "tier-clf",
+ "prompt_tokens": 333,
+ "completion_tokens": 12,
+ "cached_tokens": 5,
+ "router_type": "stage_router",
+ "routed_to": "tier_classifier",
+ }
+ ]