Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

fix(gateway): bill Messages cache tokens once in usage semantics (#75)

Jason Catce4c480256c7
diff --git a/crates/gateway/src/publication/compiler.rs b/crates/gateway/src/publication/compiler.rs
index bd542990..20081343 100644
--- a/crates/gateway/src/publication/compiler.rs
+++ b/crates/gateway/src/publication/compiler.rs
@@ -664,7 +664,7 @@ fn compile_candidate_pricing(
source_id: identity.source_id.clone().into(),
source_identity_digest: identity.source_identity_digest.clone(),
actual_offer_ref: identity.actual_offer_ref.clone().into(),
- usage_semantics: usage_semantics(profile.capability.upstream_protocol),
+ usage_semantics: usage_semantics(),
billing_context: PriceBillingContextV1::StandardTokens,
});
}
@@ -693,15 +693,15 @@ fn compile_candidate_pricing(
Ok(bindings.into())
}
-fn usage_semantics(protocol: IngressProtocol) -> UsageSemanticsV1 {
+fn usage_semantics() -> UsageSemanticsV1 {
UsageSemanticsV1 {
frame_kind: UsageFrameKindV1::Cumulative,
- input: match protocol {
- IngressProtocol::Responses | IngressProtocol::ChatCompletions => {
- InputUsageMeaningV1::IncludesExclusiveCache
- }
- IngressProtocol::Messages => InputUsageMeaningV1::UncachedOnly,
- },
+ // Every upstream decoder normalizes input to the complete input
+ // consumed by the request with cache read/creation as exclusive
+ // sub-buckets (`adapters/response/wire.rs`), so the uncached share is
+ // always derived by subtraction; declaring an uncached-only input
+ // frame here would bill the cache buckets a second time.
+ input: InputUsageMeaningV1::IncludesExclusiveCache,
output: OutputUsageMeaningV1::IncludesReasoning,
cache_buckets_exclusive: true,
}
diff --git a/crates/gateway/src/publication/tests.rs b/crates/gateway/src/publication/tests.rs
index 57a9f043..5003ef94 100644
--- a/crates/gateway/src/publication/tests.rs
+++ b/crates/gateway/src/publication/tests.rs
@@ -432,7 +432,7 @@ fn authorized_request_freezes_exact_price_identity_and_protocol_usage_semantics(
IngressProtocol::Messages,
"token-beta",
"beta",
- InputUsageMeaningV1::UncachedOnly,
+ InputUsageMeaningV1::IncludesExclusiveCache,
),
] {
let authorized = GatewayRequestAuthority::new(Arc::clone(&installer))
@@ -474,6 +474,52 @@ fn authorized_request_freezes_exact_price_identity_and_protocol_usage_semantics(
}
}
+#[test]
+fn every_upstream_protocol_freezes_complete_input_usage_semantics() {
+ use hiroute_domain::{
+ GatewayCandidatePricingIdentityV1, InputUsageMeaningV1, OutputUsageMeaningV1,
+ UsageFrameKindV1,
+ };
+
+ for protocol in [
+ IngressProtocol::Responses,
+ IngressProtocol::ChatCompletions,
+ IngressProtocol::Messages,
+ ] {
+ let mut candidate = exact_test_candidate(9, protocol, None);
+ candidate.pricing_identity = Some(GatewayCandidatePricingIdentityV1 {
+ source_id: "source-9".into(),
+ source_identity_digest: hiroute_domain::CanonicalDigest::of_bytes(b"source-9"),
+ model_configuration_id: candidate.protocol_profiles[0]
+ .capability
+ .model_configuration_id
+ .clone(),
+ actual_offer_ref: "offer-9".into(),
+ });
+ let alias = AliasPlanV1 {
+ served_model_id: "gamma".into(),
+ purpose: "gamma purpose".into(),
+ agent_plan_revision: 30,
+ protocols: vec![protocol],
+ overall_timeout_ms: 1_000,
+ max_attempts: 1,
+ routing: None,
+ candidates: vec![candidate],
+ };
+ let bindings = super::compiler::compile_pricing_bindings(&alias).unwrap();
+ assert_eq!(bindings.len(), 1);
+ let semantics = bindings[0].usage_semantics;
+ // Messages decoders normalize input to the complete input consumed by
+ // the request, so every upstream protocol must declare the cache
+ // buckets as exclusive sub-buckets of the reported input; declaring
+ // uncached-only input would bill cache tokens a second time.
+ assert_eq!(semantics.input, InputUsageMeaningV1::IncludesExclusiveCache);
+ assert_eq!(semantics.output, OutputUsageMeaningV1::IncludesReasoning);
+ assert_eq!(semantics.frame_kind, UsageFrameKindV1::Cumulative);
+ assert!(semantics.cache_buckets_exclusive);
+ }
+}
+
#[test]
fn managed_cpa_ipv6_target_compiles_as_an_exact_socket_without_dns() {
let mut value = snapshot(1, "renderer");