Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix: task dedup (#4450)

Eric Hayes772c2eb5b378
diff --git a/docker-compose.yml b/docker-compose.yml
index d5870cc129..1907c97810 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -425,8 +425,10 @@ services:
# Bun evaluates the circular @lexical/* ESM imports in an order that triggers
# a TDZ ("Cannot access 'HeadingNode'/'ElementNode' before initialization");
# bundling hoists/orders the declarations and resolves it (matching how the
- # worker is run normally — wrangler/esbuild bundles it first).
- command: ["sh", "-c", "bun build src/server.ts --target=bun --outfile=/tmp/server.js && exec bun /tmp/server.js"]
+ # worker is run normally — wrangler/esbuild bundles it first). loro-crdt
+ # stays external: bundling it emits its .wasm as a second output file
+ # (breaking --outfile) and mis-wires the wasm instantiation at runtime.
+ command: ["sh", "-c", "bun build src/server.ts --target=bun --external loro-crdt --outfile=/tmp/server.js && exec bun /tmp/server.js"]
environment:
- PORT=8096
- INTERNAL_AUTH_KEY=${INTERNAL_API_SECRET_KEY}
diff --git a/js/app/packages/block-md/component/TaskDuplicateList.tsx b/js/app/packages/block-md/component/TaskDuplicateList.tsx
index 75693f5220..9345f71e41 100644
--- a/js/app/packages/block-md/component/TaskDuplicateList.tsx
+++ b/js/app/packages/block-md/component/TaskDuplicateList.tsx
@@ -1,7 +1,10 @@
import { QUERY_FILTERS_BASE } from '@app/component/next-soup/filters/query-filters';
import { TaskListEntity } from '@app/component/next-soup/soup-view/views/tasks/TaskListEntity';
import { useFeatureFlag } from '@app/lib/analytics/posthog';
-import { ENABLE_TASK_DUPLICATES_FLAG } from '@core/constant/featureFlags';
+import {
+ ENABLE_TASK_DUPLICATES_FLAG,
+ ENABLE_TASK_DUPLICATES_OVERRIDE,
+} from '@core/constant/featureFlags';
import { ListLayoutProvider } from '@entity';
import CaretRightIcon from '@phosphor/caret-right.svg';
import CopyIcon from '@phosphor/copy.svg';
@@ -38,8 +41,6 @@ function SimilarTasksInner(props: {
const similarity = useTaskSimilaritySearchQuery(() => ({
title: props.debounced().title,
markdown: props.debounced().markdown,
- // Mirror the composer's create call, which does not share with a team.
- shareWithTeam: false,
}));
// The query retains its last data while disabled, so clear the results
@@ -124,7 +125,9 @@ export function SimilarTasksSection(props: {
content: Accessor<string>;
onOpenTask: (taskId: string) => void;
}) {
- const flag = useFeatureFlag(ENABLE_TASK_DUPLICATES_FLAG);
+ const flag = useFeatureFlag(ENABLE_TASK_DUPLICATES_FLAG, {
+ enabledOverride: ENABLE_TASK_DUPLICATES_OVERRIDE,
+ });
const [debounced, setDebounced] = createSignal<DebouncedInput>({
title: props.title(),
diff --git a/js/app/packages/queries/storage/keys.ts b/js/app/packages/queries/storage/keys.ts
index a5cf10dc46..b205c1eef3 100644
--- a/js/app/packages/queries/storage/keys.ts
+++ b/js/app/packages/queries/storage/keys.ts
@@ -81,12 +81,8 @@ export const entityKeys = createQueryKeys('entity', {
export const taskSimilaritySearchKeys = createQueryKeys(
'taskSimilaritySearch',
{
- forInput: (input: {
- title: string;
- markdown: string;
- shareWithTeam: boolean;
- }) => ({
- queryKey: [input.title, input.markdown, input.shareWithTeam],
+ forInput: (input: { title: string; markdown: string }) => ({
+ queryKey: [input.title, input.markdown],
}),
}
);
diff --git a/js/app/packages/queries/storage/task-duplicates.ts b/js/app/packages/queries/storage/task-duplicates.ts
index f93c455aee..8904711aaa 100644
--- a/js/app/packages/queries/storage/task-duplicates.ts
+++ b/js/app/packages/queries/storage/task-duplicates.ts
@@ -72,7 +72,6 @@ export function useDismissTaskDuplicatesMutation(documentId: Accessor<string>) {
type TaskSimilaritySearchInput = {
title: string;
markdown: string;
- shareWithTeam: boolean;
};
async function searchSimilarTasks(
@@ -84,7 +83,6 @@ async function searchSimilarTasks(
// compact embedding format up front rather than round-tripping through
// lexical-service on a latency-sensitive as-you-type search.
markdown: markdownToEmbeddingText(input.markdown),
- shareWithTeam: input.shareWithTeam,
});
if (result.isErr()) {
throw new Error('Failed to search for similar tasks');
diff --git a/js/app/packages/service-clients/service-storage/client.ts b/js/app/packages/service-clients/service-storage/client.ts
index ebfc0d49f1..bafd31de47 100644
--- a/js/app/packages/service-clients/service-storage/client.ts
+++ b/js/app/packages/service-clients/service-storage/client.ts
@@ -1212,11 +1212,7 @@ export const storageServiceClient = {
).map((result) => result.duplicates);
},
- async searchSimilarTasks(params: {
- taskName: string;
- markdown?: string;
- shareWithTeam?: boolean;
- }) {
+ async searchSimilarTasks(params: { taskName: string; markdown?: string }) {
return (
await dssFetch<TaskSimilaritySearchResponse>(
`/documents/similarity_search`,
@@ -1225,7 +1221,6 @@ export const storageServiceClient = {
body: JSON.stringify({
taskName: params.taskName,
markdown: params.markdown,
- shareWithTeam: params.shareWithTeam ?? false,
}),
}
)
diff --git a/rust/cloud-storage/.sqlx/query-517d22edfe5421d848804d496cf8a1ac59b76474ad1c8a695f41f324833bdcd6.json b/rust/cloud-storage/.sqlx/query-517d22edfe5421d848804d496cf8a1ac59b76474ad1c8a695f41f324833bdcd6.json
deleted file mode 100644
index 8e74d1f011..0000000000
--- a/rust/cloud-storage/.sqlx/query-517d22edfe5421d848804d496cf8a1ac59b76474ad1c8a695f41f324833bdcd6.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n SELECT values as \"values: serde_json::Value\"\n FROM entity_properties\n WHERE entity_id = $1 AND entity_type = $2 AND property_definition_id = $3\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "values: serde_json::Value",
- "type_info": "Jsonb"
- }
- ],
- "parameters": {
- "Left": [
- "Text",
- {
- "Custom": {
- "name": "property_entity_type",
- "kind": {
- "Enum": [
- "CHANNEL",
- "CHAT",
- "DOCUMENT",
- "PROJECT",
- "THREAD",
- "USER",
- "COMPANY",
- "TASK"
- ]
- }
- }
- },
- "Uuid"
- ]
- },
- "nullable": [
- true
- ]
- },
- "hash": "517d22edfe5421d848804d496cf8a1ac59b76474ad1c8a695f41f324833bdcd6"
-}
diff --git a/rust/cloud-storage/Cargo.lock b/rust/cloud-storage/Cargo.lock
index c0b5248d1c..b04ae0e588 100644
--- a/rust/cloud-storage/Cargo.lock
+++ b/rust/cloud-storage/Cargo.lock
@@ -992,6 +992,17 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
+[[package]]
+name = "automod"
+version = "1.0.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a273e731a7fb8c2268fd2932c9e9a3469f4fd171d85d070d2687190229653bd3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
[[package]]
name = "av-scenechange"
version = "0.14.1"
@@ -4854,7 +4865,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -8799,7 +8810,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -11096,7 +11107,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.12.1",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -11167,7 +11178,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -12920,6 +12931,7 @@ dependencies = [
"ai_usage",
"anyhow",
"async-trait",
+ "automod",
"aws-sdk-secretsmanager",
"clap",
"connection_gateway_client",
@@ -12934,6 +12946,7 @@ dependencies = [
"macro_middleware",
"macro_service_urls",
"model-entity",
+ "reqwest 0.13.4",
"secretsmanager_client",
"serde",
"serde_json",
@@ -12993,7 +13006,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix 1.1.4",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -14419,7 +14432,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.61.2",
]
[[package]]
diff --git a/rust/cloud-storage/Cargo.toml b/rust/cloud-storage/Cargo.toml
index fcb788345b..3fc96cf5d5 100644
--- a/rust/cloud-storage/Cargo.toml
+++ b/rust/cloud-storage/Cargo.toml
@@ -80,6 +80,7 @@ async-openai = { version = "0.36.1", features = [
async-graphql = "7.2.1"
async-stream = "0.3.6"
async-trait = "0.1.86"
+automod = "1"
aws-config = { version = "1.8.16", features = ["behavior-version-latest"] }
aws-sdk-dynamodb = "1.111.0"
aws-sdk-lambda = "1.122.0"
diff --git a/rust/cloud-storage/document_storage_service/src/config.rs b/rust/cloud-storage/document_storage_service/src/config.rs
index deec257101..7c0e38b168 100644
--- a/rust/cloud-storage/document_storage_service/src/config.rs
+++ b/rust/cloud-storage/document_storage_service/src/config.rs
@@ -30,6 +30,9 @@ env_vars! {
/// injected as `OPENAI_API_KEY` from the `openai-key` secret by the
/// infra stack, the same way `document_cognition_service` consumes it.
pub struct OpenaiApiKey;
+ /// Cohere API key used by the task-dedup reranker. Required — injected
+ /// as `COHERE_API_KEY`, following the same pattern as `OPENAI_API_KEY`.
+ pub struct CohereApiKey;
pub struct DocumentLimit;
pub struct DocumentStorageServicePresignedUrlExpirySeconds;
pub struct DocumentStorageServicePresignedUrlBrowserCacheExpirySeconds;
@@ -93,6 +96,7 @@ pub struct Config {
pub livekit_api_key: LivekitApiKey,
pub livekit_api_secret: LivekitApiSecret,
pub openai_api_key: OpenaiApiKey,
+ pub cohere_api_key: CohereApiKey,
pub github_webhook_secret_key: LocalOrRemoteSecret<GithubWebhookSecretKey>,
pub github_sync_app_pem_secret_key: LocalOrRemoteSecret<GithubSyncAppPemSecretKey>,
pub cal_webhook_secret_key: CalWebhookSecretKey,
diff --git a/rust/cloud-storage/document_storage_service/src/main.rs b/rust/cloud-storage/document_storage_service/src/main.rs
index 87c43db971..58f3591b0d 100644
--- a/rust/cloud-storage/document_storage_service/src/main.rs
+++ b/rust/cloud-storage/document_storage_service/src/main.rs
@@ -90,10 +90,10 @@ use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImp
use task_dedup::{
TaskDedupService,
outbound::{
+ cohere::CohereReranker,
connection_gateway::ConnectionGatewayTaskDedupNotifier,
judge::AgentDuplicateJudge,
postgres::{PgTaskMatchRepo, PgTaskVectorDb},
- reranker::NoOpReranker,
},
};
@@ -595,10 +595,15 @@ async fn main() -> anyhow::Result<()> {
!openai_api_key.trim().is_empty(),
"OpenAI API key is required for task dedup embeddings",
);
+ let cohere_api_key = config.cohere_api_key.as_ref().to_owned();
+ anyhow::ensure!(
+ !cohere_api_key.trim().is_empty(),
+ "Cohere API key is required for task dedup reranking",
+ );
let task_dedup_service = Arc::new(TaskDedupService::new(
TextEmbedding3Small::new(openai_api_key),
PgTaskVectorDb::new(db.clone()),
- NoOpReranker,
+ CohereReranker::new(cohere_api_key),
Arc::new(AgentDuplicateJudge::new(ai_usage::pg_recorder(db.clone()))),
Arc::new(ConnectionGatewayTaskDedupNotifier::new(
conn_gateway_client.clone(),
diff --git a/rust/cloud-storage/documents/src/inbound/axum_router/task_duplicates.rs b/rust/cloud-storage/documents/src/inbound/axum_router/task_duplicates.rs
index 24fce4fc2e..57754d7207 100644
--- a/rust/cloud-storage/documents/src/inbound/axum_router/task_duplicates.rs
+++ b/rust/cloud-storage/documents/src/inbound/axum_router/task_duplicates.rs
@@ -58,10 +58,6 @@ pub struct TaskSimilaritySearchRequest {
/// this latency-sensitive endpoint embeds the text as-is without a
/// lexical-service round trip.
pub markdown: Option<String>,
- /// Whether the task will be shared with the user's team, which widens the
- /// search scope to team tasks.
- #[serde(default)]
- pub share_with_team: bool,
}
/// Response for searching tasks similar to an unsaved draft.
@@ -111,13 +107,12 @@ pub async fn task_similarity_search_handler<T: DocumentService, Svc: EntityAcces
optional_team: OptionalMacroUserTeamExtractor<MemberTeamRole, Svc>,
Json(request): Json<TaskSimilaritySearchRequest>,
) -> Result<Json<TaskSimilaritySearchResponse>, DocumentError> {
- let team_id = if request.share_with_team {
- optional_team
- .entity_access_receipt
- .map(|team| macro_uuid::string_to_uuid(&team.entity().entity_id).unwrap())
- } else {
- None
- };
+ // Search the user's whole team, not just their own tasks: a duplicate is a
+ // duplicate no matter which teammate filed it. Users without a team fall
+ // back to owner-only scope.
+ let team_id = optional_team
+ .entity_access_receipt
+ .map(|team| macro_uuid::string_to_uuid(&team.entity().entity_id).unwrap());
// The composer renders the draft body with lexical-core's
// `markdownToEmbeddingText`, so we trust it as embedding-format here rather
// than round-tripping through lexical-service on this latency-sensitive path.
diff --git a/rust/cloud-storage/task_dedup/Cargo.toml b/rust/cloud-storage/task_dedup/Cargo.toml
index d36724fe36..69ccee71b7 100644
--- a/rust/cloud-storage/task_dedup/Cargo.toml
+++ b/rust/cloud-storage/task_dedup/Cargo.toml
@@ -46,6 +46,7 @@ embedding = { path = "../embedding" }
futures = { workspace = true }
lexical_client = { path = "../lexical_client" }
model-entity = { path = "../model-entity" }
+reqwest = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
sqlx = { workspace = true }
@@ -74,6 +75,7 @@ macro_service_urls = { path = "../macro_service_urls" }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
+automod = { workspace = true }
macro_db_migrator = { path = "../macro_db_migrator" }
macro_env_var = { path = "../macro_env_var" }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
diff --git a/rust/cloud-storage/task_dedup/justfile b/rust/cloud-storage/task_dedup/justfile
index a65003af64..606a06eeb3 100644
--- a/rust/cloud-storage/task_dedup/justfile
+++ b/rust/cloud-storage/task_dedup/justfile
@@ -4,14 +4,13 @@ import '../database.just'
# OpenAI embeddings + Anthropic judge over the labeled corpus, reporting
# precision/recall, retrieval recall@k, the vector-score threshold sweep, and
# judge variance. Needs the local macrodb up and OPENAI_API_KEY /
-# ANTHROPIC_API_KEY / CEREBRAS_API_KEY in the environment. Each measurement runs
-# its model calls concurrently, and the binaries run in parallel, for the fastest
-# wall-clock; pass a name to run just one (e.g. `just eval judge_accuracy`).
+# ANTHROPIC_API_KEY / CEREBRAS_API_KEY in the environment. Every file in
+# tests/eval/cases/ is discovered automatically (see tests/eval/main.rs). Each
+# measurement runs its model calls concurrently, and the measurements run on
+# parallel test threads, for the fastest wall-clock; pass a name to run just
+# one (e.g. `just eval judge_accuracy`).
eval filter="":
- DATABASE_URL={{ DATABASE_URL }} cargo test -p task_dedup \
- --test eval_judge_accuracy --test eval_retrieval_recall --test eval_end_to_end \
- --test eval_threshold_sweep --test eval_recall_at_k --test eval_judge_variance \
- {{ filter }} -- --ignored --nocapture
+ DATABASE_URL={{ DATABASE_URL }} cargo test -p task_dedup --test eval {{ filter }} -- --ignored --nocapture
test:
DATABASE_URL={{ DATABASE_URL }} cargo test
diff --git a/rust/cloud-storage/task_dedup/src/lib.rs b/rust/cloud-storage/task_dedup/src/lib.rs
index 0aa10a5222..9057818981 100644
--- a/rust/cloud-storage/task_dedup/src/lib.rs
+++ b/rust/cloud-storage/task_dedup/src/lib.rs
@@ -21,12 +21,12 @@ pub use domain::service::{TaskDedupConfig, TaskDedupService};
/// this crate. Its only constructors are the lexical fetch and an explicit
/// client-trusted wrapper, so wrong-format bodies cannot reach the embedder.
pub use lexical_client::parse_markdown::EmbeddingMarkdown;
+use outbound::cohere::CohereReranker;
use outbound::postgres::PgTaskVectorDb;
-use outbound::reranker::NoOpReranker;
/// The production task-dedup service: OpenAI `text-embedding-3-small` embeddings,
-/// a Postgres/pgvector store, and the no-op reranker. Consumers depend on this
+/// a Postgres/pgvector store, and the Cohere reranker. Consumers depend on this
/// concrete type so the generic [`TaskDedupService`] parameters do not leak into
/// axum state and handler signatures.
pub type PgTaskDedupService =
- TaskDedupService<DIMS, TextEmbedding3Small, PgTaskVectorDb, NoOpReranker>;
+ TaskDedupService<DIMS, TextEmbedding3Small, PgTaskVectorDb, CohereReranker>;
diff --git a/rust/cloud-storage/task_dedup/src/outbound/cohere.rs b/rust/cloud-storage/task_dedup/src/outbound/cohere.rs
new file mode 100644
index 0000000000..2802d3655c
--- /dev/null
+++ b/rust/cloud-storage/task_dedup/src/outbound/cohere.rs
@@ -0,0 +1,148 @@
+//! Cohere reranker adapter for task duplicate detection.
+
+use anyhow::Context as _;
+use embedding::{Content, RerankModel, Reranked, SearchResults};
+use serde::Deserialize;
+use serde_json::json;
+
+#[cfg(test)]
+mod test;
+
+/// The Cohere model used for reranking unless one is supplied explicitly.
+pub const DEFAULT_COHERE_RERANK_MODEL: &str = "rerank-v3.5";
+
+const COHERE_RERANK_URL: &str = "https://api.cohere.com/v2/rerank";
+
+/// Reranker backed by Cohere's [rerank API].
+///
+/// Each candidate is presented to the API as its matched field contents joined
+/// back together — the same reconstruction the service uses for judging — and
+/// comes back scored by relevance to the query.
+///
+/// [rerank API]: https://docs.cohere.com/reference/rerank
+#[derive(Clone)]
+pub struct CohereReranker {
+ client: reqwest::Client,
+ api_key: String,
+ model: String,
+}
+
+impl CohereReranker {
+ /// Creates a reranker authenticated with `api_key`, using
+ /// [`DEFAULT_COHERE_RERANK_MODEL`].
+ ///
+ /// The key is used directly — nothing is read from the environment.
+ pub fn new(api_key: impl Into<String>) -> Self {
+ Self::with_model(api_key, DEFAULT_COHERE_RERANK_MODEL)
+ }
+
+ /// Creates a reranker authenticated with `api_key` using an explicit Cohere
+ /// rerank model.
+ pub fn with_model(api_key: impl Into<String>, model: impl Into<String>) -> Self {
+ Self {
+ client: reqwest::Client::new(),
+ api_key: api_key.into(),
+ model: model.into(),
+ }
+ }
+}
+
+#[derive(Deserialize)]
+struct CohereRerankResponse {
+ results: Vec<CohereRerankResult>,
+}
+
+#[derive(Deserialize)]
+struct CohereRerankResult {
+ index: usize,
+ relevance_score: f32,
+}
+
+/// A candidate's matched field contents joined back together, mirroring how
+/// the service reconstructs candidate text for judging.
+fn document_text<T, const DIMS: usize>(result: &SearchResults<T, DIMS>) -> String {
+ result
+ .matches
+ .iter()
+ .map(|matched| matched.embedding.content.as_ref())
+ .collect::<Vec<_>>()
+ .join("\n")
+}
+
+/// Reorders `candidates` by the API's relevance results, pairing each result's
+/// `index` back to the candidate it scored. Rejects out-of-range and duplicate
+/// indices rather than dropping or double-counting candidates.
+fn order_by_relevance<T>(
+ results: Vec<CohereRerankResult>,
+ candidates: Vec<T>,
+) -> anyhow::Result<Vec<Reranked<T>>> {
+ let mut slots: Vec<Option<T>> = candidates.into_iter().map(Some).collect();
+ results
+ .into_iter()
+ .map(|result| {
+ let slot = slots
+ .get_mut(result.index)
+ .with_context(|| format!("Cohere returned out-of-range index {}", result.index))?;
+ let item = slot
+ .take()
+ .with_context(|| format!("Cohere returned duplicate index {}", result.index))?;
+ Ok(Reranked {
+ item,
+ score: result.relevance_score,
+ })
+ })
+ .collect()
+}
+
+impl<const DIMS: usize> RerankModel<DIMS> for CohereReranker {
+ async fn rerank<'a, T: Send>(
+ &self,
+ query: Content<'a>,
+ candidates: Vec<SearchResults<T, DIMS>>,
+ ) -> anyhow::Result<Vec<Reranked<T>>> {
+ // The API rejects an empty document list, so short-circuit.
+ if candidates.is_empty() {
+ return Ok(Vec::new());
+ }
+
+ let documents: Vec<String> = candidates.iter().map(document_text).collect();
+ let metadata: Vec<T> = candidates
+ .into_iter()
+ .map(|candidate| candidate.metadata)
+ .collect();
+
+ let response = self
+ .client
+ .post(COHERE_RERANK_URL)
+ .bearer_auth(&self.api_key)
+ .json(&json!({
+ "model": self.model,
+ "query": query.as_ref(),
+ "documents": documents,
+ }))
+ .send()
+ .await
+ .context("Cohere rerank request failed")?;
+
+ let status = response.status();
+ if !status.is_success() {
+ let body = response.text().await.unwrap_or_default();
+ anyhow::bail!("Cohere rerank returned {status}: {body}");
+ }
+
+ let body: CohereRerankResponse = response
+ .json()
+ .await
+ .context("Cohere rerank response was not valid JSON")?;
+
+ if body.results.len() != metadata.len() {
+ anyhow::bail!(
+ "Cohere returned {} results for {} documents",
+ body.results.len(),
+ metadata.len(),
+ );
+ }
+
+ order_by_relevance(body.results, metadata)
+ }
+}
diff --git a/rust/cloud-storage/task_dedup/src/outbound/cohere/test.rs b/rust/cloud-storage/task_dedup/src/outbound/cohere/test.rs
new file mode 100644
index 0000000000..5adbd66a91
--- /dev/null
+++ b/rust/cloud-storage/task_dedup/src/outbound/cohere/test.rs
@@ -0,0 +1,89 @@
+use std::borrow::Cow;
+
+use embedding::{LabeledEmbedding, Match};
+
+use super::*;
+
+const DIMS: usize = 3;
+
+fn search_result(metadata: &str, contents: &[&str]) -> SearchResults<String, DIMS> {
+ SearchResults {
+ metadata: metadata.to_string(),
+ matches: contents
+ .iter()
+ .map(|content| Match {
+ score: 0.5,
+ embedding: LabeledEmbedding {
+ search_key: "title",
+ content: Cow::Owned(content.to_string()),
+ embedding: [0.0; DIMS],
+ },
+ })
+ .collect(),
+ }
+}
+
+#[test]
+fn document_text_joins_matched_field_contents() {
+ let result = search_result("doc-1", &["fix the login bug", "users cannot sign in"]);
+ assert_eq!(
+ document_text(&result),
+ "fix the login bug\nusers cannot sign in"
+ );
+}
+
+#[test]
+fn document_text_of_empty_matches_is_empty() {
+ let result = search_result("doc-1", &[]);
+ assert_eq!(document_text(&result), "");
+}
+
+#[test]
+fn order_by_relevance_reorders_candidates_by_result_index() {
+ let results = vec![
+ CohereRerankResult {
+ index: 2,
+ relevance_score: 0.9,
+ },
+ CohereRerankResult {
+ index: 0,
+ relevance_score: 0.4,
+ },
+ CohereRerankResult {
+ index: 1,
+ relevance_score: 0.1,
+ },
+ ];
+ let reranked = order_by_relevance(results, vec!["a", "b", "c"]).unwrap();
+ let ordered: Vec<_> = reranked
+ .iter()
+ .map(|reranked| (reranked.item, reranked.score))
+ .collect();
+ assert_eq!(ordered, vec![("c", 0.9), ("a", 0.4), ("b", 0.1)]);
+}
+
+#[test]
+fn order_by_relevance_rejects_out_of_range_index() {
+ let results = vec![CohereRerankResult {
+ index: 3,
+ relevance_score: 0.9,
+ }];
+ let error = order_by_relevance(results, vec!["a"]).err().unwrap();
+ assert!(error.to_string().contains("out-of-range index 3"));
+}
+
+#[test]
+fn order_by_relevance_rejects_duplicate_index() {
+ let results = vec![
+ CohereRerankResult {
+ index: 0,
+ relevance_score: 0.9,
+ },
+ CohereRerankResult {
+ index: 0,
+ relevance_score: 0.4,
+ },
+ ];
+ let error = order_by_relevance(results, vec!["a", "b"]).err().unwrap();
+ assert!(error.to_string().contains("duplicate index 0"));
+}
diff --git a/rust/cloud-storage/task_dedup/src/outbound/mod.rs b/rust/cloud-storage/task_dedup/src/outbound/mod.rs
index 436e82cb47..f9ada6881d 100644
--- a/rust/cloud-storage/task_dedup/src/outbound/mod.rs
+++ b/rust/cloud-storage/task_dedup/src/outbound/mod.rs
@@ -1,6 +1,6 @@
//! Outbound adapters for task duplicate detection.
+pub mod cohere;
pub mod connection_gateway;
pub mod judge;
pub mod postgres;
-pub mod reranker;
diff --git a/rust/cloud-storage/task_dedup/src/outbound/postgres/test.rs b/rust/cloud-storage/task_dedup/src/outbound/postgres/test.rs
index b9a8333a32..113b815637 100644
--- a/rust/cloud-storage/task_dedup/src/outbound/postgres/test.rs
+++ b/rust/cloud-storage/task_dedup/src/outbound/postgres/test.rs
@@ -1,7 +1,9 @@
use std::sync::Arc;
use embedding::embedding_provider::openai::DIMS;
-use embedding::{Content, Embeddable, EmbeddingModel, LabeledEmbedding};
+use embedding::{
+ Content, Embeddable, EmbeddingModel, LabeledEmbedding, RerankModel, Reranked, SearchResults,
+};
use macro_db_migrator::MACRO_DB_MIGRATIONS;
use sqlx::PgPool;
@@ -11,7 +13,6 @@ use crate::domain::models::NewTask;
use crate::domain::ports::TaskDedupNotifier;
use crate::domain::service::TaskDedupService;
use crate::outbound::judge::LocalDuplicateJudge;
-use crate::outbound::reranker::NoOpReranker;
const OWNER: &str = "macro|[email protected]";
const TEAM_ID: Uuid = uuid::uuid!("a0000000-0000-0000-0000-000000000001");
@@ -21,6 +22,35 @@ const TASK_THREE: &str = "d1000000-0000-0000-0000-000000000003";
type TestService = TaskDedupService<DIMS, LocalEmbedder, PgTaskVectorDb, NoOpReranker>;
+/// Test-only reranker that preserves the upstream vector-similarity ordering,
+/// carrying each candidate's best vector score through unchanged, so these
+/// tests exercise the store rather than a reranking model.
+#[derive(Clone, Copy)]
+struct NoOpReranker;
+
+impl<const D: usize> RerankModel<D> for NoOpReranker {
+ async fn rerank<'a, T: Send>(
+ &self,
+ _query: Content<'a>,
+ candidates: Vec<SearchResults<T, D>>,
+ ) -> anyhow::Result<Vec<Reranked<T>>> {
+ Ok(candidates
+ .into_iter()
+ .map(|result| {
+ let score = result
+ .matches
+ .iter()
+ .map(|matched| matched.score)
+ .fold(f32::NEG_INFINITY, f32::max);
+ Reranked {
+ item: result.metadata,
+ score,
+ }
+ })
+ .collect())
+ }
+}
+
struct NoopNotifier;
#[async_trait::async_trait]
diff --git a/rust/cloud-storage/task_dedup/tests/empty_task_embedding.rs b/rust/cloud-storage/task_dedup/tests/eval/cases/empty_task_embedding.rs
similarity index 100%
rename from rust/cloud-storage/task_dedup/tests/empty_task_embedding.rs
rename to rust/cloud-storage/task_dedup/tests/eval/cases/empty_task_embedding.rs
diff --git a/rust/cloud-storage/task_dedup/tests/eval_corpus.rs b/rust/cloud-storage/task_dedup/tests/eval/cases/eval_corpus.rs
similarity index 98%
rename from rust/cloud-storage/task_dedup/tests/eval_corpus.rs
rename to rust/cloud-storage/task_dedup/tests/eval/cases/eval_corpus.rs
index df0b2736ea..7b76d2eb30 100644
--- a/rust/cloud-storage/task_dedup/tests/eval_corpus.rs
+++ b/rust/cloud-storage/task_dedup/tests/eval/cases/eval_corpus.rs
@@ -1,10 +1,9 @@
//! Offline guards that run in normal CI (no API calls): the committed corpus is
//! well-formed and covers every scenario, and the metric math is correct.
-mod eval;
-use eval::corpus::load_corpus;
-use eval::metrics::{
+use crate::util::corpus::load_corpus;
+use crate::util::metrics::{
Confusion, PairOutcome, average_precision, recall_at_k, report, threshold_sweep,
};
use task_dedup::eval::PairCase;
diff --git a/rust/cloud-storage/task_dedup/tests/eval_end_to_end.rs b/rust/cloud-storage/task_dedup/tests/eval/cases/eval_end_to_end.rs
similarity index 95%
rename from rust/cloud-storage/task_dedup/tests/eval_end_to_end.rs
rename to rust/cloud-storage/task_dedup/tests/eval/cases/eval_end_to_end.rs
index e7abdf1c0a..1d6ac3658a 100644
--- a/rust/cloud-storage/task_dedup/tests/eval_end_to_end.rs
+++ b/rust/cloud-storage/task_dedup/tests/eval/cases/eval_end_to_end.rs
@@ -17,17 +17,16 @@
//!
//! `#[ignore]` — hits OpenAI + Anthropic. Run with `just eval`.
-mod eval;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Instant;
use embedding::embedding_provider::openai::TextEmbedding3Small;
use embedding::{EmbeddingModel, VectorStore};
-use eval::corpus::{embeddable, load_corpus, seed_ids};
-use eval::harness::{EVAL_CONCURRENCY, build_service, openai_key};
-use eval::metrics::{PairOutcome, report};
-use eval::seed::{EVAL_OWNER, reset_matches, seed_documents};
+use crate::util::corpus::{embeddable, load_corpus, seed_ids};
+use crate::util::harness::{EVAL_CONCURRENCY, build_service, openai_key};
+use crate::util::metrics::{PairOutcome, report};
+use crate::util::seed::{EVAL_OWNER, reset_matches, seed_documents};
use futures::StreamExt;
use macro_db_migrator::MACRO_DB_MIGRATIONS;
use sqlx::PgPool;
@@ -36,7 +35,7 @@ use task_dedup::{EmbeddingMarkdown, NewTask, TaskDedupConfig};
#[sqlx::test(
migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../documents/fixtures", scripts("documents_test_data"))
+ fixtures(path = "../../../../documents/fixtures", scripts("documents_test_data"))
)]
#[ignore = "hits OpenAI + Anthropic; run locally with API keys"]
async fn end_to_end_detection_baseline(pool: PgPool) {
diff --git a/rust/cloud-storage/task_dedup/tests/eval_judge_accuracy.rs b/rust/cloud-storage/task_dedup/tests/eval/cases/eval_judge_accuracy.rs
similarity index 93%
rename from rust/cloud-storage/task_dedup/tests/eval_judge_accuracy.rs
rename to rust/cloud-storage/task_dedup/tests/eval/cases/eval_judge_accuracy.rs
index 6bc987e0b0..977f33e03a 100644
--- a/rust/cloud-storage/task_dedup/tests/eval_judge_accuracy.rs
+++ b/rust/cloud-storage/task_dedup/tests/eval/cases/eval_judge_accuracy.rs
@@ -5,11 +5,10 @@
//!
//! `#[ignore]` — hits Anthropic. Run with `just eval` (see `task_dedup/justfile`).
-mod eval;
-use eval::corpus::{full_text, load_corpus};
-use eval::harness::{EVAL_CONCURRENCY, openai_key};
-use eval::metrics::{PairOutcome, report};
+use crate::util::corpus::{full_text, load_corpus};
+use crate::util::harness::{EVAL_CONCURRENCY, openai_key};
+use crate::util::metrics::{PairOutcome, report};
use futures::StreamExt;
use macro_db_migrator::MACRO_DB_MIGRATIONS;
use sqlx::PgPool;
diff --git a/rust/cloud-storage/task_dedup/tests/eval_judge_variance.rs b/rust/cloud-storage/task_dedup/tests/eval/cases/eval_judge_variance.rs
similarity index 98%
rename from rust/cloud-storage/task_dedup/tests/eval_judge_variance.rs
rename to rust/cloud-storage/task_dedup/tests/eval/cases/eval_judge_variance.rs
index 7d314688ab..b28be094b7 100644
--- a/rust/cloud-storage/task_dedup/tests/eval_judge_variance.rs
+++ b/rust/cloud-storage/task_dedup/tests/eval/cases/eval_judge_variance.rs
@@ -12,10 +12,9 @@
//!
//! `#[ignore]` — hits Anthropic. Run with `just eval`.
-mod eval;
-use eval::corpus::{full_text, load_corpus};
-use eval::harness::{EVAL_CONCURRENCY, openai_key};
+use crate::util::corpus::{full_text, load_corpus};
+use crate::util::harness::{EVAL_CONCURRENCY, openai_key};
use futures::StreamExt;
use macro_db_migrator::MACRO_DB_MIGRATIONS;
use sqlx::PgPool;
diff --git a/rust/cloud-storage/task_dedup/tests/eval_recall_at_k.rs b/rust/cloud-storage/task_dedup/tests/eval/cases/eval_recall_at_k.rs
similarity index 93%
rename from rust/cloud-storage/task_dedup/tests/eval_recall_at_k.rs
rename to rust/cloud-storage/task_dedup/tests/eval/cases/eval_recall_at_k.rs
index 658be3e7a5..92bc5344f9 100644
--- a/rust/cloud-storage/task_dedup/tests/eval_recall_at_k.rs
+++ b/rust/cloud-storage/task_dedup/tests/eval/cases/eval_recall_at_k.rs
@@ -11,14 +11,13 @@
//!
//! `#[ignore]` — hits OpenAI. Run with `just eval`.
-mod eval;
use embedding::embedding_provider::openai::TextEmbedding3Small;
use embedding::{EmbeddingModel, VectorStore};
-use eval::corpus::{embeddable, load_corpus, seed_ids};
-use eval::harness::{EVAL_CONCURRENCY, build_service, openai_key};
-use eval::metrics::{fmt_ratio, recall_at_k};
-use eval::seed::{EVAL_OWNER, seed_documents};
+use crate::util::corpus::{embeddable, load_corpus, seed_ids};
+use crate::util::harness::{EVAL_CONCURRENCY, build_service, openai_key};
+use crate::util::metrics::{fmt_ratio, recall_at_k};
+use crate::util::seed::{EVAL_OWNER, seed_documents};
use futures::StreamExt;
use macro_db_migrator::MACRO_DB_MIGRATIONS;
use sqlx::PgPool;
@@ -27,7 +26,7 @@ use task_dedup::{EmbeddingMarkdown, TaskDedupConfig};
#[sqlx::test(
migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../documents/fixtures", scripts("documents_test_data"))
+ fixtures(path = "../../../../documents/fixtures", scripts("documents_test_data"))
)]
#[ignore = "hits OpenAI; run locally with API keys"]
async fn recall_at_k_baseline(pool: PgPool) {
diff --git a/rust/cloud-storage/task_dedup/tests/eval_retrieval_recall.rs b/rust/cloud-storage/task_dedup/tests/eval/cases/eval_retrieval_recall.rs
similarity index 92%
rename from rust/cloud-storage/task_dedup/tests/eval_retrieval_recall.rs
rename to rust/cloud-storage/task_dedup/tests/eval/cases/eval_retrieval_recall.rs
index 0d1755eb0d..e4607542dd 100644
--- a/rust/cloud-storage/task_dedup/tests/eval_retrieval_recall.rs
+++ b/rust/cloud-storage/task_dedup/tests/eval/cases/eval_retrieval_recall.rs
@@ -6,14 +6,13 @@
//!
//! `#[ignore]` — hits OpenAI. Run with `just eval`.
-mod eval;
use embedding::embedding_provider::openai::TextEmbedding3Small;
use embedding::{EmbeddingModel, VectorStore};
-use eval::corpus::{embeddable, load_corpus, seed_ids};
-use eval::harness::{EVAL_CONCURRENCY, build_service, openai_key};
-use eval::metrics::fmt_ratio;
-use eval::seed::{EVAL_OWNER, seed_documents};
+use crate::util::corpus::{embeddable, load_corpus, seed_ids};
+use crate::util::harness::{EVAL_CONCURRENCY, build_service, openai_key};
+use crate::util::metrics::fmt_ratio;
+use crate::util::seed::{EVAL_OWNER, seed_documents};
use futures::StreamExt;
use macro_db_migrator::MACRO_DB_MIGRATIONS;
use sqlx::PgPool;
@@ -22,7 +21,7 @@ use task_dedup::{EmbeddingMarkdown, TaskDedupConfig};
#[sqlx::test(
migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../documents/fixtures", scripts("documents_test_data"))
+ fixtures(path = "../../../../documents/fixtures", scripts("documents_test_data"))
)]
#[ignore = "hits OpenAI; run locally with API keys"]
async fn retrieval_recall_baseline(pool: PgPool) {
diff --git a/rust/cloud-storage/task_dedup/tests/eval_threshold_sweep.rs b/rust/cloud-storage/task_dedup/tests/eval/cases/eval_threshold_sweep.rs
similarity index 95%
rename from rust/cloud-storage/task_dedup/tests/eval_threshold_sweep.rs
rename to rust/cloud-storage/task_dedup/tests/eval/cases/eval_threshold_sweep.rs
index 867882c576..02322d90ae 100644
--- a/rust/cloud-storage/task_dedup/tests/eval_threshold_sweep.rs
+++ b/rust/cloud-storage/task_dedup/tests/eval/cases/eval_threshold_sweep.rs
@@ -11,16 +11,15 @@
//!
//! `#[ignore]` — hits OpenAI. Run with `just eval`.
-mod eval;
use std::collections::HashMap;
use embedding::EmbeddingModel;
use embedding::LabeledEmbedding;
use embedding::embedding_provider::openai::TextEmbedding3Small;
-use eval::corpus::{embeddable, load_corpus};
-use eval::harness::{EVAL_CONCURRENCY, openai_key};
-use eval::metrics::sweep_report;
+use crate::util::corpus::{embeddable, load_corpus};
+use crate::util::harness::{EVAL_CONCURRENCY, openai_key};
+use crate::util::metrics::sweep_report;
use futures::StreamExt;
/// Cosine similarity between two equal-length vectors.
diff --git a/rust/cloud-storage/task_dedup/tests/eval/main.rs b/rust/cloud-storage/task_dedup/tests/eval/main.rs
new file mode 100644
index 0000000000..c659aac544
--- /dev/null
+++ b/rust/cloud-storage/task_dedup/tests/eval/main.rs
@@ -0,0 +1,11 @@
+//! Entry point for the `eval` test target. Cargo auto-discovers
+//! `tests/eval/main.rs` as an integration-test target named `eval`, and
+//! `automod` expands every file in `tests/eval/cases/` into a module at compile
+//! time — drop a new `*.rs` file in `cases/` and it is picked up with no
+//! registration in `Cargo.toml`, here, or the justfile.
+
+mod util;
+
+mod cases {
+ automod::dir!("tests/eval/cases");
+}
diff --git a/rust/cloud-storage/task_dedup/tests/eval/mod.rs b/rust/cloud-storage/task_dedup/tests/eval/mod.rs
deleted file mode 100644
index d9a1bbf142..0000000000
--- a/rust/cloud-storage/task_dedup/tests/eval/mod.rs
+++ /dev/null
@@ -1,18 +0,0 @@
-//! Shared support for the semantic-eval test binaries.
-//!
-//! Each measurement lives in its own top-level test file (`eval_*.rs`) and pulls
-//! this module in with `mod eval;`. It is split by concern so no single file is a
-//! grab-bag:
-//! - [`corpus`] — load the labeled corpus and per-task helpers,
-//! - [`seed`] — insert the DB rows a measurement needs,
-//! - [`harness`] — build the real service (OpenAI embedder + Anthropic judge),
-//! - [`metrics`] — confusion matrix + text report.
-//!
-//! `#![allow(dead_code)]` because each test binary uses only the slice of this
-//! module it needs; items unused by a given binary would otherwise warn.
-#![allow(dead_code)]
-
-pub mod corpus;
-pub mod harness;
-pub mod metrics;
-pub mod seed;
diff --git a/rust/cloud-storage/task_dedup/tests/eval/corpus.rs b/rust/cloud-storage/task_dedup/tests/eval/util/corpus.rs
similarity index 79%
rename from rust/cloud-storage/task_dedup/tests/eval/corpus.rs
rename to rust/cloud-storage/task_dedup/tests/eval/util/corpus.rs
index 1b85cf76ad..f67d5aed1c 100644
--- a/rust/cloud-storage/task_dedup/tests/eval/corpus.rs
+++ b/rust/cloud-storage/task_dedup/tests/eval/util/corpus.rs
@@ -23,18 +23,19 @@ use task_dedup::eval::{CorpusTask, EvalCorpus, TaskSource};
/// Loads and merges the committed fixtures into one corpus, panicking if any
/// labeled pair references a task that is missing.
pub fn load_corpus() -> EvalCorpus {
- let mut corpus =
- EvalCorpus::from_json(include_bytes!("../../fixtures/eval/prod_title_only.json"))
- .expect("prod_title_only.json parses");
+ let mut corpus = EvalCorpus::from_json(include_bytes!(
+ "../../../fixtures/eval/prod_title_only.json"
+ ))
+ .expect("prod_title_only.json parses");
for bytes in [
- include_bytes!("../../fixtures/eval/prod_with_body.json").as_slice(),
- include_bytes!("../../fixtures/eval/prod_pairs.json").as_slice(),
- include_bytes!("../../fixtures/eval/prod_mined_pairs.json").as_slice(),
- include_bytes!("../../fixtures/eval/prod_hard_negatives.json").as_slice(),
- include_bytes!("../../fixtures/eval/prod_hard_negative_pairs.json").as_slice(),
- include_bytes!("../../fixtures/eval/synthetic.json").as_slice(),
- include_bytes!("../../fixtures/eval/synthetic_generated.json").as_slice(),
- include_bytes!("../../fixtures/eval/synthetic_hard_positives.json").as_slice(),
+ include_bytes!("../../../fixtures/eval/prod_with_body.json").as_slice(),
+ include_bytes!("../../../fixtures/eval/prod_pairs.json").as_slice(),
+ include_bytes!("../../../fixtures/eval/prod_mined_pairs.json").as_slice(),
+ include_bytes!("../../../fixtures/eval/prod_hard_negatives.json").as_slice(),
+ include_bytes!("../../../fixtures/eval/prod_hard_negative_pairs.json").as_slice(),
+ include_bytes!("../../../fixtures/eval/synthetic.json").as_slice(),
+ include_bytes!("../../../fixtures/eval/synthetic_generated.json").as_slice(),
+ include_bytes!("../../../fixtures/eval/synthetic_hard_positives.json").as_slice(),
] {
corpus.merge(EvalCorpus::from_json(bytes).expect("fixture parses"));
}
diff --git a/rust/cloud-storage/task_dedup/tests/eval/harness.rs b/rust/cloud-storage/task_dedup/tests/eval/util/harness.rs
similarity index 73%
rename from rust/cloud-storage/task_dedup/tests/eval/harness.rs
rename to rust/cloud-storage/task_dedup/tests/eval/util/harness.rs
index cfa0c61540..bd4dc66df6 100644
--- a/rust/cloud-storage/task_dedup/tests/eval/harness.rs
+++ b/rust/cloud-storage/task_dedup/tests/eval/util/harness.rs
@@ -4,13 +4,20 @@
use std::sync::Arc;
use async_trait::async_trait;
-use embedding::embedding_provider::openai::TextEmbedding3Small;
+use embedding::embedding_provider::openai::{DIMS, TextEmbedding3Small};
use sqlx::PgPool;
use task_dedup::domain::ports::TaskDedupNotifier;
use task_dedup::outbound::judge::AgentDuplicateJudge;
use task_dedup::outbound::postgres::{PgTaskMatchRepo, PgTaskVectorDb};
-use task_dedup::outbound::reranker::NoOpReranker;
-use task_dedup::{PgTaskDedupService, TaskDedupConfig, TaskDedupService};
+use task_dedup::{TaskDedupConfig, TaskDedupService};
+
+use super::rerank::NoOpReranker;
+
+/// The service shape the evals run: the production embedder, store, and judge,
+/// but the test-only [`NoOpReranker`] so measurements isolate embedding + judge
+/// quality (production `PgTaskDedupService` runs the Cohere reranker).
+pub type EvalDedupService =
+ TaskDedupService<DIMS, TextEmbedding3Small, PgTaskVectorDb, NoOpReranker>;
/// Concurrency for the parallel measurement loops. The heavy work is network
/// (OpenAI / Anthropic) and DB connections are released across awaits, so a
@@ -47,9 +54,10 @@ impl TaskDedupNotifier for NoopNotifier {
}
}
-/// Builds the production service (real embedder + judge) over `pool` with the
-/// given config. The judge records usage into the ephemeral db's ai_usage table.
-pub fn build_service(pool: &PgPool, config: TaskDedupConfig) -> PgTaskDedupService {
+/// Builds the eval service (real embedder + judge, no-op reranker) over `pool`
+/// with the given config. The judge records usage into the ephemeral db's
+/// ai_usage table.
+pub fn build_service(pool: &PgPool, config: TaskDedupConfig) -> EvalDedupService {
TaskDedupService::with_config(
config,
TextEmbedding3Small::new(openai_key()),
diff --git a/rust/cloud-storage/task_dedup/tests/eval/metrics.rs b/rust/cloud-storage/task_dedup/tests/eval/util/metrics.rs
similarity index 100%
rename from rust/cloud-storage/task_dedup/tests/eval/metrics.rs
rename to rust/cloud-storage/task_dedup/tests/eval/util/metrics.rs
diff --git a/rust/cloud-storage/task_dedup/tests/eval/util/mod.rs b/rust/cloud-storage/task_dedup/tests/eval/util/mod.rs
new file mode 100644
index 0000000000..a497c5e620
--- /dev/null
+++ b/rust/cloud-storage/task_dedup/tests/eval/util/mod.rs
@@ -0,0 +1,20 @@
+//! Shared support for the semantic-eval tests.
+//!
+//! Each measurement lives in its own file under `cases/`, auto-discovered by
+//! `automod` in `main.rs`, and reaches this module as `crate::util`. It is
+//! split by concern so no single file is a grab-bag:
+//! - [`corpus`] — load the labeled corpus and per-task helpers,
+//! - [`seed`] — insert the DB rows a measurement needs,
+//! - [`harness`] — build the real service (OpenAI embedder + Anthropic judge),
+//! - [`metrics`] — confusion matrix + text report,
+//! - [`rerank`] — the test-only no-op reranker the eval services run with.
+//!
+//! `#![allow(dead_code)]` because each measurement uses only the slice of this
+//! module it needs; items unused by a given measurement would otherwise warn.
+#![allow(dead_code)]
+
+pub mod corpus;
+pub mod harness;
+pub mod metrics;
+pub mod rerank;
+pub mod seed;
diff --git a/rust/cloud-storage/task_dedup/src/outbound/reranker.rs b/rust/cloud-storage/task_dedup/tests/eval/util/rerank.rs
similarity index 80%
rename from rust/cloud-storage/task_dedup/src/outbound/reranker.rs
rename to rust/cloud-storage/task_dedup/tests/eval/util/rerank.rs
index ec85628c54..54f53f3091 100644
--- a/rust/cloud-storage/task_dedup/src/outbound/reranker.rs
+++ b/rust/cloud-storage/task_dedup/tests/eval/util/rerank.rs
@@ -1,4 +1,4 @@
-//! Reranker adapters for task duplicate detection.
+//! Test-only reranker for the semantic evals.
use embedding::{Content, RerankModel, Reranked, SearchResults};
@@ -6,9 +6,9 @@ use embedding::{Content, RerankModel, Reranked, SearchResults};
/// every candidate back in input order, carrying its existing vector-similarity
/// score through unchanged.
///
-/// This is the default until a real cross-encoder is wired up. A production
-/// implementation — e.g. a Cohere rerank connection — implements the same
-/// [`RerankModel`] and can be swapped in without touching the service.
+/// The evals use it so measurements isolate embedding + judge quality without a
+/// reranking model (or its API key) in the loop; production uses
+/// `task_dedup::outbound::cohere::CohereReranker`.
#[derive(Clone, Copy, Default)]
pub struct NoOpReranker;
diff --git a/rust/cloud-storage/task_dedup/tests/eval/seed.rs b/rust/cloud-storage/task_dedup/tests/eval/util/seed.rs
similarity index 100%
rename from rust/cloud-storage/task_dedup/tests/eval/seed.rs
rename to rust/cloud-storage/task_dedup/tests/eval/util/seed.rs