Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

bench(tau2): add generic scope fairness check (#2172)

huangruiteng8817abdd38f9
diff --git a/benchmark/tau2/README.md b/benchmark/tau2/README.md
index 3c9e5501a..59ec971a8 100644
--- a/benchmark/tau2/README.md
+++ b/benchmark/tau2/README.md
@@ -184,10 +184,11 @@ Memory V2 cells run through a small TAU-2 agent adapter in this directory:
regenerate that step with the matched memories. The default benchmark
retrieves 6 pre-write candidates and injects 2, which keeps extra candidates
visible in traces without expanding the prompt budget;
-- optionally run an explicit scope-prompt treatment that keeps retrieved
+- optionally run an explicit generic scope-prompt treatment that keeps retrieved
memories advisory and asks the agent to preserve the current task scope before
- write-like tool calls. Configs provide per-domain files through
- `scope_prompt_files`;
+ write-like tool calls. The benchmark configs use a single benchmark-neutral
+ `scope_prompt_file`; the runner still accepts `scope_prompt_files` for custom
+ local experiments;
- emit artifact metadata to identify the OpenViking account, agent,
corpus, retrieval mode, and simulator policy used by each cell.
diff --git a/benchmark/tau2/config/prb_content_matrix_new_prompt.yaml b/benchmark/tau2/config/prb_content_matrix_new_prompt.yaml
index bfb711e74..8a0a18a39 100644
--- a/benchmark/tau2/config/prb_content_matrix_new_prompt.yaml
+++ b/benchmark/tau2/config/prb_content_matrix_new_prompt.yaml
@@ -24,9 +24,7 @@ strategies:
retrieval_mode: first_user
retrieval_top_k: 4
first_user_inject_top_k: 4
- scope_prompt_files:
- retail: benchmark/tau2/config/scope_prompts/retail_memory_scope.md
- airline: benchmark/tau2/config/scope_prompts/airline_memory_scope.md
+ scope_prompt_file: benchmark/tau2/config/scope_prompts/generic_memory_scope.md
- id: new_traj_fixed_prewrite_only
label: PR-B new trajectory fixed-count prewrite top4
@@ -42,9 +40,7 @@ strategies:
retrieval_top_k: 4
prewrite_retrieval_top_k: 4
prewrite_inject_top_k: 4
- scope_prompt_files:
- retail: benchmark/tau2/config/scope_prompts/retail_memory_scope.md
- airline: benchmark/tau2/config/scope_prompts/airline_memory_scope.md
+ scope_prompt_file: benchmark/tau2/config/scope_prompts/generic_memory_scope.md
- id: new_traj_fixed_first_user_prewrite
label: PR-B new trajectory fixed-count first-user + prewrite top4
@@ -61,9 +57,7 @@ strategies:
first_user_inject_top_k: 4
prewrite_retrieval_top_k: 4
prewrite_inject_top_k: 4
- scope_prompt_files:
- retail: benchmark/tau2/config/scope_prompts/retail_memory_scope.md
- airline: benchmark/tau2/config/scope_prompts/airline_memory_scope.md
+ scope_prompt_file: benchmark/tau2/config/scope_prompts/generic_memory_scope.md
- id: new_exp_fixed_first_user
label: PR-B new experience fixed-count first-user top2
@@ -78,9 +72,7 @@ strategies:
retrieval_mode: first_user
retrieval_top_k: 2
first_user_inject_top_k: 2
- scope_prompt_files:
- retail: benchmark/tau2/config/scope_prompts/retail_memory_scope.md
- airline: benchmark/tau2/config/scope_prompts/airline_memory_scope.md
+ scope_prompt_file: benchmark/tau2/config/scope_prompts/generic_memory_scope.md
- id: new_exp_fixed_prewrite_only
label: PR-B new experience fixed-count prewrite top2
@@ -96,9 +88,7 @@ strategies:
retrieval_top_k: 2
prewrite_retrieval_top_k: 2
prewrite_inject_top_k: 2
- scope_prompt_files:
- retail: benchmark/tau2/config/scope_prompts/retail_memory_scope.md
- airline: benchmark/tau2/config/scope_prompts/airline_memory_scope.md
+ scope_prompt_file: benchmark/tau2/config/scope_prompts/generic_memory_scope.md
- id: new_exp_fixed_first_user_prewrite
label: PR-B new experience fixed-count first-user + prewrite top2
@@ -115,9 +105,7 @@ strategies:
first_user_inject_top_k: 2
prewrite_retrieval_top_k: 2
prewrite_inject_top_k: 2
- scope_prompt_files:
- retail: benchmark/tau2/config/scope_prompts/retail_memory_scope.md
- airline: benchmark/tau2/config/scope_prompts/airline_memory_scope.md
+ scope_prompt_file: benchmark/tau2/config/scope_prompts/generic_memory_scope.md
- id: new_traj_4000_prewrite_only
label: PR-B new trajectory 4000-char prewrite
@@ -134,9 +122,7 @@ strategies:
prewrite_retrieval_top_k: 8
prewrite_inject_top_k: 8
memory_inject_max_chars: 4000
- scope_prompt_files:
- retail: benchmark/tau2/config/scope_prompts/retail_memory_scope.md
- airline: benchmark/tau2/config/scope_prompts/airline_memory_scope.md
+ scope_prompt_file: benchmark/tau2/config/scope_prompts/generic_memory_scope.md
- id: new_exp_4000_first_user_prewrite
label: PR-B new experience 4000-char first-user + prewrite
@@ -154,6 +140,4 @@ strategies:
prewrite_retrieval_top_k: 8
prewrite_inject_top_k: 8
memory_inject_max_chars: 4000
- scope_prompt_files:
- retail: benchmark/tau2/config/scope_prompts/retail_memory_scope.md
- airline: benchmark/tau2/config/scope_prompts/airline_memory_scope.md
+ scope_prompt_file: benchmark/tau2/config/scope_prompts/generic_memory_scope.md
diff --git a/benchmark/tau2/config/prb_scope_fairness.yaml b/benchmark/tau2/config/prb_scope_fairness.yaml
new file mode 100644
index 000000000..cc2f820c1
--- /dev/null
+++ b/benchmark/tau2/config/prb_scope_fairness.yaml
@@ -0,0 +1,34 @@
+extends: baseline.yaml
+
+benchmark:
+ name: tau2_prb_scope_fairness
+ strategy_concurrency: 16
+ task_max_concurrency: 5
+ corpus_prepare_concurrency: 1
+
+openviking:
+ url: ${OPENVIKING_URL:-http://localhost:1933}
+
+strategies:
+ - id: no_memory
+ label: TAU-2 no-memory same-seed baseline
+ memory_backend: none
+
+ - id: no_memory_generic_scope
+ label: TAU-2 no-memory same-seed baseline with generic memory scope prompt
+ memory_backend: none
+ scope_prompt_file: benchmark/tau2/config/scope_prompts/generic_memory_scope.md
+
+ - id: trajectory_top4_first_user_prewrite_generic_scope
+ label: Trajectory top4 first-user + pre-write with generic memory scope prompt
+ memory_backend: openviking
+ train_required: true
+ corpus_id: memory_v2_trajectory_view
+ train_memory_mode: experience_only
+ search_memory_type: trajectories
+ retrieval_mode: first_user_prewrite
+ retrieval_top_k: 4
+ first_user_inject_top_k: 4
+ prewrite_retrieval_top_k: 4
+ prewrite_inject_top_k: 4
+ scope_prompt_file: benchmark/tau2/config/scope_prompts/generic_memory_scope.md
diff --git a/benchmark/tau2/config/scope_prompts/airline_memory_scope.md b/benchmark/tau2/config/scope_prompts/airline_memory_scope.md
deleted file mode 100644
index 8847796a9..000000000
--- a/benchmark/tau2/config/scope_prompts/airline_memory_scope.md
+++ /dev/null
@@ -1,18 +0,0 @@
-<openviking_memory_scope_guard>
-OpenViking memories are advisory. Use them only when their trigger, preconditions,
-and applicability boundary match the current airline task.
-
-- Do not broaden the user's requested booking, cancellation, rebooking, flight
- update, passenger update, baggage update, insurance, or payment scope because a
- retrieved memory describes a nearby workflow.
-- Keep the current reservation scope explicit. Only use flights, passengers,
- baggage entries, cabin changes, insurance choices, payment IDs, dates, and
- amounts that are grounded in user input, recent tool observations, reservation
- state, profile/payment state, or an explicit search/lookup result.
-- Before a write tool call, verify that the selected write action matches the
- user's requested operation. Do not mix cancellation, rebooking, upgrade,
- downgrade, baggage, or passenger-update flows unless the current task asks for
- that combined operation.
-- If a memory and the current task disagree, follow the current task state and the
- domain policy.
-</openviking_memory_scope_guard>
diff --git a/benchmark/tau2/config/scope_prompts/generic_memory_scope.md b/benchmark/tau2/config/scope_prompts/generic_memory_scope.md
new file mode 100644
index 000000000..74c578969
--- /dev/null
+++ b/benchmark/tau2/config/scope_prompts/generic_memory_scope.md
@@ -0,0 +1,16 @@
+<openviking_memory_scope_guard>
+Retrieved OpenViking memories are advisory examples, not policy or hidden task
+requirements. Use a memory only when its trigger, preconditions, object scope,
+and action boundary match the current task.
+
+- Do not broaden the user's requested objective, target object, or write scope
+ because a retrieved memory describes a nearby workflow.
+- Before any write or irreversible action, verify that the selected operation
+ matches the user's current request and the latest observed state.
+- Every write argument must be grounded in user input, recent tool observations,
+ current state, profile/account state, or an explicit lookup result. Do not copy
+ identifiers, amounts, dates, object references, or action choices from memory
+ unless they are re-grounded in the current task.
+- If memory conflicts with the current task, current state, tool results, or
+ domain policy, ignore the memory and follow the current task.
+</openviking_memory_scope_guard>
diff --git a/benchmark/tau2/config/scope_prompts/retail_memory_scope.md b/benchmark/tau2/config/scope_prompts/retail_memory_scope.md
deleted file mode 100644
index 65a8d61ff..000000000
--- a/benchmark/tau2/config/scope_prompts/retail_memory_scope.md
+++ /dev/null
@@ -1,16 +0,0 @@
-<openviking_memory_scope_guard>
-OpenViking memories are advisory. Use them only when their trigger, preconditions,
-and applicability boundary match the current retail task.
-
-- Do not broaden the user's requested replacement, return, exchange, cancellation,
- address-change, or payment scope because a retrieved memory describes a nearby
- workflow.
-- If the user restricts the request to the current order, same order, observed
- order items, or a specific product variant, choose write arguments only from the
- current tool observations or an explicitly requested catalog lookup.
-- Before a write tool call, order IDs, item IDs, new item IDs, payment method IDs,
- addresses, amounts, and refund/payment direction must be grounded in user input,
- recent tool observations, profile/order state, or an explicit catalog lookup.
-- If a memory and the current task disagree, follow the current task state and the
- domain policy.
-</openviking_memory_scope_guard>
diff --git a/benchmark/tau2/config/trajectory.yaml b/benchmark/tau2/config/trajectory.yaml
index 40c378542..2538d2639 100644
--- a/benchmark/tau2/config/trajectory.yaml
+++ b/benchmark/tau2/config/trajectory.yaml
@@ -20,9 +20,7 @@ strategies:
train_memory_mode: experience_only
search_memory_type: trajectories
retrieval_mode: first_user
- scope_prompt_files:
- retail: benchmark/tau2/config/scope_prompts/retail_memory_scope.md
- airline: benchmark/tau2/config/scope_prompts/airline_memory_scope.md
+ scope_prompt_file: benchmark/tau2/config/scope_prompts/generic_memory_scope.md
- id: memory_v2_trajectory_prewrite_only
label: OpenViking Memory V2 trajectory pre-write-only recall
memory_backend: openviking
@@ -47,9 +45,7 @@ strategies:
train_memory_mode: experience_only
search_memory_type: trajectories
retrieval_mode: prewrite
- scope_prompt_files:
- retail: benchmark/tau2/config/scope_prompts/retail_memory_scope.md
- airline: benchmark/tau2/config/scope_prompts/airline_memory_scope.md
+ scope_prompt_file: benchmark/tau2/config/scope_prompts/generic_memory_scope.md
- id: memory_v2_trajectory_prewrite_scope
label: OpenViking Memory V2 trajectory first-user + pre-write recall with scope prompt
memory_backend: openviking
@@ -58,6 +54,4 @@ strategies:
train_memory_mode: experience_only
search_memory_type: trajectories
retrieval_mode: first_user_prewrite
- scope_prompt_files:
- retail: benchmark/tau2/config/scope_prompts/retail_memory_scope.md
- airline: benchmark/tau2/config/scope_prompts/airline_memory_scope.md
+ scope_prompt_file: benchmark/tau2/config/scope_prompts/generic_memory_scope.md
diff --git a/benchmark/tau2/scripts/run_eval.py b/benchmark/tau2/scripts/run_eval.py
index ea747045a..4965eb465 100755
--- a/benchmark/tau2/scripts/run_eval.py
+++ b/benchmark/tau2/scripts/run_eval.py
@@ -445,6 +445,8 @@ def _tau2_command(
]
if fixed_first_user_file is not None:
command.extend(["--fixed-first-user-file", str(fixed_first_user_file)])
+ if scope_prompt_file is not None:
+ command.extend(["--scope-prompt-file", str(scope_prompt_file)])
if task_ids:
for task_id in task_ids: