Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

E2E: print which scenario each request tag belongs to

Petr Bauch5e1244f467a2
diff --git a/console/tooling-e2e/src/agentic/agentic.e2e.test.ts b/console/tooling-e2e/src/agentic/agentic.e2e.test.ts
index 10af75ca0..b1806e6ce 100644
--- a/console/tooling-e2e/src/agentic/agentic.e2e.test.ts
+++ b/console/tooling-e2e/src/agentic/agentic.e2e.test.ts
@@ -187,7 +187,7 @@ describe('agentic lane', () => {
ledger.assertWithin(`${scenario.header.id} started`);
const names = scopes.scope();
- const observer = watch.begin(h.switch, names.tag);
+ const observer = watch.begin(h.switch, names.tag, scenario.header.id);
const turns: Turn[] = [];
let facts: FactResult[] = [];
let failure: unknown = null;
diff --git a/console/tooling-e2e/src/browser/browser.e2e.test.ts b/console/tooling-e2e/src/browser/browser.e2e.test.ts
index a263a2861..c5031befe 100644
--- a/console/tooling-e2e/src/browser/browser.e2e.test.ts
+++ b/console/tooling-e2e/src/browser/browser.e2e.test.ts
@@ -83,7 +83,7 @@ describe('browser lane', () => {
env.switch,
new IssuedSecrets(env.secretsFile)
);
- const observer = watch.begin(switchClient, names.tag);
+ const observer = watch.begin(switchClient, names.tag, scenario.header.id);
let failure: unknown = null;
let problems: string[];
let leftovers: string[];
diff --git a/console/tooling-e2e/src/core/server-log.test.ts b/console/tooling-e2e/src/core/server-log.test.ts
index b8215b0b1..1a08c6a7e 100644
--- a/console/tooling-e2e/src/core/server-log.test.ts
+++ b/console/tooling-e2e/src/core/server-log.test.ts
@@ -100,7 +100,7 @@ describe('server log watch', () => {
const file = logFile([record('info', 'booted')]);
const watch = new ServerLogWatch(file);
append(file, [record('error', 'during setup')]);
- const observer = watch.begin(tagger, 'e2e-abc123b');
+ const observer = watch.begin(tagger, 'e2e-abc123b', 'some-scenario');
append(file, [record('error', 'in the scenario', 'e2e-abc123b-1')]);
observer.stop();
expect(observer.errors(null, () => {})?.map((error) => error.message)).toEqual([
@@ -116,7 +116,7 @@ describe('server log watch', () => {
it("never lets an unreadable log replace the scenario's own failure", () => {
const file = logFile([record('info', 'booted'), record('info', 'more')]);
const watch = new ServerLogWatch(file);
- const observer = watch.begin(tagger, 'e2e-abc123b');
+ const observer = watch.begin(tagger, 'e2e-abc123b', 'some-scenario');
truncateSync(file, 10);
const said: string[] = [];
expect(observer.errors(new Error('the scenario failed'), (line) => said.push(line))).toBe(null);
@@ -126,7 +126,7 @@ describe('server log watch', () => {
it('is off, reading nothing, without a log', async () => {
const watch = new ServerLogWatch(null);
- expect(watch.begin(tagger, 'e2e-abc123b').errors(null, () => {})).toBe(null);
+ expect(watch.begin(tagger, 'e2e-abc123b', 'some-scenario').errors(null, () => {})).toBe(null);
expect(await watch.sweep()).toBe(null);
});
});
diff --git a/console/tooling-e2e/src/core/server-log.ts b/console/tooling-e2e/src/core/server-log.ts
index 333317241..99754fadf 100644
--- a/console/tooling-e2e/src/core/server-log.ts
+++ b/console/tooling-e2e/src/core/server-log.ts
@@ -209,8 +209,13 @@ export class ServerLogWatch {
* Start watching a scenario: what was logged since the last read is set
* aside as outside any scenario, and the client tags its requests with
* `tag` until the observer stops.
+ *
+ * Which scenario a tag belongs to is printed, whether or not this process
+ * reads the server's log: a deployment whose log is read elsewhere (after
+ * the run, from its log system) names a scenario by this line.
*/
- begin(client: Taggable, tag: string): ScenarioObserver {
+ begin(client: Taggable, tag: string, scenario: string): ScenarioObserver {
+ console.log(`[${scenario}] requests tagged ${tag}-<n>`);
if (this.cursor !== null) this.outside.push(...this.cursor.take());
client.tagRequests(tag);
return new ScenarioObserver(this.cursor, client, tag);
diff --git a/console/tooling-e2e/src/fast/fast.e2e.test.ts b/console/tooling-e2e/src/fast/fast.e2e.test.ts
index c3d898e3b..867ca41ce 100644
--- a/console/tooling-e2e/src/fast/fast.e2e.test.ts
+++ b/console/tooling-e2e/src/fast/fast.e2e.test.ts
@@ -57,7 +57,7 @@ describe('fast lane', () => {
const names = run.scope();
const fixtures = new Fixtures(env, names, switchClient, mattermost);
const log = (message: string): void => console.log(`[${scenario.header.id}] ${message}`);
- const observer = watch.begin(switchClient, names.tag);
+ const observer = watch.begin(switchClient, names.tag, scenario.header.id);
let failure: unknown = null;
let leftovers: string[];
try {
@@ -89,7 +89,7 @@ describe('fast lane, derived checks', () => {
if (!lane.target) return context.skip(lane.skip);
if (lane.target.env.environment === 'prod') return context.skip('sends writes; not on prod');
const { switchClient } = lane.target;
- const observer = watch.begin(switchClient, run.scope().tag);
+ const observer = watch.begin(switchClient, run.scope().tag, 'session-required');
let probe: Awaited<ReturnType<typeof probeSessionRoutes>>;
try {
probe = await probeSessionRoutes(switchClient, readSurface());
@@ -120,7 +120,7 @@ describe('fast lane, derived checks', () => {
const names = run.scope();
const fixtures = new Fixtures(env, names, switchClient, mattermost);
const log = (message: string): void => console.log(`[cross-tenant] ${message}`);
- const observer = watch.begin(switchClient, names.tag);
+ const observer = watch.begin(switchClient, names.tag, 'cross-tenant');
let failure: unknown = null;
let probe: CrossTenantProbe | null = null;
let leftovers: string[];
diff --git a/console/tooling-e2e/src/provider/provider.e2e.test.ts b/console/tooling-e2e/src/provider/provider.e2e.test.ts
index cb8d4ee20..8e22bb8f7 100644
--- a/console/tooling-e2e/src/provider/provider.e2e.test.ts
+++ b/console/tooling-e2e/src/provider/provider.e2e.test.ts
@@ -97,7 +97,7 @@ describe('provider lane', () => {
}
const h = harness!;
const log = (message: string): void => console.log(`[${scenario.header.id}] ${message}`);
- const observer = watch.begin(h.switch, scopes.scope().tag);
+ const observer = watch.begin(h.switch, scopes.scope().tag, scenario.header.id);
let result: ScenarioResult;
try {
result = await scenario.run(h);