Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
devops(test-results-db): scan artifacts run by run (#43187)
Dmitry Gozman4c50104bafbd
diff --git a/utils/test-results-db/cli.ts b/utils/test-results-db/cli.tsindex d06890cab4..0de09e560a 100644--- a/utils/test-results-db/cli.ts+++ b/utils/test-results-db/cli.ts@@ -31,10 +31,7 @@ Commands:Starts a fresh database if none exists yet.update [options] Ingest parquet artifacts that aren't in the database yet.--lookback-days <n> How many days back to scan (default 7).- --concurrency <n> Parallel downloads per batch (default 16).- --stop-after-seen <n> Stop after this many consecutive already-ingested- artifacts (default 100). The list is newest-first, so- this short-circuits the scan once caught up.+ --concurrency <n> Parallel downloads and listings per batch (default 16).truncate --max-runs <n> Keep only the newest <n> runs, delete the rest, compact.Environment:@@ -50,7 +47,6 @@ function defaultDbPath(): string {const UPDATE_OPTIONS = {'lookback-days': { type: 'string' },'concurrency': { type: 'string' },- 'stop-after-seen': { type: 'string' },} as const;const TRUNCATE_OPTIONS = {@@ -90,7 +86,6 @@ async function main(): Promise<void> {await cmdUpdate(dbPath, requireToken(), {lookbackDays: intFlag(values, 'lookback-days', 7),concurrency: intFlag(values, 'concurrency', 16),- stopAfterSeen: intFlag(values, 'stop-after-seen', 100),});break;}diff --git a/utils/test-results-db/db.ts b/utils/test-results-db/db.tsindex 0aabfa8131..6288685195 100644--- a/utils/test-results-db/db.ts+++ b/utils/test-results-db/db.ts@@ -80,6 +80,17 @@ export class TestResultsDb {await this._conn.run(`INSERT INTO ${TABLE_NAME} BY NAME ${INGEST_SELECT}`, params);}+ // When the previous ingest ran: the newest `ingested_at` stamp as epoch ms,+ // or 0 on a fresh db.+ async lastIngestedAt(): Promise<number> {+ if (!await this._tableExists())+ return 0;+ const reader = await this._conn.runAndReadAll(+ `SELECT epoch_ms(max(ingested_at)) FROM ${TABLE_NAME}`);+ const value = reader.getRows()[0][0];+ return value === null ? 0 : Number(value);+ }+async runCount(): Promise<number> {if (!await this._tableExists())return 0;diff --git a/utils/test-results-db/github.ts b/utils/test-results-db/github.tsindex 6299e29498..5417217a08 100644--- a/utils/test-results-db/github.ts+++ b/utils/test-results-db/github.ts@@ -23,10 +23,27 @@ export type Artifact = {name: string;};+export type ArtifactScan = {+ artifacts: Artifact[];+ // Runs created inside the lookback window.+ runCount: number;+ // Runs whose artifacts we actually listed.+ listedRunCount: number;+};+type ListOptions = {ingested: Set<string>;lookbackDays: number;- stopAfterSeen: number;+ // Completed runs last updated before this time (epoch ms) are not listed.+ rescanSince: number;+ concurrency: number;+};++type RawRun = {+ id: number;+ status: string;+ created_at: string;+ updated_at: string;};type RawArtifact = {@@ -36,8 +53,15 @@ type RawArtifact = {created_at: string;};-// Thin GitHub REST client over global fetch. Only the three endpoints this CLI-// needs: list artifacts, list by name, and download an artifact zip.+// A single runs query returns at most this many results, no matter how you+// page it. Larger windows have to be split into several queries.+const RUNS_QUERY_CAP = 1000;++const FETCH_ATTEMPTS = 3;++// Thin GitHub REST client over global fetch. Only the endpoints this CLI+// needs: list runs, list a run's artifacts, list artifacts by name, and+// download an artifact zip.export class GitHubClient {private _base: string;private _headers: Record<string, string>;@@ -53,56 +77,46 @@ export class GitHubClient {};}- // Return the not-yet-ingested artifacts matching `prefix`, walking the list- // from the top and stopping early once we're safely into the already-ingested- // region.+ // Return the not-yet-ingested artifacts matching `prefix` from the runs+ // created in the last `lookbackDays`.//- // The list is ordered by descending artifact id, and GitHub assigns ids as a- // monotonic creation-order sequence (verified: no inversions across a 1000-- // artifact sample). So the newest artifacts are always at the head. An artifact- // gets its id when its upload *starts* but only appears in the list once the- // upload *finalizes*, so the single way one can surface below where a prior- // scan stopped is a still-in-flight upload finalizing late -- a window bounded- // by one artifact's upload+list latency (seconds, for these KB parquet files).- // We therefore keep scanning `stopAfterSeen` artifacts past the newest- // already-ingested one as a cushion; any new (un-ingested) artifact resets the- // counter. To bury a late finalizer we'd need `stopAfterSeen` newer artifacts- // ingested above it while it uploads, i.e. an upload outlasting a whole cron- // interval -- impossible here, so this misses nothing in practice.+ // We walk run by run: list the recent runs, then each run's artifacts. The+ // repo-wide artifact listing (GET /actions/artifacts without a name filter)+ // returns 500 for large repositories, so it cannot be used here.//- // `lookbackDays` is the absolute backstop for the first run, when nothing is- // ingested yet and the cushion never triggers.- async listArtifacts(prefix: string, options: ListOptions): Promise<Artifact[]> {- const { ingested, lookbackDays, stopAfterSeen } = options;+ // Listing every run on each cron tick would be expensive, so a run is only+ // listed when it can still hold artifacts we have not seen. Artifacts are+ // uploaded by jobs, and a run's `updated_at` moves whenever a job finishes,+ // so a run that completed before `rescanSince` was already final the last+ // time we looked and everything it has is ingested. In-progress runs are+ // always listed.+ async listArtifacts(prefix: string, options: ListOptions): Promise<ArtifactScan> {+ const { ingested, lookbackDays, rescanSince, concurrency } = options;const cutoff = Date.now() - lookbackDays * 24 * 60 * 60 * 1000;- const out: Artifact[] = [];+ const runs = await this._listRecentRuns(cutoff);+ const toList = runs.filter(run => run.status !== 'completed' || Date.parse(run.updated_at) >= rescanSince);++ const artifacts: Artifact[] = [];const queued = new Set<string>();- let seen = 0;- for await (const artifact of this._paginateArtifacts('/actions/artifacts?per_page=100')) {- const createdAt = artifact.created_at ? Date.parse(artifact.created_at) : 0;- if (createdAt && createdAt < cutoff)- return out;- if (artifact.expired || !artifact.name.startsWith(prefix))- continue;- const id = String(artifact.id);- if (queued.has(id))- continue;- if (ingested.has(id)) {- if (++seen >= stopAfterSeen)- return out;- continue;+ for (const batch of chunk(toList, concurrency)) {+ const lists = await Promise.all(batch.map(run => this._listRunArtifacts(run.id)));+ for (const artifact of lists.flat()) {+ if (artifact.expired || !artifact.name.startsWith(prefix))+ continue;+ const id = String(artifact.id);+ if (ingested.has(id) || queued.has(id))+ continue;+ queued.add(id);+ artifacts.push({ id, name: artifact.name });}- seen = 0;- queued.add(id);- out.push({ id, name: artifact.name });}- return out;+ return { artifacts, runCount: runs.length, listedRunCount: toList.length };}// The newest non-expired artifact with the exact name, or null if none.async findLatestArtifact(name: string): Promise<string | null> {const query = `/actions/artifacts?name=${encodeURIComponent(name)}&per_page=100`;- for await (const artifact of this._paginateArtifacts(query)) {+ for await (const artifact of this._paginate<RawArtifact>(query, 'artifacts')) {if (!artifact.expired)return String(artifact.id);}@@ -112,24 +126,77 @@ export class GitHubClient {async downloadArtifactZip(id: string): Promise<Buffer> {// 302 -> blob storage; fetch follows it and strips the Authorization header// on the cross-origin redirect, as required by the signed URL.- const response = await fetch(`${this._base}/actions/artifacts/${id}/zip`, { headers: this._headers });+ const response = await this._fetch(`${this._base}/actions/artifacts/${id}/zip`);if (!response.ok)throw new Error(`Failed to download artifact ${id}: ${response.status} ${response.statusText}`);return Buffer.from(await response.arrayBuffer());}- private async * _paginateArtifacts(path: string): AsyncGenerator<RawArtifact> {+ // All runs created at or after `cutoff`, newest first. The `created` filter+ // takes an inclusive `from..to` range; when a range hits the result cap we+ // issue the next query ending at the oldest run seen so far.+ private async _listRecentRuns(cutoff: number): Promise<RawRun[]> {+ const runs: RawRun[] = [];+ const seen = new Set<number>();+ const from = isoSeconds(cutoff);+ let to = isoSeconds(Date.now() + 60 * 60 * 1000);+ while (true) {+ const query = `/actions/runs?created=${from}..${to}&per_page=100`;+ let count = 0;+ let oldest = to;+ for await (const run of this._paginate<RawRun>(query, 'workflow_runs')) {+ count++;+ oldest = run.created_at;+ if (seen.has(run.id))+ continue;+ seen.add(run.id);+ runs.push(run);+ }+ if (count < RUNS_QUERY_CAP || oldest === to)+ return runs;+ to = oldest;+ }+ }++ private async _listRunArtifacts(runId: number): Promise<RawArtifact[]> {+ const artifacts: RawArtifact[] = [];+ for await (const artifact of this._paginate<RawArtifact>(`/actions/runs/${runId}/artifacts?per_page=100`, 'artifacts'))+ artifacts.push(artifact);+ return artifacts;+ }++ private async * _paginate<T>(path: string, key: string): AsyncGenerator<T> {let url: string | null = `${this._base}${path}`;while (url) {- const response = await fetch(url, { headers: this._headers });+ const response = await this._fetch(url);if (!response.ok)throw new Error(`GitHub API error: ${response.status} ${response.statusText} for ${url}`);- const body = await response.json() as { artifacts?: RawArtifact[] };- for (const artifact of body.artifacts ?? [])- yield artifact;+ const body = await response.json() as Record<string, T[] | undefined>;+ for (const item of body[key] ?? [])+ yield item;url = nextPageUrl(response.headers.get('link'));}}++ // GET with a few retries on network errors and 5xx/429 responses.+ private async _fetch(url: string): Promise<Response> {+ for (let attempt = 1; ; attempt++) {+ try {+ const response = await fetch(url, { headers: this._headers });+ if (attempt >= FETCH_ATTEMPTS || (response.status < 500 && response.status !== 429))+ return response;+ } catch (error) {+ if (attempt >= FETCH_ATTEMPTS)+ throw error;+ }+ await new Promise(resolve => setTimeout(resolve, 1000 * attempt));+ }+ }+}++// ISO 8601 without milliseconds, the form GitHub's `created` filter accepts.+function isoSeconds(epochMs: number): string {+ return new Date(epochMs).toISOString().replace(/\.\d{3}Z$/, 'Z');}// Parse the `rel="next"` target out of a GitHub Link header, or null if absent.diff --git a/utils/test-results-db/update.ts b/utils/test-results-db/update.tsindex 52e732633b..73bd511217 100644--- a/utils/test-results-db/update.ts+++ b/utils/test-results-db/update.ts@@ -23,10 +23,14 @@ import { GitHubClient, chunk, extractSingle } from './github.ts';const PARQUET_ARTIFACT_PREFIX = 'parquet-report-';+// Completed runs are listed again only if they were updated after the previous+// ingest. This margin covers the time between that ingest listing the runs and+// stamping its rows, plus clock skew between GitHub and the runner.+const RESCAN_MARGIN_MS = 6 * 60 * 60 * 1000;+export type UpdateOptions = {lookbackDays: number;concurrency: number;- stopAfterSeen: number;};// Ingest the parquet artifacts that aren't in the database yet. Downloads run@@ -41,12 +45,16 @@ export async function cmdUpdate(dbPath: string, token: string, options: UpdateOpconsole.log(`Test results database`);console.log(` ${await db.rowCount()} rows from ${ingested.size} artifacts`);- const todo = await github.listArtifacts(PARQUET_ARTIFACT_PREFIX, {+ const lastIngestedAt = await db.lastIngestedAt();+ const scan = await github.listArtifacts(PARQUET_ARTIFACT_PREFIX, {ingested,lookbackDays: options.lookbackDays,- stopAfterSeen: options.stopAfterSeen,+ rescanSince: lastIngestedAt - RESCAN_MARGIN_MS,+ concurrency: options.concurrency,});+ const todo = scan.artifacts;console.log(`\nScanning for new artifacts (last ${options.lookbackDays} days)`);+ console.log(` ${scan.runCount} runs in window, ${scan.listedRunCount} listed`);console.log(` ${todo.length} new artifacts to import`);let imported = 0;