Git designed for AI agents.
#!/usr/bin/env python3
"""
provision-repos - idempotently provision benchmark repositories in a
chart-installed GitLab and write their hashed disk paths back into the
experiment's config.yml.
The in-cluster flow has no filesystem access to storage nodes, so repos
are created and imported via Rails APIs rather than cloned onto disk.
For A/B comparisons a second, identical project set (BENCH_CONTROL_GROUP)
is provisioned onto the control Gitaly's Rails storage
(CONTROL_STORAGE_NAME) through the same pipeline, so both sides start
from identically constructed state. Skipped when the storage isn't
registered in Rails.
Each set is made public, swept of stray workload refs, fetched when the
pinned `revision` is missing, pinned on its default branch, and given
the `additional_refs` zzz refs through Gitaly's write path. Re-runs are
safe: existing projects are reused.
Usage:
EXPERIMENT=incluster-master ./incluster/provision-repos
EXPERIMENT=incluster-master ./incluster/provision-repos --write-config
The first form prints the YAML snippet for review; the second edits
config.yml in place.
"""
from __future__ import annotations
import argparse
import base64
import json
import os
import shlex
import subprocess
import sys
import tempfile
import time
from pathlib import Path
# ruamel.yaml roundtrips comments and key order; fall back to PyYAML
# (already required elsewhere) - still valid YAML, just reformatted.
try:
from ruamel.yaml import YAML # type: ignore
_yaml = YAML()
_yaml.preserve_quotes = True
_yaml.indent(mapping=2, sequence=2, offset=0)
def _load(p: Path):
with p.open() as f:
return _yaml.load(f)
def _dump(data, p: Path) -> None:
with p.open("w") as f:
_yaml.dump(data, f)
except ImportError:
import yaml # type: ignore
def _load(p: Path):
with p.open() as f:
return yaml.safe_load(f)
def _dump(data, p: Path) -> None:
with p.open("w") as f:
yaml.safe_dump(data, f, sort_keys=False)
SCRIPT_DIR = Path(__file__).resolve().parent
BENCH_DIR = SCRIPT_DIR.parent
# Group namespacing the benchmark projects.
BENCH_GROUP = os.environ.get("BENCH_GROUP", "bench")
# Control-side project set for A/B comparisons (see module docstring).
BENCH_CONTROL_GROUP = os.environ.get("BENCH_CONTROL_GROUP", f"{BENCH_GROUP}-control")
CONTROL_STORAGE_NAME = os.environ.get("CONTROL_STORAGE_NAME", "benchmark-control")
# Namespace where the GitLab Helm chart lives; matches run-benchmarks-incluster.
GITLAB_NAMESPACE = os.environ.get("GITLAB_NAMESPACE", "gitlab")
# The chart ships gitlab-toolbox with app=toolbox.
TOOLBOX_SELECTOR = os.environ.get("TOOLBOX_SELECTOR", "app=toolbox")
# Sidekiq import poll loop.
POLL_INTERVAL_SECONDS = int(os.environ.get("PROVISION_POLL_INTERVAL", "15"))
POLL_TIMEOUT_SECONDS = int(os.environ.get("PROVISION_POLL_TIMEOUT", "1800"))
def die(msg: str) -> None:
print(f"error: {msg}", file=sys.stderr)
sys.exit(1)
def _kubectl_cmd() -> list[str]:
cmd = ["kubectl"]
if kc := os.environ.get("KUBECONFIG"):
cmd += [f"--kubeconfig={kc}"]
if ctx := os.environ.get("KUBE_CONTEXT"):
cmd += [f"--context={ctx}"]
return cmd
def find_toolbox_pod() -> str:
out = subprocess.run(
_kubectl_cmd()
+ [
"-n",
GITLAB_NAMESPACE,
"get",
"pods",
"-l",
TOOLBOX_SELECTOR,
"-o",
"jsonpath={.items[0].metadata.name}",
],
capture_output=True,
text=True,
check=False,
)
pod = out.stdout.strip()
if not pod:
die(
f"no toolbox pod found in namespace {GITLAB_NAMESPACE!r} "
f"matching selector {TOOLBOX_SELECTOR!r}.\n"
f" stderr: {out.stderr.strip()}\n"
" Override the lookup with GITLAB_NAMESPACE / TOOLBOX_SELECTOR."
)
return pod
def rails_runner(pod: str, script: str, env: dict[str, str] | None = None) -> str:
"""Execute a Ruby snippet via `gitlab-rails runner -` inside the toolbox pod.
The script goes over stdin (the `-` arg) to avoid quoting a multi-line
Ruby program through `kubectl exec`; parameters ride in an `env KEY=VALUE`
prefix because kubectl exec has no --env flag. Returns stdout; aborts on
non-zero exit.
"""
extra_env = env or {}
env_prefix: list[str] = []
if extra_env:
env_prefix = ["env"] + [f"{k}={v}" for k, v in extra_env.items()]
cmd = _kubectl_cmd() + [
"-n",
GITLAB_NAMESPACE,
"exec",
"-i",
pod,
"--",
*env_prefix,
"gitlab-rails",
"runner",
"-",
]
proc = subprocess.run(
cmd, input=script, capture_output=True, text=True, check=False
)
if proc.returncode != 0:
die(
"gitlab-rails runner failed:\n"
f" exit_code={proc.returncode}\n"
f" command: {shlex.join(cmd)}\n"
f" stdout:\n{proc.stdout}\n"
f" stderr:\n{proc.stderr}"
)
return proc.stdout
# Creates (or finds) projects under the group, printing one JSON object per
# line. Parameters arrive via ENV so the Ruby source needs no escaping.
# Groups::CreateService returns a ServiceResponse (payload[:group]) on newer
# GitLab but the Group directly on older versions; probe via respond_to?.
_PROVISION_SCRIPT = r"""
require 'json'
group_path = ENV.fetch('PROVISION_GROUP')
specs = JSON.parse(ENV.fetch('PROVISION_SPECS'))
user = User.find_by_username('root') || User.admins.first
unless user
STDERR.puts 'no admin user available; cannot create projects'
exit 1
end
# Public so the benchmark Job can clone anonymously over http.
# update_column skips instance-level visibility validations on purpose.
public_level = Gitlab::VisibilityLevel::PUBLIC
ns = Namespace.find_by_full_path(group_path)
unless ns
# GitLab 16.11+ requires groups/projects to belong to an Organization;
# attach the default org when present. Older versions ignore the arg.
org = nil
if defined?(::Organizations::Organization)
org = ::Organizations::Organization.find_by(path: 'default') ||
::Organizations::Organization.first
end
attrs = {
name: group_path, path: group_path, visibility_level: public_level
}
attrs[:organization_id] = org.id if org
result = Groups::CreateService.new(user, attrs).execute
ns = if result.respond_to?(:payload)
result.payload[:group] || result.payload['group']
else
result
end
if ns.nil? || (ns.respond_to?(:persisted?) && !ns.persisted?)
err = ns.respond_to?(:errors) ? ns.errors.full_messages.join(', ') : result.inspect
STDERR.puts "FAIL group #{group_path}: #{err}"
exit 1
end
end
ns.update_column(:visibility_level, public_level) if ns.visibility_level != public_level
specs.each do |spec|
name = spec.fetch('name')
url = spec.fetch('remote')
full = "#{group_path}/#{name}"
storage = ENV['PROVISION_STORAGE'].to_s
project = Project.find_by_full_path(full)
if !project.nil? && !storage.empty? && project.repository_storage != storage
puts JSON.generate(name: name,
error: "exists on storage #{project.repository_storage}, want #{storage}; re-run with --reset")
next
end
if project.nil?
project_attrs = {
name: name, path: name, namespace_id: ns.id,
visibility_level: public_level, import_url: url
}
project_attrs[:repository_storage] = storage unless storage.empty?
project_attrs[:organization_id] = ns.organization_id if ns.respond_to?(:organization_id) && ns.organization_id
project = ::Projects::CreateService.new(user, project_attrs).execute
unless project.respond_to?(:persisted?) && project.persisted?
err = project.respond_to?(:errors) ? project.errors.full_messages.join(', ') : project.inspect
puts JSON.generate(name: name, error: err)
next
end
end
if project.visibility_level != public_level
project.update_column(:visibility_level, public_level)
end
puts JSON.generate(
name: name,
full_path: project.full_path,
disk_path: project.disk_path,
gl_repository: "project-#{project.id}",
repo_storage: project.repository_storage,
import_status: project.import_status
)
end
""".strip()
# Reset destroys the whole project instead of re-importing in place: an
# MVCC repo recreated at its old hashed path reads a stale manifest and
# is permanently broken. Callers re-resolve disk_path via --write-config.
_RESET_SCRIPT = r"""
require 'json'
group_path = ENV.fetch('PROVISION_GROUP')
specs = JSON.parse(ENV.fetch('PROVISION_SPECS'))
user = User.find_by_username('root') || User.admins.first
unless user
STDERR.puts 'no admin user available; cannot destroy projects'
exit 1
end
specs.each do |spec|
name = spec.fetch('name')
project = Project.find_by_full_path("#{group_path}/#{name}")
if project.nil?
puts JSON.generate(name: name, reset: true, skipped: true, reason: 'project not found')
next
end
::Projects::DestroyService.new(project, user).execute
# Guard against the project row lingering (adjourned deletion etc.),
# which would make the recreate fail with "path already taken".
20.times do
break if Project.find_by_full_path("#{group_path}/#{name}").nil?
sleep 3
end
if Project.find_by_full_path("#{group_path}/#{name}")
puts JSON.generate(name: name, error: 'project still present after destroy')
next
end
puts JSON.generate(name: name, reset: true)
end
""".strip()
_STATUS_SCRIPT = r"""
require 'json'
group_path = ENV.fetch('PROVISION_GROUP')
names = JSON.parse(ENV.fetch('PROVISION_NAMES'))
names.each do |name|
p = Project.find_by_full_path("#{group_path}/#{name}")
if p.nil?
puts JSON.generate(name: name, missing: true)
next
end
# -1 means unreadable; lets the caller tell a failed import from Rails
# post-import bookkeeping failures.
branches = begin
p.repository.raw_repository.branch_count
rescue StandardError
-1
end
puts JSON.generate(
name: name,
import_status: p.import_status,
import_error: p.import_state&.last_error,
repo_exists: p.repository.exists?,
branches: branches
)
end
""".strip()
# Imports never fetch again, so a `revision` bumped past the import date
# can't resolve; fetch heads+tags, forced, never pruned (local-only zzz
# refs must survive), via FetchRemote. No-op when the pin resolves.
_FETCH_MISSING_REVISIONS_SCRIPT = r"""
require 'json'
group_path = ENV.fetch('PROVISION_GROUP')
specs = JSON.parse(ENV.fetch('PROVISION_SPECS'))
refmap = ['+refs/heads/*:refs/heads/*', '+refs/tags/*:refs/tags/*']
specs.each do |spec|
name = spec.fetch('name')
rev = spec['revision'].to_s
next if rev.empty?
project = Project.find_by_full_path("#{group_path}/#{name}")
if project.nil?
puts JSON.generate(name: name, error: 'project not found')
next
end
if project.repository.commit(rev)
puts JSON.generate(name: name, fetched: false)
next
end
url = spec['remote'].to_s
if url.empty?
puts JSON.generate(name: name,
error: "revision #{rev} missing and no remote to fetch from")
next
end
raw = project.repository.raw_repository
begin
begin
raw.fetch_remote(url, refmap: refmap, forced: true, prune: false)
rescue ArgumentError
# Older GitLab: fetch_remote without a prune kwarg (defaults safe here
# because the refmap only covers heads/tags, not refs/refs/zzz-*).
raw.fetch_remote(url, refmap: refmap, forced: true)
end
rescue StandardError => e
puts JSON.generate(name: name,
error: "fetch from #{url} failed: #{e.class}: #{e.message}")
next
end
begin
project.repository.expire_branches_cache
rescue StandardError
end
if project.repository.commit(rev)
puts JSON.generate(name: name, fetched: true)
else
puts JSON.generate(name: name,
error: "revision #{rev} still not found after fetching #{url} (remote rewritten?)")
end
end
""".strip()
# Pin the default branch to config.yml's `revision` (via WriteRef) so
# both sides serve identical content. No-op when already pinned.
_ENFORCE_REVISION_SCRIPT = r"""
require 'json'
group_path = ENV.fetch('PROVISION_GROUP')
specs = JSON.parse(ENV.fetch('PROVISION_SPECS'))
specs.each do |spec|
name = spec.fetch('name')
rev = spec['revision'].to_s
next if rev.empty?
project = Project.find_by_full_path("#{group_path}/#{name}")
if project.nil?
puts JSON.generate(name: name, error: 'project not found')
next
end
commit = project.repository.commit(rev)
if commit.nil?
puts JSON.generate(name: name, error: "revision #{rev} not found (remote rewritten?)")
next
end
branch = project.default_branch || 'master'
ref = "refs/heads/#{branch}"
current = project.repository.commit(branch)&.sha
if current == commit.sha
puts JSON.generate(name: name, ref: ref, revision: commit.sha, changed: false)
else
project.repository.raw_repository.write_ref(ref, commit.sha)
begin
project.repository.expire_branches_cache
rescue StandardError
end
puts JSON.generate(name: name, ref: ref, revision: commit.sha, changed: true, previous: current)
end
end
""".strip()
# Sweep refs an aborted k6 run leaves behind (refs/test/*, benchmark-*
# branches) before pinning, via ListRefs/DeleteRefs.
_SWEEP_REFS_SCRIPT = r"""
require 'json'
group_path = ENV.fetch('PROVISION_GROUP')
names = JSON.parse(ENV.fetch('PROVISION_NAMES'))
STRAY_PREFIXES = ['refs/test/', 'refs/heads/benchmark-'].freeze
names.each do |name|
project = Project.find_by_full_path("#{group_path}/#{name}")
if project.nil?
puts JSON.generate(name: name, error: 'project not found')
next
end
raw = project.repository.raw_repository
unless raw.respond_to?(:list_refs)
puts JSON.generate(name: name, swept: 0,
warning: 'list_refs unavailable on this GitLab; sweep skipped')
next
end
# List broadly, filter client-side: for-each-ref prefixes only match
# at '/' boundaries, so 'refs/heads/benchmark-' can't be a pattern.
stray = raw.list_refs(['refs/test/', 'refs/heads/'],
dynamic_timeout: Gitlab::GitalyClient.long_timeout)
.map(&:name)
.select { |n| STRAY_PREFIXES.any? { |p| n.start_with?(p) } }
if stray.empty?
puts JSON.generate(name: name, swept: 0)
next
end
stray.each_slice(1000) { |batch| raw.delete_refs(*batch) }
begin
project.repository.expire_branches_cache
rescue StandardError
end
puts JSON.generate(name: name, swept: stray.size)
end
""".strip()
# Create the refs/refs/zzz-N refs on the pinned revision via bulk
# UpdateReferences. The stream is built by hand because raw.update_refs
# sets `repository` on every message, which Gitaly rejects once a call
# spans multiple 100-ref slices.
_ADDITIONAL_REFS_SCRIPT = r"""
require 'json'
group_path = ENV.fetch('PROVISION_GROUP')
specs = JSON.parse(ENV.fetch('PROVISION_SPECS'))
batch_size = Integer(ENV.fetch('PROVISION_REF_BATCH', '50000'))
bulk_update_refs = lambda do |raw, updates|
requests = Enumerator.new do |y|
updates.each_slice(100).with_index do |slice, i|
msgs = slice.map do |u|
Gitaly::UpdateReferencesRequest::Update.new(
old_object_id: u[:old_sha],
new_object_id: u[:new_sha],
reference: u[:reference]
)
end
args = { updates: msgs }
args[:repository] = raw.gitaly_repository if i.zero?
y.yield Gitaly::UpdateReferencesRequest.new(**args)
end
end
Gitlab::GitalyClient.call(raw.storage, :ref_service, :update_references,
requests, timeout: Gitlab::GitalyClient.long_timeout)
end
bulk_delete_refs = lambda do |raw, refs|
response = Gitlab::GitalyClient.call(
raw.storage, :ref_service, :delete_refs,
Gitaly::DeleteRefsRequest.new(repository: raw.gitaly_repository, refs: refs),
timeout: Gitlab::GitalyClient.long_timeout
)
raise "DeleteRefs: #{response.git_error}" unless response.git_error.to_s.empty?
end
specs.each do |spec|
name = spec.fetch('name')
count = spec.fetch('additional_refs').to_i
next if count <= 0
project = Project.find_by_full_path("#{group_path}/#{name}")
if project.nil?
puts JSON.generate(name: name, error: 'project not found')
next
end
# Raw (uncached) RPCs throughout: Rails' repository.ref_exists? can
# serve a stale cache entry from a destroyed-and-recreated project.
raw = project.repository.raw_repository
# Skip only when the exact expected set exists. Probing sentinel refs
# (zzz-1, zzz-#{count}) instead would miss an interrupted sweep (deletion
# goes in lexicographic order, so both sentinels fall in the first batch)
# and a changed additional_refs count, stranding stale refs.
# use longer timeout so 6-figure ref deletion can still work
existing = raw.list_refs(['refs/refs/'],
dynamic_timeout: Gitlab::GitalyClient.long_timeout)
.map(&:name)
.select { |n| n.start_with?('refs/refs/zzz-') }
expected = (1..count).map { |i| "refs/refs/zzz-#{i}" }
if existing.sort == expected.sort
puts JSON.generate(name: name, refs: count, skipped: true)
next
end
# A partial set can't be resumed (creations assert the ref doesn't exist
# yet, old_object_id all-zeroes); sweep leftovers first.
unless existing.empty?
existing.each_slice(1000) { |b| bulk_delete_refs.call(raw, b) }
STDERR.puts "#{name}: swept #{existing.size} stale zzz refs"
end
anchor = spec['revision'].to_s
anchor = project.default_branch || 'master' if anchor.empty?
commit = project.repository.commit(anchor)
if commit.nil?
puts JSON.generate(name: name, error: "anchor #{anchor} not found")
next
end
sha = commit.sha
blank = '0' * sha.length
(1..count).each_slice(batch_size) do |batch|
bulk_update_refs.call(raw, batch.map { |i|
{ old_sha: blank, new_sha: sha, reference: "refs/refs/zzz-#{i}" }
})
STDERR.puts "#{name}: refs/refs/zzz-#{batch.last} of #{count}"
end
unless raw.ref_exists?("refs/refs/zzz-#{count}")
puts JSON.generate(name: name,
error: "refs/refs/zzz-#{count} still missing after creation; the write path is dropping refs")
next
end
puts JSON.generate(name: name, refs: count, anchor: sha)
rescue StandardError => e
# One slow repo must not abort the whole runner
puts JSON.generate(name: name, error: "#{e.class}: #{e.message}")
end
""".strip()
# Mint the PAT the http push transport authenticates with. Revoke and
# recreate each run: an existing token's cleartext can't be read back.
_PAT_SCRIPT = r"""
require 'json'
user = User.admins.active.first
if user.nil?
puts JSON.generate(error: 'no admin user available; cannot create PAT')
exit
end
name = 'gitaly-benchmark'
user.personal_access_tokens.active.where(name: name).find_each(&:revoke!)
token = user.personal_access_tokens.create!(
name: name,
scopes: ['api', 'write_repository'],
expires_at: 30.days.from_now
)
puts JSON.generate(token: token.token, user: user.username,
expires_at: token.expires_at)
""".strip()
# Register the public half of the ssh transport's keypair on the admin user
# so gitlab-shell can authenticate the benchmark client.
_SSH_KEY_SCRIPT = r"""
require 'json'
user = User.admins.active.first
if user.nil?
puts JSON.generate(error: 'no admin user available; cannot add an SSH key')
exit
end
title = 'gitaly-benchmark'
user.keys.where(title: title).find_each(&:destroy)
key = user.keys.new(title: title, key: ENV.fetch('BENCH_SSH_PUBKEY'))
if key.save
puts JSON.generate(user: user.username, fingerprint: key.fingerprint_sha256)
else
puts JSON.generate(error: key.errors.full_messages.join('; '))
end
""".strip()
def rails_storages(pod: str) -> list[str]:
"""Names of the Gitaly storages registered in Rails."""
out = rails_runner(
pod,
"require 'json'; puts JSON.generate(Gitlab.config.repositories.storages.keys)",
)
for line in out.splitlines():
line = line.strip()
if line.startswith("["):
return json.loads(line)
return []
def provision(pod: str, group: str, storage: str, specs: list[dict]) -> dict[str, dict]:
"""Create projects (idempotently) and return per-name metadata."""
out = rails_runner(
pod,
_PROVISION_SCRIPT,
env={
"PROVISION_GROUP": group,
"PROVISION_STORAGE": storage,
"PROVISION_SPECS": json.dumps(specs),
},
)
results: dict[str, dict] = {}
for line in out.splitlines():
line = line.strip()
if not line.startswith("{"):
# Rails prints deprecation warnings to stdout; skip non-JSON lines.
continue
rec = json.loads(line)
results[rec["name"]] = rec
# A project silently created on the wrong storage would make the
# comparison measure the wrong Gitaly; fail instead.
for rec in results.values():
if storage and not rec.get("error") and rec.get("repo_storage") != storage:
rec["error"] = (
f"created on storage {rec.get('repo_storage')!r}, wanted "
f"{storage!r} (does Projects::CreateService honor "
"repository_storage on this GitLab version?)"
)
return results
def reset_repositories(pod: str, group: str, specs: list[dict]) -> None:
"""Destroy the bench projects so provisioning recreates them fresh."""
out = rails_runner(
pod,
_RESET_SCRIPT,
env={
"PROVISION_GROUP": group,
"PROVISION_SPECS": json.dumps(specs),
},
)
reset_names = set()
errors = []
for line in out.splitlines():
line = line.strip()
if not line.startswith("{"):
continue
rec = json.loads(line)
if rec.get("error"):
errors.append(f"{rec['name']}: {rec['error']}")
continue
if rec.get("reset"):
reset_names.add(rec["name"])
state = "already absent" if rec.get("skipped") else "destroyed"
print(f" {group}/{rec['name']}: {state}", file=sys.stderr)
missing = [
s["name"]
for s in specs
if s["name"] not in reset_names
and not any(e.startswith(f"{s['name']}:") for e in errors)
]
if errors or missing:
die(
"repository reset failed:\n "
+ "\n ".join(errors + [f"{n}: no reset confirmation" for n in missing])
)
def wait_for_imports(pod: str, group: str, names: list[str]) -> dict[str, dict]:
"""Poll Sidekiq import status until every project is finished/failed."""
deadline = time.monotonic() + POLL_TIMEOUT_SECONDS
last: dict[str, dict] = {}
while True:
out = rails_runner(
pod,
_STATUS_SCRIPT,
env={
"PROVISION_GROUP": group,
"PROVISION_NAMES": json.dumps(names),
},
)
last = {}
for line in out.splitlines():
line = line.strip()
if not line.startswith("{"):
continue
rec = json.loads(line)
last[rec["name"]] = rec
pending = [
n
for n in names
if last.get(n, {}).get("import_status")
not in (None, "finished", "failed", "none")
]
for n in names:
r = last.get(n, {})
print(
f" {group}/{n}: import_status="
f"{r.get('import_status', '?')} "
f"repo_exists={r.get('repo_exists', '?')}",
file=sys.stderr,
)
if not pending:
return last
if time.monotonic() > deadline:
die(
f"timed out after {POLL_TIMEOUT_SECONDS}s waiting for imports: "
f"{pending}"
)
time.sleep(POLL_INTERVAL_SECONDS)
def fetch_missing_revisions(pod: str, group: str, specs: list[dict]) -> None:
"""Fetch a repo's remote when its pinned revision is absent locally.
Repos whose pin already resolves are untouched, so bumping `revision:`
in config.yml self-heals after the one-time import goes stale.
"""
pinned = [s for s in specs if s.get("revision")]
if not pinned:
return
out = rails_runner(
pod,
_FETCH_MISSING_REVISIONS_SCRIPT,
env={
"PROVISION_GROUP": group,
"PROVISION_SPECS": json.dumps(pinned),
},
)
errors = []
for line in out.splitlines():
line = line.strip()
if not line.startswith("{"):
continue
rec = json.loads(line)
if rec.get("error"):
errors.append(f"{rec['name']}: {rec['error']}")
continue
state = "fetched from remote" if rec.get("fetched") else "already present"
print(
f" {group}/{rec['name']}: pinned revision {state}",
file=sys.stderr,
)
if errors:
die("revision fetch failed:\n " + "\n ".join(errors))
def enforce_revisions(pod: str, group: str, specs: list[dict]) -> None:
"""Reset each project's default branch to its pinned revision."""
pinned = [s for s in specs if s.get("revision")]
if not pinned:
return
out = rails_runner(
pod,
_ENFORCE_REVISION_SCRIPT,
env={
"PROVISION_GROUP": group,
"PROVISION_SPECS": json.dumps(pinned),
},
)
errors = []
for line in out.splitlines():
line = line.strip()
if not line.startswith("{"):
continue
rec = json.loads(line)
if rec.get("error"):
errors.append(f"{rec['name']}: {rec['error']}")
continue
state = "pinned" if rec.get("changed") else "already-pinned"
print(
f" {group}/{rec['name']}: {rec['ref']} -> "
f"{rec['revision']} ({state})",
file=sys.stderr,
)
if errors:
die("revision enforcement failed:\n " + "\n ".join(errors))
def sweep_stray_refs(pod: str, group: str, names: list[str]) -> None:
"""Delete refs/test/* and benchmark-* branches left by previous runs."""
out = rails_runner(
pod,
_SWEEP_REFS_SCRIPT,
env={
"PROVISION_GROUP": group,
"PROVISION_NAMES": json.dumps(names),
},
)
errors = []
for line in out.splitlines():
line = line.strip()
if not line.startswith("{"):
continue
rec = json.loads(line)
if rec.get("error"):
errors.append(f"{rec['name']}: {rec['error']}")
continue
msg = f" {group}/{rec['name']}: swept {rec['swept']} stray refs"
if rec.get("warning"):
msg += f" ({rec['warning']})"
print(msg, file=sys.stderr)
if errors:
die("stray-ref sweep failed:\n " + "\n ".join(errors))
def create_additional_refs(pod: str, group: str, specs: list[dict]) -> None:
"""Create the zzz benchmark refs through the write path.
Mirrors the VM flow (roles/gitaly/tasks/initialize.yml) so every
provisioned project set carries the same ref load.
"""
wanted = [s for s in specs if s.get("additional_refs")]
if not wanted:
return
out = rails_runner(
pod,
_ADDITIONAL_REFS_SCRIPT,
env={
"PROVISION_GROUP": group,
"PROVISION_SPECS": json.dumps(wanted),
"PROVISION_REF_BATCH": os.environ.get("PROVISION_REF_BATCH", "50000"),
},
)
errors = []
for line in out.splitlines():
line = line.strip()
if not line.startswith("{"):
continue
rec = json.loads(line)
if rec.get("error"):
errors.append(f"{rec['name']}: {rec['error']}")
continue
state = "already-present" if rec.get("skipped") else "created"
print(
f" {group}/{rec['name']}: {rec['refs']} refs ({state})",
file=sys.stderr,
)
if errors:
die("additional-refs creation failed:\n " + "\n ".join(errors))
def mint_push_token(pod: str) -> None:
"""Mint the benchmark PAT and store it as the gitaly-benchmark-pat Secret
(key `token`), which the Job's http push transport reads via secretKeyRef.
The cleartext goes kubectl-stdin -> Secret, never printed or on disk."""
out = rails_runner(pod, _PAT_SCRIPT)
rec: dict = {}
for line in out.splitlines():
line = line.strip()
if line.startswith("{"):
rec = json.loads(line)
if rec.get("error") or not rec.get("token"):
die(f"PAT provisioning failed: {rec.get('error') or out.strip()}")
_store_secret("gitaly-benchmark-pat", {"token": rec["token"]})
print(
f" PAT for {rec['user']} (expires {rec['expires_at']}) stored in "
f"secret gitaly-benchmark-pat/{GITLAB_NAMESPACE}",
file=sys.stderr,
)
def _store_secret(name: str, data: dict[str, str]) -> None:
"""Apply an Opaque Secret. Cleartext goes over kubectl's stdin, not
argv, which is world-readable in `ps`."""
manifest = json.dumps(
{
"apiVersion": "v1",
"kind": "Secret",
"metadata": {"name": name, "namespace": GITLAB_NAMESPACE},
"type": "Opaque",
"data": {
k: base64.b64encode(v.encode()).decode() for k, v in data.items()
},
}
)
apply = subprocess.run(
_kubectl_cmd() + ["apply", "-f", "-"],
input=manifest,
capture_output=True,
text=True,
check=False,
)
if apply.returncode != 0:
die(f"storing secret {name} failed: {apply.stderr.strip()}")
def mint_ssh_key(pod: str) -> None:
"""Provision the ssh transport's keypair: public half onto the admin
user, private half into a Secret. Fresh each run (GitLab keeps only
the public half)."""
with tempfile.TemporaryDirectory() as tmp:
key_path = Path(tmp) / "id_ed25519"
keygen = subprocess.run(
["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C",
"gitaly-benchmark", "-f", str(key_path)],
capture_output=True,
text=True,
check=False,
)
if keygen.returncode != 0:
die(f"ssh-keygen failed: {keygen.stderr.strip() or keygen.stdout.strip()}")
private_key = key_path.read_text()
public_key = Path(f"{key_path}.pub").read_text().strip()
out = rails_runner(pod, _SSH_KEY_SCRIPT, env={"BENCH_SSH_PUBKEY": public_key})
rec: dict = {}
for line in out.splitlines():
line = line.strip()
if line.startswith("{"):
rec = json.loads(line)
if rec.get("error") or not rec.get("fingerprint"):
die(f"SSH key provisioning failed: {rec.get('error') or out.strip()}")
_store_secret("gitaly-benchmark-ssh-key", {"private_key": private_key})
print(
f" SSH key for {rec['user']} ({rec['fingerprint']}) stored in "
f"secret gitaly-benchmark-ssh-key/{GITLAB_NAMESPACE}",
file=sys.stderr,
)
def _gitaly_relative_path(disk_path: str) -> str:
# Rails returns disk_path without the trailing `.git`; Gitaly's
# gRPC relativePath expects it.
return disk_path if disk_path.endswith(".git") else f"{disk_path}.git"
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument(
"--write-config",
action="store_true",
help="Edit experiments/${EXPERIMENT}/config.yml in place, "
"adding `disk_path:` keys next to each repo's `name:`.",
)
ap.add_argument(
"--reset",
action="store_true",
help="Remove and re-import existing benchmark repositories before provisioning.",
)
ap.add_argument(
"--experiment",
default=os.environ.get("EXPERIMENT"),
help="Experiment directory name (defaults to $EXPERIMENT).",
)
args = ap.parse_args()
if not args.experiment:
die("EXPERIMENT must be set (or pass --experiment)")
cfg_path = BENCH_DIR / "experiments" / args.experiment / "config.yml"
if not cfg_path.is_file():
die(f"experiment config not found: {cfg_path}")
cfg = _load(cfg_path)
repos = cfg.get("repositories") or []
if not repos:
die(f"no repositories defined in {cfg_path}")
specs = []
for r in repos:
name = r.get("name")
remote = r.get("remote")
if not name or not remote:
die(
f"repository entry missing required name/remote: {r!r}\n in {cfg_path}"
)
spec = {"name": str(name), "remote": str(remote)}
if r.get("revision"):
spec["revision"] = str(r["revision"])
extra = r.get("additional_refs")
if isinstance(extra, int) and extra > 0:
spec["additional_refs"] = str(extra)
specs.append(spec)
pod = find_toolbox_pod()
# One (group, Rails storage, config.yml key prefix) per project set. The
# control set only exists when its storage is registered in Rails.
sides = [(BENCH_GROUP, "default", "")]
storages = rails_storages(pod)
if CONTROL_STORAGE_NAME in storages:
sides.append((BENCH_CONTROL_GROUP, CONTROL_STORAGE_NAME, "control_"))
else:
print(
f"[provision] storage {CONTROL_STORAGE_NAME!r} not registered in "
f"Rails (have: {', '.join(storages) or 'none'}); skipping the "
"control-side project set. Comparison runs need it: register "
"the control Gitaly as an external storage in the chart values.",
file=sys.stderr,
)
print(
"[provision] toolbox=" + pod + " sides="
+ ", ".join(f"{g} (storage={s})" for g, s, _ in sides),
file=sys.stderr,
)
results: dict[str, dict[str, dict]] = {}
for group, storage, prefix in sides:
if args.reset:
print(f"[provision] resetting repositories in {group}", file=sys.stderr)
reset_repositories(pod, group, specs)
created = provision(pod, group, storage, specs)
failures = {n: r for n, r in created.items() if r.get("error")}
if failures:
die(
f"project creation failed in {group}:\n "
+ "\n ".join(f"{n}: {r['error']}" for n, r in failures.items())
)
print(f"[provision] waiting for imports in {group}", file=sys.stderr)
status = wait_for_imports(pod, group, [s["name"] for s in specs])
failed = [
n
for n, r in status.items()
if r.get("import_status") == "failed" or r.get("missing")
]
# The MVCC backend lacks RepositoryInfo, so Rails bookkeeping can
# mark an import "failed" even when the clone succeeded. Tolerate
# readable repos; revision enforcement still catches broken ones.
for n in list(failed):
r = status[n]
if not r.get("missing") and r.get("branches", -1) > 0:
print(
f" {group}/{n}: import marked failed but repository "
f"is readable ({r['branches']} branches); continuing "
f"(error was: {r.get('import_error')})",
file=sys.stderr,
)
failed.remove(n)
if failed:
die(
f"imports failed in {group}:\n "
+ "\n ".join(
f"{n}: {status[n].get('import_error') or 'missing'}"
for n in failed
)
)
print(f"[provision] sweeping stray benchmark refs in {group}", file=sys.stderr)
sweep_stray_refs(pod, group, [s["name"] for s in specs])
print(
f"[provision] fetching remotes where pinned revisions are "
f"missing in {group}",
file=sys.stderr,
)
fetch_missing_revisions(pod, group, specs)
print(f"[provision] enforcing pinned revisions in {group}", file=sys.stderr)
enforce_revisions(pod, group, specs)
print(
f"[provision] creating additional benchmark refs in {group} "
"(can take several minutes on first run)",
file=sys.stderr,
)
create_additional_refs(pod, group, specs)
results[prefix] = created
print("[provision] minting PAT for the http push transport", file=sys.stderr)
mint_push_token(pod)
print("[provision] minting SSH key for the ssh transport", file=sys.stderr)
mint_ssh_key(pod)
# No ref-packing on either side: MVCC has no packed-refs, and packing
# only the control would give the sides different ref layouts.
# Write each side's disk_path back to its repo entry (control keys get
# the control_ prefix). gl_repository rides along for the push
# sections' pre-receive hook.
for r in repos:
for prefix, created in results.items():
meta = created.get(str(r["name"]))
if not meta or not meta.get("disk_path"):
continue
r[f"{prefix}disk_path"] = _gitaly_relative_path(meta["disk_path"])
if meta.get("gl_repository"):
r[f"{prefix}gl_repository"] = meta["gl_repository"]
if args.write_config:
_dump(cfg, cfg_path)
print(
f"[provision] wrote disk_path/gl_repository entries to {cfg_path}",
file=sys.stderr,
)
else:
print("# paste this into the corresponding entry in", cfg_path)
for r in repos:
for prefix, _created in results.items():
if f"{prefix}disk_path" not in r:
continue
print(f" {prefix}disk_path: {r[f'{prefix}disk_path']!r}")
if f"{prefix}gl_repository" in r:
print(f" {prefix}gl_repository: {r[f'{prefix}gl_repository']!r}")
return 0
if __name__ == "__main__":
sys.exit(main())