Python
feat(ows): coordinate reviewed AWS and Azure service deployments
Rohit P05773ef3f0b7
diff --git a/deploy/service/guard/guard.test.cjs b/deploy/service/guard/guard.test.cjsindex a51e5ab3..a5ea83bb 100644--- a/deploy/service/guard/guard.test.cjs+++ b/deploy/service/guard/guard.test.cjs@@ -227,3 +227,11 @@ test("invalid clocks and incomplete SDK inventories never authorize cleanup", as};await assert.rejects(api.objects(), /repeated_token/);});++test("persistent deployments never expire or delete data after provisioning deadline", async () => {+ const f = fixture();+ const s = settings({...env, OWS_AUTO_EXPIRE:"false"}, context);+ assert.equal((await run(s, f.api, 5000)).status, "not_due");+ assert.equal(f.effects.length, 0);+ assert.throws(()=>settings({...env, OWS_AUTO_EXPIRE:"maybe"}, context));+});diff --git a/deploy/service/guard/index.cjs b/deploy/service/guard/index.cjsindex a5c7c3ae..ea69a140 100644--- a/deploy/service/guard/index.cjs+++ b/deploy/service/guard/index.cjs@@ -33,6 +33,8 @@ function settings(env, context) {`arn:aws:lambda:${region}:${account}:function:${prefix}-cleanup`)throw Error("invalid_function_identity");+ if (env.OWS_AUTO_EXPIRE !== undefined && !["true", "false"].includes(env.OWS_AUTO_EXPIRE))+ throw Error("invalid_expiry_policy");const expiry = Number(env.OWS_EXPIRY);if (!Number.isSafeInteger(expiry) || expiry <= 0)throw Error("invalid_expiry");@@ -46,6 +48,7 @@ function settings(env, context) {role,prefix,expiry,+ autoExpire: env.OWS_AUTO_EXPIRE !== "false",table: prefix + "-control",};}@@ -106,7 +109,8 @@ async function run(s, api, now, event = {}) {record.version++;await api.writeControl(record, previous);}- if (now < s.expiry) return { status: "not_due", destructiveActions: 0 };+ if (now < s.expiry || s.autoExpire === false)+ return { status: "not_due", destructiveActions: 0 };if (stack) {if (stack.StackStatus === "DELETE_IN_PROGRESS")return { status: "deleting", destructiveActions: 0 };diff --git a/deploy/service/up/README.md b/deploy/service/up/README.mdnew file mode 100644index 00000000..4becb9a1--- /dev/null+++ b/deploy/service/up/README.md@@ -0,0 +1,103 @@+# Deploy OpenWorker with one command++This experimental coordinator installs a private service cluster, OpenShell and the OpenWorker service using a reviewed account setup profile. It resumes the same deployment when rerun. Fresh AWS and Azure installs with this coordinator still require live qualification; do not treat local tests as a production certificate.++Once the prerequisites below are installed, copy `aws.yaml` or `azure.yaml` to `deployment.yaml`, set your account/subscription and region, then run:++```sh+ows deploy aws up --config deployment.yaml+# or+ows deploy azure up --config deployment.yaml+```++The command resolves a supported VM size, writes a cost/permission review, and asks for approval before making cloud changes. Fixed node counts are supported; autoscaling is not. Requests with an uncertain outcome are observed using their original identity rather than blindly resubmitted.++## One-time prerequisites++Install `ows`, your cloud CLI, Docker, Helm, kubectl and OpenSSL. Azure also needs kubelogin and, for the management VM, SSH. AWS's optional operator bridge needs the Session Manager plugin. Authenticate with your own cloud identity. The command never creates long-lived cloud access keys.++An administrator prepares a target-bound setup profile at:++```text+~/.config/ows/deployments/aws/<account-id>/<region>.json+~/.config/ows/deployments/azure/<subscription-id>/<region>.json+```++Use `advanced.setupProfile` in YAML to select another file. The administrator reviews the scoped native bootstrap permission exports. AWS needs permission to create the reviewed bootstrap identity stack. Azure needs the reviewed RBAC authority at the workload/control groups and exact vault/secret scopes; its active login must match `setupObjectId`. Normal workload identities never receive these installation privileges. Azure resource groups, the Entra administrator group and the versioned database password in Key Vault already exist. The vault must use RBAC and allow ARM template deployment.++The profile contains **references and version pins**, not passwords:++```json+{+ "schemaVersion": 1,+ "provider": "aws",+ "account": "123456789012",+ "region": "us-west-2",+ "operatorArn": "arn:aws:iam::123456789012:role/approved-installer",+ "infrastructure": {},+ "release": "/absolute/path/to/release/release.json"+}+```++Replace `infrastructure` with the reviewed native AWS input described in [`harness/service-cli/examples/README.md`](../../../harness/service-cli/examples/README.md). It includes zone/version pins and a versioned S3 reference to **this CLI release's packaged cleanup guard**. For Azure, use the existing Azure infrastructure input (Kubernetes version, supported VM size, SSH public key, administrator group IDs, control group, pinned external cleanup image and Key Vault secret reference). Its profile uses `provider: azure`, the subscription as `account`, and `resourceGroup`, `setupObjectId`, `setupPrincipalType` instead of `operatorArn`. Package the matching native cleanup image before running the installer. Existing control assets are retained, not silently taken over or deleted.++This release does not assume public OpenWorker service images or a published Helm chart exist. Prepare a reviewed release manifest with `package.py --help`: it copies a chart, pinned OpenShell archive and values, platform manifest and database CA bundle, then hashes them. Supply immutable harness/runner image digests for Linux ARM64 on AWS or AMD64 on Azure. The supported OpenShell archive is 0.1.7. The CA bundle must contain 1–32 certificates. The installer publishes the selected images into its owned ECR/ACR registry. An optional UI image can be packaged, but browser ingress/UI exposure is not enabled by this initial coordinator.++## Review, budget and resume++```sh+ows deploy azure up --config deployment.yaml --plan-only+```++The private `.ows/deployments/<target-and-name-hash>/` directory contains `review.json`, the exact native `infrastructure/` permission/template bundle and the durable journal. Keep it: it holds the original deployment identities. Do not delete or edit it to retry an uncertain cloud operation.++`budget.limitUSD` is the reviewed experiment/planning budget used for cost checks. **It does not create cloud billing alerts and is not a hard cap.** Fixed costs, missing catalog rates and a separate variable-cost allowance are reported. Storage, logs, traffic, model usage and retained control resources can add costs; the allowance is not an upper bound. Persistent reviews use a 730-hour planning horizon; disposable trials use their specified lifetime. Set account billing alerts separately.++For automation, review the generated files, then supply their exact review digest:++```sh+ows deploy azure up --config deployment.yaml \+ --accept-plan <planDigest> --accept-bootstrap <planDigest>+```++AWS accepts the same options. Saved approvals apply only to that exact configuration, release and generated cloud graph. Rerunning observes unfinished operations and retains their receipts. Changed configuration requires a separate reviewed deployment/update; it is not applied in place.++## Private management access++The command can use an existing private route. If your account input enables the operator bridge, it creates the native bridge resource and connects automatically: AWS SSM port forwarding or Azure SSH restricted to the configured operator IPv4. Azure's `advanced.azure.sshPrivateKeyFile` selects the matching local key; otherwise SSH uses your existing key/agent. SSH host keys are saved in the private deployment directory. The original Kubernetes hostname and CA remain verified through a loopback proxy.++Add `--keep-access` to keep that route open after readiness checks. In another terminal use `connection.json` for the private API URL, CA, token-file path and port-forward instructions. Ctrl-C closes the local route; subsequent commands can recreate it after reconciling the original SSM session. A changed operator address, unavailable SSM session, occupied local proxy port or missing private route produces an actionable error rather than making the Kubernetes management API public.++## Model credentials and completion++Use `models.openai: {env: OPENAI_API_KEY}` or `models.anthropic: {env: ANTHROPIC_API_KEY}` (a `file` reference also works). Keys are read only when staging Kubernetes Secrets and are never written to the immutable review. The private local credential staging directory contains secrets and must not be committed.++An installation can complete without model credentials. Completion verifies component readiness; it explicitly reports `modelSmokeTest: not_run`. Submit an agent separately to qualify your chosen provider/model and application.++## Persistent deployments and disposable trials++Omitting `trial` disables automatic workload expiry. The older destructive teardown route refuses persistent deployments until a separate retention workflow is approved. It will not delete your database as a side effect of an expired provisioning deadline.++For disposable tests only, set:++```yaml+trial:+ expiresAfterHours: 6+ deleteData: true+```++This enables the existing independent cleanup controller. It retains the control/audit resources documented in the native review. A timeout or failure does not authorize deleting/recreating resources with uncertain outcomes. Read the original receipts and use the native lifecycle reconciliation/cleanup commands for the same deployment.++## Local checks++Run the CLI unit/integration suite with `cargo test --manifest-path harness/Cargo.toml -p openworker-service-cli`. Its cloud operations use fixtures. The cleanup guard and Helm rendering checks run with `node --test deploy/service/guard/guard.test.cjs deploy/service/chart.test.mjs`.++To exercise the actual `up --plan-only` entry point, package pins, approval gates and resume handling with trapped AWS commands:++```sh+python3 deploy/service/up/test-plan.py \+ --ows /absolute/path/to/ows \+ --openshell-archive /absolute/path/to/helm-chart-0.1.7.tgz+```++That check uses the real pinned OpenShell archive, fixture image references and catalog responses. It performs no real cloud calls, does not install a cluster and must not be presented as cloud qualification.diff --git a/deploy/service/up/aws.yaml b/deploy/service/up/aws.yamlnew file mode 100644index 00000000..2b29c14a--- /dev/null+++ b/deploy/service/up/aws.yaml@@ -0,0 +1,18 @@+apiVersion: ows/v1alpha1+name: my-agents+cloud:+ provider: aws+ accountId: "123456789012"+ region: us-west-2+ profile: default+nodes:+ count: 2+ requirements:+ cpu: {min: 2, max: 4}+ memoryGiB: {min: 8, max: 16}+budget:+ limitUSD: 1000+# Omit trial to keep the deployment running. For disposable test data only:+# trial: {expiresAfterHours: 6, deleteData: true}+# models:+# anthropic: {env: ANTHROPIC_API_KEY}diff --git a/deploy/service/up/azure.yaml b/deploy/service/up/azure.yamlnew file mode 100644index 00000000..9d1be8e0--- /dev/null+++ b/deploy/service/up/azure.yaml@@ -0,0 +1,17 @@+apiVersion: ows/v1alpha1+name: my-agents+cloud:+ provider: azure+ subscriptionId: "11111111-2222-4333-8444-555555555555"+ region: westus2+nodes:+ count: 2+ requirements:+ cpu: {min: 4, max: 8}+ memoryGiB: {min: 16, max: 32}+budget:+ limitUSD: 1000+# Omit trial to keep the deployment running. For disposable test data only:+# trial: {expiresAfterHours: 6, deleteData: true}+# models:+# anthropic: {env: ANTHROPIC_API_KEY}diff --git a/deploy/service/up/package.py b/deploy/service/up/package.pynew file mode 100644index 00000000..b34d6b68--- /dev/null+++ b/deploy/service/up/package.py@@ -0,0 +1,70 @@+#!/usr/bin/env python3+"""Package reviewed local deployment assets. Offline; never uploads or provisions."""+import argparse+import hashlib+import json+import pathlib+import shutil+++def sha(data):+ return hashlib.sha256(data).hexdigest()+++def pin(source, destination):+ source = pathlib.Path(source)+ if source.is_symlink() or not source.is_file() or source.stat().st_size > 16 * 1024 * 1024:+ raise ValueError('Expected a bounded regular release file')+ shutil.copyfile(source, destination)+ return {'file': str(destination), 'sha256': sha(destination.read_bytes())}+++def tree(root):+ files = {}+ for path in sorted(root.rglob('*')):+ if path.is_symlink():+ raise ValueError('Chart symlinks are not supported')+ if path.is_file():+ if path.stat().st_size > 1024 * 1024:+ raise ValueError('Chart file exceeds 1 MiB')+ files[path.relative_to(root).as_posix()] = sha(path.read_bytes())+ if 'Chart.yaml' not in files or len(files) > 128:+ raise ValueError('Expected a bounded Helm chart')+ return sha(json.dumps(files, sort_keys=True, separators=(',', ':'), ensure_ascii=False).encode())+++def main():+ p = argparse.ArgumentParser(description=__doc__)+ for arg in ('version', 'architecture', 'harness-image', 'runner-image', 'service-chart',+ 'openshell-archive', 'openshell-values', 'platform-manifest', 'database-ca', 'out'):+ p.add_argument('--' + arg, required=True)+ p.add_argument('--ui-image')+ a = p.parse_args()+ if a.architecture not in ('arm64', 'amd64'):+ p.error('architecture must be arm64 or amd64')+ for image in (a.harness_image, a.runner_image, a.ui_image):+ if image is not None:+ import re+ if not re.fullmatch(r'[a-zA-Z0-9/._:-]+@sha256:[a-f0-9]{64}', image):+ p.error('images must use immutable SHA-256 digests')+ root = pathlib.Path(a.out).absolute()+ root.mkdir(mode=0o700) # refuse overwrite of an existing release+ chart = pathlib.Path(a.service_chart)+ tree(chart) # validate before copying+ shutil.copytree(chart, root / 'chart')+ result = {'schemaVersion': 1, 'version': a.version, 'architecture': a.architecture,+ 'images': {'harness': a.harness_image, 'runner': a.runner_image, 'serviceUi': a.ui_image},+ 'serviceChart': str(root / 'chart'), 'serviceChartSha256': tree(root / 'chart')}+ for key, source, name in (+ ('openshellArchive', a.openshell_archive, 'openshell.tgz'),+ ('openshellValues', a.openshell_values, 'openshell-values.json'),+ ('platformManifest', a.platform_manifest, 'platform.json'),+ ('databaseCa', a.database_ca, 'database-ca.pem'),+ ):+ result[key] = pin(source, root / name)+ (root / 'release.json').write_text(json.dumps(result, indent=2) + '\n')+ print(root / 'release.json')+++if __name__ == '__main__':+ main()diff --git a/deploy/service/up/test-plan.py b/deploy/service/up/test-plan.pynew file mode 100644index 00000000..fbf2000f--- /dev/null+++ b/deploy/service/up/test-plan.py@@ -0,0 +1,83 @@+#!/usr/bin/env python3+"""Read-only CLI qualification with a real pinned chart and trapped AWS commands.+No cloud login, keys or network are used. This does not qualify installation.+"""+import argparse+import json+import os+import pathlib+import subprocess+import tempfile++p = argparse.ArgumentParser(description=__doc__)+p.add_argument('--ows', required=True)+p.add_argument('--openshell-archive', required=True)+a = p.parse_args()+repo = pathlib.Path(__file__).resolve().parents[3]+ows = str(pathlib.Path(a.ows).resolve())+with tempfile.TemporaryDirectory(prefix='ows-up-plan-') as tmp:+ root = pathlib.Path(tmp)+ tools = root / 'tools'+ tools.mkdir()+ aws = tools / 'aws'+ aws.write_text('''#!/usr/bin/python3+import json, os, sys+args=sys.argv[1:]; command=args[:2]+with open(os.environ['OWS_FIXTURE_CALLS'],'a') as f:f.write(json.dumps(command)+'\\n')+if command==['sts','get-caller-identity']:+ v={'Account':'123456789012','Arn':'arn:aws:iam::123456789012:role/operator'}+elif command==['ec2','describe-instance-types']:+ v={'InstanceTypes':[{'InstanceType':name,'VCpuInfo':{'DefaultVCpus':cpu},'MemoryInfo':{'SizeInMiB':mem*1024},'ProcessorInfo':{'SupportedArchitectures':['arm64']}} for name,cpu,mem in [('t4g.large',2,8),('t4g.xlarge',4,16),('t4g.2xlarge',8,32)]]}+elif command==['ec2','describe-instance-type-offerings']:+ v={'InstanceTypeOfferings':[{'InstanceType':n} for n in ['t4g.large','t4g.xlarge','t4g.2xlarge']]}+elif command==['pricing','get-products']:+ filters=json.loads(args[args.index('--filters')+1]); monthly=any(f['Value'] in ['Storage','Database Storage'] for f in filters)+ dimension={'unit':'GB-Mo' if monthly else 'Hrs','beginRange':'0','endRange':'Inf','pricePerUnit':{'USD':'0.1'}}+ v={'PriceList':[json.dumps({'terms':{'OnDemand':{'term':{'priceDimensions':{'fixture':dimension}}}}})]}+else:+ raise SystemExit('UNEXPECTED CLOUD COMMAND: '+str(command))+print(json.dumps(v))+''')+ compile(aws.read_text(), str(aws), "exec")+ aws.chmod(0o700)+ env = dict(os.environ, PATH=str(tools) + os.pathsep + os.environ.get('PATH', ''), HOME=str(root), OWS_FIXTURE_CALLS=str(root/'calls.jsonl'))+ def run(args, ok=True):+ r = subprocess.run([ows]+args, env=env, cwd=root, capture_output=True, text=True)+ if ok and r.returncode != 0:+ raise AssertionError(r.stderr)+ if not ok and r.returncode == 0:+ raise AssertionError('Unexpected success')+ return r+ run(['deploy','guard','package','--out',str(root/'guard')])+ (root/'shell.json').write_text(json.dumps({'global':{'image':{'tag':'0.1.7'}},'server':{'auth':{'allowUnauthenticatedUsers':False}}}))+ (root/'platform.json').write_text(json.dumps({'apiVersion':'v1','kind':'List','items':[]}))+ subprocess.run(['python3',str(repo/'deploy/service/up/package.py'),'--version','fixture','--architecture','arm64',+ '--harness-image','ghcr.io/example/harness@sha256:'+'a'*64,'--runner-image','ghcr.io/example/runner@sha256:'+'b'*64,+ '--service-chart',str(repo/'deploy/service/chart'),'--openshell-archive',str(pathlib.Path(a.openshell_archive).resolve()),+ '--openshell-values',str(root/'shell.json'),'--platform-manifest',str(root/'platform.json'),+ '--database-ca',str(repo/'harness/store-postgres/tests/fixtures/public-ca.pem'),'--out',str(root/'release')], check=True, capture_output=True)+ infrastructure = json.loads((repo/'harness/service-cli/examples/aws-infrastructure.json').read_text())+ infrastructure['guardCode']['sha256'] = json.loads((root/'guard/package-manifest.json').read_text())['archiveSha256']+ setup = {'schemaVersion':1,'provider':'aws','account':'123456789012','region':'us-west-2',+ 'operatorArn':'arn:aws:iam::123456789012:role/operator','infrastructure':infrastructure,'release':str(root/'release/release.json')}+ (root/'setup.json').write_text(json.dumps(setup))+ yaml = "name: fixture\ncloud: {provider: aws, accountId: '123456789012', region: us-west-2}\nnodes: {count: 2, instanceType: t4g.large}\nbudget: {limitUSD: 1000}\nadvanced:\n setupProfile: " + str(root/'setup.json') + '\n'+ (root/'deployment.yaml').write_text(yaml)+ args = ['deploy','aws','up','--config',str(root/'deployment.yaml'),'--state',str(root/'state')]+ run(args+['--plan-only'])+ first = (root/'state/review.json').read_bytes()+ review = json.loads(first)+ assert review['config']['nodes']['count'] == 2+ plan = json.loads((root/'state/infrastructure/deployment-plan.json').read_text())+ assert plan['autoExpire'] is False+ run(args+['--plan-only'])+ assert (root/'state/review.json').read_bytes() == first+ run(args+['--accept-plan','wrong'], ok=False)+ run(args+['--accept-plan',review['digest']], ok=False) # bootstrap still unapproved+ calls = (root/'calls.jsonl').read_bytes()+ (root/'deployment.yaml').write_text(yaml.replace('count: 2', 'count: 3'))+ result = run(args+['--plan-only'], ok=False)+ assert 'changed' in result.stderr+ assert (root/'calls.jsonl').read_bytes() == calls+ assert all(json.loads(line) in [['sts','get-caller-identity'],['ec2','describe-instance-types'],['ec2','describe-instance-type-offerings'],['pricing','get-products']] for line in calls.splitlines())+ print(json.dumps({'readOnlyPlan':True,'stableResume':True,'changedConfigRejected':True,'unapprovedBootstrapBlocked':True,'cloudMutations':0,'installationQualified':False}))diff --git a/harness/service-cli/src/deployment.rs b/harness/service-cli/src/deployment.rsindex 79a47561..75f79a68 100644--- a/harness/service-cli/src/deployment.rs+++ b/harness/service-cli/src/deployment.rs@@ -16,6 +16,8 @@ mod infrastructure;mod local;#[path = "deployment_preflight.rs"]mod preflight;+#[path = "deployment_up.rs"]+mod up;use anyhow::{bail, ensure, Context, Result};use clap::{Args, Subcommand};use serde::{Deserialize, Serialize};@@ -89,6 +91,8 @@ pub enum Command {}#[derive(Subcommand)]pub enum AwsCommand {+ /// Provision and install from one reviewed deployment.yaml; resumes original state.+ Up(up::Options),Plan(AwsPlan),/// Experimental lifecycle; explicit opt-in, role profile and fresh preflight.Lifecycle {@@ -143,6 +147,8 @@ pub enum GcpCommand {}#[derive(Subcommand)]pub enum AzureCommand {+ /// Provision and install from one reviewed deployment.yaml; resumes original state.+ Up(up::Options),Plan(AzurePlan),/// Export reviewed identity-only bootstrap; never runs Azure commands.Bootstrap(azure::Bootstrap),@@ -188,6 +194,8 @@ pub struct AzurePlan {}#[derive(Args)]struct PlanOptions {+ #[arg(long, default_value_t = 2)]+ node_count: u16,#[arg(long)]name: String,#[arg(long, default_value_t = 100)]@@ -240,6 +248,10 @@ struct Plan {target: Target,created_at_unix: u64,expires_at_unix: u64,+ /// None preserves legacy trial semantics; false means provisioning deadline+ /// only, never permission to destroy a persistent deployment's data.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ auto_expire: Option<bool>,budget: Budget,topology: Topology,#[serde(default, skip_serializing_if = "Option::is_none")]@@ -292,6 +304,9 @@ fn region_ok(s: &str) -> bool {&& !parts[2].starts_with('0')}impl Plan {+ fn expires_workload(&self) -> bool {+ self.auto_expire.unwrap_or(true)+ }fn validate(&self) -> Result<()> {ensure!([1, 2, 3].contains(&self.schema_version),@@ -419,11 +434,16 @@ impl Plan {&& self.budget.estimate.is_none(),"Offline plans require a positive budget, unknown estimate, and no hard-cap claim");+ ensure!(+ (1..=16).contains(&self.topology.node_count)+ && (!matches!(self.target, Target::Gcp { .. }) || self.topology.node_count == 2),+ "Use 1..16 fixed AWS/Azure nodes; the GCP reference still requires two"+ );ensure!(self.topology== Topology {cluster_mode: "new".into(),- node_count: 2,+ node_count: self.topology.node_count,max_worker_replicas: 2,architecture: if matches!(self.target, Target::Aws { .. }) {"arm64"@@ -455,6 +475,14 @@ impl Planner for AwsPlanner {let prefix = p.prefix();let review=format!("# {} deployment review\n\nDeployment: `{}`\nAWS account: `{}`\nRegion: `{}`\nResource prefix: `{}`\n\n## Proposed experiment\n\nNew EKS cluster; two ARM64 nodes; at most two harness worker replicas.\nPrivate access. No automatic node autoscaler. PostgreSQL and OpenShell are\nrequired dependencies. Nodes and worker replicas are separate capacity settings.\n\nBudget authorization: ${} USD. **No verified cost estimate; not a hard cap.**\nCreated (Unix seconds): {}. Fixed expiry (Unix seconds): {}.\n\n## Readiness\n\nThis is an offline plan. No AWS call or resource change has occurred.\nIt cannot be applied yet. Still required: complete scoped bootstrap policies,\ncaller/role identity, live price/quota/version checks, pinned service/UI/runner\nimages, infrastructure templates, network/storage inputs, secret references,\ncloud-side expiry guard, and a qualified cloud executor.\n\n`permissions-request.json` describes required authority; it is NOT an IAM policy.\nNo bootstrap.sh is emitted until its permissions and operations are implemented.\nThe bundle digest detects changes; it is not a digital signature.\n",p.name,p.deployment_id,account,region,prefix,p.budget.authorized_usd,p.created_at_unix,p.expires_at_unix);+ let review = if p.auto_expire.is_some() || p.topology.node_count != 2 {+ review.replace(+ "two ARM64 nodes",+ &format!("{} fixed ARM64 nodes", p.topology.node_count),+ )+ } else {+ review+ };let permissions = json!({"schemaVersion":1,"format":"ows.permission-requirements.v1","deploymentId":p.deployment_id,"target":p.target,"resourcePrefix":prefix,"setup":{"suggestedRoleName":format!("{prefix}-setup"),"trust":"explicit operator principal required","temporaryCredentials":true,"operations":["manage owned CloudFormation infrastructure","configure scoped EKS access and platform dependencies","install namespace-bound Helm releases"],"policyReady":false},"workload":{"suggestedRoleName":format!("{prefix}-artifacts"),"identity":"EKS Pod Identity","operations":["access only the deployment artifact bucket/prefix if enabled"],"sharedWithAgents":false,"policyReady":false},@@ -501,6 +529,12 @@ fn artifacts(plan: &Plan) -> Result<BTreeMap<String, Vec<u8>>> {}if plan.azure_infrastructure.is_some() {files.extend(azure::infrastructure_artifacts(plan)?);+ if plan.auto_expire.is_some() {+ files.insert(+ "azure-up-grants.json".into(),+ json_bytes(&azure::up_grants(plan)?)?,+ );+ }}if plan.gcp_infrastructure.is_some() {files.extend(gcp::infrastructure_artifacts(plan)?);@@ -597,6 +631,15 @@ fn result(p: &Plan, m: &Manifest) -> Value {"identityBootstrap": if p.azure_bootstrap.is_some() { "exported_azure_identity_only" } else if p.gcp_bootstrap.is_some() { "exported_gcp_identity_only" } else if p.infrastructure.is_some() { "exported_generated_execution_roles" } else if p.bootstrap.is_some() { "exported_existing_execution_role_required" } else { "not_generated" }})}pub async fn dispatch(command: &Command) -> Result<()> {+ match command {+ Command::Aws {+ command: AwsCommand::Up(o),+ } => return up::run("aws", o).await,+ Command::Azure {+ command: AzureCommand::Up(o),+ } => return up::run("azure", o).await,+ _ => {}+ }if let Command::Azure {command: AzureCommand::Installation { command },} = command@@ -667,6 +710,12 @@ pub async fn dispatch(command: &Command) -> Result<()> {}pub fn run(command: &Command) -> Result<()> {match command {+ Command::Aws {+ command: AwsCommand::Up(_),+ }+ | Command::Azure {+ command: AzureCommand::Up(_),+ } => bail!("Deployment up requires asynchronous dispatch"),Command::Azure {command:AzureCommand::Lifecycle { .. }@@ -839,6 +888,7 @@ fn create(expires_at_unix: created.checked_add(u64::from(a.ttl_hours) * 3600).context("Expiry overflow")?,+ auto_expire: None,budget: Budget {authorized_usd: a.budget,hard_cap: false,@@ -846,7 +896,7 @@ fn create(},topology: Topology {cluster_mode: "new".into(),- node_count: 2,+ node_count: a.node_count,max_worker_replicas: 2,architecture: architecture.into(),access: "private".into(),@@ -939,6 +989,7 @@ mod tests {},created_at_unix: 100,expires_at_unix: 200,+ auto_expire: None,budget: Budget {authorized_usd: 100,hard_cap: false,diff --git a/harness/service-cli/src/deployment_aws_nodes.rs b/harness/service-cli/src/deployment_aws_nodes.rsindex 41651ae6..0577f3e1 100644--- a/harness/service-cli/src/deployment_aws_nodes.rs+++ b/harness/service-cli/src/deployment_aws_nodes.rs@@ -9,7 +9,7 @@ pub(super) fn permissions(p: &Plan) -> Value {let via_template = json!({"ArnLike":{"ec2:LaunchTemplate":template},"StringEquals":{"aws:RequestedRegion":region}});let id = &p.deployment_id;json!({"Version":"2012-10-17","Statement":[- {"Sid":"TaggedWorkerInstances","Effect":"Allow","Action":"ec2:RunInstances","Resource":ec2("instance"),"Condition":{"StringEquals":{"aws:RequestedRegion":region,"aws:RequestTag/ows:deployment-id":id,"ec2:InstanceType":"t4g.large"},"ArnLike":{"ec2:LaunchTemplate":template}}},+ {"Sid":"TaggedWorkerInstances","Effect":"Allow","Action":"ec2:RunInstances","Resource":ec2("instance"),"Condition":{"StringEquals":{"aws:RequestedRegion":region,"aws:RequestTag/ows:deployment-id":id,"ec2:InstanceType":p.infrastructure.as_ref().unwrap().node_instance_type},"ArnLike":{"ec2:LaunchTemplate":template}}},{"Sid":"TaggedWorkerDisks","Effect":"Allow","Action":"ec2:RunInstances","Resource":ec2("volume"),"Condition":{"StringEquals":{"aws:RequestedRegion":region,"aws:RequestTag/ows:deployment-id":id},"ArnLike":{"ec2:LaunchTemplate":template}}},{"Sid":"OwnedWorkerTemplate","Effect":"Allow","Action":"ec2:RunInstances","Resource":template,"Condition":{"StringEquals":{"ec2:ResourceTag/ows:deployment-id":id}}},{"Sid":"OwnedWorkerSubnets","Effect":"Allow","Action":"ec2:RunInstances","Resource":ec2("subnet"),"Condition":{"StringEquals":{"ec2:ResourceTag/ows:deployment-id":id},"ArnLike":{"ec2:LaunchTemplate":template}}},diff --git a/harness/service-cli/src/deployment_azure.rs b/harness/service-cli/src/deployment_azure.rsindex 457af30b..fa4ccb7d 100644--- a/harness/service-cli/src/deployment_azure.rs+++ b/harness/service-cli/src/deployment_azure.rs@@ -9,6 +9,8 @@ mod graph;mod installation;#[path = "deployment_azure_lifecycle.rs"]mod recovery;+#[path = "deployment_up_azure_bootstrap.rs"]+mod up_bootstrap;pub(crate) use installation::{database as installation_database, intent as installation_intent,storage as installation_storage, verify as verify_installation, Binding as InstallationBinding,@@ -247,3 +249,7 @@ pub(super) fn outside(out: &Path, input: &Path) -> Result<()> {);Ok(())}++pub(super) fn up_grants(p: &Plan) -> Result<Value> {+ up_bootstrap::grants(p)+}diff --git a/harness/service-cli/src/deployment_azure_api.rs b/harness/service-cli/src/deployment_azure_api.rsindex b7eb7f1c..8f91f081 100644--- a/harness/service-cli/src/deployment_azure_api.rs+++ b/harness/service-cli/src/deployment_azure_api.rs@@ -61,6 +61,32 @@ impl Azure {fixture_origin: None,})}+ pub(super) async fn check_setup_identity(&self, p: &Plan) -> Result<()> {+ use base64::Engine;+ let token = self.token("https://management.azure.com/").await?;+ let part = token+ .split('.')+ .nth(1)+ .context("Azure caller claims unavailable")?;+ let bytes = Zeroizing::new(+ base64::engine::general_purpose::URL_SAFE_NO_PAD+ .decode(part)+ .context("Invalid caller claims")?,+ );+ let claims: Value = serde_json::from_slice(&bytes).context("Invalid caller claims")?;+ // Identity pinning on a token obtained from the trusted CLI; ARM verifies+ // its signature. This is not an independent token authentication API.+ ensure!(+ claims["tid"] == self.tenant+ && claims["oid"]+ == p.azure_bootstrap+ .as_ref()+ .context("Setup recipient missing")?+ .setup_object_id,+ "Azure login is not the exact reviewed setup principal/tenant"+ );+ Ok(())+ }async fn token(&self, resource: &str) -> Result<Zeroizing<String>> {#[cfg(test)]if self.fixture_origin.is_some() {@@ -342,7 +368,7 @@ async fn cli_token_response(}#[cfg(test)]-mod tests {+pub(super) mod tests {use super::*;use reqwest::Method;use std::{@@ -388,7 +414,7 @@ printf '%s' '{{"tenant":"{tenant}","subscription":"expected-sub","accessToken":"assert!(error.contains("Token target mismatch"));assert!(!error.contains("fixture-secret-canary"));}- fn fixture(+ pub(in super::super) fn fixture(responses: Vec<Option<String>>,) -> (Azure, Arc<Mutex<Vec<String>>>, std::thread::JoinHandle<()>) {let listener = TcpListener::bind("127.0.0.1:0").unwrap();diff --git a/harness/service-cli/src/deployment_azure_infrastructure.rs b/harness/service-cli/src/deployment_azure_infrastructure.rsindex 4341ea2b..79838c6a 100644--- a/harness/service-cli/src/deployment_azure_infrastructure.rs+++ b/harness/service-cli/src/deployment_azure_infrastructure.rs@@ -502,7 +502,7 @@ pub(crate) fn artifacts(p: &Plan) -> Result<BTreeMap<String, Vec<u8>>> {"Microsoft.ContainerService/managedClusters","2024-10-01",&format!("{}-aks", n.prefix),- json!({"dnsPrefix":n.prefix,"kubernetesVersion":c.kubernetes_version,"enableRBAC":true,"disableLocalAccounts":true,"aadProfile":{"managed":true,"enableAzureRBAC":false,"adminGroupObjectIDs":c.admin_group_object_ids,"tenantID":tenant},"apiServerAccessProfile":{"enablePrivateCluster":true,"enablePrivateClusterPublicFQDN":false,"privateDNSZone":"System"},"oidcIssuerProfile":{"enabled":true},"securityProfile":{"workloadIdentity":{"enabled":true}},"linuxProfile":{"adminUsername":"owsadmin","ssh":{"publicKeys":[{"keyData":c.ssh_public_key}]}},"agentPoolProfiles":[{"name":"system","count":2,"vmSize":c.node_vm_size,"osDiskSizeGB":64,"osType":"Linux","osSKU":"Ubuntu","mode":"System","type":"VirtualMachineScaleSets","enableAutoScaling":false,"vnetSubnetID":aks_subnet}],"networkProfile":{"networkPlugin":"azure","networkPolicy":"azure","serviceCidr":"10.73.0.0/24","dnsServiceIP":"10.73.0.10","loadBalancerSku":"standard","outboundType":"loadBalancer"},"identityProfile":{"kubeletidentity":{"resourceId":kubelet,"clientId":format!("[reference('{kubelet}', '2023-01-31').clientId]"),"objectId":principal(&kubelet)}}}),+ json!({"dnsPrefix":n.prefix,"kubernetesVersion":c.kubernetes_version,"enableRBAC":true,"disableLocalAccounts":true,"aadProfile":{"managed":true,"enableAzureRBAC":false,"adminGroupObjectIDs":c.admin_group_object_ids,"tenantID":tenant},"apiServerAccessProfile":{"enablePrivateCluster":true,"enablePrivateClusterPublicFQDN":false,"privateDNSZone":"System"},"oidcIssuerProfile":{"enabled":true},"securityProfile":{"workloadIdentity":{"enabled":true}},"linuxProfile":{"adminUsername":"owsadmin","ssh":{"publicKeys":[{"keyData":c.ssh_public_key}]}},"agentPoolProfiles":[{"name":"system","count":p.topology.node_count,"vmSize":c.node_vm_size,"osDiskSizeGB":64,"osType":"Linux","osSKU":"Ubuntu","mode":"System","type":"VirtualMachineScaleSets","enableAutoScaling":false,"vnetSubnetID":aks_subnet}],"networkProfile":{"networkPlugin":"azure","networkPolicy":"azure","serviceCidr":"10.73.0.0/24","dnsServiceIP":"10.73.0.10","loadBalancerSku":"standard","outboundType":"loadBalancer"},"identityProfile":{"kubeletidentity":{"resourceId":kubelet,"clientId":format!("[reference('{kubelet}', '2023-01-31').clientId]"),"objectId":principal(&kubelet)}}}),);// This bounded, expiring trial has no quota/budget for upgrade surge nodes.// Long-lived installations must choose their own patch/upgrade policy.@@ -606,6 +606,14 @@ pub(crate) fn artifacts(p: &Plan) -> Result<BTreeMap<String, Vec<u8>>> {assignment_actions.push(json!({"scope":resource_id,"actions":["Microsoft.ManagedIdentity/userAssignedIdentities/assign/action"],"recipientObjectId":p.azure_bootstrap.as_ref().unwrap().setup_object_id}));}let review=format!("# Azure ARM graph (experimental, live-unqualified)\n\nTarget `{subscription}` / `{location}`. Workload scope `{}`. Independent control scope `{}`. Both groups already exist and remain customer owned.\n\nOrder: bootstrap independent control and verify heartbeat; submit foundation; have a bootstrap administrator apply resource-specific bindings; submit cluster; have administrator apply federation. Normal executor role has no RBAC writes. Exported identity-assignment and Key Vault deploy grants are separate customer-reviewed prerequisites.\n\nTwo bounded AMD64 Ubuntu nodes; private AKS management API, PostgreSQL and Blob. ACR Basic remains authenticated public HTTPS (no admin credentials). Existing private network access and DNS are needed by the installer. Fixed network ranges must not overlap the customer's network. Admin group memberships/Entra privileges are customer owned.\n\nThe PostgreSQL secret is a pinned Key Vault reference, never plaintext. Existing vault requires template-deployment enabled and exact vault `Microsoft.KeyVault/vaults/deploy/action` for the deployment caller. ARM resolves the secret; no value is exported.\n\nResource tags and names are bookkeeping, not ownership authority. Cleanup uses exact IDs + persisted nonce and immutable creation receipts; missing receipts stop deletion. The AKS provider manages its node RG; we never delete arbitrary `MC_*` groups. No customer RG delete, wildcard role assignment, account key/SAS, Entra app creation or broad subscription grant is exported.\n\nThis export is not a production readiness certificate. Region/version/SKU/quota/price, Azure Policy/locks/effective IAM, actual API schema validation, private routes, cleanup and AKS/OpenShell runtime compatibility need a separately authorized live trial. No paid resource is created by export. Budget alerts/TTL are not a hard cost cap.\n",scope(p),control_scope(p));+ let review = if p.auto_expire.is_some() || p.topology.node_count != 2 {+ review.replace(+ "Two bounded AMD64 Ubuntu nodes;",+ &format!("{} fixed AMD64 Ubuntu nodes;", p.topology.node_count),+ )+ } else {+ review+ };Ok(BTreeMap::from([("azure-foundation.arm.json".into(), json_bytes(&ft)?),("azure-cluster.arm.json".into(), json_bytes(&ct)?),diff --git a/harness/service-cli/src/deployment_azure_installation.rs b/harness/service-cli/src/deployment_azure_installation.rsindex bfe96281..2eef2433 100644--- a/harness/service-cli/src/deployment_azure_installation.rs+++ b/harness/service-cli/src/deployment_azure_installation.rs@@ -6,6 +6,24 @@ use base64::Engine;use reqwest::Method;#[derive(Subcommand)]pub enum Installation {+ /// Explicit digest-bound setup/control grants for the single-command flow.+ Bootstrap {+ #[arg(long)]+ plan: PathBuf,+ #[arg(long)]+ state: PathBuf,+ #[arg(long, value_enum)]+ phase: super::up_bootstrap::UpPhase,+ #[arg(long)]+ accept_plan: String,+ #[arg(long)]+ approve_identities: bool,+ },+ /// Read owned infrastructure material for the single-command installer; no credentials.+ Material {+ #[arg(long)]+ state: PathBuf,+ },/// Observe owned AKS and export a reviewed private Helm handoff; no Kubernetes writes.Prepare {#[arg(long)]@@ -221,17 +239,23 @@ pub(crate) async fn verify(values_path.parent().unwrap(),values_path.file_name().unwrap().to_str().unwrap(),)?;+ let (local_plan, _, _) = super::recovery::up_local(&binding.state)?;+ let actual: Value = serde_json::from_slice(&kube_bytes)?;+ let (actual, original_bytes) = if local_plan.auto_expire.is_some() {+ let original = super::super::up::original_kubeconfig(&actual)?;+ let bytes = json_bytes(&original)?;+ (original, bytes)+ } else {+ (actual, kube_bytes)+ };ensure!(context == binding.context- && hash(&kube_bytes) == binding.kubeconfig_hash+ && hash(&original_bytes) == binding.kubeconfig_hash&& hash(&value_bytes) == binding.values_hash,"Azure installation inputs changed");let (p, _, expected, identity) = observed(&binding.state, provisioning).await?;- let actual: Value = serde_json::from_slice(&super::super::read_file(- path.parent().unwrap(),- path.file_name().unwrap().to_str().unwrap(),- )?)?;+ensure!(context == graph::names(&p).prefix && actual == expected,"Kubeconfig differs from original AKS endpoint/CA/Entra plugin"@@ -285,6 +309,71 @@ pub(crate) fn storage(classes: &Value) -> Result<()> {}pub(crate) async fn install(command: &Installation) -> Result<()> {match command {+ Installation::Bootstrap {+ plan,+ state,+ phase,+ accept_plan,+ approve_identities,+ } => super::up_bootstrap::run(plan, state, *phase, accept_plan, *approve_identities).await,+ Installation::Material { state } => {+ let (p, mut api, cluster, identity) = observed(state, false).await?;+ let names = graph::names(&p);+ let registry_id = graph::id(+ &p,+ "Microsoft.ContainerRegistry/registries",+ &names.registry,+ );+ let registry = api+ .arm(+ Method::GET,+ ®istry_id,+ "2023-07-01",+ None,+ &uuid::Uuid::new_v4().to_string(),+ )+ .await?;+ ensure!(registry.status == 200, "Owned registry is unavailable");+ let registry = registry.body.context("Registry observation missing")?;+ ensure!(+ registry["id"]+ .as_str()+ .is_some_and(|id| id.eq_ignore_ascii_case(®istry_id))+ && registry["properties"]["loginServer"]+ == format!("{}.azurecr.io", names.registry),+ "Owned registry identity mismatch"+ );+ let management_ip = if p+ .azure_infrastructure+ .as_ref()+ .unwrap()+ .management_access+ .is_some()+ {+ let id = graph::id(+ &p,+ "Microsoft.Network/publicIPAddresses",+ &format!("{}-management-ip", names.prefix),+ );+ let r = api+ .arm(+ Method::GET,+ &id,+ "2024-05-01",+ None,+ &uuid::Uuid::new_v4().to_string(),+ )+ .await?;+ ensure!(r.status == 200, "Owned management IP missing");+ r.body.context("Management IP observation missing")?["properties"]["ipAddress"]+ .clone()+ } else {+ Value::Null+ };+ crate::output(+ json!({"deploymentId":p.deployment_id,"cluster":cluster,"managementIp":management_ip,"artifactIdentity":identity,"registry":{"id":registry_id,"name":names.registry,"loginServer":registry["properties"]["loginServer"],"createdAt":registry["properties"]["creationDate"]},"artifactAccount":names.storage,"postgresHost":format!("{}-pg.postgres.database.azure.com",names.prefix),"target":p.target,"credentialsIncluded":false}),+ )+ }Installation::Prepare { state, config, out } => {outside(out, state)?;let mut c: Value = serde_json::from_slice(&super::super::read_file(diff --git a/harness/service-cli/src/deployment_azure_lifecycle.rs b/harness/service-cli/src/deployment_azure_lifecycle.rsindex 02a4c644..532b112f 100644--- a/harness/service-cli/src/deployment_azure_lifecycle.rs+++ b/harness/service-cli/src/deployment_azure_lifecycle.rs@@ -713,6 +713,7 @@ async fn apply(}}async fn destroy_one(c: &mut impl Cloud, s: &mut State, e: &mut String, at: u64) -> Result<()> {+ ensure!(s.plan.expires_workload(), "Persistent deployment data is retained. Review a separate data-retention/teardown plan before destruction.");if !s.destroy_requested {s.destroy_requested = true;save(c, s, e).await?;@@ -861,7 +862,7 @@ async fn heartbeat(c: &mut impl Cloud, s: &mut State, e: &mut String, at: u64) -s.heartbeat_unix = Some(at);save(c, s, e).await?;if s.completed_unix.is_none() {- if at >= s.plan.expires_at_unix || s.destroy_requested {+ if (s.plan.expires_workload() && at >= s.plan.expires_at_unix) || s.destroy_requested {destroy_one(c, s, e, at).await?;} else {observe(c, s, e).await?;@@ -945,6 +946,45 @@ async fn initialize(c: &mut impl Cloud, s: &mut State) -> Result<()> {Ok(())}+pub(super) fn up_package(root: &Path, out: &Path) -> Result<()> {+ let s = local(root)?;+ let files = control_artifacts(&s)?;+ private_dir(out)?;+ for (name, bytes) in files {+ put(out, &name, &bytes)?;+ }+ Ok(())+}+pub(super) async fn up_initialize_control(root: &Path) -> Result<bool> {+ let mut expected = local(root)?;+ let mut api = Azure::new(&expected.plan, None)?;+ let path = root.join("up-operations/control-init.json");+ if let Some((_etag, shared)) = api.read_control(&control_url(&expected.plan)).await? {+ ensure!(+ shared.nonce == expected.nonce && shared.bundle_digest == expected.bundle_digest,+ "Shared cleanup identity mismatch"+ );+ verify_job(&mut api, &shared).await?;+ super::super::up::write_private(+ &path,+ &json!({"phase":"observed","nonce":shared.nonce,"bundleDigest":shared.bundle_digest}),+ )?;+ return Ok(shared+ .heartbeat_unix+ .is_some_and(|at| now().is_ok_and(|n| at <= n && n - at <= 600)));+ }+ ensure!(!path.exists(),"Control initialization outcome unknown; observe original Blob instead of replaying creation");+ super::super::up::write_private(+ &path,+ &json!({"phase":"intent","nonce":expected.nonce,"bundleDigest":expected.bundle_digest}),+ )?;+ initialize(&mut api, &mut expected).await?;+ Ok(false)+}+pub(super) fn up_local(root: &Path) -> Result<(Plan, String, String)> {+ let s = local(root)?;+ Ok((s.plan, s.bundle_digest, s.nonce))+}pub(crate) async fn lifecycle(command: &Lifecycle) -> Result<()> {match command {Lifecycle::Prepare { plan, out } => {@@ -1815,6 +1855,22 @@ mod tests {assert!(destroy_one(&mut c, &mut s, &mut e, 160).await.is_err());assert_eq!(c.deletes(), 0);}+ #[tokio::test]+ async fn persistent_workload_is_not_deleted_at_provisioning_deadline() {+ let mut s = state();+ s.plan.auto_expire = Some(false);+ s.bundle_digest = digest(&s.plan).unwrap();+ let mut c = Fake::new(s.clone());+ let mut e = c.etag();+ apply(&mut c, &mut s, &mut e, Stage::Foundation, 150)+ .await+ .unwrap();+ heartbeat(&mut c, &mut s, &mut e, 500).await.unwrap();+ assert_eq!(c.deletes(), 0);+ assert!(destroy_one(&mut c, &mut s, &mut e, 500).await.is_err());+ assert_eq!(c.deletes(), 0);+ }+#[tokio::test]async fn expiry_runs_outside_cluster_and_stops_its_job_after_workload_cleanup() {let (mut c, mut s, mut e) = active().await;diff --git a/harness/service-cli/src/deployment_cloud.rs b/harness/service-cli/src/deployment_cloud.rsindex d9edd7f6..b27099e6 100644--- a/harness/service-cli/src/deployment_cloud.rs+++ b/harness/service-cli/src/deployment_cloud.rs@@ -14,6 +14,8 @@ mod install;mod platform_artifact;#[path = "deployment_cloud_state.rs"]mod state;+#[path = "deployment_up_aws_bootstrap.rs"]+mod up_bootstrap;pub(super) use install::{database_url, enable_strict_network, install_scope, installation_intent, verify_installation,verify_network, Binding,@@ -22,6 +24,19 @@ use state::{Journal, Provider, Stack};#[derive(Subcommand)]pub enum Command {+ /// Explicit, digest-bound identity-stack bootstrap. Normal deployment does not grant IAM.+ Bootstrap {+ #[arg(long)]+ plan: PathBuf,+ #[arg(long)]+ state: PathBuf,+ #[arg(long)]+ profile: String,+ #[arg(long)]+ accept_plan: String,+ #[arg(long)]+ approve_identities: bool,+ },PlatformArtifact {#[arg(long)]plan: PathBuf,@@ -516,7 +531,7 @@ impl Provider for Aws {}async fn guard_health(&mut self) -> Result<()> {ensure!(- self.p.infrastructure.as_ref().unwrap().guard_code.sha256 == guard::archive_sha(),+ guard::compatible(&self.p),"Guard must match this CLI's locally qualified bundled package");let Target::Aws { account, region } = &self.p.target else {@@ -699,6 +714,17 @@ fn fresh_preflight(path: &Path, p: &Plan, j: &Journal) -> Result<()> {Ok(())}pub(super) async fn run(command: &Command) -> Result<()> {+ if let Command::Bootstrap {+ plan,+ state,+ profile,+ accept_plan,+ approve_identities,+ } = command+ {+ return up_bootstrap::bootstrap(plan, state, profile, accept_plan, *approve_identities)+ .await;+ }if let Command::PlatformArtifact {plan,out,@@ -744,9 +770,7 @@ pub(super) async fn run(command: &Command) -> Result<()> {if let Command::Prepare { plan, out } = command {let (p, _) = inspect(plan)?;ensure!(- p.infrastructure- .as_ref()- .is_some_and(|i| i.guard_code.sha256 == guard::archive_sha()),+ guard::compatible(&p),"Pin this CLI's packaged guard before preparing lifecycle execution");let j = state::init(plan, out)?;diff --git a/harness/service-cli/src/deployment_cloud_install.rs b/harness/service-cli/src/deployment_cloud_install.rsindex b80ae308..4649c039 100644--- a/harness/service-cli/src/deployment_cloud_install.rs+++ b/harness/service-cli/src/deployment_cloud_install.rs@@ -129,6 +129,11 @@ pub(crate) async fn verify_installation(path.parent().unwrap(),path.file_name().unwrap().to_str().unwrap(),)?)?;+ let content = if p.auto_expire.is_some() {+ super::super::up::original_kubeconfig(&content)?+ } else {+ content+ };ensure!(context == p.prefix() && content == kubeconfig(&p, &binding.profile, &c),"Kubeconfig must match the original private cluster endpoint, CA and setup profile"diff --git a/harness/service-cli/src/deployment_cloud_state.rs b/harness/service-cli/src/deployment_cloud_state.rsindex ac6a3a95..62e2aff5 100644--- a/harness/service-cli/src/deployment_cloud_state.rs+++ b/harness/service-cli/src/deployment_cloud_state.rs@@ -362,6 +362,7 @@ pub(super) async fn destroy<P: Provider>(p: &Plan,api: &mut P,) -> Result<()> {+ ensure!(p.expires_workload(), "Persistent deployment data is retained. Review a separate data-retention/teardown plan before destruction.");let key = "create:infra";let Some(mut op) = j.operations.get(key).cloned() else {bail!("Infrastructure has no original intent; inspect incomplete bootstrap")diff --git a/harness/service-cli/src/deployment_guard.rs b/harness/service-cli/src/deployment_guard.rsindex bbdf7e18..3058d076 100644--- a/harness/service-cli/src/deployment_guard.rs+++ b/harness/service-cli/src/deployment_guard.rs@@ -63,6 +63,17 @@ fn zip(name: &[u8], data: &[u8]) -> Vec<u8> {u16b(&mut v, 0);v}+// Previously qualified trial archive (source SHA256 abaeac734788348ce3b5f2d9de9faec429fac4cb348ceedc75bd9e8d368c0743).+// Old journals may observe/finish their original guard; new persistent plans+// require the current package so an old guard cannot ignore auto-expiry policy.+pub(super) fn compatible(p: &Plan) -> bool {+ p.infrastructure.as_ref().is_some_and(|i| {+ i.guard_code.sha256 == archive_sha()+ || (p.auto_expire.is_none()+ && i.guard_code.sha256+ == "f3adf51e0bea8279c36788b5740964788d9c1e1fa4d16ef9c306f2b5038b216a")+ })+}pub(super) fn archive_sha() -> String {hash(&zip(b"index.js",diff --git a/harness/service-cli/src/deployment_infrastructure.rs b/harness/service-cli/src/deployment_infrastructure.rsindex a52f3d47..9e58dd99 100644--- a/harness/service-cli/src/deployment_infrastructure.rs+++ b/harness/service-cli/src/deployment_infrastructure.rs@@ -12,6 +12,8 @@ mod permissions;#[serde(rename_all = "camelCase", deny_unknown_fields)]pub(super) struct Inputs {pub schema_version: u16,+ #[serde(default = "default_node_type", skip_serializing_if = "default_node")]+ pub node_instance_type: String,pub availability_zones: [String; 2],pub vpc_cidr: String,pub kubernetes_version: String,@@ -51,17 +53,35 @@ pub(super) fn addon_version(s: &str) -> bool {parts.len() == 3 && parts.iter().all(|p| digits(p, 1, 4)) && digits(build, 1, 4)}+fn default_node_type() -> String {+ "t4g.large".into()+}+fn default_node(s: &str) -> bool {+ s == "t4g.large"+}+pub(super) fn worker_shape(s: &str) -> Option<(u32, u32)> {+ match s {+ "t4g.large" => Some((2, 8)),+ "t4g.xlarge" => Some((4, 16)),+ "t4g.2xlarge" => Some((8, 32)),+ _ => None,+ }+}impl Inputs {pub(super) fn validate(&self, p: &Plan) -> Result<()> {let Target::Aws { region, .. } = &p.target else {bail!("AWS infrastructure inputs require AWS target")};+ ensure!(+ worker_shape(&self.node_instance_type).is_some(),+ "Unsupported reviewed ARM64 worker size"+ );ensure!(self.schema_version == 1,"Unsupported infrastructure input schema");ensure!(- self.acknowledge_experiment_data_deletion,+ !p.expires_workload() || self.acknowledge_experiment_data_deletion,"Explicit experiment data-deletion acknowledgement required");ensure!(@@ -342,8 +362,8 @@ fn infrastructure_template(p: &Plan, i: &Inputs) -> Result<Value> {add("Nodes","AWS::EKS::Nodegroup",- json!({"ClusterName":reference("Cluster"),"NodegroupName":"workers","NodeRole":role_arn(p,"node"),"Subnets":[reference("PrivateSubnet0"),reference("PrivateSubnet1")],"AmiType":"AL2023_ARM_64_STANDARD","ReleaseVersion":i.node_release_version,"Version":i.kubernetes_version,"InstanceTypes":["t4g.large"],"CapacityType":"ON_DEMAND",- "ScalingConfig":{"MinSize":2,"MaxSize":2,"DesiredSize":2},"LaunchTemplate":{"Id":reference("LaunchTemplate"),"Version":attribute("LaunchTemplate","LatestVersionNumber")},"Tags":{"ows:deployment-id":p.deployment_id,"ows:expires-at":p.expires_at_unix.to_string()}}),+ json!({"ClusterName":reference("Cluster"),"NodegroupName":"workers","NodeRole":role_arn(p,"node"),"Subnets":[reference("PrivateSubnet0"),reference("PrivateSubnet1")],"AmiType":"AL2023_ARM_64_STANDARD","ReleaseVersion":i.node_release_version,"Version":i.kubernetes_version,"InstanceTypes":[i.node_instance_type],"CapacityType":"ON_DEMAND",+ "ScalingConfig":{"MinSize":p.topology.node_count,"MaxSize":p.topology.node_count,"DesiredSize":p.topology.node_count},"LaunchTemplate":{"Id":reference("LaunchTemplate"),"Version":attribute("LaunchTemplate","LatestVersionNumber")},"Tags":{"ows:deployment-id":p.deployment_id,"ows:expires-at":p.expires_at_unix.to_string()}}),);for (logical, name, version) in [("Cni", "vpc-cni", &i.addons.vpc_cni),@@ -510,6 +530,10 @@ fn guard_template(p: &Plan, i: &Inputs) -> Value {),]);r.get_mut("Cleanup").unwrap()["DependsOn"] = json!(["Logs", "Control"]);+ if let Some(auto_expire) = p.auto_expire {+ r.get_mut("Cleanup").unwrap()["Properties"]["Environment"]["Variables"]+ ["OWS_AUTO_EXPIRE"] = json!(auto_expire.to_string());+ }template("OpenWorker independent expiry guard; artifact and cleanup behavior require qualification",r,@@ -549,6 +573,14 @@ pub(super) fn render(p: &Plan) -> Result<BTreeMap<String, Vec<u8>>> {json!({"Sid":"RegistryLogin","Effect":"Allow","Action":["ecr:GetAuthorizationToken"],"Resource":"*","Condition":{"StringEquals":{"aws:RequestedRegion":region},"NumericLessThan":{"aws:EpochTime":p.expires_at_unix}}}),json!({"Sid":"PublishOwnImages","Effect":"Allow","Action":["ecr:BatchCheckLayerAvailability","ecr:GetDownloadUrlForLayer","ecr:BatchGetImage","ecr:DescribeRepositories","ecr:InitiateLayerUpload","ecr:UploadLayerPart","ecr:CompleteLayerUpload","ecr:PutImage"],"Resource":(["harness","runner","service-ui"].iter().map(|s|format!("arn:aws:ecr:{region}:{account}:repository/{}/{s}",p.prefix())).collect::<Vec<_>>()),"Condition":{"NumericLessThan":{"aws:EpochTime":p.expires_at_unix}}}),]);+ if p.auto_expire.is_some() && i.operator_access.is_some() {+ setup["Statement"].as_array_mut().unwrap().extend([+ json!({"Sid":"StartOwnedOperatorTunnel","Effect":"Allow","Action":"ssm:StartSession","Resource":format!("arn:aws:ec2:{region}:{account}:instance/*"),"Condition":{"StringEquals":{"ssm:resourceTag/ows:deployment-id":p.deployment_id}}}),+ json!({"Sid":"UsePortForwardDocument","Effect":"Allow","Action":"ssm:StartSession","Resource":format!("arn:aws:ssm:{region}::document/AWS-StartPortForwardingSessionToRemoteHost")}),+ json!({"Sid":"ObserveManagementSessions","Effect":"Allow","Action":"ssm:DescribeSessions","Resource":"*","Condition":{"StringEquals":{"aws:RequestedRegion":region}}}),+ json!({"Sid":"OwnManagementSessionOnly","Effect":"Allow","Action":["ssm:ResumeSession","ssm:TerminateSession","ssmmessages:OpenDataChannel"],"Resource":format!("arn:aws:ssm:{region}:{account}:session/*"),"Condition":{"StringEquals":{"ssm:resourceTag/aws:ssmmessages:session-id":"${aws:userid}"}}})+ ]);+ }boot["Resources"]["SetupRole"]["Properties"]["Policies"][0]["PolicyDocument"] = setup.clone();files.insert("setup-policy.json".into(), json_bytes(&setup)?);let resources = boot["Resources"].as_object_mut().unwrap();@@ -724,6 +756,22 @@ of the entire bundle. No cloud execution was performed to generate this review.i.availability_zones,policies.len());+ if p.auto_expire.is_some() || p.topology.node_count != 2 || i.node_instance_type != "t4g.large"+ {+ text = text.replace(+ "Two ARM64 t4g.large workers",+ &format!(+ "{} fixed ARM64 {} workers",+ p.topology.node_count, i.node_instance_type+ ),+ );+ if !p.expires_workload() {+ text = text.replace(+ "Fixed expiry",+ "Provisioning deadline (persistent data is not automatically deleted)",+ );+ }+ }if let Some(access) = &i.operator_access {text.push_str(&format!("\n## Optional SSM operator bridge\n\nOne private t4g.micro, pinned AMI {}, encrypted 8 GiB disposable disk, no public IP, no inbound ports or SSH. Its instance profile permits only regional SSM heartbeat/session channels, no parameter, database, artifact or model-secret reads. The main infrastructure stack owns the instance/security groups, so normal and expiry teardown include them. Bootstrap owns the role/profile; remove those last. The existing operator login must authorize Session Manager port forwarding; it is separate from workload identity. The added instance/storage/vCPUs are included in preflight cost/quota checks.\n",access.ami_id));}diff --git a/harness/service-cli/src/deployment_local.rs b/harness/service-cli/src/deployment_local.rsindex 6a774781..3a56a3cd 100644--- a/harness/service-cli/src/deployment_local.rs+++ b/harness/service-cli/src/deployment_local.rs@@ -165,7 +165,16 @@ struct Client<'a> {c: &'a Config,}pub(super) async fn exec(bin: &Path, a: &[String], input: Option<Vec<u8>>) -> Result<Vec<u8>> {+ exec_with_env(bin, a, input, &BTreeMap::new()).await+}+pub(super) async fn exec_with_env(+ bin: &Path,+ a: &[String],+ input: Option<Vec<u8>>,+ env: &BTreeMap<String, String>,+) -> Result<Vec<u8>> {let mut child = Process::new(bin)+ .envs(env).args(a).stdin(Stdio::piped()).stdout(Stdio::piped())@@ -188,6 +197,7 @@ pub(super) async fn exec(bin: &Path, a: &[String], input: Option<Vec<u8>>) -> Rebytes(stderr, 65536),async move {if let Some(v) = input {+ let v = zeroize::Zeroizing::new(v);stdin.write_all(&v).await?;}drop(stdin);@@ -1283,7 +1293,7 @@ pub(super) async fn verify_teardown("Installation belongs to a different cloud lifecycle/profile");ensure!(- read_json(&p.config.kubeconfig)? == *kubeconfig,+ super::up::original_kubeconfig(&read_json(&p.config.kubeconfig)?)? == *kubeconfig,"Installation cluster endpoint/CA/profile changed");let client = Client { c: &p.config };diff --git a/harness/service-cli/src/deployment_preflight_checks.rs b/harness/service-cli/src/deployment_preflight_checks.rsindex 11fb9bc1..3e42ad1c 100644--- a/harness/service-cli/src/deployment_preflight_checks.rs+++ b/harness/service-cli/src/deployment_preflight_checks.rs@@ -135,7 +135,7 @@ async fn zones(aws: &mut Aws, p: &Plan) -> Observed<Value> {"--location-type","availability-zone","--filters",- "Name=instance-type,Values=t4g.large",+ &format!("Name=instance-type,Values={}", i.node_instance_type),&filter,]),"InstanceTypeOfferings",@@ -145,11 +145,11 @@ async fn zones(aws: &mut Aws, p: &Plan) -> Observed<Value> {expect(offers.iter()- .any(|z| z["InstanceType"] == "t4g.large" && z["Location"] == *zone),+ .any(|z| z["InstanceType"] == i.node_instance_type && z["Location"] == *zone),)?;}Ok(- json!({"zones":i.availability_zones,"instanceType":"t4g.large","offered":true,"capacityReserved":false}),+ json!({"zones":i.availability_zones,"instanceType":i.node_instance_type,"offered":true,"capacityReserved":false}),)}async fn operator_access(aws: &mut Aws, p: &Plan) -> Observed<Value> {@@ -482,11 +482,16 @@ async fn quotas(aws: &mut Aws, p: &Plan) -> Observed<Value> {}// Counting every non-host On-Demand family is conservative: some families// consume separate quotas. Never understate usage by assuming their count zero.- let requested = if p.infrastructure.as_ref().unwrap().operator_access.is_some() {- 6- } else {- 4- };+ let inputs = p.infrastructure.as_ref().unwrap();+ let cores = infrastructure::worker_shape(&inputs.node_instance_type)+ .ok_or_else(invalid)?+ .0;+ let requested = cores * u32::from(p.topology.node_count)+ + if inputs.operator_access.is_some() {+ 2+ } else {+ 0+ };expect(limit >= used + f64::from(requested))?;let cluster_limit = quota(aws, "eks", "L-1194D53C").await?;let clusters = aws@@ -526,8 +531,8 @@ async fn quotas(aws: &mut Aws, p: &Plan) -> Observed<Value> {)}#[derive(Clone, Copy)]-enum Product {- Node(&'static str),+enum Product<'a> {+ Node(&'a str),Database,Cluster,Nat,@@ -535,7 +540,7 @@ enum Product {Ebs,RdsStorage,}-async fn price(aws: &mut Aws, p: &Plan, kind: Product) -> Observed<Value> {+async fn price(aws: &mut Aws, p: &Plan, kind: Product<'_>) -> Observed<Value> {let region = target(p).1;let (service, filters, unit) = match kind {Product::Node(instance) => (@@ -700,12 +705,20 @@ async fn price(aws: &mut Aws, p: &Plan, kind: Product) -> Observed<Value> {async fn cost(aws: &mut Aws, p: &Plan, allowance: u32) -> Observed<Value> {let mut components = BTreeMap::new();let mut lines = vec![- ("workerInstances", Product::Node("t4g.large"), 2.0),+ (+ "workerInstances",+ Product::Node(&p.infrastructure.as_ref().unwrap().node_instance_type),+ f64::from(p.topology.node_count),+ ),("databaseInstance", Product::Database, 1.0),("eksStandardCluster", Product::Cluster, 1.0),("natGateway", Product::Nat, 1.0),("natPublicIpv4", Product::Ip, 1.0),- ("nodeGp3Storage", Product::Ebs, 60.0),+ (+ "nodeGp3Storage",+ Product::Ebs,+ 30.0 * f64::from(p.topology.node_count),+ ),("databaseGp3Storage", Product::RdsStorage, 20.0),];if p.infrastructure.as_ref().unwrap().operator_access.is_some() {diff --git a/harness/service-cli/src/deployment_up.rs b/harness/service-cli/src/deployment_up.rsnew file mode 100644index 00000000..7c41c069--- /dev/null+++ b/harness/service-cli/src/deployment_up.rs@@ -0,0 +1,408 @@+//! One entry point over the existing provider lifecycle and shared installer.+//! Desired state, reviewed observations and effect receipts remain distinct.+use super::*;+#[path = "deployment_up_config.rs"]+mod config;+#[path = "deployment_up_native.rs"]+mod native;+#[path = "deployment_up_package.rs"]+mod package;+#[path = "deployment_up_pricing.rs"]+mod pricing;+#[path = "deployment_up_resolve.rs"]+mod resolve;+#[path = "deployment_up_state.rs"]+mod state;+use config::Config;++#[derive(Args)]+pub struct Options {+ #[arg(long)]+ config: PathBuf,+ /// Private journal directory; defaults to .ows/deployments/<target-and-name hash>.+ #[arg(long)]+ state: Option<PathBuf>,+ /// Resolve and write the review without provisioning.+ #[arg(long)]+ plan_only: bool,+ /// Noninteractive approval of this exact review digest, never a blanket yes.+ #[arg(long)]+ accept_plan: Option<String>,+ /// Separate approval of scoped identity grants in the exact reviewed plan.+ #[arg(long)]+ accept_bootstrap: Option<String>,+ #[arg(long, default_value_t = 3600)]+ timeout_seconds: u64,+ /// Keep the private management route open after installation; Ctrl-C closes it.+ #[arg(long)]+ keep_access: bool,+}+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+struct Review {+ schema_version: u16,+ config: Config,+ config_digest: String,+ deployment_id: String,+ resolution: resolve::Resolution,+ package_digest: String,+ native_bundle_digest: String,+ digest: String,+}+impl Review {+ fn digest(&self) -> Result<String> {+ let mut v = serde_json::to_value(self)?;+ v.as_object_mut().unwrap().remove("digest");+ Ok(hash(&serde_json::to_vec(&v)?))+ }+ fn validate(&self, c: &Config) -> Result<()> {+ ensure!(self.schema_version==1 && self.config==*c && self.config_digest==c.digest()? && self.digest==self.digest()?,"Deployment configuration/review changed. Review an explicit new deployment; do not overwrite the original journal.");+ self.resolution.validate(c)?;+ Ok(())+ }+}+pub(super) fn write_private(path: &Path, value: &impl Serialize) -> Result<()> {+ use std::io::Write;+ let temp = path.with_extension(format!("{}.tmp", uuid::Uuid::new_v4().simple()));+ let mut options = fs::OpenOptions::new();+ options.write(true).create_new(true);+ #[cfg(unix)]+ {+ use std::os::unix::fs::OpenOptionsExt;+ options.mode(0o600);+ }+ let mut f = options.open(&temp)?;+ f.write_all(&json_bytes(value)?)?;+ f.sync_all()?;+ fs::rename(&temp, path)?;+ #[cfg(unix)]+ fs::File::open(path.parent().context("Receipt parent missing")?)?.sync_all()?;+ Ok(())+}+pub(super) fn directory(path: &Path) -> Result<()> {+ if path.exists() {+ let m = fs::symlink_metadata(path)?;+ ensure!(+ m.is_dir() && !m.file_type().is_symlink(),+ "Deployment state must be a directory, not a symlink"+ );+ #[cfg(unix)]+ {+ use std::os::unix::fs::PermissionsExt;+ ensure!(+ m.permissions().mode() & 0o077 == 0,+ "Deployment state must be private (chmod 700)"+ );+ }+ } else {+ let mut b = fs::DirBuilder::new();+ b.recursive(true);+ #[cfg(unix)]+ {+ use std::os::unix::fs::DirBuilderExt;+ b.mode(0o700);+ }+ b.create(path)?;+ }+ Ok(())+}+/// Remove only a local transport route for comparison with the provider's+/// original kubeconfig. Hostname, CA, TLS options and authentication stay intact.+pub(super) fn original_kubeconfig(value: &Value) -> Result<Value> {+ let mut original = value.clone();+ for cluster in original["clusters"]+ .as_array_mut()+ .context("Kubeconfig clusters missing")?+ {+ let c = cluster["cluster"]+ .as_object_mut()+ .context("Cluster definition missing")?;+ if let Some(proxy) = c.remove("proxy-url") {+ let url = reqwest::Url::parse(proxy.as_str().context("Invalid local proxy")?)?;+ ensure!(+ url.scheme() == "http"+ && url.host_str() == Some("127.0.0.1")+ && url.port().is_some_and(|p| p >= 1024)+ && url.username().is_empty()+ && url.password().is_none()+ && url.path() == "/"+ && url.query().is_none()+ && url.fragment().is_none(),+ "Only the loopback management transport may differ from the original kubeconfig"+ );+ }+ }+ Ok(original)+}+pub(super) fn operation_lock(root: &Path, name: &str) -> Result<fs::File> {+ directory(root)?;+ let path = root.join(name);+ if path.exists() {+ ensure!(+ !fs::symlink_metadata(&path)?.file_type().is_symlink(),+ "Operation lock cannot be a symlink"+ );+ }+ let mut options = fs::OpenOptions::new();+ options.create(true).truncate(false).write(true);+ #[cfg(unix)]+ {+ use std::os::unix::fs::OpenOptionsExt;+ options.mode(0o600);+ }+ let f = options.open(path)?;+ f.try_lock()+ .context("Another command owns this operation journal")?;+ Ok(f)+}+pub(super) async fn run(provider: &str, o: &Options) -> Result<()> {+ let config = Config::load(&o.config, provider)?;+ let package = package::load(&config)?;+ ensure!(+ (10..=14400).contains(&o.timeout_seconds),+ "Use timeoutSeconds between 10 and 14400"+ );+ let identity = hash(+ format!(+ "{}:{}:{}:{}",+ provider,+ config.cloud.account(),+ config.cloud.region(),+ config.name+ )+ .as_bytes(),+ );+ let state = o+ .state+ .clone()+ .unwrap_or_else(|| PathBuf::from(".ows/deployments").join(&identity[..24]));+ directory(&state)?;+ let lock = fs::OpenOptions::new()+ .create(true)+ .truncate(false)+ .write(true)+ .open(state.join("up.lock"))?;+ lock.try_lock()+ .context("Another deployment command owns this journal")?;+ let review_path = state.join("review.json");+ let review: Review = if review_path.exists() {+ let value: Review = serde_json::from_slice(&fs::read(&review_path)?)?;+ value.validate(&config)?;+ ensure!(+ value.package_digest == package.digest,+ "Account setup/release changed; do not alter the original deployment review"+ );+ value+ } else {+ let resolution = resolve::live(&config, &package.setup.infrastructure).await?;+ let mut value = Review {+ schema_version: 1,+ config_digest: config.digest()?,+ config: config.clone(),+ deployment_id: format!("ows-{}", uuid::Uuid::new_v4().simple()),+ resolution,+ package_digest: package.digest.clone(),+ native_bundle_digest: String::new(),+ digest: String::new(),+ };+ value.native_bundle_digest = native::Native {+ root: state.canonicalize()?,+ review: &value,+ package: package.clone(),+ bootstrap_approved: false,+ bridge: Default::default(),+ }+ .artifact_digest()?;+ value.digest = value.digest()?;+ value.validate(&config)?;+ write_private(&review_path, &value)?;+ value+ };+ ensure!(+ native::Native {+ root: state.canonicalize()?,+ review: &review,+ package: package.clone(),+ bootstrap_approved: false,+ bridge: Default::default()+ }+ .artifact_digest()?+ == review.native_bundle_digest,+ "Generated infrastructure/permissions changed since review; do not reuse the old approval"+ );+ crate::output(+ json!({"deploymentId":review.deployment_id,"state":state,"planDigest":review.digest,"nodeCount":config.nodes.count,"instanceType":review.resolution.node.instance_type,"price":review.resolution.price,"phase":"review","mutations":0}),+ )?;+ if o.plan_only {+ native::Native {+ root: state.canonicalize()?,+ review: &review,+ package: package.clone(),+ bootstrap_approved: false,+ bridge: Default::default(),+ }+ .review_bundle()+ .await?;+ return Ok(());+ }+ if let Some(subtotal) = review.resolution.price.known_hourly_subtotal_usd {+ let minimum = subtotal * f64::from(review.resolution.price.planned_hours)+ + f64::from(review.resolution.price.variable_allowance_usd);+ ensure!(minimum<=f64::from(config.budget.alert_usd),"Known planning estimate already exceeds budget.limitUSD. Review a new budget/size/lifetime before provisioning; no mutations performed.");+ }+ // Render the exact permission payloads before either approval prompt.+ let preview = native::Native {+ root: state.canonicalize()?,+ review: &review,+ package: package.clone(),+ bootstrap_approved: false,+ bridge: Default::default(),+ };+ preview.review_bundle().await?;+ approve(+ &state,+ &review.digest,+ "provisioning",+ o.accept_plan.as_deref(),+ )?;+ approve(+ &state,+ &review.digest,+ "bootstrap",+ o.accept_bootstrap.as_deref(),+ )?;+ resolve::identity(&config).await?;+ let mut driver = native::Native {+ root: state.canonicalize()?,+ review: &review,+ package,+ bootstrap_approved: true,+ bridge: Default::default(),+ };+ let path = state.join("up-journal.json");+ let mut journal = state::Journal::open(&path, &review)?;+ if journal+ .stages+ .get(&state::Stage::PrivateAccess)+ .is_some_and(|s| s.phase == state::Phase::Complete)+ {+ driver.restore_access().await?;+ }+ let previously_complete = journal.complete();+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(o.timeout_seconds);+ loop {+ let previous = journal+ .stages+ .iter()+ .map(|(stage, s)| (*stage, s.phase))+ .collect::<Vec<_>>();+ if state::advance(&mut journal, &path, &mut driver).await? {+ // Completion receipts are history, not a claim that today’s cluster is healthy.+ if previously_complete {+ driver.verify_current().await?;+ }+ if o.keep_access {+ crate::output(+ json!({"phase":"access_ready","connection":state.join("connection.json"),"close":"Ctrl-C"}),+ )?;+ tokio::signal::ctrl_c().await?;+ }+ driver.close_access().await?;+ return crate::output(+ json!({"deploymentId":review.deployment_id,"phase":"installed","state":state,"connection":state.join("connection.json"),"modelSmokeTest":"not_run"}),+ );+ }+ let current = journal+ .stages+ .iter()+ .map(|(stage, s)| (*stage, s.phase))+ .collect::<Vec<_>>();+ if current != previous {+ eprintln!("Deployment progress: {:?}", current.last());+ }+ ensure!(std::time::Instant::now()<deadline,"Deployment is still pending. Rerun the same up command to resume; original resources and requests were retained.");+ tokio::time::sleep(std::time::Duration::from_secs(5)).await;+ }+}++fn approve(root: &Path, digest: &str, kind: &str, supplied: Option<&str>) -> Result<()> {+ use std::io::{IsTerminal, Write};+ if let Some(value) = supplied {+ ensure!(+ value == digest,+ "Approval digest does not match the reviewed deployment"+ );+ }+ let path = root.join(format!("approval-{kind}.json"));+ if path.exists() {+ let old: Value = serde_json::from_slice(&fs::read(&path)?)?;+ ensure!(+ old["digest"] == digest && old["kind"] == kind,+ "Saved approval belongs to another review"+ );+ return Ok(());+ }+ if supplied.is_none() {+ ensure!(std::io::stdin().is_terminal(), "Review the infrastructure directory, then rerun with --accept-plan <planDigest> and --accept-bootstrap <planDigest>");+ eprint!(+ "Approve {kind} for this exact review? Inspect {}. Type {}: ",+ root.join("infrastructure").display(),+ &digest[..12]+ );+ std::io::stderr().flush()?;+ let mut answer = String::new();+ std::io::stdin().read_line(&mut answer)?;+ ensure!(+ answer.trim() == &digest[..12],+ "Not approved; no cloud mutation performed"+ );+ }+ write_private(+ &path,+ &json!({"digest":digest,"kind":kind,"approvedAtUnix":now()?}),+ )+}+#[cfg(test)]+mod route_tests {+ use super::*;+ #[test]+ fn route_normalization_never_relaxes_cluster_identity_or_authentication() {+ let original = json!({"clusters":[{"cluster":{"server":"https://original.internal","certificate-authority-data":"original-ca"}}],"users":[{"user":{"exec":{"command":"aws"}}}]});+ let mut routed = original.clone();+ routed["clusters"][0]["cluster"]["proxy-url"] = json!("http://127.0.0.1:15000");+ assert_eq!(original_kubeconfig(&routed).unwrap(), original);+ routed["clusters"][0]["cluster"]["certificate-authority-data"] = json!("changed");+ assert_ne!(original_kubeconfig(&routed).unwrap(), original);+ for url in [+ "http://remote.example:15000",+ "https://127.0.0.1:15000",+ "http://[email protected]:15000",+ "http://127.0.0.1:80",+ "http://127.0.0.1:15000/path",+ ] {+ routed["clusters"][0]["cluster"]["proxy-url"] = json!(url);+ assert!(original_kubeconfig(&routed).is_err());+ }+ }+}++#[cfg(test)]+mod approval_tests {+ use super::*;+ #[test]+ fn approval_is_durable_scoped_and_rejects_changed_review() {+ let temp = tempfile::tempdir().unwrap();+ let digest = "a".repeat(64);+ approve(temp.path(), &digest, "provisioning", Some(&digest)).unwrap();+ approve(temp.path(), &digest, "provisioning", None).unwrap();+ assert!(!temp.path().join("approval-bootstrap.json").exists());+ assert!(approve(+ temp.path(),+ &"b".repeat(64),+ "provisioning",+ Some(&"b".repeat(64))+ )+ .is_err());+ assert!(approve(temp.path(), &digest, "provisioning", Some("wrong")).is_err());+ }+}diff --git a/harness/service-cli/src/deployment_up_access.rs b/harness/service-cli/src/deployment_up_access.rsnew file mode 100644index 00000000..bbc308f8--- /dev/null+++ b/harness/service-cli/src/deployment_up_access.rs@@ -0,0 +1,362 @@+//! Private management routing. Only the original cluster's TLS endpoint is+//! forwarded; this proxy is not a general-purpose internet or credential proxy.+use super::*;+use tokio::io::{AsyncReadExt, AsyncWriteExt};++pub(in super::super) struct Bridge {+ child: tokio::process::Child,+ proxy: tokio::task::JoinHandle<()>,+}+impl Drop for Bridge {+ fn drop(&mut self) {+ self.proxy.abort();+ let _ = self.child.start_kill();+ }+}+fn endpoint(kube: &Value) -> Result<String> {+ let clusters = kube["clusters"]+ .as_array()+ .filter(|a| a.len() == 1)+ .context("Expected one original cluster")?;+ let c = &clusters[0]["cluster"];+ ensure!(+ c["insecure-skip-tls-verify"] != true+ && c["certificate-authority-data"]+ .as_str()+ .is_some_and(|s| !s.is_empty()),+ "Original cluster CA required"+ );+ let u = reqwest::Url::parse(c["server"].as_str().context("Missing cluster endpoint")?)?;+ ensure!(+ u.scheme() == "https"+ && u.port_or_known_default() == Some(443)+ && u.username().is_empty()+ && u.password().is_none()+ && u.path() == "/"+ && u.query().is_none(),+ "Invalid private management endpoint"+ );+ Ok(u.host_str().context("Endpoint host missing")?.into())+}+async fn tunnel(mut incoming: tokio::net::TcpStream, host: &str, port: u16) -> Result<()> {+ let mut header = Vec::new();+ loop {+ let mut b = [0];+ incoming.read_exact(&mut b).await?;+ header.push(b[0]);+ ensure!(header.len() <= 8192, "Proxy header too large");+ if header.ends_with(b"\r\n\r\n") {+ break;+ }+ }+ let line = std::str::from_utf8(&header)?+ .split("\r\n")+ .next()+ .unwrap_or("");+ ensure!(+ line == format!("CONNECT {host}:443 HTTP/1.1")+ || line == format!("CONNECT {host}:443 HTTP/1.0"),+ "Proxy destination is outside the reviewed cluster"+ );+ let mut upstream =+ tokio::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, port)).await?;+ incoming+ .write_all(b"HTTP/1.1 200 Connection Established\r\n\r\n")+ .await?;+ tokio::io::copy_bidirectional(&mut incoming, &mut upstream).await?;+ Ok(())+}+impl Native<'_> {+ async fn probe(&self, kubeconfig: &Path) -> Result<Value> {+ let raw = self+ .tool(+ "kubectl",+ vec![+ "--kubeconfig".into(),+ arg(kubeconfig)?,+ "--request-timeout=10s".into(),+ "get".into(),+ "namespace".into(),+ "kube-system".into(),+ "-o".into(),+ "json".into(),+ ],+ None,+ )+ .await?;+ let v: Value = serde_json::from_slice(&raw)?;+ ensure!(+ v["metadata"]["name"] == "kube-system" && v["metadata"]["uid"].is_string(),+ "Cluster identity probe failed"+ );+ Ok(v)+ }+ async fn session_json(&self, args: Vec<String>) -> Result<Value> {+ let mut a = args;+ a.extend([+ "--profile".into(),+ self.aws_profile()?,+ "--region".into(),+ self.review.config.cloud.region().into(),+ "--output".into(),+ "json".into(),+ "--no-cli-pager".into(),+ ]);+ serde_json::from_slice(&self.tool("aws", a, None).await?).context("Invalid SSM observation")+ }+ async fn retire_session(&self, path: &Path) -> Result<()> {+ if !path.exists() {+ return Ok(());+ }+ let mut r = self.document(path)?;+ ensure!(+ r["deploymentId"] == self.review.deployment_id,+ "SSM receipt identity mismatch"+ );+ if r["phase"] == "terminated" {+ return Ok(());+ }+ let mut found = vec![];+ for state in ["Active", "History"] {+ let v = self+ .session_json(strings(&["ssm", "describe-sessions", "--state", state]))+ .await?;+ for s in v["Sessions"]+ .as_array()+ .context("SSM session inventory missing")?+ {+ if s["Reason"] == r["reason"] && s["Target"] == r["target"] {+ found.push(s.clone());+ }+ }+ }+ ensure!(found.len()==1,"Original SSM session is not uniquely observed. Retain receipt; do not start another session.");+ let s = &found[0];+ let id = text(s, "SessionId")?;+ ensure!(+ r.get("sessionId").is_none_or(|v| v == &id),+ "SSM session changed"+ );+ r["sessionId"] = json!(id);+ write_private(path, &r)?;+ if !["Terminated", "Failed"].contains(&s["Status"].as_str().unwrap_or("")) {+ self.session_json(strings(&["ssm", "terminate-session", "--session-id", &id]))+ .await?;+ }+ r["phase"] = json!("terminated");+ write_private(path, &r)?;+ Ok(())+ }+ pub(in super::super) async fn close_access(&self) -> Result<()> {+ self.bridge.borrow_mut().take();+ if self.provider() == "aws" {+ self.retire_session(&self.root.join("ssm-session.json"))+ .await?;+ }+ Ok(())+ }+ pub(super) async fn connect_private(&self, kubeconfig: &Path) -> Result<()> {+ let mut kube = self.document(kubeconfig)?;+ let host = endpoint(&kube)?;+ if self.bridge.borrow().is_some()+ || kube["clusters"][0]["cluster"]["proxy-url"].is_null()+ && self.probe(kubeconfig).await.is_ok()+ {+ let value = self.probe(kubeconfig).await?;+ return self.access_receipt(kubeconfig, &value);+ }+ let (plan, _) = self.plan()?;+ let has_bridge = if self.provider() == "aws" {+ plan.infrastructure+ .as_ref()+ .unwrap()+ .operator_access+ .is_some()+ } else {+ plan.azure_infrastructure+ .as_ref()+ .unwrap()+ .management_access+ .is_some()+ };+ ensure!(has_bridge,"Private management API unreachable. Connect the approved private network, or configure the operator bridge before creating a new deployment. Original resources retained.");+ let config_path = self.root.join("private-route.json");+ let route = if config_path.exists() {+ self.document(&config_path)?+ } else {+ let port = std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))?+ .local_addr()?+ .port();+ let v = json!({"deploymentId":self.review.deployment_id,"host":host,"proxyPort":port});+ write_private(&config_path, &v)?;+ v+ };+ ensure!(+ route["host"] == host && route["deploymentId"] == self.review.deployment_id,+ "Private route target changed"+ );+ let proxy_port = u16::try_from(route["proxyPort"].as_u64().context("Proxy port missing")?)?;+ let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, proxy_port))+ .await+ .context(+ "Original loopback proxy port is occupied; do not alter the pinned kubeconfig",+ )?;+ let raw_port = std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))?+ .local_addr()?+ .port();+ let (program, args) = if self.provider() == "azure" {+ let m = self.azure_material().await?;+ let ip = text(&m, "managementIp")?;+ ip.parse::<std::net::Ipv4Addr>()+ .context("Invalid owned management IP")?;+ let mut a = vec![+ "-N".into(),+ "-T".into(),+ "-o".into(),+ "BatchMode=yes".into(),+ "-o".into(),+ "ExitOnForwardFailure=yes".into(),+ "-o".into(),+ "ForwardAgent=no".into(),+ "-o".into(),+ "StrictHostKeyChecking=accept-new".into(),+ "-o".into(),+ format!(+ "UserKnownHostsFile={}",+ self.root.join("ssh-known-hosts").display()+ ),+ "-L".into(),+ format!("127.0.0.1:{raw_port}:{host}:443"),+ ];+ if let Some(key) = self+ .review+ .config+ .advanced+ .azure+ .as_ref()+ .and_then(|a| a.ssh_private_key_file.as_ref())+ {+ a.extend(["-i".into(), arg(key)?]);+ }+ a.push(format!("owsadmin@{ip}"));+ ("ssh", a)+ } else {+ let path = self.root.join("ssm-session.json");+ self.retire_session(&path).await?;+ if path.exists() {+ fs::rename(+ &path,+ self.root.join(format!(+ "ssm-session-{}.json",+ uuid::Uuid::new_v4().simple()+ )),+ )?;+ }+ let status = self.cloud_status().await?;+ let target = text(+ &status["observed"]["infra"]["outputs"],+ "OperatorInstanceId",+ )?;+ ensure!(+ target.starts_with("i-") && target[2..].bytes().all(|b| b.is_ascii_hexdigit()),+ "Invalid original operator instance"+ );+ let reason = format!("ows-{}", uuid::Uuid::new_v4().simple());+ write_private(+ &path,+ &json!({"deploymentId":self.review.deployment_id,"target":target,"reason":reason,"phase":"intent"}),+ )?;+ (+ "aws",+ vec![+ "ssm".into(),+ "start-session".into(),+ "--target".into(),+ target,+ "--document-name".into(),+ "AWS-StartPortForwardingSessionToRemoteHost".into(),+ "--reason".into(),+ reason,+ "--parameters".into(),+ serde_json::to_string(+ &json!({"host":[host],"portNumber":["443"],"localPortNumber":[raw_port.to_string()]}),+ )?,+ "--profile".into(),+ self.aws_profile()?,+ "--region".into(),+ self.review.config.cloud.region().into(),+ ],+ )+ };+ let mut command = tokio::process::Command::new(program);+ command+ .args(args)+ .envs(self.environment()?)+ .stdin(std::process::Stdio::null())+ .stdout(std::process::Stdio::null())+ .stderr(std::process::Stdio::null())+ .kill_on_drop(true);+ let mut child = command.spawn().context("Start private management bridge")?;+ let mut ready = false;+ for _ in 0..30 {+ ensure!(child.try_wait()?.is_none(),"Private bridge exited; check CLI login, bridge permissions and SSH key/plugin prerequisites");+ if tokio::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, raw_port))+ .await+ .is_ok()+ {+ ready = true;+ break;+ }+ tokio::time::sleep(std::time::Duration::from_secs(1)).await;+ }+ ensure!(ready, "Private management bridge did not become ready");+ let proxy = tokio::spawn(async move {+ while let Ok((socket, _)) = listener.accept().await {+ let host = host.clone();+ tokio::spawn(async move {+ let _ = tunnel(socket, &host, raw_port).await;+ });+ }+ });+ *self.bridge.borrow_mut() = Some(Bridge { child, proxy });+ let proxy_url = format!("http://127.0.0.1:{proxy_port}");+ let current = &kube["clusters"][0]["cluster"]["proxy-url"];+ ensure!(+ current.is_null() || current == &proxy_url,+ "Unexpected kubeconfig proxy"+ );+ if current.is_null() {+ kube["clusters"][0]["cluster"]["proxy-url"] = json!(proxy_url);+ write_private(kubeconfig, &kube)?;+ }+ let value = self.probe(kubeconfig).await?;+ self.access_receipt(kubeconfig, &value)+ }+ fn access_receipt(&self, kubeconfig: &Path, value: &Value) -> Result<()> {+ let path = self.root.join("private-access.json");+ if path.exists() {+ ensure!(+ self.document(&path)?["clusterUid"] == value["metadata"]["uid"],+ "Original cluster was replaced"+ );+ }+ write_private(+ &path,+ &json!({"kubeconfig":kubeconfig,"clusterUid":value["metadata"]["uid"],"tlsVerified":true}),+ )+ }+}+#[cfg(test)]+mod tests {+ use super::*;+ #[test]+ fn private_endpoint_requires_original_ca_and_https() {+ let mut v = json!({"clusters":[{"cluster":{"server":"https://original.internal","certificate-authority-data":"CA"}}]});+ assert_eq!(endpoint(&v).unwrap(), "original.internal");+ v["clusters"][0]["cluster"]["insecure-skip-tls-verify"] = json!(true);+ assert!(endpoint(&v).is_err());+ v["clusters"][0]["cluster"]["insecure-skip-tls-verify"] = json!(false);+ v["clusters"][0]["cluster"]["server"] = json!("https://[email protected]");+ assert!(endpoint(&v).is_err());+ }+}diff --git a/harness/service-cli/src/deployment_up_aws_bootstrap.rs b/harness/service-cli/src/deployment_up_aws_bootstrap.rsnew file mode 100644index 00000000..d8a1846c--- /dev/null+++ b/harness/service-cli/src/deployment_up_aws_bootstrap.rs@@ -0,0 +1,168 @@+//! Explicit identity-stack bootstrap with original request and template receipts.+//! This runs only after a separate digest-bound approval for IAM changes.+use super::*;+#[derive(Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+struct Receipt {+ deployment_id: String,+ bundle_digest: String,+ request_id: String,+ arn: Option<String>,+ complete: bool,+}+pub(super) async fn bootstrap(+ plan: &Path,+ root: &Path,+ profile: &str,+ accept: &str,+ approve: bool,+) -> Result<()> {+ let _lock = super::super::up::operation_lock(root, "bootstrap.lock")?;+ let (p, m) = inspect(plan)?;+ ensure!(+ m.bundle_digest == accept && approve,+ "Explicit approval of the exact identity bundle is required"+ );+ let Target::Aws { account, region } = &p.target else {+ bail!("AWS bundle required")+ };+ let api = Aws {+ p: p.clone(),+ profile: profile.into(),+ root: root.into(),+ preflight: None,+ };+ let who = api.json("sts", "get-caller-identity", json!({})).await?;+ let expected = &p+ .bootstrap+ .as_ref()+ .context("Bootstrap identity missing")?+ .operator_arn;+ let arn = who["Arn"].as_str().context("AWS caller ARN missing")?;+ let role = expected+ .strip_prefix(&format!("arn:aws:iam::{account}:role/"))+ .map(|s| s.rsplit('/').next().unwrap());+ ensure!(+ who["Account"] == *account+ && (arn == expected+ || role.is_some_and(|r| arn+ .strip_prefix(&format!("arn:aws:sts::{account}:assumed-role/{r}/"))+ .is_some_and(|s| !s.is_empty() && !s.contains('/')))),+ "Bootstrap requires the exact reviewed operator/account"+ );+ let expected_template: Value = serde_json::from_slice(&read_file(plan, "bootstrap.cfn.json")?)?;+ let file = root.join("bootstrap-receipt.json");+ let mut receipt: Option<Receipt> = if file.exists() {+ Some(serde_json::from_slice(&fs::read(&file)?)?)+ } else {+ None+ };+ if let Some(r) = &receipt {+ ensure!(+ r.deployment_id == p.deployment_id && r.bundle_digest == m.bundle_digest,+ "Bootstrap receipt belongs to another deployment"+ );+ }+ let name = format!("{}-bootstrap", p.prefix());+ let selector = receipt+ .as_ref()+ .and_then(|r| r.arn.as_deref())+ .unwrap_or(&name);+ let found = api+ .call(+ "cloudformation",+ "describe-stacks",+ json!({"StackName":selector}),+ None,+ )+ .await?;+ if let Some(value) = found {+ let row = value["Stacks"]+ .as_array()+ .filter(|a| a.len() == 1)+ .map(|a| &a[0])+ .context("Ambiguous bootstrap stack")?;+ let r = receipt+ .as_mut()+ .context("Existing identity stack has no original local intent; refuse adoption")?;+ let observed = row["StackId"].as_str().context("Missing stack ARN")?;+ ensure!(+ row["StackName"] == name+ && observed.starts_with(&format!(+ "arn:aws:cloudformation:{region}:{account}:stack/{name}/"+ ))+ && r.arn.as_ref().is_none_or(|a| a == observed),+ "Bootstrap stack replaced or out of scope"+ );+ ensure!(+ row["Tags"].as_array().is_some_and(|a| a+ .iter()+ .any(|t| t["Key"] == "ows:deployment-id" && t["Value"] == p.deployment_id)),+ "Bootstrap ownership tag mismatch"+ );+ let remote = api+ .json(+ "cloudformation",+ "get-template",+ json!({"StackName":observed,"TemplateStage":"Original"}),+ )+ .await?;+ let template = match &remote["TemplateBody"] {+ Value::String(s) => serde_json::from_str(s)?,+ v @ Value::Object(_) => v.clone(),+ _ => bail!("Original bootstrap template missing"),+ };+ ensure!(+ template == expected_template,+ "Bootstrap template differs from review"+ );+ let events = api+ .json(+ "cloudformation",+ "describe-stack-events",+ json!({"StackName":observed}),+ )+ .await?;+ ensure!(+ events["StackEvents"]+ .as_array()+ .is_some_and(|a| a.iter().any(|e| e["StackId"] == observed+ && e["PhysicalResourceId"] == observed+ && e["ResourceType"] == "AWS::CloudFormation::Stack"+ && e["ClientRequestToken"] == r.request_id)),+ "Original bootstrap request token not observed; inspect before continuing"+ );+ r.arn = Some(observed.into());+ r.complete = row["StackStatus"] == "CREATE_COMPLETE";+ super::super::up::write_private(&file, r)?;+ ensure!(+ r.complete || row["StackStatus"] == "CREATE_IN_PROGRESS",+ "Identity stack failed; inspect original stack without recreating it"+ );+ return crate::output(+ json!({"phase":if r.complete{"ready"}else{"creating"},"stackArn":r.arn,"requestId":r.request_id}),+ );+ }+ ensure!(receipt.is_none(),"Original bootstrap request not observed; outcome remains unknown. Do not replay CreateStack.");+ let mut r = Receipt {+ deployment_id: p.deployment_id.clone(),+ bundle_digest: m.bundle_digest,+ request_id: format!("{}-bootstrap", p.deployment_id),+ arn: None,+ complete: false,+ };+ super::super::up::write_private(&file, &r)?;+ let result=api.json("cloudformation","create-stack",json!({"StackName":name,"TemplateBody":serde_json::to_string(&expected_template)?,"Capabilities":["CAPABILITY_NAMED_IAM"],"ClientRequestToken":r.request_id,"Tags":[{"Key":"ows:deployment-id","Value":p.deployment_id},{"Key":"ows:expires-at","Value":p.expires_at_unix.to_string()}]})).await?;+ let arn = result["StackId"]+ .as_str()+ .context("Bootstrap create outcome unknown: stack ARN missing")?;+ ensure!(+ arn.starts_with(&format!(+ "arn:aws:cloudformation:{region}:{account}:stack/{name}/"+ )),+ "Unexpected bootstrap stack ARN"+ );+ r.arn = Some(arn.into());+ super::super::up::write_private(&file, &r)?;+ crate::output(json!({"phase":"creating","stackArn":r.arn,"requestId":r.request_id}))+}diff --git a/harness/service-cli/src/deployment_up_azure_bootstrap.rs b/harness/service-cli/src/deployment_up_azure_bootstrap.rsnew file mode 100644index 00000000..09c4c741--- /dev/null+++ b/harness/service-cli/src/deployment_up_azure_bootstrap.rs@@ -0,0 +1,744 @@+//! Azure setup uses the same fixed resource graph and exact role payloads as+//! the reviewed exporters. Every additional write has its own durable intent.+use super::*;+use api::Azure;+use reqwest::Method;+#[derive(Clone, Copy, clap::ValueEnum)]+pub enum UpPhase {+ Setup,+ Control,+ Foundation,+ Federation,+}+fn save(path: &Path, v: &Value) -> Result<()> {+ super::super::up::write_private(path, v)+}+fn expected_properties(v: &Value) -> Value {+ let mut props = v.get("properties").cloned().unwrap_or(Value::Null);+ if let Some(o) = props.as_object_mut() {+ for k in [+ "createdOn",+ "updatedOn",+ "createdBy",+ "updatedBy",+ "provisioningState",+ "scope",+ ] {+ o.remove(k);+ }+ }+ props+}+async fn put_exact(+ api: &mut Azure,+ dir: &Path,+ label: &str,+ id: &str,+ version: &str,+ body: Value,+) -> Result<()> {+ let path = dir.join(format!("{label}.json"));+ let old: Option<Value> = if path.exists() {+ Some(serde_json::from_slice(&fs::read(&path)?)?)+ } else {+ None+ };+ let body_hash = hash(&json_bytes(&body)?);+ if let Some(r) = &old {+ ensure!(+ r["id"] == id && r["bodyHash"] == body_hash,+ "Original role/grant request changed"+ );+ }+ let current = api+ .arm(+ Method::GET,+ id,+ version,+ None,+ &uuid::Uuid::new_v4().to_string(),+ )+ .await?;+ if current.status == 200 {+ let observed = current.body.context("Role observation missing")?;+ let expected = expected_properties(&body);+ let actual = expected_properties(&observed);+ let good = expected+ .as_object()+ .is_some_and(|m| m.iter().all(|(k, v)| actual.get(k) == Some(v)));+ ensure!(+ good,+ "Existing role/grant differs from reviewed payload; refusing overwrite"+ );+ let mut receipt =+ old.unwrap_or_else(|| json!({"id":id,"bodyHash":body_hash,"preexisting":true}));+ receipt["phase"] = json!("observed");+ save(&path, &receipt)?;+ return Ok(());+ }+ ensure!(+ current.status == 404,+ "Unable to prove role/grant absence; no write attempted"+ );+ if path.exists() {+ let old: Value = serde_json::from_slice(&fs::read(&path)?)?;+ ensure!(+ old["phase"] == "rejected" && [401, 403].contains(&old["status"].as_u64().unwrap_or(0)),+ "Prior role/grant outcome unknown; observe original request before retrying"+ );+ ensure!(+ old["id"] == id && old["bodyHash"] == hash(&json_bytes(&body)?),+ "Role/grant request changed"+ );+ }+ let request = uuid::Uuid::new_v4().to_string();+ let mut receipt =+ json!({"id":id,"bodyHash":hash(&json_bytes(&body)?),"requestId":request,"phase":"intent"});+ if let Some(previous) = old {+ receipt["previousAttempts"] = json!([previous]);+ }+ save(&path, &receipt)?;+ let response = api+ .arm(Method::PUT, id, version, Some(&body), &request)+ .await?;+ if [401, 403].contains(&response.status) {+ receipt["phase"] = json!("rejected");+ receipt["status"] = json!(response.status);+ save(&path, &receipt)?;+ bail!("Administrator permission is missing for an exact reviewed role/grant; no automatic retry");+ }+ ensure!(+ [200, 201].contains(&response.status),+ "Role/grant outcome not confirmed; inspect original intent"+ );+ receipt["phase"] = json!("accepted");+ save(&path, &receipt)?;+ Ok(())+}+async fn deploy(+ api: &mut Azure,+ dir: &Path,+ identity: (&Plan, &str, &str),+ scope: &str,+ label: &str,+ mut template: Value,+ parameters: Value,+) -> Result<bool> {+ let (p, nonce, bundle) = identity;+ let path = dir.join(format!("deployment-{label}.json"));+ let name = format!("{}-up-{label}", graph::names(p).prefix);+ let id = format!("{scope}/providers/Microsoft.Resources/deployments/{name}");+ for field in ["owsDeployment", "owsNonce", "owsBundle"] {+ template["parameters"][field] = json!({"type":"string"});+ }+ let mut params = parameters;+ params["owsDeployment"] = json!({"value":p.deployment_id});+ params["owsNonce"] = json!({"value":nonce});+ params["owsBundle"] = json!({"value":bundle});+ let body = json!({"properties":{"mode":"Incremental","template":template,"parameters":params}});+ let before = api+ .arm(+ Method::GET,+ &id,+ "2022-09-01",+ None,+ &uuid::Uuid::new_v4().to_string(),+ )+ .await?;+ if before.status == 200 {+ ensure!(+ path.exists(),+ "Existing ARM deployment lacks original local intent; refusing adoption"+ );+ let mut receipt: Value = serde_json::from_slice(&fs::read(&path)?)?;+ ensure!(+ receipt["bodyHash"] == hash(&json_bytes(&body)?),+ "Original ARM deployment template changed"+ );+ let live = before.body.context("ARM deployment observation missing")?;+ ensure!(+ parameter_values(&live["properties"]["parameters"])? == parameter_values(¶ms)?,+ "Original ARM deployment parameters changed"+ );+ let exported = api+ .arm(+ Method::POST,+ &format!("{id}/exportTemplate"),+ "2022-09-01",+ None,+ &uuid::Uuid::new_v4().to_string(),+ )+ .await?;+ ensure!(+ exported.status == 200+ && exported+ .body+ .as_ref()+ .is_some_and(|v| v["template"] == body["properties"]["template"]),+ "Original ARM template differs or could not be verified; do not advance"+ );+ let correlation = live["properties"]["correlationId"]+ .as_str()+ .context("ARM correlation missing")?;+ ensure!(+ receipt+ .get("correlationId")+ .is_none_or(|old| old == correlation),+ "ARM deployment replaced"+ );+ receipt["correlationId"] = json!(correlation);+ save(&path, &receipt)?;+ match live["properties"]["provisioningState"].as_str() {+ Some("Succeeded") => return Ok(true),+ Some("Failed" | "Canceled") => bail!(+ "Reviewed ARM deployment failed; inspect the original deployment before continuing"+ ),+ _ => return Ok(false),+ }+ }+ ensure!(+ before.status == 404 && !path.exists(),+ "ARM deployment outcome unknown; do not resubmit the original request"+ );+ let request = uuid::Uuid::new_v4().to_string();+ save(+ &path,+ &json!({"id":id,"requestId":request,"bodyHash":hash(&json_bytes(&body)?),"phase":"intent"}),+ )?;+ let response = api+ .arm(Method::PUT, &id, "2022-09-01", Some(&body), &request)+ .await?;+ ensure!(+ [200, 201, 202].contains(&response.status),+ "ARM deployment outcome not confirmed; inspect original request"+ );+ Ok(false)+}+fn file(root: &Path, name: &str) -> Result<Value> {+ Ok(serde_json::from_slice(&super::super::read_file(+ root, name,+ )?)?)+}+fn path(url: &str) -> Result<&str> {+ url.strip_prefix("https://management.azure.com")+ .and_then(|s| s.split('?').next())+ .context("Invalid generated role URL")+}+fn principal_id(body: &Value) -> Result<&str> {+ body["properties"]["principalId"]+ .as_str()+ .filter(|s| uuid::Uuid::parse_str(s).is_ok())+ .context("Managed identity principal missing")+}+fn resolve_references(v: &mut Value, identity: &str) {+ match v {+ Value::String(s) if s.starts_with("[reference(") && s.ends_with(".principalId]") => {+ *s = identity.into()+ }+ Value::Object(o) => {+ for v in o.values_mut() {+ resolve_references(v, identity)+ }+ }+ Value::Array(a) => {+ for v in a {+ resolve_references(v, identity)+ }+ }+ _ => {}+ }+}+pub(super) async fn run(+ plan: &Path,+ state: &Path,+ phase: UpPhase,+ accept: &str,+ approved: bool,+) -> Result<()> {+ let (p, m) = super::super::inspect(plan)?;+ ensure!(+ approved && accept == m.bundle_digest,+ "Explicit approval of the exact Azure identity/control bundle is required"+ );+ let (bound, bundle, nonce) = recovery::up_local(state)?;+ ensure!(+ bound == p && bundle == m.bundle_digest,+ "Azure local lifecycle identity mismatch"+ );+ let dir = state.join("up-operations");+ super::super::up::directory(&dir)?;+ let _lock = super::super::up::operation_lock(&dir, "bootstrap.lock")?;+ let mut api = Azure::new(&p, None)?;+ api.check_setup_identity(&p).await?;+ let work = scope(&p);+ let control = graph::control_scope(&p);+ let names = graph::names(&p);+ match phase {+ UpPhase::Setup => {+ put_exact(+ &mut api,+ &dir,+ "setup-role",+ path(&role_url(&p))?,+ "2022-04-01",+ file(plan, "azure-setup-role.json")?,+ )+ .await?;+ put_exact(+ &mut api,+ &dir,+ "setup-assignment",+ path(&assignment_url(&p))?,+ "2022-04-01",+ file(plan, "azure-setup-assignment.json")?,+ )+ .await?;+ let executor = file(plan, "azure-executor-identity.json")?;+ put_exact(+ &mut api,+ &dir,+ "executor-role",+ path(+ executor["roleUrl"]+ .as_str()+ .context("Executor role URL missing")?,+ )?,+ "2022-04-01",+ file(plan, "azure-executor-role.json")?,+ )+ .await?;+ put_exact(+ &mut api,+ &dir,+ "executor-assignment",+ path(+ executor["assignmentUrl"]+ .as_str()+ .context("Executor assignment URL missing")?,+ )?,+ "2022-04-01",+ json!({"properties":executor["assignment"]["properties"]}),+ )+ .await?;+ apply_grants(&mut api, &dir, plan, "setup").await?;+ crate::output(json!({"phase":"ready","scope":work}))+ }+ UpPhase::Control => {+ if dir.join("control-init.json").exists() {+ let ready = recovery::up_initialize_control(state).await?;+ return crate::output(json!({"phase":if ready{"ready"}else{"waiting_heartbeat"}}));+ }+ let out = state.join("control-package");+ if !out.exists() {+ recovery::up_package(state, &out)?;+ }+ let mut identity_template = file(&out, "azure-control.arm.json")?;+ identity_template["resources"]+ .as_array_mut()+ .context("Control resources missing")?+ .retain(|r| r["type"] == "Microsoft.ManagedIdentity/userAssignedIdentities");+ if !deploy(+ &mut api,+ &dir,+ (&p, &nonce, &bundle),+ &control,+ "control-identity",+ identity_template,+ json!({}),+ )+ .await?+ {+ return crate::output(json!({"phase":"creating_identity"}));+ }+ apply_grants(&mut api, &dir, plan, "control-identity").await?;+ let ready = deploy(+ &mut api,+ &dir,+ (&p, &nonce, &bundle),+ &control,+ "control",+ file(&out, "azure-control.arm.json")?,+ json!({}),+ )+ .await?;+ if !ready {+ return crate::output(json!({"phase":"creating"}));+ }+ let identity_path = format!(+ "{control}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{}-cleanup",+ names.prefix+ );+ let identity = api+ .arm(Method::GET, &identity_path, "2023-01-31", None, &nonce)+ .await?;+ ensure!(identity.status == 200, "Cleanup identity not ready");+ let identity = identity.body.context("Cleanup identity missing")?;+ let principal = principal_id(&identity)?;+ let bindings = file(&out, "azure-control-bindings.json")?;+ for (i, assignment) in bindings["assignments"]+ .as_array()+ .context("Control bindings missing")?+ .iter()+ .enumerate()+ {+ let mut body = json!({"properties":{"principalId":assignment["principalId"],"principalType":assignment["principalType"],"roleDefinitionId":bindings["roleDefinitionId"]}});+ resolve_references(&mut body, principal);+ let id = format!(+ "{}/providers/Microsoft.Authorization/roleAssignments/{}",+ bindings["scope"].as_str().unwrap(),+ assignment["guid"].as_str().unwrap()+ );+ put_exact(+ &mut api,+ &dir,+ &format!("control-blob-{i}"),+ &id,+ "2022-04-01",+ body,+ )+ .await?;+ }+ for (label, role, assignment) in [+ (+ "workload",+ "azure-cleanup-role.json",+ "azure-cleanup-assignment.json",+ ),+ ("stop", "azure-cleanup-stop-role.json", ""),+ ] {+ let role = file(&out, role)?;+ let role_guid = guid(+ &p,+ if label == "workload" {+ "cleanup-role"+ } else {+ "cleanup-stop-role"+ },+ );+ let role_scope = if label == "workload" { &work } else { &control };+ let role_path = format!(+ "{role_scope}/providers/Microsoft.Authorization/roleDefinitions/{role_guid}"+ );+ put_exact(+ &mut api,+ &dir,+ &format!("cleanup-{label}-role"),+ &role_path,+ "2022-04-01",+ json!({"properties":role["properties"]}),+ )+ .await?;+ if !assignment.is_empty() {+ let mut body = file(&out, assignment)?;+ resolve_references(&mut body, principal);+ let id = format!(+ "{work}/providers/Microsoft.Authorization/roleAssignments/{}",+ guid(&p, "cleanup-workload")+ );+ put_exact(+ &mut api,+ &dir,+ "cleanup-workload-assignment",+ &id,+ "2022-04-01",+ body,+ )+ .await?;+ }+ }+ let stop = file(&out, "azure-cleanup-stop-role.json")?;+ let Target::Azure { subscription, .. } = &p.target else {+ unreachable!()+ };+ let stop_assignment = format!(+ "{}/providers/Microsoft.Authorization/roleAssignments/{}",+ stop["assignmentScope"].as_str().unwrap(),+ stop["assignmentGuid"].as_str().unwrap()+ );+ let stop_role=format!("/subscriptions/{subscription}/providers/Microsoft.Authorization/roleDefinitions/{}",stop["roleDefinitionGuid"].as_str().unwrap());+ put_exact(&mut api,&dir,"cleanup-stop-assignment",&stop_assignment,"2022-04-01",json!({"properties":{"principalId":principal,"principalType":"ServicePrincipal","roleDefinitionId":stop_role}})).await?;+ let registry = file(&out, "azure-control-registry-binding.json")?;+ if let Some(scope) = registry["scope"].as_str() {+ let id = format!(+ "{scope}/providers/Microsoft.Authorization/roleAssignments/{}",+ registry["assignmentGuid"].as_str().unwrap()+ );+ put_exact(&mut api,&dir,"cleanup-registry-pull",&id,"2022-04-01",json!({"properties":{"principalId":principal,"principalType":"ServicePrincipal","roleDefinitionId":registry["roleDefinitionId"]}})).await?;+ }+ let ready = recovery::up_initialize_control(state).await?;+ crate::output(json!({"phase":if ready{"ready"}else{"waiting_heartbeat"}}))+ }+ UpPhase::Foundation | UpPhase::Federation => {+ apply_grants(+ &mut api,+ &dir,+ plan,+ if matches!(phase, UpPhase::Foundation) {+ "foundation"+ } else {+ "federation"+ },+ )+ .await?;+ let label = if matches!(phase, UpPhase::Foundation) {+ "bindings"+ } else {+ "federation"+ };+ let ready = deploy(+ &mut api,+ &dir,+ (&p, &nonce, &bundle),+ &work,+ label,+ file(plan, &format!("azure-{label}.arm.json"))?,+ json!({}),+ )+ .await?;+ crate::output(json!({"phase":if ready{"ready"}else{"creating"}}))+ }+ }+}++/// Complete, reviewable supplemental authority. No subscription-wide roles,+/// role-assignment writes or secret enumeration are granted to the executor.+pub(super) fn grants(p: &Plan) -> Result<Value> {+ let Target::Azure { subscription, .. } = &p.target else {+ bail!("Azure required")+ };+ let i = p+ .azure_bootstrap+ .as_ref()+ .context("Setup identity missing")?;+ let n = graph::names(p);+ let control = graph::control_scope(p);+ let work = scope(p);+ let input = serde_json::to_value(p.azure_infrastructure.as_ref().unwrap())?;+ let vault = input["postgresPassword"]["keyVaultId"].as_str().unwrap();+ let secret = format!(+ "{vault}/secrets/{}",+ input["postgresPassword"]["secretName"].as_str().unwrap()+ );+ let mut out = vec![];+ let mut custom = |label: &str,+ phase: &str,+ scope: &str,+ actions: Vec<&str>,+ data: Vec<&str>| {+ let role = guid(p, &format!("up-{label}-role"));+ let assignment = guid(p, &format!("up-{label}-grant"));+ let definition_scope = scope.split("/providers/").next().unwrap();+ let role_id=format!("/subscriptions/{subscription}/providers/Microsoft.Authorization/roleDefinitions/{role}");+ out.push(json!({"label":format!("{label}-role"),"phase":phase,"id":format!("{definition_scope}/providers/Microsoft.Authorization/roleDefinitions/{role}"),"body":{"properties":{"roleName":format!("{}-{label}",n.prefix),"type":"CustomRole","assignableScopes":[definition_scope],"permissions":[{"actions":actions,"notActions":[],"dataActions":data,"notDataActions":[]}]}}}));+ out.push(json!({"label":format!("{label}-grant"),"phase":phase,"id":format!("{scope}/providers/Microsoft.Authorization/roleAssignments/{assignment}"),"body":{"properties":{"principalId":i.setup_object_id,"principalType":i.setup_principal_type.azure(),"roleDefinitionId":role_id}}}));+ };+ custom(+ "control-setup",+ "setup",+ &control,+ vec![+ "Microsoft.Resources/deployments/read",+ "Microsoft.Resources/deployments/write",+ "Microsoft.Resources/deployments/operations/read",+ "Microsoft.Resources/deployments/operationStatuses/read",+ "Microsoft.Resources/deployments/exportTemplate/action",+ "Microsoft.ManagedIdentity/userAssignedIdentities/read",+ "Microsoft.ManagedIdentity/userAssignedIdentities/write",+ "Microsoft.Storage/storageAccounts/read",+ "Microsoft.Storage/storageAccounts/write",+ "Microsoft.Storage/storageAccounts/blobServices/containers/read",+ "Microsoft.Storage/storageAccounts/blobServices/containers/write",+ "Microsoft.App/managedEnvironments/read",+ "Microsoft.App/managedEnvironments/write",+ "Microsoft.App/managedEnvironments/join/action",+ "Microsoft.App/jobs/read",+ "Microsoft.App/jobs/write",+ ],+ vec![],+ );+ custom(+ "vault-deploy",+ "setup",+ vault,+ vec![+ "Microsoft.KeyVault/vaults/read",+ "Microsoft.KeyVault/vaults/deploy/action",+ ],+ vec![],+ );+ custom(+ "workload-inspection",+ "setup",+ &work,+ vec!["Microsoft.Resources/deployments/exportTemplate/action"],+ vec![],+ );+ custom(+ "cluster-credentials",+ "federation",+ &format!(+ "{work}/providers/Microsoft.ContainerService/managedClusters/{}-aks",+ n.prefix+ ),+ vec![+ "Microsoft.ContainerService/managedClusters/read",+ "Microsoft.ContainerService/managedClusters/listClusterUserCredential/action",+ ],+ vec![],+ );+ custom(+ "artifact-federation",+ "foundation",+ &format!(+ "{work}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{}-artifacts",+ n.prefix+ ),+ vec![+ "Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials/read",+ "Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials/write",+ ],+ vec![],+ );+ // Secret-level assignment: no permission to enumerate/read other secrets.+ out.push(json!({"label":"database-secret-read","phase":"setup","id":format!("{secret}/providers/Microsoft.Authorization/roleAssignments/{}",guid(p,"up-secret-read")),"body":{"properties":{"principalId":i.setup_object_id,"principalType":i.setup_principal_type.azure(),"roleDefinitionId":format!("/subscriptions/{subscription}/providers/Microsoft.Authorization/roleDefinitions/4633458b-17de-408a-b874-0445c86b69e6")}}}));+ for (label, phase, id) in [+ (+ "cleanup",+ "control-identity",+ format!(+ "{control}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{}-cleanup",+ n.prefix+ ),+ ),+ (+ "cluster",+ "foundation",+ format!(+ "{work}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{}-cluster",+ n.prefix+ ),+ ),+ (+ "kubelet",+ "foundation",+ format!(+ "{work}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{}-kubelet",+ n.prefix+ ),+ ),+ ] {+ out.push(json!({"label":format!("{label}-assign"),"phase":phase,"id":format!("{id}/providers/Microsoft.Authorization/roleAssignments/{}",guid(p,&format!("up-{label}-assign"))),"body":{"properties":{"principalId":i.setup_object_id,"principalType":i.setup_principal_type.azure(),"roleDefinitionId":format!("/subscriptions/{subscription}/providers/Microsoft.Authorization/roleDefinitions/f1a07417-d97a-45cb-824c-7a7467783830")}}}));+ }+ out.push(json!({"label":"registry-push","phase":"foundation","id":format!("{work}/providers/Microsoft.ContainerRegistry/registries/{}/providers/Microsoft.Authorization/roleAssignments/{}",n.registry,guid(p,"up-registry-push")),"body":{"properties":{"principalId":i.setup_object_id,"principalType":i.setup_principal_type.azure(),"roleDefinitionId":format!("/subscriptions/{subscription}/providers/Microsoft.Authorization/roleDefinitions/8311e382-0749-4cb8-b61a-304f252e45ec")}}}));+ Ok(json!({"schemaVersion":1,"deploymentId":p.deployment_id,"grants":out}))+}+async fn apply_grants(api: &mut Azure, dir: &Path, plan: &Path, phase: &str) -> Result<()> {+ let v = file(plan, "azure-up-grants.json")?;+ for g in v["grants"]+ .as_array()+ .context("Reviewed grants missing")?+ .iter()+ .filter(|g| g["phase"] == phase)+ {+ put_exact(+ api,+ dir,+ g["label"].as_str().context("Grant label missing")?,+ g["id"].as_str().context("Grant scope missing")?,+ "2022-04-01",+ g["body"].clone(),+ )+ .await?;+ }+ Ok(())+}++fn parameter_values(parameters: &Value) -> Result<BTreeMap<String, Value>> {+ parameters+ .as_object()+ .context("ARM parameter map missing")?+ .iter()+ .map(|(name, value)| {+ Ok((+ name.clone(),+ value+ .get("value")+ .cloned()+ .context("ARM parameter value missing")?,+ ))+ })+ .collect()+}++#[cfg(test)]+mod tests {+ use super::*;+ fn response(status: u16, body: Value) -> String {+ let body = body.to_string();+ format!("HTTP/1.1 {status} fixture\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",body.len())+ }+ const ID:&str="/subscriptions/66666666-7777-4888-8999-aaaaaaaaaaaa/resourceGroups/workload/providers/Microsoft.Authorization/roleAssignments/original";+ #[tokio::test]+ async fn lost_grant_response_is_never_replayed_on_absence() {+ let (mut api, log, thread) = api::tests::fixture(vec![+ Some(response(404, json!({}))),+ None,+ Some(response(404, json!({}))),+ ]);+ let root = tempfile::tempdir().unwrap();+ let body = json!({"properties":{"principalId":"same","roleDefinitionId":"exact"}});+ assert!(put_exact(+ &mut api,+ root.path(),+ "grant",+ ID,+ "2022-04-01",+ body.clone()+ )+ .await+ .is_err());+ let first: Value =+ serde_json::from_slice(&fs::read(root.path().join("grant.json")).unwrap()).unwrap();+ assert_eq!(first["phase"], "intent");+ assert!(+ put_exact(&mut api, root.path(), "grant", ID, "2022-04-01", body)+ .await+ .is_err()+ );+ thread.join().unwrap();+ assert_eq!(+ log.lock()+ .unwrap()+ .iter()+ .filter(|r| r.starts_with("PUT "))+ .count(),+ 1+ );+ }+ #[tokio::test]+ async fn observed_grant_preserves_original_request_and_refuses_drift() {+ let body = json!({"properties":{"principalId":"same","roleDefinitionId":"exact"}});+ let (mut api, _, thread) = api::tests::fixture(vec![Some(response(200, body.clone()))]);+ let root = tempfile::tempdir().unwrap();+ let receipt = json!({"id":ID,"bodyHash":hash(&json_bytes(&body).unwrap()),"requestId":"original-request","phase":"accepted","previousAttempts":[{"requestId":"rejected-request"}]});+ save(&root.path().join("grant.json"), &receipt).unwrap();+ put_exact(&mut api, root.path(), "grant", ID, "2022-04-01", body)+ .await+ .unwrap();+ thread.join().unwrap();+ let r: Value =+ serde_json::from_slice(&fs::read(root.path().join("grant.json")).unwrap()).unwrap();+ assert_eq!(r["requestId"], "original-request");+ assert_eq!(r["previousAttempts"], receipt["previousAttempts"]);+ assert!(put_exact(+ &mut api,+ root.path(),+ "grant",+ ID,+ "2022-04-01",+ json!({"properties":{"principalId":"other"}})+ )+ .await+ .is_err());+ }+}diff --git a/harness/service-cli/src/deployment_up_azure_images.rs b/harness/service-cli/src/deployment_up_azure_images.rsnew file mode 100644index 00000000..89c16d46--- /dev/null+++ b/harness/service-cli/src/deployment_up_azure_images.rs@@ -0,0 +1,265 @@+//! Managed-identity/OAuth registry publication. Intent is durable before push;+//! an absent tag after an uncertain push is not automatic retry permission.+use super::*;+use zeroize::Zeroizing;+struct Temporary(PathBuf);+impl Drop for Temporary {+ fn drop(&mut self) {+ let _ = fs::remove_dir_all(&self.0);+ }+}+impl Native<'_> {+ async fn az_json(&self, args: Vec<String>) -> Result<Value> {+ let mut a = args;+ a.extend([+ "--subscription".into(),+ self.review.config.cloud.account().into(),+ "--only-show-errors".into(),+ "-o".into(),+ "json".into(),+ ]);+ let bytes = Zeroizing::new(self.tool("az", a, None).await?);+ serde_json::from_slice(&bytes).context("Invalid Azure CLI response")+ }+ async fn acr_tag(&self, registry: &str, repo: &str, tag: &str) -> Result<Option<String>> {+ let names = self+ .az_json(strings(&["acr", "repository", "list", "--name", registry]))+ .await?;+ ensure!(+ names+ .as_array()+ .is_some_and(|a| a.iter().all(Value::is_string)),+ "Invalid registry repository inventory"+ );+ if !names.as_array().unwrap().iter().any(|s| s == repo) {+ return Ok(None);+ }+ let tags = self+ .az_json(strings(&[+ "acr",+ "repository",+ "show-tags",+ "--name",+ registry,+ "--repository",+ repo,+ "--detail",+ ]))+ .await?;+ let values = tags.as_array().context("Invalid registry tag inventory")?;+ let matching: Vec<_> = values.iter().filter(|v| v["name"] == tag).collect();+ ensure!(matching.len() <= 1, "Ambiguous registry tag");+ let Some(t) = matching.first() else {+ return Ok(None);+ };+ let digest = t["digest"]+ .as_str()+ .context("Registry manifest digest missing")?;+ ensure!(+ digest.starts_with("sha256:")+ && digest.len() == 71+ && digest[7..]+ .bytes()+ .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)),+ "Invalid registry manifest digest"+ );+ Ok(Some(digest.into()))+ }+ pub(super) async fn azure_images(&self) -> Result<Observation> {+ let material = self.azure_material().await?;+ let registry = text(&material["registry"], "name")?;+ let host = text(&material["registry"], "loginServer")?;+ ensure!(+ host == format!("{registry}.azurecr.io")+ && material["registry"]["createdAt"].is_string(),+ "Owned ACR identity missing"+ );+ let directory_path = self+ .root+ .join(format!(".registry-auth-{}", uuid::Uuid::new_v4().simple()));+ directory(&directory_path)?;+ let _temporary = Temporary(directory_path.clone());+ let mut auth_env = self.environment()?;+ auth_env.insert("DOCKER_CONFIG".into(), arg(&directory_path)?);+ let mut login = self+ .az_json(strings(&[+ "acr",+ "login",+ "--name",+ ®istry,+ "--expose-token",+ ]))+ .await?;+ ensure!(+ login["loginServer"] == host,+ "Registry token audience mismatch"+ );+ let token = Zeroizing::new(+ login+ .as_object_mut()+ .context("Invalid registry login")?+ .remove("accessToken")+ .and_then(|v| {+ if let Value::String(s) = v {+ Some(s)+ } else {+ None+ }+ })+ .context("Registry token missing")?,+ );+ super::super::super::local::exec_with_env(+ Path::new("docker"),+ &strings(&[+ "login",+ &host,+ "--username",+ "00000000-0000-0000-0000-000000000000",+ "--password-stdin",+ ]),+ Some(token.as_bytes().to_vec()),+ &auth_env,+ )+ .await?;+ let mut results = BTreeMap::<String, Value>::new();+ for (role, source) in [+ ("harness", Some(&self.package.release.images.harness)),+ ("runner", Some(&self.package.release.images.runner)),+ (+ "service-ui",+ self.package.release.images.service_ui.as_ref(),+ ),+ ] {+ let Some(source) = source else { continue };+ self.tool(+ "docker",+ strings(&["pull", "--platform", "linux/amd64", source]),+ None,+ )+ .await?;+ let local: Value = serde_json::from_slice(+ &self+ .tool("docker", strings(&["image", "inspect", source]), None)+ .await?,+ )?;+ let local = local+ .as_array()+ .filter(|a| a.len() == 1)+ .map(|a| &a[0])+ .context("Ambiguous local image")?;+ ensure!(+ local["Architecture"] == "amd64" && local["Os"] == "linux",+ "Release image platform mismatch"+ );+ let id = text(local, "Id")?;+ let tag = format!(+ "ows-{}",+ &hash(format!("{}:{role}:{id}", self.review.deployment_id).as_bytes())[..32]+ );+ let target = format!("{host}/{role}:{tag}");+ let path = self.root.join(format!("image-{role}.json"));+ let mut receipt = if path.exists() {+ self.document(&path)?+ } else {+ json!({"source":source,"configDigest":id,"registryCreatedAt":material["registry"]["createdAt"],"target":target,"dispatched":false})+ };+ ensure!(+ receipt["source"] == *source+ && receipt["configDigest"] == id+ && receipt["registryCreatedAt"] == material["registry"]["createdAt"]+ && receipt["target"] == target,+ "Original image publication identity changed"+ );+ let mut remote = self.acr_tag(®istry, role, &tag).await?;+ if remote.is_none() {+ ensure!(+ receipt["dispatched"] == false,+ "Registry push outcome unknown; inspect original tag. No automatic push retry."+ );+ self.tool("docker", strings(&["tag", source, &target]), None)+ .await?;+ receipt["dispatched"] = json!(true);+ write_private(&path, &receipt)?;+ super::super::super::local::exec_with_env(+ Path::new("docker"),+ &strings(&["push", &target]),+ None,+ &auth_env,+ )+ .await?;+ remote = self.acr_tag(®istry, role, &tag).await?;+ }+ let digest =+ remote.context("Original pushed tag not observed; inspect before retry")?;+ let pinned = format!("{host}/{role}@{digest}");+ let raw = super::super::super::local::exec_with_env(+ Path::new("docker"),+ &strings(&["buildx", "imagetools", "inspect", "--raw", &pinned]),+ None,+ &auth_env,+ )+ .await?;+ let manifest: Value = serde_json::from_slice(&raw)?;+ ensure!(+ manifest["config"]["digest"] == id+ && (format!("sha256:{}", hash(&raw)) == digest+ || raw+ .strip_suffix(b"\n")+ .is_some_and(|b| format!("sha256:{}", hash(b)) == digest)),+ "Remote image content differs from the reviewed local image"+ );+ receipt["image"] = json!(pinned);+ write_private(&path, &receipt)?;+ results.insert(role.into(), receipt);+ write_private(&self.root.join("published-images.json"), &results)?;+ }+ Ok(Observation::Complete(json!(results)))+ }+ pub(super) async fn azure_database_url(&self) -> Result<Zeroizing<String>> {+ let (p, _) = self.plan()?;+ let value = serde_json::to_value(p)?;+ let secret = &value["azureInfrastructure"]["postgresPassword"];+ let vault = text(secret, "keyVaultId")?;+ let vault = vault.rsplit('/').next().context("Vault identity missing")?;+ let mut args = strings(&[+ "keyvault",+ "secret",+ "show",+ "--vault-name",+ vault,+ "--name",+ secret["secretName"].as_str().unwrap(),+ "--version",+ secret["secretVersion"].as_str().unwrap(),+ "--query",+ "value",+ "--output",+ "tsv",+ "--only-show-errors",+ "--subscription",+ self.review.config.cloud.account(),+ ]);+ let raw = Zeroizing::new(self.tool("az", std::mem::take(&mut args), None).await?);+ let password = std::str::from_utf8(&raw)+ .context("Invalid secret encoding")?+ .trim_end_matches(['\r', '\n']);+ ensure!(+ !password.is_empty() && password.len() <= 4096,+ "Invalid managed database secret"+ );+ let encoded = password+ .bytes()+ .map(|b| {+ if b.is_ascii_alphanumeric() || b"-._~".contains(&b) {+ (b as char).to_string()+ } else {+ format!("%{b:02X}")+ }+ })+ .collect::<String>();+ let host = text(&self.azure_material().await?, "postgresHost")?;+ Ok(Zeroizing::new(format!(+ "postgresql://owsadmin:{encoded}@{host}:5432/postgres?sslmode=require"+ )))+ }+}diff --git a/harness/service-cli/src/deployment_up_config.rs b/harness/service-cli/src/deployment_up_config.rsnew file mode 100644index 00000000..7356f097--- /dev/null+++ b/harness/service-cli/src/deployment_up_config.rs@@ -0,0 +1,359 @@+//! Human-authored desired state. Secret values and resolved cloud observations+//! never belong in this document. Unknown options fail before any provider call.+use super::*;++#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Config {+ #[serde(default = "schema")]+ pub api_version: String,+ pub name: String,+ pub cloud: Cloud,+ pub nodes: Nodes,+ pub budget: Budget,+ #[serde(default)]+ pub trial: Option<Trial>,+ #[serde(default)]+ pub models: BTreeMap<String, Secret>,+ #[serde(default)]+ pub advanced: Advanced,+}+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Trial {+ pub expires_after_hours: u16,+ pub delete_data: bool,+}+fn schema() -> String {+ "ows/v1alpha1".into()+}+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]+#[serde(tag = "provider", rename_all = "lowercase", deny_unknown_fields)]+pub(super) enum Cloud {+ #[serde(rename_all = "camelCase")]+ Aws {+ account_id: String,+ region: String,+ #[serde(default = "profile")]+ profile: String,+ },+ #[serde(rename_all = "camelCase")]+ Azure {+ subscription_id: String,+ region: String,+ },+}+fn profile() -> String {+ "default".into()+}+impl Cloud {+ pub fn provider(&self) -> &'static str {+ match self {+ Self::Aws { .. } => "aws",+ Self::Azure { .. } => "azure",+ }+ }+ pub fn account(&self) -> &str {+ match self {+ Self::Aws { account_id, .. } => account_id,+ Self::Azure {+ subscription_id, ..+ } => subscription_id,+ }+ }+ pub fn region(&self) -> &str {+ match self {+ Self::Aws { region, .. } | Self::Azure { region, .. } => region,+ }+ }+}+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Nodes {+ pub count: u16,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub instance_type: Option<String>,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub requirements: Option<Requirements>,+}+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Requirements {+ pub cpu: Range,+ #[serde(rename = "memoryGiB")]+ pub memory_gi_b: Range,+}+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]+#[serde(deny_unknown_fields)]+pub(super) struct Range {+ pub min: u32,+ pub max: u32,+}+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Budget {+ #[serde(rename = "limitUSD", alias = "alertUSD")]+ pub alert_usd: u32,+}+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Secret {+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub env: Option<String>,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub file: Option<PathBuf>,+}+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Advanced {+ /// A reviewed release package can replace locally packaged repository assets.+ pub release: Option<PathBuf>,+ pub setup_profile: Option<PathBuf>,+ pub network_cidr: Option<String>,+ pub kubernetes_version: Option<String>,+ pub azure: Option<AzureOptions>,+ pub aws: Option<AwsOptions>,+}+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct AzureOptions {+ pub tenant_id: Option<String>,+ pub resource_group: Option<String>,+ pub control_resource_group: Option<String>,+ #[serde(default)]+ pub admin_group_object_ids: Vec<String>,+ pub ssh_public_key_file: Option<PathBuf>,+ pub ssh_private_key_file: Option<PathBuf>,+}+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct AwsOptions {+ pub operator_arn: Option<String>,+}++pub(super) fn safe_name(s: &str) -> bool {+ !s.is_empty()+ && s.len() <= 63+ && s.as_bytes()[0].is_ascii_alphanumeric()+ && s.bytes()+ .all(|b| b.is_ascii_alphanumeric() || b"_-".contains(&b))+}+impl Config {+ pub fn load(path: &Path, provider: &str) -> Result<Self> {+ let bytes = fs::read(path).context("Read deployment.yaml")?;+ ensure!(bytes.len() <= 64 * 1024, "deployment.yaml exceeds 64 KiB");+ // Serde's raw error may echo an inline credential; never relay input text.+ let mut config:Self=serde_yaml::from_slice(&bytes).map_err(|_|anyhow::anyhow!(+ "Invalid deployment.yaml schema. Check field names/types; credentials must use env or file references."))?;+ let base = path+ .canonicalize()?+ .parent()+ .context("Configuration directory missing")?+ .to_path_buf();+ let resolve = |p: &mut Option<PathBuf>| {+ if let Some(value) = p {+ if value.is_relative() {+ *value = base.join(&*value);+ }+ }+ };+ resolve(&mut config.advanced.setup_profile);+ resolve(&mut config.advanced.release);+ if let Some(a) = &mut config.advanced.azure {+ resolve(&mut a.ssh_public_key_file);+ resolve(&mut a.ssh_private_key_file);+ }+ for reference in config.models.values_mut() {+ resolve(&mut reference.file);+ }+ config.validate(provider)?;+ Ok(config)+ }+ pub fn validate(&self, provider: &str) -> Result<()> {+ if let Some(t) = &self.trial {+ ensure!((1..=72).contains(&t.expires_after_hours)&&t.delete_data,"A disposable trial requires 1..72 hours and explicit deleteData:true; omit trial for persistent deployment");+ }+ ensure!(+ self.api_version == "ows/v1alpha1",+ "Unsupported deployment apiVersion"+ );+ ensure!(super::super::name_ok(&self.name),"Use a lowercase deployment name (3..30 characters), starting with a letter and ending with a letter or digit");+ ensure!(+ self.cloud.provider() == provider,+ "Cloud in deployment.yaml differs from the selected command"+ );+ ensure!(+ safe_name(self.cloud.region())+ && self+ .cloud+ .region()+ .bytes()+ .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'),+ "Invalid cloud region"+ );+ match &self.cloud {+ Cloud::Aws {+ account_id,+ profile,+ ..+ } => ensure!(+ account_id.len() == 12+ && account_id.bytes().all(|b| b.is_ascii_digit())+ && safe_name(profile),+ "Use an exact AWS account ID and a named CLI profile"+ ),+ Cloud::Azure {+ subscription_id, ..+ } => ensure!(+ uuid::Uuid::parse_str(subscription_id)+ .is_ok_and(|id| !id.is_nil() && id.to_string() == *subscription_id),+ "Use the exact lowercase Azure subscription UUID"+ ),+ }+ ensure!(+ (1..=16).contains(&self.nodes.count),+ "Fixed node count must be between 1 and 16; autoscaling is not supported yet"+ );+ ensure!(+ self.nodes.instance_type.is_some() != self.nodes.requirements.is_some(),+ "Choose exactly one of nodes.instanceType or nodes.requirements"+ );+ if let Some(size) = &self.nodes.instance_type {+ ensure!(+ !size.is_empty()+ && size.len() <= 80+ && size+ .bytes()+ .all(|b| b.is_ascii_alphanumeric() || b"._-".contains(&b)),+ "Invalid instanceType"+ );+ }+ if let Some(r) = &self.nodes.requirements {+ for range in [&r.cpu, &r.memory_gi_b] {+ ensure!(+ range.min > 0 && range.min <= range.max && range.max <= 1024,+ "Node size ranges must be positive, ordered and bounded"+ );+ }+ }+ ensure!(+ (1..=100_000).contains(&self.budget.alert_usd),+ "budget.limitUSD must be between 1 and 100000; it is not a hard cap"+ );+ ensure!(+ self.advanced.azure.is_none() || provider == "azure",+ "Azure options cannot be used with AWS"+ );+ ensure!(+ self.advanced.aws.is_none() || provider == "aws",+ "AWS options cannot be used with Azure"+ );+ ensure!(+ provider == "aws" || self.advanced.network_cidr.is_none(),+ "Azure network CIDR override is not supported by the current fixed network template"+ );+ if let Some(a) = &self.advanced.azure {+ ensure!(+ a.admin_group_object_ids.len() <= 4+ && a.admin_group_object_ids+ .iter()+ .all(|s| uuid::Uuid::parse_str(s)+ .is_ok_and(|id| !id.is_nil() && id.to_string() == *s)),+ "Use at most four exact Entra administrator group IDs"+ );+ for name in [&a.resource_group, &a.control_resource_group]+ .into_iter()+ .flatten()+ {+ ensure!(safe_name(name), "Invalid Azure resource group name");+ }+ if let Some(id) = &a.tenant_id {+ ensure!(+ uuid::Uuid::parse_str(id).is_ok_and(|v| !v.is_nil() && v.to_string() == *id),+ "Invalid Azure tenant UUID"+ );+ }+ }+ for (name, key) in &self.models {+ ensure!(+ ["anthropic", "openai"].contains(&name.as_str()),+ "Unsupported model credential name"+ );+ ensure!(+ key.env.is_some() != key.file.is_some(),+ "Model credentials require exactly one env or file reference"+ );+ if let Some(env) = &key.env {+ ensure!(+ !env.is_empty()+ && env.len() <= 128+ && env+ .bytes()+ .all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_'),+ "Use an uppercase environment variable name, not a credential value"+ );+ }+ if let Some(path) = &key.file {+ ensure!(+ !path.as_os_str().is_empty(),+ "Credential file path is empty"+ );+ }+ }+ Ok(())+ }+ pub fn digest(&self) -> Result<String> {+ Ok(hash(&serde_json::to_vec(self)?))+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ pub(super) fn yaml() -> &'static str {+ "name: example\ncloud:\n provider: azure\n subscriptionId: 11111111-2222-4333-8444-555555555555\n region: westus2\nnodes:\n count: 2\n requirements:\n cpu: {min: 4, max: 8}\n memoryGiB: {min: 16, max: 32}\nbudget:\n limitUSD: 100\n"+ }+ #[test]+ fn minimal_config_is_strict_and_contains_no_credentials() {+ let c: Config = serde_yaml::from_str(yaml()).unwrap();+ c.validate("azure").unwrap();+ assert!(c.validate("aws").is_err());+ assert!(serde_yaml::from_str::<Config>(+ &yaml().replace("count: 2", "count: 2\n autoscaling: true")+ )+ .is_err());+ let mut v = serde_json::to_value(&c).unwrap();+ v["nodes"]["instanceType"] = json!("Standard_D4s_v5");+ assert!(serde_json::from_value::<Config>(v)+ .unwrap()+ .validate("azure")+ .is_err());+ let mut c = c;+ c.nodes.count = 0;+ assert!(c.validate("azure").is_err());+ }+ #[test]+ fn inline_secret_errors_do_not_echo_input() {+ let dir = tempfile::tempdir().unwrap();+ let p = dir.path().join("deployment.yaml");+ fs::write(+ &p,+ format!(+ "{}models:\n openai:\n apiKey: secret-test-value\n",+ yaml()+ ),+ )+ .unwrap();+ let error = Config::load(&p, "azure").unwrap_err().to_string();+ assert!(!error.contains("secret-test-value"));+ let mut c: Config = serde_yaml::from_str(yaml()).unwrap();+ c.models.insert(+ "openai".into(),+ Secret {+ env: Some("OPENAI_API_KEY".into()),+ file: None,+ },+ );+ c.validate("azure").unwrap();+ }+}diff --git a/harness/service-cli/src/deployment_up_install.rs b/harness/service-cli/src/deployment_up_install.rsnew file mode 100644index 00000000..9aecd42e--- /dev/null+++ b/harness/service-cli/src/deployment_up_install.rs@@ -0,0 +1,267 @@+//! Shared service/OpenShell rendering and private credential staging.+use super::*;+impl Native<'_> {+ pub(super) async fn tool(+ &self,+ program: &str,+ args: Vec<String>,+ input: Option<Vec<u8>>,+ ) -> Result<Vec<u8>> {+ super::super::super::local::exec_with_env(+ Path::new(program),+ &args,+ input,+ &self.environment()?,+ )+ .await+ }+ pub(super) async fn images(&self) -> Result<Observation> {+ if self.provider() != "aws" {+ return self.azure_images().await;+ }+ let mut receipts = BTreeMap::<String, Value>::new();+ for (role, source) in [+ ("harness", Some(&self.package.release.images.harness)),+ ("runner", Some(&self.package.release.images.runner)),+ (+ "service-ui",+ self.package.release.images.service_ui.as_ref(),+ ),+ ] {+ let Some(source) = source else { continue };+ // A local content-addressed pull is repeatable; the native publisher+ // separately reconciles the original registry mutation and never replays it.+ self.tool(+ "docker",+ strings(&["pull", "--platform", "linux/arm64", source]),+ None,+ )+ .await?;+ let mut a = strings(&["deploy", "aws", "lifecycle", "publish-image", "--state"]);+ a.extend([+ arg(&self.cloud_state())?,+ "--profile".into(),+ self.aws_profile()?,+ "--role".into(),+ role.into(),+ "--source".into(),+ source.clone(),+ "--allow-experimental-cloud".into(),+ ]);+ receipts.insert(role.into(), self.command(a).await?);+ write_private(&self.root.join("published-images.json"), &receipts)?;+ }+ Ok(Observation::Complete(json!(receipts)))+ }+ pub(super) async fn azure_material(&self) -> Result<Value> {+ let mut a = strings(&["deploy", "azure", "installation", "material", "--state"]);+ a.push(arg(&self.cloud_state())?);+ self.command(a).await+ }+ pub(super) async fn render_installation(&self) -> Result<()> {+ let c = &self.review.config;+ let (plan, _) = self.plan()?;+ let images = self.document(&self.root.join("published-images.json"))?;+ let harness = text(&images["harness"], "image")?;+ let runner = text(&images["runner"], "image")?;+ let fullname = "ow-gateway";+ let ns = "ow-openshell";+ let endpoint = format!("https://{fullname}.{ns}.svc.cluster.local:8080");+ let artifacts = if self.provider() == "aws" {+ let status = self.cloud_status().await?;+ let outputs = &status["observed"]["infra"]["outputs"];+ json!({"kind":"s3","deployment":plan.deployment_id,"bucket":outputs["ArtifactBucket"],"region":c.cloud.region()})+ } else {+ let m = self.azure_material().await?;+ json!({"kind":"azure","deployment":plan.deployment_id,"account":m["artifactAccount"],"container":"artifacts","tenant_id":m["target"]["tenant"],"client_id":m["artifactIdentity"]["properties"]["clientId"],"federated_token_file":"/var/run/secrets/azure/tokens/azure-identity-token"})+ };+ let provider = c+ .models+ .keys()+ .next()+ .map(String::as_str)+ .unwrap_or("openai");+ let model = match provider {+ "anthropic" => "claude-sonnet-4-6",+ _ => "unconfigured",+ };+ let connections:Vec<Value>=c.models.keys().enumerate().map(|(i,p)|{+ let id=if i==0{"primary"}else{p.as_str()};+ json!({"id":id,"provider":p,"identity":format!("{}-{id}",plan.deployment_id),"key_env":if p=="anthropic"{"ANTHROPIC_API_KEY"}else{"OPENAI_API_KEY"}})+ }).collect();+ let values = json!({"image":harness,"apiReplicas":1,"worker":{"replicas":1,"ownership":"manual","maxReplicas":2,"slots":2,"drainSeconds":30},"terminationGracePeriodSeconds":90,"secrets":{"model":if c.models.is_empty(){""}else{"ow-service-model"}},"config":{"service":{"id":plan.deployment_id,"namespace":"local","agent":"installed-tools","legacy_compatibility":false},"database_env":"OW_PG_URL","schema":"ow_service","database_loopback_plaintext":false,"database_ca":"/run/service/pg-ca.crt","state_root":"/state","bundle":"/opt/openworker/bundle","runner":"/opt/openworker/openworker","model":model,"scripted":false,"model_provider":provider,"model_connections":connections,"api_token_file":"/bootstrap/admin-token","actor":"deployment-admin","openshell":{"identity":plan.deployment_id,"endpoint":endpoint,"tls_server_name":format!("{fullname}.{ns}.svc.cluster.local"),"certificates":"/openshell","workspace":"default","image":runner},"artifacts":artifacts}});+ self.exact_file("service-values.json", &values)?;+ let mut shell: Value =+ serde_json::from_slice(&fs::read(&self.package.release.openshell_values.file)?)?;+ shell["fullnameOverride"] = json!(fullname);+ shell["sandbox"]["image"] = json!({"repository":runner.split('@').next().unwrap(),"digest":runner.split('@').nth(1).unwrap()});+ shell["server"]["workspaceStorageClass"] = json!(if self.provider() == "aws" {+ "ows-workspaces"+ } else {+ "managed-csi"+ });+ shell["pkiInitJob"]["serverDnsNames"] = json!([+ format!("{fullname}.{ns}.svc.cluster.local"),+ format!("{fullname}.{ns}.svc")+ ]);+ self.exact_file("openshell-values.json", &shell)?;+ let platform = if self.provider() == "aws" {+ let path = self.root.join("platform.json");+ if !path.exists() {+ let mut args =+ strings(&["deploy", "aws", "lifecycle", "platform-artifact", "--plan"]);+ args.extend([+ arg(&self.plan_root())?,+ "--out".into(),+ arg(&path)?,+ "--openshell-namespace".into(),+ ns.into(),+ "--openshell-release".into(),+ "ow-shell".into(),+ ]);+ self.command(args).await?;+ }+ json!({"file":path,"sha256":hash(&fs::read(&path)?)})+ } else {+ serde_json::to_value(&self.package.release.platform_manifest)?+ };+ let config = json!({"schemaVersion":1,"kubeconfig":self.root.join("handoff/kubeconfig.json"),"context":"pending-owned-handoff","helm":"helm","service":{"namespace":"ow-service","release":"ows","chart":self.package.release.service_chart,"valuesFile":self.root.join("service-values.json"),"fixtureDatabase":false},"openshell":{"mode":"install","namespace":ns,"release":"ow-shell","fullname":fullname,"archive":self.package.release.openshell_archive.file,"valuesFile":self.root.join("openshell-values.json")},"platform":platform});+ self.exact_file("installation-input.json", &config)?;+ Ok(())+ }+ fn exact_file(&self, name: &str, value: &Value) -> Result<()> {+ let p = self.root.join(name);+ if p.exists() {+ ensure!(+ self.document(&p)? == *value,+ "Generated installer input changed; retain the original handoff"+ );+ } else {+ write_private(&p, value)?;+ }+ Ok(())+ }+ pub(super) async fn handoff(&self) -> Result<Observation> {+ self.render_installation().await?;+ let path = self.root.join("handoff");+ let name = if self.provider() == "aws" {+ "installer-config.json"+ } else {+ "cluster-install.json"+ };+ if !path.join(name).exists() {+ ensure!(!path.exists(),"Partial installation handoff; inspect the original native receipt before continuing");+ let mut a = if self.provider() == "aws" {+ strings(&["deploy", "aws", "lifecycle", "installation"])+ } else {+ strings(&["deploy", "azure", "installation", "prepare"])+ };+ a.extend([+ "--state".into(),+ arg(&self.cloud_state())?,+ "--config".into(),+ arg(&self.root.join("installation-input.json"))?,+ "--out".into(),+ arg(&path)?,+ ]);+ if self.provider() == "aws" {+ a.extend(["--profile".into(), self.aws_profile()?]);+ }+ self.command(a).await?;+ }+ self.connect_private(&path.join("kubeconfig.json")).await?;+ if !self.install_state().exists() {+ let mut args = strings(&["deploy", "cluster", "plan", "--config"]);+ args.extend([+ arg(&path.join(name))?,+ "--out".into(),+ arg(&self.install_state())?,+ ]);+ self.command(args).await?;+ }+ Ok(Observation::Complete(+ json!({"installerState":self.install_state(),"managementApi":"private"}),+ ))+ }+ pub(super) async fn secrets(&self) -> Result<Observation> {+ let secret_dir = self.root.join("credentials");+ directory(&secret_dir)?;+ let cert = secret_dir.join("api.crt");+ let key = secret_dir.join("api.key");+ if !cert.exists() || !key.exists() {+ ensure!(+ !cert.exists() && !key.exists(),+ "Partial TLS generation; inspect private credential files before continuing"+ );+ self.tool("openssl",vec!["req".into(),"-x509".into(),"-newkey".into(),"rsa:3072".into(),"-nodes".into(),"-days".into(),"30".into(),"-subj".into(),"/CN=ows-api.ow-service.svc.cluster.local".into(),"-addext".into(),"subjectAltName=DNS:localhost,IP:127.0.0.1,DNS:ows-api.ow-service.svc.cluster.local,DNS:ows-api.ow-service.svc".into(),"-keyout".into(),arg(&key)?,"-out".into(),arg(&cert)?],None).await?;+ }+ #[cfg(unix)]+ {+ use std::os::unix::fs::PermissionsExt;+ fs::set_permissions(&key, fs::Permissions::from_mode(0o600))?;+ }+ let mut model = serde_json::Map::new();+ for (name, reference) in &self.review.config.models {+ let value = if let Some(env) = &reference.env {+ std::env::var(env).with_context(|| {+ format!("Missing model credential environment variable {env}")+ })?+ } else {+ fs::read_to_string(reference.file.as_ref().unwrap())+ .context("Read private model credential file")?+ };+ let value = value.trim();+ ensure!(+ !value.is_empty() && value.len() <= 32768 && !value.contains(['\r', '\n']),+ "Invalid model credential format"+ );+ model.insert(name.clone(), json!(value));+ }+ let mut input = json!({"database":{"pg-ca.crt":fs::read_to_string(&self.package.release.database_ca.file)?},"apiTLS":{"tls.crt":fs::read_to_string(&cert)?,"tls.key":fs::read_to_string(&key)?},"model":model});+ if self.review.config.models.is_empty() {+ input["model"] = Value::Null;+ }+ if self.provider() == "azure" {+ input["database"]["database-url"] = json!(self.azure_database_url().await?.as_str());+ }+ let path = secret_dir.join("installation-secrets.json");+ write_private(&path, &input)?;+ let native = self.document(&self.install_state().join("local-plan.json"))?;+ let mut a = strings(&["deploy", "cluster", "secrets", "--state"]);+ a.extend([+ arg(&self.install_state())?,+ "--accept-plan".into(),+ text(&native, "digest")?,+ "--input".into(),+ arg(&path)?,+ ]);+ let result = self.command(a).await?;+ Ok(Observation::Complete(result))+ }+ pub(super) async fn verify(&self) -> Result<Observation> {+ let result = self.installer("status", &[]).await?;+ let kube = self.root.join("handoff/kubeconfig.json");+ for component in ["api", "worker", "controller", "platform"] {+ self.tool(+ "kubectl",+ vec![+ "--kubeconfig".into(),+ arg(&kube)?,+ "-n".into(),+ "ow-service".into(),+ "rollout".into(),+ "status".into(),+ format!("deployment/ows-{component}"),+ "--timeout=180s".into(),+ ],+ None,+ )+ .await?;+ }+ let connection = json!({"deploymentId":self.review.deployment_id,"serviceId":self.review.deployment_id,"namespace":"local","url":"https://ows-api.ow-service.svc.cluster.local","access":"private cluster network","kubeconfig":kube,"caFile":self.root.join("credentials/api.crt"),"tokenFile":self.install_state().join("bootstrap-token"),"privatePreview":{"command":"kubectl","arguments":["--kubeconfig",arg(&kube)?,"-n","ow-service","port-forward","service/ows-api","8443:443","--address=127.0.0.1"],"url":"https://localhost:8443"},"modelSmokeTest":"not_run"});+ write_private(&self.root.join("connection.json"), &connection)?;+ Ok(Observation::Complete(+ json!({"nativeStatus":result,"rolloutsReady":true,"modelSmokeTest":"not_run","connectionFile":self.root.join("connection.json")}),+ ))+ }+}diff --git a/harness/service-cli/src/deployment_up_native.rs b/harness/service-cli/src/deployment_up_native.rsnew file mode 100644index 00000000..6a0bc3e9--- /dev/null+++ b/harness/service-cli/src/deployment_up_native.rs@@ -0,0 +1,709 @@+//! Adapter to the existing ownership-aware cloud and Kubernetes engines. The+//! coordinator never bypasses their fresh preflight, scope or receipt validation.+use super::*;+use config::Cloud;+use state::{Driver, Observation, Phase, Stage, Step};+#[path = "deployment_up_access.rs"]+mod access;+#[path = "deployment_up_azure_images.rs"]+mod azure_images;+#[path = "deployment_up_install.rs"]+mod install;+pub(super) struct Native<'a> {+ pub root: PathBuf,+ pub review: &'a Review,+ pub package: package::Snapshot,+ pub bootstrap_approved: bool,+ pub bridge: std::cell::RefCell<Option<access::Bridge>>,+}+fn arg(path: &Path) -> Result<String> {+ Ok(path+ .to_str()+ .context("Deployment paths must be UTF-8")?+ .into())+}+fn text(v: &Value, key: &str) -> Result<String> {+ Ok(v[key]+ .as_str()+ .with_context(|| format!("Missing native receipt field {key}"))?+ .into())+}+fn strings(args: &[&str]) -> Vec<String> {+ args.iter().map(|s| s.to_string()).collect()+}+impl Native<'_> {+ fn environment(&self) -> Result<BTreeMap<String, String>> {+ let mut env = BTreeMap::new();+ if self.root.join("aws-config").is_file() {+ env.insert(+ "AWS_CONFIG_FILE".into(),+ arg(&self.root.join("aws-config"))?,+ );+ }+ env.insert("AWS_MAX_ATTEMPTS".into(), "1".into());+ env.insert("AWS_PAGER".into(), String::new());+ env.insert("AWS_CLI_AUTO_PROMPT".into(), "off".into());+ env.insert("AZURE_EXTENSION_USE_DYNAMIC_INSTALL".into(), "no".into());+ if self.root.join("tools").is_dir() {+ env.insert(+ "PATH".into(),+ format!(+ "{}:{}",+ self.root.join("tools").display(),+ std::env::var("PATH").unwrap_or_default()+ ),+ );+ }+ Ok(env)+ }+ async fn command(&self, args: Vec<String>) -> Result<Value> {+ let bytes = super::super::local::exec_with_env(+ &std::env::current_exe()?,+ &args,+ None,+ &self.environment()?,+ )+ .await?;+ serde_json::from_slice(&bytes).context("Invalid native receipt")+ }+ fn provider(&self) -> &str {+ self.review.config.cloud.provider()+ }+ fn plan_root(&self) -> PathBuf {+ self.root.join("infrastructure")+ }+ fn cloud_state(&self) -> PathBuf {+ self.root.join("cloud")+ }+ fn install_state(&self) -> PathBuf {+ self.root.join("installation")+ }+ fn document(&self, path: &Path) -> Result<Value> {+ Ok(serde_json::from_slice(&fs::read(path)?)?)+ }+ fn plan(&self) -> Result<(super::super::Plan, super::super::Manifest)> {+ super::super::inspect(&self.plan_root())+ }+ fn aws_profile(&self) -> Result<String> {+ ensure!(self.provider() == "aws", "AWS profile required");+ Ok(format!("ow-{}", &self.review.deployment_id[4..16]))+ }+ fn native_plan(&self) -> Result<super::super::Plan> {+ let c = &self.review.config;+ let package = &self.package;+ let created = self.review.resolution.observed_at_unix;+ let mut value = json!({"schemaVersion":if self.provider()=="aws"{3}else{2},"deploymentId":self.review.deployment_id,"name":c.name,"createdAtUnix":created,"expiresAtUnix":created+u64::from(c.trial.as_ref().map(|t|t.expires_after_hours).unwrap_or(72))*3600,"autoExpire":c.trial.is_some(),"budget":{"authorizedUsd":c.budget.alert_usd,"hardCap":false,"estimate":null},"topology":{"clusterMode":"new","nodeCount":c.nodes.count,"maxWorkerReplicas":2,"architecture":self.review.resolution.node.architecture,"access":"private","nodeAutoscaling":false}});+ let mut inputs = package.setup.infrastructure.clone();+ if let Some(version) = &c.advanced.kubernetes_version {+ inputs["kubernetesVersion"] = json!(version);+ }+ if let Some(cidr) = &c.advanced.network_cidr {+ ensure!(+ self.provider() == "aws",+ "Custom network CIDR is supported only by the AWS template"+ );+ inputs["vpcCidr"] = json!(cidr);+ }+ match &c.cloud {+ Cloud::Aws {+ account_id, region, ..+ } => {+ value["target"] = json!({"provider":"aws","account":account_id,"region":region});+ let operator = c+ .advanced+ .aws+ .as_ref()+ .and_then(|a| a.operator_arn.as_ref())+ .or(package.setup.operator_arn.as_ref())+ .context("AWS setup profile needs its approved IAM operatorArn")?;+ let prefix = format!("ows-{}-{}", c.name, &self.review.deployment_id[4..16]);+ value["bootstrap"] = json!({"operatorArn":operator,"executionRoleArn":format!("arn:aws:iam::{account_id}:role/{prefix}-executor")});+ inputs["nodeInstanceType"] = json!(self.review.resolution.node.instance_type);+ inputs["acknowledgeExperimentDataDeletion"] = json!(c.trial.is_some());+ value["infrastructure"] = inputs;+ }+ Cloud::Azure {+ subscription_id,+ region,+ } => {+ value["target"] = json!({"provider":"azure","tenant":self.review.resolution.identity["tenantId"],"subscription":subscription_id,"location":region});+ let azure = c.advanced.azure.as_ref();+ if let Some(a) = azure {+ if let Some(group) = &a.control_resource_group {+ inputs["controlResourceGroup"] = json!(group);+ }+ if !a.admin_group_object_ids.is_empty() {+ inputs["adminGroupObjectIds"] = json!(a.admin_group_object_ids);+ }+ if let Some(key) = &a.ssh_public_key_file {+ inputs["sshPublicKey"] = json!(fs::read_to_string(key)?.trim());+ }+ }+ value["azureBootstrap"] = json!({"resourceGroup":azure.and_then(|a| a.resource_group.as_ref()).or(package.setup.resource_group.as_ref()).context("Azure setup profile needs an approved workload resourceGroup")?,"setupObjectId":package.setup.setup_object_id.as_ref().context("Azure setup profile needs an exact setupObjectId")?,"setupPrincipalType":package.setup.setup_principal_type.as_deref().unwrap_or("ServicePrincipal")});+ inputs["nodeVmSize"] = json!(self.review.resolution.node.instance_type);+ value["azureInfrastructure"] = inputs;+ }+ }+ let p: super::super::Plan =+ serde_json::from_value(value).context("Invalid reviewed cloud setup inputs")?;+ p.validate()?;+ if self.provider() == "aws" {+ ensure!(+ super::super::guard::compatible(&p),+ "Setup profile must reference the guard archive packaged by this CLI release"+ );+ }+ Ok(p)+ }+ pub(super) fn artifact_digest(&self) -> Result<String> {+ let files = super::super::artifacts(&self.native_plan()?)?;+ let hashes: BTreeMap<_, _> = files.iter().map(|(n, b)| (n, hash(b))).collect();+ Ok(hash(&serde_json::to_vec(&hashes)?))+ }+ pub(super) async fn review_bundle(&self) -> Result<()> {+ self.bundle().await?;+ Ok(())+ }+ pub(super) async fn restore_access(&self) -> Result<()> {+ self.connect_private(&self.root.join("handoff/kubeconfig.json"))+ .await+ }+ pub(super) async fn verify_current(&self) -> Result<()> {+ self.connect_private(&self.root.join("handoff/kubeconfig.json"))+ .await?;+ self.verify().await?;+ Ok(())+ }+ async fn bundle(&self) -> Result<Value> {+ let expected = self.native_plan()?;+ ensure!(+ self.artifact_digest()? == self.review.native_bundle_digest,+ "Native cloud graph changed since approval"+ );+ if self.plan_root().exists() {+ let (p, m) = self.plan()?;+ ensure!(+ p == expected,+ "Original infrastructure bundle differs from review"+ );+ return Ok(json!({"bundleDigest":m.bundle_digest}));+ }+ let temporary = self+ .root+ .join(format!(".infrastructure-{}", uuid::Uuid::new_v4().simple()));+ super::super::write_bundle(&temporary, &expected)?;+ fs::rename(&temporary, self.plan_root())?;+ Ok(json!({"bundleDigest":self.plan()?.1.bundle_digest}))+ }+ async fn prepare_state(&self) -> Result<()> {+ let file = if self.provider() == "aws" {+ "cloud-journal.json"+ } else {+ "azure-state.json"+ };+ if self.cloud_state().join(file).exists() {+ return Ok(());+ }+ ensure!(+ !self.cloud_state().exists(),+ "Partial local lifecycle state; inspect before continuation"+ );+ let temp = self+ .root+ .join(format!(".cloud-{}", uuid::Uuid::new_v4().simple()));+ let mut args = strings(&["deploy", self.provider(), "lifecycle", "prepare", "--plan"]);+ args.push(arg(&self.plan_root())?);+ args.extend(["--out".into(), arg(&temp)?]);+ self.command(args).await?;+ fs::rename(temp, self.cloud_state())?;+ Ok(())+ }+ async fn cloud_status(&self) -> Result<Value> {+ let mut a = strings(&["deploy", self.provider(), "lifecycle", "status", "--state"]);+ a.push(arg(&self.cloud_state())?);+ if self.provider() == "aws" {+ a.extend(["--profile".into(), self.aws_profile()?]);+ }+ self.command(a).await+ }+ async fn cloud_apply(&self, stage: &str) -> Result<Value> {+ let (_, m) = self.plan()?;+ let report = self.root.join("preflight/preflight.json");+ let stale = if report.exists() {+ let r = self.document(&report)?;+ r["observedAtUnix"]+ .as_u64()+ .is_none_or(|at| now().is_ok_and(|n| at > n || n - at > 300))+ } else {+ true+ };+ if stale {+ let r = self.preflight().await?;+ ensure!(+ r["preflightReady"] == true || r["checksPassed"] == true,+ "Fresh cloud prerequisites are not ready; original requests are retained"+ );+ }+ let mut a = strings(&["deploy", self.provider(), "lifecycle", "apply", "--state"]);+ a.push(arg(&self.cloud_state())?);+ a.extend([+ if self.provider() == "aws" {+ "--accept-plan"+ } else {+ "--accept"+ }+ .into(),+ m.bundle_digest,+ "--allow-experimental-cloud".into(),+ ]);+ if self.provider() == "aws" {+ a.extend([+ "--profile".into(),+ self.aws_profile()?,+ "--preflight".into(),+ arg(&self.root.join("preflight/preflight.json"))?,+ ]);+ } else {+ a.extend(["--stage".into(), stage.into()]);+ }+ self.command(a).await+ }+ fn aws_config(&self) -> Result<()> {+ let Cloud::Aws {+ profile, region, ..+ } = &self.review.config.cloud+ else {+ return Ok(());+ };+ let (plan, _) = self.plan()?;+ let source = std::env::var_os("AWS_CONFIG_FILE")+ .map(PathBuf::from)+ .unwrap_or_else(|| {+ PathBuf::from(std::env::var_os("HOME").unwrap_or_default()).join(".aws/config")+ });+ let mut content = if source.exists() {+ fs::read_to_string(source)?+ } else {+ String::new()+ };+ let name = self.aws_profile()?;+ ensure!(+ !content.contains(&format!("[profile {name}]")),+ "Generated setup profile collides with the original AWS configuration"+ );+ content.push_str(&format!("\n[profile {name}]\nrole_arn = arn:aws:iam::{}:role/{}-setup\nsource_profile = {profile}\nregion = {region}\nrole_session_name = {name}\n",self.review.config.cloud.account(),plan.prefix()));+ let path = self.root.join("aws-config");+ if path.exists() {+ ensure!(+ fs::read_to_string(&path)? == content,+ "AWS login configuration changed; inspect the pinned setup profile before resuming"+ );+ return Ok(());+ }+ use std::io::Write;+ let mut options = fs::OpenOptions::new();+ options.write(true).create_new(true);+ #[cfg(unix)]+ {+ use std::os::unix::fs::OpenOptionsExt;+ options.mode(0o600);+ }+ options.open(path)?.write_all(content.as_bytes())?;+ Ok(())+ }+ async fn aws_bootstrap(&self) -> Result<Observation> {+ if !self.bootstrap_approved {+ return Ok(Observation::Blocked(format!("Identity setup needs separate approval. Review {} and rerun with --accept-bootstrap {}. Normal setup never silently grants IAM.",self.plan_root().join("permission-review.json").display(),self.review.digest)));+ }+ let Cloud::Aws { profile, .. } = &self.review.config.cloud else {+ bail!("AWS configuration required")+ };+ let (_, manifest) = self.plan()?;+ let mut args = strings(&["deploy", "aws", "lifecycle", "bootstrap", "--plan"]);+ args.extend([+ arg(&self.plan_root())?,+ "--state".into(),+ arg(&self.root)?,+ "--profile".into(),+ profile.clone(),+ "--accept-plan".into(),+ manifest.bundle_digest,+ "--approve-identities".into(),+ ]);+ let receipt = self.command(args).await?;+ if receipt["phase"] == "ready" {+ self.aws_config()?;+ Ok(Observation::Complete(receipt))+ } else {+ Ok(Observation::Pending)+ }+ }+ async fn azure_bootstrap(&self, phase: &str) -> Result<Observation> {+ if !self.bootstrap_approved {+ return Ok(Observation::Blocked(format!("Review the exact Azure role/control payloads under {}, then approve with --accept-bootstrap {}. No implicit administrator grants are made.",self.plan_root().display(),self.review.digest)));+ }+ let (_, manifest) = self.plan()?;+ let mut a = strings(&["deploy", "azure", "installation", "bootstrap", "--plan"]);+ a.extend([+ arg(&self.plan_root())?,+ "--state".into(),+ arg(&self.cloud_state())?,+ "--phase".into(),+ phase.into(),+ "--accept-plan".into(),+ manifest.bundle_digest,+ "--approve-identities".into(),+ ]);+ let receipt = self.command(a).await?;+ Ok(if receipt["phase"] == "ready" {+ Observation::Complete(receipt)+ } else {+ Observation::Pending+ })+ }+ async fn preflight(&self) -> Result<Value> {+ let output = self+ .root+ .join(format!("preflight-{}", uuid::Uuid::new_v4().simple()));+ let mut args = strings(&["deploy", self.provider(), "preflight", "--plan"]);+ args.push(arg(&self.plan_root())?);+ args.extend([+ "--out".into(),+ arg(&output)?,+ "--timeout-seconds".into(),+ "60".into(),+ "--overall-timeout-seconds".into(),+ "180".into(),+ ]);+ if let Cloud::Aws { profile, .. } = &self.review.config.cloud {+ args.extend([+ "--profile".into(),+ profile.clone(),+ "--check-setup-role".into(),+ ]);+ }+ let result = self.command(args).await;+ let report = self.document(&output.join("preflight.json"))?;+ let (_, manifest) = self.plan()?;+ ensure!(+ report["deploymentId"] == self.review.deployment_id+ && report["bundleDigest"] == manifest.bundle_digest,+ "Preflight report is not bound to the original reviewed graph"+ );+ let _ = result;+ let dest = self.root.join("preflight");+ if dest.exists() {+ fs::rename(+ &dest,+ self.root.join(format!(+ "preflight-previous-{}",+ uuid::Uuid::new_v4().simple()+ )),+ )?;+ }+ fs::rename(output, &dest)?;+ Ok(report)+ }+ async fn installer(&self, verb: &str, extra: &[&str]) -> Result<Value> {+ let p = self.document(&self.install_state().join("local-plan.json"))?;+ let mut a = strings(&["deploy", "cluster", verb, "--state"]);+ a.push(arg(&self.install_state())?);+ if verb != "status" {+ a.extend(["--accept-plan".into(), text(&p, "digest")?]);+ }+ a.extend(extra.iter().map(|s| s.to_string()));+ self.command(a).await+ }+ fn ready_install(&self, stage: Stage) -> Result<Option<Value>> {+ let path = self.install_state().join("journal.json");+ if !path.exists() {+ return Ok(None);+ }+ let j = self.document(&path)?;+ let phase = j["phase"].as_str().unwrap_or("");+ let ready = match stage {+ Stage::Platform => j["operations"].as_object().is_some_and(|m| {+ m.iter()+ .filter(|(k, _)| k.starts_with("platform:"))+ .all(|(_, v)| v["phase"] == "complete")+ && m.keys().any(|k| k.starts_with("platform:"))+ }),+ Stage::OpenShell => ["openshell_ready", "installed"].contains(&phase),+ Stage::Secrets => j["operations"].as_object().is_some_and(|m| {+ [+ "secret:ow-service-database",+ "secret:ow-service-tls",+ "secret:ow-service-bootstrap",+ ]+ .iter()+ .all(|k| m.get(*k).is_some_and(|v| v["phase"] == "complete"))+ }),+ Stage::Service => phase == "installed",+ _ => false,+ };+ Ok(ready.then_some(json!({"phase":phase,"installer":self.install_state()})))+ }+}+impl Driver for Native<'_> {+ async fn observe(&mut self, stage: Stage, step: Option<&Step>) -> Result<Observation> {+ self.package.validate(&self.review.config)?;+ match stage {+ Stage::Bundle => {+ if self.plan_root().exists() {+ let r = self.bundle().await?;+ self.prepare_state().await?;+ Ok(Observation::Complete(r))+ } else {+ Ok(Observation::Absent)+ }+ }+ Stage::Preflight => Ok(Observation::Absent), // read-only; always obtain fresh observations+ Stage::Permissions+ if self.provider() == "aws"+ && self.root.join("bootstrap-receipt.json").exists() =>+ {+ self.aws_bootstrap().await+ }+ Stage::Permissions => Ok(Observation::Absent),+ Stage::Control | Stage::IdentityBindings if self.provider() == "azure" => {+ Ok(Observation::Absent)+ }+ Stage::Images => Ok(Observation::Absent),+ Stage::PrivateAccess => Ok(Observation::Absent),+ Stage::Verify => Ok(Observation::Absent),+ Stage::Control if self.provider() == "aws" => Ok(Observation::Absent),+ Stage::IdentityBindings if self.provider() == "aws" => Ok(Observation::Complete(+ json!({"managedBy":"reviewed CloudFormation graph"}),+ )),+ Stage::Foundation | Stage::Cluster => {+ if !self.cloud_state().exists() {+ return Ok(Observation::Absent);+ }+ let status = self.cloud_status().await?;+ if self.provider() == "aws" {+ if status["observed"]["infra"]["status"] == "CREATE_COMPLETE" {+ return Ok(Observation::Complete(status));+ }+ let journal = self.document(&self.cloud_state().join("cloud-journal.json"))?;+ let uncertain = journal["operations"]+ .as_object()+ .is_some_and(|ops| ops.values().any(|o| o["status"] == "unknown"));+ if uncertain {+ return Ok(Observation::Unknown("Inspect original AWS lifecycle receipts; do not create a replacement stack".into()));+ }+ // Native provision reconciles existing create intents, refuses+ // unknown effects and advances only dependencies already ready.+ Ok(Observation::Absent)+ } else {+ let name = if stage == Stage::Foundation {+ "foundation"+ } else {+ "cluster"+ };+ let op = status["stages"]+ .as_array()+ .and_then(|a| a.iter().find(|v| v["stage"] == name))+ .context("Azure stage receipt missing")?;+ match op["phase"].as_str(){Some("active")=>Ok(Observation::Complete(status)),Some("prepared")=>Ok(Observation::Absent),Some("creating"|"unknown"|"intent")=>{+ let(_,m)=self.plan()?;let mut a=strings(&["deploy","azure","lifecycle","reconcile","--state"]);a.extend([arg(&self.cloud_state())?,"--accept".into(),m.bundle_digest,"--allow-experimental-cloud".into()]);self.command(a).await?;Ok(Observation::Pending)+ },_=>Ok(Observation::Blocked("Azure reported a failed deployment; inspect the original ARM operation before continuing".into()))}+ }+ }+ Stage::Platform | Stage::OpenShell | Stage::Secrets | Stage::Service => {+ if let Some(receipt) = self.ready_install(stage)? {+ return Ok(Observation::Complete(receipt));+ }+ // All mutation retry decisions belong to the existing installer,+ // which checks namespace UIDs and persisted Helm/Secret intents.+ Ok(Observation::Absent)+ }+ _ => {+ if step.is_some_and(|s| s.phase == Phase::Intent || s.phase == Phase::Pending) {+ Ok(Observation::Unknown(+ "Read the original provider receipt before retrying this stage".into(),+ ))+ } else {+ Ok(Observation::Absent)+ }+ }+ }+ }+ async fn execute(&mut self, stage: Stage, _request: &str) -> Result<Observation> {+ match stage {+ Stage::Bundle => {+ let r = self.bundle().await?;+ self.prepare_state().await?;+ Ok(Observation::Complete(r))+ }+ Stage::Permissions if self.provider() == "aws" => self.aws_bootstrap().await,+ Stage::Control if self.provider() == "aws" => {+ let r = self.cloud_apply("foundation").await?;+ Ok(+ if ["infra_creating", "infrastructure_ready"]+ .contains(&r["phase"].as_str().unwrap_or(""))+ {+ Observation::Complete(r)+ } else {+ Observation::Pending+ },+ )+ }+ Stage::Permissions if self.provider() == "azure" => self.azure_bootstrap("setup").await,+ Stage::Control if self.provider() == "azure" => self.azure_bootstrap("control").await,+ Stage::IdentityBindings if self.provider() == "azure" => {+ self.azure_bootstrap("foundation").await+ }+ Stage::Images => {+ if self.provider() == "azure" {+ match self.azure_bootstrap("federation").await? {+ Observation::Complete(_) => {}+ other => return Ok(other),+ }+ }+ self.images().await+ }+ Stage::PrivateAccess => self.handoff().await,+ Stage::Secrets => self.secrets().await,+ Stage::Verify => self.verify().await,+ Stage::Preflight => {+ let r = self.preflight().await?;+ if r["preflightReady"] == true || r["checksPassed"] == true {+ Ok(Observation::Complete(r))+ } else {+ Ok(Observation::Blocked(format!("Cloud prerequisites need attention. Review {} and rerun the same up command; no infrastructure request was submitted.",self.root.join("preflight/preflight.json").display())))+ }+ }+ Stage::Foundation => {+ let r = self.cloud_apply("foundation").await?;+ Ok(+ if self.provider() == "aws" && r["phase"] == "infrastructure_ready" {+ Observation::Complete(r)+ } else {+ Observation::Pending+ },+ )+ }+ Stage::Cluster => {+ self.cloud_apply("cluster").await?;+ Ok(Observation::Pending)+ }+ Stage::Platform => {+ let r = self+ .installer("platform", &["--allow-experimental-cloud"])+ .await?;+ Ok(Observation::Complete(r))+ }+ Stage::OpenShell => {+ let r = self.installer("apply", &["--openshell-only"]).await?;+ Ok(Observation::Complete(r))+ }+ Stage::Service => {+ let r = self.installer("apply", &[]).await?;+ ensure!(r["phase"] == "installed", "Service installation not ready");+ Ok(Observation::Complete(r))+ }+ _ => Ok(Observation::Blocked(format!(+ "Stage {stage:?} needs account installation wiring before provisioning can proceed"+ ))),+ }+ }+}++#[cfg(test)]+mod contract_tests {+ use super::*;+ fn fixture(cloud: &str) -> (Review, package::Snapshot) {+ let aws = cloud == "aws";+ let account = if aws {+ "123456789012"+ } else {+ "11111111-2222-4333-8444-555555555555"+ };+ let mut infrastructure: Value = if aws {+ serde_json::from_str(include_str!("../examples/aws-infrastructure.json")).unwrap()+ } else {+ json!({"kubernetesVersion":"1.33.4","nodeVmSize":"Standard_D4s_v5","sshPublicKey":"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","adminGroupObjectIds":["11111111-1111-4111-8111-111111111111"],"controlResourceGroup":"example-control","guardImage":format!("ghcr.io/test/ows@sha256:{}","a".repeat(64)),"postgresPassword":{"keyVaultId":format!("/subscriptions/{account}/resourceGroups/example-secrets/providers/Microsoft.KeyVault/vaults/example-vault"),"secretName":"postgres","secretVersion":"a".repeat(32)}})+ };+ if aws {+ infrastructure["guardCode"]["sha256"] =+ json!(super::super::super::guard::archive_sha());+ infrastructure["operatorAccess"] =+ json!({"mode":"ssm","amiId":"ami-0123456789abcdef0"});+ }+ let target = if aws {+ json!({"provider":"aws","accountId":account,"region":"us-west-2"})+ } else {+ json!({"provider":"azure","subscriptionId":account,"region":"westus2"})+ };+ let size = if aws { "t4g.xlarge" } else { "Standard_D4s_v5" };+ let arch = if aws { "arm64" } else { "amd64" };+ let config:Config=serde_json::from_value(json!({"name":"example","cloud":target,"nodes":{"count":3,"instanceType":size},"budget":{"limitUSD":1000}})).unwrap();+ let pin = json!({"file":"/fixture","sha256":"a".repeat(64)});+ let package:package::Snapshot=serde_json::from_value(json!({"digest":"fixture","setup":{"schemaVersion":1,"provider":cloud,"account":account,"region":config.cloud.region(),"infrastructure":infrastructure,"operatorArn":"arn:aws:iam::123456789012:role/operator","setupObjectId":"bbbbbbbb-cccc-4ddd-8eee-ffffffffffff","setupPrincipalType":"User","resourceGroup":"example-workload","release":"/release.json"},"release":{"schemaVersion":1,"version":"test","architecture":arch,"images":{"harness":format!("ghcr.io/test/harness@sha256:{}","a".repeat(64)),"runner":format!("ghcr.io/test/runner@sha256:{}","b".repeat(64))},"serviceChart":"/chart","serviceChartSha256":"c".repeat(64),"openshellArchive":pin,"openshellValues":pin,"platformManifest":pin,"databaseCa":pin}})).unwrap();+ let review=Review{schema_version:1,config_digest:config.digest().unwrap(),config,deployment_id:"ows-0123456789abcdef0123456789abcdef".into(),package_digest:"fixture".into(),native_bundle_digest:String::new(),digest:String::new(),resolution:serde_json::from_value(json!({"node":{"instanceType":size,"cpu":4,"memoryGiB":16,"architecture":arch,"eligible":true,"hourlyUsd":0.2},"price":{"currency":"USD","workerHourlyUsd":0.6,"totalHourlyUsd":null,"knownHourlySubtotalUsd":0.6,"components":{},"variableAllowanceUsd":25,"plannedHours":730,"unpricedComponents":["network"],"hardCap":false},"observedAtUnix":1000,"identity":{"tenantId":"11111111-2222-4333-8444-555555555555"}})).unwrap()};+ (review, package)+ }+ #[test]+ fn both_native_graphs_preserve_fixed_capacity_and_persistent_policy() {+ for cloud in ["aws", "azure"] {+ let (review, package) = fixture(cloud);+ let native = Native {+ root: PathBuf::from("/fixture"),+ review: &review,+ package,+ bootstrap_approved: false,+ bridge: Default::default(),+ };+ let plan = native.native_plan().unwrap();+ assert_eq!(plan.topology.node_count, 3);+ assert_eq!(plan.auto_expire, Some(false));+ assert!(!plan.expires_workload());+ let files = super::super::super::artifacts(&plan).unwrap();+ if cloud == "aws" {+ let graph: Value =+ serde_json::from_slice(&files["infrastructure.cfn.json"]).unwrap();+ let group = graph["Resources"]+ .as_object()+ .unwrap()+ .values()+ .find(|v| v["Type"] == "AWS::EKS::Nodegroup")+ .unwrap();+ assert_eq!(+ group["Properties"]["ScalingConfig"],+ json!({"DesiredSize":3,"MinSize":3,"MaxSize":3})+ );+ assert_eq!(group["Properties"]["InstanceTypes"], json!(["t4g.xlarge"]));+ let setup: Value = serde_json::from_slice(&files["setup-policy.json"]).unwrap();+ assert!(setup["Statement"]+ .as_array()+ .unwrap()+ .iter()+ .any(|s| s["Sid"] == "OwnManagementSessionOnly"+ && s["Condition"]["StringEquals"]+ ["ssm:resourceTag/aws:ssmmessages:session-id"]+ == "${aws:userid}"));+ } else {+ let graph: Value =+ serde_json::from_slice(&files["azure-cluster.arm.json"]).unwrap();+ let cluster = graph["resources"]+ .as_array()+ .unwrap()+ .iter()+ .find(|v| v["type"] == "Microsoft.ContainerService/managedClusters")+ .unwrap();+ assert_eq!(cluster["properties"]["agentPoolProfiles"][0]["count"], 3);+ let grants: Value = serde_json::from_slice(&files["azure-up-grants.json"]).unwrap();+ for g in grants["grants"].as_array().unwrap() {+ assert!(g["id"].as_str().unwrap().contains("/resourceGroups/"));+ assert!(!g.to_string().contains("roleAssignments/write"));+ if g["label"].as_str().unwrap().ends_with("-grant") {+ assert_eq!(+ g["body"]["properties"]["principalId"],+ "bbbbbbbb-cccc-4ddd-8eee-ffffffffffff"+ );+ }+ }+ }+ }+ }+}diff --git a/harness/service-cli/src/deployment_up_package.rs b/harness/service-cli/src/deployment_up_package.rsnew file mode 100644index 00000000..3a967875--- /dev/null+++ b/harness/service-cli/src/deployment_up_package.rs@@ -0,0 +1,258 @@+//! Release assets and account prerequisites are installed once, separately from+//! application desired state. Neither document can inject executable commands.+use super::*;+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct FilePin {+ pub file: PathBuf,+ pub sha256: String,+}+impl FilePin {+ fn validate(&self) -> Result<()> {+ let meta = fs::symlink_metadata(&self.file)?;+ ensure!(+ meta.is_file() && !meta.file_type().is_symlink() && meta.len() <= 16 * 1024 * 1024,+ "Release file must be a bounded regular file"+ );+ ensure!(+ hash(&fs::read(&self.file)?) == self.sha256,+ "Release artifact checksum mismatch"+ );+ Ok(())+ }+}+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Images {+ pub harness: String,+ pub runner: String,+ pub service_ui: Option<String>,+}+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Release {+ pub schema_version: u16,+ pub version: String,+ pub architecture: String,+ pub images: Images,+ pub service_chart: PathBuf,+ pub service_chart_sha256: String,+ pub openshell_archive: FilePin,+ pub openshell_values: FilePin,+ pub platform_manifest: FilePin,+ pub database_ca: FilePin,+}+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Setup {+ pub schema_version: u16,+ pub provider: String,+ pub account: String,+ pub region: String,+ /// Existing provider input schema: pinned versions, scoped bootstrap identities+ /// and secret-manager references. Never plaintext database/model passwords.+ pub infrastructure: Value,+ pub operator_arn: Option<String>,+ pub setup_object_id: Option<String>,+ pub setup_principal_type: Option<String>,+ pub resource_group: Option<String>,+ pub release: PathBuf,+}+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Snapshot {+ pub setup: Setup,+ pub release: Release,+ pub digest: String,+}+fn image(s: &str) -> bool {+ s.len() < 300+ && s.split_once("@sha256:").is_some_and(|(name, d)| {+ name.contains('/')+ && name+ .bytes()+ .all(|b| b.is_ascii_alphanumeric() || b"/._-:".contains(&b))+ && d.len() == 64+ && d.bytes()+ .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))+ })+}+fn tree(root: &Path) -> Result<String> {+ let meta = fs::symlink_metadata(root)?;+ ensure!(+ meta.is_dir() && !meta.file_type().is_symlink(),+ "Release chart root must be a real directory"+ );+ fn walk(root: &Path, dir: &Path, out: &mut BTreeMap<String, String>) -> Result<()> {+ ensure!(+ dir.strip_prefix(root)?.components().count() <= 16,+ "Release chart nesting too deep"+ );+ for p in fs::read_dir(dir)? {+ let p = p?;+ let meta = fs::symlink_metadata(p.path())?;+ ensure!(+ !meta.file_type().is_symlink(),+ "Release chart cannot contain symlinks"+ );+ if meta.is_dir() {+ walk(root, &p.path(), out)?;+ } else {+ ensure!(+ meta.is_file() && meta.len() <= 1024 * 1024,+ "Invalid chart file"+ );+ let name = p+ .path()+ .strip_prefix(root)?+ .components()+ .map(|c| c.as_os_str().to_string_lossy())+ .collect::<Vec<_>>()+ .join("/");+ out.insert(name, hash(&fs::read(p.path())?));+ ensure!(out.len() <= 128, "Release chart too large");+ }+ }+ Ok(())+ }+ let mut files = BTreeMap::new();+ walk(root, root, &mut files)?;+ ensure!(+ files.contains_key("Chart.yaml"),+ "Service chart missing Chart.yaml"+ );+ Ok(hash(&serde_json::to_vec(&files)?))+}+impl Snapshot {+ pub fn validate(&self, c: &Config) -> Result<()> {+ ensure!(+ self.setup.schema_version == 1+ && self.setup.provider == c.cloud.provider()+ && self.setup.account == c.cloud.account()+ && self.setup.region == c.cloud.region(),+ "Setup profile belongs to another cloud target"+ );+ ensure!(+ self.release.schema_version == 1+ && self.release.architecture+ == if c.cloud.provider() == "aws" {+ "arm64"+ } else {+ "amd64"+ },+ "Release architecture/provider mismatch"+ );+ for i in [&self.release.images.harness, &self.release.images.runner]+ .into_iter()+ .chain(self.release.images.service_ui.iter())+ {+ ensure!(image(i), "Release images must be pinned by SHA-256");+ }+ for f in [+ &self.release.openshell_archive,+ &self.release.openshell_values,+ &self.release.platform_manifest,+ &self.release.database_ca,+ ] {+ f.validate()?;+ }+ ensure!(+ self.release.openshell_archive.sha256+ == "5ce3e63648d1a80f96f748063e44444069667ae9049457b8aded35cdb2a6171a",+ "New deployment requires the reviewed OpenShell 0.1.7 archive"+ );+ let values: Value =+ serde_json::from_slice(&fs::read(&self.release.openshell_values.file)?)?;+ ensure!(+ values["global"]["image"]["tag"] == "0.1.7"+ && values["server"]["auth"]["allowUnauthenticatedUsers"] == false,+ "OpenShell release/authentication mismatch"+ );+ let ca = fs::read(&self.release.database_ca.file)?;+ let certificates =+ reqwest::Certificate::from_pem_bundle(&ca).context("Invalid database CA bundle")?;+ ensure!(+ !certificates.is_empty() && certificates.len() <= 32,+ "Database CA bundle must contain 1..32 certificates"+ );+ ensure!(+ tree(&self.release.service_chart)? == self.release.service_chart_sha256,+ "Service chart changed since release packaging"+ );+ ensure!(+ self.digest == hash(&serde_json::to_vec(&(&self.setup, &self.release))?),+ "Release/setup review digest changed"+ );+ Ok(())+ }+}+pub(super) fn load(c: &Config) -> Result<Snapshot> {+ let path = if let Some(p) = &c.advanced.setup_profile {+ p.clone()+ } else {+ let home =+ std::env::var_os("HOME").context("HOME unavailable; set advanced.setupProfile")?;+ PathBuf::from(home)+ .join(".config/ows/deployments")+ .join(c.cloud.provider())+ .join(c.cloud.account())+ .join(format!("{}.json", c.cloud.region()))+ };+ ensure!(path.is_file(),"Account setup profile is missing at {}. Install the reviewed account/release prerequisites once (see deploy/service/up/README.md), then rerun the same up command. No cloud resources were created.",path.display());+ ensure!(+ fs::metadata(&path)?.len() <= 65536,+ "Setup profile exceeds 64 KiB"+ );+ let setup: Setup = serde_json::from_slice(&fs::read(&path)?).map_err(|_| {+ anyhow::anyhow!("Invalid account setup profile; use references, not inline credentials")+ })?;+ let release_path = c.advanced.release.as_ref().unwrap_or(&setup.release);+ ensure!(+ fs::metadata(release_path)?.len() <= 65536,+ "Release manifest exceeds 64 KiB"+ );+ let release: Release = serde_json::from_slice(&fs::read(release_path)?)+ .context("Read reviewed release manifest")?;+ let digest = hash(&serde_json::to_vec(&(&setup, &release))?);+ let s = Snapshot {+ setup,+ release,+ digest,+ };+ s.validate(c)?;+ Ok(s)+}++#[cfg(test)]+mod tests {+ use super::*;+ #[test]+ fn changed_chart_or_file_changes_release_identity() {+ let d = tempfile::tempdir().unwrap();+ let p = d.path();+ fs::write(p.join("Chart.yaml"), "name: example").unwrap();+ let initial = tree(p).unwrap();+ fs::write(p.join("values.yaml"), "image: changed").unwrap();+ assert_ne!(tree(p).unwrap(), initial);+ let pin = FilePin {+ file: p.join("Chart.yaml"),+ sha256: hash(b"name: example"),+ };+ pin.validate().unwrap();+ fs::write(&pin.file, "changed").unwrap();+ assert!(pin.validate().is_err());+ assert!(!image("ghcr.io/example/service:latest"));+ assert!(image(&format!(+ "ghcr.io/example/service@sha256:{}",+ "a".repeat(64)+ )));+ }+ #[cfg(unix)]+ #[test]+ fn release_chart_cannot_follow_links() {+ let d = tempfile::tempdir().unwrap();+ fs::write(d.path().join("Chart.yaml"), "name: test").unwrap();+ std::os::unix::fs::symlink(d.path(), d.path().join("loop")).unwrap();+ assert!(tree(d.path()).is_err());+ }+}diff --git a/harness/service-cli/src/deployment_up_pricing.rs b/harness/service-cli/src/deployment_up_pricing.rsnew file mode 100644index 00000000..f6b66e48--- /dev/null+++ b/harness/service-cli/src/deployment_up_pricing.rs@@ -0,0 +1,532 @@+//! Public catalog observations, not a bill or spending cap. Missing and ambiguous+//! prices stay unknown; variable allowances are visible and are never upper bounds.+use super::*;+use config::{Cloud, Config};+use resolve::Node;+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Line {+ pub hourly_usd: f64,+ pub quantity: f64,+ pub source: String,+ pub meter: String,+}+fn rate(v: &Value) -> Option<f64> {+ let n = v.as_f64().or_else(|| v.as_str()?.parse().ok())?;+ (n.is_finite() && n > 0.0).then_some(n)+}+fn unique(rows: Vec<(String, f64)>) -> Option<(String, f64)> {+ (rows.len() == 1).then(|| rows[0].clone())+}+pub(super) fn aws_rate(value: &Value, unit: &str) -> Option<(String, f64)> {+ let mut rates = vec![];+ for row in value["PriceList"].as_array()? {+ let p: Value = if let Some(s) = row.as_str() {+ serde_json::from_str(s).ok()?+ } else {+ row.clone()+ };+ for term in p["terms"]["OnDemand"].as_object()?.values() {+ for (id, d) in term["priceDimensions"].as_object()? {+ if d["unit"] == unit && d["beginRange"] == "0" && d["endRange"] == "Inf" {+ rates.push((id.clone(), rate(&d["pricePerUnit"]["USD"])?));+ }+ }+ }+ }+ unique(rates)+}+async fn aws(+ c: &Config,+ service: &str,+ filters: &[(&str, &str)],+ unit: &str,+) -> Result<Option<(String, f64)>> {+ let filters: Vec<_> = filters+ .iter()+ .map(|(field, value)| json!({"Type":"TERM_MATCH","Field":field,"Value":value}))+ .collect();+ let value = resolve::read(+ c,+ &["pricing", "get-products"],+ vec![+ "--service-code".into(),+ service.into(),+ "--filters".into(),+ serde_json::to_string(&filters)?,+ ],+ )+ .await?;+ Ok(aws_rate(&value, unit))+}+async fn retail(c: &Config, service: &str, extra: &str) -> Result<Vec<Value>> {+ retail_region(c.cloud.region(), service, extra).await+}+async fn retail_region(region: &str, service: &str, extra: &str) -> Result<Vec<Value>> {+ use futures_util::StreamExt;+ let client = reqwest::Client::builder()+ .redirect(reqwest::redirect::Policy::none())+ .retry(reqwest::retry::never())+ .timeout(std::time::Duration::from_secs(30))+ .build()?;+ let mut url = reqwest::Url::parse("https://prices.azure.com/api/retail/prices")?;+ url.query_pairs_mut().append_pair("$filter",&format!("serviceName eq '{service}' and armRegionName eq '{}' and priceType eq 'Consumption'{extra}",region));+ let mut seen = std::collections::BTreeSet::new();+ let mut rows = vec![];+ for _ in 0..16 {+ ensure!(+ url.scheme() == "https"+ && url.host_str() == Some("prices.azure.com")+ && url.username().is_empty()+ && url.password().is_none()+ && url.port().is_none_or(|p| p == 443)+ && url.path() == "/api/retail/prices"+ && seen.insert(url.to_string()),+ "Unexpected/repeated Azure price page"+ );+ let response = client.get(url.clone()).send().await?;+ ensure!(+ response.status().is_success(),+ "Azure price catalog unavailable"+ );+ let mut bytes = vec![];+ let mut body = response.bytes_stream();+ while let Some(chunk) = body.next().await {+ bytes.extend(chunk?);+ ensure!(+ bytes.len() <= 8 * 1024 * 1024,+ "Azure price response too large"+ );+ }+ let value: Value = serde_json::from_slice(&bytes)?;+ rows.extend(+ value["Items"]+ .as_array()+ .context("Azure price items missing")?+ .iter()+ .cloned(),+ );+ ensure!(rows.len() <= 16000, "Azure price inventory too large");+ match value["NextPageLink"].as_str().filter(|s| !s.is_empty()) {+ Some(next) => url = reqwest::Url::parse(next)?,+ None => return Ok(rows),+ }+ }+ bail!("Azure price pagination incomplete")+}+pub(super) fn azure_rate(+ rows: &[Value],+ region: &str,+ unit: &str,+ select: impl Fn(&Value) -> bool,+) -> Option<(String, f64)> {+ let at = now().ok()? as i64;+ let mut rates = vec![];+ for row in rows {+ if row["armRegionName"] != region+ || row["currencyCode"] != "USD"+ || row["type"] != "Consumption"+ || row["unitOfMeasure"] != unit+ || row["tierMinimumUnits"].as_f64() != Some(0.0)+ || row["isPrimaryMeterRegion"] == false+ || !select(row)+ {+ continue;+ }+ let started = chrono::DateTime::parse_from_rfc3339(row["effectiveStartDate"].as_str()?)+ .ok()?+ .timestamp();+ if started > at {+ continue;+ }+ rates.push((row["meterId"].as_str()?.into(), rate(&row["retailPrice"])?));+ }+ unique(rates)+}+pub(super) async fn price_nodes(c: &Config, nodes: &mut [Node]) {+ for node in nodes.iter_mut().filter(|n| n.eligible) {+ let result = match &c.cloud {+ Cloud::Aws { region, .. } => {+ aws(+ c,+ "AmazonEC2",+ &[+ ("regionCode", region),+ ("instanceType", &node.instance_type),+ ("operatingSystem", "Linux"),+ ("tenancy", "Shared"),+ ("preInstalledSw", "NA"),+ ("capacitystatus", "Used"),+ ],+ "Hrs",+ )+ .await+ }+ Cloud::Azure { region, .. } => {+ let rows = retail(+ c,+ "Virtual Machines",+ &format!(" and armSkuName eq '{}'", node.instance_type),+ )+ .await;+ rows.map(|rows| {+ azure_rate(&rows, region, "1 Hour", |r| {+ r["armSkuName"] == node.instance_type+ && !r["productName"].as_str().unwrap_or("").contains("Windows")+ && !r["meterName"].as_str().unwrap_or("").contains("Spot")+ && !r["meterName"]+ .as_str()+ .unwrap_or("")+ .contains("Low Priority")+ })+ })+ }+ };+ node.hourly_usd = result.ok().flatten().map(|(_, rate)| rate);+ }+}+pub(super) async fn estimate(c: &Config, node: &Node, infrastructure: &Value) -> resolve::Price {+ let mut components = BTreeMap::new();+ let mut missing = vec![];+ let mut add =+ |name: &str, quote: Option<(String, f64)>, quantity: f64, divisor: f64, source: &str| {+ if let Some((meter, value)) = quote {+ components.insert(+ name.to_string(),+ Line {+ hourly_usd: value / divisor,+ quantity,+ source: source.into(),+ meter,+ },+ );+ } else {+ missing.push(name.into());+ }+ };+ add(+ "workerInstances",+ node.hourly_usd.map(|v| (node.instance_type.clone(), v)),+ f64::from(c.nodes.count),+ 1.0,+ "provider retail catalog",+ );+ match &c.cloud {+ Cloud::Aws { region, .. } => {+ for (name, service, filters, unit, quantity, divisor) in [+ (+ "database",+ "AmazonRDS",+ vec![+ ("regionCode", region.as_str()),+ ("instanceType", "db.t4g.micro"),+ ("databaseEngine", "PostgreSQL"),+ ("deploymentOption", "Single-AZ"),+ ],+ "Hrs",+ 1.0,+ 1.0,+ ),+ (+ "cluster",+ "AmazonEKS",+ vec![+ ("regionCode", region.as_str()),+ ("operation", "CreateOperation"),+ ],+ "Hrs",+ 1.0,+ 1.0,+ ),+ (+ "nodeDisks",+ "AmazonEC2",+ vec![+ ("regionCode", region.as_str()),+ ("productFamily", "Storage"),+ ("volumeApiName", "gp3"),+ ],+ "GB-Mo",+ 30.0 * f64::from(c.nodes.count),+ 730.0,+ ),+ ] {+ add(+ name,+ aws(c, service, &filters, unit).await.ok().flatten(),+ quantity,+ divisor,+ "AWS Price List OnDemand USD",+ );+ }+ for (name, service, filters, unit, quantity, divisor) in [+ (+ "natGateway",+ "AmazonEC2",+ vec![+ ("regionCode", region.as_str()),+ ("productFamily", "NAT Gateway"),+ ],+ "Hrs",+ 1.0,+ 1.0,+ ),+ (+ "publicIPv4",+ "AmazonVPC",+ vec![+ ("regionCode", region.as_str()),+ ("group", "VPCPublicIPv4Address"),+ ],+ "Hrs",+ 1.0,+ 1.0,+ ),+ (+ "databaseDisks",+ "AmazonRDS",+ vec![+ ("regionCode", region.as_str()),+ ("productFamily", "Database Storage"),+ ("databaseEngine", "PostgreSQL"),+ ("deploymentOption", "Single-AZ"),+ ("volumeType", "General Purpose-GP3"),+ ],+ "GB-Mo",+ 20.0,+ 730.0,+ ),+ ] {+ add(+ name,+ aws(c, service, &filters, unit).await.ok().flatten(),+ quantity,+ divisor,+ "AWS Price List OnDemand USD",+ );+ }+ if infrastructure+ .get("operatorAccess")+ .is_some_and(|v| !v.is_null())+ {+ add(+ "managementInstance",+ aws(+ c,+ "AmazonEC2",+ &[+ ("regionCode", region),+ ("instanceType", "t4g.micro"),+ ("operatingSystem", "Linux"),+ ("tenancy", "Shared"),+ ("preInstalledSw", "NA"),+ ("capacitystatus", "Used"),+ ],+ "Hrs",+ )+ .await+ .ok()+ .flatten(),+ 1.0,+ 1.0,+ "AWS Price List OnDemand USD",+ );+ add(+ "managementDisk",+ aws(+ c,+ "AmazonEC2",+ &[+ ("regionCode", region),+ ("productFamily", "Storage"),+ ("volumeApiName", "gp3"),+ ],+ "GB-Mo",+ )+ .await+ .ok()+ .flatten(),+ 8.0,+ 730.0,+ "AWS Price List OnDemand USD",+ );+ }+ }+ Cloud::Azure { region, .. } => {+ let pg = retail(c, "Azure Database for PostgreSQL", "")+ .await+ .unwrap_or_default();+ add(+ "database",+ azure_rate(&pg, region, "1 Hour", |r| {+ r["productName"]=="Azure Database for PostgreSQL Flexible Server Burstable BS Series Compute"&&r["skuName"]=="B1MS"+ }),+ 1.0,+ 1.0,+ "Azure Retail Prices USD",+ );+ add(+ "databaseStorage",+ azure_rate(&pg, region, "1 GB/Month", |r| {+ r["productName"] == "Azure Database for PostgreSQL Flex Server Storage"+ && r["skuName"] == "Storage"+ }),+ 32.0,+ 730.0,+ "Azure Retail Prices USD",+ );+ let registry = retail(c, "Container Registry", "")+ .await+ .unwrap_or_default();+ add(+ "registry",+ azure_rate(®istry, region, "1/Day", |r| {+ r["meterName"] == "Basic Registry Unit"+ }),+ 1.0,+ 24.0,+ "Azure Retail Prices USD",+ );+ let network = retail(c, "Virtual Network", "").await.unwrap_or_default();+ add(+ "outboundIPv4",+ azure_rate(&network, region, "1 Hour", |r| {+ r["productName"] == "IP Addresses"+ && r["meterName"] == "Standard IPv4 Static Public IP"+ }),+ 1.0,+ 1.0,+ "Azure Retail Prices USD",+ );+ let disk = retail(+ c,+ "Storage",+ " and productName eq 'Premium SSD Managed Disks'",+ )+ .await+ .unwrap_or_default();+ add(+ "nodeDisks",+ azure_rate(&disk, region, "1/Month", |r| {+ r["meterName"] == "P6 LRS Disk"+ }),+ f64::from(c.nodes.count),+ 730.0,+ "Azure Retail Prices USD; Premium SSD 64 GiB assumption",+ );+ let lb = retail_region("Global", "Load Balancer", "")+ .await+ .unwrap_or_default();+ add(+ "loadBalancer",+ azure_rate(&lb, "Global", "1 Hour", |r| {+ r["meterName"] == "Standard Included LB Rules and Outbound Rules"+ }),+ 1.0,+ 1.0,+ "Azure Retail Prices USD",+ );+ let private = retail_region(+ "Global",+ "Virtual Network",+ " and productName eq 'Virtual Network Private Link'",+ )+ .await+ .unwrap_or_default();+ add(+ "blobPrivateEndpoint",+ azure_rate(&private, "Global", "1 Hour", |r| {+ r["meterName"] == "Standard Private Endpoint"+ }),+ 1.0,+ 1.0,+ "Azure Retail Prices USD",+ );+ if infrastructure+ .get("managementAccess")+ .is_some_and(|v| !v.is_null())+ {+ let vm = retail(+ c,+ "Virtual Machines",+ " and armSkuName eq 'Standard_D2as_v7'",+ )+ .await+ .unwrap_or_default();+ add(+ "managementInstance",+ azure_rate(&vm, region, "1 Hour", |r| {+ !r["productName"].as_str().unwrap_or("").contains("Windows")+ && !r["meterName"].as_str().unwrap_or("").contains("Spot")+ && !r["meterName"]+ .as_str()+ .unwrap_or("")+ .contains("Low Priority")+ }),+ 1.0,+ 1.0,+ "Azure Retail Prices USD",+ );+ add(+ "managementDisk",+ azure_rate(&disk, region, "1/Month", |r| {+ r["meterName"] == "P4 LRS Disk"+ }),+ 1.0,+ 730.0,+ "Azure Retail Prices USD; 32 GiB assumption",+ );+ add(+ "managementIPv4",+ azure_rate(&network, region, "1 Hour", |r| {+ r["productName"] == "IP Addresses"+ && r["meterName"] == "Standard IPv4 Static Public IP"+ }),+ 1.0,+ 1.0,+ "Azure Retail Prices USD",+ );+ }+ }+ }+ let subtotal: f64 = components.values().map(|l| l.hourly_usd * l.quantity).sum();+ resolve::Price {+ currency: "USD".into(),+ worker_hourly_usd: node.hourly_usd.map(|v| v * f64::from(c.nodes.count)),+ total_hourly_usd: if missing.is_empty() {+ Some(subtotal)+ } else {+ None+ },+ known_hourly_subtotal_usd: if components.is_empty() {+ None+ } else {+ Some(subtotal)+ },+ components,+ unpriced_components: missing,+ variable_allowance_usd: 25,+ planned_hours: c+ .trial+ .as_ref()+ .map(|t| u32::from(t.expires_after_hours))+ .unwrap_or(730),+ hard_cap: false,+ }+}+#[cfg(test)]+mod tests {+ use super::*;+ #[test]+ fn ambiguous_or_missing_price_is_not_zero() {+ assert!(aws_rate(&json!({"PriceList":[]}), "Hrs").is_none());+ let row = json!({"product":{"sku":"x"},"terms":{"OnDemand":{"a":{"priceDimensions":{"m":{"unit":"Hrs","beginRange":"0","endRange":"Inf","pricePerUnit":{"USD":"0.2"}}}}}}});+ assert_eq!(+ aws_rate(&json!({"PriceList":[row.clone()]}), "Hrs")+ .unwrap()+ .1,+ 0.2+ );+ assert!(aws_rate(&json!({"PriceList":[row.clone(),row]}), "Hrs").is_none());+ }+}diff --git a/harness/service-cli/src/deployment_up_resolve.rs b/harness/service-cli/src/deployment_up_resolve.rsnew file mode 100644index 00000000..7ccd00ab--- /dev/null+++ b/harness/service-cli/src/deployment_up_resolve.rs@@ -0,0 +1,459 @@+//! Read-only machine resolution. Freeze an exact eligible choice before effects.+use super::*;+use config::{Cloud, Config};+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Node {+ pub instance_type: String,+ pub cpu: u32,+ #[serde(rename = "memoryGiB")]+ pub memory_gi_b: u32,+ pub architecture: String,+ pub eligible: bool,+ pub hourly_usd: Option<f64>,+}+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Price {+ pub currency: String,+ pub worker_hourly_usd: Option<f64>,+ pub total_hourly_usd: Option<f64>,+ pub known_hourly_subtotal_usd: Option<f64>,+ pub components: BTreeMap<String, pricing::Line>,+ pub variable_allowance_usd: u32,+ pub planned_hours: u32,+ pub unpriced_components: Vec<String>,+ pub hard_cap: bool,+}+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Resolution {+ pub node: Node,+ pub price: Price,+ pub observed_at_unix: u64,+ pub identity: Value,+}+impl Resolution {+ pub fn validate(&self, c: &Config) -> Result<()> {+ ensure!(+ eligible(c, &self.node),+ "Resolved node no longer matches the reviewed configuration"+ );+ ensure!(+ self.price.currency == "USD" && !self.price.hard_cap,+ "Invalid price currency or hard-cap claim"+ );+ for n in [+ self.node.hourly_usd,+ self.price.worker_hourly_usd,+ self.price.total_hourly_usd,+ ]+ .into_iter()+ .flatten()+ {+ ensure!(+ n.is_finite() && n > 0.0,+ "Unknown/invalid prices must not be represented as zero"+ );+ }+ ensure!(+ self.observed_at_unix > 0,+ "Missing resolution observation time"+ );+ Ok(())+ }+}+fn eligible(c: &Config, n: &Node) -> bool {+ let architecture = if c.cloud.provider() == "aws" {+ "arm64"+ } else {+ "amd64"+ };+ n.eligible+ && n.architecture == architecture+ && n.cpu >= 2+ && n.memory_gi_b >= 8+ && u32::from(c.nodes.count) * n.cpu >= 4+ && u32::from(c.nodes.count) * n.memory_gi_b >= 16+ && match (&c.nodes.instance_type, &c.nodes.requirements) {+ (Some(size), None) => size == &n.instance_type,+ (None, Some(r)) => {+ r.cpu.min <= n.cpu+ && n.cpu <= r.cpu.max+ && r.memory_gi_b.min <= n.memory_gi_b+ && n.memory_gi_b <= r.memory_gi_b.max+ }+ _ => false,+ }+}+pub(super) fn choose(c: &Config, nodes: Vec<Node>) -> Result<Node> {+ let mut candidates: Vec<_> = nodes.into_iter().filter(|n| eligible(c, n)).collect();+ ensure!(!candidates.is_empty(),"No reviewed node size satisfies your requirements and regional availability. Change the range/type or request quota; no larger fallback was selected.");+ // Do not label an unknown price cheapest. Deterministic resource ordering is+ // an explicit fallback and the review marks the estimate incomplete.+ candidates.sort_by(|a, b| {+ match (a.hourly_usd, b.hourly_usd) {+ (Some(a), Some(b)) => a.total_cmp(&b),+ (Some(_), None) => std::cmp::Ordering::Less,+ (None, Some(_)) => std::cmp::Ordering::Greater,+ _ => a.cpu.cmp(&b.cpu).then(a.memory_gi_b.cmp(&b.memory_gi_b)),+ }+ .then(a.instance_type.cmp(&b.instance_type))+ });+ Ok(candidates.remove(0))+}+pub(super) async fn read(c: &Config, command: &[&str], args: Vec<String>) -> Result<Value> {+ let (bin, mut all) = match &c.cloud {+ Cloud::Aws {+ region, profile, ..+ } => {+ ensure!(+ matches!(+ command,+ ["sts", "get-caller-identity"]+ | ["ec2", "describe-instance-types"]+ | ["ec2", "describe-instance-type-offerings"]+ | ["ec2", "describe-availability-zones"]+ | ["pricing", "get-products"]+ | ["eks", "describe-cluster-versions"]+ | ["eks", "describe-addon-versions"]+ | ["ssm", "get-parameter"]+ | ["rds", "describe-db-engine-versions"]+ ),+ "Unapproved read operation"+ );+ let r = if command == ["pricing", "get-products"] {+ "us-east-1"+ } else {+ region+ };+ (+ "aws",+ [+ command.iter().map(|s| s.to_string()).collect::<Vec<_>>(),+ vec![+ "--profile".into(),+ profile.clone(),+ "--region".into(),+ r.into(),+ "--output".into(),+ "json".into(),+ "--no-cli-pager".into(),+ ],+ ]+ .concat(),+ )+ }+ Cloud::Azure {+ subscription_id, ..+ } => {+ ensure!(+ matches!(+ command,+ ["account", "show"]+ | ["cloud", "show"]+ | ["vm", "list-skus"]+ | ["vm", "list-usage"]+ | ["aks", "get-versions"]+ | ["ad", "signed-in-user", "show"]+ ),+ "Unapproved read operation"+ );+ let mut v = command.iter().map(|s| s.to_string()).collect::<Vec<_>>();+ if command != ["cloud", "show"] && command != ["ad", "signed-in-user", "show"] {+ v.extend(["--subscription".into(), subscription_id.clone()]);+ }+ v.extend([+ "--output".into(),+ "json".into(),+ "--only-show-errors".into(),+ ]);+ ("az", v)+ }+ };+ all.extend(args);+ let raw=super::super::local::exec(Path::new(bin),&all,None).await.context("Cloud prerequisite read failed. Check CLI login, target permissions and connectivity; no provisioning occurred.")?;+ serde_json::from_slice(&raw).context("Cloud returned an invalid prerequisite response")+}+pub(super) async fn identity(c: &Config) -> Result<Value> {+ match &c.cloud {+ Cloud::Aws { account_id, .. } => {+ let v = read(c, &["sts", "get-caller-identity"], vec![]).await?;+ ensure!(+ v["Account"] == *account_id+ && v["Arn"].as_str().is_some_and(+ |s| s.starts_with("arn:aws:") && s.contains(&format!(":{account_id}:"))+ ),+ "AWS login differs from the configured account"+ );+ Ok(json!({"accountId":account_id,"principal":v["Arn"]}))+ }+ Cloud::Azure {+ subscription_id, ..+ } => {+ let cloud = read(c, &["cloud", "show"], vec![]).await?;+ ensure!(+ cloud["name"] == "AzureCloud",+ "Only Azure public cloud is qualified by this release"+ );+ let v = read(c, &["account", "show"], vec![]).await?;+ ensure!(+ v["id"] == *subscription_id && v["state"] == "Enabled",+ "Azure login/subscription is not the configured enabled target"+ );+ let tenant = v["tenantId"].as_str().context("Azure tenant missing")?;+ ensure!(+ uuid::Uuid::parse_str(tenant).is_ok_and(|id| !id.is_nil()),+ "Invalid Azure tenant"+ );+ if let Some(wanted) = c.advanced.azure.as_ref().and_then(|a| a.tenant_id.as_ref()) {+ ensure!(wanted == tenant, "Azure tenant mismatch");+ }+ Ok(json!({"subscriptionId":subscription_id,"tenantId":tenant}))+ }+ }+}+fn capability<'a>(v: &'a Value, name: &str) -> Option<&'a str> {+ v["capabilities"]+ .as_array()?+ .iter()+ .find(|r| r["name"] == name)?["value"]+ .as_str()+}+pub(super) fn azure_nodes(c: &Config, v: &Value) -> Result<Vec<Node>> {+ let rows = v.as_array().context("Azure SKU inventory malformed")?;+ let mut out = vec![];+ for (name, cpu, mem) in [+ ("Standard_D4s_v5", 4, 16),+ ("Standard_D8s_v5", 8, 32),+ ("Standard_D4as_v7", 4, 16),+ ] {+ let Some(row) = rows+ .iter()+ .find(|r| r["name"] == name && r["resourceType"] == "virtualMachines")+ else {+ continue;+ };+ let locations = row["locations"]+ .as_array()+ .context("Azure SKU locations missing")?;+ let restrictions = row["restrictions"]+ .as_array()+ .context("Azure SKU restrictions missing")?;+ let restricted = restrictions.iter().any(|r| r["type"] != "Zone");+ let capable = capability(row, "vCPUs").and_then(|s| s.parse::<u32>().ok()) == Some(cpu)+ && capability(row, "MemoryGB").and_then(|s| s.parse::<u32>().ok()) == Some(mem)+ && capability(row, "CpuArchitectureType") == Some("x64");+ out.push(Node {+ instance_type: name.into(),+ cpu,+ memory_gi_b: mem,+ architecture: "amd64".into(),+ eligible: !restricted && capable && locations.iter().any(|r| r == c.cloud.region()),+ hourly_usd: None,+ });+ }+ Ok(out)+}+pub(super) fn aws_nodes(types: &Value, offers: &Value) -> Result<Vec<Node>> {+ let rows = types["InstanceTypes"]+ .as_array()+ .context("AWS type inventory malformed")?;+ let offers = offers["InstanceTypeOfferings"]+ .as_array()+ .context("AWS offerings malformed")?;+ let mut out = vec![];+ for name in ["t4g.large", "t4g.xlarge", "t4g.2xlarge"] {+ let Some(r) = rows.iter().find(|r| r["InstanceType"] == name) else {+ continue;+ };+ let (cpu, mem) = super::super::infrastructure::worker_shape(name).unwrap();+ let ok = r["VCpuInfo"]["DefaultVCpus"] == cpu+ && r["MemoryInfo"]["SizeInMiB"] == mem * 1024+ && r["ProcessorInfo"]["SupportedArchitectures"]+ .as_array()+ .is_some_and(|a| a.iter().any(|s| s == "arm64"));+ out.push(Node {+ instance_type: name.into(),+ cpu,+ memory_gi_b: mem,+ architecture: "arm64".into(),+ eligible: ok && offers.iter().any(|r| r["InstanceType"] == name),+ hourly_usd: None,+ });+ }+ Ok(out)+}+pub(super) async fn live(c: &Config, infrastructure: &Value) -> Result<Resolution> {+ eprintln!("Checking cloud identity, supported node sizes and quota…");+ let identity = identity(c).await?;+ let mut candidates = match &c.cloud {+ Cloud::Azure { region, .. } => azure_nodes(+ c,+ &read(+ c,+ &["vm", "list-skus"],+ vec![+ "--location".into(),+ region.clone(),+ "--resource-type".into(),+ "virtualMachines".into(),+ "--size".into(),+ "Standard_D".into(),+ "--all".into(),+ ],+ )+ .await?,+ )?,+ Cloud::Aws { .. } => {+ let types = read(+ c,+ &["ec2", "describe-instance-types"],+ vec![+ "--instance-types".into(),+ "t4g.large".into(),+ "t4g.xlarge".into(),+ "t4g.2xlarge".into(),+ ],+ )+ .await?;+ let offers = read(+ c,+ &["ec2", "describe-instance-type-offerings"],+ vec![+ "--location-type".into(),+ "region".into(),+ "--filters".into(),+ "Name=instance-type,Values=t4g.large,t4g.xlarge,t4g.2xlarge".into(),+ ],+ )+ .await?;+ aws_nodes(&types, &offers)?+ }+ };+ if c.cloud.provider() == "azure" {+ let usage = read(+ c,+ &["vm", "list-usage"],+ vec!["--location".into(), c.cloud.region().into()],+ )+ .await?;+ let management = infrastructure+ .get("managementAccess")+ .is_some_and(|v| !v.is_null());+ for node in &mut candidates {+ let family = match node.instance_type.as_str() {+ "Standard_D4s_v5" | "Standard_D8s_v5" => "standardDSv5Family",+ _ => "standardDasv7Family",+ };+ node.eligible &= (!management || node.instance_type == "Standard_D4as_v7")+ && quota_available(+ &usage,+ "cores",+ u64::from(c.nodes.count) * u64::from(node.cpu) + if management { 2 } else { 0 },+ )+ && quota_available(+ &usage,+ family,+ u64::from(c.nodes.count) * u64::from(node.cpu) + if management { 2 } else { 0 },+ );+ }+ }+ eprintln!("Reading price catalogs; unavailable rates will remain marked unknown…");+ pricing::price_nodes(c, &mut candidates).await;+ let node = choose(c, candidates)?;+ let price = pricing::estimate(c, &node, infrastructure).await;+ Ok(Resolution {+ node,+ price,+ observed_at_unix: now()?,+ identity,+ })+}+#[cfg(test)]+mod tests {+ use super::*;+ fn config() -> Config {+ serde_json::from_value(json!({"name":"example","cloud":{"provider":"azure","subscriptionId":"11111111-2222-4333-8444-555555555555","region":"westus2"},"nodes":{"count":2,"requirements":{"cpu":{"min":4,"max":8},"memoryGiB":{"min":16,"max":32}}},"budget":{"alertUSD":100}})).unwrap()+ }+ fn node(name: &str, cpu: u32, price: Option<f64>) -> Node {+ Node {+ instance_type: name.into(),+ cpu,+ memory_gi_b: cpu * 4,+ architecture: "amd64".into(),+ eligible: true,+ hourly_usd: price,+ }+ }+ #[test]+ fn resolution_obeys_bounds_and_never_claims_unknown_is_free() {+ let c = config();+ assert_eq!(+ choose(+ &c,+ vec![node("small", 4, None), node("known", 8, Some(0.2))]+ )+ .unwrap()+ .instance_type,+ "known"+ );+ assert!(choose(&c, vec![node("too-big", 16, Some(0.1))]).is_err());+ let mut bad = node("wrong-arch", 4, None);+ bad.architecture = "arm64".into();+ assert!(choose(&c, vec![bad]).is_err());+ }+ #[test]+ fn subscription_restrictions_and_unverified_capabilities_exclude_node() {+ let c = config();+ let sku = json!([{"name":"Standard_D4s_v5","resourceType":"virtualMachines","locations":["westus2"],"restrictions":[{"type":"Location"}],"capabilities":[{"name":"vCPUs","value":"4"},{"name":"MemoryGB","value":"16"},{"name":"CpuArchitectureType","value":"x64"}]}]);+ assert!(choose(&c, azure_nodes(&c, &sku).unwrap()).is_err());+ }+}++fn quota_available(v: &Value, name: &str, needed: u64) -> bool {+ let Some(rows) = v.as_array() else {+ return false;+ };+ let rows: Vec<_> = rows+ .iter()+ .filter(|r| {+ r["name"]["value"]+ .as_str()+ .is_some_and(|n| n.eq_ignore_ascii_case(name))+ })+ .collect();+ if rows.len() != 1 {+ return false;+ }+ let count = |v: &Value| v.as_u64().or_else(|| v.as_str()?.parse().ok());+ match (count(&rows[0]["limit"]), count(&rows[0]["currentValue"])) {+ (Some(limit), Some(used)) => limit.checked_sub(used).is_some_and(|n| n >= needed),+ _ => false,+ }+}++#[cfg(test)]+mod quota_tests {+ use super::*;+ #[test]+ fn quota_requires_unambiguous_remaining_capacity() {+ let row = json!({"name":{"value":"standardDasv7Family"},"limit":12,"currentValue":2});+ assert!(quota_available(+ &json!([row.clone()]),+ "StandardDasv7Family",+ 10+ ));+ assert!(!quota_available(+ &json!([row.clone()]),+ "StandardDasv7Family",+ 11+ ));+ assert!(!quota_available(+ &json!([row.clone(), row]),+ "StandardDasv7Family",+ 1+ ));+ assert!(!quota_available(&json!([]), "cores", 1));+ }+}diff --git a/harness/service-cli/src/deployment_up_state.rs b/harness/service-cli/src/deployment_up_state.rsnew file mode 100644index 00000000..ec9bbc50--- /dev/null+++ b/harness/service-cli/src/deployment_up_state.rs@@ -0,0 +1,300 @@+//! Restartable orchestration of native lifecycle stages. A stage receipt does not+//! replace provider ownership receipts. Every resumed effect is observed first.+use super::*;+#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]+#[serde(rename_all = "snake_case")]+pub(super) enum Stage {+ Bundle,+ Preflight,+ Permissions,+ Control,+ Foundation,+ IdentityBindings,+ Cluster,+ Images,+ PrivateAccess,+ Platform,+ OpenShell,+ Secrets,+ Service,+ Verify,+}+pub(super) const ORDER: [Stage; 14] = [+ Stage::Bundle,+ Stage::Permissions,+ Stage::Preflight,+ Stage::Control,+ Stage::Foundation,+ Stage::IdentityBindings,+ Stage::Cluster,+ Stage::Images,+ Stage::PrivateAccess,+ Stage::Platform,+ Stage::OpenShell,+ Stage::Secrets,+ Stage::Service,+ Stage::Verify,+];+#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]+#[serde(rename_all = "snake_case")]+pub(super) enum Phase {+ Intent,+ Pending,+ Complete,+ Blocked,+}+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Step {+ pub request_id: String,+ pub phase: Phase,+ pub receipt: Option<Value>,+ pub action: Option<String>,+}+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub(super) struct Journal {+ pub schema_version: u16,+ pub review_digest: String,+ pub deployment_id: String,+ pub stages: BTreeMap<Stage, Step>,+}+#[derive(Debug)]+pub(super) enum Observation {+ /// Independently proven no request/effect; native receipt contract permits start.+ Absent,+ Pending,+ Complete(Value),+ Blocked(String),+ /// No deletion/recreation/replay is permitted on this observation.+ Unknown(String),+}+pub(super) trait Driver {+ async fn observe(&mut self, stage: Stage, step: Option<&Step>) -> Result<Observation>;+ async fn execute(&mut self, stage: Stage, request_id: &str) -> Result<Observation>;+}+impl Journal {+ pub fn open(path: &Path, review: &Review) -> Result<Self> {+ if !path.exists() {+ return Ok(Self {+ schema_version: 1,+ review_digest: review.digest.clone(),+ deployment_id: review.deployment_id.clone(),+ stages: BTreeMap::new(),+ });+ }+ let j: Self = serde_json::from_slice(&fs::read(path)?)?;+ ensure!(+ j.schema_version == 1+ && j.review_digest == review.digest+ && j.deployment_id == review.deployment_id,+ "Deployment journal identity mismatch"+ );+ for step in j.stages.values() {+ ensure!(+ uuid::Uuid::parse_str(&step.request_id).is_ok_and(|id| !id.is_nil()),+ "Invalid stage request receipt"+ );+ }+ Ok(j)+ }+ pub fn complete(&self) -> bool {+ ORDER.iter().all(|s| {+ self.stages+ .get(s)+ .is_some_and(|v| v.phase == Phase::Complete)+ })+ }+}+fn record(+ j: &mut Journal,+ path: &Path,+ stage: Stage,+ phase: Phase,+ receipt: Option<Value>,+ action: Option<String>,+) -> Result<()> {+ let step = j.stages.entry(stage).or_insert_with(|| Step {+ request_id: uuid::Uuid::new_v4().to_string(),+ phase: Phase::Intent,+ receipt: None,+ action: None,+ });+ step.phase = phase;+ step.receipt = receipt;+ step.action = action;+ write_private(path, j)+}+/// Advances at most one cloud stage. Effects are never retried because a local+/// process failed: the native adapter must first prove no-effect or completion.+pub(super) async fn advance(+ j: &mut Journal,+ path: &Path,+ driver: &mut impl Driver,+) -> Result<bool> {+ let Some(stage) = ORDER+ .iter()+ .find(|s| j.stages.get(s).is_none_or(|v| v.phase != Phase::Complete))+ .copied()+ else {+ return Ok(true);+ };+ let observed = driver.observe(stage, j.stages.get(&stage)).await?;+ let outcome = match observed {+ Observation::Absent => {+ record(j, path, stage, Phase::Intent, None, None)?;+ let id = j.stages[&stage].request_id.clone();+ // On failure the durable intent remains. The next invocation calls+ // observe, never execute directly.+ driver.execute(stage, &id).await?+ }+ other => other,+ };+ match outcome {+ Observation::Complete(receipt) => {+ record(j, path, stage, Phase::Complete, Some(receipt), None)?+ }+ Observation::Pending => record(j, path, stage, Phase::Pending, None, None)?,+ Observation::Blocked(action) => {+ record(j, path, stage, Phase::Blocked, None, Some(action.clone()))?;+ bail!("{action}");+ }+ Observation::Unknown(action) => {+ record(j, path, stage, Phase::Pending, None, Some(action.clone()))?;+ bail!("Outcome unknown; inspect original stage {stage:?}. {action}");+ }+ Observation::Absent => bail!(+ "Stage did not provide a durable outcome after execution; reconcile before continuing"+ ),+ }+ Ok(j.complete())+}+#[cfg(test)]+mod tests {+ use super::*;+ struct Fake {+ effects: usize,+ lost: bool,+ remote: bool,+ unknown: bool,+ }+ impl Driver for Fake {+ async fn observe(&mut self, _: Stage, _: Option<&Step>) -> Result<Observation> {+ Ok(if self.unknown {+ Observation::Unknown("read failed".into())+ } else if self.remote {+ Observation::Complete(json!({"original":true}))+ } else {+ Observation::Absent+ })+ }+ async fn execute(&mut self, _: Stage, _: &str) -> Result<Observation> {+ self.effects += 1;+ self.remote = true;+ if self.lost {+ bail!("lost after effect")+ }+ Ok(Observation::Pending)+ }+ }+ fn journal() -> Journal {+ Journal {+ schema_version: 1,+ review_digest: "digest".into(),+ deployment_id: "original".into(),+ stages: BTreeMap::new(),+ }+ }+ #[tokio::test]+ async fn lost_effect_is_reconciled_after_process_restart_without_duplicate() {+ let dir = tempfile::tempdir().unwrap();+ let p = dir.path().join("journal.json");+ let mut j = journal();+ let mut d = Fake {+ effects: 0,+ lost: true,+ remote: false,+ unknown: false,+ };+ assert!(advance(&mut j, &p, &mut d).await.is_err());+ assert_eq!(j.stages[&Stage::Bundle].phase, Phase::Intent);+ let id = j.stages[&Stage::Bundle].request_id.clone();+ let mut j: Journal = serde_json::from_slice(&fs::read(&p).unwrap()).unwrap();+ advance(&mut j, &p, &mut d).await.unwrap();+ assert_eq!(d.effects, 1);+ assert_eq!(j.stages[&Stage::Bundle].request_id, id);+ assert_eq!(j.stages[&Stage::Bundle].phase, Phase::Complete);+ }+ #[tokio::test]+ async fn unknown_observation_never_executes_and_blocks_following_stages() {+ let dir = tempfile::tempdir().unwrap();+ let p = dir.path().join("journal.json");+ let mut j = journal();+ let mut d = Fake {+ effects: 0,+ lost: false,+ remote: false,+ unknown: true,+ };+ assert!(advance(&mut j, &p, &mut d).await.is_err());+ assert_eq!(d.effects, 0);+ assert_eq!(j.stages.len(), 1);+ }+}++#[cfg(test)]+mod boundary_tests {+ use super::*;+ struct Fault {+ effects: BTreeMap<Stage, usize>,+ fail: Stage,+ lost: bool,+ }+ impl Driver for Fault {+ async fn observe(&mut self, s: Stage, _: Option<&Step>) -> Result<Observation> {+ Ok(if self.effects.contains_key(&s) {+ Observation::Complete(json!({"original":true}))+ } else {+ Observation::Absent+ })+ }+ async fn execute(&mut self, s: Stage, _: &str) -> Result<Observation> {+ *self.effects.entry(s).or_default() += 1;+ if s == self.fail && !self.lost {+ self.lost = true;+ bail!("crash after effect")+ }+ Ok(Observation::Complete(json!({"ready":true})))+ }+ }+ #[tokio::test]+ async fn every_stage_survives_restart_without_repeating_effects() {+ for failed in ORDER {+ let temp = tempfile::tempdir().unwrap();+ let path = temp.path().join("journal.json");+ let mut j = Journal {+ schema_version: 1,+ review_digest: "r".into(),+ deployment_id: "d".into(),+ stages: BTreeMap::new(),+ };+ let mut driver = Fault {+ effects: BTreeMap::new(),+ fail: failed,+ lost: false,+ };+ for _ in 0..ORDER.len() + 2 {+ match advance(&mut j, &path, &mut driver).await {+ Ok(true) => break,+ Ok(false) => (),+ Err(_) => {+ j = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap();+ }+ }+ }+ assert!(j.complete(), "{failed:?}");+ assert_eq!(driver.effects.len(), ORDER.len());+ assert!(driver.effects.values().all(|n| *n == 1));+ }+ }+}diff --git a/harness/service-cli/tests/deployment_up.rs b/harness/service-cli/tests/deployment_up.rsnew file mode 100644index 00000000..0600fd50--- /dev/null+++ b/harness/service-cli/tests/deployment_up.rs@@ -0,0 +1,74 @@+//! Public entry points must reject unsupported configuration before cloud tools.+use std::{fs, process::Command};+const BIN: &str = env!("CARGO_BIN_EXE_ows");+#[test]+fn both_entry_points_explain_review_resume_and_private_access() {+ for cloud in ["aws", "azure"] {+ let output = Command::new(BIN)+ .args(["deploy", cloud, "up", "--help"])+ .output()+ .unwrap();+ assert!(output.status.success());+ let help = String::from_utf8(output.stdout).unwrap();+ for option in [+ "--config",+ "--plan-only",+ "--accept-plan",+ "--accept-bootstrap",+ "--keep-access",+ ] {+ assert!(help.contains(option), "{cloud}: {option}");+ }+ }+}+#[test]+fn inline_secret_or_wrong_cloud_fails_without_login_or_secret_echo() {+ let temp = tempfile::tempdir().unwrap();+ let config = temp.path().join("deployment.yaml");+ let base="name: example\ncloud: {provider: aws, accountId: '123456789012', region: us-west-2}\nnodes: {count: 2, instanceType: t4g.large}\nbudget: {limitUSD: 100}\n";+ for (cloud, text, expected) in [+ ("azure", base.to_owned(), "differs"),+ (+ "aws",+ format!("{base}models: {{openai: {{apiKey: NEVER_ECHO_THIS_SECRET}}}}"),+ "schema",+ ),+ ] {+ fs::write(&config, text).unwrap();+ let r = Command::new(BIN)+ .env_clear()+ .env("HOME", temp.path())+ .env("PATH", temp.path())+ .args(["deploy", cloud, "up", "--config", config.to_str().unwrap()])+ .output()+ .unwrap();+ assert!(!r.status.success());+ let stderr = String::from_utf8_lossy(&r.stderr);+ assert!(stderr.contains(expected), "{stderr}");+ assert!(!stderr.contains("NEVER_ECHO_THIS_SECRET"));+ }+}+#[test]+fn absent_account_profile_produces_prerequisite_instructions_without_cloud_calls() {+ let temp = tempfile::tempdir().unwrap();+ let config = temp.path().join("deployment.yaml");+ fs::write(&config,"name: example\ncloud: {provider: azure, subscriptionId: '11111111-2222-4333-8444-555555555555', region: westus2}\nnodes: {count: 2, instanceType: Standard_D4as_v7}\nbudget: {limitUSD: 100}\n").unwrap();+ let r = Command::new(BIN)+ .env_clear()+ .env("HOME", temp.path())+ .env("PATH", temp.path())+ .args([+ "deploy",+ "azure",+ "up",+ "--config",+ config.to_str().unwrap(),+ "--plan-only",+ ])+ .output()+ .unwrap();+ assert!(!r.status.success());+ let error = String::from_utf8_lossy(&r.stderr);+ assert!(error.contains("setup profile is missing"), "{error}");+ assert!(!temp.path().join(".ows").exists());+}