:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(wizard): pin cloud runs to wizard 2.86.0 and log failure messages (#121741)
Rafael Audibert89c2dea98e0c
diff --git a/products/wizard/backend/facade/config.py b/products/wizard/backend/facade/config.pyindex 6ba92a5f23a9..0a8d87c75d48 100644--- a/products/wizard/backend/facade/config.py+++ b/products/wizard/backend/facade/config.py@@ -1,6 +1,6 @@from datetime import timedelta-DEFAULT_WIZARD_VERSION = "2.78.0"+DEFAULT_WIZARD_VERSION = "2.86.0"LATEST_WIZARD_VERSION = "latest"WIZARD_REGISTRY_VERSION = 1WIZARD_RUN_DEADLINE = timedelta(minutes=85)diff --git a/products/wizard/backend/logic/artifacts/service.py b/products/wizard/backend/logic/artifacts/service.pyindex bd0e38c9e278..5f5c15c76707 100644--- a/products/wizard/backend/logic/artifacts/service.py+++ b/products/wizard/backend/logic/artifacts/service.py@@ -1,9 +1,10 @@-import loggingfrom hashlib import sha256from uuid import UUIDfrom django.conf import settings+import structlog+from posthog.dataclasses import frozenfrom posthog.storage import object_storage@@ -20,7 +21,7 @@ from products.wizard.backend.logic.artifacts.config import MAX_GIT_DIFF_BYTES, Mfrom products.wizard.backend.logic.runs import store as run_storefrom products.wizard.backend.observability.service import wizard_observability as run_observability-logger = logging.getLogger(__name__)+logger = structlog.get_logger(__name__)@frozen@@ -110,7 +111,9 @@ def get_git_diff_artifact_content(team_id: int, run_id: UUID, artifact_id: UUID)if content_hash != stored_artifact.content_hash:logger.error("wizard_git_diff_integrity_check_failed",- extra={"team_id": team_id, "run_id": str(run_id), "artifact_id": str(artifact_id)},+ team_id=team_id,+ run_id=str(run_id),+ artifact_id=str(artifact_id),)raise ValueError("Stored Wizard diff failed its integrity check")diff --git a/products/wizard/backend/logic/reconciliation/service.py b/products/wizard/backend/logic/reconciliation/service.pyindex d5c5618acf74..0785d4c44800 100644--- a/products/wizard/backend/logic/reconciliation/service.py+++ b/products/wizard/backend/logic/reconciliation/service.py@@ -1,9 +1,10 @@-import loggingfrom time import monotonicfrom django.db.models import Qfrom django.utils import timezone+import structlog+from posthog.dataclasses import frozenfrom products.wizard.backend.facade.enums import (@@ -25,7 +26,7 @@ from products.wizard.backend.logic.workers import lifecycle as worker_lifecyclefrom products.wizard.backend.models import WizardRun, WizardWorkerfrom products.wizard.backend.observability.service import wizard_observability-logger = logging.getLogger(__name__)+logger = structlog.get_logger(__name__)@frozen@@ -69,7 +70,9 @@ def reconcile_pending_dispatches() -> ReconciliationSummary:except WizardRunDispatchError as error:logger.exception("wizard_run_redispatch_failed",- extra={"team_id": team_id, "run_id": str(run_id), "exhausted": error.exhausted},+ team_id=team_id,+ run_id=str(run_id),+ exhausted=error.exhausted,)if error.exhausted:try:@@ -77,11 +80,12 @@ def reconcile_pending_dispatches() -> ReconciliationSummary:except (IllegalStatusTransitionError, WizardRunNotFoundError, ValueError):logger.exception("wizard_run_dispatch_exhaustion_failed",- extra={"team_id": team_id, "run_id": str(run_id)},+ team_id=team_id,+ run_id=str(run_id),)continueexcept (WizardRunNotFoundError, ValueError):- logger.exception("wizard_run_redispatch_failed", extra={"team_id": team_id, "run_id": str(run_id)})+ logger.exception("wizard_run_redispatch_failed", team_id=team_id, run_id=str(run_id))continuereconciled += 1@@ -111,7 +115,8 @@ def reconcile_pending_cancellations() -> ReconciliationSummary:except (WizardRunNotFoundError, ValueError):logger.exception("wizard_run_cancellation_reconciliation_failed",- extra={"team_id": team_id, "run_id": str(run_id)},+ team_id=team_id,+ run_id=str(run_id),)return _summary(len(pending), reconciled)@@ -132,7 +137,7 @@ def reconcile_expired_runs() -> ReconciliationSummary:try:run = lifecycle.fail_run(team_id, run_id, error_code=WizardRunErrorCode.TIMEOUT)except (IllegalStatusTransitionError, WizardRunNotFoundError, ValueError):- logger.exception("wizard_run_expiration_failed", extra={"team_id": team_id, "run_id": str(run_id)})+ logger.exception("wizard_run_expiration_failed", team_id=team_id, run_id=str(run_id))continuewizard_observability.run_past_deadline(run)if workflow_id is not None:@@ -174,7 +179,7 @@ def reconcile_pending_worker_cleanup() -> ReconciliationSummary:try:worker_lifecycle.cleanup_worker(team_id, run_id, sandbox_id)except WizardWorkerCleanupError:- logger.exception("wizard_worker_reconciliation_failed", extra={"team_id": team_id, "run_id": str(run_id)})+ logger.exception("wizard_worker_reconciliation_failed", team_id=team_id, run_id=str(run_id))continuereconciled += 1diff --git a/products/wizard/backend/logic/registry/service.py b/products/wizard/backend/logic/registry/service.pyindex a5b045017f63..f9767d874175 100644--- a/products/wizard/backend/logic/registry/service.py+++ b/products/wizard/backend/logic/registry/service.py@@ -1,5 +1,4 @@-import logging-+import structlogimport posthoganalyticsfrom posthoganalytics.request import APIError, RequestsConnectionError, RequestsTimeout@@ -9,7 +8,7 @@ from products.wizard.backend.logic.registry.config import FALLBACK_REGISTRY, REGfrom products.wizard.backend.logic.registry.parser import parse_registry_payloadfrom products.wizard.backend.metrics import report_registry_fallback-logger = logging.getLogger(__name__)+logger = structlog.get_logger(__name__)def get_registry(*, distinct_id: str, organization_id: str) -> tuple[WizardProgram, ...]:@@ -23,14 +22,14 @@ def get_registry(*, distinct_id: str, organization_id: str) -> tuple[WizardProgrsend_feature_flag_events=False,)except (APIError, RequestsConnectionError, RequestsTimeout):- logger.warning("wizard_registry_fallback", extra={"reason": "request_failed"}, exc_info=True)+ logger.warning("wizard_registry_fallback", reason="request_failed", exc_info=True)report_registry_fallback("request_failed")return FALLBACK_REGISTRY.programstry:registry = parse_registry_payload(payload)except ValueError:- logger.warning("wizard_registry_fallback", extra={"reason": "invalid_payload"}, exc_info=True)+ logger.warning("wizard_registry_fallback", reason="invalid_payload", exc_info=True)report_registry_fallback("invalid_payload")return FALLBACK_REGISTRY.programsreturn registry.programsdiff --git a/products/wizard/backend/logic/runs/cancellation.py b/products/wizard/backend/logic/runs/cancellation.pyindex 1843cff9515e..c416c67dc2ba 100644--- a/products/wizard/backend/logic/runs/cancellation.py+++ b/products/wizard/backend/logic/runs/cancellation.py@@ -1,12 +1,13 @@-import loggingfrom uuid import UUID+import structlog+from products.wizard.backend.logic.runs import storefrom products.wizard.backend.observability.tracing import annotate_run_span, wizard_spanfrom products.wizard.backend.temporal import client as temporal_clientfrom products.wizard.backend.temporal.errors import WizardTemporalError-logger = logging.getLogger(__name__)+logger = structlog.get_logger(__name__)@wizard_span("wizard.run.cancel")@@ -19,7 +20,7 @@ def dispatch_cancellation(team_id: int, run_id: UUID) -> bool:with wizard_span("wizard.run.cancel_workflow"):temporal_client.cancel_wizard_run_workflow(run_id)except WizardTemporalError:- logger.exception("wizard_run_cancellation_failed", extra={"team_id": team_id, "run_id": str(run_id)})+ logger.exception("wizard_run_cancellation_failed", team_id=team_id, run_id=str(run_id))return Falsestore.mark_cancellation_dispatched(team_id, run_id)diff --git a/products/wizard/backend/logic/runs/lifecycle.py b/products/wizard/backend/logic/runs/lifecycle.pyindex 895e7104dd58..aba656ff56f7 100644--- a/products/wizard/backend/logic/runs/lifecycle.py+++ b/products/wizard/backend/logic/runs/lifecycle.py@@ -1,4 +1,3 @@-import loggingfrom dataclasses import replacefrom datetime import datetimefrom functools import partial@@ -7,6 +6,8 @@ from uuid import UUIDfrom django.db import transaction as database_transactionfrom django.utils import timezone+import structlog+from posthog.models import Team, Userfrom products.wizard.backend.facade.config import DEFAULT_WIZARD_VERSION@@ -49,7 +50,7 @@ from products.wizard.backend.logic.runs.transitions import transitionfrom products.wizard.backend.observability.service import wizard_observability as run_observabilityfrom products.wizard.backend.observability.tracing import annotate_run_span, wizard_span-logger = logging.getLogger(__name__)+logger = structlog.get_logger(__name__)def create_run(params: CreateWizardRunInput) -> WizardRunDTO:@@ -154,7 +155,9 @@ def _dispatch_cloud_run(team_id: int, run_id: UUID) -> None:except WizardRunDispatchError as error:logger.exception("wizard_run_dispatch_failed",- extra={"team_id": team_id, "run_id": str(run_id), "exhausted": error.exhausted},+ team_id=team_id,+ run_id=str(run_id),+ exhausted=error.exhausted,)if error.exhausted:fail_run(team_id, run_id, error_code=WizardRunErrorCode.DISPATCH_FAILED.value)diff --git a/products/wizard/backend/logic/runs/repository_access.py b/products/wizard/backend/logic/runs/repository_access.pyindex d8e1faa7ecf9..31c65a9fb7a8 100644--- a/products/wizard/backend/logic/runs/repository_access.py+++ b/products/wizard/backend/logic/runs/repository_access.py@@ -1,13 +1,13 @@-import logging-from django.core.cache import cache+import structlog+from products.tasks.backend.facade import repo_selectionfrom products.wizard.backend.facade.errors import MissingGitHubIntegrationError, RepositoryNotAccessibleErrorfrom products.wizard.backend.logic.runs.validation import validate_git_repository_namefrom products.wizard.backend.observability.tracing import wizard_span-logger = logging.getLogger(__name__)+logger = structlog.get_logger(__name__)REPOSITORY_CACHE_MIN_REFRESH_INTERVAL_SECONDS = 60@@ -48,7 +48,8 @@ def refresh_repository_cache(team_id: int) -> bool:logger.warning("wizard_repository_cache_refresh_failed",exc_info=True,- extra={"team_id": team_id, "integration_id": github.integration.id},+ team_id=team_id,+ integration_id=github.integration.id,)return Falsereturn Truediff --git a/products/wizard/backend/logic/runs/store.py b/products/wizard/backend/logic/runs/store.pyindex bd5a9b73a29d..b3ccaf082cda 100644--- a/products/wizard/backend/logic/runs/store.py+++ b/products/wizard/backend/logic/runs/store.py@@ -1,4 +1,3 @@-import loggingfrom uuid import UUIDfrom django.db import (@@ -7,6 +6,8 @@ from django.db import ()from django.utils import timezone+import structlog+from products.wizard.backend.facade.config import WIZARD_RUN_DEADLINEfrom products.wizard.backend.facade.contracts import (ListWizardRunsInput,@@ -35,7 +36,7 @@ from products.wizard.backend.logic.runs.mappers import record_to_run, workspace_from products.wizard.backend.logic.runs.pubsub import publish_run_updatefrom products.wizard.backend.models import WizardRun-logger = logging.getLogger(__name__)+logger = structlog.get_logger(__name__)def _get_run_record(team_id: int, run_id: UUID) -> WizardRun:@@ -191,7 +192,8 @@ def list_runs(params: ListWizardRunsInput) -> WizardRunPage:except ValueError:logger.exception("wizard_run_deserialization_failed",- extra={"team_id": params.team_id, "run_id": str(run.id)},+ team_id=params.team_id,+ run_id=str(run.id),)return WizardRunPage(results=tuple(results), count=runs.count())diff --git a/products/wizard/backend/logic/sessions/lifecycle.py b/products/wizard/backend/logic/sessions/lifecycle.pyindex d9c543831a63..b94f5000b2f9 100644--- a/products/wizard/backend/logic/sessions/lifecycle.py+++ b/products/wizard/backend/logic/sessions/lifecycle.py@@ -2,12 +2,12 @@Business logic for Wizard sessions."""-import loggingfrom functools import partialfrom typing import Anyfrom django.db import IntegrityError, transaction+import structlogfrom kombu.exceptions import OperationalErrorfrom products.wizard.backend.facade.contracts import UpsertWizardSessionInput, WizardSessionDTO@@ -19,7 +19,7 @@ from products.wizard.backend.metrics import report_session_upsertedfrom products.wizard.backend.models import WizardSessionfrom products.wizard.backend.tasks.tasks import sync_wizard_event_definitions-logger = logging.getLogger(__name__)+logger = structlog.get_logger(__name__)def upsert_session(params: UpsertWizardSessionInput) -> tuple[WizardSessionDTO, bool]:diff --git a/products/wizard/backend/logic/workers/config.py b/products/wizard/backend/logic/workers/config.pyindex 2d5d5b160a8b..0107669eae37 100644--- a/products/wizard/backend/logic/workers/config.py+++ b/products/wizard/backend/logic/workers/config.py@@ -7,6 +7,7 @@ from products.tasks.backend.facade.sandbox import SandboxTemplateWIZARD_TIMEOUT_SECONDS = 45 * 60WIZARD_TIMEOUT_EXIT_CODE = 124WIZARD_ERROR_DETAIL_LENGTH = 2000+WIZARD_ERROR_MESSAGE_LENGTH = 500WIZARD_ERROR_OUTPUT_PREFIX = "phw-error:"MAX_HANDOFF_BODY_BYTES = 60_000LOCAL_WIZARD_BUILD_TIMEOUT_SECONDS = 15 * 60diff --git a/products/wizard/backend/logic/workers/lifecycle.py b/products/wizard/backend/logic/workers/lifecycle.pyindex 0dbf2b297d12..6a2de5398e72 100644--- a/products/wizard/backend/logic/workers/lifecycle.py+++ b/products/wizard/backend/logic/workers/lifecycle.py@@ -1,9 +1,10 @@-import loggingfrom uuid import UUIDfrom django.core.exceptions import ObjectDoesNotExistfrom django.db import DatabaseError+import structlog+from products.tasks.backend.facade.sandbox import SandboxCleanupErrorfrom products.wizard.backend.facade.errors import WizardRunNotFoundErrorfrom products.wizard.backend.logic.runs import store as run_store@@ -16,7 +17,7 @@ from products.wizard.backend.logic.workers.contracts import WizardWorkerUsageMeafrom products.wizard.backend.observability.contracts import WizardWorkerCleanupOutcomefrom products.wizard.backend.observability.service import wizard_observability-logger = logging.getLogger(__name__)+logger = structlog.get_logger(__name__)def cleanup_worker(team_id: int, run_id: UUID, sandbox_id: str) -> None:@@ -47,7 +48,9 @@ def _record_worker_usage(except (DatabaseError, ObjectDoesNotExist, ValueError):logger.exception("wizard_worker_usage_recording_failed",- extra={"team_id": team_id, "run_id": str(run_id), "sandbox_id": sandbox_id},+ team_id=team_id,+ run_id=str(run_id),+ sandbox_id=sandbox_id,)@@ -69,7 +72,9 @@ def _record_cleanup_failure(team_id: int, run_id: UUID, sandbox_id: str) -> Noneexcept DatabaseError:logger.exception("wizard_worker_cleanup_failure_recording_failed",- extra={"team_id": team_id, "run_id": str(run_id), "sandbox_id": sandbox_id},+ team_id=team_id,+ run_id=str(run_id),+ sandbox_id=sandbox_id,)@@ -80,7 +85,8 @@ def _report_worker_usage(team_id: int, run_id: UUID) -> None:except (DatabaseError, ObjectDoesNotExist, ValueError, WizardRunNotFoundError):logger.exception("wizard_worker_usage_reporting_failed",- extra={"team_id": team_id, "run_id": str(run_id)},+ team_id=team_id,+ run_id=str(run_id),)returndiff --git a/products/wizard/backend/logic/workers/service.py b/products/wizard/backend/logic/workers/service.pyindex 72cef9a68611..98bfdf84b4f4 100644--- a/products/wizard/backend/logic/workers/service.py+++ b/products/wizard/backend/logic/workers/service.py@@ -1,4 +1,3 @@-import loggingfrom collections.abc import Callablefrom datetime import timedeltafrom pathlib import Path@@ -7,6 +6,8 @@ from uuid import UUIDfrom django.conf import settingsfrom django.utils import timezone+import structlog+from posthog.dataclasses import frozenfrom posthog.models.user import Userfrom posthog.temporal.oauth import create_wizard_oauth_access_token_for_user@@ -55,7 +56,7 @@ from products.wizard.backend.logic.workers.contracts import (WizardWorkerUsageMeasurement,)from products.wizard.backend.logic.workers.local_package import build_local_wizard_source_archive-from products.wizard.backend.logic.workers.wizard_error_output import stderr_to_wizard_error_code+from products.wizard.backend.logic.workers.wizard_error_output import WizardErrorLine, stderr_to_wizard_errorfrom products.wizard.backend.observability.service import wizard_observabilityfrom products.wizard.backend.observability.tracing import annotate_run_span, wizard_span@@ -66,7 +67,7 @@ from .repository_publisher import (stage_publishable_changes,)-logger = logging.getLogger(__name__)+logger = structlog.get_logger(__name__)@frozen@@ -117,12 +118,12 @@ class WizardWorkerExecutionError(Exception):stage: str,exit_code: int,detail: str | None = None,- wizard_error_code: str | None = None,+ wizard_error: WizardErrorLine | None = None,) -> None:self.stage = stageself.exit_code = exit_codeself.detail = detail- self.wizard_error_code = wizard_error_code+ self.wizard_error = wizard_errormessage = f"Wizard Worker {stage} failed with exit code {exit_code}."if detail:@@ -165,13 +166,11 @@ def _start_cpu_billing_sampler(sandbox: SandboxBase) -> None:try:started = sandbox.start_cpu_billing_sampler()except (SandboxExecutionError, SandboxNotFoundError, SandboxTimeoutError):- logger.warning(- "wizard_worker_cpu_billing_sampler_start_failed", extra={"sandbox_id": sandbox.id}, exc_info=True- )+ logger.warning("wizard_worker_cpu_billing_sampler_start_failed", sandbox_id=sandbox.id, exc_info=True)returnif not started:- logger.warning("wizard_worker_cpu_billing_sampler_start_failed", extra={"sandbox_id": sandbox.id})+ logger.warning("wizard_worker_cpu_billing_sampler_start_failed", sandbox_id=sandbox.id)@wizard_span("wizard.repository.prepare")@@ -346,7 +345,7 @@ def measure_worker_usage(sandbox_id: str) -> WizardWorkerUsageMeasurement | Nonetry:sandbox = get_sandbox_class().get_by_id(sandbox_id)except (SandboxExecutionError, SandboxNotFoundError, SandboxTimeoutError):- logger.exception("wizard_worker_usage_measurement_failed", extra={"sandbox_id": sandbox_id})+ logger.exception("wizard_worker_usage_measurement_failed", sandbox_id=sandbox_id)return Nonesandbox.config.cpu_cores = SANDBOX_CPU_CORES@@ -367,7 +366,7 @@ def _read_worker_usage(read_usage: Callable[[], int | None], sandbox_id: str) ->try:return read_usage()except Exception:- logger.exception("wizard_worker_usage_read_failed", extra={"sandbox_id": sandbox_id})+ logger.exception("wizard_worker_usage_read_failed", sandbox_id=sandbox_id)return None@@ -384,7 +383,7 @@ def _raise_for_failure(stage,exit_code,_failure_detail(stdout, stderr, sensitive_values),- stderr_to_wizard_error_code(stderr),+ stderr_to_wizard_error(stderr),)diff --git a/products/wizard/backend/logic/workers/wizard_error_output.py b/products/wizard/backend/logic/workers/wizard_error_output.pyindex af56c8c9e043..4178e6557f30 100644--- a/products/wizard/backend/logic/workers/wizard_error_output.py+++ b/products/wizard/backend/logic/workers/wizard_error_output.py@@ -1,10 +1,41 @@+import reimport json+from posthog.dataclasses import frozen+from products.wizard.backend.facade.validation import is_wizard_error_code-from products.wizard.backend.logic.workers.config import WIZARD_ERROR_OUTPUT_PREFIX+from products.wizard.backend.logic.workers.config import WIZARD_ERROR_MESSAGE_LENGTH, WIZARD_ERROR_OUTPUT_PREFIX++_REDACTED = "[REDACTED]"++# The Wizard copies raw exception text into this message, and the worker does not know the tokens the+# sandbox holds (the gateway token is minted inside it), so credentials are redacted by shape.+_CREDENTIAL_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (+ (re.compile(r"\bph[a-z]_[A-Za-z0-9_-]{8,}"), _REDACTED),+ (re.compile(r"\b(?:gh[oprsu]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{16,})"), _REDACTED),+ (re.compile(r"\bsk-[A-Za-z0-9_-]{16,}"), _REDACTED),+ (re.compile(r"(?<=://)[^/\s@]+@"), f"{_REDACTED}@"),+ (re.compile(r"(?i)\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]{8,}"), rf"\1 {_REDACTED}"),+ (+ re.compile(r"""(?i)((?:api[_-]?key|token|secret|password)["']?\s*[:=]\s*["']?)[^\s"',;&]+"""),+ rf"\1{_REDACTED}",+ ),+)++def _redact_credentials(text: str) -> str:+ for pattern, replacement in _CREDENTIAL_PATTERNS:+ text = pattern.sub(replacement, text)+ return text-def stderr_to_wizard_error_code(stderr: str) -> str | None:++@frozen+class WizardErrorLine:+ code: str+ message: str | None+++def stderr_to_wizard_error(stderr: str) -> WizardErrorLine | None:for line in reversed(stderr.splitlines()):value = line.strip()if not value.startswith(WIZARD_ERROR_OUTPUT_PREFIX):@@ -22,6 +53,14 @@ def stderr_to_wizard_error_code(stderr: str) -> str | None:if not isinstance(code, str) or not is_wizard_error_code(code):return None- return code+ message = payload.get("message")+ if not isinstance(message, str) or not message:+ return WizardErrorLine(code=code, message=None)++ # A stderr line can hold 64KB, so redaction runs on a bounded window to keep its cost fixed. The+ # window is twice the kept length, so a credential that crosses the final cut is still redacted+ # whole before the cut drops its tail.+ window = message[: 2 * WIZARD_ERROR_MESSAGE_LENGTH]+ return WizardErrorLine(code=code, message=_redact_credentials(window)[:WIZARD_ERROR_MESSAGE_LENGTH])return Nonediff --git a/products/wizard/backend/observability/service.py b/products/wizard/backend/observability/service.pyindex fe632cceb79c..bd7b8c1740be 100644--- a/products/wizard/backend/observability/service.py+++ b/products/wizard/backend/observability/service.py@@ -5,14 +5,17 @@ from uuid import UUIDfrom django.db import DatabaseError+import structlog+from products.wizard.backend.facade.contracts import WizardRunArtifactDTO, WizardRunDTO, WizardRunPullRequestArtifactDTOfrom products.wizard.backend.facade.enums import WizardRunStatusfrom products.wizard.backend.logic.workers.contracts import WizardWorkerTelemetry+from products.wizard.backend.logic.workers.wizard_error_output import WizardErrorLinefrom products.wizard.backend.observability import events, metricsfrom products.wizard.backend.observability.contracts import WizardRunDispatchOutcome, WizardWorkerCleanupOutcomefrom products.wizard.backend.observability.worker_usage import worker_usage_observation-logger = logging.getLogger(__name__)+logger = structlog.get_logger(__name__)class WizardObservability:@@ -31,16 +34,16 @@ class WizardObservability:try:metric()except ValueError:- logger.exception(metric_error or f"{name}_metric_failed", extra=context)+ logger.exception(metric_error or f"{name}_metric_failed", **context)if event is not None:try:event()except (DatabaseError, ValueError):- logger.exception(event_error or f"{name}_event_failed", extra=context)+ logger.exception(event_error or f"{name}_event_failed", **context)if level is not None:- logger.log(level, name, extra=context)+ logger.log(level, name, **context)def run_created(self, run: WizardRunDTO) -> None:self._observe(@@ -122,7 +125,7 @@ class WizardObservability:try:usage = worker_usage_observation(telemetry)except ValueError:- logger.exception("wizard_worker_usage_mapping_failed", extra=self._run_context(run))+ logger.exception("wizard_worker_usage_mapping_failed", **self._run_context(run))returnself._observe(@@ -142,6 +145,18 @@ class WizardObservability:event_error="wizard_worker_usage_event_failed",)+ def wizard_execution_failed(self, run: WizardRunDTO, exit_code: int, wizard_error: WizardErrorLine | None) -> None:+ self._observe(+ name="wizard_execution_failed",+ context={+ **self._run_context(run),+ "exit_code": exit_code,+ "wizard_error_code": wizard_error.code if wizard_error is not None else None,+ "wizard_error_message": wizard_error.message if wizard_error is not None else None,+ },+ level=logging.WARNING,+ )+def git_diff_omitted(self, run: WizardRunDTO, size_bytes: int) -> None:self._observe(name="wizard_git_diff_omitted",diff --git a/products/wizard/backend/tasks/reconciliation.py b/products/wizard/backend/tasks/reconciliation.pyindex a2d2c8e2561f..f836e7a6311c 100644--- a/products/wizard/backend/tasks/reconciliation.py+++ b/products/wizard/backend/tasks/reconciliation.py@@ -1,6 +1,6 @@-import loggingfrom time import monotonic+import structlogfrom celery import shared_taskfrom prometheus_client import Gauge@@ -10,7 +10,7 @@ from posthog.tasks.utils import PushGatewayTaskfrom products.wizard.backend.logic.reconciliation import service as reconciliationfrom products.wizard.backend.tasks.config import RECONCILE_WIZARD_RUNS_TASK-logger = logging.getLogger(__name__)+logger = structlog.get_logger(__name__)@shared_task(@@ -32,17 +32,15 @@ def reconcile_wizard_runs(self: PushGatewayTask) -> None:logger.info("wizard_run_reconciliation_completed",- extra={- "duration_seconds": round(monotonic() - started_at, 3),- "operations": {- operation: {- "scanned": result.scanned,- "reconciled": result.reconciled,- "failed": result.failed,- "batch_limit_reached": result.batch_limit_reached,- }- for operation, result in results.items()- },+ duration_seconds=round(monotonic() - started_at, 3),+ operations={+ operation: {+ "scanned": result.scanned,+ "reconciled": result.reconciled,+ "failed": result.failed,+ "batch_limit_reached": result.batch_limit_reached,+ }+ for operation, result in results.items()},)diff --git a/products/wizard/backend/tasks/tasks.py b/products/wizard/backend/tasks/tasks.pyindex 57967d958627..f90cec8615bc 100644--- a/products/wizard/backend/tasks/tasks.py+++ b/products/wizard/backend/tasks/tasks.py@@ -1,8 +1,8 @@-import loggingfrom typing import Anyfrom django.db import DatabaseError+import structlogfrom celery import shared_taskfrom celery.app.task import Task@@ -15,7 +15,7 @@ from products.event_definitions.backend.logic.placeholder import (from products.wizard.backend.facade.enums import WizardSessionRunPhasefrom products.wizard.backend.models import WizardSession-logger = logging.getLogger(__name__)+logger = structlog.get_logger(__name__)MAX_PLANNED_EVENT_DEFINITIONS = 50MAX_PLANNED_EVENT_CANDIDATES = 500@@ -74,6 +74,8 @@ def sync_wizard_event_definitions(self: Task, team_id: int, session_id: str) ->countdown = min(2**self.request.retries, 60)logger.warning("Retrying wizard event definition sync",- extra={"team_id": team_id, "session_id": session_id, "retry": self.request.retries},+ team_id=team_id,+ session_id=session_id,+ retry=self.request.retries,)raise self.retry(exc=error, countdown=countdown)diff --git a/products/wizard/backend/temporal/activities/execution.py b/products/wizard/backend/temporal/activities/execution.pyindex a4568d7c59ad..2917578debd6 100644--- a/products/wizard/backend/temporal/activities/execution.py+++ b/products/wizard/backend/temporal/activities/execution.py@@ -7,6 +7,7 @@ from products.wizard.backend.facade import api as wizard_facadefrom products.wizard.backend.facade.enums import WizardRunStagefrom products.wizard.backend.logic.workers import service as cloud_workerfrom products.wizard.backend.logic.workers.service import WizardExecutionRequest+from products.wizard.backend.observability.service import wizard_observabilityfrom products.wizard.backend.observability.tracing import annotate_run_spanfrom products.wizard.backend.temporal.activities.errors import (WIZARD_WORKER_EXECUTION_ERROR_TYPE,@@ -41,8 +42,11 @@ def execute_wizard(input: PreparedGitRepositoryWorkspace) -> None:non_retryable=True,) from errorexcept cloud_worker.WizardWorkerExecutionError as error:+ # sanitize_activity_errors drops this error's message before Temporal records it, so this log+ # line is the only place the wizard's own failure message survives.+ wizard_observability.wizard_execution_failed(run, error.exit_code, error.wizard_error)raise ApplicationError(str(error),- type=error.wizard_error_code or WIZARD_WORKER_EXECUTION_ERROR_TYPE,+ type=error.wizard_error.code if error.wizard_error is not None else WIZARD_WORKER_EXECUTION_ERROR_TYPE,non_retryable=True,) from errordiff --git a/products/wizard/backend/tests/runs/test_cloud_activities.py b/products/wizard/backend/tests/runs/test_cloud_activities.pyindex 058f6a8be599..a9ae54e4a5ca 100644--- a/products/wizard/backend/tests/runs/test_cloud_activities.py+++ b/products/wizard/backend/tests/runs/test_cloud_activities.py@@ -15,6 +15,7 @@ from opentelemetry.sdk.trace import TracerProviderfrom opentelemetry.sdk.trace.export import SimpleSpanProcessorfrom opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporterfrom opentelemetry.trace import StatusCode+from structlog.testing import capture_logsfrom temporalio.contrib.opentelemetry import OpenTelemetryInterceptorfrom temporalio.exceptions import ApplicationError, ApplicationErrorCategory, CancelledErrorfrom temporalio.testing import ActivityEnvironment@@ -46,6 +47,7 @@ from products.wizard.backend.logic.workers.service import (WizardWorkerResult,WizardWorkerTimeoutError,)+from products.wizard.backend.logic.workers.wizard_error_output import WizardErrorLinefrom products.wizard.backend.temporal.activities.errors import (WIZARD_REPOSITORY_ACCESS_ERROR_TYPE,WIZARD_WORKER_EXECUTION_ERROR_TYPE,@@ -298,7 +300,11 @@ def test_clone_repository_keeps_transient_clone_failure_retryable(team, user) ->True,),(- WizardWorkerExecutionError("execution", 1, wizard_error_code="PHW_DETECT_NO_POSTHOG_SDK"),+ WizardWorkerExecutionError(+ "execution",+ 1,+ wizard_error=WizardErrorLine(code="PHW_DETECT_NO_POSTHOG_SDK", message="Missing SDK"),+ ),"PHW_DETECT_NO_POSTHOG_SDK",True,),@@ -339,6 +345,7 @@ def test_execute_wizard_maps_worker_error(worker_error: Exception, error_type: sside_effect=worker_error,) as execute,pytest.raises(ApplicationError) as error,+ capture_logs() as logs,):get_run.return_value.program.command = ()get_run.return_value.program.wizard_version = "2.60.0"@@ -361,6 +368,16 @@ def test_execute_wizard_maps_worker_error(worker_error: Exception, error_type: sassert error.value.__cause__ is Noneassert error.value.__context__ is Noneassert not error.value.details+ failure_logs = [log for log in logs if log["event"] == "wizard_execution_failed"]+ if isinstance(worker_error, WizardWorkerExecutionError):+ assert len(failure_logs) == 1+ expected_error = worker_error.wizard_error+ assert failure_logs[0]["log_level"] == "warning"+ assert failure_logs[0]["wizard_error_code"] == (expected_error.code if expected_error else None)+ assert failure_logs[0]["wizard_error_message"] == (expected_error.message if expected_error else None)+ else:+ assert not failure_logs+ assert "fake-wizard-api-key-for-test" not in str(logs)if isinstance(worker_error, ApplicationError):assert error.value.next_retry_delay == worker_error.next_retry_delayassert error.value.category == worker_error.categorydiff --git a/products/wizard/backend/tests/runs/test_cloud_worker.py b/products/wizard/backend/tests/runs/test_cloud_worker.pyindex cbe1f05c9108..c73f7d5ad45a 100644--- a/products/wizard/backend/tests/runs/test_cloud_worker.py+++ b/products/wizard/backend/tests/runs/test_cloud_worker.py@@ -48,6 +48,7 @@ from products.wizard.backend.logic.workers.service import (prepare_local_wizard,provision_wizard_worker,)+from products.wizard.backend.logic.workers.wizard_error_output import WizardErrorLinefrom products.wizard.backend.observability.tracing import wizard_span@@ -452,7 +453,7 @@ def test_execute_wizard_surfaces_wizard_error_code(get_sandbox_class: MagicMock)with pytest.raises(WizardWorkerExecutionError) as error:execute_wizard(request)- assert error.value.wizard_error_code == "PHW_DETECT_NO_POSTHOG_SDK"+ assert error.value.wizard_error == WizardErrorLine(code="PHW_DETECT_NO_POSTHOG_SDK", message="Missing SDK")@patch("products.wizard.backend.logic.workers.service.stage_publishable_changes")diff --git a/products/wizard/backend/tests/runs/test_observability.py b/products/wizard/backend/tests/runs/test_observability.pyindex a7922a17a620..e66d4a3736a1 100644--- a/products/wizard/backend/tests/runs/test_observability.py+++ b/products/wizard/backend/tests/runs/test_observability.py@@ -1,4 +1,3 @@-import loggingfrom dataclasses import replacefrom datetime import UTC, datetime, timedeltafrom uuid import UUID, uuid4@@ -12,6 +11,7 @@ from django.test import override_settingsfrom parameterized import parameterizedfrom prometheus_client import REGISTRY+from structlog.testing import capture_logsfrom products.wizard.backend.facade.contracts import (CreateWizardRunInput,@@ -71,7 +71,7 @@ class TestWizardObservability(TestCase):patch.object(metrics.WIZARD_RUN_DISPATCH_ATTEMPTS_TOTAL, "labels") as counter,patch.object(events, "ph_background_capture") as background_capture,patch.object(events.User.objects, "filter") as users,- self.assertLogs("products.wizard.backend.observability.service", level=logging.INFO) as logs,+ capture_logs() as logs,):counter.return_value.inc.side_effect = ValueError("Metric unavailable") if metric_fails else Noneusers.return_value.values_list.return_value.first.return_value = "example-user"@@ -82,11 +82,13 @@ class TestWizardObservability(TestCase):counter.return_value.inc.assert_called_once_with()background_capture.return_value.assert_called_once()self.assertEqual(background_capture.return_value.call_args.kwargs["event"], "wizard run dispatch finished")- messages = [record.getMessage() for record in logs.records]+ messages = [log["event"] for log in logs]self.assertEqual("wizard_run_dispatch_metric_failed" in messages, metric_fails)self.assertEqual("wizard_run_dispatch_event_failed" in messages, event_fails)self.assertEqual(messages[-1], "wizard_run_dispatch_finished")- self.assertEqual(logs.records[-1].levelno, logging.INFO)+ self.assertEqual(logs[-1]["log_level"], "info")+ self.assertEqual(logs[-1]["run_id"], str(run.id))+ self.assertEqual(logs[-1]["outcome"], WizardRunDispatchOutcome.SUCCEEDED.value)def _cloud_run() -> WizardRunDTO:diff --git a/products/wizard/backend/tests/runs/test_wizard_error_output.py b/products/wizard/backend/tests/runs/test_wizard_error_output.pyindex f134a815e5b5..13a3c865a47c 100644--- a/products/wizard/backend/tests/runs/test_wizard_error_output.py+++ b/products/wizard/backend/tests/runs/test_wizard_error_output.py@@ -1,6 +1,8 @@+import json+import pytest-from products.wizard.backend.logic.workers.wizard_error_output import stderr_to_wizard_error_code+from products.wizard.backend.logic.workers.wizard_error_output import WizardErrorLine, stderr_to_wizard_error@pytest.mark.parametrize(@@ -8,18 +10,48 @@ from products.wizard.backend.logic.workers.wizard_error_output import stderr_to_(('Wizard output\nphw-error: {"code":"PHW_DETECT_NO_POSTHOG_SDK","message":"Missing SDK"}',- "PHW_DETECT_NO_POSTHOG_SDK",+ WizardErrorLine(code="PHW_DETECT_NO_POSTHOG_SDK", message="Missing SDK"),),('phw-error: {"code":"PHW_DETECT_NO_FRAMEWORK"}\nphw-error: {"code":"PHW_AGENT_NO_PROGRESS"}',- "PHW_AGENT_NO_PROGRESS",+ WizardErrorLine(code="PHW_AGENT_NO_PROGRESS", message=None),+ ),+ (+ 'phw-error: {"code":"PHW_AGENT_API_ERROR","message":' + '"' + "x" * 600 + '"}',+ WizardErrorLine(code="PHW_AGENT_API_ERROR", message="x" * 500),+ ),+ (+ 'phw-error: {"code":"PHW_FUTURE_ERROR","message":{"nested":true}}',+ WizardErrorLine(code="PHW_FUTURE_ERROR", message=None),+ ),+ *(+ (+ 'phw-error: {"code":"PHW_AGENT_API_ERROR","message":' + json.dumps(message) + "}",+ WizardErrorLine(code="PHW_AGENT_API_ERROR", message=expected_message),+ )+ for message, expected_message in (+ ("Gateway refused pha_FakeOAuthToken1234 for team 2", "Gateway refused [REDACTED] for team 2"),+ ("clone failed: gho_FAKEFAKEFAKEFAKEFAKE1234", "clone failed: [REDACTED]"),+ (+ "fetch https://x-access-token:[email protected]/org/repo.git failed",+ "fetch https://[REDACTED]@github.com/org/repo.git failed",+ ),+ ("401 with Authorization: Bearer fake.jwt.value-1234", "401 with Authorization: Bearer [REDACTED]"),+ ('config {"apiKey": "fake-key-value"} rejected', 'config {"apiKey": "[REDACTED]"} rejected'),+ ("refresh failed: access_token=fake-access-value", "refresh failed: access_token=[REDACTED]"),+ ("x" * 490 + " pha_FakeOAuthToken1234", "x" * 490 + " [REDACTED"),+ (+ "OpenAI API error (400): 400 invalid request body",+ "OpenAI API error (400): 400 invalid request body",+ ),+ ("Headless mode requires --api-key (phx_xxx)", "Headless mode requires --api-key (phx_xxx)"),+ )),- ('phw-error: {"code":"PHW_FUTURE_ERROR"}', "PHW_FUTURE_ERROR"),('phw-error: {"code":"detect_no_framework"}', None),('phw-error: {"message":"Missing code"}', None),("phw-error: not-json", None),("ordinary stderr", None),),)-def test_stderr_to_wizard_error_code(stderr: str, expected: str | None) -> None:- assert stderr_to_wizard_error_code(stderr) == expected+def test_stderr_to_wizard_error(stderr: str, expected: WizardErrorLine | None) -> None:+ assert stderr_to_wizard_error(stderr) == expecteddiff --git a/products/wizard/docs/reliability.md b/products/wizard/docs/reliability.mdindex 16577ad64cd9..6709fcb6561b 100644--- a/products/wizard/docs/reliability.md+++ b/products/wizard/docs/reliability.md@@ -21,4 +21,10 @@ The Wizard writes a final `phw-error:` record to standard error when a program fThe backend stores only the code. The UI owns the user message and recovery steps. If the record is absent or invalid, use the platform error for the failed stage.+The execution activity also logs a `wizard_execution_failed` warning with the run ID, Wizard version, exit code, and the record's code and message.+The worker redacts credential-shaped values in the message, such as PostHog and GitHub tokens, URL credentials, and `Authorization` values. It then cuts the message to 500 characters.+It does not enter Temporal workflow history, run metadata, or analytics events.+To investigate a failed cloud run, search the worker logs for its run ID.+The CLI's own events, including its `$exception`, carry the same ID as `task_run_id`.+Collect latency data before you set latency SLOs. Review these targets after 30 to 60 days of production data.