Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(wizard): pin cloud runs to wizard 2.86.0 and log failure messages (#121741)

Rafael Audibert89c2dea98e0c
diff --git a/products/wizard/backend/facade/config.py b/products/wizard/backend/facade/config.py
index 6ba92a5f23a9..0a8d87c75d48 100644
--- a/products/wizard/backend/facade/config.py
+++ b/products/wizard/backend/facade/config.py
@@ -1,6 +1,6 @@
from datetime import timedelta
-DEFAULT_WIZARD_VERSION = "2.78.0"
+DEFAULT_WIZARD_VERSION = "2.86.0"
LATEST_WIZARD_VERSION = "latest"
WIZARD_REGISTRY_VERSION = 1
WIZARD_RUN_DEADLINE = timedelta(minutes=85)
diff --git a/products/wizard/backend/logic/artifacts/service.py b/products/wizard/backend/logic/artifacts/service.py
index bd0e38c9e278..5f5c15c76707 100644
--- a/products/wizard/backend/logic/artifacts/service.py
+++ b/products/wizard/backend/logic/artifacts/service.py
@@ -1,9 +1,10 @@
-import logging
from hashlib import sha256
from uuid import UUID
from django.conf import settings
+import structlog
+
from posthog.dataclasses import frozen
from posthog.storage import object_storage
@@ -20,7 +21,7 @@ from products.wizard.backend.logic.artifacts.config import MAX_GIT_DIFF_BYTES, M
from products.wizard.backend.logic.runs import store as run_store
from products.wizard.backend.observability.service import wizard_observability as run_observability
-logger = logging.getLogger(__name__)
+logger = structlog.get_logger(__name__)
@frozen
@@ -110,7 +111,9 @@ def get_git_diff_artifact_content(team_id: int, run_id: UUID, artifact_id: UUID)
if content_hash != stored_artifact.content_hash:
logger.error(
"wizard_git_diff_integrity_check_failed",
- extra={"team_id": team_id, "run_id": str(run_id), "artifact_id": str(artifact_id)},
+ team_id=team_id,
+ run_id=str(run_id),
+ artifact_id=str(artifact_id),
)
raise ValueError("Stored Wizard diff failed its integrity check")
diff --git a/products/wizard/backend/logic/reconciliation/service.py b/products/wizard/backend/logic/reconciliation/service.py
index d5c5618acf74..0785d4c44800 100644
--- a/products/wizard/backend/logic/reconciliation/service.py
+++ b/products/wizard/backend/logic/reconciliation/service.py
@@ -1,9 +1,10 @@
-import logging
from time import monotonic
from django.db.models import Q
from django.utils import timezone
+import structlog
+
from posthog.dataclasses import frozen
from products.wizard.backend.facade.enums import (
@@ -25,7 +26,7 @@ from products.wizard.backend.logic.workers import lifecycle as worker_lifecycle
from products.wizard.backend.models import WizardRun, WizardWorker
from products.wizard.backend.observability.service import wizard_observability
-logger = logging.getLogger(__name__)
+logger = structlog.get_logger(__name__)
@frozen
@@ -69,7 +70,9 @@ def reconcile_pending_dispatches() -> ReconciliationSummary:
except WizardRunDispatchError as error:
logger.exception(
"wizard_run_redispatch_failed",
- extra={"team_id": team_id, "run_id": str(run_id), "exhausted": error.exhausted},
+ team_id=team_id,
+ run_id=str(run_id),
+ exhausted=error.exhausted,
)
if error.exhausted:
try:
@@ -77,11 +80,12 @@ def reconcile_pending_dispatches() -> ReconciliationSummary:
except (IllegalStatusTransitionError, WizardRunNotFoundError, ValueError):
logger.exception(
"wizard_run_dispatch_exhaustion_failed",
- extra={"team_id": team_id, "run_id": str(run_id)},
+ team_id=team_id,
+ run_id=str(run_id),
)
continue
except (WizardRunNotFoundError, ValueError):
- logger.exception("wizard_run_redispatch_failed", extra={"team_id": team_id, "run_id": str(run_id)})
+ logger.exception("wizard_run_redispatch_failed", team_id=team_id, run_id=str(run_id))
continue
reconciled += 1
@@ -111,7 +115,8 @@ def reconcile_pending_cancellations() -> ReconciliationSummary:
except (WizardRunNotFoundError, ValueError):
logger.exception(
"wizard_run_cancellation_reconciliation_failed",
- extra={"team_id": team_id, "run_id": str(run_id)},
+ team_id=team_id,
+ run_id=str(run_id),
)
return _summary(len(pending), reconciled)
@@ -132,7 +137,7 @@ def reconcile_expired_runs() -> ReconciliationSummary:
try:
run = lifecycle.fail_run(team_id, run_id, error_code=WizardRunErrorCode.TIMEOUT)
except (IllegalStatusTransitionError, WizardRunNotFoundError, ValueError):
- logger.exception("wizard_run_expiration_failed", extra={"team_id": team_id, "run_id": str(run_id)})
+ logger.exception("wizard_run_expiration_failed", team_id=team_id, run_id=str(run_id))
continue
wizard_observability.run_past_deadline(run)
if workflow_id is not None:
@@ -174,7 +179,7 @@ def reconcile_pending_worker_cleanup() -> ReconciliationSummary:
try:
worker_lifecycle.cleanup_worker(team_id, run_id, sandbox_id)
except WizardWorkerCleanupError:
- logger.exception("wizard_worker_reconciliation_failed", extra={"team_id": team_id, "run_id": str(run_id)})
+ logger.exception("wizard_worker_reconciliation_failed", team_id=team_id, run_id=str(run_id))
continue
reconciled += 1
diff --git a/products/wizard/backend/logic/registry/service.py b/products/wizard/backend/logic/registry/service.py
index a5b045017f63..f9767d874175 100644
--- a/products/wizard/backend/logic/registry/service.py
+++ b/products/wizard/backend/logic/registry/service.py
@@ -1,5 +1,4 @@
-import logging
-
+import structlog
import posthoganalytics
from posthoganalytics.request import APIError, RequestsConnectionError, RequestsTimeout
@@ -9,7 +8,7 @@ from products.wizard.backend.logic.registry.config import FALLBACK_REGISTRY, REG
from products.wizard.backend.logic.registry.parser import parse_registry_payload
from products.wizard.backend.metrics import report_registry_fallback
-logger = logging.getLogger(__name__)
+logger = structlog.get_logger(__name__)
def get_registry(*, distinct_id: str, organization_id: str) -> tuple[WizardProgram, ...]:
@@ -23,14 +22,14 @@ def get_registry(*, distinct_id: str, organization_id: str) -> tuple[WizardProgr
send_feature_flag_events=False,
)
except (APIError, RequestsConnectionError, RequestsTimeout):
- logger.warning("wizard_registry_fallback", extra={"reason": "request_failed"}, exc_info=True)
+ logger.warning("wizard_registry_fallback", reason="request_failed", exc_info=True)
report_registry_fallback("request_failed")
return FALLBACK_REGISTRY.programs
try:
registry = parse_registry_payload(payload)
except ValueError:
- logger.warning("wizard_registry_fallback", extra={"reason": "invalid_payload"}, exc_info=True)
+ logger.warning("wizard_registry_fallback", reason="invalid_payload", exc_info=True)
report_registry_fallback("invalid_payload")
return FALLBACK_REGISTRY.programs
return registry.programs
diff --git a/products/wizard/backend/logic/runs/cancellation.py b/products/wizard/backend/logic/runs/cancellation.py
index 1843cff9515e..c416c67dc2ba 100644
--- a/products/wizard/backend/logic/runs/cancellation.py
+++ b/products/wizard/backend/logic/runs/cancellation.py
@@ -1,12 +1,13 @@
-import logging
from uuid import UUID
+import structlog
+
from products.wizard.backend.logic.runs import store
from products.wizard.backend.observability.tracing import annotate_run_span, wizard_span
from products.wizard.backend.temporal import client as temporal_client
from products.wizard.backend.temporal.errors import WizardTemporalError
-logger = logging.getLogger(__name__)
+logger = structlog.get_logger(__name__)
@wizard_span("wizard.run.cancel")
@@ -19,7 +20,7 @@ def dispatch_cancellation(team_id: int, run_id: UUID) -> bool:
with wizard_span("wizard.run.cancel_workflow"):
temporal_client.cancel_wizard_run_workflow(run_id)
except WizardTemporalError:
- logger.exception("wizard_run_cancellation_failed", extra={"team_id": team_id, "run_id": str(run_id)})
+ logger.exception("wizard_run_cancellation_failed", team_id=team_id, run_id=str(run_id))
return False
store.mark_cancellation_dispatched(team_id, run_id)
diff --git a/products/wizard/backend/logic/runs/lifecycle.py b/products/wizard/backend/logic/runs/lifecycle.py
index 895e7104dd58..aba656ff56f7 100644
--- a/products/wizard/backend/logic/runs/lifecycle.py
+++ b/products/wizard/backend/logic/runs/lifecycle.py
@@ -1,4 +1,3 @@
-import logging
from dataclasses import replace
from datetime import datetime
from functools import partial
@@ -7,6 +6,8 @@ from uuid import UUID
from django.db import transaction as database_transaction
from django.utils import timezone
+import structlog
+
from posthog.models import Team, User
from products.wizard.backend.facade.config import DEFAULT_WIZARD_VERSION
@@ -49,7 +50,7 @@ from products.wizard.backend.logic.runs.transitions import transition
from products.wizard.backend.observability.service import wizard_observability as run_observability
from products.wizard.backend.observability.tracing import annotate_run_span, wizard_span
-logger = logging.getLogger(__name__)
+logger = structlog.get_logger(__name__)
def create_run(params: CreateWizardRunInput) -> WizardRunDTO:
@@ -154,7 +155,9 @@ def _dispatch_cloud_run(team_id: int, run_id: UUID) -> None:
except WizardRunDispatchError as error:
logger.exception(
"wizard_run_dispatch_failed",
- extra={"team_id": team_id, "run_id": str(run_id), "exhausted": error.exhausted},
+ team_id=team_id,
+ run_id=str(run_id),
+ exhausted=error.exhausted,
)
if error.exhausted:
fail_run(team_id, run_id, error_code=WizardRunErrorCode.DISPATCH_FAILED.value)
diff --git a/products/wizard/backend/logic/runs/repository_access.py b/products/wizard/backend/logic/runs/repository_access.py
index d8e1faa7ecf9..31c65a9fb7a8 100644
--- a/products/wizard/backend/logic/runs/repository_access.py
+++ b/products/wizard/backend/logic/runs/repository_access.py
@@ -1,13 +1,13 @@
-import logging
-
from django.core.cache import cache
+import structlog
+
from products.tasks.backend.facade import repo_selection
from products.wizard.backend.facade.errors import MissingGitHubIntegrationError, RepositoryNotAccessibleError
from products.wizard.backend.logic.runs.validation import validate_git_repository_name
from products.wizard.backend.observability.tracing import wizard_span
-logger = logging.getLogger(__name__)
+logger = structlog.get_logger(__name__)
REPOSITORY_CACHE_MIN_REFRESH_INTERVAL_SECONDS = 60
@@ -48,7 +48,8 @@ def refresh_repository_cache(team_id: int) -> bool:
logger.warning(
"wizard_repository_cache_refresh_failed",
exc_info=True,
- extra={"team_id": team_id, "integration_id": github.integration.id},
+ team_id=team_id,
+ integration_id=github.integration.id,
)
return False
return True
diff --git a/products/wizard/backend/logic/runs/store.py b/products/wizard/backend/logic/runs/store.py
index bd5a9b73a29d..b3ccaf082cda 100644
--- a/products/wizard/backend/logic/runs/store.py
+++ b/products/wizard/backend/logic/runs/store.py
@@ -1,4 +1,3 @@
-import logging
from uuid import UUID
from django.db import (
@@ -7,6 +6,8 @@ from django.db import (
)
from django.utils import timezone
+import structlog
+
from products.wizard.backend.facade.config import WIZARD_RUN_DEADLINE
from products.wizard.backend.facade.contracts import (
ListWizardRunsInput,
@@ -35,7 +36,7 @@ from products.wizard.backend.logic.runs.mappers import record_to_run, workspace_
from products.wizard.backend.logic.runs.pubsub import publish_run_update
from products.wizard.backend.models import WizardRun
-logger = logging.getLogger(__name__)
+logger = structlog.get_logger(__name__)
def _get_run_record(team_id: int, run_id: UUID) -> WizardRun:
@@ -191,7 +192,8 @@ def list_runs(params: ListWizardRunsInput) -> WizardRunPage:
except ValueError:
logger.exception(
"wizard_run_deserialization_failed",
- extra={"team_id": params.team_id, "run_id": str(run.id)},
+ team_id=params.team_id,
+ run_id=str(run.id),
)
return WizardRunPage(results=tuple(results), count=runs.count())
diff --git a/products/wizard/backend/logic/sessions/lifecycle.py b/products/wizard/backend/logic/sessions/lifecycle.py
index d9c543831a63..b94f5000b2f9 100644
--- a/products/wizard/backend/logic/sessions/lifecycle.py
+++ b/products/wizard/backend/logic/sessions/lifecycle.py
@@ -2,12 +2,12 @@
Business logic for Wizard sessions.
"""
-import logging
from functools import partial
from typing import Any
from django.db import IntegrityError, transaction
+import structlog
from kombu.exceptions import OperationalError
from products.wizard.backend.facade.contracts import UpsertWizardSessionInput, WizardSessionDTO
@@ -19,7 +19,7 @@ from products.wizard.backend.metrics import report_session_upserted
from products.wizard.backend.models import WizardSession
from products.wizard.backend.tasks.tasks import sync_wizard_event_definitions
-logger = logging.getLogger(__name__)
+logger = structlog.get_logger(__name__)
def upsert_session(params: UpsertWizardSessionInput) -> tuple[WizardSessionDTO, bool]:
diff --git a/products/wizard/backend/logic/workers/config.py b/products/wizard/backend/logic/workers/config.py
index 2d5d5b160a8b..0107669eae37 100644
--- a/products/wizard/backend/logic/workers/config.py
+++ b/products/wizard/backend/logic/workers/config.py
@@ -7,6 +7,7 @@ from products.tasks.backend.facade.sandbox import SandboxTemplate
WIZARD_TIMEOUT_SECONDS = 45 * 60
WIZARD_TIMEOUT_EXIT_CODE = 124
WIZARD_ERROR_DETAIL_LENGTH = 2000
+WIZARD_ERROR_MESSAGE_LENGTH = 500
WIZARD_ERROR_OUTPUT_PREFIX = "phw-error:"
MAX_HANDOFF_BODY_BYTES = 60_000
LOCAL_WIZARD_BUILD_TIMEOUT_SECONDS = 15 * 60
diff --git a/products/wizard/backend/logic/workers/lifecycle.py b/products/wizard/backend/logic/workers/lifecycle.py
index 0dbf2b297d12..6a2de5398e72 100644
--- a/products/wizard/backend/logic/workers/lifecycle.py
+++ b/products/wizard/backend/logic/workers/lifecycle.py
@@ -1,9 +1,10 @@
-import logging
from uuid import UUID
from django.core.exceptions import ObjectDoesNotExist
from django.db import DatabaseError
+import structlog
+
from products.tasks.backend.facade.sandbox import SandboxCleanupError
from products.wizard.backend.facade.errors import WizardRunNotFoundError
from products.wizard.backend.logic.runs import store as run_store
@@ -16,7 +17,7 @@ from products.wizard.backend.logic.workers.contracts import WizardWorkerUsageMea
from products.wizard.backend.observability.contracts import WizardWorkerCleanupOutcome
from products.wizard.backend.observability.service import wizard_observability
-logger = logging.getLogger(__name__)
+logger = structlog.get_logger(__name__)
def cleanup_worker(team_id: int, run_id: UUID, sandbox_id: str) -> None:
@@ -47,7 +48,9 @@ def _record_worker_usage(
except (DatabaseError, ObjectDoesNotExist, ValueError):
logger.exception(
"wizard_worker_usage_recording_failed",
- extra={"team_id": team_id, "run_id": str(run_id), "sandbox_id": sandbox_id},
+ team_id=team_id,
+ run_id=str(run_id),
+ sandbox_id=sandbox_id,
)
@@ -69,7 +72,9 @@ def _record_cleanup_failure(team_id: int, run_id: UUID, sandbox_id: str) -> None
except DatabaseError:
logger.exception(
"wizard_worker_cleanup_failure_recording_failed",
- extra={"team_id": team_id, "run_id": str(run_id), "sandbox_id": sandbox_id},
+ team_id=team_id,
+ run_id=str(run_id),
+ sandbox_id=sandbox_id,
)
@@ -80,7 +85,8 @@ def _report_worker_usage(team_id: int, run_id: UUID) -> None:
except (DatabaseError, ObjectDoesNotExist, ValueError, WizardRunNotFoundError):
logger.exception(
"wizard_worker_usage_reporting_failed",
- extra={"team_id": team_id, "run_id": str(run_id)},
+ team_id=team_id,
+ run_id=str(run_id),
)
return
diff --git a/products/wizard/backend/logic/workers/service.py b/products/wizard/backend/logic/workers/service.py
index 72cef9a68611..98bfdf84b4f4 100644
--- a/products/wizard/backend/logic/workers/service.py
+++ b/products/wizard/backend/logic/workers/service.py
@@ -1,4 +1,3 @@
-import logging
from collections.abc import Callable
from datetime import timedelta
from pathlib import Path
@@ -7,6 +6,8 @@ from uuid import UUID
from django.conf import settings
from django.utils import timezone
+import structlog
+
from posthog.dataclasses import frozen
from posthog.models.user import User
from posthog.temporal.oauth import create_wizard_oauth_access_token_for_user
@@ -55,7 +56,7 @@ from products.wizard.backend.logic.workers.contracts import (
WizardWorkerUsageMeasurement,
)
from products.wizard.backend.logic.workers.local_package import build_local_wizard_source_archive
-from products.wizard.backend.logic.workers.wizard_error_output import stderr_to_wizard_error_code
+from products.wizard.backend.logic.workers.wizard_error_output import WizardErrorLine, stderr_to_wizard_error
from products.wizard.backend.observability.service import wizard_observability
from products.wizard.backend.observability.tracing import annotate_run_span, wizard_span
@@ -66,7 +67,7 @@ from .repository_publisher import (
stage_publishable_changes,
)
-logger = logging.getLogger(__name__)
+logger = structlog.get_logger(__name__)
@frozen
@@ -117,12 +118,12 @@ class WizardWorkerExecutionError(Exception):
stage: str,
exit_code: int,
detail: str | None = None,
- wizard_error_code: str | None = None,
+ wizard_error: WizardErrorLine | None = None,
) -> None:
self.stage = stage
self.exit_code = exit_code
self.detail = detail
- self.wizard_error_code = wizard_error_code
+ self.wizard_error = wizard_error
message = f"Wizard Worker {stage} failed with exit code {exit_code}."
if detail:
@@ -165,13 +166,11 @@ def _start_cpu_billing_sampler(sandbox: SandboxBase) -> None:
try:
started = sandbox.start_cpu_billing_sampler()
except (SandboxExecutionError, SandboxNotFoundError, SandboxTimeoutError):
- logger.warning(
- "wizard_worker_cpu_billing_sampler_start_failed", extra={"sandbox_id": sandbox.id}, exc_info=True
- )
+ logger.warning("wizard_worker_cpu_billing_sampler_start_failed", sandbox_id=sandbox.id, exc_info=True)
return
if not started:
- logger.warning("wizard_worker_cpu_billing_sampler_start_failed", extra={"sandbox_id": sandbox.id})
+ logger.warning("wizard_worker_cpu_billing_sampler_start_failed", sandbox_id=sandbox.id)
@wizard_span("wizard.repository.prepare")
@@ -346,7 +345,7 @@ def measure_worker_usage(sandbox_id: str) -> WizardWorkerUsageMeasurement | None
try:
sandbox = get_sandbox_class().get_by_id(sandbox_id)
except (SandboxExecutionError, SandboxNotFoundError, SandboxTimeoutError):
- logger.exception("wizard_worker_usage_measurement_failed", extra={"sandbox_id": sandbox_id})
+ logger.exception("wizard_worker_usage_measurement_failed", sandbox_id=sandbox_id)
return None
sandbox.config.cpu_cores = SANDBOX_CPU_CORES
@@ -367,7 +366,7 @@ def _read_worker_usage(read_usage: Callable[[], int | None], sandbox_id: str) ->
try:
return read_usage()
except Exception:
- logger.exception("wizard_worker_usage_read_failed", extra={"sandbox_id": sandbox_id})
+ logger.exception("wizard_worker_usage_read_failed", sandbox_id=sandbox_id)
return None
@@ -384,7 +383,7 @@ def _raise_for_failure(
stage,
exit_code,
_failure_detail(stdout, stderr, sensitive_values),
- stderr_to_wizard_error_code(stderr),
+ stderr_to_wizard_error(stderr),
)
diff --git a/products/wizard/backend/logic/workers/wizard_error_output.py b/products/wizard/backend/logic/workers/wizard_error_output.py
index af56c8c9e043..4178e6557f30 100644
--- a/products/wizard/backend/logic/workers/wizard_error_output.py
+++ b/products/wizard/backend/logic/workers/wizard_error_output.py
@@ -1,10 +1,41 @@
+import re
import json
+from posthog.dataclasses import frozen
+
from products.wizard.backend.facade.validation import is_wizard_error_code
-from products.wizard.backend.logic.workers.config import WIZARD_ERROR_OUTPUT_PREFIX
+from products.wizard.backend.logic.workers.config import WIZARD_ERROR_MESSAGE_LENGTH, WIZARD_ERROR_OUTPUT_PREFIX
+
+_REDACTED = "[REDACTED]"
+
+# The Wizard copies raw exception text into this message, and the worker does not know the tokens the
+# sandbox holds (the gateway token is minted inside it), so credentials are redacted by shape.
+_CREDENTIAL_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
+ (re.compile(r"\bph[a-z]_[A-Za-z0-9_-]{8,}"), _REDACTED),
+ (re.compile(r"\b(?:gh[oprsu]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{16,})"), _REDACTED),
+ (re.compile(r"\bsk-[A-Za-z0-9_-]{16,}"), _REDACTED),
+ (re.compile(r"(?<=://)[^/\s@]+@"), f"{_REDACTED}@"),
+ (re.compile(r"(?i)\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]{8,}"), rf"\1 {_REDACTED}"),
+ (
+ re.compile(r"""(?i)((?:api[_-]?key|token|secret|password)["']?\s*[:=]\s*["']?)[^\s"',;&]+"""),
+ rf"\1{_REDACTED}",
+ ),
+)
+
+def _redact_credentials(text: str) -> str:
+ for pattern, replacement in _CREDENTIAL_PATTERNS:
+ text = pattern.sub(replacement, text)
+ return text
-def stderr_to_wizard_error_code(stderr: str) -> str | None:
+
+@frozen
+class WizardErrorLine:
+ code: str
+ message: str | None
+
+
+def stderr_to_wizard_error(stderr: str) -> WizardErrorLine | None:
for line in reversed(stderr.splitlines()):
value = line.strip()
if not value.startswith(WIZARD_ERROR_OUTPUT_PREFIX):
@@ -22,6 +53,14 @@ def stderr_to_wizard_error_code(stderr: str) -> str | None:
if not isinstance(code, str) or not is_wizard_error_code(code):
return None
- return code
+ message = payload.get("message")
+ if not isinstance(message, str) or not message:
+ return WizardErrorLine(code=code, message=None)
+
+ # A stderr line can hold 64KB, so redaction runs on a bounded window to keep its cost fixed. The
+ # window is twice the kept length, so a credential that crosses the final cut is still redacted
+ # whole before the cut drops its tail.
+ window = message[: 2 * WIZARD_ERROR_MESSAGE_LENGTH]
+ return WizardErrorLine(code=code, message=_redact_credentials(window)[:WIZARD_ERROR_MESSAGE_LENGTH])
return None
diff --git a/products/wizard/backend/observability/service.py b/products/wizard/backend/observability/service.py
index fe632cceb79c..bd7b8c1740be 100644
--- a/products/wizard/backend/observability/service.py
+++ b/products/wizard/backend/observability/service.py
@@ -5,14 +5,17 @@ from uuid import UUID
from django.db import DatabaseError
+import structlog
+
from products.wizard.backend.facade.contracts import WizardRunArtifactDTO, WizardRunDTO, WizardRunPullRequestArtifactDTO
from products.wizard.backend.facade.enums import WizardRunStatus
from products.wizard.backend.logic.workers.contracts import WizardWorkerTelemetry
+from products.wizard.backend.logic.workers.wizard_error_output import WizardErrorLine
from products.wizard.backend.observability import events, metrics
from products.wizard.backend.observability.contracts import WizardRunDispatchOutcome, WizardWorkerCleanupOutcome
from products.wizard.backend.observability.worker_usage import worker_usage_observation
-logger = logging.getLogger(__name__)
+logger = structlog.get_logger(__name__)
class WizardObservability:
@@ -31,16 +34,16 @@ class WizardObservability:
try:
metric()
except ValueError:
- logger.exception(metric_error or f"{name}_metric_failed", extra=context)
+ logger.exception(metric_error or f"{name}_metric_failed", **context)
if event is not None:
try:
event()
except (DatabaseError, ValueError):
- logger.exception(event_error or f"{name}_event_failed", extra=context)
+ logger.exception(event_error or f"{name}_event_failed", **context)
if level is not None:
- logger.log(level, name, extra=context)
+ logger.log(level, name, **context)
def run_created(self, run: WizardRunDTO) -> None:
self._observe(
@@ -122,7 +125,7 @@ class WizardObservability:
try:
usage = worker_usage_observation(telemetry)
except ValueError:
- logger.exception("wizard_worker_usage_mapping_failed", extra=self._run_context(run))
+ logger.exception("wizard_worker_usage_mapping_failed", **self._run_context(run))
return
self._observe(
@@ -142,6 +145,18 @@ class WizardObservability:
event_error="wizard_worker_usage_event_failed",
)
+ def wizard_execution_failed(self, run: WizardRunDTO, exit_code: int, wizard_error: WizardErrorLine | None) -> None:
+ self._observe(
+ name="wizard_execution_failed",
+ context={
+ **self._run_context(run),
+ "exit_code": exit_code,
+ "wizard_error_code": wizard_error.code if wizard_error is not None else None,
+ "wizard_error_message": wizard_error.message if wizard_error is not None else None,
+ },
+ level=logging.WARNING,
+ )
+
def git_diff_omitted(self, run: WizardRunDTO, size_bytes: int) -> None:
self._observe(
name="wizard_git_diff_omitted",
diff --git a/products/wizard/backend/tasks/reconciliation.py b/products/wizard/backend/tasks/reconciliation.py
index a2d2c8e2561f..f836e7a6311c 100644
--- a/products/wizard/backend/tasks/reconciliation.py
+++ b/products/wizard/backend/tasks/reconciliation.py
@@ -1,6 +1,6 @@
-import logging
from time import monotonic
+import structlog
from celery import shared_task
from prometheus_client import Gauge
@@ -10,7 +10,7 @@ from posthog.tasks.utils import PushGatewayTask
from products.wizard.backend.logic.reconciliation import service as reconciliation
from products.wizard.backend.tasks.config import RECONCILE_WIZARD_RUNS_TASK
-logger = logging.getLogger(__name__)
+logger = structlog.get_logger(__name__)
@shared_task(
@@ -32,17 +32,15 @@ def reconcile_wizard_runs(self: PushGatewayTask) -> None:
logger.info(
"wizard_run_reconciliation_completed",
- extra={
- "duration_seconds": round(monotonic() - started_at, 3),
- "operations": {
- operation: {
- "scanned": result.scanned,
- "reconciled": result.reconciled,
- "failed": result.failed,
- "batch_limit_reached": result.batch_limit_reached,
- }
- for operation, result in results.items()
- },
+ duration_seconds=round(monotonic() - started_at, 3),
+ operations={
+ operation: {
+ "scanned": result.scanned,
+ "reconciled": result.reconciled,
+ "failed": result.failed,
+ "batch_limit_reached": result.batch_limit_reached,
+ }
+ for operation, result in results.items()
},
)
diff --git a/products/wizard/backend/tasks/tasks.py b/products/wizard/backend/tasks/tasks.py
index 57967d958627..f90cec8615bc 100644
--- a/products/wizard/backend/tasks/tasks.py
+++ b/products/wizard/backend/tasks/tasks.py
@@ -1,8 +1,8 @@
-import logging
from typing import Any
from django.db import DatabaseError
+import structlog
from celery import shared_task
from celery.app.task import Task
@@ -15,7 +15,7 @@ from products.event_definitions.backend.logic.placeholder import (
from products.wizard.backend.facade.enums import WizardSessionRunPhase
from products.wizard.backend.models import WizardSession
-logger = logging.getLogger(__name__)
+logger = structlog.get_logger(__name__)
MAX_PLANNED_EVENT_DEFINITIONS = 50
MAX_PLANNED_EVENT_CANDIDATES = 500
@@ -74,6 +74,8 @@ def sync_wizard_event_definitions(self: Task, team_id: int, session_id: str) ->
countdown = min(2**self.request.retries, 60)
logger.warning(
"Retrying wizard event definition sync",
- extra={"team_id": team_id, "session_id": session_id, "retry": self.request.retries},
+ team_id=team_id,
+ session_id=session_id,
+ retry=self.request.retries,
)
raise self.retry(exc=error, countdown=countdown)
diff --git a/products/wizard/backend/temporal/activities/execution.py b/products/wizard/backend/temporal/activities/execution.py
index a4568d7c59ad..2917578debd6 100644
--- a/products/wizard/backend/temporal/activities/execution.py
+++ b/products/wizard/backend/temporal/activities/execution.py
@@ -7,6 +7,7 @@ from products.wizard.backend.facade import api as wizard_facade
from products.wizard.backend.facade.enums import WizardRunStage
from products.wizard.backend.logic.workers import service as cloud_worker
from products.wizard.backend.logic.workers.service import WizardExecutionRequest
+from products.wizard.backend.observability.service import wizard_observability
from products.wizard.backend.observability.tracing import annotate_run_span
from products.wizard.backend.temporal.activities.errors import (
WIZARD_WORKER_EXECUTION_ERROR_TYPE,
@@ -41,8 +42,11 @@ def execute_wizard(input: PreparedGitRepositoryWorkspace) -> None:
non_retryable=True,
) from error
except cloud_worker.WizardWorkerExecutionError as error:
+ # sanitize_activity_errors drops this error's message before Temporal records it, so this log
+ # line is the only place the wizard's own failure message survives.
+ wizard_observability.wizard_execution_failed(run, error.exit_code, error.wizard_error)
raise ApplicationError(
str(error),
- type=error.wizard_error_code or WIZARD_WORKER_EXECUTION_ERROR_TYPE,
+ type=error.wizard_error.code if error.wizard_error is not None else WIZARD_WORKER_EXECUTION_ERROR_TYPE,
non_retryable=True,
) from error
diff --git a/products/wizard/backend/tests/runs/test_cloud_activities.py b/products/wizard/backend/tests/runs/test_cloud_activities.py
index 058f6a8be599..a9ae54e4a5ca 100644
--- a/products/wizard/backend/tests/runs/test_cloud_activities.py
+++ b/products/wizard/backend/tests/runs/test_cloud_activities.py
@@ -15,6 +15,7 @@ from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from opentelemetry.trace import StatusCode
+from structlog.testing import capture_logs
from temporalio.contrib.opentelemetry import OpenTelemetryInterceptor
from temporalio.exceptions import ApplicationError, ApplicationErrorCategory, CancelledError
from temporalio.testing import ActivityEnvironment
@@ -46,6 +47,7 @@ from products.wizard.backend.logic.workers.service import (
WizardWorkerResult,
WizardWorkerTimeoutError,
)
+from products.wizard.backend.logic.workers.wizard_error_output import WizardErrorLine
from products.wizard.backend.temporal.activities.errors import (
WIZARD_REPOSITORY_ACCESS_ERROR_TYPE,
WIZARD_WORKER_EXECUTION_ERROR_TYPE,
@@ -298,7 +300,11 @@ def test_clone_repository_keeps_transient_clone_failure_retryable(team, user) ->
True,
),
(
- WizardWorkerExecutionError("execution", 1, wizard_error_code="PHW_DETECT_NO_POSTHOG_SDK"),
+ WizardWorkerExecutionError(
+ "execution",
+ 1,
+ wizard_error=WizardErrorLine(code="PHW_DETECT_NO_POSTHOG_SDK", message="Missing SDK"),
+ ),
"PHW_DETECT_NO_POSTHOG_SDK",
True,
),
@@ -339,6 +345,7 @@ def test_execute_wizard_maps_worker_error(worker_error: Exception, error_type: s
side_effect=worker_error,
) as execute,
pytest.raises(ApplicationError) as error,
+ capture_logs() as logs,
):
get_run.return_value.program.command = ()
get_run.return_value.program.wizard_version = "2.60.0"
@@ -361,6 +368,16 @@ def test_execute_wizard_maps_worker_error(worker_error: Exception, error_type: s
assert error.value.__cause__ is None
assert error.value.__context__ is None
assert not error.value.details
+ failure_logs = [log for log in logs if log["event"] == "wizard_execution_failed"]
+ if isinstance(worker_error, WizardWorkerExecutionError):
+ assert len(failure_logs) == 1
+ expected_error = worker_error.wizard_error
+ assert failure_logs[0]["log_level"] == "warning"
+ assert failure_logs[0]["wizard_error_code"] == (expected_error.code if expected_error else None)
+ assert failure_logs[0]["wizard_error_message"] == (expected_error.message if expected_error else None)
+ else:
+ assert not failure_logs
+ assert "fake-wizard-api-key-for-test" not in str(logs)
if isinstance(worker_error, ApplicationError):
assert error.value.next_retry_delay == worker_error.next_retry_delay
assert error.value.category == worker_error.category
diff --git a/products/wizard/backend/tests/runs/test_cloud_worker.py b/products/wizard/backend/tests/runs/test_cloud_worker.py
index cbe1f05c9108..c73f7d5ad45a 100644
--- a/products/wizard/backend/tests/runs/test_cloud_worker.py
+++ b/products/wizard/backend/tests/runs/test_cloud_worker.py
@@ -48,6 +48,7 @@ from products.wizard.backend.logic.workers.service import (
prepare_local_wizard,
provision_wizard_worker,
)
+from products.wizard.backend.logic.workers.wizard_error_output import WizardErrorLine
from products.wizard.backend.observability.tracing import wizard_span
@@ -452,7 +453,7 @@ def test_execute_wizard_surfaces_wizard_error_code(get_sandbox_class: MagicMock)
with pytest.raises(WizardWorkerExecutionError) as error:
execute_wizard(request)
- assert error.value.wizard_error_code == "PHW_DETECT_NO_POSTHOG_SDK"
+ assert error.value.wizard_error == WizardErrorLine(code="PHW_DETECT_NO_POSTHOG_SDK", message="Missing SDK")
@patch("products.wizard.backend.logic.workers.service.stage_publishable_changes")
diff --git a/products/wizard/backend/tests/runs/test_observability.py b/products/wizard/backend/tests/runs/test_observability.py
index a7922a17a620..e66d4a3736a1 100644
--- a/products/wizard/backend/tests/runs/test_observability.py
+++ b/products/wizard/backend/tests/runs/test_observability.py
@@ -1,4 +1,3 @@
-import logging
from dataclasses import replace
from datetime import UTC, datetime, timedelta
from uuid import UUID, uuid4
@@ -12,6 +11,7 @@ from django.test import override_settings
from parameterized import parameterized
from prometheus_client import REGISTRY
+from structlog.testing import capture_logs
from products.wizard.backend.facade.contracts import (
CreateWizardRunInput,
@@ -71,7 +71,7 @@ class TestWizardObservability(TestCase):
patch.object(metrics.WIZARD_RUN_DISPATCH_ATTEMPTS_TOTAL, "labels") as counter,
patch.object(events, "ph_background_capture") as background_capture,
patch.object(events.User.objects, "filter") as users,
- self.assertLogs("products.wizard.backend.observability.service", level=logging.INFO) as logs,
+ capture_logs() as logs,
):
counter.return_value.inc.side_effect = ValueError("Metric unavailable") if metric_fails else None
users.return_value.values_list.return_value.first.return_value = "example-user"
@@ -82,11 +82,13 @@ class TestWizardObservability(TestCase):
counter.return_value.inc.assert_called_once_with()
background_capture.return_value.assert_called_once()
self.assertEqual(background_capture.return_value.call_args.kwargs["event"], "wizard run dispatch finished")
- messages = [record.getMessage() for record in logs.records]
+ messages = [log["event"] for log in logs]
self.assertEqual("wizard_run_dispatch_metric_failed" in messages, metric_fails)
self.assertEqual("wizard_run_dispatch_event_failed" in messages, event_fails)
self.assertEqual(messages[-1], "wizard_run_dispatch_finished")
- self.assertEqual(logs.records[-1].levelno, logging.INFO)
+ self.assertEqual(logs[-1]["log_level"], "info")
+ self.assertEqual(logs[-1]["run_id"], str(run.id))
+ self.assertEqual(logs[-1]["outcome"], WizardRunDispatchOutcome.SUCCEEDED.value)
def _cloud_run() -> WizardRunDTO:
diff --git a/products/wizard/backend/tests/runs/test_wizard_error_output.py b/products/wizard/backend/tests/runs/test_wizard_error_output.py
index f134a815e5b5..13a3c865a47c 100644
--- a/products/wizard/backend/tests/runs/test_wizard_error_output.py
+++ b/products/wizard/backend/tests/runs/test_wizard_error_output.py
@@ -1,6 +1,8 @@
+import json
+
import pytest
-from products.wizard.backend.logic.workers.wizard_error_output import stderr_to_wizard_error_code
+from products.wizard.backend.logic.workers.wizard_error_output import WizardErrorLine, stderr_to_wizard_error
@pytest.mark.parametrize(
@@ -8,18 +10,48 @@ from products.wizard.backend.logic.workers.wizard_error_output import stderr_to_
(
(
'Wizard output\nphw-error: {"code":"PHW_DETECT_NO_POSTHOG_SDK","message":"Missing SDK"}',
- "PHW_DETECT_NO_POSTHOG_SDK",
+ WizardErrorLine(code="PHW_DETECT_NO_POSTHOG_SDK", message="Missing SDK"),
),
(
'phw-error: {"code":"PHW_DETECT_NO_FRAMEWORK"}\nphw-error: {"code":"PHW_AGENT_NO_PROGRESS"}',
- "PHW_AGENT_NO_PROGRESS",
+ WizardErrorLine(code="PHW_AGENT_NO_PROGRESS", message=None),
+ ),
+ (
+ 'phw-error: {"code":"PHW_AGENT_API_ERROR","message":' + '"' + "x" * 600 + '"}',
+ WizardErrorLine(code="PHW_AGENT_API_ERROR", message="x" * 500),
+ ),
+ (
+ 'phw-error: {"code":"PHW_FUTURE_ERROR","message":{"nested":true}}',
+ WizardErrorLine(code="PHW_FUTURE_ERROR", message=None),
+ ),
+ *(
+ (
+ 'phw-error: {"code":"PHW_AGENT_API_ERROR","message":' + json.dumps(message) + "}",
+ WizardErrorLine(code="PHW_AGENT_API_ERROR", message=expected_message),
+ )
+ for message, expected_message in (
+ ("Gateway refused pha_FakeOAuthToken1234 for team 2", "Gateway refused [REDACTED] for team 2"),
+ ("clone failed: gho_FAKEFAKEFAKEFAKEFAKE1234", "clone failed: [REDACTED]"),
+ (
+ "fetch https://x-access-token:[email protected]/org/repo.git failed",
+ "fetch https://[REDACTED]@github.com/org/repo.git failed",
+ ),
+ ("401 with Authorization: Bearer fake.jwt.value-1234", "401 with Authorization: Bearer [REDACTED]"),
+ ('config {"apiKey": "fake-key-value"} rejected', 'config {"apiKey": "[REDACTED]"} rejected'),
+ ("refresh failed: access_token=fake-access-value", "refresh failed: access_token=[REDACTED]"),
+ ("x" * 490 + " pha_FakeOAuthToken1234", "x" * 490 + " [REDACTED"),
+ (
+ "OpenAI API error (400): 400 invalid request body",
+ "OpenAI API error (400): 400 invalid request body",
+ ),
+ ("Headless mode requires --api-key (phx_xxx)", "Headless mode requires --api-key (phx_xxx)"),
+ )
),
- ('phw-error: {"code":"PHW_FUTURE_ERROR"}', "PHW_FUTURE_ERROR"),
('phw-error: {"code":"detect_no_framework"}', None),
('phw-error: {"message":"Missing code"}', None),
("phw-error: not-json", None),
("ordinary stderr", None),
),
)
-def test_stderr_to_wizard_error_code(stderr: str, expected: str | None) -> None:
- assert stderr_to_wizard_error_code(stderr) == expected
+def test_stderr_to_wizard_error(stderr: str, expected: WizardErrorLine | None) -> None:
+ assert stderr_to_wizard_error(stderr) == expected
diff --git a/products/wizard/docs/reliability.md b/products/wizard/docs/reliability.md
index 16577ad64cd9..6709fcb6561b 100644
--- a/products/wizard/docs/reliability.md
+++ b/products/wizard/docs/reliability.md
@@ -21,4 +21,10 @@ The Wizard writes a final `phw-error:` record to standard error when a program f
The backend stores only the code. The UI owns the user message and recovery steps. If the record is absent or invalid, use the platform error for the failed stage.
+The execution activity also logs a `wizard_execution_failed` warning with the run ID, Wizard version, exit code, and the record's code and message.
+The worker redacts credential-shaped values in the message, such as PostHog and GitHub tokens, URL credentials, and `Authorization` values. It then cuts the message to 500 characters.
+It does not enter Temporal workflow history, run metadata, or analytics events.
+To investigate a failed cloud run, search the worker logs for its run ID.
+The CLI's own events, including its `$exception`, carry the same ID as `task_run_id`.
+
Collect latency data before you set latency SLOs. Review these targets after 30 to 60 days of production data.