Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Balance slow backend source slices by measured runtime
pulse-triage[bot]09a296f9ac73
diff --git a/.gitignore b/.gitignoreindex 90201eb37f..b4dbaa1064 100644--- a/.gitignore+++ b/.gitignore@@ -275,6 +275,8 @@ scripts/release_control/*!scripts/release_control/run_go_package_slice.py!scripts/release_control/run_go_package_slice_test.py!scripts/release_control/run_go_rest_shard.py+!scripts/release_control/monitoring-test-seconds.txt+!scripts/release_control/installtests-test-seconds.txt!scripts/release_control/repo_file_io.py!scripts/release_control/repo_file_io_test.py!scripts/release_control/record_rc_to_ga_rehearsal.pydiff --git a/scripts/release_control/installtests-test-seconds.txt b/scripts/release_control/installtests-test-seconds.txtnew file mode 100644index 0000000000..f2b21d1472--- /dev/null+++ b/scripts/release_control/installtests-test-seconds.txt@@ -0,0 +1,105 @@+# Scheduling estimates from complete source-listed race executions, not coverage.+# Natural Build and Test run 41127638037, jobs 124311583060/124311582969,+# source 367d46162b05f834a975375e23747def2405480b (30 October 2026).+# Root elapsed times can overlap; these are cut-point hints, not serial loads+# or a predicted wall-time saving. The selector preserves source order.+# Unlisted/new/renamed tests use the mean sub-0.5s cost; stale names are ignored.+# Kept with main controls so mechanical patch CI carries the same scheduler.+DEFAULT_WEIGHT 0.07+TestAgentBuildCacheDoesNotResurrectPulseAgentPackage 0.52+TestAutoUpdateAssetTransport 60.14+TestAutoUpdateBackupTransaction 9.33+TestAutoUpdateDownloadFailures 1.54+TestAutoUpdateEditionProbeHasHardDeadline 6.01+TestAutoUpdateNativeSystemdHarnessContracts 12.42+TestAutoUpdateReleaseMetadataAdmission 0.67+TestAutoUpdateRunningExecutableIdentity 7.33+TestAutoUpdateServiceDiscoveryAndInstallerHandoff 7.60+TestAutoUpdateVersionProbeHasHardDeadline 6.01+TestBackendAPIShardsKeepRequiredCheckExhaustive 0.99+TestBackfillReleaseAssetsRewritesIntegrityMetadataLocally 21.94+TestCIDocsMirrorPreflight 0.68+TestCIIssueMetadataUsesSourceOnlyTriageChecks 4.67+TestCITestCompilationPreservesChecks 1.16+TestDemoReachabilityStopsBeforeUnreadyOrFailedTailnetProbes 0.68+TestDemoTransactionConnectedRecoveryControls 1.39+TestFrontendCINativeUpdaterKeepsSecretFreeRequiredVerification 0.77+TestFrontendChangeClassificationPreservesStrictAudit 2.76+TestGuidanceRecipeRunnerBoundsEachPhaseAndStopsOwnedChildren 1.53+TestGuidanceRecipeRunnerRetainsIncompletePipeResult 2.01+TestGuidanceRecipeRunnerSeparatesPreparationFromCurl 0.51+TestInstallAutoUpdateAssetsKeepsWorkingHelperWhenStagingCopyFails 0.60+TestInstallAutoUpdateAssetsWritesUnitsAtomically 0.56+TestInstallPS1ParsesWithPowerShell 2.96+TestInstallPS1ServiceRemovalRuntime 3.57+TestInstallPS1UninstallRegistrationRuntime 57.00+TestInstallSHActionRunnerAtomicCancelUsesPrivateGoTransportAndRequires204 74.27+TestInstallSHActionRunnerConfigBindsKnownAgentIDImmediately 1.23+TestInstallSHActionRunnerConfigFailureLeavesOnlyCompleteFiles 0.54+TestInstallSHActionRunnerPersistsResolvedIdentityOutsideCollectorState 70.78+TestInstallSHActionRunnerPostCommitReadinessFailureRetainsReplacement 1.00+TestInstallSHActionRunnerRecoveredCollectorTrust 4.42+TestInstallSHActionRunnerSelfRevokeUsesPrivateCredential 1.60+TestInstallSHAgentIDRecoveryRejectsSymlinkFIFOAndOversizedState 108.58+TestInstallSHCombinesRecoveredProcessArgsAndEnvConnectionState 0.71+TestInstallSHDiscoversCustomStateDirFromGeneratedLaunchdPlist 0.68+TestInstallSHDiscoversCustomStateDirFromGeneratedSystemdUnit 1.80+TestInstallSHLegacyAgentIDRecoveryAccountsForEveryByte 2.86+TestInstallSHNASBootOwnersQuiesceBeforeRecoveryErasure 20.67+TestInstallSHOfflineInstallerRequiresValidServedSignature 1.11+TestInstallSHPreflightChecksAgentDownloadArtifact 3.41+TestInstallSHQNAPBootReplacementPreservesUnrelatedJobs 1.00+TestInstallSHRecoversLegacyDefaultTokenFileForSavedStateUpdate 0.83+TestInstallSHRecoversV5FreeBSDRCServiceState 1.00+TestInstallSHRecoversV5ProcessArgsForSavedStateUpdate 0.64+TestInstallSHRecoversV5ProcessArgsWithoutProcfs 1.11+TestInstallSHRegistrationRetryWindowOutlastsFirstReportCycle 0.60+TestInstallSHRejectsPartialRecoveredProcessConnectionState 0.65+TestInstallSHRetargetDoesNotRecoverLegacyServiceTrust 0.52+TestInstallSHRetargetPreservesIdentityWithoutOldEndpointTrust 0.75+TestInstallSHSavedInstallerTamperAndUntrustedStateFailClosed 0.70+TestInstallSHUninstallRetainsCredentialsWhenSavedURLIsMissing 1.17+TestInstallSHUnraidBootReplacementPreservesUnrelatedJobs 2.12+TestInstallSHUnraidGoEntryDoesNotBlockLaterBootLines 2.35+TestInstallSHUnraidGoEntryReplacesOnlyPulseLaunches 1.29+TestInstallSHUpdateModeMergesExplicitURLWithRunningV5ProcessState 0.91+TestInstallSHVerifyAgentServerRegistrationDetectsRejectedToken 100.36+TestManualUpdateChannelAdmission 3.06+TestNativeAgentRemovalSystemdRecipeRuntime 12.11+TestNativeInstallerFixtureToolsAreRealAndStepLocal 0.91+TestNativeInstallerWorkflowAdmitsServerUpgradeSource 0.66+TestNativeUnixContractRunnerPartitionsWithoutDroppingChecks 4.92+TestNpmAuditNativeScopeArgumentsPreserveFindings 0.76+TestProviderMSPStartupCommandReadbackSeparatesStderr 6.43+TestProviderMSPUpgradeFromBundleRepinsToTheBundleRelease 0.67+TestPulseAPIGuidanceTimedOutPOSTIsNotRepeated 32.04+TestPulseAgentStateDirLifecycleIntegration 20.13+TestPythonFixtureControlsBoundedParallelExecution 12.47+TestRenderedQNAPWatchdogRejectsASecondSupervisor 3.03+TestRenderedQNAPWatchdogRunsUnifiedStoredRuntimeBinary 3.06+TestRenderedSystemdUnitDoesNotPregrantCommandCapabilities 0.50+TestRootInstallExistingAutoUpdatesRequireAffirmativeChoice 1.88+TestRootInstallExistingMenuRequiresExactAutomaticIntent 2.43+TestRootInstallProxmoxFailurePreservesRecovery 1.40+TestRootInstallReleaseSelectionRejectsChartAndUnknownMetadata 5.15+TestRootInstallRemovalFilesystemFailures 3.56+TestRootInstallResetDoesNotStartAnIdleServerWhenTimerWasActive 0.71+TestRootInstallResetHandlesAbsentOptionalUnits 0.58+TestRootInstallScriptConfigBackupPublicationPreservesPriorSnapshots 1.32+TestRootInstallScriptSweepsOnlyStaleServerStaging 1.26+TestRootInstallScriptUpdateFlowsRepairHalfRemovedInstall 1.16+TestRootInstallServiceDiscoveryFailsClosedWithoutShortReads 16.48+TestRootInstallTemporaryInputsRemainPrivateAndAuthenticated 2.37+TestRootInstallTimerDiscoveryReachesConsentPreservingRefresh 15.87+TestRootlessQualificationCleanupFailureCannotEmitPassingPacket 0.73+TestSafeProfileExitTrapRestoresUncommittedTransaction 0.84+TestSafeProfileFailsClosedOnEffectiveSystemdOverrides 0.91+TestSafeProfileTransactionCommitAndFailureRollback 1.24+TestSafeProfileValidatesEveryEffectiveSystemdBoundary 1.32+TestSetupAutoUpdatesRendersExecutableExecCondition 0.71+TestTypedHelperIsolationDoesNotDependOnArchitectureOrPVE 10.19+TestUninstallSensorProxyIncompleteInventoryNeverStartsKeyCleanup 0.55+TestUninstallSensorProxyLocalKeyWritersPreserveRecovery 1.17+TestWindowsAgentLifecycleNativeExitChecksStopFollowingCommands 15.84+TestWindowsAgentLifecycleObservationRuntime 4.71+TestWindowsAgentLifecycleSavedRemovalInvokerRuntime 8.32diff --git a/scripts/release_control/monitoring-test-seconds.txt b/scripts/release_control/monitoring-test-seconds.txtnew file mode 100644index 0000000000..1f7f74752c--- /dev/null+++ b/scripts/release_control/monitoring-test-seconds.txt@@ -0,0 +1,197 @@+# Scheduling estimates from complete source-listed race executions, not coverage.+# Natural Build and Test run 41127638037, jobs 124311583060/124311582969,+# source 367d46162b05f834a975375e23747def2405480b (30 October 2026).+# Root elapsed times can overlap; these are cut-point hints, not serial loads+# or a predicted wall-time saving. The selector preserves source order.+# Unlisted/new/renamed tests use the mean sub-0.5s cost; stale names are ignored.+# Kept with main controls so mechanical patch CI carries the same scheduler.+DEFAULT_WEIGHT 0.04+TestAgentContainerReadingsAreNotShownForANodeSplitFromTheAgent 0.96+TestAgentFleetDiagnosticsDoesNotValidateSyntheticMockCredentials 1.63+TestAgentLookupAllocationIgnoresUnrelatedKubernetesMetadata 1.07+TestAgentLookupReportsKeepCompleteKubernetesInventory 0.86+TestAgentReportRefreshFoldsReportsWithinWindow 1.11+TestApplyDockerReportSkipsMetricsHistoryInMockMode 1.33+TestApplyHostReportSkipsMetricsAndSMARTWritesInMockMode 2.17+TestApplyHostReportStampsTheTokenOnlyOnHostsItHolds 1.76+TestBackupHistoryBackfillKeepsLiveTaskPayloadBounded 0.50+TestBackupHistoryUnreadTaskCannotCompleteRunningArchive 2.15+TestBoundMockMetricsHistoryGraphCapsLargeEstateGuestsAcrossTheEstate 1.10+TestBroadcastMetricsSnapshotConnectedContent 14.44+TestBroadcastProxmoxPlatformWireMatchesParent 0.79+TestCheckMockAlertsEvaluatesHostAgentsBeforeLinkedNodes 3.61+TestCheckMockAlertsEvaluatesPhysicalDisks 2.38+TestClusterFirstUsePasswordEndpointProjection 11.09+TestClusterNodeInventoryEndpointProjection 33.17+TestClusterNodeProbeEndpointProjection 11.06+TestClusterPinConflictsSurviveMonitorLifecycle 0.64+TestClusterRecoverySessionEndpointProjection 22.09+TestConfirmedSplitOfAManualLinkClearsTheAgentsContainerReadings 0.62+TestCurrentUnifiedStateViewCachedBetweenMockTicks 4.80+TestDockerPruneIgnoresTheRegistryMockModeLeft 4.24+TestEnteringMockModeClearsEveryRunningTenantMonitor 6.53+TestGetGuestMetricsForChartBatch_DownsamplesDenseSeededMockHistory 2.35+TestGetGuestMetricsForChartBatch_PrefersSeededMockHistoryInMockMode 1.81+TestGetGuestMetricsForChart_PrefersSeededMockHistoryInMockMode 2.07+TestGetGuestMetricsForChart_SupplementsPartialMockHistoryPerSeries 2.17+TestGetGuestMetricsForChart_SynthesizesMockHostTemperature 1.89+TestGetGuestMetricsForChart_UsesCanonicalMockSamplerInMockMode 2.17+TestGetPhysicalDiskTemperatureCharts_PrefersSeededMockHistoryInMockMode 3.03+TestGetStorageMetricsForChart_DownsamplesDenseSeededMockHistory 2.56+TestGetStorageMetricsForChart_PrefersSeededMockHistoryInMockMode 2.76+TestGetStorageMetricsForChart_UsesCanonicalMockSamplerInMockMode 4.25+TestGuestAgentOptionalReadOrderingContract 47.21+TestGuestDiskObservationContract 4.96+TestGuestHistoryObservationContinuity 2.33+TestGuestHistoryPreservesIndependentCurrentMemory 1.15+TestGuestInventoryFailurePreservesOwnedReadingsAndRecovers 0.60+TestGuestMeminfoIntegrityPollingContract 2.17+TestGuestMemoryLifecycleRestartPublishesNewSample 1.90+TestGuestMetadataCleanupSuppressionContract 1.76+TestGuestMetadataRateLimitRecoveryContract 1.06+TestGuestMetadataRejectionRecoveryContract 1.34+TestGuestMetadataRetirementPollingKeepsSuppression 0.93+TestGuestPollOwnershipHTTPWithdrawalPreservesSharedFence 1.00+TestGuestPollOwnershipRejectsObsoletePublication 1.16+TestGuestWindowsMeminfoPollingContract 10.85+TestGuestWindowsRestartPollingContract 47.08+TestLeavingMockModeClearsEveryRunningTenantMonitor 12.34+TestLeavingMockModeDiscardsAnInFlightBackupEvaluation 5.48+TestLeavingMockModeDiscardsAnInFlightConnectionCheck 2.19+TestLeavingMockModeDiscardsAnInFlightFrontendRefresh 2.23+TestLeavingMockModeRefusesTheRestOfAnInFlightPass 2.72+TestLeavingMockModeReleasesFixtureAgentNodeLinks 2.67+TestLeavingMockModeReleasesFixtureAgentNodeLinksOnEveryRunningMonitor 16.01+TestLeavingMockModeRemovesFixtureDockerHostAlerts 1.63+TestLeavingMockModeWaitsForAnInFlightEvaluation 2.22+TestLivePointReadsNilAndMockSemantics 1.74+TestLiveRefreshRejectsCollectedRecordsFromMockMode 4.73+TestManualNodeLinkKeepsTheSplitWhenItsIntentCannotPersist 0.59+TestManualNodeLinkReplacesAnOperatorSplit 0.75+TestManualNodeLinkSurvivesAnUnreadableSplitStore 0.58+TestMetadataMonitorProjectionParity 5.54+TestMockChartCacheInvalidatesAfterMockHistoryRefresh 2.59+TestMockChartHistorySharesTimestampsAcrossMetrics 4.29+TestMockDataVersionViewStillRebuildsAfterFixtureTicks 1.62+TestMockDockerContainerSeedCoversDrawerHistorySeries 6.94+TestMockDockerHostLeavingFixtureUsesRemovalLifecycle 2.55+TestMockDockerHostSeedCoversTheSameSeriesAsTheSyntheticGenerator 8.20+TestMockFixtureCurrentRevisionCannotReviveAnEndedEpoch 1.79+TestMockFixturePassFromBeforeAnEstateRebuildIsRejected 1.77+TestMockFixturePassPausedAcrossDisableAndReenableIsRejected 3.70+TestMockGuestChartHistoryCarriesMemoryUsedBeyondSeededWindow 1.57+TestMockGuestChartHistorySkipsMemoryUsedForNonProxmoxGuests 1.63+TestMockHostAgentLeavingFixtureUsesRemovalLifecycle 2.77+TestMockMetricsTargetFollowsAStructuralFixtureChange 1.45+TestMockMetricsTargetFollowsAnOperatorLink 1.23+TestMockMetricsTargetForUnknownIDDoesNotRebuildTheViewAfterFixtureTicks 2.42+TestMockMetricsTargetLookupsDoNotRebuildTheViewAfterFixtureTicks 2.36+TestMockMetricsTargetMatchesTheRegistryResolution 0.85+TestMockMetricsTargetRebuildsWhenTheLinkListMoves 0.70+TestMockModeDiscardsRealHostReports 2.17+TestMockModeDropsRealHostContinuity 4.12+TestMockModeSwitchLeavesAnUnjoinedMonitorsRuntimeToItsStart 4.88+TestMockNodeSeedCoversDrawerHistorySeries 7.50+TestMockOwnedUnifiedMetricSyncDefersToCanonicalSamplerInMockMode 1.59+TestMockStorageChartReadsTheEstateOnlyForAnUncachedChart 1.00+TestMockStructureConsumersDoNotRebuildTheViewAfterFixtureTicks 2.82+TestMockStructureReadStateFollowsAStructuralFixtureChange 1.21+TestMockStructureReadStateFollowsAnOperatorLink 1.16+TestMockUnifiedViewAppliesOperatorManualLinks 16.54+TestMockUnifiedViewFollowsALinkCreationTimeChange 2.15+TestMonitorBuildBroadcastFrontendStateUsesCanonicalMockUnifiedResources 4.14+TestMonitorDiscoveryEventsStayWithOwningOrganization 0.71+TestMonitorGetUnifiedReadStateOrSnapshotUsesCanonicalMockUnifiedResources 3.20+TestMonitorSeverityDelivery 14.68+TestMonitorStartJoinsConnectionAlertChecksBeforeReturning 10.67+TestMonitorStartJoinsMockAlertChecksBeforeReturning 0.81+TestMonitorStartedAfterAMockModeRoundTripDropsWhatItRaisedMeanwhile 1.38+TestMonitorStartedAfterAMockModeSwitchLeavesTheModeItWasBuiltIn 1.32+TestMonitor_GetConnectionStatuses_MockMode_Extra 1.37+TestMonitor_HandleAlertEscalated_QuietHoursSuppressesNotification 0.57+TestMonitor_HandleAlertResolved_SuppressesRecoveryWhenFiringNeverDelivered 1.55+TestMonitor_PollPVEBackupsAndSnapshots_DoesNotStarveSnapshotsAfterStorageTimeout 60.00+TestMonitor_PollStorageBackupsWithNodes_Coverage 2.00+TestMonitor_SetMockMode_Advanced_Extra 1.26+TestMonitor_Start_Extra 2.86+TestNewMonitorRoutesStartupCustomSensorWarningBeforeStart 0.90+TestNewRestoresSavedNotificationChoices 0.81+TestNew_MockModeClientInitializationRespectsKeepRealPollingSetting 2.44+TestOperatorIntentIdentityKeepsOneOverlayPerGeneration 1.48+TestOperatorSplitKeepsAgentSMARTOffTheProxmoxDisk 3.68+TestOperatorSplitStopsMonitoringTreatingNodeAndAgentAsLinked 1.16+TestOptionalDenialCadenceControls 3.65+TestPBSConflictingSnapshotIdentityRetainsUnreadScope 8.55+TestPBSDatastoreDiscoveryFailureQualifiesRetainedHistory 3.04+TestPBSDatastoreDiscoverySkippedScopeIsNotEmpty 0.76+TestPBSFailedRefreshDoesNotRenewCarriedPointEvidence 4.30+TestPBSMetricObservationGapNotificationReceipts 1.43+TestPBSMissingInventoryArrayRetainsUnreadScope 8.38+TestPBSNamespaceDiscoveryContinuityAndAccessWithdrawal 0.91+TestPBSNamespaceDiscoveryFailureRetainsRecoveryHistory 1.38+TestPBSNamespaceMixedFailuresDoNotResurrectDeniedScope 0.69+TestPBSNamespaceTransientGroupReadKeepsLiveInventory 4.77+TestPBSObservationHistoryAcrossMonitorReplacement 2.26+TestPBSOrdinaryPollCancellationDoesNotRenewUnpublishedCache 0.66+TestPBSOrdinaryPollNewestRemovalInvalidatesCappedCache 17.05+TestPBSOrdinaryPollRootNamespaceContainerPostureLifecycle 1.47+TestPBSOrdinaryPollRootNamespacePostureLifecycle 1.55+TestPBSPartialMetricsWebhookLifecycle 2.91+TestPBSRetryableReadLifecycle 4.39+TestPBSRetryableReadWithoutCacheDoesNotInventFiles 0.84+TestPMGOptionalReadFailureKeepsNormalSchedule 1.01+TestPVEBackupIncompletePollRetainsRecoveryHistory 0.95+TestPVEDisabledLocalStorageDoesNotLoseKnownHistory 3.48+TestPVELocalArchiveIdentityLifecycle 2.41+TestPVELocalArchiveSharedIdentityControls 0.70+TestPVELocalStorageReadScopeSurvivesPartialPoll 4.87+TestPVELocalStorageScopeKeepsAcceptedSharedAbsence 0.53+TestPVELocalStorageUnreadNodeDoesNotRetainRemovedSiblingStorage 0.52+TestPVEProtectionDisabledStorageIsNotAnEmptyRead 1.34+TestPVESnapshotHTTPInvalidResponseRetainsHistory 7.89+TestPVESnapshotIncompletePollRetainsRecoveryHistory 3.21+TestPVEStorageBudgetAcceptedEmptyDoesNotKeepRawRows 1.21+TestPVEStorageBudgetDoesNotInventAlertReadFailure 2.29+TestPVEStorageBudgetOwnerWithdrawalStillBlocksPublication 3.34+TestPVEStorageBudgetPublishesAcceptedListings 26.19+TestPVEStorageBudgetUnreadNodeKeepsUnscopedPBSConfirmation 1.37+TestPVEStorageHTTPAllUnreadCannotClaimEmpty 0.78+TestPVEStorageHTTPInvalidResponseRetainsHistory 10.05+TestPVEStorageRunningArchiveOrigin 1.97+TestPollPBSBackupsBoundsLargeTopologyAcrossPolls 2.50+TestPollVMsWithNodes_SkipsNativeGuestMetricWritesInMockMode 2.31+TestPrepareMockMetricsHistoryBoundsFullDemoEstateBeforeCaching 2.03+TestPreviousStateCarryIgnoresTheRegistryMockModeLeft 4.27+TestRecordGuestMetric_SkipsNativeWritesInMockMode 1.31+TestReloadPreservesLiveAgentCredentialInventory 2.24+TestReloadableMonitor_Lifecycle_Coverage 1.15+TestReplicationLifecycleDeadlineAndOrdinaryRecovery 10.10+TestReplicationLifecycleObsoleteScheduledCompletion 1.65+TestReportUsesTheLinkTheStateKeptWhenASplitLandsMidReport 0.80+TestResourceWiringModeBoundProvidersDoNotCrossTheSwitchHookGap 2.10+TestSLO_GetGuestMetricsForChartBatch_DoesNotStitchSparseStoreTailOntoCoveredInMemorySeries 2.83+TestSeedMockMetricsHistory_SeedsTrueNASMetricsStore 0.75+TestSeedMockMetricsHistory_SeedsVMwareMetricsStore 1.20+TestSeedMockMetricsHistory_UsesCanonicalMockFixtureGraphForLegacyAndProviderFixtures 1.42+TestSetMockModeAlignsAMonitorWhoseModeTheFlagAlreadyHolds 1.78+TestSnapshotCollectionFailureIsNotRemoval 0.75+TestSnapshotPollWorkersCancelQueuedReadsWithoutLosingAcceptedListings 0.65+TestSplitAgentSensorsDoNotReachTheNodeThroughItsName 0.58+TestStartMockMetricsSampler_PrewarmsDefaultWorkloadChartCaches 0.76+TestStartMockMetricsSampler_SeedsCanonicalMockResourceHistory 2.34+TestStartMockMetricsSampler_SeedsMetricsStoreWhenOptedIn 0.55+TestStorageInvestigationHistoryKeepsRequestedWindowAndStoreErrors 1.16+TestStorageMissingCapacityRestartNotificationReceipts 0.74+TestStoreValueProjectionPreservesChartSampling 2.89+TestSyncUnifiedAgentMetricsSkipsMockOwnedProviderHistoryWhenMockEnabled 2.38+TestSyncUnifiedAppContainerMetricsSkipsMockOwnedTrueNASHistoryWhenMockEnabled 1.99+TestSyncUnifiedStorageAndDiskMetricsSkipMockOwnedTrueNASHistoryWhenMockEnabled 2.03+TestTenantDeletionRetainsBlockedAlertPassThenRemovesCompletedRuntime 0.68+TestTenantDeletionRetainsBlockedGuestWriterOwnership 2.00+TestTrueNASPollerHonorsConfiguredPollInterval 1.02+TestVMFilesystemAliasUsagePollingAndHistory 4.84+TestVMFilesystemIncompletePollingPreservesHistoryAndBreach 8.30+TestVMFilesystemWindowsVolumePollingHistory 2.33+TestVzdumpCachedGuestReadFailureDoesNotRenewRunningIntent 0.93+TestVzdumpJobLogSchedulingPersistentReadsDoNotStarveHistory 0.50+TestVzdumpTaskLogMalformedReadRetainsAcceptedEvidence 3.84diff --git a/scripts/release_control/run_go_package_slice.py b/scripts/release_control/run_go_package_slice.pyindex fd0d3a54c7..df0181cd05 100644--- a/scripts/release_control/run_go_package_slice.py+++ b/scripts/release_control/run_go_package_slice.py@@ -2,15 +2,17 @@"""Run one exhaustive, order-preserving slice of a slow backend test package.The two fixed packages have outgrown whole-package CI balancing. Source supplies-the test inventory; no timing file or maintained test-name allowlist defines-coverage. The existing JSON recorder checks every selected root's completion,-including explicit skips, and preserves failures and unfinished output.+the test inventory; measured weights only move the contiguous boundary. Reuse+the API selector without changing source order, worker counts or test deadlines.+The existing JSON recorder checks every selected root's completion, including+explicit skips, and preserves failures and unfinished output."""from __future__ import annotationsimport argparseimport json+import osfrom pathlib import Pathimport reimport subprocess@@ -21,6 +23,11 @@ ROOT = Path(__file__).resolve().parents[2]MODULE = "github.com/rcourtman/pulse-go-rewrite/"PACKAGES = ("internal/monitoring", "scripts/installtests")RECORDER = ROOT / ".github/scripts/record-internal-api-test-seconds.py"+SELECTOR = ROOT / ".github/scripts/select-internal-api-shard.sh"+WEIGHTS = {+ package: ROOT / f"scripts/release_control/{package.rsplit('/', 1)[-1]}-test-seconds.txt"+ for package in PACKAGES+}def source_tests(output: str) -> list[str]:@@ -31,10 +38,25 @@ def source_tests(output: str) -> list[str]:return names-def select(names: list[str], index: int) -> list[str]:+def select(names: list[str], index: int, weights: Path | None = None) -> list[str]:if index not in (0, 1) or len(names) < 2:raise ValueError("two nonempty slices and an index of 0 or 1 are required")- return names[len(names) * index // 2:len(names) * (index + 1) // 2]+ if weights is None:+ return names[len(names) * index // 2:len(names) * (index + 1) // 2]+ # The same selector already keeps API tests in contiguous source order.+ # Ignore ambient overrides: both workers must use the committed weights+ # and the same default for new, renamed or unmeasured source tests.+ env = {key: value for key, value in os.environ.items() if key != "DEFAULT_WEIGHT"}+ result = subprocess.run(["bash", str(SELECTOR), str(weights), "2", str(index)],+ input="\n".join(names) + "\n", capture_output=True,+ text=True, env=env, check=True)+ selected = result.stdout.splitlines()+ # Fail before test execution on a broken selector, never fall back to an+ # apparently successful partial selection. The source list is authority.+ if (not selected or len(selected) >= len(names) or+ selected != (names[:len(selected)] if index == 0 else names[-len(selected):])):+ raise ValueError("weighted selection is not a nonempty contiguous source slice")+ return selecteddef test_filter(names: list[str], selected: list[str]) -> list[str]:@@ -65,10 +87,14 @@ def main() -> int:return listing.returncodetry:names = source_tests(listing.stdout)- selected = select(names, args.index)+ weights = WEIGHTS[args.package]+ selected = select(names, args.index, weights)filters = test_filter(names, selected)- except ValueError as error:+ except (OSError, ValueError, subprocess.CalledProcessError) as error:print(f"Backend slice selection failed: {error}", file=sys.stderr)+ if isinstance(error, subprocess.CalledProcessError):+ sys.stdout.write(error.stdout or "")+ sys.stderr.write(error.stderr or "")return 2(args.output / "listed.tests").write_text("\n".join(names) + "\n")selection = args.output / "selected.tests"@@ -86,6 +112,7 @@ def main() -> int:test_exit = process.wait()receipt = {"package": MODULE + args.package, "index": args.index, "count": 2,"listed_roots": len(names), "selected_roots": len(selected),+ "weights": str(weights.relative_to(ROOT)),"test_exit": test_exit, "recorder_exit": recorded.returncode}(args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n")print(json.dumps(receipt), flush=True)diff --git a/scripts/release_control/run_go_package_slice_test.py b/scripts/release_control/run_go_package_slice_test.pyindex 2200d00c4a..e13f61161f 100644--- a/scripts/release_control/run_go_package_slice_test.py+++ b/scripts/release_control/run_go_package_slice_test.py@@ -9,6 +9,7 @@ import shutilimport subprocessimport tempfileimport unittest+from unittest.mock import patchROOT = Path(__file__).resolve().parents[2]@@ -26,6 +27,46 @@ def job(name):class SliceTest(unittest.TestCase):+ def test_measured_boundary_balances_estimates_without_changing_source_coverage(self):+ with tempfile.TemporaryDirectory() as temporary:+ weights = Path(temporary) / 'weights'+ names = [f'TestZ{i}' for i in range(7)]+ for costs in ([1]*7, [100, 1, 1, 1, 1, 1, 1], [1, 1, 1, 100, 1, 1, 1],+ [1, 1, 1, 1, 1, 1, 100], [3, 8, 2, 15, 4, 7, 11]):+ weights.write_text('DEFAULT_WEIGHT 1\nTestDeleted 99999\n' ++ '\n'.join(f'{n} {c}' for n, c in zip(names, costs)))+ with patch.dict(os.environ, {'DEFAULT_WEIGHT': '99999'}):+ pieces = [runner.select(names, i, weights) for i in (0, 1)]+ self.assertTrue(all(pieces))+ self.assertEqual(pieces[0] + pieces[1], names)+ cut = len(pieces[0])+ self.assertEqual(max(sum(costs[:cut]), sum(costs[cut:])),+ min(max(sum(costs[:j]), sum(costs[j:]))+ for j in range(1, len(names))))+ # Weights can be stale or empty. They cannot define coverage or+ # a different default in one worker's environment.+ weights.write_text('# no measured source tests\nDEFAULT_WEIGHT 1\nTestDeleted 100\n')+ self.assertEqual([runner.select(names, i, weights) for i in (0, 1)],+ [names[:4], names[4:]])++ def test_unavailable_invalid_or_noncontiguous_selection_cannot_admit_tests(self):+ with tempfile.TemporaryDirectory() as temporary:+ weights = Path(temporary) / 'weights'+ names = ['TestA', 'TestB', 'TestC']+ with self.assertRaises(subprocess.CalledProcessError):+ runner.select(names, 0, weights)+ for bad in ('TestA NaN', 'TestA 0', 'TestA -1', 'broken line extra'):+ weights.write_text(bad+'\n')+ with self.assertRaises(subprocess.CalledProcessError):+ runner.select(names, 0, weights)+ for index, selected in ((0, ''), (0, 'TestB\n'), (1, 'TestA\n'),+ (0, 'TestA\nTestA\n'), (1, 'TestB\nTestForeign\n'),+ (0, '\n'.join(names)+'\n')):+ with patch.object(runner.subprocess, 'run', return_value=+ subprocess.CompletedProcess([], 0, selected, '')):+ with self.assertRaises(ValueError):+ runner.select(names, index, weights)+def test_slices_are_contiguous_disjoint_exhaustive_and_keep_source_order(self):for count in (2, 3, 17, 10000):names = [f"TestZ{i}" for i in range(count)]@@ -86,7 +127,8 @@ if events and '-list' not in args:if '-list' not in args and '-json' not in args:print('ok ordinary packages 0.01s')sys.exit(int(os.environ.get('ORDINARY_EXIT','0')))-names = ['TestZ', 'TestA', 'FuzzB', 'ExampleC', 'TestLast']+names = json.loads(os.environ.get('SOURCE_NAMES',+ '["TestZ", "TestA", "FuzzB", "ExampleC", "TestLast"]'))if '-list' in args:if os.environ.get('LISTING_EXIT'):print('original listing failure', file=sys.stderr)@@ -143,7 +185,24 @@ sys.exit(int(os.environ.get('TEST_EXIT','0')))result, target = self.execute(MODE="skip")self.assertEqual(result.returncode, 0, result.stdout + result.stderr)sidecar = json.loads((target / "timings.txt.failures.json").read_text())- self.assertEqual(sidecar['skipped_top_level_tests']['names'], ['TestA', 'TestZ'])+ self.assertEqual(sidecar['skipped_top_level_tests']['names'], ['FuzzB', 'TestA', 'TestZ'])++ def test_actual_runner_uses_package_weights_not_their_names_as_an_allowlist(self):+ for package, weights in runner.WEIGHTS.items():+ heavy = max((line.split() for line in weights.read_text().splitlines()+ if line and not line.startswith(('#', 'DEFAULT_WEIGHT'))),+ key=lambda fields: float(fields[1]))[0]+ names = [heavy] + [f'TestNewUnmeasured{i}' for i in range(6)]+ pieces = []+ for index in (0, 1):+ result, target = self.execute(package, index,+ SOURCE_NAMES=json.dumps(names), DEFAULT_WEIGHT='99999')+ self.assertEqual(result.returncode, 0, result.stdout+result.stderr)+ pieces.append((target / 'selected.tests').read_text().splitlines())+ receipt = json.loads((target / 'receipt.json').read_text())+ self.assertEqual(receipt['listed_roots'], len(names))+ self.assertEqual(receipt['weights'], str(weights.relative_to(ROOT)))+ self.assertEqual(pieces, [names[:1], names[1:]])def test_actual_pipeline_cannot_hide_go_recorder_or_incomplete_coverage_failure(self):for options in ({"TEST_EXIT": "9"}, {"LISTING_EXIT": "7"},@@ -180,6 +239,9 @@ class WorkflowTest(unittest.TestCase):root = fixture.rootfor name in ('scripts/release_control/run_go_package_slice.py','scripts/release_control/run_go_rest_shard.py',+ 'scripts/release_control/monitoring-test-seconds.txt',+ 'scripts/release_control/installtests-test-seconds.txt',+ '.github/scripts/select-internal-api-shard.sh','.github/scripts/record-internal-api-test-seconds.py','.github/scripts/select-go-package-shard.py','.github/scripts/go-package-test-seconds.txt'):@@ -282,6 +344,8 @@ class WorkflowTest(unittest.TestCase):self.assertTrue(RUNNER.is_relative_to(ROOT / 'scripts/release_control'))self.assertTrue(SHARD.is_relative_to(ROOT / 'scripts/release_control'))self.assertTrue(Path(__file__).is_relative_to(ROOT / 'scripts/release_control'))+ self.assertTrue(all(path.is_relative_to(ROOT / 'scripts/release_control')+ for path in runner.WEIGHTS.values()))self.assertIn('python3 scripts/release_control/run_go_package_slice_test.py', job('backend'))