Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

ci(backend): reject stale skip allowances across shard reports (#14536)

Nicholas Tindle80c4e7af7b90
diff --git a/.github/scripts/backend-allowed-skips.json b/.github/scripts/backend-allowed-skips.json
index 43f6f4bdca..39c4cc232d 100644
--- a/.github/scripts/backend-allowed-skips.json
+++ b/.github/scripts/backend-allowed-skips.json
@@ -1,138 +1,145 @@
-[
- "backend.api.features.orgs.regression_test.TestPR11RealtimeTenancy.test_subscribe_to_session_validates_org_membership",
- "backend.api.features.orgs.regression_test.TestPR15MarketplaceOrg.test_slug_unique_per_org",
- "backend.api.features.orgs.regression_test.TestPR18Cutover.test_all_agent_graphs_have_organization_id",
- "backend.api.features.orgs.regression_test.TestPR18Cutover.test_all_executions_have_organization_id",
- "backend.api.features.orgs.regression_test.TestPR18Cutover.test_all_store_listings_have_owning_org_id",
- "backend.api.features.orgs.regression_test.TestPR18Cutover.test_get_execution_requires_org_membership",
- "backend.api.features.orgs.regression_test.TestPR18Cutover.test_get_graph_requires_org_membership",
- "backend.api.features.orgs.regression_test.TestPR18Cutover.test_list_api_keys_scoped_by_org_cutover",
- "backend.api.features.orgs.regression_test.TestPR18Cutover.test_list_executions_scoped_by_team",
- "backend.api.features.orgs.regression_test.TestPR18Cutover.test_list_graphs_excludes_other_org_graphs",
- "backend.api.features.orgs.regression_test.TestPR18Cutover.test_list_graphs_returns_only_org_graphs",
- "backend.api.features.orgs.regression_test.TestPR18Cutover.test_read_by_user_id_fallback_removed",
- "backend.blocks.airtable._api_test.test_create_and_update_field",
- "backend.blocks.airtable._api_test.test_create_update_table",
- "backend.blocks.airtable._api_test.test_invalid_field_type",
- "backend.blocks.airtable._api_test.test_record_management",
- "backend.blocks.airtable._api_test.test_webhook_management",
- "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_call_tool_list_api_endpoints",
- "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_call_tool_search",
- "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_initialize",
- "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_list_tools",
- "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_sse_response_handling",
- "backend.blocks.test.test_block.test_available_blocks[HumanInTheLoopBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[CountdownTimerBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[ExecuteCodeBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[GetWeatherInformationBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[GithubListStargazersBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[Slant3DSlicerBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistCloseTaskBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistCreateLabelBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistCreateProjectBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistCreateTaskBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteCommentBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteLabelBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteProjectBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteSectionBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteTaskBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetCommentBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetCommentsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetLabelBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetProjectBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetSectionBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetSharedLabelsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetTaskBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetTasksBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistListCollaboratorsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistListLabelsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistListSectionsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistRemoveSharedLabelsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistRenameSharedLabelsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistReopenTaskBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistUpdateCommentBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistUpdateLabelBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistUpdateProjectBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TodoistUpdateTaskBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterAddListMemberBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterBookmarkTweetBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterCreateListBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterDeleteListBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterDeleteTweetBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterFollowListBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterFollowUserBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetBlockedUsersBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetBookmarkedTweetsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetFollowersBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetFollowingBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetHomeTimelineBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetLikedTweetsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetLikingUsersBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetListBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetListMembersBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetListMembershipsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetListTweetsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetMutedUsersBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetOwnedListsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetPinnedListsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetQuoteTweetsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetRetweetersBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetSpaceBuyersBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetSpaceByIdBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetSpacesBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetSpaceTweetsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetTweetBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetTweetsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetUserBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetUserMentionsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetUsersBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetUserTweetsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterHideReplyBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterLikeTweetBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterMuteUserBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterPinListBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterPostTweetBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterRemoveBookmarkTweetBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterRemoveListMemberBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterRemoveRetweetBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterRetweetBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterSearchRecentTweetsBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterSearchSpacesBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnfollowListBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnfollowUserBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnhideReplyBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnlikeTweetBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnmuteUserBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnpinListBlock]",
- "backend.blocks.test.test_block.test_block_ids_valid[TwitterUpdateListBlock]",
- "backend.copilot.baseline.service_test.test_baseline_multi_turn",
- "backend.copilot.graphiti.client_test.TestHyphenInGroupIdRegression.test_build_fulltext_query_escapes_hyphens_in_group_id",
- "backend.copilot.sdk.cli_openrouter_compat_test.test_bare_cli_does_not_send_openrouter_incompatible_features",
- "backend.copilot.sdk.tool_adapter_test.TestBug1DuplicateExecution.test_old_code_duplicates_on_arg_mismatch",
- "backend.copilot.sdk.tool_adapter_test.TestBug2FIFODesync.test_old_code_fifo_desync_on_denial",
- "backend.copilot.sdk.tool_adapter_test.TestBug3CancelRace.test_old_code_cancel_arrives_too_late",
- "backend.copilot.service_test.test_sdk_resume_multi_turn",
- "backend.copilot.test_copilot_e2e.test_session_state_persistence",
- "backend.copilot.tools.agent_browser_integration_test.test_chromium_executable_env_is_set",
- "backend.copilot.tools.agent_browser_integration_test.test_close_session",
- "backend.copilot.tools.agent_browser_integration_test.test_concurrent_independent_sessions",
- "backend.copilot.tools.agent_browser_integration_test.test_fill_form_field",
- "backend.copilot.tools.agent_browser_integration_test.test_get_title_after_navigate",
- "backend.copilot.tools.agent_browser_integration_test.test_get_url_after_navigate",
- "backend.copilot.tools.agent_browser_integration_test.test_navigate_returns_success",
- "backend.copilot.tools.agent_browser_integration_test.test_screenshot_produces_valid_png",
- "backend.copilot.tools.agent_browser_integration_test.test_scroll_down",
- "backend.copilot.tools.agent_browser_integration_test.test_snapshot_returns_interactive_elements",
- "backend.copilot.tools.agent_browser_integration_test.test_tool_act_fill_and_click",
- "backend.copilot.tools.agent_browser_integration_test.test_tool_act_missing_action",
- "backend.copilot.tools.agent_browser_integration_test.test_tool_act_missing_target",
- "backend.copilot.tools.agent_browser_integration_test.test_tool_act_scroll",
- "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://[::1]/]",
- "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://0.0.0.0/]",
- "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://10.0.0.1/]",
- "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://127.0.0.1/]",
- "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://169.254.169.254/]",
- "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_missing_url",
- "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_returns_response",
- "test.test_data_creator.test_main_function_runs_without_errors"
-]
+{
+ "common": [
+ "backend.api.features.orgs.regression_test.TestPR11RealtimeTenancy.test_subscribe_to_session_validates_org_membership",
+ "backend.api.features.orgs.regression_test.TestPR15MarketplaceOrg.test_slug_unique_per_org",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_all_agent_graphs_have_organization_id",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_all_executions_have_organization_id",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_all_store_listings_have_owning_org_id",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_get_execution_requires_org_membership",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_get_graph_requires_org_membership",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_list_api_keys_scoped_by_org_cutover",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_list_executions_scoped_by_team",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_list_graphs_excludes_other_org_graphs",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_list_graphs_returns_only_org_graphs",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_read_by_user_id_fallback_removed",
+ "backend.blocks.airtable._api_test.test_create_and_update_field",
+ "backend.blocks.airtable._api_test.test_create_update_table",
+ "backend.blocks.airtable._api_test.test_invalid_field_type",
+ "backend.blocks.airtable._api_test.test_record_management",
+ "backend.blocks.airtable._api_test.test_webhook_management",
+ "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_call_tool_list_api_endpoints",
+ "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_call_tool_search",
+ "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_initialize",
+ "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_list_tools",
+ "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_sse_response_handling",
+ "backend.blocks.test.test_block.test_available_blocks[HumanInTheLoopBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[CountdownTimerBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[ExecuteCodeBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[GetWeatherInformationBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[GithubListStargazersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[Slant3DSlicerBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistCloseTaskBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistCreateLabelBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistCreateProjectBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistCreateTaskBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteCommentBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteLabelBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteProjectBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteSectionBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteTaskBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetCommentBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetCommentsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetLabelBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetProjectBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetSectionBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetSharedLabelsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetTaskBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetTasksBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistListCollaboratorsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistListLabelsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistListSectionsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistRemoveSharedLabelsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistRenameSharedLabelsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistReopenTaskBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistUpdateCommentBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistUpdateLabelBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistUpdateProjectBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistUpdateTaskBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterAddListMemberBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterBookmarkTweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterCreateListBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterDeleteListBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterDeleteTweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterFollowListBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterFollowUserBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetBlockedUsersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetBookmarkedTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetFollowersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetFollowingBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetHomeTimelineBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetLikedTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetLikingUsersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetListBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetListMembersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetListMembershipsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetListTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetMutedUsersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetOwnedListsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetPinnedListsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetQuoteTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetRetweetersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetSpaceBuyersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetSpaceByIdBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetSpacesBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetSpaceTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetTweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetUserBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetUserMentionsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetUsersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetUserTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterHideReplyBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterLikeTweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterMuteUserBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterPinListBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterPostTweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterRemoveBookmarkTweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterRemoveListMemberBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterRemoveRetweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterRetweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterSearchRecentTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterSearchSpacesBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnfollowListBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnfollowUserBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnhideReplyBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnlikeTweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnmuteUserBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnpinListBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterUpdateListBlock]",
+ "backend.copilot.baseline.service_test.test_baseline_multi_turn",
+ "backend.copilot.graphiti.client_test.TestHyphenInGroupIdRegression.test_build_fulltext_query_escapes_hyphens_in_group_id",
+ "backend.copilot.sdk.cli_openrouter_compat_test.test_bare_cli_does_not_send_openrouter_incompatible_features",
+ "backend.copilot.sdk.tool_adapter_test.TestBug1DuplicateExecution.test_old_code_duplicates_on_arg_mismatch",
+ "backend.copilot.sdk.tool_adapter_test.TestBug2FIFODesync.test_old_code_fifo_desync_on_denial",
+ "backend.copilot.sdk.tool_adapter_test.TestBug3CancelRace.test_old_code_cancel_arrives_too_late",
+ "backend.copilot.service_test.test_sdk_resume_multi_turn",
+ "backend.copilot.test_copilot_e2e.test_session_state_persistence",
+ "backend.copilot.tools.agent_browser_integration_test.test_chromium_executable_env_is_set",
+ "backend.copilot.tools.agent_browser_integration_test.test_close_session",
+ "backend.copilot.tools.agent_browser_integration_test.test_concurrent_independent_sessions",
+ "backend.copilot.tools.agent_browser_integration_test.test_fill_form_field",
+ "backend.copilot.tools.agent_browser_integration_test.test_get_title_after_navigate",
+ "backend.copilot.tools.agent_browser_integration_test.test_get_url_after_navigate",
+ "backend.copilot.tools.agent_browser_integration_test.test_navigate_returns_success",
+ "backend.copilot.tools.agent_browser_integration_test.test_screenshot_produces_valid_png",
+ "backend.copilot.tools.agent_browser_integration_test.test_scroll_down",
+ "backend.copilot.tools.agent_browser_integration_test.test_snapshot_returns_interactive_elements",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_act_fill_and_click",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_act_missing_action",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_act_missing_target",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_act_scroll",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://[::1]/]",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://0.0.0.0/]",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://10.0.0.1/]",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://127.0.0.1/]",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://169.254.169.254/]",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_missing_url",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_returns_response",
+ "test.test_data_creator.test_main_function_runs_without_errors"
+ ],
+ "python_versions": {
+ "3.11": [],
+ "3.12": [],
+ "3.13": []
+ }
+}
diff --git a/.github/scripts/detect_overlaps.py b/.github/scripts/detect_overlaps.py
index 20b7bbc514..ac89a4478c 100644
--- a/.github/scripts/detect_overlaps.py
+++ b/.github/scripts/detect_overlaps.py
@@ -350,7 +350,7 @@ def find_file_overlap_candidates(
current_files: list[str], other_prs: list[dict], max_age_days: int = 14
) -> list[tuple[dict, list[str]]]:
"""Find PRs that share files with the current PR."""
- from datetime import datetime, timezone, timedelta
+ from datetime import datetime, timedelta, timezone
current_files_set = set(f for f in current_files if not should_ignore_file(f))
candidates = []
@@ -1139,7 +1139,7 @@ def post_or_update_comment(pr_number: int, body: str):
"""
result = run_gh(["api", "graphql", "-f", f"query={mutation}"], check=False)
if result.returncode == 0:
- print(f"Updated existing overlap comment")
+ print("Updated existing overlap comment")
else:
# Fallback to posting new comment
print(
diff --git a/.github/scripts/test_validate_backend_skips.py b/.github/scripts/test_validate_backend_skips.py
new file mode 100644
index 0000000000..a02ea58b21
--- /dev/null
+++ b/.github/scripts/test_validate_backend_skips.py
@@ -0,0 +1,299 @@
+import fnmatch
+import json
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+from xml.etree import ElementTree
+
+import yaml
+
+from backend_test_shard import SHARDS
+from validate_backend_skips import main
+from validate_junit import load_skip_policy
+from validate_junit import main as validate_junit_main
+
+
+class AggregateSkipTests(unittest.TestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temporary.cleanup)
+ self.directory = Path(self.temporary.name)
+ self.reports = self.directory / "reports"
+ self.policy_file = self.directory / "policy.json"
+ self.policy = {
+ "common": ["backend.example.test_shared"],
+ "python_versions": {"3.11": [], "3.12": []},
+ }
+ for version in self.policy["python_versions"]:
+ for shard in (*SHARDS, "remainder"):
+ skips = ["test_shared"] if shard == "remainder" else []
+ self.write_report(version, shard, skips)
+
+ def write_report(self, version, shard, skips):
+ report = self.report_path(version, shard)
+ report.parent.mkdir(parents=True, exist_ok=True)
+ root = ElementTree.Element("testsuite", name=shard)
+ ElementTree.SubElement(
+ root, "testcase", classname="backend.example", name=f"{shard}_passes"
+ )
+ for name in skips:
+ case = ElementTree.SubElement(
+ root, "testcase", classname="backend.example", name=name
+ )
+ ElementTree.SubElement(case, "skipped")
+ ElementTree.ElementTree(root).write(report)
+ return report
+
+ def report_path(self, version, shard):
+ return (
+ self.reports
+ / f"backend-test-reports-py{version}-{shard}"
+ / f"junit-{shard}.xml"
+ )
+
+ def run_validator(self):
+ self.policy_file.write_text(json.dumps(self.policy))
+ with patch("sys.stderr") as stderr:
+ result = main(
+ [
+ "--allow-skips-from",
+ str(self.policy_file),
+ "--reports-dir",
+ str(self.reports),
+ ]
+ )
+ return result, "".join(call.args[0] for call in stderr.write.call_args_list)
+
+ def test_entry_used_only_by_a_different_shard_passes(self):
+ self.assertEqual(self.run_validator()[0], 0)
+
+ def test_entry_used_by_no_shard_fails_with_actionable_diagnostic(self):
+ self.policy["common"].append("backend.example.test_removed")
+ result, diagnostic = self.run_validator()
+ self.assertEqual(result, 1)
+ self.assertIn("Python 3.11", diagnostic)
+ self.assertIn("stale", diagnostic)
+ self.assertIn("backend.example.test_removed", diagnostic)
+ self.assertIn("policy.json", diagnostic)
+
+ def test_a_common_entry_cannot_be_hidden_by_another_python_version(self):
+ self.write_report("3.12", "remainder", [])
+ result, diagnostic = self.run_validator()
+ self.assertEqual(result, 1)
+ self.assertIn("Python 3.12", diagnostic)
+ self.assertIn("backend.example.test_shared", diagnostic)
+
+ def test_explicit_python_version_specific_skip_passes(self):
+ self.policy["python_versions"]["3.12"] = ["backend.example.test_version_only"]
+ self.write_report("3.12", "data", ["test_version_only"])
+ self.assertEqual(self.run_validator()[0], 0)
+
+ def test_same_skip_can_be_explicit_for_multiple_versions(self):
+ for version in self.policy["python_versions"]:
+ self.policy["python_versions"][version] = [
+ "backend.example.test_version_only"
+ ]
+ self.write_report(version, "data", ["test_version_only"])
+ self.assertEqual(self.run_validator()[0], 0)
+
+ def test_python_specific_allowance_cannot_permit_skip_in_another_version(self):
+ self.policy["python_versions"]["3.12"] = ["backend.example.test_version_only"]
+ self.write_report("3.11", "data", ["test_version_only"])
+ self.write_report("3.12", "data", ["test_version_only"])
+ result, diagnostic = self.run_validator()
+ self.assertEqual(result, 1)
+ self.assertIn("unapproved skipped test IDs", diagnostic)
+ self.assertIn("Python 3.11", diagnostic)
+
+ def test_missing_shard_report_fails_even_when_all_skips_are_present(self):
+ self.report_path("3.12", "copilot").unlink()
+ result, diagnostic = self.run_validator()
+ self.assertEqual(result, 1)
+ self.assertIn("Python 3.12, shard copilot", diagnostic)
+ self.assertIn("missing or empty", diagnostic)
+
+ def test_missing_entire_python_version_fails(self):
+ for shard in (*SHARDS, "remainder"):
+ report = self.report_path("3.12", shard)
+ report.unlink()
+ report.parent.rmdir()
+ result, diagnostic = self.run_validator()
+ self.assertEqual(result, 1)
+ self.assertIn("missing", diagnostic)
+ self.assertIn("py3.12", diagnostic)
+
+ def test_unknown_or_duplicate_shard_artifact_fails(self):
+ self.write_report("3.11", "data-copy", [])
+ result, diagnostic = self.run_validator()
+ self.assertEqual(result, 1)
+ self.assertIn("unexpected", diagnostic)
+ self.assertIn("data-copy", diagnostic)
+
+ def test_unconfigured_python_version_reports_fail(self):
+ self.write_report("3.13", "data", [])
+ result, diagnostic = self.run_validator()
+ self.assertEqual(result, 1)
+ self.assertIn("unexpected", diagnostic)
+ self.assertIn("py3.13", diagnostic)
+
+ def test_invalid_or_incomplete_reports_fail(self):
+ cases = (
+ ("", "missing or empty"),
+ ("<testsuite>", "no element found"),
+ ('<testsuite tests="0"/>', "no test cases"),
+ (
+ '<testsuite tests="2"><testcase name="only"/></testsuite>',
+ "count is 2",
+ ),
+ (
+ '<testsuite><testcase name="bad"><failure/></testcase></testsuite>',
+ "1 failures",
+ ),
+ )
+ for content, expected in cases:
+ with self.subTest(content=content):
+ self.report_path("3.11", "data").write_text(content)
+ result, diagnostic = self.run_validator()
+ self.assertEqual(result, 1)
+ self.assertIn(expected, diagnostic)
+
+ def test_new_skip_remains_rejected(self):
+ self.write_report("3.11", "data", ["test_unapproved"])
+ result, diagnostic = self.run_validator()
+ self.assertEqual(result, 1)
+ self.assertIn("unapproved skipped test IDs", diagnostic)
+
+ def test_aggregate_validation_requires_explicit_versions(self):
+ self.policy = ["backend.example.test_shared"]
+ result, diagnostic = self.run_validator()
+ self.assertEqual(result, 1)
+ self.assertIn("explicit python_versions", diagnostic)
+
+ def test_per_shard_validator_uses_the_same_version_specific_policy(self):
+ self.policy["python_versions"]["3.12"] = ["backend.example.test_version_only"]
+ report = self.write_report("3.12", "data", ["test_version_only"])
+ self.policy_file.write_text(json.dumps(self.policy))
+ arguments = ["--allow-skips-from", str(self.policy_file), str(report)]
+ self.assertEqual(
+ validate_junit_main(arguments + ["--python-version", "3.12"]), 0
+ )
+ with patch("sys.stderr"):
+ self.assertEqual(
+ validate_junit_main(arguments + ["--python-version", "3.11"]), 1
+ )
+ self.assertEqual(validate_junit_main(arguments), 1)
+ self.assertEqual(
+ validate_junit_main(arguments + ["--python-version", "3.13"]), 1
+ )
+
+ def test_invalid_policy_is_rejected_before_validation(self):
+ invalid = (
+ {},
+ {"common": [], "python_versions": {}},
+ {"common": [], "python_versions": {"3.11.1": []}},
+ {
+ "common": ["backend.test", "backend.test"],
+ "python_versions": {"3.11": []},
+ },
+ {
+ "common": [],
+ "python_versions": {"3.11": ["backend.test", "backend.test"]},
+ },
+ {"common": ["backend.test"], "python_versions": {"3.11": ["backend.test"]}},
+ {"common": [], "python_versions": {"3.11": [""]}},
+ {"common": [], "python_versions": {"3.11": []}, "typo": []},
+ )
+ for policy in invalid:
+ with self.subTest(policy=policy):
+ self.policy = policy
+ self.assertEqual(self.run_validator()[0], 1)
+
+
+class WorkflowSkipPolicyTests(unittest.TestCase):
+ def setUp(self):
+ self.github = Path(__file__).resolve().parents[1]
+ self.workflow = yaml.safe_load(
+ (self.github / "workflows/platform-backend-ci.yml").read_text()
+ )
+
+ def test_explicit_policy_versions_match_every_backend_python_matrix(self):
+ policy = load_skip_policy(self.github / "scripts/backend-allowed-skips.json")
+ for name in ("test", "type-check"):
+ versions = self.workflow["jobs"][name]["strategy"]["matrix"][
+ "python-version"
+ ]
+ self.assertCountEqual(versions, policy.python_versions)
+
+ def test_aggregate_job_uses_unmerged_artifacts_from_every_matrix_shard(self):
+ jobs = self.workflow["jobs"]
+ aggregate = jobs["validate-skip-policy"]
+ self.assertEqual(aggregate["needs"], "test")
+ self.assertEqual(aggregate["if"], "${{ always() }}")
+ self.assertEqual(aggregate["runs-on"], "ubuntu-latest")
+ download = next(
+ step
+ for step in aggregate["steps"]
+ if step.get("uses", "").startswith("actions/download-artifact@")
+ )["with"]
+ self.assertFalse(download.get("merge-multiple", False))
+ upload = next(
+ step
+ for step in jobs["test"]["steps"]
+ if step.get("uses", "").startswith("actions/upload-artifact@")
+ )["with"]["name"]
+ matrix = jobs["test"]["strategy"]["matrix"]
+ for version in matrix["python-version"]:
+ for shard in matrix["test-shard"]:
+ artifact = upload.replace(
+ "${{ matrix.python-version }}", version
+ ).replace("${{ matrix.test-shard }}", shard)
+ self.assertEqual(artifact, f"backend-test-reports-py{version}-{shard}")
+ self.assertTrue(fnmatch.fnmatchcase(artifact, download["pattern"]))
+
+ def test_aggregate_step_refuses_partial_reports_from_unsuccessful_jobs(self):
+ job = self.workflow["jobs"]["validate-skip-policy"]
+ validate = job["steps"][-1]
+ self.assertEqual(validate["if"], "${{ always() }}")
+ self.assertEqual(
+ validate["env"]["SHARD_JOB_RESULT"], "${{ needs.test.result }}"
+ )
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ python = root / "python3"
+ python.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > invocation\n')
+ python.chmod(0o755)
+ for result in ("failure", "cancelled", "skipped", "success"):
+ with self.subTest(result=result):
+ invocation = root / "invocation"
+ invocation.unlink(missing_ok=True)
+ run = subprocess.run(
+ ["bash", "-e", "-o", "pipefail", "-c", validate["run"]],
+ cwd=root,
+ env={
+ **os.environ,
+ "PATH": f"{root}:{os.environ['PATH']}",
+ "SHARD_JOB_RESULT": result,
+ },
+ capture_output=True,
+ text=True,
+ )
+ if result == "success":
+ self.assertEqual(run.returncode, 0, run.stderr)
+ arguments = invocation.read_text().splitlines()
+ self.assertEqual(
+ arguments[0], ".github/scripts/validate_backend_skips.py"
+ )
+ self.assertEqual(arguments[-1], "backend-shard-reports")
+ else:
+ self.assertNotEqual(run.returncode, 0)
+ self.assertIn("reports may be incomplete", run.stderr)
+ self.assertFalse(invocation.exists())
+
+ def test_new_helper_and_test_changes_trigger_backend_ci(self):
+ events = self.workflow["on"] if "on" in self.workflow else self.workflow[True]
+ for event in ("push", "pull_request"):
+ for name in ("validate_backend_skips.py", "test_validate_backend_skips.py"):
+ self.assertIn(f".github/scripts/{name}", events[event]["paths"])
diff --git a/.github/scripts/validate_backend_skips.py b/.github/scripts/validate_backend_skips.py
new file mode 100644
index 0000000000..112f79c5a6
--- /dev/null
+++ b/.github/scripts/validate_backend_skips.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+
+import argparse
+import sys
+from pathlib import Path
+from xml.etree import ElementTree
+
+from backend_test_shard import SHARDS
+from validate_junit import load_skip_policy, validate_path
+
+
+def validate_shard_reports(
+ directory: Path, version: str, allowed_skips: set[str]
+) -> set[str]:
+ observed = set()
+ for shard in (*SHARDS, "remainder"):
+ report = (
+ directory
+ / f"backend-test-reports-py{version}-{shard}"
+ / f"junit-{shard}.xml"
+ )
+ try:
+ validate_path(report, False, allowed_skips)
+ root = ElementTree.parse(report).getroot()
+ except (OSError, ElementTree.ParseError, ValueError) as exc:
+ raise ValueError(f"Python {version}, shard {shard}: {exc}") from exc
+ observed.update(
+ f"{case.get('classname', '')}.{case.get('name', '')}"
+ for case in root.iter("testcase")
+ if case.find("skipped") is not None
+ )
+ return observed
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--allow-skips-from", type=Path, required=True)
+ parser.add_argument("--reports-dir", type=Path, required=True)
+ args = parser.parse_args(argv)
+ try:
+ policy = load_skip_policy(args.allow_skips_from)
+ if policy.python_versions is None:
+ raise ValueError("aggregate validation requires explicit python_versions")
+ expected = {
+ f"backend-test-reports-py{version}-{shard}"
+ for version in policy.python_versions
+ for shard in (*SHARDS, "remainder")
+ }
+ actual = {
+ path.name for path in args.reports_dir.glob("backend-test-reports-py*")
+ }
+ if expected != actual:
+ raise ValueError(
+ f"incomplete shard artifacts; missing: {sorted(expected - actual)}; "
+ f"unexpected: {sorted(actual - expected)}"
+ )
+ problems = []
+ for version in policy.python_versions:
+ allowed = policy.for_python_version(version)
+ try:
+ observed = validate_shard_reports(args.reports_dir, version, allowed)
+ if stale := allowed - observed:
+ raise ValueError(
+ f"Python {version} has stale skip IDs in {args.allow_skips_from}: "
+ f"{', '.join(sorted(stale))}. Remove obsolete allowances or "
+ "configure legitimate Python-version-specific skips explicitly."
+ )
+ except ValueError as exc:
+ problems.append(str(exc))
+ continue
+ print(
+ f"Python {version}: all four shard reports match {len(allowed)} skip IDs"
+ )
+ if problems:
+ raise ValueError("\n".join(problems))
+ except (OSError, ValueError) as exc:
+ print(f"Invalid backend skip reports: {exc}", file=sys.stderr)
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.github/scripts/validate_junit.py b/.github/scripts/validate_junit.py
index 6971292161..efaf563b21 100644
--- a/.github/scripts/validate_junit.py
+++ b/.github/scripts/validate_junit.py
@@ -20,6 +20,65 @@ class JUnitSummary:
return self.tests - self.failures - self.errors - self.skipped
+@dataclass(frozen=True)
+class SkipPolicy:
+ common: frozenset[str]
+ python_versions: dict[str, frozenset[str]] | None = None
+
+ def for_python_version(self, version: str | None) -> set[str]:
+ if self.python_versions is None:
+ return set(self.common)
+ if version not in self.python_versions:
+ raise ValueError(
+ f"an explicitly configured Python version is required; got {version!r}"
+ )
+ return set(self.common | self.python_versions[version])
+
+
+def _skip_ids(entries: object) -> frozenset[str]:
+ if (
+ not isinstance(entries, list)
+ or any(
+ not isinstance(entry, str)
+ or "." not in entry
+ or not entry.split(".", 1)[0].strip()
+ or not entry.rsplit(".", 1)[-1].strip()
+ for entry in entries
+ )
+ or len(set(entries)) != len(entries)
+ ):
+ raise ValueError(
+ "skip allowlist must contain unique nonempty classname.name IDs"
+ )
+ return frozenset(entries)
+
+
+def load_skip_policy(path: Path) -> SkipPolicy:
+ entries = json.loads(path.read_text(encoding="utf-8"))
+ if isinstance(entries, list):
+ return SkipPolicy(_skip_ids(entries))
+ if not isinstance(entries, dict) or set(entries) != {"common", "python_versions"}:
+ raise ValueError("skip policy must contain common and python_versions")
+ common = _skip_ids(entries["common"])
+ versions = entries["python_versions"]
+ if not isinstance(versions, dict) or not versions:
+ raise ValueError(
+ "python_versions must explicitly configure every tested version"
+ )
+ version_skips = {}
+ for version, ids in versions.items():
+ parts = version.split(".")
+ if len(parts) != 2 or not all(part.isdigit() for part in parts):
+ raise ValueError(f"invalid Python version: {version!r}")
+ specific = _skip_ids(ids)
+ if duplicate := common & specific:
+ raise ValueError(
+ f"Python {version} repeats common skip IDs: {', '.join(sorted(duplicate))}"
+ )
+ version_skips[version] = specific
+ return SkipPolicy(common, version_skips)
+
+
def _declared_count(element: ElementTree.Element, key: str, label: str) -> int | None:
raw = element.get(key)
if raw is None:
@@ -154,6 +213,9 @@ def parse_args(argv: list[str]) -> argparse.Namespace:
type=Path,
help="reject skips not listed as exact classname.name IDs in this JSON file",
)
+ parser.add_argument(
+ "--python-version", help="Python version that produced the reports"
+ )
parser.add_argument("reports", nargs="+", type=Path)
return parser.parse_args(argv)
@@ -164,22 +226,9 @@ def main(argv: list[str] | None = None) -> int:
allowed_skips = None
if args.allow_skips_from:
try:
- entries = json.loads(args.allow_skips_from.read_text(encoding="utf-8"))
- if (
- not isinstance(entries, list)
- or any(
- not isinstance(entry, str)
- or "." not in entry
- or not entry.split(".", 1)[0].strip()
- or not entry.rsplit(".", 1)[-1].strip()
- for entry in entries
- )
- or len(set(entries)) != len(entries)
- ):
- raise ValueError(
- "skip allowlist must contain unique nonempty classname.name IDs"
- )
- allowed_skips = set(entries)
+ allowed_skips = load_skip_policy(args.allow_skips_from).for_python_version(
+ args.python_version
+ )
except (OSError, ValueError) as exc:
print(f"Invalid skip allowlist: {exc}", file=sys.stderr)
return 1
diff --git a/.github/workflows/platform-backend-ci.yml b/.github/workflows/platform-backend-ci.yml
index 734a30ea04..5aea17e9d6 100644
--- a/.github/workflows/platform-backend-ci.yml
+++ b/.github/workflows/platform-backend-ci.yml
@@ -9,6 +9,8 @@ on:
- ".github/scripts/docker-pull-with-retry.sh"
- ".github/scripts/validate_junit.py"
- ".github/scripts/test_validate_junit.py"
+ - ".github/scripts/validate_backend_skips.py"
+ - ".github/scripts/test_validate_backend_skips.py"
- ".github/scripts/backend_test_shard.py"
- ".github/scripts/test_backend_test_shard.py"
- ".github/scripts/backend-allowed-skips.json"
@@ -22,6 +24,8 @@ on:
- ".github/scripts/docker-pull-with-retry.sh"
- ".github/scripts/validate_junit.py"
- ".github/scripts/test_validate_junit.py"
+ - ".github/scripts/validate_backend_skips.py"
+ - ".github/scripts/test_validate_backend_skips.py"
- ".github/scripts/backend_test_shard.py"
- ".github/scripts/test_backend_test_shard.py"
- ".github/scripts/backend-allowed-skips.json"
@@ -571,6 +575,7 @@ jobs:
run: >-
python ../../.github/scripts/validate_junit.py --synthesize-invalid
--allow-skips-from ../../.github/scripts/backend-allowed-skips.json
+ --python-version "${{ matrix.python-version }}"
"junit-${TEST_SHARD}.xml"
- name: Validate coverage reports
@@ -620,3 +625,41 @@ jobs:
# the backend service, docker composes, and examples
RABBITMQ_DEFAULT_USER: "rabbitmq_user_default"
RABBITMQ_DEFAULT_PASS: "k0VMxyIJF9S35f3x2uaw5IWAl6Y536O7"
+
+ validate-skip-policy:
+ name: Validate backend skip policy
+ needs: test
+ if: ${{ always() }}
+ permissions:
+ contents: read
+ actions: read
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ defaults:
+ run:
+ working-directory: .
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v6
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+
+ - name: Download backend shard reports
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ pattern: backend-test-reports-py*
+ path: backend-shard-reports
+
+ - name: Validate complete skip policy for every Python version
+ if: ${{ always() }}
+ env:
+ SHARD_JOB_RESULT: ${{ needs.test.result }}
+ run: |
+ if [[ "$SHARD_JOB_RESULT" != "success" ]]; then
+ echo "Backend shard jobs did not all succeed; reports may be incomplete." >&2
+ exit 1
+ fi
+ python3 .github/scripts/validate_backend_skips.py \
+ --allow-skips-from .github/scripts/backend-allowed-skips.json \
+ --reports-dir backend-shard-reports