🙌 OpenHands: AI-Driven Development
Automate good first issue labeling in issue triage (#14498)
Engel Nystd04c1bb31c0d
diff --git a/.github/workflows/issue-duplicate-checker.yml b/.github/workflows/issue-duplicate-checker.ymlindex 2fcda8d72d..752391123a 100644--- a/.github/workflows/issue-duplicate-checker.yml+++ b/.github/workflows/issue-duplicate-checker.yml@@ -15,10 +15,11 @@ on:options:- smoke-clone- issue-check+ - good-first-issue-check- auto-closedefault: smoke-cloneissue_number:- description: Existing issue number to analyze when mode is issue-check+ description: Existing issue number to analyze for manual issue-check modesrequired: falsetype: numberclose_after_days:@@ -57,13 +58,19 @@ jobs:issue-duplicate-check:if: |- github.event_name == 'issues' ||+ (github.event_name == 'issues' && github.event.action == 'opened') ||(github.event_name == 'workflow_dispatch' && inputs.mode == 'issue-check' && inputs.issue_number != null)runs-on: ubuntu-latesttimeout-minutes: 35concurrency:group: issue-duplicate-check-${{ github.repository }}-${{ github.event.issue.number || inputs.issue_number }}cancel-in-progress: false+ outputs:+ should_comment: ${{ steps.parsed_result.outputs.should_comment }}+ is_duplicate: ${{ steps.parsed_result.outputs.is_duplicate }}+ auto_close_candidate: ${{ steps.parsed_result.outputs.auto_close_candidate }}+ confidence: ${{ steps.parsed_result.outputs.confidence }}+ classification: ${{ steps.parsed_result.outputs.classification }}steps:- name: Checkout repositoryuses: actions/checkout@v6@@ -430,6 +437,234 @@ jobs:await ensureCandidateLabelOnIssue();}+ issue-good-first-issue-check:+ if: |+ (+ github.event_name == 'issues' ||+ (github.event_name == 'workflow_dispatch' && inputs.mode == 'good-first-issue-check' && inputs.issue_number != null)+ ) &&+ (needs.issue-duplicate-check.result == 'success' || needs.issue-duplicate-check.result == 'skipped')+ needs:+ - issue-duplicate-check+ runs-on: ubuntu-latest+ timeout-minutes: 35+ concurrency:+ group: issue-good-first-issue-check-${{ github.repository }}-${{ github.event.issue.number || inputs.issue_number }}+ cancel-in-progress: false+ steps:+ - name: Checkout repository+ uses: actions/checkout@v6++ - name: Set up Python+ uses: actions/setup-python@v6+ with:+ python-version: '3.13'++ - name: Validate good first issue check inputs+ env:+ OPENHANDS_API_KEY: ${{ secrets.OPENHANDS_API_KEY }}+ ISSUE_NUMBER: ${{ github.event.issue.number || inputs.issue_number }}+ run: |+ if [ -z "$OPENHANDS_API_KEY" ]; then+ echo "Error: OPENHANDS_API_KEY secret is required"+ exit 1+ fi+ if [ -z "$ISSUE_NUMBER" ]; then+ echo "Error: ISSUE_NUMBER is required"+ exit 1+ fi++ - name: Determine whether good first issue auto-label check should run+ id: precheck+ uses: actions/github-script@v9+ env:+ ISSUE_NUMBER: ${{ github.event.issue.number || inputs.issue_number }}+ DUPLICATE_SHOULD_COMMENT: ${{ needs.issue-duplicate-check.outputs.should_comment }}+ DUPLICATE_CLASSIFICATION: ${{ needs.issue-duplicate-check.outputs.classification }}+ with:+ github-token: ${{ secrets.OPENHANDS_BOT_GITHUB_PAT_PUBLIC || github.token }}+ script: |+ const issueNumber = Number(process.env.ISSUE_NUMBER);+ const duplicateShouldComment = process.env.DUPLICATE_SHOULD_COMMENT === 'true';+ const duplicateClassification = process.env.DUPLICATE_CLASSIFICATION || '';+ const { data: issue } = await github.rest.issues.get({+ owner: context.repo.owner,+ repo: context.repo.repo,+ issue_number: issueNumber,+ });+ const labelNames = (issue.labels || []).map((label) => (+ typeof label === 'string' ? label : label.name+ )).filter(Boolean);++ let skipReason = '';+ if (issue.pull_request) {+ skipReason = `#${issueNumber} is a pull request, not an issue.`;+ } else if (issue.state !== 'open' || issue.locked) {+ skipReason = `#${issueNumber} is not an open, unlocked issue.`;+ } else if (labelNames.includes('good first issue')) {+ skipReason = 'Issue is already labeled good first issue.';+ } else if (labelNames.includes('duplicate-candidate')) {+ skipReason = 'Issue is marked duplicate-candidate, so good first issue auto-labeling is blocked.';+ } else if (+ duplicateShouldComment ||+ ['duplicate', 'overlapping-scope'].includes(duplicateClassification)+ ) {+ skipReason = `Duplicate check classified the issue as ${duplicateClassification || 'overlapping-scope'} and blocked good first issue auto-labeling.`;+ }++ core.setOutput('issue_url', issue.html_url || '');+ core.setOutput('labels_json', JSON.stringify(labelNames));+ core.setOutput('skip_reason', skipReason);+ core.setOutput('should_run', skipReason ? 'false' : 'true');++ - name: Summarize skipped good first issue check+ if: steps.precheck.outputs.should_run != 'true'+ run: |+ {+ echo "## Good first issue check"+ echo+ echo "- Issue: #${{ github.event.issue.number || inputs.issue_number }}"+ echo "- Skipped: ${{ steps.precheck.outputs.skip_reason }}"+ } >> "$GITHUB_STEP_SUMMARY"++ - name: Run OpenHands good first issue conversation+ if: steps.precheck.outputs.should_run == 'true'+ id: run_check+ env:+ OPENHANDS_API_KEY: ${{ secrets.OPENHANDS_API_KEY }}+ GITHUB_TOKEN: ${{ secrets.OPENHANDS_BOT_GITHUB_PAT_PUBLIC || github.token }}+ ISSUE_NUMBER: ${{ github.event.issue.number || inputs.issue_number }}+ OUTPUT_PATH: ${{ runner.temp }}/good-first-issue-check-result.json+ REPOSITORY: ${{ github.repository }}+ run: |+ python scripts/issue_good_first_issue_check_openhands.py \+ --repository "$REPOSITORY" \+ --issue-number "$ISSUE_NUMBER" \+ --output "$OUTPUT_PATH"+ test -f "$OUTPUT_PATH" || {+ echo "Error: Output file not created"+ exit 1+ }+ echo "result_path=$OUTPUT_PATH" >> "$GITHUB_OUTPUT"++ - name: Parse good first issue check result+ if: steps.precheck.outputs.should_run == 'true'+ id: parsed_result+ env:+ RESULT_PATH: ${{ steps.run_check.outputs.result_path }}+ run: |+ python - <<'PY'+ import json+ import os+ import sys+ from pathlib import Path++ try:+ result = json.loads(Path(os.environ['RESULT_PATH']).read_text())+ except (FileNotFoundError, json.JSONDecodeError) as exc:+ print(+ f"Error: Failed to read good first issue result: {exc}",+ file=sys.stderr,+ )+ raise SystemExit(1) from exc+ output_path = Path(os.environ['GITHUB_OUTPUT'])+ summary_path = Path(os.environ['GITHUB_STEP_SUMMARY'])++ def write_multiline(name: str, value: str) -> None:+ delimiter = f"EOF_{os.urandom(8).hex()}"+ with output_path.open('a', encoding='utf-8') as fh:+ fh.write(f"{name}<<{delimiter}\n{value}\n{delimiter}\n")++ with output_path.open('a', encoding='utf-8') as fh:+ fh.write(+ f"should_apply_label={'true' if result.get('should_apply_label') else 'false'}\n"+ )+ fh.write(f"confidence={result.get('confidence', '')}\n")+ fh.write(f"conversation_url={result.get('conversation_url', '')}\n")++ write_multiline('summary', str(result.get('summary', '')).strip())+ write_multiline(+ 'criteria_met_json',+ json.dumps(result.get('criteria_met', []), ensure_ascii=False),+ )+ write_multiline(+ 'disqualifiers_json',+ json.dumps(result.get('disqualifiers', []), ensure_ascii=False),+ )++ criteria_lines = [+ f"- {item}" for item in result.get('criteria_met', [])+ ]+ disqualifier_lines = [+ f"- {item}" for item in result.get('disqualifiers', [])+ ]++ summary_path.write_text(+ "\n".join(+ [+ "## Good first issue check result",+ "",+ f"- Repository: {result.get('repository')}",+ f"- Issue: #{result.get('issue_number')}",+ f"- Should apply label: {result.get('should_apply_label')}",+ f"- Confidence: {result.get('confidence')}",+ f"- Conversation: {result.get('conversation_url')}",+ "",+ "### Summary",+ result.get('summary', ''),+ "",+ "### Criteria met",+ *(criteria_lines or ["- None"]),+ "",+ "### Disqualifiers",+ *(disqualifier_lines or ["- None"]),+ ]+ )+ + "\n",+ encoding='utf-8',+ )+ PY++ - name: Apply good first issue label+ if: |+ steps.precheck.outputs.should_run == 'true' &&+ steps.parsed_result.outputs.should_apply_label == 'true'+ uses: actions/github-script@v9+ env:+ ISSUE_NUMBER: ${{ github.event.issue.number || inputs.issue_number }}+ with:+ github-token: ${{ secrets.OPENHANDS_BOT_GITHUB_PAT_PUBLIC || github.token }}+ script: |+ const issueNumber = Number(process.env.ISSUE_NUMBER);+ const labelName = 'good first issue';+ const { data: issue } = await github.rest.issues.get({+ owner: context.repo.owner,+ repo: context.repo.repo,+ issue_number: issueNumber,+ });+ const labelNames = (issue.labels || []).map((label) => (+ typeof label === 'string' ? label : label.name+ )).filter(Boolean);+ if (labelNames.includes(labelName)) {+ core.info(`Issue #${issueNumber} already has the ${labelName} label.`);+ return;+ }+ await github.rest.issues.addLabels({+ owner: context.repo.owner,+ repo: context.repo.repo,+ issue_number: issueNumber,+ labels: [labelName],+ });++ - name: Summarize good first issue label action+ if: steps.precheck.outputs.should_run == 'true'+ run: |+ if [ "${{ steps.parsed_result.outputs.should_apply_label }}" = "true" ]; then+ echo "- Applied label: good first issue" >> "$GITHUB_STEP_SUMMARY"+ else+ echo "- Label not applied" >> "$GITHUB_STEP_SUMMARY"+ fi+auto-close-duplicates:if: |github.event_name == 'schedule' ||diff --git a/AGENTS.md b/AGENTS.mdindex ee2881a6d5..92e9bb8122 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -497,3 +497,9 @@ Called by `workspace.get_llm()` in the SDK to retrieve LLM config with the API k- `openhands/sdk/llm/llm.py`: `LLM.api_key` accepts `SecretSource` (including `LookupSecret`)- `openhands/workspace/cloud/workspace.py`: `get_llm()` and `get_secrets()` return LookupSecret-backed objects- Tests: `tests/sdk/llm/test_llm_secret_source_api_key.py`, `tests/workspace/test_cloud_workspace_sdk_settings.py`++### Issue Triage Automation++- `.github/workflows/issue-duplicate-checker.yml` now has a second job that auto-applies `good first issue` after the duplicate check completes.+- The duplicate check is used only as a veto/guardrail for `good first issue` automation: duplicate or overlapping-scope issues should not be auto-labeled.+- The OpenHands classifier logic for newcomer suitability lives in `scripts/issue_good_first_issue_check_openhands.py`, with focused unit coverage in `tests/unit/test_issue_good_first_issue_check_openhands.py`.diff --git a/ISSUE_TRIAGE.md b/ISSUE_TRIAGE.mdindex dee67b10cd..250414ceae 100644--- a/ISSUE_TRIAGE.md+++ b/ISSUE_TRIAGE.md@@ -10,7 +10,15 @@ These are the procedures and guidelines on how issues are triaged in this repo b* **Critical**: Affecting all users or potential security issues.## Difficulty-* Issues good for newcomers may be tagged with **good first issue**.+* Issues good for newcomers may be tagged with **good first issue** by maintainers or by the automated triage workflow when the issue clearly meets the criteria below.+* The `welcome-good-first-issue` workflow only posts a welcome comment after the label is present; it does not decide whether the label should be applied.+* The automated triage workflow should be conservative and should not auto-apply **good first issue** to issues that look like duplicates or overlapping-scope reports.+* Use **good first issue** only when all of the following are true:+ * The work is narrow in scope and should be solved as a single issue rather than a multi-step project.+ * The bug, request, or expected outcome is already clear enough that a contributor should not need major discovery work before starting.+ * The likely fix stays within a well-bounded area of the repo and does not require deep architecture knowledge, cross-repo coordination, enterprise-only context, migrations, or infrastructure work. Exclude enterprise/ directory from these.+ * The validation path is straightforward: the PR author should be able to give clear **How to Test** steps, and if the change is user-facing, screenshots or video evidence should be practical to provide.+* Avoid **good first issue** for broad design discussions, umbrella tracking issues, items missing reproduction steps or acceptance criteria, and changes that are likely to require significant maintainer guidance.## Not Enough Information* User is asked to provide more information (logs, how to reproduce, etc.) when the issue is not clear.diff --git a/scripts/issue_good_first_issue_check_openhands.py b/scripts/issue_good_first_issue_check_openhands.pynew file mode 100644index 0000000000..28e71488ca--- /dev/null+++ b/scripts/issue_good_first_issue_check_openhands.py@@ -0,0 +1,568 @@+#!/usr/bin/env python3+from __future__ import annotations++import argparse+import json+import os+import re+import sys+import time+import urllib.error+import urllib.parse+import urllib.request+from pathlib import Path+from typing import Any++OPENHANDS_BASE_URL = os.environ.get('OPENHANDS_BASE_URL', 'https://app.all-hands.dev')+REPOSITORY_PATTERN = re.compile(r'^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$')+GITHUB_API_BASE_URL = os.environ.get('GITHUB_API_BASE_URL', 'https://api.github.com')+FAILED_EXECUTION_STATUSES = {+ 'error',+ 'errored',+ 'failed',+ 'stopped',+}+SUCCESSFUL_TERMINAL_EXECUTION_STATUSES = {+ 'completed',+ 'finished',+}+TERMINAL_EXECUTION_STATUSES = (+ FAILED_EXECUTION_STATUSES | SUCCESSFUL_TERMINAL_EXECUTION_STATUSES+)+EVENT_SEARCH_LIMIT = 1000+EVENT_SEARCH_LIMIT_HIT_MESSAGE = (+ f'Event search returned at least {EVENT_SEARCH_LIMIT} events; results may be '+ 'incomplete'+)+++def parse_args() -> argparse.Namespace:+ parser = argparse.ArgumentParser(+ description=(+ 'Start an OpenHands Cloud conversation that decides whether a '+ 'GitHub issue should be labeled good first issue.'+ )+ )+ parser.add_argument(+ '--repository', required=True, help='Repository in owner/repo form'+ )+ parser.add_argument(+ '--issue-number', required=True, type=int, help='Issue number to inspect'+ )+ parser.add_argument(+ '--output',+ default='good-first-issue-check-result.json',+ help='Path where the JSON result should be written',+ )+ parser.add_argument(+ '--poll-interval-seconds',+ default=5,+ type=int,+ help='Polling interval while waiting for the conversation to finish',+ )+ parser.add_argument(+ '--max-wait-seconds',+ default=900,+ type=int,+ help=(+ 'Maximum time to wait per polling phase; if a start task must be awaited '+ 'first, the total runtime can approach twice this value'+ ),+ )+ return parser.parse_args()+++def github_headers() -> dict[str, str]:+ headers = {+ 'Accept': 'application/vnd.github+json',+ 'User-Agent': 'openhands-good-first-issue-check',+ 'X-GitHub-Api-Version': '2022-11-28',+ }+ github_token = os.environ.get('GITHUB_TOKEN')+ if github_token:+ headers['Authorization'] = f'Bearer {github_token}'+ return headers+++def openhands_headers() -> dict[str, str]:+ api_key = os.environ.get('OPENHANDS_API_KEY')+ if not api_key:+ raise RuntimeError('OPENHANDS_API_KEY environment variable is required')+ return {+ 'Authorization': f'Bearer {api_key}',+ 'Content-Type': 'application/json',+ }+++def request_json(+ base_url: str,+ path: str,+ *,+ method: str = 'GET',+ headers: dict[str, str] | None = None,+ body: dict[str, Any] | None = None,+) -> Any:+ data = json.dumps(body).encode('utf-8') if body is not None else None+ request = urllib.request.Request(+ f'{base_url}{path}',+ data=data,+ headers=headers or {},+ method=method,+ )+ try:+ with urllib.request.urlopen(request, timeout=60) as response:+ return json.load(response)+ except urllib.error.HTTPError as exc:+ error_body = exc.read().decode('utf-8', errors='replace')+ raise RuntimeError(+ f'{method} {base_url}{path} failed with HTTP {exc.code}: {error_body}'+ ) from exc+ except json.JSONDecodeError as exc:+ raise RuntimeError(+ f'Failed to parse JSON from {method} {base_url}{path}: {exc}'+ ) from exc+ except urllib.error.URLError as exc:+ raise RuntimeError(f'{method} {base_url}{path} failed: {exc}') from exc+++def fetch_issue(repository: str, issue_number: int) -> dict[str, Any]:+ if not REPOSITORY_PATTERN.fullmatch(repository):+ raise ValueError(f'Invalid repository format: {repository}')+ return request_json(+ GITHUB_API_BASE_URL,+ f'/repos/{repository}/issues/{issue_number}',+ headers=github_headers(),+ )+++def escape_json_text(value: str | None) -> str:+ return json.dumps(value or '', ensure_ascii=False)+++def extract_label_names(issue: dict[str, Any]) -> list[str]:+ labels = issue.get('labels')+ if not isinstance(labels, list):+ return []++ label_names: list[str] = []+ for label in labels:+ if isinstance(label, str):+ candidate = label.strip()+ elif isinstance(label, dict):+ candidate = str(label.get('name') or '').strip()+ else:+ candidate = ''+ if candidate:+ label_names.append(candidate)+ return sorted(set(label_names), key=str.lower)+++def build_prompt(repository: str, issue: dict[str, Any]) -> str:+ issue_number = issue['number']+ issue_title = issue.get('title', '')+ issue_body = issue.get('body') or ''+ issue_url = issue.get('html_url', '')+ issue_labels = extract_label_names(issue)+ issue_title_json = escape_json_text(issue_title)+ issue_body_json = escape_json_text(issue_body)+ issue_labels_json = json.dumps(issue_labels, ensure_ascii=False)++ return '\n'.join(+ [+ 'You are deciding whether a GitHub issue should be labeled `good first issue` for newcomers to this repository.',+ '',+ 'Be conservative. This label is auto-applied by automation, so false positives are more harmful than false negatives.',+ '',+ f'Repository: {repository}',+ f'Issue number: #{issue_number}',+ f'Issue URL: {issue_url}',+ f'Issue labels (JSON array): {issue_labels_json}',+ f'Issue title (JSON-escaped string): {issue_title_json}',+ f'Issue body (JSON-escaped string): {issue_body_json}',+ '',+ 'Use the repository, issue details, labels, and any code/documentation context you inspect to decide whether this issue is truly appropriate for a newcomer.',+ '',+ 'All of the following must be true to apply `good first issue`:',+ '1. The work is narrow in scope and should be solved as a single issue rather than a multi-step project.',+ '2. The bug, request, or expected outcome is already clear enough that a contributor should not need major discovery work before starting.',+ '3. The likely fix stays within a well-bounded area of the repo and does not require deep architecture knowledge, cross-repo coordination, enterprise-only context, migrations, security-sensitive changes, or infrastructure work.',+ '4. The validation path is straightforward: the PR author should be able to give clear How to Test steps, and if the change is user-facing, screenshots or video evidence should be practical to provide.',+ '5. A reviewer should be able to evaluate the change with the normal PR review workflow, without special credentials, paid services, or hard-to-reproduce environments.',+ '',+ 'Do NOT apply `good first issue` for broad design discussions, umbrella tracking issues, items missing reproduction steps or acceptance criteria, or changes likely to require significant maintainer guidance.',+ 'Do NOT treat “not a duplicate” as positive evidence on its own. Only use the criteria above.',+ 'Do not post comments, do not modify files, and do not change repository state.',+ 'Return exactly one JSON object and nothing else. Do not wrap it in markdown fences.',+ '',+ 'Return schema:',+ '{',+ f' "issue_number": {issue_number},',+ ' "should_apply_label": true or false,',+ ' "confidence": "high" | "medium" | "low",',+ ' "summary": "short explanation",',+ ' "criteria_met": ["criterion that is satisfied"],',+ ' "disqualifiers": ["reason the issue should not be auto-labeled"]',+ '}',+ '',+ 'Rules:',+ '- `should_apply_label` must be true only when all required criteria are satisfied.',+ '- If any material uncertainty remains, return `should_apply_label: false`.',+ '- `should_apply_label` must be false unless confidence is `high`.',+ '- `criteria_met` and `disqualifiers` should each contain at most 5 short strings.',+ '- If the issue is missing enough detail to judge newcomer suitability confidently, return `should_apply_label: false`.',+ '- If the issue appears enterprise-only, security-sensitive, infrastructure-heavy, or broadly architectural, return `should_apply_label: false`.',+ ]+ )+++def start_conversation(+ prompt: str, repository: str, issue_number: int+) -> dict[str, Any]:+ body = {+ 'title': f'Good first issue check #{issue_number}',+ 'selected_repository': repository,+ 'initial_message': {+ 'content': [+ {+ 'type': 'text',+ 'text': prompt,+ }+ ]+ },+ }+ return request_json(+ OPENHANDS_BASE_URL,+ '/api/v1/app-conversations',+ method='POST',+ headers=openhands_headers(),+ body=body,+ )+++def extract_first_item(payload: Any) -> dict[str, Any] | None:+ if isinstance(payload, list):+ first_item = payload[0] if payload else None+ return first_item if isinstance(first_item, dict) else None+ if not isinstance(payload, dict):+ return None++ items = payload.get('items')+ if isinstance(items, list):+ first_item = items[0] if items else None+ return first_item if isinstance(first_item, dict) else None+ return payload+++def poll_start_task(+ start_task_id: str, poll_interval_seconds: int, max_wait_seconds: int+) -> dict[str, Any]:+ deadline = time.time() + max_wait_seconds+ while time.time() < deadline:+ payload = request_json(+ OPENHANDS_BASE_URL,+ f'/api/v1/app-conversations/start-tasks?ids={urllib.parse.quote(start_task_id)}',+ headers={'Authorization': openhands_headers()['Authorization']},+ )+ item = extract_first_item(payload)+ if item is None:+ time.sleep(poll_interval_seconds)+ continue+ status = item.get('status')+ if status == 'READY' and item.get('app_conversation_id'):+ return item+ if status in {'ERROR', 'FAILED'}:+ raise RuntimeError(f'OpenHands start task failed: {json.dumps(item)}')+ time.sleep(poll_interval_seconds)+ raise TimeoutError(+ f'Timed out waiting for start task {start_task_id} to become ready'+ )+++def poll_conversation(+ app_conversation_id: str, poll_interval_seconds: int, max_wait_seconds: int+) -> dict[str, Any]:+ deadline = time.time() + max_wait_seconds+ while time.time() < deadline:+ payload = request_json(+ OPENHANDS_BASE_URL,+ f'/api/v1/app-conversations?ids={urllib.parse.quote(app_conversation_id)}',+ headers={'Authorization': openhands_headers()['Authorization']},+ )+ item = extract_first_item(payload)+ if item is None:+ time.sleep(poll_interval_seconds)+ continue+ execution_status = str(item.get('execution_status', '')).lower()+ if execution_status in FAILED_EXECUTION_STATUSES:+ raise RuntimeError(+ 'OpenHands conversation ended with '+ f'{execution_status}: {json.dumps(item)}'+ )+ if execution_status in SUCCESSFUL_TERMINAL_EXECUTION_STATUSES:+ return item+ time.sleep(poll_interval_seconds)+ raise TimeoutError(+ f'Timed out waiting for conversation {app_conversation_id} to finish running'+ )+++def validate_event_search_results(events: list[dict[str, Any]]) -> list[dict[str, Any]]:+ if len(events) >= EVENT_SEARCH_LIMIT:+ raise RuntimeError(EVENT_SEARCH_LIMIT_HIT_MESSAGE)+ return events+++def fetch_app_server_events(app_conversation_id: str) -> list[dict[str, Any]]:+ payload = request_json(+ OPENHANDS_BASE_URL,+ f'/api/v1/conversation/{urllib.parse.quote(app_conversation_id)}/events/search?limit={EVENT_SEARCH_LIMIT}',+ headers={'Authorization': openhands_headers()['Authorization']},+ )+ if isinstance(payload, dict):+ items = payload.get('items')+ return validate_event_search_results(items) if isinstance(items, list) else []+ if isinstance(payload, list):+ return validate_event_search_results(payload)+ return []+++def fetch_agent_server_events(+ app_conversation_id: str, agent_server_url: str, session_api_key: str+) -> list[dict[str, Any]]:+ payload = request_json(+ agent_server_url,+ f'/api/conversations/{urllib.parse.quote(app_conversation_id)}/events/search?limit={EVENT_SEARCH_LIMIT}',+ headers={'X-Session-API-Key': session_api_key},+ )+ if isinstance(payload, dict):+ items = payload.get('items')+ return validate_event_search_results(items) if isinstance(items, list) else []+ if isinstance(payload, list):+ return validate_event_search_results(payload)+ return []+++def fetch_agent_server_final_response(+ app_conversation_id: str, agent_server_url: str, session_api_key: str+) -> str:+ payload = request_json(+ agent_server_url,+ f'/api/conversations/{urllib.parse.quote(app_conversation_id)}/agent_final_response',+ headers={'X-Session-API-Key': session_api_key},+ )+ if not isinstance(payload, dict):+ return ''+ return str(payload.get('response') or '').strip()+++def extract_agent_server_url(conversation_url: str) -> str | None:+ marker = '/api/conversations/'+ if marker not in conversation_url:+ return None+ return conversation_url.rsplit(marker, 1)[0]+++def extract_last_agent_text(events: list[dict[str, Any]]) -> str:+ agent_events = [+ event+ for event in events+ if event.get('kind') == 'MessageEvent' and event.get('source') == 'agent'+ ]+ if not agent_events:+ raise RuntimeError(+ 'No assistant text message was found in the conversation events'+ )++ llm_message = agent_events[-1].get('llm_message')+ if not isinstance(llm_message, dict):+ raise RuntimeError('Last agent message has no llm_message field')+ content = llm_message.get('content')+ if not isinstance(content, list):+ raise RuntimeError('Last agent message content is not a list')++ text_parts: list[str] = []+ for part in content:+ if not isinstance(part, dict):+ continue+ if part.get('type') == 'text' and part.get('text'):+ text_parts.append(str(part['text']))+ if not text_parts:+ raise RuntimeError('Last agent message contains no text content')+ return ''.join(text_parts).strip()+++def parse_agent_json(text: str) -> dict[str, Any]:+ cleaned = text.strip()+ try:+ return json.loads(cleaned)+ except json.JSONDecodeError:+ decoder = json.JSONDecoder()+ for start, character in enumerate(cleaned):+ if character != '{':+ continue+ try:+ candidate, end = decoder.raw_decode(cleaned[start:])+ except json.JSONDecodeError:+ continue+ trailing = cleaned[start + end :].strip()+ if trailing not in {'', '```'}:+ continue+ if isinstance(candidate, dict):+ return candidate+ raise ValueError('No valid JSON object found in the agent response')+++def as_bool(value: Any) -> bool:+ if isinstance(value, bool):+ return value+ if isinstance(value, str):+ return value.strip().lower() in {'true', '1', 'yes'}+ if isinstance(value, (int, float)):+ return bool(value)+ return False+++def normalize_string_list(value: Any) -> list[str]:+ if not isinstance(value, list):+ return []++ normalized: list[str] = []+ for item in value:+ text = str(item).strip() if item is not None else ''+ if text:+ normalized.append(text)+ if len(normalized) == 5:+ break+ return normalized+++def normalize_result(result: dict[str, Any]) -> dict[str, Any]:+ normalized = dict(result)+ normalized['should_apply_label'] = as_bool(normalized.get('should_apply_label'))++ confidence = str(normalized.get('confidence') or 'low').strip().lower()+ if confidence not in {'high', 'medium', 'low'}:+ confidence = 'low'+ normalized['confidence'] = confidence++ normalized['summary'] = str(normalized.get('summary') or '').strip()+ normalized['criteria_met'] = normalize_string_list(normalized.get('criteria_met'))+ normalized['disqualifiers'] = normalize_string_list(normalized.get('disqualifiers'))++ if confidence != 'high':+ normalized['should_apply_label'] = False+ if normalized['disqualifiers']:+ normalized['should_apply_label'] = False++ return normalized+++def main() -> int:+ args = parse_args()+ issue = fetch_issue(args.repository, args.issue_number)+ if issue.get('pull_request'):+ raise RuntimeError(f'#{args.issue_number} is a pull request, not an issue')++ prompt = build_prompt(args.repository, issue)+ start_task = start_conversation(prompt, args.repository, args.issue_number)+ app_conversation_id = start_task.get('app_conversation_id')+ conversation_url = ''++ if not app_conversation_id:+ task_id = start_task.get('id')+ if not task_id:+ raise RuntimeError(f'Missing id in start task response: {start_task}')+ ready_task = poll_start_task(+ task_id,+ args.poll_interval_seconds,+ args.max_wait_seconds,+ )+ app_conversation_id = ready_task.get('app_conversation_id')+ if not app_conversation_id:+ raise RuntimeError(f'Missing app_conversation_id in response: {ready_task}')++ conversation = poll_conversation(+ app_conversation_id,+ args.poll_interval_seconds,+ args.max_wait_seconds,+ )+ conversation_url = (+ conversation.get('conversation_url')+ or f'{OPENHANDS_BASE_URL}/conversations/{app_conversation_id}'+ )+ session_api_key_value = conversation.get('session_api_key')+ if session_api_key_value and not isinstance(session_api_key_value, str):+ raise RuntimeError(+ 'session_api_key had unexpected type in the OpenHands conversation: '+ f'{type(session_api_key_value).__name__}'+ )+ session_api_key = session_api_key_value or ''+ agent_server_url = extract_agent_server_url(conversation_url)++ agent_text = ''+ if agent_server_url and session_api_key:+ try:+ agent_text = fetch_agent_server_final_response(+ app_conversation_id,+ agent_server_url,+ session_api_key,+ )+ except RuntimeError:+ agent_text = ''+ if not agent_text:+ events = fetch_app_server_events(app_conversation_id)+ try:+ agent_text = extract_last_agent_text(events)+ except RuntimeError as exc:+ if not session_api_key:+ raise RuntimeError(+ 'App server events did not contain assistant text and '+ 'session_api_key was missing from the OpenHands conversation'+ ) from exc+ if not agent_server_url:+ raise RuntimeError(+ 'App server events did not contain assistant text and cannot '+ 'extract agent server URL from conversation URL: '+ f'{conversation_url}'+ ) from exc+ events = fetch_agent_server_events(+ app_conversation_id,+ agent_server_url,+ session_api_key,+ )+ agent_text = extract_last_agent_text(events)+ result = normalize_result(parse_agent_json(agent_text))++ result['issue_number'] = args.issue_number+ result['repository'] = args.repository+ result['app_conversation_id'] = app_conversation_id+ result['conversation_url'] = conversation_url+ result['agent_response'] = agent_text++ output_path = Path(args.output)+ try:+ output_path.write_text(json.dumps(result, indent=2, ensure_ascii=False) + '\n')+ except OSError as exc:+ raise RuntimeError(f'Failed to write output to {output_path}: {exc}') from exc++ print(+ json.dumps(+ {+ 'issue_number': result.get('issue_number'),+ 'should_apply_label': result.get('should_apply_label'),+ 'confidence': result.get('confidence'),+ 'conversation_url': result.get('conversation_url'),+ 'output': str(output_path),+ },+ ensure_ascii=False,+ )+ )+ return 0+++if __name__ == '__main__':+ try:+ raise SystemExit(main())+ except Exception as exc: # pragma: no cover - CLI boundary+ print(f'Error: {exc}', file=sys.stderr)+ raise SystemExit(1)diff --git a/tests/unit/test_issue_good_first_issue_check_openhands.py b/tests/unit/test_issue_good_first_issue_check_openhands.pynew file mode 100644index 0000000000..52980765e7--- /dev/null+++ b/tests/unit/test_issue_good_first_issue_check_openhands.py@@ -0,0 +1,291 @@+from __future__ import annotations++import argparse+import importlib.util+import itertools+import json+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[2]+MODULE_COUNTER = itertools.count()+++def load_module(script_name: str):+ path = ROOT / 'scripts' / script_name+ module_name = f'test_{path.stem}_{next(MODULE_COUNTER)}'+ spec = importlib.util.spec_from_file_location(module_name, path)+ if spec is None or spec.loader is None:+ raise AssertionError(f'Unable to load module from {path}')+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def make_agent_message(text: str) -> dict:+ return {+ 'kind': 'MessageEvent',+ 'source': 'agent',+ 'llm_message': {'content': [{'type': 'text', 'text': text}]},+ }+++def test_extract_label_names_normalizes_issue_labels():+ module = load_module('issue_good_first_issue_check_openhands.py')++ label_names = module.extract_label_names(+ {+ 'labels': [+ {'name': 'bug'},+ 'good first issue',+ {'name': 'Bug'},+ {'name': ''},+ None,+ ]+ }+ )++ assert label_names == ['Bug', 'bug', 'good first issue']+++def test_build_prompt_includes_issue_labels_and_criteria():+ module = load_module('issue_good_first_issue_check_openhands.py')++ prompt = module.build_prompt(+ 'OpenHands/OpenHands',+ {+ 'number': 123,+ 'title': 'Clarify triage behavior',+ 'body': 'The issue needs a clear automated path.',+ 'html_url': 'https://github.com/OpenHands/OpenHands/issues/123',+ 'labels': [{'name': 'bug'}, {'name': 'frontend'}],+ },+ )++ assert 'Issue labels (JSON array): ["bug", "frontend"]' in prompt+ assert 'Do NOT treat “not a duplicate” as positive evidence on its own.' in prompt+ assert 'All of the following must be true to apply `good first issue`:' in prompt+++def test_normalize_result_requires_high_confidence_and_no_disqualifiers():+ module = load_module('issue_good_first_issue_check_openhands.py')++ normalized = module.normalize_result(+ {+ 'should_apply_label': True,+ 'confidence': 'medium',+ 'summary': 'Looks promising',+ 'criteria_met': ['narrow scope', 'clear request'],+ 'disqualifiers': [],+ }+ )++ assert normalized['should_apply_label'] is False+ assert normalized['confidence'] == 'medium'++ normalized = module.normalize_result(+ {+ 'should_apply_label': True,+ 'confidence': 'high',+ 'summary': 'Looks promising',+ 'criteria_met': ['narrow scope', 'clear request'],+ 'disqualifiers': ['needs enterprise context'],+ }+ )++ assert normalized['should_apply_label'] is False+ assert normalized['disqualifiers'] == ['needs enterprise context']+++def test_normalize_result_limits_string_lists():+ module = load_module('issue_good_first_issue_check_openhands.py')++ normalized = module.normalize_result(+ {+ 'should_apply_label': True,+ 'confidence': 'high',+ 'summary': 'Looks promising',+ 'criteria_met': ['1', '2', '3', '4', '5', '6'],+ 'disqualifiers': [None, '', 'too broad', 42, ''],+ }+ )++ assert normalized['criteria_met'] == ['1', '2', '3', '4', '5']+ assert normalized['disqualifiers'] == ['too broad', '42']+ assert normalized['should_apply_label'] is False+++def test_good_first_issue_main_writes_result_from_final_response(monkeypatch, tmp_path):+ module = load_module('issue_good_first_issue_check_openhands.py')+ output_path = tmp_path / 'result.json'++ monkeypatch.setattr(+ module,+ 'parse_args',+ lambda: argparse.Namespace(+ repository='OpenHands/OpenHands',+ issue_number=123,+ output=str(output_path),+ poll_interval_seconds=1,+ max_wait_seconds=10,+ ),+ )+ monkeypatch.setattr(+ module,+ 'fetch_issue',+ lambda repository, issue_number: {+ 'number': issue_number,+ 'title': 'Issue title',+ 'body': 'Issue body',+ 'html_url': f'https://github.com/{repository}/issues/{issue_number}',+ 'labels': [{'name': 'bug'}],+ },+ )+ monkeypatch.setattr(+ module,+ 'start_conversation',+ lambda *args, **kwargs: {'app_conversation_id': 'conv-123'},+ )+ monkeypatch.setattr(+ module,+ 'poll_conversation',+ lambda app_conversation_id, poll_interval_seconds, max_wait_seconds: {+ 'conversation_url': 'https://runtime.example/api/conversations/conv-123',+ 'session_api_key': 'session-key',+ },+ )+ monkeypatch.setattr(+ module,+ 'fetch_agent_server_final_response',+ lambda app_conversation_id, agent_server_url, session_api_key: json.dumps(+ {+ 'should_apply_label': True,+ 'confidence': 'high',+ 'summary': 'Narrow, clear, and easy to validate.',+ 'criteria_met': ['narrow scope', 'clear expected outcome'],+ 'disqualifiers': [],+ }+ ),+ )+ monkeypatch.setattr(+ module,+ 'fetch_app_server_events',+ lambda app_conversation_id: pytest.fail(+ 'fetch_app_server_events should not run'+ ),+ )+ monkeypatch.setattr(+ module,+ 'fetch_agent_server_events',+ lambda *args, **kwargs: pytest.fail('fetch_agent_server_events should not run'),+ )++ assert module.main() == 0++ result = json.loads(output_path.read_text())+ assert result['should_apply_label'] is True+ assert result['confidence'] == 'high'+ assert result['criteria_met'] == ['narrow scope', 'clear expected outcome']+ assert result['repository'] == 'OpenHands/OpenHands'+++def test_good_first_issue_main_falls_back_to_app_server_events(monkeypatch, tmp_path):+ module = load_module('issue_good_first_issue_check_openhands.py')+ output_path = tmp_path / 'result.json'++ monkeypatch.setattr(+ module,+ 'parse_args',+ lambda: argparse.Namespace(+ repository='OpenHands/OpenHands',+ issue_number=123,+ output=str(output_path),+ poll_interval_seconds=1,+ max_wait_seconds=10,+ ),+ )+ monkeypatch.setattr(+ module,+ 'fetch_issue',+ lambda repository, issue_number: {+ 'number': issue_number,+ 'title': 'Issue title',+ 'body': 'Issue body',+ 'html_url': f'https://github.com/{repository}/issues/{issue_number}',+ 'labels': [{'name': 'enhancement'}],+ },+ )+ monkeypatch.setattr(+ module,+ 'start_conversation',+ lambda *args, **kwargs: {'app_conversation_id': 'conv-123'},+ )+ monkeypatch.setattr(+ module,+ 'poll_conversation',+ lambda app_conversation_id, poll_interval_seconds, max_wait_seconds: {+ 'conversation_url': 'https://runtime.example/api/conversations/conv-123',+ 'session_api_key': 'session-key',+ },+ )+ monkeypatch.setattr(+ module,+ 'fetch_agent_server_final_response',+ lambda *args, **kwargs: '',+ )+ monkeypatch.setattr(+ module,+ 'fetch_app_server_events',+ lambda app_conversation_id: [+ make_agent_message(+ json.dumps(+ {+ 'should_apply_label': False,+ 'confidence': 'medium',+ 'summary': 'Needs more discovery before a newcomer can pick it up.',+ 'criteria_met': ['clear user-facing outcome'],+ 'disqualifiers': ['scope is still too broad'],+ }+ )+ )+ ],+ )+ monkeypatch.setattr(+ module,+ 'fetch_agent_server_events',+ lambda *args, **kwargs: pytest.fail('fetch_agent_server_events should not run'),+ )++ assert module.main() == 0++ result = json.loads(output_path.read_text())+ assert result['should_apply_label'] is False+ assert result['disqualifiers'] == ['scope is still too broad']+++def test_good_first_issue_main_rejects_pull_requests(monkeypatch, tmp_path):+ module = load_module('issue_good_first_issue_check_openhands.py')++ monkeypatch.setattr(+ module,+ 'parse_args',+ lambda: argparse.Namespace(+ repository='OpenHands/OpenHands',+ issue_number=123,+ output=str(tmp_path / 'result.json'),+ poll_interval_seconds=1,+ max_wait_seconds=10,+ ),+ )+ monkeypatch.setattr(+ module,+ 'fetch_issue',+ lambda repository, issue_number: {+ 'number': issue_number,+ 'pull_request': {'url': 'https://example.test/pr/123'},+ },+ )++ with pytest.raises(RuntimeError, match='is a pull request, not an issue'):+ module.main()