Scrapy, a fast high-level web crawling & scraping framework for Python.
Switch the spam PR check to scrapy/unattended-pr-guard 0.2.0 (#8286)
Adrianeba114d98ecd
diff --git a/.github/workflows/close-spam-prs.yml b/.github/workflows/close-spam-prs.ymlindex 5b1168111..68aa151cf 100644--- a/.github/workflows/close-spam-prs.yml+++ b/.github/workflows/close-spam-prs.yml@@ -1,48 +1,4 @@name: Close spam PRs-# Closes as spam pull requests whose author's public activity suggests that an-# LLM is writing them without supervision, and records the evidence in the-# workflow run summary so that reviewing the decision does not require reading-# a user profile. Reopening and removing the label is all it takes to overrule-# it.-#-# Four independent signals, any of which is enough to close. Each one abstains-# when the data it needs is unavailable, so a missing signal never counts-# against an author:-#-# - Rejection burst: pull requests of theirs closed unmerged elsewhere within-# the last month. Volume of rejections in absolute terms separates spraying-# from ordinary contribution far better than a merge ratio does, since-# ratios reward authors who accumulate merges in trivial repositories.-# - Spray breadth: unrelated repositories they open pull requests against-# within one week. Breadth catches an agent on its first day, before any of-# its pull requests have been closed, and it comes from the event feed, so it-# also covers authors that the search API refuses to return.-# - Assistant voice: their recent comments across GitHub read as assistant-# output rather than as a developer talking, by section headings, bullet-# lists, em dash density or stock acknowledgement phrases.-# - Agent branch: the branch name carries an agent prefix.-#-# The signals build up over time: the author's other pull requests get closed-# elsewhere, and the spray reaches more repositories after this one. So besides-# scoring each pull request when it is opened, the same scoring runs every few-# hours over the pull requests opened within the last few days. That pass-# leaves alone any pull request that a maintainer has commented on, reviewed-# or reopened, so a human decision is never overturned.-#-# Authors that the organisations behind this repository already trust are left-# alone before any of that runs: public members of those organisations, and-# authors with a track record of pull requests merged into their repositories.-# Trust from a merge record rather than from a list of names keeps the exemption-# in step with who is actually contributing.-#-# Deliberately not used: account age, fork age, follower count, total pull-# request count and cross-repository merge ratio. All of them were measured-# against hand-labelled pull requests and either failed to separate or, in the-# case of the merge ratio, inverted on held-out data.-#-# The workflow only reads pull request and public activity metadata through the-# API, it never checks out or runs pull request code, so pull_request_target is-# safe here.on: # zizmor: ignore[dangerous-triggers]pull_request_target:types: [opened]@@ -53,243 +9,15 @@ permissions:contents: readpull-requests: writejobs:- close-spam-pr:- name: Close PRs if the author's activity suggests unsupervised LLM use+ unattended-pr-guard:runs-on: ubuntu-lateststeps:- - name: Score the authors and close the PRs- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0+ - uses: scrapy/unattended-pr-guard@2ea7eff575dd0912d03cf3e3ae7acf65052c6445 # 0.2.0with:- github-token: ${{ secrets.GITHUB_TOKEN }}- script: |- const LABEL = 'spam';- const RETRIES = 5;- const RETRY_WAIT_MS = 60000;- const REJECTION_WINDOW_DAYS = 30;- const RECHECK_WINDOW_DAYS = 3;- const MIN_REJECTIONS = 1;- const MIN_COMMENTS = 2;- const MAX_REPOS_PER_WEEK = 2;- const VOICE = { structure: 0.10, emDashPerKChar: 0.30, acknowledgement: 0.40 };- const EVENT_PAGES = 3;- const TRUSTED_ORGS = ['scrapy', 'scrapy-plugins', 'scrapinghub', 'zytedata'];- const MIN_TRUSTED_MERGES = 10;- const AGENT_BRANCH = /^(agent|codex|claude|cursor|devin|copilot|jules|bot)[\/_-]/i;-- const { owner, repo } = context.repo;- const now = new Date();- const daysAgo = date => (now - new Date(date)) / 86400000;-- // Rate and abuse limits reset on the order of a minute, so waiting- // is enough; other errors are not worth retrying.- const retriable = new Set([403, 429, 500, 502, 503, 504]);- const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));- async function withRetries(description, call) {- for (let attempt = 1; ; attempt++) {- try {- return await call();- } catch (error) {- if (!retriable.has(error.status) || attempt > RETRIES) throw error;- const reset = Number(error.response?.headers?.['x-ratelimit-reset']) * 1000 - Date.now();- const after = Number(error.response?.headers?.['retry-after']) * 1000;- const wait = Math.min(Math.max(after || reset || RETRY_WAIT_MS, RETRY_WAIT_MS), 15 * RETRY_WAIT_MS);- core.info(`${description} failed with ${error.status}, retrying in ${Math.round(wait / 1000)}s (attempt ${attempt}/${RETRIES}).`);- await sleep(wait);- }- }- }- // Accounts excluded from search, deleted users and the like leave a- // signal unmeasurable rather than negative.- const orNull = promise => promise.catch(error => {- if ([404, 410, 422].includes(error.status)) return null;- throw error;- });-- async function score(pr) {- const author = pr.user.login;-- // author_association only reports membership of the organisation- // that owns this repository, and only when it is public, so trust- // in the author is established here instead.- const trustedOrg = (await Promise.all(TRUSTED_ORGS.map(org =>- orNull(withRetries(`Checking public membership of ${org}`, () =>- github.rest.orgs.checkPublicMembershipForUser({ org, username: author }),- )).then(response => response && org),- ))).find(Boolean);- if (trustedOrg) {- core.info(`Skipping PR #${pr.number} by ${author} (public member of ${trustedOrg}).`);- return;- }- // Repeating a qualifier narrows the search instead of widening it,- // hence the explicit disjunction.- const trustedMerges = await orNull(withRetries('Counting merged PRs in trusted organisations', () =>- github.rest.search.issuesAndPullRequests({- q: `author:${author} type:pr is:merged`- + ` (${TRUSTED_ORGS.map(org => `org:${org}`).join(' OR ')})`,- advanced_search: 'true', per_page: 1,- }).then(response => response.data.total_count),- ));- if (trustedMerges >= MIN_TRUSTED_MERGES) {- core.info(`Skipping PR #${pr.number} by ${author}`- + ` (${trustedMerges} PR(s) merged into ${TRUSTED_ORGS.join(', ')}).`);- return;- }-- // Signal 1: pull requests closed unmerged elsewhere, recently.- const search = await orNull(withRetries('Searching for PRs by the author', () =>- github.rest.search.issuesAndPullRequests({- q: `author:${author} type:pr`, advanced_search: 'true',- sort: 'created', order: 'desc', per_page: 100,- }).then(response => response.data),- ));- let rejections = null;- if (search) {- rejections = search.items.filter(item => {- const itemOwner = item.repository_url.split('/repos/')[1].split('/')[0].toLowerCase();- return itemOwner !== author.toLowerCase()- && item.state === 'closed' && !item.pull_request?.merged_at- && daysAgo(item.created_at) <= REJECTION_WINDOW_DAYS;- }).map(item => item.html_url);- }-- // Signal 2: how their recent comments across GitHub read.- const events = [];- for (let page = 1; page <= EVENT_PAGES; page++) {- const batch = await orNull(withRetries(`Reading public events page ${page}`, () =>- github.rest.activity.listPublicEventsForUser({- username: author, per_page: 100, page,- }).then(response => response.data),- ));- if (!batch?.length) break;- events.push(...batch);- if (batch.length < 100) break;- }- const comments = events- .filter(event => ['IssueCommentEvent', 'PullRequestReviewCommentEvent'].includes(event.type))- .map(event => event.payload?.comment?.body)- .filter(Boolean);-- // Signal 3: how many unrelated projects they open pull requests- // against in a single week. Breadth rather than volume: a focused- // contributor sends many pull requests to few repositories, while- // an unattended agent sprays a few across many. Taken from the- // event feed, which unlike search covers authors that search- // refuses to return.- const weeks = {};- for (const event of events) {- if (event.type !== 'PullRequestEvent' || event.payload?.action !== 'opened') continue;- const name = event.repo?.name;- if (!name || name.toLowerCase().startsWith(`${author.toLowerCase()}/`)) continue;- const week = Math.floor(new Date(event.created_at) / (7 * 86400000));- (weeks[week] ??= new Set()).add(name);- }- const breadth = events.length- ? Math.max(0, ...Object.values(weeks).map(repos => repos.size))- : null;- const STRUCTURE = [/^\s*#{2,3}\s/m, /^\s*[-*]\s.+\n\s*[-*]\s/m, /\*\*[^*]+\*\*/, /```/];- const ACKNOWLEDGEMENT = [- /thanks for (the )?(review|feedback|pointing|catching|flagging|clarif)/i,- /you'?re (absolutely )?right/i, /great catch/i, /that makes sense/i,- /i'?ll (continue|investigate|update|submit|look into|make sure)/i,- /let me know (if|whether)/i, /happy to (update|adjust|revise|change)/i,- /i understand that/i, /thanks for your time/i, /just following up/i,- /hope (this|that) helps/i, /please let me know/i, /i'?ve (updated|addressed|fixed)/i,- ];- let voice = null;- if (comments.length >= MIN_COMMENTS) {- const chars = comments.reduce((total, body) => total + body.length, 0);- const rate = patterns => comments.filter(body => patterns.some(re => re.test(body))).length / comments.length;- voice = {- comments: comments.length,- structure: rate(STRUCTURE),- acknowledgement: rate(ACKNOWLEDGEMENT),- emDashPerKChar: 1000 * comments.reduce((total, body) => total + (body.match(/—/g) || []).length, 0) / chars,- };- }-- const reasons = [];- if (rejections && rejections.length >= MIN_REJECTIONS) {- reasons.push(`${rejections.length} PR(s) of theirs closed unmerged elsewhere in the last`- + ` ${REJECTION_WINDOW_DAYS} days: ${rejections.slice(0, 10).join(' ')}`);- }- if (voice && (voice.structure > VOICE.structure- || voice.emDashPerKChar > VOICE.emDashPerKChar- || voice.acknowledgement > VOICE.acknowledgement)) {- reasons.push(`comment style over ${voice.comments} recent comments:`- + ` ${(100 * voice.structure).toFixed(0)}% structured,`- + ` ${(100 * voice.acknowledgement).toFixed(0)}% stock acknowledgements,`- + ` ${voice.emDashPerKChar.toFixed(2)} em dashes per 1000 characters`);- }- if (breadth !== null && breadth > MAX_REPOS_PER_WEEK) {- reasons.push(`opened pull requests against ${breadth} unrelated repositories within a week`);- }- if (AGENT_BRANCH.test(pr.head?.ref || '')) {- reasons.push(`branch name carries an agent prefix: ${pr.head.ref}`);- }-- await core.summary- .addHeading(`PR #${pr.number} by ${author}`, 3)- .addList([- rejections === null- ? 'recent rejections elsewhere: unmeasurable, the author cannot be searched'- : `recent rejections elsewhere: ${rejections.length}`,- voice === null- ? `comment style: unmeasurable, fewer than ${MIN_COMMENTS} recent comments found`- : `comment style: ${(100 * voice.structure).toFixed(0)}% structured,`- + ` ${(100 * voice.acknowledgement).toFixed(0)}% stock acknowledgements,`- + ` ${voice.emDashPerKChar.toFixed(2)} em dashes per 1000 characters`- + ` over ${voice.comments} comments`,- breadth === null- ? 'repositories per week: unmeasurable, no public events found'- : `repositories per week, at most: ${breadth}`,- `branch: ${pr.head?.ref ?? 'unknown'}`,- `verdict: ${reasons.length ? `closed as ${LABEL}` : 'left open'}`,- ])- .addRaw(reasons.length ? `\n${reasons.map(reason => `- ${reason}`).join('\n')}\n` : '')- .write();-- if (!reasons.length) {- core.info(`Leaving PR #${pr.number} open.`);- return;- }- await withRetries('Adding the label', () =>- github.rest.issues.addLabels({ owner, repo, issue_number: pr.number, labels: [LABEL] }),- );- await withRetries('Closing the PR', () =>- github.rest.pulls.update({ owner, repo, pull_number: pr.number, state: 'closed' }),- );- core.info(`Closed PR #${pr.number} as ${LABEL}: ${reasons.join(' | ')}`);- }-- // A maintainer who has commented on, reviewed or reopened a pull- // request has already made the call.- async function attended(pr) {- const timeline = await withRetries(`Reading the timeline of PR #${pr.number}`, () =>- github.paginate(github.rest.issues.listEventsForTimeline, {- owner, repo, issue_number: pr.number, per_page: 100,- }),- );- return timeline.some(event => event.event === 'reopened'- || (['commented', 'reviewed'].includes(event.event)- && event.user?.login !== pr.user.login && event.user?.type !== 'Bot'));- }-- const opened = context.payload.pull_request;- const prs = opened ? [opened] : (await withRetries('Listing open PRs', () =>- github.paginate(github.rest.pulls.list, {- owner, repo, state: 'open', sort: 'created', direction: 'desc', per_page: 100,- }),- )).filter(pr => daysAgo(pr.created_at) <= RECHECK_WINDOW_DAYS);- core.info(`Scoring ${prs.length} PR(s).`);- for (const pr of prs) {- if (pr.user.type === 'Bot'- || ['MEMBER', 'OWNER', 'COLLABORATOR'].includes(pr.author_association)) {- core.info(`Skipping PR #${pr.number} by ${pr.user.login} (${pr.user.type}, ${pr.author_association}).`);- continue;- }- if (!opened && await attended(pr)) {- core.info(`Skipping PR #${pr.number} by ${pr.user.login} (a maintainer has already looked at it).`);- continue;- }- await score(pr);- }+ trusted-orgs: scrapy,scrapy-plugins,scrapinghub,zytedata+ trusted-users: >-+ adnan-awan,akhterwahab,alexldide,ankitjavalkar,DiogoCastro,+ diogosuguimoto,emarondan,gutsytechster,haideriqbal1,+ hammadqureshi-web,hassaan-naushahi,hygorcluz,imhamzaa,kishan3,+ Nykakin,serhii73+ label: spam