A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Scan every pull request for secrets and personal data
Fei24118b8eac7a6463
diff --git a/.github/workflows/test.yml b/.github/workflows/test.ymlindex aafe8688..8bcc79de 100644--- a/.github/workflows/test.yml+++ b/.github/workflows/test.yml@@ -40,13 +40,27 @@ jobs:- run: node scripts/build-client.mjs --check- run: npm run lint+ scan:+ name: Leak scan+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v4+ - uses: actions/setup-node@v4+ with:+ node-version: 24+ # Scan exactly the tracked files, as they would be published.+ - run: mkdir -p "$RUNNER_TEMP/tree" && git archive HEAD | tar -x -C "$RUNNER_TEMP/tree"+ - run: node scripts/scan.mjs "$RUNNER_TEMP/tree"+ env:+ SCAN_EXTRA_PATTERNS: ${{ secrets.SCAN_EXTRA_PATTERNS }}+# The one check that branch rules require. A job whose needs failed is skipped,# and GitHub counts a skipped required check as passed, so this job always runs- # and fails unless every matrix entry passed.+ # and fails unless every matrix entry and the leak scan passed.result:name: Test result- needs: test+ needs: [test, scan]if: always()runs-on: ubuntu-lateststeps:- - run: test "${{ needs.test.result }}" = success+ - run: test "${{ needs.test.result }}" = success && test "${{ needs.scan.result }}" = successdiff --git a/.gitignore b/.gitignoreindex 07e2f8cc..3dc75482 100644--- a/.gitignore+++ b/.gitignore@@ -9,3 +9,6 @@ node_modules/.dsh-home//shots/.playwright-mcp/++# Extra patterns for scripts/scan.mjs that must not be named in the repository.+.scan-extradiff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex b49e5cba..cc28361c 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -130,7 +130,13 @@ node scripts/build-client.mjs --check # client.js matches src/clientnpm run lint # oxlint```-CI runs the same three checks on every pull request, on Linux and Windows.+CI runs the same three checks on every pull request, on Linux and Windows. It also scans the tracked files for keys, tokens, personal paths and e-mail addresses, and for material that must stay local. To run the scan yourself, from the repository root:++```sh+mkdir -p /tmp/ds-bot-tree && git archive HEAD | tar -x -C /tmp/ds-bot-tree && node scripts/scan.mjs /tmp/ds-bot-tree+```++A "must fix" finding fails CI; "can stay" findings are for reading. The branch rules require the single check **Test result**, which passes only when all the checks pass.For a change to the server side (`index.js`, `src/host/`), also run the smoke test. It starts the mock model and a DSH Web with the plugin linked from your checkout, and walks through the main flows (Bots, group chats, conversation parts, condensing, settings). `CONTEXT_WINDOW` and `CHECK_RATIO` give the mock models a small window, so a few long replies reach the check line:diff --git a/CONTRIBUTING.zh.md b/CONTRIBUTING.zh.mdindex d8e32094..75f4d0b9 100644--- a/CONTRIBUTING.zh.md+++ b/CONTRIBUTING.zh.md@@ -130,7 +130,13 @@ node scripts/build-client.mjs --check # client.js 与 src/client 一致npm run lint # oxlint```-CI 在每个 Pull Request 上,在 Linux 和 Windows 上跑同样的三项检查。+CI 在每个 Pull Request 上,在 Linux 和 Windows 上跑同样的三项检查。它还会扫描入库的文件,查找密钥、token、个人路径和邮箱,以及只应留在本地的材料。想自己扫描,在仓库根目录运行:++```sh+mkdir -p /tmp/ds-bot-tree && git archive HEAD | tar -x -C /tmp/ds-bot-tree && node scripts/scan.mjs /tmp/ds-bot-tree+```++「must fix」的结果会让 CI 失败;「can stay」的结果仅供阅读。分支规则只要求一项检查 **Test result**,所有检查都通过它才通过。改服务端(`index.js`、`src/host/`)时,还要跑冒烟测试。它启动模拟模型和一个链接到你本地插件的 DSH Web,走一遍主要流程(Bot、群聊、对话分段、压缩、设置)。`CONTEXT_WINDOW` 和 `CHECK_RATIO` 让模拟模型的上下文窗口变小,几条长回复就能到达检查线:diff --git a/scripts/scan.mjs b/scripts/scan.mjsnew file mode 100644index 00000000..3863e7f5--- /dev/null+++ b/scripts/scan.mjs@@ -0,0 +1,184 @@+#!/usr/bin/env node+// Scans a copy of the repository tree for secrets, personal data and private material.+//+// git archive HEAD | tar -x -C /tmp/tree && node scripts/scan.mjs /tmp/tree [--json <file>] [--all]+//+// Findings are "must fix" (exit code 1) or "can stay" (read and accept). Matched secrets+// are never printed in full. --all prints every location instead of the first few per rule.+//+// Extra "must fix" patterns, one regular expression per line, come from the SCAN_EXTRA_PATTERNS+// environment variable and from a `.scan-extra` file at the root of the checkout (ignored by+// Git). They hold markers that must not appear in this repository and must not be named in it+// either, such as a maintainer's machine or account names.+import { existsSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'+import { dirname, join, relative } from 'node:path'+import { fileURLToPath } from 'node:url'++const MUST = 'must fix'+const KEEP = 'can stay'++const SELF = 'scripts/scan.mjs'+const PLACEHOLDER = /mock|test|fake|dummy|example|placeholder|should-not|xxx|your|<|\$\{|redact|sample/i+const VENDORS = /\b(qwen|minimax|stepfun|kimi|moonshot|zhipu|glm|mimo|anthropic|claude|openai|gemini|mistral|xai|zai)\b/i+// Files that define the look (characters, inks, logos, themes). Vendor names there are+// brand references; elsewhere they name model routes and can stay.+const LOOK_FILES = /(^|\/)(src\/client\/(inks|characters|mark|themes|styles)\.js|dev\/(mascot-outlines|characters)\.mjs)$/+const NOT_PUBLIC = /(^|[\s(`'"]|\.\.\/)(spec|research|backup|shots|attachments|workspace)\/|\.dsh-home/+// Migration code and its tests must name the legacy ids (old package name, old theme and+// color keys) to move saved data over; there, and on lines that say "legacy" (旧版), old names stay.+const LEGACY_FILES = /(^|\/)(src\/client\/storage\.js|test\/legacy[\w-]*\.test\.js)$/++const extra = extraPatterns()++const TEXT_RULES = [+ // Secrets.+ { id: 'secret-openai-style', level: MUST, re: /\bsk-[A-Za-z0-9_-]{20,}/g, secret: true },+ { id: 'secret-github', level: MUST, re: /\b(gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{20,})/g, secret: true },+ { id: 'secret-aws', level: MUST, re: /\bAKIA[0-9A-Z]{16}\b/g, secret: true },+ { id: 'secret-google', level: MUST, re: /\bAIza[0-9A-Za-z_-]{35}\b/g, secret: true },+ { id: 'secret-slack', level: MUST, re: /\bxox[abposr]-[A-Za-z0-9-]{10,}/g, secret: true },+ { id: 'secret-private-key', level: MUST, re: /-----BEGIN [A-Z ]*PRIVATE KEY-----/g, secret: true },+ { id: 'secret-bearer', level: MUST, re: /\bBearer\s+[A-Za-z0-9._~+/-]{24,}=*/g, secret: true },+ {+ id: 'secret-assignment', level: MUST, secret: true,+ re: /\b(api[_-]?key|secret|token|password|passwd)["']?\s*[:=]\s*["'][A-Za-z0-9_\-./+=]{16,}["']/gi,+ },+ // Personal data.+ { id: 'personal-path', level: MUST, re: /\/home\/(?!<|user\b|runner\b|node\b)[a-z_][a-z0-9_-]*\/|\/Users\/(?!<|Shared\b)[A-Za-z][\w.-]*\/|[A-Za-z]:\\Users\\(?!<|Public\b)[^\\\s`"'<>]+/g },+ { id: 'personal-email', level: MUST, re: /\b[\w.+-]+@(qq|163|126|gmail|outlook|hotmail|foxmail|icloud|yahoo)\.(com|cn)\b/gi },+ { id: 'other-email', level: KEEP, re: /\b[\w.+-]+@(?!(\d|example\.|users\.noreply\.github\.com|localhost|(qq|163|126|gmail|outlook|hotmail|foxmail|icloud|yahoo)\.))[\w-]+\.[a-z]{2,}\b/gi },+ ...(extra ? [{ id: 'extra-pattern', level: MUST, re: extra, legacyOk: true }] : []),+ // Only original material: no vendor brands in the look, no third-party logos.+ { id: 'vendor-look-identifier', level: MUST, re: /\b(KIMI|MINIMAX|STEPFUN|QWEN|MIMO|ZAI|ZHIPU|GLM|DEEPSEEK)_(BODY|WAVE|STAR|MARK|LOGO|INK|INKS|COLOR|COLORS)\b|\bLAB_(INKS|COLORS)\b/g, legacyOk: true },+ { id: 'vendor-in-look-file', level: MUST, re: new RegExp(VENDORS.source, 'gi'), files: LOOK_FILES, legacyOk: true },+ { id: 'vendor-brand', level: MUST, re: /brand[^\n]{0,40}\b(qwen|minimax|stepfun|kimi|zhipu|mimo|deepseek)\b|\b(qwen|minimax|stepfun|kimi|zhipu|mimo)\b[^\n]{0,40}brand/gi, legacyOk: true },+ // A test may name the logo to check that the look no longer uses it.+ { id: 'third-party-logo', level: MUST, re: /WhaleLogo|favicon\.svg/g, notFiles: /(^|\/)test\// },+ { id: 'vendor-name', level: KEEP, re: new RegExp(VENDORS.source, 'gi'), notFiles: LOOK_FILES, summary: true },+ // Former names of the project and the package.+ { id: 'old-project-name', level: MUST, re: /DSH Bot\b/g, legacyOk: true },+ { id: 'old-package-name', level: MUST, re: /\bdsh-bot\b/g, legacyOk: true },+ { id: 'legacy-name-in-migration', level: KEEP, re: /\bdsh-bot\b|DeepSeek Bot\b/gi, onlyLegacy: true, summary: true },+ // Drafts and local-only material.+ { id: 'draft-marker', level: MUST, re: /TBD\(|DRAFT for|草稿/g },+ { id: 'not-public-path-in-docs', level: MUST, re: new RegExp(NOT_PUBLIC.source, 'g'), files: /\.md$/ },+ { id: 'not-public-path-in-code', level: KEEP, re: new RegExp(NOT_PUBLIC.source, 'g'), notFiles: /\.md$/ },+ { id: 'public-ip', level: KEEP, re: /\b(?!(127|10|0)\.)(?!192\.168\.)(?!172\.(1[6-9]|2\d|3[01])\.)(?!192\.0\.2\.|198\.51\.100\.|203\.0\.113\.|169\.254\.)(\d{1,3}\.){3}\d{1,3}\b/g, unless: /version|v\d|@\d/i },+]++const FORBIDDEN_FILES = [+ { re: /^(spec|research|backup|shots|attachments|workspace|deepseek-harness)\//, why: 'local-only folder' },+ { re: /(^|\/)\.dsh-(home|test)/, why: 'DSH data' },+ { re: /(^|\/)node_modules\//, why: 'installed packages' },+ { re: /\.(env|pem|key|p12|sqlite|db)$/, why: 'secret or data file' },+ { re: /^\.scan-extra$/, why: 'local scan patterns' },+]++const args = process.argv.slice(2)+const root = args.find(a => !a.startsWith('--') && args[args.indexOf(a) - 1] !== '--json')+if (!root) {+ console.error('usage: node scripts/scan.mjs <dir> [--json <file>] [--all]')+ process.exit(2)+}+const jsonOut = args.includes('--json') ? args[args.indexOf('--json') + 1] : null+const showAll = args.includes('--all')++const findings = []+const files = walk(root).map(path => relative(root, path).split('\\').join('/')).sort()++for (const file of files) {+ for (const rule of FORBIDDEN_FILES) {+ if (rule.re.test(file)) add(MUST, 'forbidden-file', file, 0, rule.why)+ }+ // The scanner names the patterns it looks for.+ if (file === SELF) continue+ const data = readFileSync(join(root, file))+ if (data.includes(0)) continue+ const lines = data.toString('utf8').split('\n')+ lines.forEach((line, index) => {+ for (const rule of TEXT_RULES) {+ if (rule.files && !rule.files.test(file)) continue+ if (rule.notFiles && rule.notFiles.test(file)) continue+ if (rule.unless && rule.unless.test(line)) continue+ const legacy = LEGACY_FILES.test(file) || /legacy|旧版/i.test(line)+ if (rule.legacyOk && legacy) continue+ if (rule.onlyLegacy && !legacy) continue+ rule.re.lastIndex = 0+ for (const match of line.matchAll(rule.re)) {+ if (rule.secret && PLACEHOLDER.test(match[0])) continue+ const shown = rule.secret ? `${match[0].slice(0, 6)}… (${match[0].length} chars)` : snippet(line, match.index, match[0].length)+ add(rule.level, rule.id, file, index + 1, shown, rule.summary)+ }+ }+ })+}++// Package checks: the license and the fields that npm and DSH read.+if (!files.includes('LICENSE')) add(MUST, 'license', 'LICENSE', 0, 'no LICENSE at the root')+if (!files.includes('ds-bot/package.json')) add(MUST, 'plugin-folder', '.', 0, 'no ds-bot/package.json')+else {+ const pkg = JSON.parse(readFileSync(join(root, 'ds-bot/package.json'), 'utf8'))+ if (pkg.name !== 'ds-bot') add(MUST, 'package', 'ds-bot/package.json', 0, `name is "${pkg.name}", expected "ds-bot"`)+ if (!files.includes('ds-bot/LICENSE')) add(MUST, 'package', 'ds-bot/LICENSE', 0, 'the package has no LICENSE file')+ if (!(pkg.files ?? []).includes('LICENSE')) add(MUST, 'package', 'ds-bot/package.json', 0, '"files" does not list LICENSE')+ for (const field of ['repository', 'homepage', 'bugs']) {+ if (!pkg[field]) add(MUST, 'package', 'ds-bot/package.json', 0, `no "${field}" field`)+ }+ for (const [name, range] of Object.entries(pkg.peerDependencies ?? {})) {+ if (range !== '*') add(KEEP, 'package', 'ds-bot/package.json', 0, `peer ${name} is "${range}", not "*"`)+ }+}+for (const name of ['README.md', 'README.zh.md', 'CONTRIBUTING.md', 'CHANGELOG.md', 'AGENTS.md', '.gitignore']) {+ if (!files.includes(name)) add(MUST, 'root-file', name, 0, 'missing at the root')+}++report()++function extraPatterns() {+ const local = join(dirname(fileURLToPath(import.meta.url)), '..', '.scan-extra')+ const lines = [process.env.SCAN_EXTRA_PATTERNS ?? '', existsSync(local) ? readFileSync(local, 'utf8') : '']+ .flatMap(text => text.split(/\r?\n/))+ .map(line => line.trim())+ .filter(line => line && !line.startsWith('#'))+ return lines.length ? new RegExp(lines.map(line => `(?:${line})`).join('|'), 'gi') : null+}++function add(level, id, file, line, text, summary = false) {+ findings.push({ level, id, file, line, text, summary })+}++function report() {+ const byLevel = level => findings.filter(f => f.level === level)+ console.log(`scanned ${files.length} files in ${root}${extra ? ' (with extra patterns)' : ''}`)+ for (const level of [MUST, KEEP]) {+ const list = byLevel(level)+ console.log(`\n== ${level}: ${list.length}`)+ const rules = Map.groupBy(list, f => f.id)+ for (const [id, items] of rules) {+ const perFile = Map.groupBy(items, f => f.file)+ console.log(`- ${id}: ${items.length} in ${perFile.size} files`)+ if (items[0].summary) {+ for (const [file, hits] of perFile) console.log(` ${file}: ${hits.length}`)+ continue+ }+ const shown = showAll ? items : items.slice(0, 6)+ for (const f of shown) console.log(` ${f.file}${f.line ? `:${f.line}` : ''} ${f.text}`)+ if (shown.length < items.length) console.log(` … ${items.length - shown.length} more (--all or --json)`)+ }+ }+ if (jsonOut) writeFileSync(jsonOut, JSON.stringify(findings, null, 2))+ process.exit(byLevel(MUST).length > 0 ? 1 : 0)+}++function snippet(line, at, length) {+ const start = Math.max(0, at - 40)+ const end = Math.min(line.length, at + length + 40)+ return `${start > 0 ? '…' : ''}${line.slice(start, end).trim()}${end < line.length ? '…' : ''}`+}++function walk(dir) {+ return readdirSync(dir).flatMap(name => {+ const path = join(dir, name)+ return statSync(path).isDirectory() ? walk(path) : [path]+ })+}