RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
CI: add check for ASCII, disallow none ASCII code check-in (#21784)
Wang Qie13fe445e531
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.ymlindex fb6d35d0e1..ef7992ee7a 100644--- a/.github/workflows/sep-tests.yml+++ b/.github/workflows/sep-tests.yml@@ -17,11 +17,7 @@ on:# We use pull_request_target so that GITEE_TOKEN (and other repo secrets) are available when building# ragflow:nightly from a fork PR, while still running the full test suite on PRs labeled with `ci`.pull_request:- types: [synchronize, labeled]- paths-ignore:- - 'docs/**'- - '*.md'- - '*.mdx'+ types: [opened, reopened, ready_for_review, synchronize, labeled]schedule:- cron: '0 16 * * *' # This schedule runs every 16:00:00Z(00:00:00+08:00)@@ -67,6 +63,13 @@ jobs:fetch-tags: trueallow-unsafe-pr-checkout: true+ - name: Check added lines for non-ASCII characters+ if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}+ env:+ PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}+ PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}+ run: go run ./tools/checkascii "$PR_BASE_SHA" "$PR_HEAD_SHA"+- name: Check workflow duplicationif: ${{ !cancelled() && !failure() }}run: |diff --git a/tools/checkascii/README.md b/tools/checkascii/README.mdnew file mode 100644index 0000000000..5a008566e0--- /dev/null+++ b/tools/checkascii/README.md@@ -0,0 +1,41 @@+# PR ASCII check++`ragflow_preflight` rejects non-ASCII bytes on added lines in a PR's diff+from its merge base. Unchanged lines and deleted lines do not fail the check.+The preflight keeps its existing `ci` label requirement for non-draft PRs,+including documentation-only changes.+This is an ASCII rule, so it also rejects smart quotes, accented letters,+emoji, and typographic dashes. Use ASCII punctuation and Unicode escapes+for required string literals in application source code.++Run locally from the repository root:++```sh+go run ./tools/checkascii origin/main HEAD+bash build.sh --test ./tools/checkascii+```++The checker reads committed content rather than working-tree files. Failures+produce a GitHub Actions annotation for every offending added line, with the+file and new line number in both the annotation metadata and the visible error+message (for example, `web/src/app.tsx:42: Non-ASCII characters on added line.`).+Invalid Git references and other Git errors fail the check. Git binary patches are+skipped. Renamed files are treated as additions, including moves out of+excluded paths.++Exclusions:++- Go `*_test.go` files.+- Frontend JavaScript/TypeScript `*.test.*` and `*.spec.*` files under `web/`,+ files in `web/**/__tests__/` or `web/**/__mocks__/`, and `web/jest-setup.ts`.+- `web/src/locales/*.ts`, except `web/src/locales/en.ts`.+- `README_*.md` directly in the repository root.+- `internal/agent/templates/*.json`: localized agent names and descriptions.+- `internal/rag/res/ner.json` and `internal/rag/res/synonym.json`: multilingual+ entity and synonym dictionaries.+- Data files under a `testdata`, `fixtures`, or `golden` directory with extension+ `.json`, `.txt`, `.md`, `.html`, `.csv`, `.tsv`, `.xml`, `.svg`, or `.b64`:+ multilingual parser inputs, DSL examples, and expected results.++Other source files remain checked, including fixture generators. English+documentation, prompts, and regular application code have no blanket exclusion.diff --git a/tools/checkascii/main.go b/tools/checkascii/main.gonew file mode 100644index 0000000000..8908a3c5f0--- /dev/null+++ b/tools/checkascii/main.go@@ -0,0 +1,141 @@+// Command checkascii rejects non-ASCII bytes on lines added by a pull request.+package main++import (+ "bytes"+ "context"+ "fmt"+ "io"+ "os"+ "os/exec"+ "os/signal"+ "path"+ "regexp"+ "strconv"+ "strings"+)++var hunkHeader = regexp.MustCompile(`^@@ -[0-9]+(?:,[0-9]+)? \+([0-9]+)(?:,[0-9]+)? @@`)+var frontendTestFile = regexp.MustCompile(`\.(test|spec)\.([jt]sx?|[cm][jt]s)$`)++func main() {+ if len(os.Args) != 3 {+ fmt.Fprintln(os.Stderr, "usage: checkascii <base-ref> <head-ref>")+ os.Exit(2)+ }+ ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)+ defer cancel()+ if err := check(ctx, ".", os.Args[1], os.Args[2], os.Stdout); err != nil {+ fmt.Fprintln(os.Stderr, err)+ os.Exit(1)+ }+}++func check(ctx context.Context, dir, base, head string, out io.Writer) error {+ // Resolve refs before constructing the range so invalid refs fail closed.+ refs := []string{base, head}+ for i, ref := range refs {+ commit, err := gitOutput(ctx, dir, "rev-parse", "--verify", "--end-of-options", ref+"^{commit}")+ if err != nil {+ return err+ }+ refs[i] = strings.TrimSpace(string(commit))+ }+ prRange := refs[0] + "..." + refs[1]+ files, err := gitOutput(ctx, dir, "diff", "--no-ext-diff", "--no-textconv", "--no-renames", "--diff-filter=AMT", "--name-only", "-z", prRange, "--")+ if err != nil {+ return err+ }+ failures := 0+ escape := strings.NewReplacer("%", "%25", "\r", "%0D", "\n", "%0A", ":", "%3A", ",", "%2C")+ escapeMessage := strings.NewReplacer("%", "%25", "\r", "%0D", "\n", "%0A")+ for _, name := range bytes.Split(files, []byte{0}) {+ file := string(name)+ if file == "" || isExcluded(file) {+ continue+ }+ // Inspect committed PR content, independent of the checked-out merge tree.+ // Git omits binary patches; renames are additions so moving a file cannot+ // bypass the policy by moving it out of an excluded directory.+ diff, err := gitOutput(ctx, dir, "diff", "--no-color", "--no-ext-diff", "--no-textconv", "--no-renames", "--unified=0", prRange, "--", file)+ if err != nil {+ return err+ }+ lineNumber := 0+ for _, line := range bytes.Split(diff, []byte{'\n'}) {+ if match := hunkHeader.FindSubmatch(line); match != nil {+ lineNumber, err = strconv.Atoi(string(match[1]))+ if err != nil {+ return fmt.Errorf("parse diff line number for %q: %w", file, err)+ }+ continue+ }+ if lineNumber == 0 || len(line) == 0 {+ continue+ }+ switch line[0] {+ case '+':+ for _, b := range line[1:] {+ if b >= 0x80 {+ if _, err := fmt.Fprintf(out, "::error file=%s,line=%d::%s:%d: Non-ASCII characters on added line. Use English ASCII text or Unicode escapes in string literals.\n", escape.Replace(file), lineNumber, escapeMessage.Replace(file), lineNumber); err != nil {+ return err+ }+ failures+++ break+ }+ }+ lineNumber+++ case ' ':+ lineNumber+++ }+ }+ }+ if failures > 0 {+ return fmt.Errorf("non-ASCII check failed: %d added line(s); see tools/checkascii/README.md for exclusions", failures)+ }+ return nil+}++func gitOutput(ctx context.Context, dir string, args ...string) ([]byte, error) {+ cmd := exec.CommandContext(ctx, "git", append([]string{"--literal-pathspecs"}, args...)...)+ cmd.Dir = dir+ var stderr bytes.Buffer+ cmd.Stderr = &stderr+ output, err := cmd.Output()+ if err != nil {+ return nil, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))+ }+ return output, nil+}++func isExcluded(file string) bool {+ if strings.HasSuffix(file, "_test.go") {+ return true+ }+ if strings.HasPrefix(file, "web/") && (frontendTestFile.MatchString(file) || strings.Contains(file, "/__tests__/") || strings.Contains(file, "/__mocks__/") || file == "web/jest-setup.ts") {+ return true+ }+ if path.Dir(file) == "web/src/locales" && path.Ext(file) == ".ts" {+ return path.Base(file) != "en.ts"+ }+ if path.Dir(file) == "." && strings.HasPrefix(file, "README_") && strings.HasSuffix(file, ".md") {+ return true+ }+ if path.Dir(file) == "internal/agent/templates" && path.Ext(file) == ".json" {+ return true+ }+ if file == "internal/rag/res/ner.json" || file == "internal/rag/res/synonym.json" {+ return true+ }+ // Fixture data needs literal multilingual text. Source code in these+ // directories is still checked unless it matches a test exclusion above.+ switch path.Ext(file) {+ case ".json", ".txt", ".md", ".html", ".csv", ".tsv", ".xml", ".svg", ".b64":+ for _, part := range strings.Split(path.Dir(file), "/") {+ if part == "testdata" || part == "fixtures" || part == "golden" {+ return true+ }+ }+ }+ return false+}diff --git a/tools/checkascii/main_test.go b/tools/checkascii/main_test.gonew file mode 100644index 0000000000..c33d07ad5b--- /dev/null+++ b/tools/checkascii/main_test.go@@ -0,0 +1,192 @@+package main++import (+ "bytes"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "testing"+)++func TestCheck(t *testing.T) {+ tests := []struct {+ name string+ file string+ before string+ after string+ removed bool+ lines []string+ }{+ {name: "ASCII edit beside existing Unicode", file: "code.go", before: "// \u4e2d\u6587\n// old\n", after: "// \u4e2d\u6587\n// new\n"},+ {name: "removed Unicode line", file: "code.go", before: "// \u4e2d\u6587\n// ASCII\n", after: "// ASCII\n"},+ {name: "removed file", file: "removed.go", before: "// \u4e2d\u6587\n", removed: true},+ {name: "ASCII string escape", file: "code.go", after: `var text = "\u4e2d\u6587"`},+ {name: "new code", file: "code.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},+ {name: "multiple hunks", file: "code.go", before: "one\ntwo\nthree\nfour\nfive\n", after: "one\n\u00e9\nthree\nfour\n\u2014\n", lines: []string{"2", "5"}},+ {name: "no final newline", file: "code.go", after: "// \u4e2d\u6587", lines: []string{"1"}},+ {name: "content resembles diff header", file: "code.go", after: "++ \u4e2d\u6587\n", lines: []string{"1"}},+ {name: "long line", file: "long.txt", after: strings.Repeat("a", 128*1024) + "\u00e9\n", lines: []string{"1"}},+ {name: "binary file", file: "image.bin", after: "\x00\xff\xfe\n"},+ {name: "translated locale", file: "web/src/locales/zh.ts", after: "// \u4e2d\u6587\n"},+ {name: "English locale", file: "web/src/locales/en.ts", after: "// \u2019\n", lines: []string{"1"}},+ {name: "nested locale code", file: "web/src/locales/nested/helper.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},+ {name: "translated root README", file: "README_zh.md", after: "\u4e2d\u6587\n"},+ {name: "English README", file: "README.md", after: "\u2014\n", lines: []string{"1"}},+ {name: "nested translated README", file: "docs/README_zh.md", after: "\u4e2d\u6587\n", lines: []string{"1"}},+ {name: "localized agent template", file: "internal/agent/templates/example.json", after: "{\"zh\": \"\u4e2d\u6587\"}\n"},+ {name: "entity dictionary", file: "internal/rag/res/ner.json", after: "{\"\u4e2d\u6587\": \"entity\"}\n"},+ {name: "synonym dictionary", file: "internal/rag/res/synonym.json", after: "{\"\u4e2d\u6587\": \"synonym\"}\n"},+ {name: "other resource", file: "internal/rag/res/config.json", after: "{\"value\": \"\u4e2d\u6587\"}\n", lines: []string{"1"}},+ {name: "parser input", file: "internal/parser/testdata/sample.zh.html", after: "<p>\u4e2d\u6587</p>\n"},+ {name: "fixture data", file: "test/fixtures/example.json", after: "{\"value\": \"\u4e2d\u6587\"}\n"},+ {name: "golden data", file: "internal/example/golden/result.txt", after: "\u4e2d\u6587\n"},+ {name: "Go test code", file: "internal/example/example_test.go", after: "// \u4e2d\u6587\n"},+ {name: "Go test in fixture directory", file: "internal/parser/testdata/example_test.go", after: "// \u4e2d\u6587\n"},+ {name: "frontend TypeScript test", file: "web/src/utils/example.test.ts", after: "// \u4e2d\u6587\n"},+ {name: "frontend React test", file: "web/src/components/example.test.tsx", after: "// \u4e2d\u6587\n"},+ {name: "frontend JavaScript test", file: "web/src/utils/example.test.js", after: "// \u4e2d\u6587\n"},+ {name: "frontend JSX spec", file: "web/src/components/example.spec.jsx", after: "// \u4e2d\u6587\n"},+ {name: "frontend module spec", file: "web/src/utils/example.spec.mjs", after: "// \u4e2d\u6587\n"},+ {name: "frontend grouped test", file: "web/src/utils/__tests__/example.ts", after: "// \u4e2d\u6587\n"},+ {name: "frontend mock", file: "web/__mocks__/example.js", after: "// \u4e2d\u6587\n"},+ {name: "frontend test setup", file: "web/jest-setup.ts", after: "// \u4e2d\u6587\n"},+ {name: "frontend application code", file: "web/src/utils/example.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},+ {name: "frontend application test helper", file: "web/src/utils/test-helper.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},+ {name: "test-like directory", file: "web/src/utils/__tests__extra/example.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},+ {name: "test-like filename", file: "web/src/utils/example.test.ts.backup", after: "// \u4e2d\u6587\n", lines: []string{"1"}},+ {name: "fixture generator", file: "internal/parser/testdata/generate.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},+ {name: "filename with spaces and quotes", file: "code with \"quotes\".go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},+ {name: "literal pathspec", file: ":(exclude)file.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},+ {name: "annotation escaping", file: "code%,\n.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ dir := initRepo(t)+ if tt.before != "" {+ writeFile(t, dir, tt.file, tt.before)+ }+ git(t, dir, "add", ".")+ git(t, dir, "commit", "--allow-empty", "-qm", "base")+ base := git(t, dir, "rev-parse", "HEAD")+ if tt.removed {+ if err := os.Remove(filepath.Join(dir, tt.file)); err != nil {+ t.Fatal(err)+ }+ } else {+ writeFile(t, dir, tt.file, tt.after)+ }+ git(t, dir, "add", ".")+ git(t, dir, "commit", "-qm", "PR")+ var out bytes.Buffer+ err := check(t.Context(), dir, base, "HEAD", &out)+ if (err != nil) != (len(tt.lines) > 0) {+ t.Fatalf("check returned %v, output: %s", err, &out)+ }+ if got := strings.Count(out.String(), "::error file="); got != len(tt.lines) {+ t.Fatalf("got %d annotations, want %d: %s", got, len(tt.lines), &out)+ }+ for _, line := range tt.lines {+ if !strings.Contains(out.String(), ",line="+line+"::") {+ t.Errorf("missing annotation at line %s: %s", line, &out)+ }+ file := strings.NewReplacer("%", "%25", "\r", "%0D", "\n", "%0A").Replace(tt.file)+ if !strings.Contains(out.String(), "::"+file+":"+line+": Non-ASCII characters on added line.") {+ t.Errorf("missing visible file and line %s:%s: %s", tt.file, line, &out)+ }+ }+ if tt.name == "annotation escaping" && !strings.Contains(out.String(), "file=code%25%2C%0A.go,line=1::") {+ t.Errorf("unescaped annotation path: %s", &out)+ }+ })+ }+}++func TestCheckUsesMergeBaseAndCommittedHead(t *testing.T) {+ dir := initRepo(t)+ writeFile(t, dir, "code.go", "// ASCII\n")+ git(t, dir, "add", ".")+ git(t, dir, "commit", "-qm", "common ancestor")+ ancestor := git(t, dir, "rev-parse", "HEAD")+ writeFile(t, dir, "upstream.go", "// \u4e2d\u6587\n")+ git(t, dir, "add", ".")+ git(t, dir, "commit", "-qm", "target branch change")+ base := git(t, dir, "rev-parse", "HEAD")+ git(t, dir, "checkout", "-qb", "pr", ancestor)+ writeFile(t, dir, "code.go", "// changed ASCII\n")+ git(t, dir, "add", ".")+ git(t, dir, "commit", "-qm", "PR change")+ head := git(t, dir, "rev-parse", "HEAD")+ writeFile(t, dir, "code.go", "// uncommitted \u4e2d\u6587\n")+ var out bytes.Buffer+ if err := check(t.Context(), dir, base, head, &out); err != nil {+ t.Fatalf("check included changes outside the committed PR diff: %v: %s", err, &out)+ }+ git(t, dir, "add", ".")+ git(t, dir, "commit", "-qm", "PR Unicode change")+ head = git(t, dir, "rev-parse", "HEAD")+ git(t, dir, "checkout", "--detach", base)+ if err := check(t.Context(), dir, base, head, &out); err == nil {+ t.Fatal("check ignored committed PR content absent from the working tree")+ }+}++func TestCheckRenameOutOfExclusion(t *testing.T) {+ dir := initRepo(t)+ writeFile(t, dir, "web/src/locales/zh.ts", "// \u4e2d\u6587\n")+ git(t, dir, "add", ".")+ git(t, dir, "commit", "-qm", "base")+ base := git(t, dir, "rev-parse", "HEAD")+ git(t, dir, "mv", "web/src/locales/zh.ts", "web/src/locales/en.ts")+ git(t, dir, "commit", "-qm", "rename")+ var out bytes.Buffer+ if err := check(t.Context(), dir, base, "HEAD", &out); err == nil {+ t.Fatal("check allowed Unicode to be renamed into en.ts")+ }+ if !strings.Contains(out.String(), "file=web/src/locales/en.ts,line=1::") {+ t.Fatalf("missing rename annotation: %s", &out)+ }+}++func TestCheckInvalidRefs(t *testing.T) {+ dir := initRepo(t)+ git(t, dir, "commit", "--allow-empty", "-qm", "base")+ for _, refs := range [][2]string{{"missing-base", "HEAD"}, {"HEAD", "missing-head"}} {+ var out bytes.Buffer+ if err := check(t.Context(), dir, refs[0], refs[1], &out); err == nil {+ t.Errorf("check accepted invalid refs %v", refs)+ }+ }+}++func initRepo(t *testing.T) string {+ t.Helper()+ dir := t.TempDir()+ git(t, dir, "init", "-q")+ git(t, dir, "config", "user.name", "ASCII check test")+ git(t, dir, "config", "user.email", "[email protected]")+ return dir+}++func git(t *testing.T, dir string, args ...string) string {+ t.Helper()+ cmd := exec.Command("git", append([]string{"--literal-pathspecs", "-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false"}, args...)...)+ cmd.Dir = dir+ cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")+ out, err := cmd.CombinedOutput()+ if err != nil {+ t.Fatalf("git %v: %v: %s", args, err, out)+ }+ return strings.TrimSpace(string(out))+}++func writeFile(t *testing.T, dir, file, text string) {+ t.Helper()+ fullPath := filepath.Join(dir, file)+ if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(fullPath, []byte(text), 0o644); err != nil {+ t.Fatal(err)+ }+}