Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

CI: add check for ASCII, disallow none ASCII code check-in (#21784)

Wang Qie13fe445e531
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml
index fb6d35d0e1..ef7992ee7a 100644
--- a/.github/workflows/sep-tests.yml
+++ b/.github/workflows/sep-tests.yml
@@ -17,11 +17,7 @@ on:
# We use pull_request_target so that GITEE_TOKEN (and other repo secrets) are available when building
# ragflow:nightly from a fork PR, while still running the full test suite on PRs labeled with `ci`.
pull_request:
- types: [synchronize, labeled]
- paths-ignore:
- - 'docs/**'
- - '*.md'
- - '*.mdx'
+ types: [opened, reopened, ready_for_review, synchronize, labeled]
schedule:
- cron: '0 16 * * *' # This schedule runs every 16:00:00Z(00:00:00+08:00)
@@ -67,6 +63,13 @@ jobs:
fetch-tags: true
allow-unsafe-pr-checkout: true
+ - name: Check added lines for non-ASCII characters
+ if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
+ env:
+ PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ run: go run ./tools/checkascii "$PR_BASE_SHA" "$PR_HEAD_SHA"
+
- name: Check workflow duplication
if: ${{ !cancelled() && !failure() }}
run: |
diff --git a/tools/checkascii/README.md b/tools/checkascii/README.md
new file mode 100644
index 0000000000..5a008566e0
--- /dev/null
+++ b/tools/checkascii/README.md
@@ -0,0 +1,41 @@
+# PR ASCII check
+
+`ragflow_preflight` rejects non-ASCII bytes on added lines in a PR's diff
+from its merge base. Unchanged lines and deleted lines do not fail the check.
+The preflight keeps its existing `ci` label requirement for non-draft PRs,
+including documentation-only changes.
+This is an ASCII rule, so it also rejects smart quotes, accented letters,
+emoji, and typographic dashes. Use ASCII punctuation and Unicode escapes
+for required string literals in application source code.
+
+Run locally from the repository root:
+
+```sh
+go run ./tools/checkascii origin/main HEAD
+bash build.sh --test ./tools/checkascii
+```
+
+The checker reads committed content rather than working-tree files. Failures
+produce a GitHub Actions annotation for every offending added line, with the
+file and new line number in both the annotation metadata and the visible error
+message (for example, `web/src/app.tsx:42: Non-ASCII characters on added line.`).
+Invalid Git references and other Git errors fail the check. Git binary patches are
+skipped. Renamed files are treated as additions, including moves out of
+excluded paths.
+
+Exclusions:
+
+- Go `*_test.go` files.
+- Frontend JavaScript/TypeScript `*.test.*` and `*.spec.*` files under `web/`,
+ files in `web/**/__tests__/` or `web/**/__mocks__/`, and `web/jest-setup.ts`.
+- `web/src/locales/*.ts`, except `web/src/locales/en.ts`.
+- `README_*.md` directly in the repository root.
+- `internal/agent/templates/*.json`: localized agent names and descriptions.
+- `internal/rag/res/ner.json` and `internal/rag/res/synonym.json`: multilingual
+ entity and synonym dictionaries.
+- Data files under a `testdata`, `fixtures`, or `golden` directory with extension
+ `.json`, `.txt`, `.md`, `.html`, `.csv`, `.tsv`, `.xml`, `.svg`, or `.b64`:
+ multilingual parser inputs, DSL examples, and expected results.
+
+Other source files remain checked, including fixture generators. English
+documentation, prompts, and regular application code have no blanket exclusion.
diff --git a/tools/checkascii/main.go b/tools/checkascii/main.go
new file mode 100644
index 0000000000..8908a3c5f0
--- /dev/null
+++ b/tools/checkascii/main.go
@@ -0,0 +1,141 @@
+// Command checkascii rejects non-ASCII bytes on lines added by a pull request.
+package main
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "os/signal"
+ "path"
+ "regexp"
+ "strconv"
+ "strings"
+)
+
+var hunkHeader = regexp.MustCompile(`^@@ -[0-9]+(?:,[0-9]+)? \+([0-9]+)(?:,[0-9]+)? @@`)
+var frontendTestFile = regexp.MustCompile(`\.(test|spec)\.([jt]sx?|[cm][jt]s)$`)
+
+func main() {
+ if len(os.Args) != 3 {
+ fmt.Fprintln(os.Stderr, "usage: checkascii <base-ref> <head-ref>")
+ os.Exit(2)
+ }
+ ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+ defer cancel()
+ if err := check(ctx, ".", os.Args[1], os.Args[2], os.Stdout); err != nil {
+ fmt.Fprintln(os.Stderr, err)
+ os.Exit(1)
+ }
+}
+
+func check(ctx context.Context, dir, base, head string, out io.Writer) error {
+ // Resolve refs before constructing the range so invalid refs fail closed.
+ refs := []string{base, head}
+ for i, ref := range refs {
+ commit, err := gitOutput(ctx, dir, "rev-parse", "--verify", "--end-of-options", ref+"^{commit}")
+ if err != nil {
+ return err
+ }
+ refs[i] = strings.TrimSpace(string(commit))
+ }
+ prRange := refs[0] + "..." + refs[1]
+ files, err := gitOutput(ctx, dir, "diff", "--no-ext-diff", "--no-textconv", "--no-renames", "--diff-filter=AMT", "--name-only", "-z", prRange, "--")
+ if err != nil {
+ return err
+ }
+ failures := 0
+ escape := strings.NewReplacer("%", "%25", "\r", "%0D", "\n", "%0A", ":", "%3A", ",", "%2C")
+ escapeMessage := strings.NewReplacer("%", "%25", "\r", "%0D", "\n", "%0A")
+ for _, name := range bytes.Split(files, []byte{0}) {
+ file := string(name)
+ if file == "" || isExcluded(file) {
+ continue
+ }
+ // Inspect committed PR content, independent of the checked-out merge tree.
+ // Git omits binary patches; renames are additions so moving a file cannot
+ // bypass the policy by moving it out of an excluded directory.
+ diff, err := gitOutput(ctx, dir, "diff", "--no-color", "--no-ext-diff", "--no-textconv", "--no-renames", "--unified=0", prRange, "--", file)
+ if err != nil {
+ return err
+ }
+ lineNumber := 0
+ for _, line := range bytes.Split(diff, []byte{'\n'}) {
+ if match := hunkHeader.FindSubmatch(line); match != nil {
+ lineNumber, err = strconv.Atoi(string(match[1]))
+ if err != nil {
+ return fmt.Errorf("parse diff line number for %q: %w", file, err)
+ }
+ continue
+ }
+ if lineNumber == 0 || len(line) == 0 {
+ continue
+ }
+ switch line[0] {
+ case '+':
+ for _, b := range line[1:] {
+ if b >= 0x80 {
+ if _, err := fmt.Fprintf(out, "::error file=%s,line=%d::%s:%d: Non-ASCII characters on added line. Use English ASCII text or Unicode escapes in string literals.\n", escape.Replace(file), lineNumber, escapeMessage.Replace(file), lineNumber); err != nil {
+ return err
+ }
+ failures++
+ break
+ }
+ }
+ lineNumber++
+ case ' ':
+ lineNumber++
+ }
+ }
+ }
+ if failures > 0 {
+ return fmt.Errorf("non-ASCII check failed: %d added line(s); see tools/checkascii/README.md for exclusions", failures)
+ }
+ return nil
+}
+
+func gitOutput(ctx context.Context, dir string, args ...string) ([]byte, error) {
+ cmd := exec.CommandContext(ctx, "git", append([]string{"--literal-pathspecs"}, args...)...)
+ cmd.Dir = dir
+ var stderr bytes.Buffer
+ cmd.Stderr = &stderr
+ output, err := cmd.Output()
+ if err != nil {
+ return nil, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
+ }
+ return output, nil
+}
+
+func isExcluded(file string) bool {
+ if strings.HasSuffix(file, "_test.go") {
+ return true
+ }
+ if strings.HasPrefix(file, "web/") && (frontendTestFile.MatchString(file) || strings.Contains(file, "/__tests__/") || strings.Contains(file, "/__mocks__/") || file == "web/jest-setup.ts") {
+ return true
+ }
+ if path.Dir(file) == "web/src/locales" && path.Ext(file) == ".ts" {
+ return path.Base(file) != "en.ts"
+ }
+ if path.Dir(file) == "." && strings.HasPrefix(file, "README_") && strings.HasSuffix(file, ".md") {
+ return true
+ }
+ if path.Dir(file) == "internal/agent/templates" && path.Ext(file) == ".json" {
+ return true
+ }
+ if file == "internal/rag/res/ner.json" || file == "internal/rag/res/synonym.json" {
+ return true
+ }
+ // Fixture data needs literal multilingual text. Source code in these
+ // directories is still checked unless it matches a test exclusion above.
+ switch path.Ext(file) {
+ case ".json", ".txt", ".md", ".html", ".csv", ".tsv", ".xml", ".svg", ".b64":
+ for _, part := range strings.Split(path.Dir(file), "/") {
+ if part == "testdata" || part == "fixtures" || part == "golden" {
+ return true
+ }
+ }
+ }
+ return false
+}
diff --git a/tools/checkascii/main_test.go b/tools/checkascii/main_test.go
new file mode 100644
index 0000000000..c33d07ad5b
--- /dev/null
+++ b/tools/checkascii/main_test.go
@@ -0,0 +1,192 @@
+package main
+
+import (
+ "bytes"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestCheck(t *testing.T) {
+ tests := []struct {
+ name string
+ file string
+ before string
+ after string
+ removed bool
+ lines []string
+ }{
+ {name: "ASCII edit beside existing Unicode", file: "code.go", before: "// \u4e2d\u6587\n// old\n", after: "// \u4e2d\u6587\n// new\n"},
+ {name: "removed Unicode line", file: "code.go", before: "// \u4e2d\u6587\n// ASCII\n", after: "// ASCII\n"},
+ {name: "removed file", file: "removed.go", before: "// \u4e2d\u6587\n", removed: true},
+ {name: "ASCII string escape", file: "code.go", after: `var text = "\u4e2d\u6587"`},
+ {name: "new code", file: "code.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
+ {name: "multiple hunks", file: "code.go", before: "one\ntwo\nthree\nfour\nfive\n", after: "one\n\u00e9\nthree\nfour\n\u2014\n", lines: []string{"2", "5"}},
+ {name: "no final newline", file: "code.go", after: "// \u4e2d\u6587", lines: []string{"1"}},
+ {name: "content resembles diff header", file: "code.go", after: "++ \u4e2d\u6587\n", lines: []string{"1"}},
+ {name: "long line", file: "long.txt", after: strings.Repeat("a", 128*1024) + "\u00e9\n", lines: []string{"1"}},
+ {name: "binary file", file: "image.bin", after: "\x00\xff\xfe\n"},
+ {name: "translated locale", file: "web/src/locales/zh.ts", after: "// \u4e2d\u6587\n"},
+ {name: "English locale", file: "web/src/locales/en.ts", after: "// \u2019\n", lines: []string{"1"}},
+ {name: "nested locale code", file: "web/src/locales/nested/helper.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
+ {name: "translated root README", file: "README_zh.md", after: "\u4e2d\u6587\n"},
+ {name: "English README", file: "README.md", after: "\u2014\n", lines: []string{"1"}},
+ {name: "nested translated README", file: "docs/README_zh.md", after: "\u4e2d\u6587\n", lines: []string{"1"}},
+ {name: "localized agent template", file: "internal/agent/templates/example.json", after: "{\"zh\": \"\u4e2d\u6587\"}\n"},
+ {name: "entity dictionary", file: "internal/rag/res/ner.json", after: "{\"\u4e2d\u6587\": \"entity\"}\n"},
+ {name: "synonym dictionary", file: "internal/rag/res/synonym.json", after: "{\"\u4e2d\u6587\": \"synonym\"}\n"},
+ {name: "other resource", file: "internal/rag/res/config.json", after: "{\"value\": \"\u4e2d\u6587\"}\n", lines: []string{"1"}},
+ {name: "parser input", file: "internal/parser/testdata/sample.zh.html", after: "<p>\u4e2d\u6587</p>\n"},
+ {name: "fixture data", file: "test/fixtures/example.json", after: "{\"value\": \"\u4e2d\u6587\"}\n"},
+ {name: "golden data", file: "internal/example/golden/result.txt", after: "\u4e2d\u6587\n"},
+ {name: "Go test code", file: "internal/example/example_test.go", after: "// \u4e2d\u6587\n"},
+ {name: "Go test in fixture directory", file: "internal/parser/testdata/example_test.go", after: "// \u4e2d\u6587\n"},
+ {name: "frontend TypeScript test", file: "web/src/utils/example.test.ts", after: "// \u4e2d\u6587\n"},
+ {name: "frontend React test", file: "web/src/components/example.test.tsx", after: "// \u4e2d\u6587\n"},
+ {name: "frontend JavaScript test", file: "web/src/utils/example.test.js", after: "// \u4e2d\u6587\n"},
+ {name: "frontend JSX spec", file: "web/src/components/example.spec.jsx", after: "// \u4e2d\u6587\n"},
+ {name: "frontend module spec", file: "web/src/utils/example.spec.mjs", after: "// \u4e2d\u6587\n"},
+ {name: "frontend grouped test", file: "web/src/utils/__tests__/example.ts", after: "// \u4e2d\u6587\n"},
+ {name: "frontend mock", file: "web/__mocks__/example.js", after: "// \u4e2d\u6587\n"},
+ {name: "frontend test setup", file: "web/jest-setup.ts", after: "// \u4e2d\u6587\n"},
+ {name: "frontend application code", file: "web/src/utils/example.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
+ {name: "frontend application test helper", file: "web/src/utils/test-helper.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
+ {name: "test-like directory", file: "web/src/utils/__tests__extra/example.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
+ {name: "test-like filename", file: "web/src/utils/example.test.ts.backup", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
+ {name: "fixture generator", file: "internal/parser/testdata/generate.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
+ {name: "filename with spaces and quotes", file: "code with \"quotes\".go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
+ {name: "literal pathspec", file: ":(exclude)file.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
+ {name: "annotation escaping", file: "code%,\n.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dir := initRepo(t)
+ if tt.before != "" {
+ writeFile(t, dir, tt.file, tt.before)
+ }
+ git(t, dir, "add", ".")
+ git(t, dir, "commit", "--allow-empty", "-qm", "base")
+ base := git(t, dir, "rev-parse", "HEAD")
+ if tt.removed {
+ if err := os.Remove(filepath.Join(dir, tt.file)); err != nil {
+ t.Fatal(err)
+ }
+ } else {
+ writeFile(t, dir, tt.file, tt.after)
+ }
+ git(t, dir, "add", ".")
+ git(t, dir, "commit", "-qm", "PR")
+ var out bytes.Buffer
+ err := check(t.Context(), dir, base, "HEAD", &out)
+ if (err != nil) != (len(tt.lines) > 0) {
+ t.Fatalf("check returned %v, output: %s", err, &out)
+ }
+ if got := strings.Count(out.String(), "::error file="); got != len(tt.lines) {
+ t.Fatalf("got %d annotations, want %d: %s", got, len(tt.lines), &out)
+ }
+ for _, line := range tt.lines {
+ if !strings.Contains(out.String(), ",line="+line+"::") {
+ t.Errorf("missing annotation at line %s: %s", line, &out)
+ }
+ file := strings.NewReplacer("%", "%25", "\r", "%0D", "\n", "%0A").Replace(tt.file)
+ if !strings.Contains(out.String(), "::"+file+":"+line+": Non-ASCII characters on added line.") {
+ t.Errorf("missing visible file and line %s:%s: %s", tt.file, line, &out)
+ }
+ }
+ if tt.name == "annotation escaping" && !strings.Contains(out.String(), "file=code%25%2C%0A.go,line=1::") {
+ t.Errorf("unescaped annotation path: %s", &out)
+ }
+ })
+ }
+}
+
+func TestCheckUsesMergeBaseAndCommittedHead(t *testing.T) {
+ dir := initRepo(t)
+ writeFile(t, dir, "code.go", "// ASCII\n")
+ git(t, dir, "add", ".")
+ git(t, dir, "commit", "-qm", "common ancestor")
+ ancestor := git(t, dir, "rev-parse", "HEAD")
+ writeFile(t, dir, "upstream.go", "// \u4e2d\u6587\n")
+ git(t, dir, "add", ".")
+ git(t, dir, "commit", "-qm", "target branch change")
+ base := git(t, dir, "rev-parse", "HEAD")
+ git(t, dir, "checkout", "-qb", "pr", ancestor)
+ writeFile(t, dir, "code.go", "// changed ASCII\n")
+ git(t, dir, "add", ".")
+ git(t, dir, "commit", "-qm", "PR change")
+ head := git(t, dir, "rev-parse", "HEAD")
+ writeFile(t, dir, "code.go", "// uncommitted \u4e2d\u6587\n")
+ var out bytes.Buffer
+ if err := check(t.Context(), dir, base, head, &out); err != nil {
+ t.Fatalf("check included changes outside the committed PR diff: %v: %s", err, &out)
+ }
+ git(t, dir, "add", ".")
+ git(t, dir, "commit", "-qm", "PR Unicode change")
+ head = git(t, dir, "rev-parse", "HEAD")
+ git(t, dir, "checkout", "--detach", base)
+ if err := check(t.Context(), dir, base, head, &out); err == nil {
+ t.Fatal("check ignored committed PR content absent from the working tree")
+ }
+}
+
+func TestCheckRenameOutOfExclusion(t *testing.T) {
+ dir := initRepo(t)
+ writeFile(t, dir, "web/src/locales/zh.ts", "// \u4e2d\u6587\n")
+ git(t, dir, "add", ".")
+ git(t, dir, "commit", "-qm", "base")
+ base := git(t, dir, "rev-parse", "HEAD")
+ git(t, dir, "mv", "web/src/locales/zh.ts", "web/src/locales/en.ts")
+ git(t, dir, "commit", "-qm", "rename")
+ var out bytes.Buffer
+ if err := check(t.Context(), dir, base, "HEAD", &out); err == nil {
+ t.Fatal("check allowed Unicode to be renamed into en.ts")
+ }
+ if !strings.Contains(out.String(), "file=web/src/locales/en.ts,line=1::") {
+ t.Fatalf("missing rename annotation: %s", &out)
+ }
+}
+
+func TestCheckInvalidRefs(t *testing.T) {
+ dir := initRepo(t)
+ git(t, dir, "commit", "--allow-empty", "-qm", "base")
+ for _, refs := range [][2]string{{"missing-base", "HEAD"}, {"HEAD", "missing-head"}} {
+ var out bytes.Buffer
+ if err := check(t.Context(), dir, refs[0], refs[1], &out); err == nil {
+ t.Errorf("check accepted invalid refs %v", refs)
+ }
+ }
+}
+
+func initRepo(t *testing.T) string {
+ t.Helper()
+ dir := t.TempDir()
+ git(t, dir, "init", "-q")
+ git(t, dir, "config", "user.name", "ASCII check test")
+ git(t, dir, "config", "user.email", "[email protected]")
+ return dir
+}
+
+func git(t *testing.T, dir string, args ...string) string {
+ t.Helper()
+ cmd := exec.Command("git", append([]string{"--literal-pathspecs", "-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false"}, args...)...)
+ cmd.Dir = dir
+ cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("git %v: %v: %s", args, err, out)
+ }
+ return strings.TrimSpace(string(out))
+}
+
+func writeFile(t *testing.T, dir, file, text string) {
+ t.Helper()
+ fullPath := filepath.Join(dir, file)
+ if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(fullPath, []byte(text), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}