Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Revert "CI: add check for ASCII, disallow none ASCII code check-in" (#21873)

Wang Qi543f57ea9716
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml
index 52ce71b4be..01d950e335 100644
--- a/.github/workflows/sep-tests.yml
+++ b/.github/workflows/sep-tests.yml
@@ -17,7 +17,11 @@ on:
# We use pull_request_target so that GITEE_TOKEN (and other repo secrets) are available when building
# ragflow:nightly from a fork PR, while still running the full test suite on PRs labeled with `ci`.
pull_request:
- types: [opened, reopened, ready_for_review, synchronize, labeled]
+ types: [synchronize, labeled]
+ paths-ignore:
+ - 'docs/**'
+ - '*.md'
+ - '*.mdx'
schedule:
- cron: '0 16 * * *' # This schedule runs every 16:00:00Z(00:00:00+08:00)
@@ -63,13 +67,6 @@ jobs:
fetch-tags: true
allow-unsafe-pr-checkout: true
- - name: Check added lines for non-ASCII characters
- if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
- env:
- PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
- PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
- run: go run ./tools/checkascii "$PR_BASE_SHA" "$PR_HEAD_SHA"
-
- name: Check workflow duplication
if: ${{ !cancelled() && !failure() }}
run: |
diff --git a/tools/checkascii/README.md b/tools/checkascii/README.md
deleted file mode 100644
index 5a008566e0..0000000000
--- a/tools/checkascii/README.md
+++ /dev/null
@@ -1,41 +0,0 @@
-# PR ASCII check
-
-`ragflow_preflight` rejects non-ASCII bytes on added lines in a PR's diff
-from its merge base. Unchanged lines and deleted lines do not fail the check.
-The preflight keeps its existing `ci` label requirement for non-draft PRs,
-including documentation-only changes.
-This is an ASCII rule, so it also rejects smart quotes, accented letters,
-emoji, and typographic dashes. Use ASCII punctuation and Unicode escapes
-for required string literals in application source code.
-
-Run locally from the repository root:
-
-```sh
-go run ./tools/checkascii origin/main HEAD
-bash build.sh --test ./tools/checkascii
-```
-
-The checker reads committed content rather than working-tree files. Failures
-produce a GitHub Actions annotation for every offending added line, with the
-file and new line number in both the annotation metadata and the visible error
-message (for example, `web/src/app.tsx:42: Non-ASCII characters on added line.`).
-Invalid Git references and other Git errors fail the check. Git binary patches are
-skipped. Renamed files are treated as additions, including moves out of
-excluded paths.
-
-Exclusions:
-
-- Go `*_test.go` files.
-- Frontend JavaScript/TypeScript `*.test.*` and `*.spec.*` files under `web/`,
- files in `web/**/__tests__/` or `web/**/__mocks__/`, and `web/jest-setup.ts`.
-- `web/src/locales/*.ts`, except `web/src/locales/en.ts`.
-- `README_*.md` directly in the repository root.
-- `internal/agent/templates/*.json`: localized agent names and descriptions.
-- `internal/rag/res/ner.json` and `internal/rag/res/synonym.json`: multilingual
- entity and synonym dictionaries.
-- Data files under a `testdata`, `fixtures`, or `golden` directory with extension
- `.json`, `.txt`, `.md`, `.html`, `.csv`, `.tsv`, `.xml`, `.svg`, or `.b64`:
- multilingual parser inputs, DSL examples, and expected results.
-
-Other source files remain checked, including fixture generators. English
-documentation, prompts, and regular application code have no blanket exclusion.
diff --git a/tools/checkascii/main.go b/tools/checkascii/main.go
deleted file mode 100644
index 8908a3c5f0..0000000000
--- a/tools/checkascii/main.go
+++ /dev/null
@@ -1,141 +0,0 @@
-// Command checkascii rejects non-ASCII bytes on lines added by a pull request.
-package main
-
-import (
- "bytes"
- "context"
- "fmt"
- "io"
- "os"
- "os/exec"
- "os/signal"
- "path"
- "regexp"
- "strconv"
- "strings"
-)
-
-var hunkHeader = regexp.MustCompile(`^@@ -[0-9]+(?:,[0-9]+)? \+([0-9]+)(?:,[0-9]+)? @@`)
-var frontendTestFile = regexp.MustCompile(`\.(test|spec)\.([jt]sx?|[cm][jt]s)$`)
-
-func main() {
- if len(os.Args) != 3 {
- fmt.Fprintln(os.Stderr, "usage: checkascii <base-ref> <head-ref>")
- os.Exit(2)
- }
- ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
- defer cancel()
- if err := check(ctx, ".", os.Args[1], os.Args[2], os.Stdout); err != nil {
- fmt.Fprintln(os.Stderr, err)
- os.Exit(1)
- }
-}
-
-func check(ctx context.Context, dir, base, head string, out io.Writer) error {
- // Resolve refs before constructing the range so invalid refs fail closed.
- refs := []string{base, head}
- for i, ref := range refs {
- commit, err := gitOutput(ctx, dir, "rev-parse", "--verify", "--end-of-options", ref+"^{commit}")
- if err != nil {
- return err
- }
- refs[i] = strings.TrimSpace(string(commit))
- }
- prRange := refs[0] + "..." + refs[1]
- files, err := gitOutput(ctx, dir, "diff", "--no-ext-diff", "--no-textconv", "--no-renames", "--diff-filter=AMT", "--name-only", "-z", prRange, "--")
- if err != nil {
- return err
- }
- failures := 0
- escape := strings.NewReplacer("%", "%25", "\r", "%0D", "\n", "%0A", ":", "%3A", ",", "%2C")
- escapeMessage := strings.NewReplacer("%", "%25", "\r", "%0D", "\n", "%0A")
- for _, name := range bytes.Split(files, []byte{0}) {
- file := string(name)
- if file == "" || isExcluded(file) {
- continue
- }
- // Inspect committed PR content, independent of the checked-out merge tree.
- // Git omits binary patches; renames are additions so moving a file cannot
- // bypass the policy by moving it out of an excluded directory.
- diff, err := gitOutput(ctx, dir, "diff", "--no-color", "--no-ext-diff", "--no-textconv", "--no-renames", "--unified=0", prRange, "--", file)
- if err != nil {
- return err
- }
- lineNumber := 0
- for _, line := range bytes.Split(diff, []byte{'\n'}) {
- if match := hunkHeader.FindSubmatch(line); match != nil {
- lineNumber, err = strconv.Atoi(string(match[1]))
- if err != nil {
- return fmt.Errorf("parse diff line number for %q: %w", file, err)
- }
- continue
- }
- if lineNumber == 0 || len(line) == 0 {
- continue
- }
- switch line[0] {
- case '+':
- for _, b := range line[1:] {
- if b >= 0x80 {
- if _, err := fmt.Fprintf(out, "::error file=%s,line=%d::%s:%d: Non-ASCII characters on added line. Use English ASCII text or Unicode escapes in string literals.\n", escape.Replace(file), lineNumber, escapeMessage.Replace(file), lineNumber); err != nil {
- return err
- }
- failures++
- break
- }
- }
- lineNumber++
- case ' ':
- lineNumber++
- }
- }
- }
- if failures > 0 {
- return fmt.Errorf("non-ASCII check failed: %d added line(s); see tools/checkascii/README.md for exclusions", failures)
- }
- return nil
-}
-
-func gitOutput(ctx context.Context, dir string, args ...string) ([]byte, error) {
- cmd := exec.CommandContext(ctx, "git", append([]string{"--literal-pathspecs"}, args...)...)
- cmd.Dir = dir
- var stderr bytes.Buffer
- cmd.Stderr = &stderr
- output, err := cmd.Output()
- if err != nil {
- return nil, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
- }
- return output, nil
-}
-
-func isExcluded(file string) bool {
- if strings.HasSuffix(file, "_test.go") {
- return true
- }
- if strings.HasPrefix(file, "web/") && (frontendTestFile.MatchString(file) || strings.Contains(file, "/__tests__/") || strings.Contains(file, "/__mocks__/") || file == "web/jest-setup.ts") {
- return true
- }
- if path.Dir(file) == "web/src/locales" && path.Ext(file) == ".ts" {
- return path.Base(file) != "en.ts"
- }
- if path.Dir(file) == "." && strings.HasPrefix(file, "README_") && strings.HasSuffix(file, ".md") {
- return true
- }
- if path.Dir(file) == "internal/agent/templates" && path.Ext(file) == ".json" {
- return true
- }
- if file == "internal/rag/res/ner.json" || file == "internal/rag/res/synonym.json" {
- return true
- }
- // Fixture data needs literal multilingual text. Source code in these
- // directories is still checked unless it matches a test exclusion above.
- switch path.Ext(file) {
- case ".json", ".txt", ".md", ".html", ".csv", ".tsv", ".xml", ".svg", ".b64":
- for _, part := range strings.Split(path.Dir(file), "/") {
- if part == "testdata" || part == "fixtures" || part == "golden" {
- return true
- }
- }
- }
- return false
-}
diff --git a/tools/checkascii/main_test.go b/tools/checkascii/main_test.go
deleted file mode 100644
index c33d07ad5b..0000000000
--- a/tools/checkascii/main_test.go
+++ /dev/null
@@ -1,192 +0,0 @@
-package main
-
-import (
- "bytes"
- "os"
- "os/exec"
- "path/filepath"
- "strings"
- "testing"
-)
-
-func TestCheck(t *testing.T) {
- tests := []struct {
- name string
- file string
- before string
- after string
- removed bool
- lines []string
- }{
- {name: "ASCII edit beside existing Unicode", file: "code.go", before: "// \u4e2d\u6587\n// old\n", after: "// \u4e2d\u6587\n// new\n"},
- {name: "removed Unicode line", file: "code.go", before: "// \u4e2d\u6587\n// ASCII\n", after: "// ASCII\n"},
- {name: "removed file", file: "removed.go", before: "// \u4e2d\u6587\n", removed: true},
- {name: "ASCII string escape", file: "code.go", after: `var text = "\u4e2d\u6587"`},
- {name: "new code", file: "code.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
- {name: "multiple hunks", file: "code.go", before: "one\ntwo\nthree\nfour\nfive\n", after: "one\n\u00e9\nthree\nfour\n\u2014\n", lines: []string{"2", "5"}},
- {name: "no final newline", file: "code.go", after: "// \u4e2d\u6587", lines: []string{"1"}},
- {name: "content resembles diff header", file: "code.go", after: "++ \u4e2d\u6587\n", lines: []string{"1"}},
- {name: "long line", file: "long.txt", after: strings.Repeat("a", 128*1024) + "\u00e9\n", lines: []string{"1"}},
- {name: "binary file", file: "image.bin", after: "\x00\xff\xfe\n"},
- {name: "translated locale", file: "web/src/locales/zh.ts", after: "// \u4e2d\u6587\n"},
- {name: "English locale", file: "web/src/locales/en.ts", after: "// \u2019\n", lines: []string{"1"}},
- {name: "nested locale code", file: "web/src/locales/nested/helper.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
- {name: "translated root README", file: "README_zh.md", after: "\u4e2d\u6587\n"},
- {name: "English README", file: "README.md", after: "\u2014\n", lines: []string{"1"}},
- {name: "nested translated README", file: "docs/README_zh.md", after: "\u4e2d\u6587\n", lines: []string{"1"}},
- {name: "localized agent template", file: "internal/agent/templates/example.json", after: "{\"zh\": \"\u4e2d\u6587\"}\n"},
- {name: "entity dictionary", file: "internal/rag/res/ner.json", after: "{\"\u4e2d\u6587\": \"entity\"}\n"},
- {name: "synonym dictionary", file: "internal/rag/res/synonym.json", after: "{\"\u4e2d\u6587\": \"synonym\"}\n"},
- {name: "other resource", file: "internal/rag/res/config.json", after: "{\"value\": \"\u4e2d\u6587\"}\n", lines: []string{"1"}},
- {name: "parser input", file: "internal/parser/testdata/sample.zh.html", after: "<p>\u4e2d\u6587</p>\n"},
- {name: "fixture data", file: "test/fixtures/example.json", after: "{\"value\": \"\u4e2d\u6587\"}\n"},
- {name: "golden data", file: "internal/example/golden/result.txt", after: "\u4e2d\u6587\n"},
- {name: "Go test code", file: "internal/example/example_test.go", after: "// \u4e2d\u6587\n"},
- {name: "Go test in fixture directory", file: "internal/parser/testdata/example_test.go", after: "// \u4e2d\u6587\n"},
- {name: "frontend TypeScript test", file: "web/src/utils/example.test.ts", after: "// \u4e2d\u6587\n"},
- {name: "frontend React test", file: "web/src/components/example.test.tsx", after: "// \u4e2d\u6587\n"},
- {name: "frontend JavaScript test", file: "web/src/utils/example.test.js", after: "// \u4e2d\u6587\n"},
- {name: "frontend JSX spec", file: "web/src/components/example.spec.jsx", after: "// \u4e2d\u6587\n"},
- {name: "frontend module spec", file: "web/src/utils/example.spec.mjs", after: "// \u4e2d\u6587\n"},
- {name: "frontend grouped test", file: "web/src/utils/__tests__/example.ts", after: "// \u4e2d\u6587\n"},
- {name: "frontend mock", file: "web/__mocks__/example.js", after: "// \u4e2d\u6587\n"},
- {name: "frontend test setup", file: "web/jest-setup.ts", after: "// \u4e2d\u6587\n"},
- {name: "frontend application code", file: "web/src/utils/example.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
- {name: "frontend application test helper", file: "web/src/utils/test-helper.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
- {name: "test-like directory", file: "web/src/utils/__tests__extra/example.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
- {name: "test-like filename", file: "web/src/utils/example.test.ts.backup", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
- {name: "fixture generator", file: "internal/parser/testdata/generate.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
- {name: "filename with spaces and quotes", file: "code with \"quotes\".go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
- {name: "literal pathspec", file: ":(exclude)file.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
- {name: "annotation escaping", file: "code%,\n.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- dir := initRepo(t)
- if tt.before != "" {
- writeFile(t, dir, tt.file, tt.before)
- }
- git(t, dir, "add", ".")
- git(t, dir, "commit", "--allow-empty", "-qm", "base")
- base := git(t, dir, "rev-parse", "HEAD")
- if tt.removed {
- if err := os.Remove(filepath.Join(dir, tt.file)); err != nil {
- t.Fatal(err)
- }
- } else {
- writeFile(t, dir, tt.file, tt.after)
- }
- git(t, dir, "add", ".")
- git(t, dir, "commit", "-qm", "PR")
- var out bytes.Buffer
- err := check(t.Context(), dir, base, "HEAD", &out)
- if (err != nil) != (len(tt.lines) > 0) {
- t.Fatalf("check returned %v, output: %s", err, &out)
- }
- if got := strings.Count(out.String(), "::error file="); got != len(tt.lines) {
- t.Fatalf("got %d annotations, want %d: %s", got, len(tt.lines), &out)
- }
- for _, line := range tt.lines {
- if !strings.Contains(out.String(), ",line="+line+"::") {
- t.Errorf("missing annotation at line %s: %s", line, &out)
- }
- file := strings.NewReplacer("%", "%25", "\r", "%0D", "\n", "%0A").Replace(tt.file)
- if !strings.Contains(out.String(), "::"+file+":"+line+": Non-ASCII characters on added line.") {
- t.Errorf("missing visible file and line %s:%s: %s", tt.file, line, &out)
- }
- }
- if tt.name == "annotation escaping" && !strings.Contains(out.String(), "file=code%25%2C%0A.go,line=1::") {
- t.Errorf("unescaped annotation path: %s", &out)
- }
- })
- }
-}
-
-func TestCheckUsesMergeBaseAndCommittedHead(t *testing.T) {
- dir := initRepo(t)
- writeFile(t, dir, "code.go", "// ASCII\n")
- git(t, dir, "add", ".")
- git(t, dir, "commit", "-qm", "common ancestor")
- ancestor := git(t, dir, "rev-parse", "HEAD")
- writeFile(t, dir, "upstream.go", "// \u4e2d\u6587\n")
- git(t, dir, "add", ".")
- git(t, dir, "commit", "-qm", "target branch change")
- base := git(t, dir, "rev-parse", "HEAD")
- git(t, dir, "checkout", "-qb", "pr", ancestor)
- writeFile(t, dir, "code.go", "// changed ASCII\n")
- git(t, dir, "add", ".")
- git(t, dir, "commit", "-qm", "PR change")
- head := git(t, dir, "rev-parse", "HEAD")
- writeFile(t, dir, "code.go", "// uncommitted \u4e2d\u6587\n")
- var out bytes.Buffer
- if err := check(t.Context(), dir, base, head, &out); err != nil {
- t.Fatalf("check included changes outside the committed PR diff: %v: %s", err, &out)
- }
- git(t, dir, "add", ".")
- git(t, dir, "commit", "-qm", "PR Unicode change")
- head = git(t, dir, "rev-parse", "HEAD")
- git(t, dir, "checkout", "--detach", base)
- if err := check(t.Context(), dir, base, head, &out); err == nil {
- t.Fatal("check ignored committed PR content absent from the working tree")
- }
-}
-
-func TestCheckRenameOutOfExclusion(t *testing.T) {
- dir := initRepo(t)
- writeFile(t, dir, "web/src/locales/zh.ts", "// \u4e2d\u6587\n")
- git(t, dir, "add", ".")
- git(t, dir, "commit", "-qm", "base")
- base := git(t, dir, "rev-parse", "HEAD")
- git(t, dir, "mv", "web/src/locales/zh.ts", "web/src/locales/en.ts")
- git(t, dir, "commit", "-qm", "rename")
- var out bytes.Buffer
- if err := check(t.Context(), dir, base, "HEAD", &out); err == nil {
- t.Fatal("check allowed Unicode to be renamed into en.ts")
- }
- if !strings.Contains(out.String(), "file=web/src/locales/en.ts,line=1::") {
- t.Fatalf("missing rename annotation: %s", &out)
- }
-}
-
-func TestCheckInvalidRefs(t *testing.T) {
- dir := initRepo(t)
- git(t, dir, "commit", "--allow-empty", "-qm", "base")
- for _, refs := range [][2]string{{"missing-base", "HEAD"}, {"HEAD", "missing-head"}} {
- var out bytes.Buffer
- if err := check(t.Context(), dir, refs[0], refs[1], &out); err == nil {
- t.Errorf("check accepted invalid refs %v", refs)
- }
- }
-}
-
-func initRepo(t *testing.T) string {
- t.Helper()
- dir := t.TempDir()
- git(t, dir, "init", "-q")
- git(t, dir, "config", "user.name", "ASCII check test")
- git(t, dir, "config", "user.email", "[email protected]")
- return dir
-}
-
-func git(t *testing.T, dir string, args ...string) string {
- t.Helper()
- cmd := exec.Command("git", append([]string{"--literal-pathspecs", "-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false"}, args...)...)
- cmd.Dir = dir
- cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
- out, err := cmd.CombinedOutput()
- if err != nil {
- t.Fatalf("git %v: %v: %s", args, err, out)
- }
- return strings.TrimSpace(string(out))
-}
-
-func writeFile(t *testing.T, dir, file, text string) {
- t.Helper()
- fullPath := filepath.Join(dir, file)
- if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
- t.Fatal(err)
- }
- if err := os.WriteFile(fullPath, []byte(text), 0o644); err != nil {
- t.Fatal(err)
- }
-}