RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Revert "CI: add check for ASCII, disallow none ASCII code check-in" (#21873)
Wang Qi543f57ea9716
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.ymlindex 52ce71b4be..01d950e335 100644--- a/.github/workflows/sep-tests.yml+++ b/.github/workflows/sep-tests.yml@@ -17,7 +17,11 @@ on:# We use pull_request_target so that GITEE_TOKEN (and other repo secrets) are available when building# ragflow:nightly from a fork PR, while still running the full test suite on PRs labeled with `ci`.pull_request:- types: [opened, reopened, ready_for_review, synchronize, labeled]+ types: [synchronize, labeled]+ paths-ignore:+ - 'docs/**'+ - '*.md'+ - '*.mdx'schedule:- cron: '0 16 * * *' # This schedule runs every 16:00:00Z(00:00:00+08:00)@@ -63,13 +67,6 @@ jobs:fetch-tags: trueallow-unsafe-pr-checkout: true- - name: Check added lines for non-ASCII characters- if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}- env:- PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}- PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}- run: go run ./tools/checkascii "$PR_BASE_SHA" "$PR_HEAD_SHA"-- name: Check workflow duplicationif: ${{ !cancelled() && !failure() }}run: |diff --git a/tools/checkascii/README.md b/tools/checkascii/README.mddeleted file mode 100644index 5a008566e0..0000000000--- a/tools/checkascii/README.md+++ /dev/null@@ -1,41 +0,0 @@-# PR ASCII check--`ragflow_preflight` rejects non-ASCII bytes on added lines in a PR's diff-from its merge base. Unchanged lines and deleted lines do not fail the check.-The preflight keeps its existing `ci` label requirement for non-draft PRs,-including documentation-only changes.-This is an ASCII rule, so it also rejects smart quotes, accented letters,-emoji, and typographic dashes. Use ASCII punctuation and Unicode escapes-for required string literals in application source code.--Run locally from the repository root:--```sh-go run ./tools/checkascii origin/main HEAD-bash build.sh --test ./tools/checkascii-```--The checker reads committed content rather than working-tree files. Failures-produce a GitHub Actions annotation for every offending added line, with the-file and new line number in both the annotation metadata and the visible error-message (for example, `web/src/app.tsx:42: Non-ASCII characters on added line.`).-Invalid Git references and other Git errors fail the check. Git binary patches are-skipped. Renamed files are treated as additions, including moves out of-excluded paths.--Exclusions:--- Go `*_test.go` files.-- Frontend JavaScript/TypeScript `*.test.*` and `*.spec.*` files under `web/`,- files in `web/**/__tests__/` or `web/**/__mocks__/`, and `web/jest-setup.ts`.-- `web/src/locales/*.ts`, except `web/src/locales/en.ts`.-- `README_*.md` directly in the repository root.-- `internal/agent/templates/*.json`: localized agent names and descriptions.-- `internal/rag/res/ner.json` and `internal/rag/res/synonym.json`: multilingual- entity and synonym dictionaries.-- Data files under a `testdata`, `fixtures`, or `golden` directory with extension- `.json`, `.txt`, `.md`, `.html`, `.csv`, `.tsv`, `.xml`, `.svg`, or `.b64`:- multilingual parser inputs, DSL examples, and expected results.--Other source files remain checked, including fixture generators. English-documentation, prompts, and regular application code have no blanket exclusion.diff --git a/tools/checkascii/main.go b/tools/checkascii/main.godeleted file mode 100644index 8908a3c5f0..0000000000--- a/tools/checkascii/main.go+++ /dev/null@@ -1,141 +0,0 @@-// Command checkascii rejects non-ASCII bytes on lines added by a pull request.-package main--import (- "bytes"- "context"- "fmt"- "io"- "os"- "os/exec"- "os/signal"- "path"- "regexp"- "strconv"- "strings"-)--var hunkHeader = regexp.MustCompile(`^@@ -[0-9]+(?:,[0-9]+)? \+([0-9]+)(?:,[0-9]+)? @@`)-var frontendTestFile = regexp.MustCompile(`\.(test|spec)\.([jt]sx?|[cm][jt]s)$`)--func main() {- if len(os.Args) != 3 {- fmt.Fprintln(os.Stderr, "usage: checkascii <base-ref> <head-ref>")- os.Exit(2)- }- ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)- defer cancel()- if err := check(ctx, ".", os.Args[1], os.Args[2], os.Stdout); err != nil {- fmt.Fprintln(os.Stderr, err)- os.Exit(1)- }-}--func check(ctx context.Context, dir, base, head string, out io.Writer) error {- // Resolve refs before constructing the range so invalid refs fail closed.- refs := []string{base, head}- for i, ref := range refs {- commit, err := gitOutput(ctx, dir, "rev-parse", "--verify", "--end-of-options", ref+"^{commit}")- if err != nil {- return err- }- refs[i] = strings.TrimSpace(string(commit))- }- prRange := refs[0] + "..." + refs[1]- files, err := gitOutput(ctx, dir, "diff", "--no-ext-diff", "--no-textconv", "--no-renames", "--diff-filter=AMT", "--name-only", "-z", prRange, "--")- if err != nil {- return err- }- failures := 0- escape := strings.NewReplacer("%", "%25", "\r", "%0D", "\n", "%0A", ":", "%3A", ",", "%2C")- escapeMessage := strings.NewReplacer("%", "%25", "\r", "%0D", "\n", "%0A")- for _, name := range bytes.Split(files, []byte{0}) {- file := string(name)- if file == "" || isExcluded(file) {- continue- }- // Inspect committed PR content, independent of the checked-out merge tree.- // Git omits binary patches; renames are additions so moving a file cannot- // bypass the policy by moving it out of an excluded directory.- diff, err := gitOutput(ctx, dir, "diff", "--no-color", "--no-ext-diff", "--no-textconv", "--no-renames", "--unified=0", prRange, "--", file)- if err != nil {- return err- }- lineNumber := 0- for _, line := range bytes.Split(diff, []byte{'\n'}) {- if match := hunkHeader.FindSubmatch(line); match != nil {- lineNumber, err = strconv.Atoi(string(match[1]))- if err != nil {- return fmt.Errorf("parse diff line number for %q: %w", file, err)- }- continue- }- if lineNumber == 0 || len(line) == 0 {- continue- }- switch line[0] {- case '+':- for _, b := range line[1:] {- if b >= 0x80 {- if _, err := fmt.Fprintf(out, "::error file=%s,line=%d::%s:%d: Non-ASCII characters on added line. Use English ASCII text or Unicode escapes in string literals.\n", escape.Replace(file), lineNumber, escapeMessage.Replace(file), lineNumber); err != nil {- return err- }- failures++- break- }- }- lineNumber++- case ' ':- lineNumber++- }- }- }- if failures > 0 {- return fmt.Errorf("non-ASCII check failed: %d added line(s); see tools/checkascii/README.md for exclusions", failures)- }- return nil-}--func gitOutput(ctx context.Context, dir string, args ...string) ([]byte, error) {- cmd := exec.CommandContext(ctx, "git", append([]string{"--literal-pathspecs"}, args...)...)- cmd.Dir = dir- var stderr bytes.Buffer- cmd.Stderr = &stderr- output, err := cmd.Output()- if err != nil {- return nil, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))- }- return output, nil-}--func isExcluded(file string) bool {- if strings.HasSuffix(file, "_test.go") {- return true- }- if strings.HasPrefix(file, "web/") && (frontendTestFile.MatchString(file) || strings.Contains(file, "/__tests__/") || strings.Contains(file, "/__mocks__/") || file == "web/jest-setup.ts") {- return true- }- if path.Dir(file) == "web/src/locales" && path.Ext(file) == ".ts" {- return path.Base(file) != "en.ts"- }- if path.Dir(file) == "." && strings.HasPrefix(file, "README_") && strings.HasSuffix(file, ".md") {- return true- }- if path.Dir(file) == "internal/agent/templates" && path.Ext(file) == ".json" {- return true- }- if file == "internal/rag/res/ner.json" || file == "internal/rag/res/synonym.json" {- return true- }- // Fixture data needs literal multilingual text. Source code in these- // directories is still checked unless it matches a test exclusion above.- switch path.Ext(file) {- case ".json", ".txt", ".md", ".html", ".csv", ".tsv", ".xml", ".svg", ".b64":- for _, part := range strings.Split(path.Dir(file), "/") {- if part == "testdata" || part == "fixtures" || part == "golden" {- return true- }- }- }- return false-}diff --git a/tools/checkascii/main_test.go b/tools/checkascii/main_test.godeleted file mode 100644index c33d07ad5b..0000000000--- a/tools/checkascii/main_test.go+++ /dev/null@@ -1,192 +0,0 @@-package main--import (- "bytes"- "os"- "os/exec"- "path/filepath"- "strings"- "testing"-)--func TestCheck(t *testing.T) {- tests := []struct {- name string- file string- before string- after string- removed bool- lines []string- }{- {name: "ASCII edit beside existing Unicode", file: "code.go", before: "// \u4e2d\u6587\n// old\n", after: "// \u4e2d\u6587\n// new\n"},- {name: "removed Unicode line", file: "code.go", before: "// \u4e2d\u6587\n// ASCII\n", after: "// ASCII\n"},- {name: "removed file", file: "removed.go", before: "// \u4e2d\u6587\n", removed: true},- {name: "ASCII string escape", file: "code.go", after: `var text = "\u4e2d\u6587"`},- {name: "new code", file: "code.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},- {name: "multiple hunks", file: "code.go", before: "one\ntwo\nthree\nfour\nfive\n", after: "one\n\u00e9\nthree\nfour\n\u2014\n", lines: []string{"2", "5"}},- {name: "no final newline", file: "code.go", after: "// \u4e2d\u6587", lines: []string{"1"}},- {name: "content resembles diff header", file: "code.go", after: "++ \u4e2d\u6587\n", lines: []string{"1"}},- {name: "long line", file: "long.txt", after: strings.Repeat("a", 128*1024) + "\u00e9\n", lines: []string{"1"}},- {name: "binary file", file: "image.bin", after: "\x00\xff\xfe\n"},- {name: "translated locale", file: "web/src/locales/zh.ts", after: "// \u4e2d\u6587\n"},- {name: "English locale", file: "web/src/locales/en.ts", after: "// \u2019\n", lines: []string{"1"}},- {name: "nested locale code", file: "web/src/locales/nested/helper.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},- {name: "translated root README", file: "README_zh.md", after: "\u4e2d\u6587\n"},- {name: "English README", file: "README.md", after: "\u2014\n", lines: []string{"1"}},- {name: "nested translated README", file: "docs/README_zh.md", after: "\u4e2d\u6587\n", lines: []string{"1"}},- {name: "localized agent template", file: "internal/agent/templates/example.json", after: "{\"zh\": \"\u4e2d\u6587\"}\n"},- {name: "entity dictionary", file: "internal/rag/res/ner.json", after: "{\"\u4e2d\u6587\": \"entity\"}\n"},- {name: "synonym dictionary", file: "internal/rag/res/synonym.json", after: "{\"\u4e2d\u6587\": \"synonym\"}\n"},- {name: "other resource", file: "internal/rag/res/config.json", after: "{\"value\": \"\u4e2d\u6587\"}\n", lines: []string{"1"}},- {name: "parser input", file: "internal/parser/testdata/sample.zh.html", after: "<p>\u4e2d\u6587</p>\n"},- {name: "fixture data", file: "test/fixtures/example.json", after: "{\"value\": \"\u4e2d\u6587\"}\n"},- {name: "golden data", file: "internal/example/golden/result.txt", after: "\u4e2d\u6587\n"},- {name: "Go test code", file: "internal/example/example_test.go", after: "// \u4e2d\u6587\n"},- {name: "Go test in fixture directory", file: "internal/parser/testdata/example_test.go", after: "// \u4e2d\u6587\n"},- {name: "frontend TypeScript test", file: "web/src/utils/example.test.ts", after: "// \u4e2d\u6587\n"},- {name: "frontend React test", file: "web/src/components/example.test.tsx", after: "// \u4e2d\u6587\n"},- {name: "frontend JavaScript test", file: "web/src/utils/example.test.js", after: "// \u4e2d\u6587\n"},- {name: "frontend JSX spec", file: "web/src/components/example.spec.jsx", after: "// \u4e2d\u6587\n"},- {name: "frontend module spec", file: "web/src/utils/example.spec.mjs", after: "// \u4e2d\u6587\n"},- {name: "frontend grouped test", file: "web/src/utils/__tests__/example.ts", after: "// \u4e2d\u6587\n"},- {name: "frontend mock", file: "web/__mocks__/example.js", after: "// \u4e2d\u6587\n"},- {name: "frontend test setup", file: "web/jest-setup.ts", after: "// \u4e2d\u6587\n"},- {name: "frontend application code", file: "web/src/utils/example.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},- {name: "frontend application test helper", file: "web/src/utils/test-helper.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},- {name: "test-like directory", file: "web/src/utils/__tests__extra/example.ts", after: "// \u4e2d\u6587\n", lines: []string{"1"}},- {name: "test-like filename", file: "web/src/utils/example.test.ts.backup", after: "// \u4e2d\u6587\n", lines: []string{"1"}},- {name: "fixture generator", file: "internal/parser/testdata/generate.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},- {name: "filename with spaces and quotes", file: "code with \"quotes\".go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},- {name: "literal pathspec", file: ":(exclude)file.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},- {name: "annotation escaping", file: "code%,\n.go", after: "// \u4e2d\u6587\n", lines: []string{"1"}},- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- dir := initRepo(t)- if tt.before != "" {- writeFile(t, dir, tt.file, tt.before)- }- git(t, dir, "add", ".")- git(t, dir, "commit", "--allow-empty", "-qm", "base")- base := git(t, dir, "rev-parse", "HEAD")- if tt.removed {- if err := os.Remove(filepath.Join(dir, tt.file)); err != nil {- t.Fatal(err)- }- } else {- writeFile(t, dir, tt.file, tt.after)- }- git(t, dir, "add", ".")- git(t, dir, "commit", "-qm", "PR")- var out bytes.Buffer- err := check(t.Context(), dir, base, "HEAD", &out)- if (err != nil) != (len(tt.lines) > 0) {- t.Fatalf("check returned %v, output: %s", err, &out)- }- if got := strings.Count(out.String(), "::error file="); got != len(tt.lines) {- t.Fatalf("got %d annotations, want %d: %s", got, len(tt.lines), &out)- }- for _, line := range tt.lines {- if !strings.Contains(out.String(), ",line="+line+"::") {- t.Errorf("missing annotation at line %s: %s", line, &out)- }- file := strings.NewReplacer("%", "%25", "\r", "%0D", "\n", "%0A").Replace(tt.file)- if !strings.Contains(out.String(), "::"+file+":"+line+": Non-ASCII characters on added line.") {- t.Errorf("missing visible file and line %s:%s: %s", tt.file, line, &out)- }- }- if tt.name == "annotation escaping" && !strings.Contains(out.String(), "file=code%25%2C%0A.go,line=1::") {- t.Errorf("unescaped annotation path: %s", &out)- }- })- }-}--func TestCheckUsesMergeBaseAndCommittedHead(t *testing.T) {- dir := initRepo(t)- writeFile(t, dir, "code.go", "// ASCII\n")- git(t, dir, "add", ".")- git(t, dir, "commit", "-qm", "common ancestor")- ancestor := git(t, dir, "rev-parse", "HEAD")- writeFile(t, dir, "upstream.go", "// \u4e2d\u6587\n")- git(t, dir, "add", ".")- git(t, dir, "commit", "-qm", "target branch change")- base := git(t, dir, "rev-parse", "HEAD")- git(t, dir, "checkout", "-qb", "pr", ancestor)- writeFile(t, dir, "code.go", "// changed ASCII\n")- git(t, dir, "add", ".")- git(t, dir, "commit", "-qm", "PR change")- head := git(t, dir, "rev-parse", "HEAD")- writeFile(t, dir, "code.go", "// uncommitted \u4e2d\u6587\n")- var out bytes.Buffer- if err := check(t.Context(), dir, base, head, &out); err != nil {- t.Fatalf("check included changes outside the committed PR diff: %v: %s", err, &out)- }- git(t, dir, "add", ".")- git(t, dir, "commit", "-qm", "PR Unicode change")- head = git(t, dir, "rev-parse", "HEAD")- git(t, dir, "checkout", "--detach", base)- if err := check(t.Context(), dir, base, head, &out); err == nil {- t.Fatal("check ignored committed PR content absent from the working tree")- }-}--func TestCheckRenameOutOfExclusion(t *testing.T) {- dir := initRepo(t)- writeFile(t, dir, "web/src/locales/zh.ts", "// \u4e2d\u6587\n")- git(t, dir, "add", ".")- git(t, dir, "commit", "-qm", "base")- base := git(t, dir, "rev-parse", "HEAD")- git(t, dir, "mv", "web/src/locales/zh.ts", "web/src/locales/en.ts")- git(t, dir, "commit", "-qm", "rename")- var out bytes.Buffer- if err := check(t.Context(), dir, base, "HEAD", &out); err == nil {- t.Fatal("check allowed Unicode to be renamed into en.ts")- }- if !strings.Contains(out.String(), "file=web/src/locales/en.ts,line=1::") {- t.Fatalf("missing rename annotation: %s", &out)- }-}--func TestCheckInvalidRefs(t *testing.T) {- dir := initRepo(t)- git(t, dir, "commit", "--allow-empty", "-qm", "base")- for _, refs := range [][2]string{{"missing-base", "HEAD"}, {"HEAD", "missing-head"}} {- var out bytes.Buffer- if err := check(t.Context(), dir, refs[0], refs[1], &out); err == nil {- t.Errorf("check accepted invalid refs %v", refs)- }- }-}--func initRepo(t *testing.T) string {- t.Helper()- dir := t.TempDir()- git(t, dir, "init", "-q")- git(t, dir, "config", "user.name", "ASCII check test")- git(t, dir, "config", "user.email", "[email protected]")- return dir-}--func git(t *testing.T, dir string, args ...string) string {- t.Helper()- cmd := exec.Command("git", append([]string{"--literal-pathspecs", "-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false"}, args...)...)- cmd.Dir = dir- cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")- out, err := cmd.CombinedOutput()- if err != nil {- t.Fatalf("git %v: %v: %s", args, err, out)- }- return strings.TrimSpace(string(out))-}--func writeFile(t *testing.T, dir, file, text string) {- t.Helper()- fullPath := filepath.Join(dir, file)- if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {- t.Fatal(err)- }- if err := os.WriteFile(fullPath, []byte(text), 0o644); err != nil {- t.Fatal(err)- }-}