Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

20.3 KB
Raw Blame
/*
 *  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

import { Operator } from '@/constants/agent';
import { DSL, RAGFlowNodeType } from '@/interfaces/database/agent';
import {
  getInitialExtractorValues,
  initialCompilationValues,
  initialExtractorValues,
  initialGeneralChunkerValues,
  initialParserValues,
  initialTitleChunkerValues,
  initialTokenChunkerValues,
  initialTokenizerValues,
} from '@/pages/agent/constant/pipeline';
import {
  normalizeTokenChunkerFormValues,
  transformCompilationParams,
  transformExtractorParams,
  transformGeneralChunkerParams,
  transformParserParams,
  transformTitleChunkerParams,
  transformTokenChunkerParams,
} from '@/pages/agent/utils';
import { cloneDeep, isEmpty, omit } from 'lodash';

export const FileNodeId = 'File';

export function getOperatorType(operatorId: string): Operator {
  return (operatorId.split(':')[0] || operatorId) as Operator;
}

// The dataset-level metadata group stored at parser_config.metadata. The group
// is shaped {enabled, metadata, built_in_metadata}; the boolean enabled flag
// and the built_in_metadata array identify it among other values.
function isDatasetMetadataGroup(value: any): boolean {
  return (
    typeof value === 'object' &&
    value !== null &&
    !Array.isArray(value) &&
    typeof value.enabled === 'boolean' &&
    Array.isArray(value.built_in_metadata)
  );
}

export function transformParserConfigSetups(
  setups: Record<string, any> | undefined,
): any[] {
  if (!setups || typeof setups !== 'object') {
    return [];
  }

  // parser_config entries saved by the Python-era frontend (or derived from
  // a Python canvas) nest the per-family setups under a "setups" key; the Go
  // shape lists file families at the top level. Lift the nested group —
  // top-level families win — and drop the non-family protocol keys so they
  // never render as a bogus file format. The vision options
  // (enable_vision_enhancement / vlm) are global params-level keys, not
  // families, so they are dropped here too (the caller lifts them back onto
  // the form's top level).
  let source = setups;
  const nested = source.setups;
  if (nested && typeof nested === 'object' && !Array.isArray(nested)) {
    source = { ...nested, ...omit(source, ['setups']) };
  }

  return Object.entries(
    omit(source, [
      'outputs',
      'allowed_output_format',
      'enable_vision_enhancement',
      'vlm',
    ]),
  )
    .map(([fileFormat, config]) => {
      const { pages, ...rest } = (config ?? {}) as Record<string, any>;
      const normalizedPages = Array.isArray(pages)
        ? pages.map(([from, to]: [number, number]) => ({ from, to }))
        : pages;
      return {
        fileFormat,
        ...rest,
        ...(normalizedPages !== undefined ? { pages: normalizedPages } : {}),
      } as Record<string, any>;
    })
    .sort((a, b) => (a.order_index ?? Infinity) - (b.order_index ?? Infinity));
}

function transformLevelsToRules(
  levels: any[],
): Array<{ levels: Array<{ expression: string }> }> {
  if (!Array.isArray(levels)) return [];
  return levels
    .map((levelGroup) => {
      if (Array.isArray(levelGroup)) {
        const filteredExpressions = levelGroup.filter(
          (expr: string) => expr && expr.trim() !== '',
        );
        if (filteredExpressions.length === 0) return null;
        return {
          levels: filteredExpressions.map((expression: string) => ({
            expression,
          })),
        };
      }
      return { levels: [{ expression: '' }] };
    })
    .filter((rule) => rule !== null);
}

/**
 * Converts Extractor config from API/DSL format to form format: flattens the
 * DSL prompts array into the form's single field and normalizes the nested
 * per-feature groups the extractor schema reads (schema.ExtractorParam).
 * Legacy flat keys (auto_keywords, keywords_sys_prompt, enable_summary, ...)
 * are accepted as read fallbacks and never re-emitted.
 *
 * DSL:  { prompts: [{ content: "text", role: "user" }] }
 * Form: { prompts: "text" }
 */
export function transformExtractorConfigToForm(
  config: Record<string, any> | undefined,
): Record<string, any> {
  if (!config) return {};

  const result = { ...config };
  if (Array.isArray(config.prompts) && config.prompts.length > 0) {
    result.prompts = config.prompts[0]?.content ?? '';
  }

  const isSummaryEnabled =
    config.summary?.enabled !== undefined
      ? Boolean(config.summary?.enabled)
      : config.enable_summary === 1 || config.enable_summary === true;

  // The metadata group is stored under "metadata" — the shape the Go
  // Extractor reads (schema.ExtractorParam). Accept two historical shapes:
  // the transitional "metadata_config" key, and legacy flat nodes carrying
  // enable_metadata + metadata[] + built_in_metadata[] (flat "metadata" is
  // an array there, which distinguishes it from the group object).
  const metadataGroup =
    config.metadata !== undefined && !Array.isArray(config.metadata)
      ? config.metadata
      : config.metadata_config;

  const isMetadataEnabled =
    metadataGroup?.enabled !== undefined
      ? Boolean(metadataGroup.enabled)
      : config.enable_metadata === 1 || config.enable_metadata === true;

  // The enabled switch is authoritative when the group carries it; legacy
  // configs without it derive enabled from top_n > 0, matching the Go
  // extractor's resolveExtractToggle.
  const keywordsTopN =
    config.keywords?.top_n ??
    config.auto_keywords ??
    initialExtractorValues.keywords.top_n;
  const questionsTopN =
    config.questions?.top_n ??
    config.auto_questions ??
    initialExtractorValues.questions.top_n;
  const tagsTopN =
    config.tags?.top_n ?? config.auto_tags ?? initialExtractorValues.tags.top_n;

  result.keywords = {
    enabled:
      config.keywords?.enabled !== undefined
        ? Boolean(config.keywords.enabled)
        : keywordsTopN > 0,
    top_n: keywordsTopN,
    system_prompt:
      config.keywords?.system_prompt ?? config.keywords_sys_prompt ?? '',
  };
  result.questions = {
    enabled:
      config.questions?.enabled !== undefined
        ? Boolean(config.questions.enabled)
        : questionsTopN > 0,
    top_n: questionsTopN,
    system_prompt:
      config.questions?.system_prompt ?? config.questions_sys_prompt ?? '',
  };
  result.tags = {
    enabled:
      config.tags?.enabled !== undefined
        ? Boolean(config.tags.enabled)
        : tagsTopN > 0,
    top_n: tagsTopN,
    tag_file_id: config.tags?.tag_file_id ?? config.tag_file_id ?? '',
  };
  result.summary = {
    enabled: isSummaryEnabled,
    system_prompt: config.summary?.system_prompt ?? config.sys_prompt ?? '',
  };

  result.metadata = {
    enabled: isMetadataEnabled,
    metadata:
      metadataGroup?.metadata ??
      (Array.isArray(config.metadata) ? config.metadata : []),
    built_in_metadata:
      metadataGroup?.built_in_metadata ?? config.built_in_metadata ?? [],
  };
  // Drop the historical keys so they don't linger in form state and get
  // re-emitted into the DSL on save — the extractor only reads the group.
  delete result.metadata_config;
  delete result.enable_metadata;
  delete result.built_in_metadata;

  return result;
}

/**
 * Converts TokenChunker config from API/DSL format to form format.
 * DSL:  { delimiters: ["\n"], overlapped_percent: 0.1, table_context_size: 10, image_context_size: 20 }
 * Form: { delimiters: [{ value: "\n" }], overlapped_percent: 10, image_table_context_window: 15, enable_children: false }
 */
function transformTokenChunkerConfigToForm(
  config: Record<string, any> | undefined,
  { seedLegacyDelimiter = true } = {},
): Record<string, any> {
  if (!config) return {};

  const result = { ...config };

  // Convert string array delimiters to object arrays. The legacy re-seed and
  // mode normalization happen in normalizeTokenChunkerFormValues below, which
  // still sees the raw delimiter_mode.
  if (Array.isArray(config.delimiters)) {
    result.delimiters = config.delimiters.map((d: string) => ({ value: d }));
  }
  if (Array.isArray(config.children_delimiters)) {
    result.children_delimiters = config.children_delimiters.map(
      (d: string) => ({ value: d }),
    );
  }

  // Convert overlapped_percent from 0-1 scale to 0-100 scale
  if (typeof config.overlapped_percent === 'number') {
    result.overlapped_percent = Math.round(config.overlapped_percent * 100);
  }

  // Merge table_context_size and image_context_size into image_table_context_window
  const tableSize = Number(config.table_context_size ?? 0);
  const imageSize = Number(config.image_context_size ?? 0);
  result.image_table_context_window = Math.max(tableSize, imageSize);

  const normalized = normalizeTokenChunkerFormValues(result, {
    seedLegacyDelimiter,
  });

  // Clean up DSL-only fields not in form schema
  delete normalized.table_context_size;
  delete normalized.image_context_size;

  return normalized;
}

function transformGeneralChunkerConfigToForm(
  config: Record<string, any> | undefined,
): Record<string, any> {
  // The general chunker never had the legacy 'token_size' tab, so an empty
  // saved list is always a deliberate choice — skip the legacy re-seed.
  const result = transformTokenChunkerConfigToForm(config, {
    seedLegacyDelimiter: false,
  });
  result.table_context_size = Number(config?.table_context_size ?? 0);
  result.image_context_size = Number(config?.image_context_size ?? 0);
  delete result.image_table_context_window;
  delete result.delimiter_mode;
  return result;
}

/**
 * Converts TitleChunker config from API/DSL format to form format.
 * DSL:  { method: "hierarchy", hierarchy: "3", levels: [...], chunk_token_cap, include_heading_content, root_chunk_as_heading }
 * Form: { method, hierarchyHierarchy, hierarchyGroup, hierarchyChunkTokenCap, groupChunkTokenCap, include_heading_content, root_chunk_as_heading, hierarchyRules, groupRules }
 */
function transformTitleChunkerConfigToForm(
  config: Record<string, any> | undefined,
): Record<string, any> {
  if (!config) return {};

  const result = { ...config };
  const method = config.method ?? 'hierarchy';

  // Convert legacy `hierarchy` (single field) to split fields
  let hierarchy = config.hierarchy;
  if (typeof hierarchy === 'number') {
    hierarchy = String(hierarchy);
  }

  if (method === 'hierarchy') {
    result.hierarchyHierarchy = config.hierarchyHierarchy || hierarchy || '3';
    result.hierarchyGroup = config.hierarchyGroup || '0';
  } else {
    result.hierarchyHierarchy = config.hierarchyHierarchy || hierarchy || '3';
    result.hierarchyGroup = config.hierarchyGroup || hierarchy || '0';
  }

  // Convert `levels` to `hierarchyRules`/`groupRules`
  if (config.levels && Array.isArray(config.levels) && !config.hierarchyRules) {
    result.hierarchyRules = transformLevelsToRules(config.levels);
  }
  if (config.levels && Array.isArray(config.levels) && !config.groupRules) {
    result.groupRules = transformLevelsToRules(config.levels);
  }

  // Split the backend-facing shared `chunk_token_cap` into the per-method
  // fields the form edits (mirroring the rules/hierarchy split above).
  if (config.chunk_token_cap !== undefined) {
    if (result.hierarchyChunkTokenCap === undefined) {
      result.hierarchyChunkTokenCap = config.chunk_token_cap;
    }
    if (result.groupChunkTokenCap === undefined) {
      result.groupChunkTokenCap = config.chunk_token_cap;
    }
  }

  // Clean up DSL-only fields
  delete result.hierarchy;
  delete result.levels;
  delete result.rules;
  delete result.chunk_token_cap;

  return result;
}

/**
 * Tokenizer is a passthrough — both API and form formats are identical.
 */
function transformTokenizerConfigToForm(
  config: Record<string, any> | undefined,
): Record<string, any> {
  return config ?? {};
}

/**
 * Dispatches reverse transform by operator type.
 * Converts DSL-format config to form-format config.
 */
export function transformApiConfigToForm(
  operatorType: string,
  config: Record<string, any> | undefined,
): Record<string, any> {
  switch (operatorType) {
    case Operator.Parser:
      // Lift the global vision options back onto the form's top level; omit
      // them when absent so an explicit undefined cannot clobber template or
      // default baselines downstream (Object.assign/spread copies undefined).
      return {
        setups: transformParserConfigSetups(config),
        ...(config?.enable_vision_enhancement !== undefined
          ? { enable_vision_enhancement: config.enable_vision_enhancement }
          : {}),
        ...(config?.vlm !== undefined ? { vlm: config.vlm } : {}),
      };
    case Operator.Extractor:
      return transformExtractorConfigToForm(config);
    case Operator.Tokenizer:
      return transformTokenizerConfigToForm(config);
    case Operator.TokenChunker:
      return transformTokenChunkerConfigToForm(config);
    case Operator.GeneralChunker:
      return transformGeneralChunkerConfigToForm(config);
    case Operator.TitleChunker:
    case Operator.ManualChunker:
      return transformTitleChunkerConfigToForm(config);
    default:
      return config ?? {};
  }
}

/**
 * Converts a saved parser_config (API format, keyed by operator id) to the
 * form format used by the operator tabs. Configs without pipeline keys
 * (built-in parse type) are returned as-is.
 *
 * The dataset-level metadata group is authoritative for Extractor nodes,
 * mirroring buildOperatorNode, so the values seeded into the outer form match
 * what the operator tabs initialize from.
 */
export function transformSavedParserConfigToForm(
  parserConfig?: Record<string, any>,
): Record<string, any> | undefined {
  if (
    !parserConfig ||
    typeof parserConfig !== 'object' ||
    Array.isArray(parserConfig) ||
    !Object.keys(parserConfig).some((key) => key.includes(':'))
  ) {
    return parserConfig;
  }

  const formParserConfig: Record<string, any> = {};
  for (const [operatorId, config] of Object.entries(parserConfig)) {
    const operatorType = getOperatorType(operatorId);
    const apiConfig =
      operatorType === Operator.Extractor &&
      isDatasetMetadataGroup(parserConfig.metadata)
        ? { ...config, metadata: parserConfig.metadata }
        : (config as Record<string, any>);
    formParserConfig[operatorId] = transformApiConfigToForm(
      operatorType,
      apiConfig,
    );
  }
  return formParserConfig;
}

/**
 * Dispatches forward transform by operator type.
 * Converts form-format config to DSL format for saving.
 */
export function transformFormConfigToApi(
  operatorType: string,
  config: Record<string, any> | undefined,
): Record<string, any> {
  if (!config) return {};

  switch (operatorType) {
    case Operator.Parser:
      return transformParserParams(config as any);
    case Operator.Extractor:
      return transformExtractorParams(config as any);
    case Operator.Compiler:
      return transformCompilationParams(config as Record<string, any>);
    case Operator.Tokenizer:
      return config; // passthrough for Tokenizer
    case Operator.TokenChunker:
      return transformTokenChunkerParams(config as any);
    case Operator.GeneralChunker:
      return transformGeneralChunkerParams(config as any);
    case Operator.TitleChunker:
      return transformTitleChunkerParams(config as any);
    case Operator.ManualChunker:
      // These fields are UI-only for the title chunker and are not read by
      // the backend ManualChunker component (manual.go pins method=group,
      // ignores the token cap).
      return transformTitleChunkerParams(
        omit(config, [
          'include_heading_content',
          'chunk_token_cap',
          'hierarchyChunkTokenCap',
          'groupChunkTokenCap',
        ]) as any,
      );
    default:
      return config;
  }
}

export function normalizeOperatorForm(
  operatorId: string,
  rawForm: Record<string, any> | undefined,
): Record<string, any> {
  const operatorType = getOperatorType(operatorId);

  switch (operatorType) {
    case Operator.Parser: {
      return {
        ...cloneDeep(initialParserValues),
        ...rawForm,
        setups: rawForm?.setups?.length
          ? rawForm.setups
          : cloneDeep(initialParserValues.setups),
      };
    }
    case Operator.TitleChunker:
    case Operator.ManualChunker:
      return {
        ...cloneDeep(initialTitleChunkerValues),
        ...rawForm,
      };
    case Operator.TokenChunker:
      return {
        ...cloneDeep(initialTokenChunkerValues),
        ...rawForm,
      };
    case Operator.GeneralChunker:
      return {
        ...cloneDeep(initialGeneralChunkerValues),
        ...rawForm,
      };
    case Operator.Extractor:
      return {
        ...cloneDeep(getInitialExtractorValues()),
        ...rawForm,
      };
    case Operator.Compiler:
      return {
        ...cloneDeep(initialCompilationValues),
        ...rawForm,
      };
    case Operator.Tokenizer:
      return {
        ...cloneDeep(initialTokenizerValues),
        ...rawForm,
      };
    default:
      return rawForm ?? {};
  }
}

export function buildOperatorNode(
  dslNode: RAGFlowNodeType,
  pipelineParserConfig: Record<string, any> = {},
): RAGFlowNodeType {
  const operatorId = dslNode.id;
  const operatorType = getOperatorType(operatorId);

  // Form-format config from DSL graph
  const configFromDsl = dslNode.data?.form;

  const rawForm = {
    ...configFromDsl, // template defaults from DSL (form format)
  };

  if (!isEmpty(pipelineParserConfig)) {
    let apiConfig = pipelineParserConfig[operatorId];
    // Dataset-level auto-metadata lives at parser_config.metadata; the
    // backend preserves that object and re-scopes it into every Extractor
    // node on write, so it is the authoritative source for the metadata
    // toggle's initial state — prefer it over a possibly stale per-node copy.
    if (
      operatorType === Operator.Extractor &&
      isDatasetMetadataGroup(pipelineParserConfig.metadata)
    ) {
      apiConfig = {
        ...apiConfig,
        metadata: pipelineParserConfig.metadata,
      };
    }
    // user overrides from API (now also form format)
    Object.assign(
      rawForm,
      transformApiConfigToForm(operatorType, apiConfig), // Convert API config to form format, then merge (DSL template is baseline, API overrides)
    );
  }

  return {
    ...dslNode,
    id: '',
    data: {
      ...dslNode.data,
      label: operatorType,
      operatorId,
      form: normalizeOperatorForm(operatorId, rawForm),
    },
  };
}

export function buildPipelineOperatorNodes(
  dsl?: DSL,
  pipelineParserConfig: Record<string, any> = {},
): RAGFlowNodeType[] {
  if (!dsl?.graph?.nodes || !dsl?.graph?.edges) {
    return [];
  }

  // Build source → target map from edges (pipeline is linear)
  const sourceToTarget = new Map<string, string>();
  for (const edge of dsl.graph.edges) {
    sourceToTarget.set(edge.source, edge.target);
  }

  // Follow the chain starting from File, collecting node IDs in order
  const orderedIds: string[] = [];
  let currentId: string | undefined = FileNodeId;
  while (currentId) {
    orderedIds.push(currentId);
    currentId = sourceToTarget.get(currentId);
  }

  // Build a lookup from node ID → node
  const nodeById = new Map<string, RAGFlowNodeType>();
  for (const node of dsl.graph.nodes) {
    nodeById.set(node.id, node);
  }

  // Map ordered IDs to nodes, excluding File
  return orderedIds
    .filter((id) => id !== FileNodeId)
    .map((id) => nodeById.get(id))
    .filter((node): node is RAGFlowNodeType => node !== undefined)
    .map((node) => buildOperatorNode(node, pipelineParserConfig));
}

/**
 * Builds a parser_config record from operator nodes, keyed by operatorId.
 * Each entry is the node's merged form (DSL template defaults + overrides),
 * i.e. exactly what the operator tabs display.
 */
export function buildParserConfigFromNodes(
  operatorNodes: RAGFlowNodeType[],
): Record<string, any> {
  return Object.fromEntries(
    operatorNodes.map((node) => [
      (node.data as Record<string, any>)?.operatorId || node.data?.label || '',
      node.data?.form ?? {},
    ]),
  );
}