LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
from enum import Enum
class PromptServiceConstants:
"""Constants used in the prompt service."""
WORD = "word"
SYNONYMS = "synonyms"
OUTPUTS = "outputs"
TOOL_ID = "tool_id"
RUN_ID = "run_id"
EXECUTION_ID = "execution_id"
FILE_NAME = "file_name"
FILE_HASH = "file_hash"
NAME = "name"
ACTIVE = "active"
PROMPT = "prompt"
CHUNK_SIZE = "chunk-size"
PROMPTX = "promptx"
VECTOR_DB = "vector-db"
EMBEDDING = "embedding"
X2TEXT_ADAPTER = "x2text_adapter"
# Extract-file path that is never rewritten by summarize-as-source /
# smart-table overrides — the page-image reader keys on this.
EXTRACT_FILE_PATH = "extract_file_path"
# Per-prompt stamp of the x2text adapter's output mode (LLMWhisperer
# only); lets image-mode detection skip the platform-service call.
X2TEXT_OUTPUT_MODE = "x2text_output_mode"
CHUNK_OVERLAP = "chunk-overlap"
LLM = "llm"
IS_ASSERT = "is_assert"
ASSERTION_FAILURE_PROMPT = "assertion_failure_prompt"
RETRIEVAL_STRATEGY = "retrieval-strategy"
TYPE = "type"
NUMBER = "number"
EMAIL = "email"
DATE = "date"
BOOLEAN = "boolean"
JSON = "json"
PREAMBLE = "preamble"
SIMILARITY_TOP_K = "similarity-top-k"
PROMPT_TOKENS = "prompt_tokens"
COMPLETION_TOKENS = "completion_tokens"
TOTAL_TOKENS = "total_tokens"
RESPONSE = "response"
POSTAMBLE = "postamble"
GRAMMAR = "grammar"
PLATFORM_SERVICE_API_KEY = "PLATFORM_SERVICE_API_KEY"
EMBEDDING_SUFFIX = "embedding_suffix"
EVAL_SETTINGS = "eval_settings"
EVAL_SETTINGS_EVALUATE = "evaluate"
EVAL_SETTINGS_MONITOR_LLM = "monitor_llm"
EVAL_SETTINGS_EXCLUDE_FAILED = "exclude_failed"
TOOL_SETTINGS = "tool_settings"
LOG_EVENTS_ID = "log_events_id"
CHALLENGE_LLM = "challenge_llm"
CHALLENGE = "challenge"
ENABLE_CHALLENGE = "enable_challenge"
EXTRACTION = "extraction"
# Reserved namespace for file-level (whole-document) metrics so they cannot
# collide with user-defined output/prompt names at the top level of the
# metrics dict. Mirrors the "file" key of the API response.
FILE = "_file"
# Text extraction (X2Text) of the source document. Named to stay distinct
# from "extraction_llm", the extraction-purpose LLM call beside it.
TEXT_EXTRACTION = "text_extraction"
# Wire-format key every metrics producer reports its duration under.
TIME_TAKEN = "time_taken(s)"
SUMMARIZE = "summarize"
SINGLE_PASS_EXTRACTION = "single-pass-extraction"
SIMPLE_PROMPT_STUDIO = "simple-prompt-studio"
LLM_USAGE_REASON = "llm_usage_reason"
METADATA = "metadata"
OUTPUT = "output"
CONTEXT = "context"
INCLUDE_METADATA = "include_metadata"
TABLE = "table"
TABLE_SETTINGS = "table_settings"
EPILOGUE = "epilogue"
PLATFORM_POSTAMBLE = "platform_postamble"
WORD_CONFIDENCE_POSTAMBLE = "word_confidence_postamble"
HIGHLIGHT_DATA_PLUGIN = "highlight-data"
SUMMARIZE_AS_SOURCE = "summarize_as_source"
VARIABLE_MAP = "variable_map"
RECORD = "record"
CUSTOM_DATA = "custom_data"
TEXT = "text"
ENABLE_HIGHLIGHT = "enable_highlight"
ENABLE_WORD_CONFIDENCE = "enable_word_confidence"
FILE_PATH = "file_path"
HIGHLIGHT_DATA = "highlight_data"
CONFIDENCE_DATA = "confidence_data"
WORD_CONFIDENCE_DATA = "word_confidence_data"
REQUIRED_FIELDS = "required_fields"
REQUIRED = "required"
EXECUTION_SOURCE = "execution_source"
METRICS = "metrics"
CAPTURE_METRICS = "capture_metrics"
LINE_ITEM = "line-item"
LINE_NUMBERS = "line_numbers"
WHISPER_HASH = "whisper_hash"
PAID_FEATURE_MSG = (
"It is a cloud / enterprise feature. If you have purchased a plan and still "
"face this issue, please contact support"
)
NO_CONTEXT_ERROR = (
"Couldn't fetch context from vector DB. "
"This happens usually due to a delay by the Vector DB "
"provider to confirm writes to DB. "
"Please try again after some time"
)
COMBINED_PROMPT = "combined_prompt"
TOOL = "tool"
JSON_POSTAMBLE = "JSON_POSTAMBLE"
DEFAULT_JSON_POSTAMBLE = "Wrap the final JSON result inbetween §§§ like below example:\n§§§\n<FINAL_JSON_RESULT>\n§§§"
DOCUMENT_TYPE = "document_type"
# Webhook postprocessing settings
ENABLE_POSTPROCESSING_WEBHOOK = "enable_postprocessing_webhook"
POSTPROCESSING_WEBHOOK_URL = "postprocessing_webhook_url"
class RunLevel(Enum):
"""Different stages of prompt execution.
Comprises of prompt run and response evaluation stages.
"""
RUN = "RUN"
EVAL = "EVAL"
CHALLENGE = "CHALLENGE"
TABLE_EXTRACTION = "TABLE_EXTRACTION"
class DBTableV2:
"""Database tables."""
ORGANIZATION = "organization"
ADAPTER_INSTANCE = "adapter_instance"
PROMPT_STUDIO_REGISTRY = "prompt_studio_registry"
PLATFORM_KEY = "platform_key"
TOKEN_USAGE = "usage"
class FileStorageKeys:
"""File storage keys."""
PERMANENT_REMOTE_STORAGE = "PERMANENT_REMOTE_STORAGE"
TEMPORARY_REMOTE_STORAGE = "TEMPORARY_REMOTE_STORAGE"
class FileStorageType(Enum):
"""File storage type."""
PERMANENT = "permanent"
TEMPORARY = "temporary"
class ExecutionSource(Enum):
"""Execution source."""
IDE = "ide"
TOOL = "tool"
class VariableType(str, Enum):
"""Type of variable."""
STATIC = "STATIC"
DYNAMIC = "DYNAMIC"
CUSTOM_DATA = "CUSTOM_DATA"
class RetrievalStrategy(str, Enum):
"""Available retrieval strategies for prompt service."""
SIMPLE = "simple"
SUBQUESTION = "subquestion"
FUSION = "fusion"
RECURSIVE = "recursive"
ROUTER = "router"
KEYWORD_TABLE = "keyword_table"
AUTOMERGING = "automerging"
class VariableConstants:
"""Constants for variable extraction."""
VARIABLE_REGEX = "{{(.+?)}}"
DYNAMIC_VARIABLE_DATA_REGEX = r"\[(.*?)\]"
DYNAMIC_VARIABLE_URL_REGEX = (
r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»"
"'']))"
) # noqa: E501
CUSTOM_DATA_VARIABLE_REGEX = r"custom_data\.([a-zA-Z0-9_\.]+)"
class IndexingConstants:
TOOL_ID = "tool_id"
EMBEDDING_INSTANCE_ID = "embedding_instance_id"
VECTOR_DB_INSTANCE_ID = "vector_db_instance_id"
X2TEXT_INSTANCE_ID = "x2text_instance_id"
FILE_PATH = "file_path"
CHUNK_SIZE = "chunk_size"
CHUNK_OVERLAP = "chunk_overlap"
REINDEX = "reindex"
FILE_HASH = "file_hash"
OUTPUT_FILE_PATH = "output_file_path"
ENABLE_HIGHLIGHT = "enable_highlight"
ENABLE_WORD_CONFIDENCE = "enable_word_confidence"
USAGE_KWARGS = "usage_kwargs"
PROCESS_TEXT = "process_text"
EXTRACTED_TEXT = "extracted_text"
TAGS = "tags"
EXECUTION_SOURCE = "execution_source"
DOC_ID = "doc_id"
TOOL_EXECUTION_METATADA = "tool_execution_metadata"
EXECUTION_DATA_DIR = "execution_data_dir"
METADATA_FILE = "METADATA.json"