100+ AI Agents, Agent Skills and RAG Apps - Free and Open Source.
Create wfgy_rag_failure_clinic.py
PSBigBig × MiniPS0d33e77903b3
diff --git a/rag_tutorials/wfgy_rag_failure_clinic/wfgy_rag_failure_clinic.py b/rag_tutorials/wfgy_rag_failure_clinic/wfgy_rag_failure_clinic.pynew file mode 100644index 00000000..4d6a0617--- /dev/null+++ b/rag_tutorials/wfgy_rag_failure_clinic/wfgy_rag_failure_clinic.py@@ -0,0 +1,216 @@+"""+WFGY RAG Failure Clinic+Minimal interactive demo for the WFGY 16 Problem Map inside awesome-llm-apps.+"""++import os+import textwrap+from getpass import getpass++import requests+from openai import OpenAI++PROBLEM_MAP_URL = "https://raw.githubusercontent.com/onestardao/WFGY/main/ProblemMap/README.md"+TXTOS_URL = "https://raw.githubusercontent.com/onestardao/WFGY/main/OS/TXTOS.txt"+WFGY_PROBLEM_MAP_HOME = "https://github.com/onestardao/WFGY/tree/main/ProblemMap"+WFGY_REPO = "https://github.com/onestardao/WFGY"+++EXAMPLE_1 = """=== Example 1 — retrieval hallucination (No.1 style) ===++Context:+You have a simple RAG chatbot that answers questions from a product FAQ.+The FAQ only covers billing rules for your SaaS product and does NOT mention anything about cryptocurrency.++User prompt:+"Can I pay my subscription with Bitcoin?"++Retrieved context (from vector store):+- "We only accept major credit cards and PayPal."+- "All payments are processed in USD."++Model answer:+"Yes, you can pay with Bitcoin. We support several cryptocurrencies through a third-party payment gateway."++Logs:+No errors. Retrieval shows the FAQ chunks above, but the model still confidently invents Bitcoin support.+"""+++EXAMPLE_2 = """=== Example 2 — bootstrap ordering / infra race (No.14 style) ===++Context:+You have a RAG API with three services: api-gateway, rag-worker, and vector-db (for example Qdrant or FAISS).+In local docker compose everything works.++Deployment:+In production, services are deployed on Kubernetes.++Symptom:+Right after a fresh deploy, api-gateway returns 500 errors for the first few minutes.+Logs show connection timeouts from api-gateway to vector-db.++After a few minutes, the errors disappear and the system behaves normally.+You suspect a startup race between api-gateway and vector-db but are not sure how to fix it properly.+"""+++EXAMPLE_3 = """=== Example 3 — secrets / config drift around first deploy (No.16 style) ===++Context:+You added a new environment variable for the RAG pipeline: SECRET_RAG_KEY.+This is required by middleware that signs outgoing requests to an internal search API.++Local:+On developer machines, SECRET_RAG_KEY is defined in .env and everything works.++Production:+You deployed a new version but forgot to add SECRET_RAG_KEY to the production environment.+The first requests after deploy fail with 500 errors and "missing secret" messages in the logs.++After hot-patching the secret into production, the errors stop.+However, similar "first deploy breaks because of missing config" incidents keep happening.+"""+++def fetch_text(url: str) -> str:+ resp = requests.get(url, timeout=30)+ resp.raise_for_status()+ return resp.text+++def build_system_prompt(problem_map: str, txtos: str) -> str:+ header = """+You are an LLM debugger that follows the WFGY 16 Problem Map.++Goal:+Given a description of a bug or failure in an LLM or RAG pipeline, you must:+- Map it to exactly one primary Problem Map number (No.1–No.16).+- Optionally propose one secondary candidate if it is very close.+- Explain your reasoning in plain language.+- Propose a minimal structural fix, not just prompt tweaking.+- When possible, point the user toward the relevant WFGY Problem Map documents.++You are not allowed to invent new problem categories.+You must choose from the sixteen WFGY Problem Map entries only.++About the three built-in examples:+- Example 1 is a clean retrieval hallucination pattern. It should map primarily to No.1.+- Example 2 is a bootstrap ordering or infra race pattern. It should map primarily to No.14.+- Example 3 is a first deploy secrets / config drift pattern. It should map primarily to No.16.+"""+ return (+ textwrap.dedent(header).strip()+ + "\n\n=== TXTOS excerpt ===\n"+ + txtos[:4000]+ + "\n\n=== Problem Map excerpt ===\n"+ + problem_map[:4000]+ )+++def load_wfgy_assets() -> str:+ print("Downloading WFGY Problem Map and TXTOS prompt ...")+ problem_map_text = fetch_text(PROBLEM_MAP_URL)+ txtos_text = fetch_text(TXTOS_URL)+ system_prompt = build_system_prompt(problem_map_text, txtos_text)+ print("Loaded WFGY assets. Ready to debug.\n")+ return system_prompt+++def make_client_and_model():+ api_key = os.getenv("OPENAI_API_KEY")+ if not api_key:+ api_key = getpass("Enter your OpenAI-compatible API key: ").strip()++ base_url = os.getenv("OPENAI_BASE_URL", "").strip()+ if not base_url:+ base_url = "https://api.openai.com/v1"++ model_name = os.getenv("OPENAI_MODEL", "").strip()+ if not model_name:+ model_name = input("Model name (press Enter for gpt-4o): ").strip() or "gpt-4o"++ client = OpenAI(api_key=api_key, base_url=base_url)+ print(f"\nUsing base URL: {base_url}")+ print(f"Using model: {model_name}\n")+ return client, model_name+++def choose_bug_description() -> str:+ print("Choose an example or paste your own bug description:")+ print(" [1] Example 1 — retrieval hallucination (No.1 style)")+ print(" [2] Example 2 — bootstrap ordering / infra race (No.14 style)")+ print(" [3] Example 3 — secrets / config drift (No.16 style)")+ print(" [p] Paste my own RAG / LLM bug\n")++ choice = input("Your choice: ").strip().lower()+ print()++ if choice == "1":+ return EXAMPLE_1+ if choice == "2":+ return EXAMPLE_2+ if choice == "3":+ return EXAMPLE_3++ print("Paste your bug description. End with an empty line.")+ lines = []+ while True:+ try:+ line = input()+ except EOFError:+ break+ if not line.strip():+ break+ lines.append(line)+ user_bug = "\n".join(lines).strip()+ if not user_bug:+ print("No bug description detected, aborting this round.\n")+ return user_bug+++def run_once(client: OpenAI, model_name: str, system_prompt: str) -> None:+ bug = choose_bug_description()+ if not bug:+ return++ print("Running diagnosis ...\n")++ completion = client.chat.completions.create(+ model=model_name,+ temperature=0.2,+ messages=[+ {"role": "system", "content": system_prompt},+ {+ "role": "user",+ "content": (+ "Here is the bug description. "+ "Follow the WFGY 16 Problem Map rules described above.\n\n"+ + bug+ ),+ },+ ],+ )++ reply = completion.choices[0].message.content or ""+ print(reply)+ print("\nFor detailed checklists, visit:")+ print(f"- Problem Map home: {WFGY_PROBLEM_MAP_HOME}")+ print(f"- Full WFGY repo: {WFGY_REPO}\n")+++def main():+ system_prompt = load_wfgy_assets()+ client, model_name = make_client_and_model()++ while True:+ run_once(client, model_name, system_prompt)+ again = input("Debug another bug? (y/n): ").strip().lower()+ if again != "y":+ print("Session finished. Goodbye.")+ break+ print()+++if __name__ == "__main__":+ main()