Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

10.8 KB
Raw Blame
"""V2 filter: chunk position + keyword rules + GPT-4.1 for hard cases.

Tiers:
  Tier 1 (auto-keep): chunk c3+ or page 1+  → ~27K
  Tier 2 (keyword-keep): c0-c2 queries with strong specificity signals → ~15K
  Tier 3 (GPT-4.1 strict): c0-c2 borderline queries → ~5-10K from ~60K
  Tier 4 (auto-remove): c0-c2 generic queries

Target: ~45-55K high-quality records (40-50% of original)
"""

import argparse
import asyncio
import json
import logging
import os
import re
import time

import httpx

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
logger = logging.getLogger(__name__)


def parse_chunk_pos(path):
    m = re.search(r"chunk_(\d+)_(\d+)\.png", path)
    return (int(m.group(1)), int(m.group(2))) if m else (0, 0)


# ---- Keyword signals ----

SPECIFIC_KEYWORDS = [
    # Temporal anchors
    r"\b(in |during |as of )\d{4}\b",
    r"\b(january|february|march|april|may|june|july|august|september|october|november|december)\s+\d",
    r"\b\d{1,2}(st|nd|rd|th)\s+(of\s+)?(january|february|march|april|may|june|july|august|september|october|november|december)",
    # Episode/season
    r"\b(episode|season|series|chapter|volume|issue|part)\s+\d",
    r"\bseason\s+\d+\s+(episode|ep)\s+\d",
    # Rankings/ordinals
    r"\b(first|second|third|fourth|fifth|\d+(st|nd|rd|th))\s+(person|place|team|winner|recipient|player|entry)",
    r"\b(rank|ranked|ranking)\s+\d",
    # Statistics/measurements
    r"\bhow (many|much|long|far|old|tall|high|fast|deep)\b",
    r"\b(score|scored|goals|points|votes|runs|wickets|fouls|assists)\b",
    r"\b(percentage|rate|ratio|proportion|frequency)\b",
    r"\b(population|elevation|area|height|weight|length|distance|speed|temperature)\b",
    r"\b(salary|price|cost|revenue|budget|fine|award|prize)\s+(of|for|in|was)",
    # Specific entities in context
    r"\b(award|prize|medal|trophy|honour)\s+(in|for|winner|recipient)\s+\d{4}",
    r"\b(won|received|awarded|earned)\s+(the|a)\s+\w+\s+(award|prize|medal)",
    r"\bwho (won|received|got|earned)\b.*\b\d{4}\b",
    r"\bwhat (year|date|day|month)\b",
    r"\bwhen did\b.*\b(first|last|final)\b",
    # Table/list signals
    r"\baccording to\b",
    r"\bwhat is the name of the\b.*\b(who|that|which)\b",
    r"\bhex(adecimal)?\s*(code|color|value)\b",
    r"\bboiling point|melting point|molar\b",
    r"\bstrike rate|batting average|bowling\b",
]
SPECIFIC_RE = [re.compile(p, re.IGNORECASE) for p in SPECIFIC_KEYWORDS]

GENERIC_STRONG = [
    r"^what is [\w\s]{3,30}\??$",
    r"^who is [\w\s]{3,30}\??$",
    r"^where is [\w\s]{3,30}\??$",
    r"\bknown for\b",
    r"\bfamous for\b",
    r"\bwhat (type|kind|genre|style) of\b",
    r"^describe\b",
    r"^explain\b",
    r"\bwhat is the (main|primary|official) (language|religion|currency)\b",
    r"\bwhat is the capital of\b",
]
GENERIC_STRONG_RE = [re.compile(p, re.IGNORECASE) for p in GENERIC_STRONG]


def has_specific_signal(query):
    return any(p.search(query) for p in SPECIFIC_RE)


def has_generic_signal(query):
    return any(p.search(query) for p in GENERIC_STRONG_RE)


# ---- GPT-4.1 classification ----

GPT41_SYSTEM = """You filter queries for a visual retrieval training dataset.
We need queries whose answers are in SPECIFIC sections of Wikipedia articles, NOT the introduction.

Rate each query: 1 = answer requires finding a specific fact in a table/list/deep section, 0 = answerable from intro.

STRICT RULES — mark 0 unless the query clearly needs a deep section:
- "What is X?" → 0 (intro)
- "When was X born/founded?" → 0 (infobox)
- "Who is the CEO of X?" → 0 (intro/infobox)
- "What award did X win in 2014?" → 1 (specific year in awards section)
- "How many goals in the 2018 final?" → 1 (match statistics)
- "In episode 7, who..." → 1 (specific episode)
- "What is the boiling point of..." → 1 (properties table)

Respond ONLY with a JSON array of 0s and 1s."""

GPT41_USER = """Rate {n} queries. JSON array of 0/1 only.

{block}"""


async def gpt41_batch(client, queries, batch_idx, semaphore):
    async with semaphore:
        block = "\n".join(f"[{i}] {q}" for i, q in enumerate(queries))
        for attempt in range(3):
            try:
                resp = await client.post(
                    "/chat/completions",
                    json={
                        "model": "gpt-4.1-mini",
                        "messages": [
                            {"role": "system", "content": GPT41_SYSTEM},
                            {
                                "role": "user",
                                "content": GPT41_USER.format(
                                    n=len(queries), block=block
                                ),
                            },
                        ],
                        "temperature": 0,
                        "max_tokens": 300,
                    },
                    timeout=60,
                )
                resp.raise_for_status()
                content = resp.json()["choices"][0]["message"]["content"].strip()
                if content.startswith("```"):
                    content = content.split("\n", 1)[1].rsplit("```", 1)[0].strip()
                labels = json.loads(content)
                if isinstance(labels, list):
                    while len(labels) < len(queries):
                        labels.append(0)
                    return batch_idx, [int(bool(x)) for x in labels[: len(queries)]]
                return batch_idx, [0] * len(queries)
            except Exception as e:
                if attempt == 2:
                    logger.warning(f"Batch {batch_idx}: {e}")
                    return batch_idx, [0] * len(queries)
                await asyncio.sleep(2**attempt)


async def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--input", required=True)
    parser.add_argument("--output", required=True)
    parser.add_argument("--batch-size", type=int, default=40)
    parser.add_argument("--concurrency", type=int, default=25)
    parser.add_argument("--api-key", default=None)
    parser.add_argument("--base-url", default="https://us.api.openai.com/v1")
    args = parser.parse_args()

    api_key = args.api_key or os.environ.get("OPENAI_API_KEY")

    logger.info(f"Loading {args.input}")
    records = []
    with open(args.input) as f:
        for line in f:
            records.append(json.loads(line))
    N = len(records)
    logger.info(f"Loaded {N} records")

    # ---- Tier assignments ----
    tier1_keep = set()  # deep chunks
    tier2_keep = set()  # keyword-specific in c0-c2
    tier4_remove = set()  # clearly generic
    tier3_classify = []  # borderline → LLM

    for i, r in enumerate(records):
        page, chunk = parse_chunk_pos(r["chunk_path"])
        q = r["query"].strip()

        if page >= 1 or chunk >= 3:
            tier1_keep.add(i)
        elif has_generic_signal(q) and not has_specific_signal(q):
            tier4_remove.add(i)
        elif has_specific_signal(q):
            tier2_keep.add(i)
        elif len(q.split()) < 6:
            tier4_remove.add(i)
        else:
            tier3_classify.append(i)

    logger.info(
        f"Tier 1 (deep chunk auto-keep): {len(tier1_keep)} ({len(tier1_keep) / N * 100:.1f}%)"
    )
    logger.info(
        f"Tier 2 (keyword-specific keep): {len(tier2_keep)} ({len(tier2_keep) / N * 100:.1f}%)"
    )
    logger.info(
        f"Tier 3 (LLM classify): {len(tier3_classify)} ({len(tier3_classify) / N * 100:.1f}%)"
    )
    logger.info(
        f"Tier 4 (generic remove): {len(tier4_remove)} ({len(tier4_remove) / N * 100:.1f}%)"
    )

    # ---- LLM on tier 3 ----
    classify_pairs = [(idx, records[idx]["query"]) for idx in tier3_classify]
    batches = []
    for i in range(0, len(classify_pairs), args.batch_size):
        batches.append(classify_pairs[i : i + args.batch_size])

    logger.info(f"Running LLM on {len(batches)} batches...")
    llm_keep = set()
    semaphore = asyncio.Semaphore(args.concurrency)

    async with httpx.AsyncClient(
        base_url=args.base_url,
        headers={"Authorization": f"Bearer {api_key}"},
        timeout=90,
    ) as client:
        tasks = [
            gpt41_batch(client, [q for _, q in b], bi, semaphore)
            for bi, b in enumerate(batches)
        ]
        done = 0
        t0 = time.time()
        for coro in asyncio.as_completed(tasks):
            bi, labels = await coro
            for (orig_idx, _), label in zip(batches[bi], labels):
                if label == 1:
                    llm_keep.add(orig_idx)
            done += 1
            if done % 200 == 0 or done == len(tasks):
                elapsed = time.time() - t0
                rate = done / elapsed if elapsed > 0 else 1
                eta = (len(tasks) - done) / rate
                logger.info(
                    f"  LLM: {done}/{len(tasks)}, kept {len(llm_keep)} "
                    f"({len(llm_keep) / max(1, done * args.batch_size) * 100:.0f}%), "
                    f"ETA {eta:.0f}s"
                )

    # ---- Combine ----
    all_keep = tier1_keep | tier2_keep | llm_keep
    logger.info("\n=== FINAL RESULTS ===")
    logger.info(f"Total kept: {len(all_keep)}/{N} = {len(all_keep) / N * 100:.1f}%")
    logger.info(f"  Tier 1 (deep chunks):    {len(tier1_keep)}")
    logger.info(f"  Tier 2 (keyword):        {len(tier2_keep)}")
    logger.info(f"  Tier 3 (LLM approved):   {len(llm_keep)}")
    logger.info(f"  Removed (generic+LLM):   {N - len(all_keep)}")

    # Position breakdown
    from collections import Counter

    pos = Counter()
    for i in all_keep:
        p, c = parse_chunk_pos(records[i]["chunk_path"])
        pos[f"p{p}_c{c}"] += 1
    logger.info("Chunk position breakdown:")
    for k, v in sorted(pos.items(), key=lambda x: -x[1])[:12]:
        logger.info(f"  {k}: {v}")

    # Write
    os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
    kept = 0
    with open(args.output, "w") as f:
        for i, rec in enumerate(records):
            if i in all_keep:
                f.write(json.dumps(rec, ensure_ascii=False) + "\n")
                kept += 1
    logger.info(f"Wrote {kept} to {args.output}")

    # Sample for inspection
    import random

    random.seed(42)
    sample_file = args.output.replace(".jsonl", "_sample200.jsonl")
    sample = random.sample(sorted(all_keep), min(200, len(all_keep)))
    with open(sample_file, "w") as f:
        for i in sample:
            f.write(json.dumps(records[i], ensure_ascii=False) + "\n")
    logger.info(f"Sample saved to {sample_file}")

    # Also save REMOVED sample for comparison
    removed = set(range(N)) - all_keep
    removed_sample = random.sample(sorted(removed), min(200, len(removed)))
    removed_file = args.output.replace(".jsonl", "_removed_sample200.jsonl")
    with open(removed_file, "w") as f:
        for i in removed_sample:
            f.write(json.dumps(records[i], ensure_ascii=False) + "\n")
    logger.info(f"Removed sample saved to {removed_file}")


if __name__ == "__main__":
    asyncio.run(main())