Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

8.0 KB
Raw Blame
"""Trace parsers for LangChain / OpenAI-style / Soup-serve JSONL logs."""

from __future__ import annotations

import json
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable, Iterator, Optional

from soup_cli.data.traces.messages import content_text, conversation_prompt

SUPPORTED_FORMATS: tuple[str, ...] = ("langchain", "openai", "soup-serve")

# The per-trace signal vocabulary a producer may write. Distinct from
# ``pair_builder.SUPPORTED_SIGNALS``, which names pair *modes*: a trace can carry
# ``thumbs_down`` (a signal) even though no ``--signal thumbs_down`` exists (#1440).
TRACE_SIGNALS: frozenset[str] = frozenset({
    "thumbs_up",
    "thumbs_down",
    "regenerated",
    "user_edit",
})


@dataclass(frozen=True)
class Trace:
    """A single request/response pair extracted from a log."""

    trace_id: str
    prompt: str
    output: str
    signal: str = "none"
    regen_order: Optional[int] = None
    edited_output: Optional[str] = None
    metadata: dict[str, Any] = field(default_factory=dict)


# ---------------------------------------------------------------------------
# LangChain
# ---------------------------------------------------------------------------


def _langchain_prompt(inputs: Any) -> str:
    """Flatten LangChain ``inputs`` into a string prompt."""
    if isinstance(inputs, list):
        return conversation_prompt(inputs)
    if not isinstance(inputs, dict):
        return ""
    messages = inputs.get("messages")
    if isinstance(messages, list):
        return conversation_prompt(messages)
    for key in ("prompt", "input", "query"):
        value = inputs.get(key)
        if isinstance(value, str):
            return value
    return json.dumps(inputs, default=str)


def _langchain_output(outputs: Any) -> str:
    """Extract the assistant response from LangChain ``outputs``."""
    if not isinstance(outputs, dict):
        return ""
    gens = outputs.get("generations")
    if isinstance(gens, list) and gens:
        first = gens[0]
        if isinstance(first, list) and first:
            item = first[0]
        else:
            item = first
        if isinstance(item, dict):
            text = item.get("text") or item.get("content") or ""
            return content_text(text)
    for key in ("output", "response", "text", "content"):
        value = outputs.get(key)
        if isinstance(value, (str, list)):
            return content_text(value)
    return ""


def _langchain_signal(feedback: Any) -> str:
    """Map LangChain feedback to our signal vocabulary."""
    if not isinstance(feedback, list):
        return "none"
    for entry in feedback:
        if not isinstance(entry, dict):
            continue
        if entry.get("key") == "thumbs":
            score = entry.get("score")
            if score == 1 or score == "up":
                return "thumbs_up"
            if score == 0 or score == "down":
                return "thumbs_down"
    return "none"


def parse_langchain(events: Iterable[Any]) -> Iterator[Trace]:
    for idx, event in enumerate(events):
        if not isinstance(event, dict):
            continue
        outputs = event.get("outputs")
        if not outputs:
            continue
        prompt = _langchain_prompt(event.get("inputs"))
        output = _langchain_output(outputs)
        if not prompt or not output:
            continue
        trace_id = str(event.get("id") or f"lc-{idx}")
        signal = _langchain_signal(event.get("feedback"))
        yield Trace(
            trace_id=trace_id, prompt=prompt, output=output, signal=signal,
        )


# ---------------------------------------------------------------------------
# OpenAI-style
# ---------------------------------------------------------------------------


def _openai_prompt(messages: Any) -> str:
    return conversation_prompt(messages)


def _openai_output(choices: Any) -> str:
    if not isinstance(choices, list) or not choices:
        return ""
    first = choices[0]
    if not isinstance(first, dict):
        return ""
    msg = first.get("message")
    if isinstance(msg, dict):
        return content_text(msg.get("content"))
    return content_text(first.get("text"))


def _openai_signal(feedback: Any) -> str:
    if not isinstance(feedback, dict):
        return "none"
    rating = feedback.get("rating")
    if rating in ("up", 1, "thumbs_up"):
        return "thumbs_up"
    if rating in ("down", 0, "thumbs_down"):
        return "thumbs_down"
    return "none"


def parse_openai(events: Iterable[Any]) -> Iterator[Trace]:
    # Track regenerations for the same prompt via "regenerated_from"
    regen_orders: dict[str, int] = {}
    for idx, event in enumerate(events):
        if not isinstance(event, dict):
            continue
        prompt = _openai_prompt(event.get("messages"))
        output = _openai_output(event.get("choices"))
        if not prompt or not output:
            continue
        trace_id = str(event.get("id") or f"oai-{idx}")
        signal = _openai_signal(event.get("feedback"))
        regen_from = event.get("regenerated_from")
        if regen_from:
            order = regen_orders.get(prompt, 0) + 1
            regen_orders[prompt] = order
            signal = "regenerated" if signal == "none" else signal
            yield Trace(
                trace_id=trace_id, prompt=prompt, output=output,
                signal=signal, regen_order=order,
            )
        else:
            regen_orders.setdefault(prompt, 0)
            yield Trace(
                trace_id=trace_id, prompt=prompt, output=output,
                signal=signal, regen_order=regen_orders[prompt],
            )


# ---------------------------------------------------------------------------
# Soup-serve (directory of JSONL files)
# ---------------------------------------------------------------------------


def _record_signal(event: dict[str, Any]) -> str:
    """The signal a ``soup ingest`` record carries.

    ``soup ingest`` writes a top-level ``signal`` field in the canonical
    vocabulary (``TraceRecord.to_dict``); ``soup serve --trace-log`` records
    none, and a hand-written log may still carry the older
    ``feedback.rating``. #1440: reading only ``feedback`` meant Soup's own
    ingest stream was read as ``none`` for every row, so
    ``soup data from-traces`` wrote 0 pairs from it and exited 0.
    """
    signal = event.get("signal")
    if isinstance(signal, str) and signal in TRACE_SIGNALS:
        return signal
    return _openai_signal(event.get("feedback"))


def _edited_output(event: dict[str, Any]) -> Optional[str]:
    """The user's edited answer, under the names a log would use."""
    feedback = event.get("feedback")
    candidates = [
        event.get("edited_output"),
        event.get("edited_response"),
        feedback.get("edited_output") if isinstance(feedback, dict) else None,
    ]
    for value in candidates:
        if isinstance(value, str) and value.strip():
            return value
    return None


def parse_soup_serve(path: str) -> Iterator[Trace]:
    dir_path = Path(path)
    if not dir_path.is_dir():
        return
    for file_path in sorted(dir_path.glob("*.jsonl")):
        # utf-8-sig strips a leading BOM (common on Windows-written JSONL);
        # plain utf-8 left the BOM on line 1, so json.loads failed and the
        # first record of every BOM'd file was silently dropped.
        for line in file_path.read_text(encoding="utf-8-sig").splitlines():
            if not line.strip():
                continue
            try:
                event = json.loads(line)
            except json.JSONDecodeError:
                continue
            if not isinstance(event, dict):
                continue
            prompt = str(event.get("prompt") or "")
            output = str(event.get("response") or event.get("output") or "")
            if not prompt or not output:
                continue
            signal = _record_signal(event)
            yield Trace(
                trace_id=str(event.get("id") or file_path.stem),
                prompt=prompt, output=output, signal=signal,
                edited_output=_edited_output(event),
            )