Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

7.7 KB
Raw Blame
#!/usr/bin/env python3
"""
Continuous anomaly detection with an XERJ _ml datafeed.

The on-demand `_score` API scores a series once, on request. A *datafeed* turns
that into a live monitor: it references a detector, and once started it re-scores
the source index every `frequency`, appending only the NEW flagged buckets to the
job's results store — which you poll with
`GET /_ml/anomaly_detectors/{job}/results/records`. Same transparent moving
mean/stddev z-score, now on a timer.

This demo proves the "continuous" part end to end:
  1. ingest a metric series with one spike, create a detector + datafeed,
  2. _start it (one scoring pass runs immediately) and read the first anomaly,
  3. ingest a SECOND spike into the live index,
  4. poll until the background scorer picks it up — with no second _start.

Run:  python3 datafeed_demo.py                       # assumes XERJ on :9200
      XERJ_URL=http://localhost:9200 python3 datafeed_demo.py

Uses only the Python 3 standard library. No pip installs.
"""

import datetime
import os
import sys
import time
import json
import urllib.error
import urllib.request

# Primary env var is XERJ_URL (default :9200); XERJ is kept as a legacy alias.
XERJ = os.environ.get("XERJ_URL", os.environ.get("XERJ", "http://localhost:9200")).rstrip("/")
INDEX = "cpu_datafeed"
DETECTOR = "cpu-spike"
DATAFEED = "cpu-spike-feed"
FREQUENCY = "2s"  # re-score cadence for the background scorer


def call(method, path, body=None):
    data = None
    headers = {}
    if body is not None:
        data = body.encode() if isinstance(body, str) else json.dumps(body).encode()
        headers["Content-Type"] = (
            "application/x-ndjson" if isinstance(body, str) else "application/json"
        )
    req = urllib.request.Request(f"{XERJ}{path}", data=data, headers=headers, method=method)
    with urllib.request.urlopen(req) as resp:
        return json.loads(resp.read())


def get_text(path):
    """GET a plain-text endpoint (e.g. the _cat family) and return it stripped."""
    req = urllib.request.Request(f"{XERJ}{path}", method="GET")
    with urllib.request.urlopen(req) as resp:
        return resp.read().decode().strip()


def ingest(readings):
    """Ingest [(minute, cpu)] as one doc per minute (one reading per bucket)."""
    lines = []
    for minute, cpu in readings:
        ts = "2026-07-06T00:%02d:00Z" % minute
        lines.append(json.dumps({"index": {"_index": INDEX}}))
        lines.append(json.dumps({"@timestamp": ts, "cpu": cpu}))
    call("POST", "/_bulk?refresh=true", "\n".join(lines) + "\n")


def when(record):
    ts = datetime.datetime.fromtimestamp(record["timestamp"] / 1000, datetime.timezone.utc)
    return ts.strftime("%Y-%m-%d %H:%M UTC")


def show(records):
    print("  when                   actual   expected   record_score")
    for r in sorted(records, key=lambda r: r["timestamp"]):
        print("  %-20s %8.1f %10.1f %14.1f"
              % (when(r), r["actual"], r["expected"], r["record_score"]))


def poll_records(min_count, timeout_s=15):
    """Poll results/records until at least `min_count` are stored (or time out)."""
    deadline = time.time() + timeout_s
    while time.time() < deadline:
        recs = call("GET", f"/_ml/anomaly_detectors/{DETECTOR}/results/records")["records"]
        if len(recs) >= min_count:
            return recs
        time.sleep(1)
    return call("GET", f"/_ml/anomaly_detectors/{DETECTOR}/results/records")["records"]


def main():
    # Fresh index: @timestamp (bucketing clock) + cpu (the metric we average).
    try:
        call("DELETE", f"/{INDEX}")
    except urllib.error.HTTPError:
        pass
    call("PUT", f"/{INDEX}", {
        "mappings": {"properties": {
            "@timestamp": {"type": "date"},
            "cpu": {"type": "double"},
        }}
    })

    # Baseline ~20% CPU for 16 minutes, with a runaway spike to 96 at minute 12.
    baseline = [19, 21, 20, 20, 22, 18, 21, 20, 19, 21, 20, 22, 96, 20, 21, 19]
    ingest(list(enumerate(baseline)))

    # A detector is which index, which clock field, what metric, bucket size,
    # and how many stddevs count as anomalous.
    call("PUT", f"/_ml/anomaly_detectors/{DETECTOR}", {
        "source_index": INDEX,
        "time_field": "@timestamp",
        "function": "mean",
        "field": "cpu",
        "bucket_span": "1m",
        "anomaly_threshold": 3.0,
        "description": "per-minute CPU mean vs moving baseline",
    })

    # A datafeed points a detector at a live index and runs it on a timer.
    feed = call("PUT", f"/_ml/datafeeds/{DATAFEED}", {
        "job_id": DETECTOR,
        "frequency": FREQUENCY,
    })
    print(f"datafeed {feed['datafeed_id']} -> job {feed['job_id']} "
          f"(every {feed['frequency']}), state={feed['state']}")
    print("  raw datafeed config:")
    for line in json.dumps(feed, indent=2, sort_keys=True).splitlines():
        print("  " + line)
    print()

    # Start it: one scoring pass runs synchronously now, then a background task
    # re-scores every `frequency`.
    call("POST", f"/_ml/datafeeds/{DATAFEED}/_start")
    first = poll_records(1)
    print(f"after _start — {len(first)} anomaly record(s) from the first pass:")
    show(first)
    # Dump the full first record so every field the recipe documents
    # (timestamp, typical, std_dev, z_score, record_score) is a value THIS run
    # actually produced — no hand-copied JSON.
    print("\n  raw first record:")
    for line in json.dumps(first[0], indent=2, sort_keys=True).splitlines():
        print("  " + line)

    # Now the "continuous" part: a NEW spike lands in the live index at minute
    # 20. We do NOT touch the datafeed — the running background scorer should
    # find it on its next tick.
    print(f"\ninjecting a new spike at 00:20, waiting for the {FREQUENCY} scorer...")
    ingest([(16, 20), (17, 21), (18, 19), (19, 20), (20, 88), (21, 20)])
    after = poll_records(len(first) + 1)
    new = [r for r in after if r not in first]
    print(f"datafeed picked up {len(new)} new anomaly record(s) on its own:")
    show(new)

    # Filter + cap the stored records (section 5 of the recipe).
    high = call("GET", f"/_ml/anomaly_detectors/{DETECTOR}/results/records?record_score=90")["records"]
    capped = call("GET", f"/_ml/anomaly_detectors/{DETECTOR}/results/records?size=1")["records"]
    print(f"\nfilter record_score>=90 -> {len(high)} record(s); size=1 cap -> {len(capped)} record(s)")

    # Stop the datafeed (idempotent cleanup for re-runs).
    stopped = call("POST", f"/_ml/datafeeds/{DATAFEED}/_stop")
    fed = call("GET", f"/_ml/datafeeds/{DATAFEED}")["datafeeds"][0]
    print(f"_stop -> {stopped} · datafeed state now: {fed['state']}")

    # A stopped datafeed can be deleted; a started one returns 409 (ES parity).
    cat = get_text("/_cat/ml/datafeeds")
    print(f"_cat/ml/datafeeds -> {cat!r}")
    deleted = call("DELETE", f"/_ml/datafeeds/{DATAFEED}")
    print(f"DELETE datafeed -> {deleted}")

    # Verify: both spikes were detected, the second WITHOUT re-starting.
    spikes = {when(r): r["actual"] for r in after}
    assert any(96.0 == v for v in spikes.values()), "first spike (00:12) missing"
    assert any(88.0 == v for v in spikes.values()), "continuously-detected spike (00:20) missing"
    assert len(after) >= 2, f"expected >=2 anomaly records, got {len(after)}"
    assert len(high) == len(after), "record_score>=90 should keep both 100.0-score records"
    assert len(capped) == 1, "size=1 should cap to a single record"
    assert fed["state"] == "stopped", "datafeed did not stop"
    print(f"\nOK: {len(after)} anomalies detected continuously; datafeed stopped cleanly.")


if __name__ == "__main__":
    try:
        main()
    except AssertionError as e:
        print(f"FAILED: {e}", file=sys.stderr)
        sys.exit(1)