Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
7.5 KB
Raw Blame
/**
 * The from-offset tail read for appended source text (t_a9cd4221).
 *
 * A growing source (a chat log, a session transcript) is re-extracted one
 * appended delta at a time instead of whole, keyed on the extraction
 * watermark the content manifest records ({@link ./content-manifest.ts}).
 * This module is the byte discipline that makes such a watermark honest:
 *
 *   - Deltas are measured and sliced in BYTES, but the emitted text is cut
 *     only at UTF-8-safe boundaries: the start snaps forward past any
 *     partial multi-byte sequence a mid-codepoint offset can land on, and
 *     the end is cut after the LAST complete newline, so an unfinished
 *     trailing line stays uncovered for the next delta instead of being
 *     covered by two deltas that each carry a fragment of it.
 *   - `fromByte`/`toByte` are the exact byte range the returned text was
 *     decoded from. A watermark stamped at `toByte` therefore never
 *     overlaps the next delta and never claims bytes the text does not
 *     carry. When no complete line exists in the appended range, the
 *     delta covers nothing (`toByte === fromByte`) - and what happens to
 *     the watermark then is the CALLER's decision, not this module's.
 *     The production caller ({@link ./ingest.ts}) stamps the extraction
 *     watermark at the announced delta's `toByte`, so an empty delta
 *     leaves the watermark at `fromByte`: the unterminated trailing
 *     fragment stays unstamped, and the next append's delta starts AT it
 *     rather than past it - the fragment is stitched into the series the
 *     first time an append ends with a newline. The ingest lane accepts
 *     the trade because extraction itself stays whole-source - the
 *     fragment is extracted, just never announced as a delta on its own.
 *   - A file that SHRANK below the offset (rotated, truncated, rewritten)
 *     falls back to the full current content with the shrink NAMED - the
 *     full-reprocess direction, never a bogus negative slice.
 *   - The read is bounded: past `maxBytes` it refuses with the shared
 *     bounded-read refusal vocabulary ({@link SOURCE_UNREAD}), the same
 *     names {@link ./ingest.ts} answers bounded reads with.
 *
 * One descriptor per read, positioned reads past the offset: a delta of a
 * large log costs the delta's bytes, not the file's.
 *
 * Language-agnostic: bytes in, decoded UTF-8 text out; no natural-language
 * content is inspected.
 */

import { closeSync, fstatSync, openSync, readSync } from "node:fs";

import {
  PRE_EXTRACT_MAX_SOURCE_BYTES,
  SOURCE_OPEN_FLAGS,
  SOURCE_UNREAD,
  type SourceUnread,
} from "./source-read.ts";

/** Bytes pulled per positioned read, so a delta never over-allocates. */
const READ_CHUNK_BYTES = 65_536;

/** UTF-8 continuation byte: 0b10xxxxxx - never a codepoint start. */
function isContinuationByte(byte: number): boolean {
  return (byte & 0b1100_0000) === 0b1000_0000;
}

/** What one tail read concluded. */
export type TailReadResult =
  | {
      /** Text was appended past `fromByte` (possibly only up to a line boundary). */
      readonly kind: "appended";
      /** The decoded delta text; empty when no complete line exists yet. */
      readonly text: string;
      /** First byte of the source the returned text was decoded from. */
      readonly fromByte: number;
      /** One past the last byte the text covers - the watermark position. */
      readonly toByte: number;
    }
  | {
      /** The file shrank below `fromByte`; the FULL content is the answer. */
      readonly kind: "shrank";
      readonly text: string;
      readonly fromByte: 0;
      readonly toByte: number;
    }
  | {
      /** The read refused: not a regular file, or past the ceiling. */
      readonly kind: "unread";
      readonly unread: SourceUnread;
    };

export interface TailReadOptions {
  /**
   * The most text one read may pull, in bytes. Absent: the same ceiling the
   * code-structure pre-extract pass applies - a tail past it is a
   * reprocess-scale change, not a delta.
   */
  readonly maxBytes?: number;
}

/**
 * Read the text appended to `absolute` since `fromByte`, boundary-safe and
 * bounded. A missing file throws (the caller that opened the ingest knows
 * the source existed moments ago; a vanished source is its own named error,
 * exactly as for {@link ./ingest.ts}'s bounded read).
 */
export function readTail(
  absolute: string,
  fromByte: number,
  opts: TailReadOptions = {},
): TailReadResult {
  const maxBytes = opts.maxBytes ?? PRE_EXTRACT_MAX_SOURCE_BYTES;
  const fd = openSync(absolute, SOURCE_OPEN_FLAGS);
  try {
    const stat = fstatSync(fd);
    if (!stat.isFile()) return { kind: "unread", unread: SOURCE_UNREAD.notAFile };
    if (stat.size < fromByte) {
      const full = readFrom(fd, 0, maxBytes);
      if (full.overCeiling) return { kind: "unread", unread: SOURCE_UNREAD.tooLarge };
      return {
        kind: "shrank",
        text: Buffer.from(full.bytes).toString("utf8"),
        fromByte: 0,
        toByte: full.bytes.length,
      };
    }
    const tail = readFrom(fd, fromByte, maxBytes);
    if (tail.overCeiling) return { kind: "unread", unread: SOURCE_UNREAD.tooLarge };
    return appendedDelta(tail.bytes, fromByte);
  } finally {
    closeSync(fd);
  }
}

/**
 * Cut the appended bytes into an honest delta: decode from the first
 * UTF-8-safe boundary, through the last complete newline, and report the
 * exact byte range the text covers. Exported so the single-pass append
 * snapshot ({@link ./ingest.ts}) cuts the tail bytes it already read with
 * the same discipline {@link readTail} applies, never a second cut rule.
 */
export interface AppendedCut {
  /** The decoded delta text; empty when no complete line exists yet. */
  readonly text: string;
  /** First byte of the source the returned text was decoded from. */
  readonly fromByte: number;
  /** One past the last byte the text covers - the watermark position. */
  readonly toByte: number;
}

export function cutAppendedDelta(bytes: Uint8Array, fromByte: number): AppendedCut {
  let start = 0;
  while (start < bytes.length && isContinuationByte(bytes[start]!)) start++;
  // One past the last complete newline; `start` when the range holds no
  // complete line, in which case the delta covers nothing.
  let end = bytes.length;
  while (end > start && bytes[end - 1] !== 0x0a) end--;
  return {
    text: Buffer.from(bytes.subarray(0, end).subarray(start)).toString("utf8"),
    fromByte: fromByte + start,
    toByte: fromByte + end,
  };
}

/**
 * Shape {@link cutAppendedDelta} into a full tail-read answer.
 */
function appendedDelta(bytes: Uint8Array, fromByte: number): TailReadResult {
  return { kind: "appended", ...cutAppendedDelta(bytes, fromByte) };
}

/**
 * Positioned reads from `start` through EOF, at most `maxBytes` bytes.
 * `overCeiling` is the named refusal: the caller answers with
 * {@link SOURCE_UNREAD.tooLarge} rather than truncating silently.
 */
function readFrom(
  fd: number,
  start: number,
  maxBytes: number,
): { readonly bytes: Uint8Array; readonly overCeiling: boolean } {
  const chunks: Buffer[] = [];
  let filled = 0;
  let position = start;
  for (;;) {
    const chunk = Buffer.allocUnsafe(Math.min(READ_CHUNK_BYTES, maxBytes + 1 - filled));
    const count = readSync(fd, chunk, 0, chunk.length, position);
    if (count === 0) break;
    chunks.push(count === chunk.length ? chunk : chunk.subarray(0, count));
    filled += count;
    position += count;
    if (filled > maxBytes) return { bytes: Buffer.concat(chunks), overCeiling: true };
  }
  return { bytes: Buffer.concat(chunks), overCeiling: false };
}