Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(retention): bound the search_requests log like every other log (#1525)

tashdroideb93fc3bb30f
diff --git a/.env.example b/.env.example
index 8337c34f..a631fae5 100644
--- a/.env.example
+++ b/.env.example
@@ -279,6 +279,13 @@ REQUEST_ANALYTICS_MAX_ROWS=100000
# SERVER_LOGS_RETENTION_DAYS=7
# SERVER_LOGS_MAX_ROWS=50000
+# Web-search attempt log behind the dashboard's Search page (search_requests).
+# Every search attempt appends a row (an agent web_search can issue up to 20),
+# so the log is bounded by age and count on the same pass as the analytics
+# prune. 0 disables that half.
+# SEARCH_LOGS_RETENTION_DAYS=90
+# SEARCH_LOGS_MAX_ROWS=100000
+
# Per-request caller identity (client IP + User-Agent) recorded into request
# analytics and shown in the dashboard's "Recent calls" table. Set to false
# to store nulls instead (aggregate analytics are unaffected).
diff --git a/docs/en/env/01-variables.md b/docs/en/env/01-variables.md
index 43c2b643..c8bd714c 100644
--- a/docs/en/env/01-variables.md
+++ b/docs/en/env/01-variables.md
@@ -118,6 +118,8 @@ In Docker, `127.0.0.1` is the container, not your machine — see [03-outbound-p
| `SERVER_LOGS_MAX_ROWS` | `50000` | Persisted server logs row cap. Set to `0` to disable this limit. |
| `QUOTA_OBSERVATIONS_RETENTION_DAYS` | `30` | Age bound for the provider quota observation audit trail (`provider_quota_observations` behind `provider_quota_state`, `.env.example:230-233`, `server/src/services/request-retention.ts:22-24`). Only the newest row per pool is ever read (indexed seek via `20260901_000002` composite index; previously window-functioned entire log). History older than this is pruned daily. `0` disables age pruning. Split from analytics retention in `77e0ecc`. |
| `QUOTA_OBSERVATIONS_MAX_ROWS` | `200000` | Count bound for the same audit trail (`QUOTA_OBSERVATIONS_MAX_ROWS=200000`, `.env.example:230-233`). Keeps the newest N rows, pruning the excess daily via `ORDER BY created_at DESC, rowid DESC LIMIT … OFFSET maxRows`. `0` disables count pruning. Together the two knobs are `getQuotaObservationRetentionConfig()`; pruned on the daily gate (24h interval, or next 60s tick while backlog remains) in budgeted chunks of `5k` rows / `250ms` (`QUOTA_OBSERVATIONS_PRUNE_CHUNK` / `QUOTA_OBSERVATIONS_PRUNE_BUDGET_MS`, `server/src/services/request-retention.ts:89-90`, refined in `4a8f095` from 20k/870ms). The count bound is far above live pool count, so keeping newest row per pool is guaranteed. |
+| `SEARCH_LOGS_RETENTION_DAYS` | `90` | Age bound for the web-search attempt log (`search_requests` behind the dashboard's Search page). Every search attempt appends a row; the widest analytics window is 90d, so older rows are pure disk growth. Pruned on the same 60s pass as the analytics prune. `0` disables age pruning. |
+| `SEARCH_LOGS_MAX_ROWS` | `100000` | Count bound for the same log: keeps the newest N rows. `0` disables count pruning. Together the two knobs are `getSearchLogRetentionConfig()`. |
| `FREEAPI_DB_PATH` | Default location next to the server build | Optional SQLite location override. Useful on hosts where only one directory is mounted persistently, or to keep the DB outside `server/data`. Example: `/app/server/data/freellmapi.db`. |
| `FREEAPI_DB_BACKUP_PATH` | Unset | Optional encrypted SQLite backup target (file path). On startup, FreeLLMAPI restores this backup if the configured DB file is missing; while running, it uploads a fresh backup periodically. |
| `FREEAPI_DB_BACKUP_URL` | Unset | HTTP(S) backup target, alternative to the path above. |
diff --git a/server/src/__tests__/services/request-retention.test.ts b/server/src/__tests__/services/request-retention.test.ts
index 750d17e3..7c3a9d01 100644
--- a/server/src/__tests__/services/request-retention.test.ts
+++ b/server/src/__tests__/services/request-retention.test.ts
@@ -4,8 +4,22 @@ import { getRequestAnalyticsRetentionConfig, pruneRequestAnalytics } from '../..
const ORIGINAL_RETENTION_DAYS = process.env.REQUEST_ANALYTICS_RETENTION_DAYS;
const ORIGINAL_MAX_ROWS = process.env.REQUEST_ANALYTICS_MAX_ROWS;
+const ORIGINAL_SEARCH_LOGS_RETENTION_DAYS = process.env.SEARCH_LOGS_RETENTION_DAYS;
+const ORIGINAL_SEARCH_LOGS_MAX_ROWS = process.env.SEARCH_LOGS_MAX_ROWS;
function restoreEnv() {
+ if (ORIGINAL_SEARCH_LOGS_RETENTION_DAYS === undefined) {
+ delete process.env.SEARCH_LOGS_RETENTION_DAYS;
+ } else {
+ process.env.SEARCH_LOGS_RETENTION_DAYS = ORIGINAL_SEARCH_LOGS_RETENTION_DAYS;
+ }
+
+ if (ORIGINAL_SEARCH_LOGS_MAX_ROWS === undefined) {
+ delete process.env.SEARCH_LOGS_MAX_ROWS;
+ } else {
+ process.env.SEARCH_LOGS_MAX_ROWS = ORIGINAL_SEARCH_LOGS_MAX_ROWS;
+ }
+
if (ORIGINAL_RETENTION_DAYS === undefined) {
delete process.env.REQUEST_ANALYTICS_RETENTION_DAYS;
} else {
@@ -149,4 +163,39 @@ describe('request analytics retention', () => {
expect((db.prepare("SELECT COUNT(*) AS n FROM response_cache WHERE cache_key = 'cache-expired'").get() as { n: number }).n).toBe(0);
expect((db.prepare("SELECT COUNT(*) AS n FROM response_cache WHERE cache_key = 'cache-live'").get() as { n: number }).n).toBe(1);
});
+
+ it('bounds the web-search attempt log by age and count', () => {
+ process.env.SEARCH_LOGS_RETENTION_DAYS = '7';
+ process.env.SEARCH_LOGS_MAX_ROWS = '2';
+ const db = getDb();
+ const insert = db.prepare(`
+ INSERT INTO search_requests (engine_id, platform, status, latency_ms, created_at)
+ VALUES ('tavily-search', 'tavily', 'ok', 10, ?)
+ `);
+ insert.run('2026-05-01 00:00:00'); // older than 7d from May 8
+ insert.run('2026-05-07 00:00:00');
+ insert.run('2026-05-07 23:00:00');
+ insert.run('2026-05-08 00:00:00');
+
+ const result = pruneRequestAnalytics({
+ db,
+ force: true,
+ now: new Date('2026-05-08T12:00:00Z'),
+ });
+ // Age prune drops May 1; the count cap then trims the rest to the newest 2.
+ expect(result.deleted).toBe(2);
+ const remaining = db.prepare(`SELECT created_at FROM search_requests ORDER BY created_at`).all() as Array<{ created_at: string }>;
+ expect(remaining.map(r => r.created_at)).toEqual(['2026-05-07 23:00:00', '2026-05-08 00:00:00']);
+ });
+
+ it('keeps search rows when both knobs are 0', () => {
+ process.env.SEARCH_LOGS_RETENTION_DAYS = '0';
+ process.env.SEARCH_LOGS_MAX_ROWS = '0';
+ const db = getDb();
+ db.prepare(`INSERT INTO search_requests (engine_id, platform, status, latency_ms, created_at)
+ VALUES ('tavily-search', 'tavily', 'ok', 10, '2020-01-01 00:00:00')`).run();
+
+ pruneRequestAnalytics({ db, force: true, now: new Date('2026-05-08T12:00:00Z') });
+ expect((db.prepare('SELECT COUNT(*) AS n FROM search_requests').get() as { n: number }).n).toBe(1);
+ });
});
diff --git a/server/src/services/request-retention.ts b/server/src/services/request-retention.ts
index 1c0cebac..0d6517ce 100644
--- a/server/src/services/request-retention.ts
+++ b/server/src/services/request-retention.ts
@@ -21,6 +21,12 @@ const DEFAULT_SERVER_LOGS_MAX_ROWS = 50_000;
// month. Pruned on the daily gate: the delete is a range over created_at.
const DEFAULT_QUOTA_OBSERVATIONS_RETENTION_DAYS = 30;
const DEFAULT_QUOTA_OBSERVATIONS_MAX_ROWS = 200_000;
+// search_requests (#1174) is the per-attempt web-search log behind the Search
+// dashboard. Before this it was the one log with no bound at all: every agent
+// web_search (up to 20 per request) appended a row forever. The widest UI
+// window is 90d, so the default age bound matches the `requests` table.
+const DEFAULT_SEARCH_RETENTION_DAYS = 90;
+const DEFAULT_SEARCH_MAX_ROWS = 100_000;
type RetentionDb = ReturnType<typeof getDb>;
@@ -80,6 +86,19 @@ export function getQuotaObservationRetentionConfig(): QuotaObservationRetentionC
};
}
+export interface SearchLogRetentionConfig {
+ retentionDays: number;
+ maxRows: number;
+}
+
+/** Same env convention as the pairs above; 0 on either knob disables that half. */
+export function getSearchLogRetentionConfig(): SearchLogRetentionConfig {
+ return {
+ retentionDays: readNonNegativeInt('SEARCH_LOGS_RETENTION_DAYS', DEFAULT_SEARCH_RETENTION_DAYS),
+ maxRows: readNonNegativeInt('SEARCH_LOGS_MAX_ROWS', DEFAULT_SEARCH_MAX_ROWS),
+ };
+}
+
// The first sweep on an install that predates the prune may face hundreds of
// thousands of rows; deleting them in one statement held the loop for ~4.5s on
// a 470k-row log. Work is chunked under a wall-clock budget instead, and a
@@ -228,6 +247,43 @@ export function pruneExpiredClaims(db: RetentionDb, nowMs: number): number {
return deleted;
}
+/**
+ * Age- and count-bound the web-search attempt log (#1174).
+ *
+ * Folded into the same 60s pass as the server logs. search_requests grew on
+ * every agent web_search attempt with no prune of any kind; its analytics only
+ * answer windows up to 90d ('all' included), so older rows are pure disk
+ * growth. Guarded against the table being absent like every other prune here.
+ */
+export function pruneSearchLogs(db: RetentionDb, nowMs: number): number {
+ const hasTable = !!db
+ .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='search_requests'")
+ .get();
+ if (!hasTable) return 0;
+
+ const { retentionDays, maxRows } = getSearchLogRetentionConfig();
+ let deleted = 0;
+
+ if (retentionDays > 0) {
+ const cutoff = toSqliteTimestamp(new Date(nowMs - retentionDays * DAY_MS));
+ deleted += db.prepare('DELETE FROM search_requests WHERE created_at < ?').run(cutoff).changes;
+ }
+
+ if (maxRows > 0) {
+ deleted += db.prepare(`
+ DELETE FROM search_requests
+ WHERE id IN (
+ SELECT id
+ FROM search_requests
+ ORDER BY created_at DESC, id DESC
+ LIMIT -1 OFFSET ?
+ )
+ `).run(maxRows).changes;
+ }
+
+ return deleted;
+}
+
export function pruneRequestAnalytics(options: {
db?: RetentionDb;
force?: boolean;
@@ -268,6 +324,8 @@ export function pruneRequestAnalytics(options: {
// Expired idempotency claims and cache rows, on the same 60s gate. Both are
// TTL'd replay stores with no reader past expiry and no other sweeper.
deleted += pruneExpiredClaims(db, nowMs);
+ // Web-search attempt log, same gate (see pruneSearchLogs).
+ deleted += pruneSearchLogs(db, nowMs);
// Hourly aggregate prune (gated once per day). The UI's widest window is
// 30d, so we keep at most 30 days of hourly buckets (~720 rows). The table