Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): keep the keyless daily credit window fixed from first use (#5466)
Erik Engervall931aeabc13b9
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.tsindex f7da83517..7e37b5d2f 100644--- a/apps/api/src/__tests__/snips/v2/keyless.test.ts+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts@@ -14,6 +14,12 @@ import * as schema from "../../../db/schema";import { and, desc, eq, gt } from "drizzle-orm";import request from "supertest";import { decryptKeylessSignupToken } from "../../../lib/keyless-signup-link";+import {+ adjustKeylessCredits,+ chargeKeylessCredits,+ keylessTeamId,+ reserveKeylessCredits,+} from "../../../lib/keyless";// The keyless tier is disabled unless both limits are configured. The harness// passes the shell env through to the server, so we read the same env here and@@ -925,3 +931,68 @@ describeIf(SPUR_ENABLED)("Keyless free tier — Spur IP reputation", () => {scrapeTimeout,);});++// The daily credit window opens on first use and must not move after that.+describe("Keyless credit window", () => {+ const ip = "198.51.100.21";+ const key = `keyless_credits:${ip}`;+ const teamId = keylessTeamId(ip);++ async function expectWindowKept(previousTtl: number) {+ const ttl = await redisRateLimitClient.ttl(key);+ expect(ttl).toBeGreaterThan(0);+ expect(ttl).toBeLessThanOrEqual(previousTtl);+ return ttl;+ }++ beforeEach(async () => {+ await redisRateLimitClient.del(key);+ });++ afterAll(async () => {+ await redisRateLimitClient.del(key);+ });++ it("keeps the window's expiry when a reservation is reconciled", async () => {+ await redisRateLimitClient.set(key, "5", "EX", 600);++ expect(await adjustKeylessCredits(teamId, 3)).toBe(8);+ const ttl = await expectWindowKept(600);++ expect(await adjustKeylessCredits(teamId, -20)).toBe(0);+ await expectWindowKept(ttl);+ });++ it("does not open a window when a refund finds no counter", async () => {+ expect(await adjustKeylessCredits(teamId, -3)).toBe(0);+ expect(await redisRateLimitClient.exists(key)).toBe(0);+ });++ it("opens the window when reconciliation finds none", async () => {+ expect(await adjustKeylessCredits(teamId, 2)).toBe(2);+ expect(await redisRateLimitClient.ttl(key)).toBeGreaterThan(600);++ await redisRateLimitClient.set(key, "5");+ expect(await adjustKeylessCredits(teamId, 1)).toBe(6);+ expect(await redisRateLimitClient.ttl(key)).toBeGreaterThan(600);+ });++ it("keeps the window's expiry when a completed request is charged", async () => {+ await redisRateLimitClient.set(key, "0", "EX", 600);++ await chargeKeylessCredits(teamId, 2);+ expect(await redisRateLimitClient.get(key)).toBe("2");+ await expectWindowKept(600);+ });++ itIf(KEYLESS_CREDITS_PER_DAY > 0)(+ "keeps the window's expiry when credits are reserved after a full refund",+ async () => {+ await redisRateLimitClient.set(key, "0", "EX", 600);++ expect((await reserveKeylessCredits(teamId, 1)).ok).toBe(true);+ expect(await redisRateLimitClient.get(key)).toBe("1");+ await expectWindowKept(600);+ },+ );+});diff --git a/apps/api/src/lib/__tests__/keyless-usage-log.test.ts b/apps/api/src/lib/__tests__/keyless-usage-log.test.tsindex e2502637c..520f8a701 100644--- a/apps/api/src/lib/__tests__/keyless-usage-log.test.ts+++ b/apps/api/src/lib/__tests__/keyless-usage-log.test.ts@@ -7,13 +7,12 @@// carrying the client IP (the durable `keyless_credit_usage` row is deferred// to the firecrawl-db migration), writes no DB row, and charges nothing;// billable requests keep writing rows exactly as before.-const { dbInsert, insertValues, redisIncrby, redisExpire } = vi.hoisted(() => {+const { dbInsert, insertValues, redisEval } = vi.hoisted(() => {const insertValues = vi.fn().mockResolvedValue(undefined);return {insertValues,dbInsert: vi.fn(() => ({ values: insertValues })),- redisIncrby: vi.fn().mockResolvedValue(1),- redisExpire: vi.fn().mockResolvedValue(1),+ redisEval: vi.fn().mockResolvedValue([1, 1]),};});@@ -25,12 +24,10 @@ vi.mock("../../db/connection", () => ({vi.mock("../../services/rate-limiter", () => ({redisRateLimitClient: {- incrby: redisIncrby,- expire: redisExpire,get: vi.fn().mockResolvedValue(null),incr: vi.fn().mockResolvedValue(1),ttl: vi.fn().mockResolvedValue(-1),- eval: vi.fn().mockResolvedValue([1, 1]),+ eval: redisEval,},}));@@ -161,7 +158,7 @@ describe("chargeKeylessCredits", () => {await chargeKeylessCredits(KEYLESS_TEAM, 0);- expect(redisIncrby).not.toHaveBeenCalled();+ expect(redisEval).not.toHaveBeenCalled();expect(dbInsert).not.toHaveBeenCalled();expect(info).toHaveBeenCalledWith(expect.stringContaining("Keyless zero-credit usage"),@@ -172,7 +169,13 @@ describe("chargeKeylessCredits", () => {it("charges the credit budget for a billable request", async () => {await chargeKeylessCredits(KEYLESS_TEAM, 4);- expect(redisIncrby).toHaveBeenCalledWith(`keyless_credits:${IP}`, 4);+ expect(redisEval).toHaveBeenCalledWith(+ expect.any(String),+ 1,+ `keyless_credits:${IP}`,+ 4,+ 86400,+ );expect(insertValues).toHaveBeenCalledWith(expect.objectContaining({ ip: IP, credits_used: 4 }),);@@ -181,7 +184,7 @@ describe("chargeKeylessCredits", () => {it("does nothing at all for a team that is not keyless", async () => {await chargeKeylessCredits("some-real-team-id", 5);- expect(redisIncrby).not.toHaveBeenCalled();+ expect(redisEval).not.toHaveBeenCalled();expect(dbInsert).not.toHaveBeenCalled();});});diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex 72d23e319..8755c0d80 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -382,7 +382,7 @@ if current + projected > limit thenreturn {0, current}endlocal total = redis.call("INCRBY", KEYS[1], projected)-if total == projected then+if redis.call("TTL", KEYS[1]) < 0 thenredis.call("EXPIRE", KEYS[1], ttl)endreturn {1, total}@@ -405,6 +405,7 @@ return {1, total}/*** Reconcile a prior keyless reservation to actual credits. Delta may be* negative; the counter is clamped at zero to tolerate retries or races.+ * The daily window keeps the expiry set on first use.*/export async function adjustKeylessCredits(teamId: string,@@ -423,11 +424,17 @@ export async function adjustKeylessCredits(local current = tonumber(redis.call("GET", KEYS[1]) or "0")local delta = tonumber(ARGV[1])local ttl = tonumber(ARGV[2])+if delta <= 0 and redis.call("EXISTS", KEYS[1]) == 0 then+ return 0+endlocal next = current + deltaif next < 0 thennext = 0end-redis.call("SET", KEYS[1], next, "EX", ttl)+redis.call("SET", KEYS[1], next, "KEEPTTL")+if redis.call("TTL", KEYS[1]) < 0 then+ redis.call("EXPIRE", KEYS[1], ttl)+endreturn next`,1,@@ -655,10 +662,18 @@ export async function chargeKeylessCredits(const inc = Math.ceil(credits);try {const key = creditsKey(ip);- const total = await redisRateLimitClient.incrby(key, inc);- if (total === inc) {- await redisRateLimitClient.expire(key, DAY_SECONDS);- }+ await redisRateLimitClient.eval(+ `+redis.call("INCRBY", KEYS[1], ARGV[1])+if redis.call("TTL", KEYS[1]) < 0 then+ redis.call("EXPIRE", KEYS[1], ARGV[2])+end+`,+ 1,+ key,+ inc,+ DAY_SECONDS,+ );} catch {// Counter is best-effort; a missed charge just means the IP gets a few// extra free credits today.