TypeScript
import { NodeCrypto, NodeFileSystem } from "@effect/platform-node"
import { SqliteClient } from "@effect/sql-sqlite-node"
import { assert, describe, expect, it } from "@effect/vitest"
import { Effect, Exit, Fiber, FileSystem, Latch, Layer, Option } from "effect"
import {
Entity,
EntityAddress,
EntityId,
EntityType,
Envelope,
Message,
MessageStorage,
RunnerHealth,
Runners,
RunnerStorage,
ShardId,
Sharding,
ShardingConfig,
Snowflake,
SqlMessageStorage
} from "effect/cluster"
import { Reactivity } from "effect/reactivity"
import { SqlClient, Statement } from "effect/sql"
import { TestClock } from "effect/testing"
import { MssqlContainer } from "../fixtures/mssql-utils.ts"
import { MysqlContainer } from "../fixtures/mysql2-utils.ts"
import { PgContainer } from "../fixtures/pg-utils.ts"
import {
GetUserRpc,
LongKeyRpc,
makeAckChunk,
makeChunkReply,
makeReply,
makeRequest,
PrimaryKeyTest,
StreamRpc
} from "./MessageStorageTest.ts"
const TestEntity = Entity.make("test", [GetUserRpc])
const TestEntityLayer = TestEntity.toLayer({
GetUser: () => Effect.void
})
const StorageLayer = SqlMessageStorage.layer.pipe(
Layer.provideMerge(Snowflake.layerGenerator),
Layer.provide([ShardingConfig.layerDefaults, NodeCrypto.layerDigest])
)
const truncate = Effect.gen(function*() {
const sql = yield* SqlClient.SqlClient
yield* sql`DELETE FROM cluster_replies`
yield* sql`DELETE FROM cluster_messages`
})
describe("SqlMessageStorage", () => {
it.effect("preserves ids above 2^53 with a number-returning SQLite driver", () => {
let id = 9007199254740993n
return Effect.gen(function*() {
const storage = yield* MessageStorage.MessageStorage
const request = yield* makeRequest({
rpc: PrimaryKeyTest,
payload: PrimaryKeyTest.payloadSchema.make({ id: 123 })
})
const requestId = request.envelope.requestId
yield* storage.saveRequest(request)
const shards = [request.envelope.address.shardId]
const messages = yield* storage.unprocessedMessages(shards)
assert.strictEqual(messages.length, 1)
assert.strictEqual(messages[0].envelope.requestId, requestId)
assert.deepStrictEqual(yield* storage.unprocessedMessages(shards), [])
const reply = yield* makeReply(request)
yield* storage.saveReply(reply)
assert.deepStrictEqual(yield* storage.repliesFor([request]), [reply.reply])
yield* storage.resetShards(shards)
assert.deepStrictEqual(yield* storage.unprocessedMessages(shards), [])
const duplicate = yield* storage.saveRequest(
yield* makeRequest({ rpc: PrimaryKeyTest, payload: request.envelope.payload })
)
assert(duplicate._tag === "Duplicate")
assert.strictEqual(duplicate.originalId, requestId)
assert.deepStrictEqual(duplicate.lastReceivedReply, Option.some(reply.reply))
assert.deepStrictEqual(
yield* storage.requestIdForPrimaryKey({
address: request.envelope.address,
tag: request.envelope.tag,
id: "123"
}),
Option.some(requestId)
)
}).pipe(
Effect.provide(SqlMessageStorage.layer.pipe(
Layer.provide(NumberSqliteLayer),
Layer.provide([ShardingConfig.layerDefaults, NodeCrypto.layer])
)),
Effect.provideService(Snowflake.Generator, {
nextUnsafe: () => {
const next = Snowflake.Snowflake(id)
id += 2n
return next
},
setMachineId: () => Effect.void
})
)
})
;([
["pg", Layer.orDie(PgContainer.layerClient)],
["mysql", Layer.orDie(MysqlContainer.layerClient)],
["mssql", Layer.orDie(MssqlContainer.layerClient)],
["sqlite", Layer.orDie(SqliteLayer)]
] as const).forEach(([label, layer]) => {
// Tests truncate this backend's shared tables.
it.layer(StorageLayer.pipe(Layer.provideMerge(layer)), {
concurrent: false,
timeout: 120000
})(label, (it) => {
if (label === "pg") {
for (const reset of ["resetShards", "resetAddresses"] as const) {
it.effect(`${reset} does not deadlock with a concurrent message claim`, () =>
Effect.gen(function*() {
yield* truncate
const sql = yield* SqlClient.SqlClient
const storage = yield* SqlMessageStorage.makeEncoded().pipe(Effect.provide(NodeCrypto.layerDigest))
// Heap order is the reverse of the claim query's rowid order.
yield* sql`
INSERT INTO cluster_messages
(id, rowid, shard_id, entity_type, entity_id, kind, tag, payload, headers, processed, request_id)
VALUES
(2, 2, 'default:1', 'test', '1', 0, 'GetUser', '{}', '{}', FALSE, 2),
(1, 1, 'default:1', 'test', '1', 0, 'GetUser', '{}', '{}', FALSE, 1)
`
// Pause after the reset locks row 2, so the claim can lock row 1.
yield* sql`
CREATE FUNCTION cluster_test_slow_reset() RETURNS trigger AS $$
BEGIN
IF NEW.last_read IS NULL AND NEW.rowid = 2 THEN
PERFORM pg_sleep(2);
END IF;
RETURN NEW;
END $$ LANGUAGE plpgsql
`
yield* sql`
CREATE TRIGGER cluster_test_slow_reset BEFORE UPDATE ON cluster_messages
FOR EACH ROW EXECUTE FUNCTION cluster_test_slow_reset()
`
yield* Effect.gen(function*() {
const resetFiber = yield* Effect.gen(function*() {
// Make the unordered UPDATE use heap order regardless of indexes.
yield* sql`SET LOCAL enable_indexscan = off`
yield* sql`SET LOCAL enable_bitmapscan = off`
yield* reset === "resetShards"
? storage.resetShards(["default:1"])
: storage.resetAddresses([EntityAddress.make({
shardId: ShardId.make("default", 1),
entityType: EntityType.make("test"),
entityId: EntityId.make("1")
})])
}).pipe(sql.withTransaction, Effect.exit, Effect.forkChild)
yield* Effect.gen(function*() {
while (true) {
const waiting = yield* sql`
SELECT 1 FROM pg_stat_activity
WHERE pid <> pg_backend_pid() AND wait_event = 'PgSleep'
AND query LIKE '%cluster_messages%'
`
if (waiting.length > 0) return
yield* Effect.sleep("10 millis")
}
}).pipe(Effect.timeout("5 seconds"))
const claim = yield* Effect.exit(storage.unprocessedMessages(["default:1"], Date.now()))
const resetResult = yield* Fiber.join(resetFiber)
expect(Exit.isSuccess(resetResult)).toBe(true)
assert(Exit.isSuccess(claim), String(claim))
expect(claim.value.map((message) => message.envelope.requestId)).toEqual(["1", "2"])
expect(yield* storage.unprocessedMessages(["default:1"], Date.now())).toHaveLength(0)
}).pipe(Effect.ensuring(
Effect.gen(function*() {
yield* sql`DROP TRIGGER cluster_test_slow_reset ON cluster_messages`
yield* sql`DROP FUNCTION cluster_test_slow_reset()`
yield* truncate
}).pipe(Effect.orDie)
))
}).pipe(TestClock.withLive))
}
it.effect("creates an index for insertion-ordered message reads", () =>
Effect.gen(function*() {
const sql = yield* SqlClient.SqlClient
const indexes = yield* sql<{ indexname: string }>`
SELECT indexname
FROM pg_indexes
WHERE tablename = 'cluster_messages'
AND indexname = 'cluster_messages_rowid_idx'
`
expect(indexes).toHaveLength(1)
}))
}
if (label === "mssql") {
it.effect("rejects duplicate exit replies when another table has the same index name", () =>
Effect.gen(function*() {
const sql = yield* SqlClient.SqlClient
yield* sql`CREATE TABLE collision_other (id INT)`
yield* sql`CREATE INDEX collision_replies_one_exit ON collision_other (id)`
const storage = yield* SqlMessageStorage.makeEncoded({ prefix: "collision" }).pipe(
Effect.provide(NodeCrypto.layer)
)
const exit = { _tag: "WithExit", requestId: "1", exit: { _tag: "Success", value: null } } as const
yield* storage.saveReply({ ...exit, id: "1" })
const error = yield* Effect.flip(storage.saveReply({ ...exit, id: "2" }))
expect(error._tag).toEqual("PersistenceError")
}))
}
it.effect("resetRequests with no IDs leaves existing claims untouched", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const request = yield* makeRequest()
yield* storage.saveRequest(request)
expect(yield* storage.unprocessedMessages([request.envelope.address.shardId])).toHaveLength(1)
yield* storage.resetRequests([])
expect(yield* storage.unprocessedMessages([request.envelope.address.shardId])).toHaveLength(0)
}))
it.effect("resetRequests releases only the selected request at a shared address", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const selected = yield* makeRequest()
const unrelated = yield* makeRequest()
yield* storage.saveRequest(selected)
yield* storage.saveRequest(unrelated)
const shards = [selected.envelope.address.shardId]
expect(yield* storage.unprocessedMessages(shards)).toHaveLength(2)
yield* storage.resetRequests([selected.envelope.requestId])
const messages = yield* storage.unprocessedMessages(shards)
expect(messages.map((message) => message.envelope.requestId)).toEqual([selected.envelope.requestId])
expect(yield* storage.unprocessedMessages(shards)).toHaveLength(0)
}))
it.effect("resetRequests preserves chunk replies, exit replies, and completed state", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const streaming = yield* makeRequest({ rpc: StreamRpc, payload: StreamRpc.payloadSchema.make({ id: 123 }) })
const completed = yield* makeRequest()
yield* storage.saveRequest(streaming)
yield* storage.saveRequest(completed)
const shards = [streaming.envelope.address.shardId]
expect(yield* storage.unprocessedMessages(shards)).toHaveLength(2)
yield* storage.saveReply(yield* makeChunkReply(streaming))
yield* storage.saveReply(yield* makeReply(completed))
const replies = yield* storage.repliesFor([streaming, completed])
expect(replies).toHaveLength(2)
expect(yield* storage.unprocessedMessages(shards)).toHaveLength(0)
const sql = yield* SqlClient.SqlClient
const processed = yield* sql`SELECT processed FROM cluster_messages ORDER BY rowid`
yield* storage.resetRequests([streaming.envelope.requestId, completed.envelope.requestId])
const messages = yield* storage.unprocessedMessages(shards)
// Replies keep both requests out of the SQL read loop.
expect(messages).toHaveLength(0)
expect(yield* storage.repliesFor([streaming, completed])).toEqual(replies)
expect(yield* sql`SELECT processed FROM cluster_messages ORDER BY rowid`).toEqual(processed)
yield* truncate
}))
it.effect("clearReplies requeues when the expected reply is current", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const request = yield* makeRequest()
const reply = yield* makeReply(request)
yield* storage.saveRequest(request)
yield* storage.saveReply(reply)
yield* storage.clearReplies(request.envelope.requestId, { expectedReplyId: reply.reply.id })
expect(yield* storage.repliesFor([request])).toHaveLength(0)
expect(yield* storage.unprocessedMessages([request.envelope.address.shardId])).toHaveLength(1)
}))
it.effect("clearReplies with a stale expected reply preserves a newer completion and processed state", () =>
Effect.gen(function*() {
yield* truncate
const sql = yield* SqlClient.SqlClient
const storage = yield* MessageStorage.MessageStorage
const request = yield* makeRequest({ rpc: StreamRpc, payload: StreamRpc.payloadSchema.make({ id: 123 }) })
const oldReply = yield* makeChunkReply(request)
const completed = yield* makeReply(request)
yield* storage.saveRequest(request)
yield* storage.saveReply(oldReply)
yield* storage.saveReply(completed)
const before = yield* sql`SELECT processed, last_reply_id FROM cluster_messages WHERE id = ${
String(request.envelope.requestId)
}`.pipe(Effect.provideService(SqlClient.SafeIntegers, true))
yield* storage.clearReplies(request.envelope.requestId, { expectedReplyId: oldReply.reply.id })
expect((yield* storage.repliesFor([request])).map((r) => r.id)).toEqual([
oldReply.reply.id,
completed.reply.id
])
expect(
yield* sql`SELECT processed, last_reply_id FROM cluster_messages WHERE id = ${
String(request.envelope.requestId)
}`.pipe(Effect.provideService(SqlClient.SafeIntegers, true))
).toEqual(before)
expect(yield* storage.unprocessedMessages([request.envelope.address.shardId])).toHaveLength(0)
yield* truncate
}))
it.effect("saveRequest", () =>
Effect.gen(function*() {
const storage = yield* MessageStorage.MessageStorage
const request = yield* makeRequest({ payload: { id: 1 } })
const result = yield* storage.saveRequest(request)
expect(result._tag).toEqual("Success")
for (let i = 2; i <= 5; i++) {
yield* storage.saveRequest(yield* makeRequest({ payload: { id: i } }))
}
yield* storage.saveReply(yield* makeReply(request))
let messages = yield* storage.unprocessedMessages([request.envelope.address.shardId])
expect(messages).toHaveLength(4)
expect(messages.map((m: any) => m.envelope.payload.id)).toEqual([2, 3, 4, 5])
for (let i = 6; i <= 10; i++) {
yield* storage.saveRequest(yield* makeRequest({ payload: { id: i } }))
}
messages = yield* storage.unprocessedMessages([request.envelope.address.shardId])
expect(messages).toHaveLength(5)
expect(messages.map((m: any) => m.envelope.payload.id)).toEqual([6, 7, 8, 9, 10])
}))
it.effect("saveReply + saveRequest duplicate", () =>
Effect.gen(function*() {
const sql = yield* SqlClient.SqlClient
const storage = yield* MessageStorage.MessageStorage
const request = yield* makeRequest({
rpc: StreamRpc,
payload: StreamRpc.payloadSchema.make({ id: 123 })
})
let result = yield* storage.saveRequest(request)
expect(result._tag).toEqual("Success")
let chunk = yield* makeChunkReply(request, 0)
yield* storage.saveReply(chunk)
const ackChunk = yield* makeAckChunk(request, chunk)
yield* storage.saveEnvelope(ackChunk)
chunk = yield* makeChunkReply(request, 1)
yield* storage.saveReply(chunk)
result = yield* storage.saveRequest(
yield* makeRequest({
rpc: StreamRpc,
payload: StreamRpc.payloadSchema.make({ id: 123 })
})
)
assert(result._tag === "Duplicate" && Option.isSome(result.lastReceivedReply))
expect(result.lastReceivedReply.value._tag).toEqual("Chunk")
// get the un-acked chunk
const replies = yield* storage.repliesFor([request])
expect(replies).toHaveLength(1)
yield* storage.saveReply(yield* makeReply(request))
result = yield* storage.saveRequest(
yield* makeRequest({
rpc: StreamRpc,
payload: StreamRpc.payloadSchema.make({ id: 123 })
})
)
assert(result._tag === "Duplicate" && Option.isSome(result.lastReceivedReply))
expect(result.lastReceivedReply.value._tag).toEqual("WithExit")
// duplicate WithExit
const fiber = yield* storage.saveReply(yield* makeReply(request)).pipe(Effect.forkChild)
yield* TestClock.adjust(1)
while (!fiber.pollUnsafe()) {
yield* sql`SELECT 1`
yield* TestClock.adjust(1000)
}
const error = yield* Effect.flip(Fiber.join(fiber))
expect(error._tag).toEqual("PersistenceError")
}))
it.effect("detects duplicates", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
yield* storage.saveRequest(
yield* makeRequest({
rpc: PrimaryKeyTest,
payload: PrimaryKeyTest.payloadSchema.make({ id: 123 })
})
)
const result = yield* storage.saveRequest(
yield* makeRequest({
rpc: PrimaryKeyTest,
payload: PrimaryKeyTest.payloadSchema.make({ id: 123 })
})
)
expect(result._tag).toEqual("Duplicate")
}))
it.effect("hashes primary keys longer than the message_id column", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const longId = "long-key-".repeat(50)
const request = yield* makeRequest({
rpc: LongKeyRpc,
payload: LongKeyRpc.payloadSchema.make({ id: longId })
})
const result = yield* storage.saveRequest(request)
expect(result._tag).toEqual("Success")
const duplicate = yield* storage.saveRequest(
yield* makeRequest({
rpc: LongKeyRpc,
payload: LongKeyRpc.payloadSchema.make({ id: longId })
})
)
expect(duplicate._tag).toEqual("Duplicate")
const requestId = yield* storage.requestIdForPrimaryKey({
address: request.envelope.address,
tag: request.envelope.tag,
id: longId
})
expect(requestId).toEqual(Option.some(request.envelope.requestId))
const sql = yield* SqlClient.SqlClient
const rows = yield* sql<{ message_id: string }>`SELECT message_id FROM cluster_messages`
expect(rows).toHaveLength(1)
expect(rows[0].message_id).toMatch(/^[0-9a-f]{64}$/)
}))
it.effect("keeps primary keys within the column width as plaintext", () =>
Effect.gen(function*() {
yield* truncate
const sql = yield* SqlClient.SqlClient
const storage = yield* MessageStorage.MessageStorage
const request = yield* makeRequest({
rpc: PrimaryKeyTest,
payload: PrimaryKeyTest.payloadSchema.make({ id: 456 })
})
yield* storage.saveRequest(request)
// rows written by previous versions store the plaintext composed
// key, so short keys must stay byte-identical to keep deduplicating
const plaintext = Envelope.primaryKey(request.envelope)!
const rows = yield* sql<{ message_id: string }>`SELECT message_id FROM cluster_messages`
expect(rows).toHaveLength(1)
expect(rows[0].message_id).toEqual(plaintext)
const result = yield* storage.saveRequest(
yield* makeRequest({
rpc: PrimaryKeyTest,
payload: PrimaryKeyTest.payloadSchema.make({ id: 456 })
})
)
assert(result._tag === "Duplicate")
expect(result.originalId).toEqual(request.envelope.requestId)
const requestId = yield* storage.requestIdForPrimaryKey({
address: request.envelope.address,
tag: request.envelope.tag,
id: "456"
})
expect(requestId).toEqual(Option.some(request.envelope.requestId))
}))
if (label === "sqlite") {
// sqlite's TEXT message_id column stored over-long plaintext keys
// before hashing, so the legacy fallback must also cover keys longer
// than the 255-character limit of the width-enforcing dialects
it.effect("detects duplicates for legacy long-key plaintext rows", () =>
Effect.gen(function*() {
yield* truncate
const sql = yield* SqlClient.SqlClient
const storage = yield* MessageStorage.MessageStorage
const longId = "legacy-long-key-".repeat(30)
const request = yield* makeRequest({
rpc: LongKeyRpc,
payload: LongKeyRpc.payloadSchema.make({ id: longId })
})
yield* storage.saveRequest(request)
// simulate a row written before message_id values were hashed
const plaintext = Envelope.primaryKey(request.envelope)!
expect(plaintext.length).toBeGreaterThan(255)
yield* sql`UPDATE cluster_messages SET message_id = ${plaintext} WHERE id = ${
String(request.envelope.requestId)
}`
const result = yield* storage.saveRequest(
yield* makeRequest({
rpc: LongKeyRpc,
payload: LongKeyRpc.payloadSchema.make({ id: longId })
})
)
assert(result._tag === "Duplicate")
expect(result.originalId).toEqual(request.envelope.requestId)
const requestId = yield* storage.requestIdForPrimaryKey({
address: request.envelope.address,
tag: request.envelope.tag,
id: longId
})
expect(requestId).toEqual(Option.some(request.envelope.requestId))
}))
}
it.effect("unprocessedMessages", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const request = yield* makeRequest()
yield* storage.saveRequest(request)
let messages = yield* storage.unprocessedMessages([request.envelope.address.shardId])
expect(messages).toHaveLength(1)
messages = yield* storage.unprocessedMessages([request.envelope.address.shardId])
expect(messages).toHaveLength(0)
yield* storage.saveRequest(yield* makeRequest())
messages = yield* storage.unprocessedMessages([request.envelope.address.shardId])
expect(messages).toHaveLength(1)
}))
it.effect("unprocessedMessages honors the limit and claims only returned rows", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const shardId = ShardId.make("default", 1)
for (let i = 1; i <= 5; i++) {
yield* storage.saveRequest(yield* makeRequest({ payload: { id: i }, entityId: String(i) }))
}
const limited = yield* storage.unprocessedMessages([shardId], { limit: 3 })
expect(limited).toHaveLength(3)
expect(limited.map((m: any) => m.envelope.payload.id)).toEqual([1, 2, 3])
// rows beyond the limit were not claimed and are returned by the
// next read
const rest = yield* storage.unprocessedMessages([shardId])
expect(rest.map((m: any) => m.envelope.payload.id)).toEqual([4, 5])
}))
it.effect("unprocessedMessages filters by address", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const shardId = ShardId.make("default", 1)
const address = (entityId: string) =>
EntityAddress.make({
shardId,
entityType: EntityType.make("test"),
entityId: EntityId.make(entityId)
})
for (let i = 1; i <= 4; i++) {
yield* storage.saveRequest(yield* makeRequest({ payload: { id: i }, entityId: String(i) }))
}
const filtered = yield* storage.unprocessedMessages([shardId], {
addresses: [address("2"), address("4")]
})
expect(filtered.map((m: any) => m.envelope.payload.id)).toEqual([2, 4])
// the filtered read must not claim the other addresses
const rest = yield* storage.unprocessedMessages([shardId])
expect(rest.map((m: any) => m.envelope.payload.id)).toEqual([1, 3])
}))
it.effect("unprocessedMessages filters addresses by shard", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const shardOne = ShardId.make("default", 1)
const shardTwo = ShardId.make("default", 2)
yield* storage.saveRequest(yield* makeRequest({ payload: { id: 1 }, shardId: shardOne }))
yield* storage.saveRequest(yield* makeRequest({ payload: { id: 2 }, shardId: shardTwo }))
const messages = yield* storage.unprocessedMessages([shardOne, shardTwo], {
addresses: [EntityAddress.make({
shardId: shardOne,
entityType: EntityType.make("test"),
entityId: EntityId.make("1")
})]
})
expect(messages.map((message: any) => message.envelope.payload.id)).toEqual([1])
const rest = yield* storage.unprocessedMessages([shardOne, shardTwo])
expect(rest.map((message: any) => message.envelope.payload.id)).toEqual([2])
}))
it.effect("encoded unprocessedMessages fails closed for empty addresses", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const request = yield* makeRequest()
yield* storage.saveRequest(request)
const encoded = yield* SqlMessageStorage.makeEncoded().pipe(Effect.provide(NodeCrypto.layerDigest))
const shardId = request.envelope.address.shardId.toString()
const messages = yield* encoded.unprocessedMessages([shardId], Date.now(), { addresses: [] })
expect(messages).toHaveLength(0)
const unfiltered = yield* encoded.unprocessedMessages([shardId], Date.now())
expect(unfiltered).toHaveLength(1)
}))
it.effect("encoded resetAddresses fails closed for an empty address list", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const request = yield* makeRequest()
yield* storage.saveRequest(request)
const encoded = yield* SqlMessageStorage.makeEncoded().pipe(Effect.provide(NodeCrypto.layerDigest))
const shardId = request.envelope.address.shardId.toString()
const claimed = yield* encoded.unprocessedMessages([shardId], Date.now())
expect(claimed).toHaveLength(1)
yield* encoded.resetAddresses([])
const messages = yield* encoded.unprocessedMessages([shardId], Date.now())
expect(messages).toHaveLength(0)
}))
it.effect("resetAddresses releases claims in one batch", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const shardId = ShardId.make("default", 1)
const address = (entityId: string) =>
EntityAddress.make({
shardId,
entityType: EntityType.make("test"),
entityId: EntityId.make(entityId)
})
for (let i = 1; i <= 4; i++) {
yield* storage.saveRequest(yield* makeRequest({ payload: { id: i }, entityId: String(i) }))
}
yield* storage.unprocessedMessages([shardId], { limit: 3 })
yield* storage.resetAddresses([address("1"), address("3")])
const messages = yield* storage.unprocessedMessages([shardId])
expect(messages.map((m: any) => m.envelope.payload.id)).toEqual([1, 3, 4])
}))
it.effect("unprocessedMessages excludes complete requests", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const request = yield* makeRequest()
yield* storage.saveRequest(request)
yield* storage.saveReply(yield* makeReply(request))
const messages = yield* storage.unprocessedMessages([request.envelope.address.shardId])
expect(messages).toHaveLength(0)
}))
it.effect("repliesFor", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const request = yield* makeRequest()
yield* storage.saveRequest(request)
let replies = yield* storage.repliesFor([request])
expect(replies).toHaveLength(0)
yield* storage.saveReply(yield* makeReply(request))
replies = yield* storage.repliesFor([request])
expect(replies).toHaveLength(1)
expect(replies[0].requestId).toEqual(request.envelope.requestId)
}))
it.effect("registerReplyHandler", () =>
Effect.gen(function*() {
const storage = yield* MessageStorage.MessageStorage
const latch = yield* Latch.make()
const request = yield* makeRequest()
yield* storage.saveRequest(request)
const fiber = yield* storage.registerReplyHandler(
new Message.OutgoingRequest({
...request,
respond: () => latch.open
})
).pipe(Effect.forkChild)
yield* TestClock.adjust(1)
yield* storage.saveReply(yield* makeReply(request))
yield* latch.await
yield* Fiber.await(fiber)
}))
it.effect("unprocessedMessagesById", () =>
Effect.gen(function*() {
yield* truncate
const storage = yield* MessageStorage.MessageStorage
const request = yield* makeRequest({
rpc: StreamRpc,
payload: StreamRpc.payloadSchema.make({ id: 123 })
})
yield* storage.saveRequest(request)
let messages = yield* storage.unprocessedMessagesById([request.envelope.requestId])
expect(messages).toHaveLength(1)
const chunk = yield* makeChunkReply(request)
yield* storage.saveReply(chunk)
const ack = yield* makeAckChunk(request, chunk)
yield* storage.saveEnvelope(ack)
const encoded = yield* SqlMessageStorage.makeEncoded().pipe(Effect.provide(NodeCrypto.layerDigest))
const acknowledgements = yield* encoded.unprocessedMessagesById([ack.envelope.id], 0)
expect(acknowledgements).toHaveLength(1)
assert(acknowledgements[0].envelope._tag === "AckChunk")
expect(acknowledgements[0].envelope.replyId).toEqual(String(chunk.reply.id))
yield* storage.saveReply(yield* makeReply(request))
messages = yield* storage.unprocessedMessagesById([request.envelope.requestId])
expect(messages).toHaveLength(0)
}))
})
})
it.effect("keeps held messages readable while entity layers build", () =>
Effect.gen(function*() {
const config = ShardingConfig.layer({
entityRegistrationTimeout: 6000,
entityTerminationTimeout: 0,
entityMessagePollInterval: 100,
refreshAssignmentsInterval: 0
})
const delayedEnv = TestEntityLayer.pipe(
Layer.provide(Layer.effectDiscard(Effect.sleep(10_000))),
Layer.provideMerge(Sharding.layer),
Layer.provide(RunnerStorage.layerMemory),
Layer.provide(RunnerHealth.layerNoop),
Layer.provide(Runners.layerNoop),
Layer.provide(config)
)
const storage = yield* MessageStorage.MessageStorage
const request = yield* makeRequest()
yield* storage.saveRequest(request)
const fiber = yield* Effect.never.pipe(
Effect.provide(delayedEnv),
Effect.scoped,
Effect.forkChild({ startImmediately: true })
)
yield* TestClock.adjust(7500)
expect(yield* storage.repliesFor([request])).toHaveLength(0)
yield* TestClock.adjust(2500)
yield* TestClock.adjust(100)
expect(yield* storage.repliesFor([request])).toHaveLength(1)
yield* Fiber.interrupt(fiber)
}).pipe(Effect.provide(StorageLayer.pipe(
Layer.provideMerge(SqliteLayer)
))), { timeout: 15_000 })
})
const SqliteLayer = Effect.gen(function*() {
const fs = yield* FileSystem.FileSystem
const dir = yield* fs.makeTempDirectoryScoped()
return SqliteClient.layer({
filename: dir + "/test.db"
})
}).pipe(Layer.unwrap, Layer.provide(NodeFileSystem.layer))
const NumberSqliteLayer = Layer.effect(
SqlClient.SqlClient,
Effect.gen(function*() {
const sqlite = yield* SqliteClient.make({ filename: ":memory:" })
const connection = yield* sqlite.reserve
// Model drivers such as Durable Objects that return INTEGERs as numbers, even with SafeIntegers.
const roundRows = (rows: ReadonlyArray<Record<string, unknown>>) =>
rows.map((row) =>
Object.fromEntries(
Object.entries(row).map(([key, value]) => [key, typeof value === "bigint" ? Number(value) : value])
)
)
return yield* SqlClient.make({
driver: "sqlite-number",
compiler: Statement.makeCompilerSqlite(),
spanAttributes: [],
acquirer: Effect.succeed({
...connection,
execute: (...args) => connection.execute(...args).pipe(Effect.map(roundRows)),
executeUnprepared: (...args) => connection.executeUnprepared(...args).pipe(Effect.map(roundRows))
})
})
})
).pipe(Layer.provide(Reactivity.layer))