Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(control-api): stress the external GFS limiter at the real budgets

Alfredo Lopez91885e1f645a
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index c0f7b6769..7c2a1ed14 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.611",
+ "version": "0.1.612",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.611",
+ "version": "0.1.612",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index b9757d8d4..7480cba97 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.611",
+ "version": "0.1.612",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/test/config.externalGfsBudgets.test.ts b/control-api/test/config.externalGfsBudgets.test.ts
index f13d73d90..33432d113 100644
--- a/control-api/test/config.externalGfsBudgets.test.ts
+++ b/control-api/test/config.externalGfsBudgets.test.ts
@@ -131,9 +131,11 @@ describe('external GFS actor budgets', () => {
expect(Number(preferredPartMib)).toBe(UPLOAD_PREFERRED_PART_MIB)
expect(MAX_UPLOAD_MUTATIONS).toBe(27)
- // 27 is a floor: larger parts, retried 429s and reconciliation add more.
+ // 27 is a floor for one upload at the preferred part size: a smaller part
+ // size, retried 429s and reconciliation add mutations (a larger part size
+ // removes some). The budget is sized for three such uploads in one minute.
const { config } = await loadConfigModuleWith({})
- expect(config.externalGfsOperationRlPerMin).toBeGreaterThanOrEqual(MAX_UPLOAD_MUTATIONS)
+ expect(3 * MAX_UPLOAD_MUTATIONS).toBeLessThanOrEqual(config.externalGfsOperationRlPerMin)
})
it('L12: defaults to 480/480/120/120 reads and 90 operations per minute', async () => {
diff --git a/control-api/test/fixtures/external-gfs-desktop-reads-2026-09-18.json b/control-api/test/fixtures/external-gfs-desktop-reads-2026-09-18.json
new file mode 100644
index 000000000..beb813b60
--- /dev/null
+++ b/control-api/test/fixtures/external-gfs-desktop-reads-2026-09-18.json
@@ -0,0 +1,223 @@
+{
+ "description": "Desktop external GFS resource-class reads on cluster clerum, 2026-09-18 (UTC), the day of the Desktop 429 incident. Each entry is the request's log time in milliseconds since 00:00 UTC; nothing else about the request is kept.",
+ "offsetsMs": [
+ 1978538, 1986761, 1990501, 1994750, 1999585, 2028160, 2029717, 2029749, 2029891, 2029894,
+ 2030047, 2030047, 2030137, 2030138, 2030235, 2030236, 2030419, 2030437, 2030438, 2030532,
+ 2030537, 2034222, 2035454, 2035458, 2035462, 2035520, 2035532, 2035533, 2035632, 2035634,
+ 2035638, 2035642, 2035643, 2035645, 2035647, 2035719, 2037866, 2525622, 2525721, 2540070,
+ 2540155, 2602460, 2618647, 2629123, 2743548, 2927492, 3006574, 3042130, 3107792, 3123288,
+ 3129202, 3135569, 3140778, 3244659, 6437148, 6509820, 6511749, 6516534, 6582257, 6598528,
+ 9264229, 9353239, 9422163, 16802775, 16885324, 16888864, 16888922, 16889076, 16889115, 16889209,
+ 16889212, 16889232, 16889236, 16889241, 16889339, 16889419, 16889435, 16889519, 16889533,
+ 16889630, 16890445, 16895285, 16895329, 16895477, 16895630, 16895635, 16895783, 16895784,
+ 16895806, 16895814, 16895815, 16895824, 16895826, 16895838, 16895842, 16895844, 16895846,
+ 16898055, 16899486, 16899522, 16899669, 16899820, 16899837, 16899843, 16899843, 16899849,
+ 16899924, 16899931, 16899931, 16899931, 16899932, 16899932, 16899936, 16899938, 16903966,
+ 16907386, 16952938, 16952979, 16953167, 16953652, 16953731, 16953737, 16953831, 16953933,
+ 16953941, 16954031, 16954036, 16954037, 16954120, 16954130, 16954133, 16954134, 16955452,
+ 16955461, 16955609, 16956130, 16956139, 16956232, 16956233, 16956239, 16956242, 16956245,
+ 16956332, 16956335, 16956345, 16956348, 16956431, 16956433, 16958160, 16958168, 16958337,
+ 16958645, 16958647, 16958732, 16958733, 16958735, 16958741, 16958741, 16958746, 16958831,
+ 16958832, 16958832, 16958833, 16958835, 16965578, 16965607, 16965748, 16965886, 16965890,
+ 16965893, 16965897, 16966028, 16966041, 16966047, 16966051, 16966052, 16966057, 16966057,
+ 16966069, 16966121, 16970621, 16973328, 16974763, 16974785, 16974937, 16975091, 16975095,
+ 16975099, 16975103, 16975106, 16975107, 16975118, 16975244, 16975248, 16975252, 16975255,
+ 16975260, 16975294, 16977275, 16977281, 16977421, 16977572, 16977580, 16977584, 16977588,
+ 16977590, 16977620, 16977631, 16977632, 16977634, 16977643, 16977646, 16977648, 16977731,
+ 16996392, 16996403, 17004893, 17004913, 17052354, 17052372, 17052516, 17052672, 17052685,
+ 17052707, 17052818, 17052818, 17052845, 17053038, 17053038, 17053041, 17053130, 17053136,
+ 17053137, 17053139, 17077142, 17077169, 17077281, 17077418, 17077422, 17077427, 17077447,
+ 17077561, 17077577, 17077583, 17077586, 17077589, 17077600, 17077606, 17077613, 17077631,
+ 17094232, 17094262, 17130362, 17130379, 17130510, 17130663, 17130665, 17130850, 17130930,
+ 17130944, 17131030, 17131041, 17131118, 17131334, 17131334, 17131335, 17131336, 17131338,
+ 17148321, 17148376, 17148994, 17148998, 17149144, 17149531, 17149536, 17149537, 17149603,
+ 17149603, 17149623, 17149630, 17149640, 17149640, 17149641, 17149642, 17149646, 17149646,
+ 17150812, 17150829, 17150967, 17151130, 17151133, 17151136, 17151147, 17151154, 17151218,
+ 17151231, 17151232, 17151233, 17151234, 17151238, 17151238, 17151240, 17152530, 17152530,
+ 17152671, 17152810, 17152817, 17152820, 17152822, 17152827, 17152831, 17152833, 17152835,
+ 17152846, 17152848, 17152849, 17152851, 17152852, 17158827, 17158858, 17158998, 17159151,
+ 17159152, 17159155, 17159159, 17159160, 17159171, 17159323, 17159338, 17159341, 17159344,
+ 17159344, 17159347, 17159365, 17160517, 17160523, 17160667, 17160813, 17160821, 17160822,
+ 17160828, 17160828, 17160831, 17160835, 17160845, 17160851, 17160853, 17160918, 17160922,
+ 17160931, 17176445, 17176466, 17176610, 17176763, 17176771, 17176779, 17176832, 17176834,
+ 17176836, 17176940, 17176941, 17176945, 17176956, 17176958, 17176961, 17176978, 17179979,
+ 17180005, 17180143, 17180287, 17180289, 17180296, 17180298, 17180299, 17180300, 17180304,
+ 17180308, 17180334, 17180339, 17180341, 17180344, 17180345, 17212431, 17212563, 17241535,
+ 17241539, 17365431, 17375739, 17375850, 17375879, 17376031, 17376051, 17376052, 17376135,
+ 17376136, 17376219, 17376230, 17376233, 17376236, 17376335, 17376337, 17376338, 17377205,
+ 17378599, 17378606, 17378745, 17378761, 17378832, 17378833, 17378837, 17378838, 17378851,
+ 17378854, 17378857, 17378858, 17378859, 17378939, 17380795, 17387632, 17391552, 17391633,
+ 17391779, 17391940, 17391955, 17392133, 17392142, 17392152, 17392156, 17392232, 17392234,
+ 17392236, 17392240, 17392243, 17392243, 17392245, 17394546, 17394556, 17394743, 17394945,
+ 17395036, 17395036, 17395039, 17395043, 17395046, 17395048, 17395051, 17395131, 17395135,
+ 17439461, 17439476, 17439618, 17439774, 17439778, 17439794, 17439830, 17439843, 17439847,
+ 17439929, 17439944, 17439948, 17440018, 17440018, 17440030, 17440030, 17440945, 17440955,
+ 17441092, 17441238, 17441240, 17441246, 17441250, 17441254, 17441333, 17441334, 17441336,
+ 17441361, 17441435, 17441928, 17546226, 17546257, 17546402, 17546555, 17546624, 17546634,
+ 17546637, 17546639, 17546643, 17546739, 17546835, 17546837, 17546844, 17546845, 17546920,
+ 17546933, 17546951, 17547023, 17547168, 17547305, 17547308, 17547315, 17547319, 17547320,
+ 17547334, 17547339, 17547339, 17547344, 17547344, 17547971, 17549355, 17549356, 17549488,
+ 17549642, 17549644, 17549650, 17549652, 17549731, 17549733, 17549736, 17549740, 17549742,
+ 17549819, 17549829, 17549830, 17549834, 17552454, 17552464, 17552605, 17552748, 17552755,
+ 17552759, 17552761, 17552763, 17552831, 17552833, 17552839, 17552839, 17552841, 17552844,
+ 17552845, 17552940, 17679956, 17683828, 17683885, 17684028, 17684035, 17684035, 17684040,
+ 17684119, 17684130, 17684131, 17684134, 17684141, 17684222, 17684226, 17684230, 17684301,
+ 18518715, 18520166, 18520261, 18520310, 18520470, 18520494, 18520496, 18520531, 18520533,
+ 18520539, 18520543, 18520547, 18520631, 18520636, 18520640, 18520640, 18521965, 18524534,
+ 18524720, 18524866, 18525009, 18525029, 18525162, 18525177, 18525231, 18525237, 18525245,
+ 18525331, 18525333, 18525336, 18525338, 18525339, 18525341, 18526082, 18588292, 18588306,
+ 18588621, 18588779, 18588781, 18588837, 18589021, 18589032, 18589036, 18589036, 18589039,
+ 18589137, 18589218, 18589232, 18589333, 18589337, 18589420, 19066795, 19066957, 19066981,
+ 19067115, 19070667, 19070691, 19070840, 19071015, 19071018, 19071018, 19071034, 19071045,
+ 19071118, 19071131, 19071160, 19071165, 19071169, 19071173, 19071242, 19071244, 19071784,
+ 20488089, 22460772, 22467355, 22467383, 22467521, 22467678, 22467679, 22467703, 22467736,
+ 22467737, 22467738, 22467744, 22467831, 22467834, 22467841, 22467842, 22467848, 22468994,
+ 22482461, 22484332, 22484584, 22484605, 22484849, 22484866, 22484872, 22484877, 22484899,
+ 22484902, 22484908, 22484930, 22484940, 22484943, 22484946, 22484948, 22488723, 22488756,
+ 30961421, 30964363, 30964368, 30964546, 30964550, 30976683, 30976778, 30977006, 30977012,
+ 30978681, 31068568, 31072392, 31072684, 31072864, 31072892, 31081456, 31081578, 31081848,
+ 31084660, 31084685, 31091763, 31100796, 31110955, 31111052, 31116102, 31116126, 31116130,
+ 31129425, 31129445, 31129496, 32879160, 32879204, 32879669, 36023762, 36027274, 36027340,
+ 36027527, 36027532, 36027560, 36027631, 36029341, 36029367, 36030032, 36030035, 36030062,
+ 36049448, 36049632, 36049638, 36050537, 36050538, 36050555, 36050636, 36052444, 36052446,
+ 36052456, 36052538, 36053394, 36053395, 36053402, 36053408, 36054625, 36077541, 36077559,
+ 36077561, 36079498, 36079504, 36079513, 36079535, 36081602, 36081605, 36081610, 36081645,
+ 36081648, 36091457, 36091506, 36091508, 36113741, 36113754, 36115370, 36115402, 36115403,
+ 36115449, 36115449, 36154290, 36154330, 36154336, 37306056, 37306058, 37308123, 37311549,
+ 37311570, 37311733, 37311764, 37311802, 37311833, 37318770, 37335330, 37338414, 37339699,
+ 37339735, 37339933, 37339938, 37339948, 37339963, 37341561, 37341567, 37341567, 37341568,
+ 37341648, 37342874, 37342890, 37342892, 37342895, 37345150, 37345158, 37345159, 37354808,
+ 37570479, 37570614, 38093708, 38096207, 38096235, 38096424, 38096427, 38096471, 38096707,
+ 38549235, 38551057, 38551109, 38551133, 38551136, 38551316, 38551329, 38552527, 38552534,
+ 38552551, 38552561, 38552653, 38552654, 38552723, 38552828, 38553110, 38553132, 38553232,
+ 38553563, 38553651, 38553742, 38553756, 38553840, 38553926, 38553927, 38553928, 38554047,
+ 38554058, 38554142, 38554143, 38554144, 38554232, 38554260, 38554266, 38554266, 38554338,
+ 38554338, 38554345, 38554418, 38554432, 38554432, 38554440, 38554441, 38554441, 38554441,
+ 38554442, 38554518, 38554633, 38554635, 38554640, 38554734, 38554833, 38554934, 38554940,
+ 38555031, 38555032, 38555034, 38555038, 38555040, 38555041, 38555045, 39540820, 39542565,
+ 39542617, 39542797, 39542920, 39543021, 39543202, 39544778, 39607643, 39607670, 39614637,
+ 39615167, 39615451, 39615629, 39619207, 39619646, 39619744, 39619915, 39694859, 39694860,
+ 39695358, 39695423, 39695591, 39695617, 39697189, 39697190, 39697216, 39697258, 39697262,
+ 39709679, 39710122, 39710190, 39710372, 39710432, 39710434, 39710775, 39710789, 39710971,
+ 39710981, 39714716, 39715307, 39715454, 39715636, 39715647, 39737399, 39737450, 39822665,
+ 39822700, 39826382, 39828365, 39828395, 39828567, 39828625, 39828640, 39828662, 39830466,
+ 39830473, 39830534, 39840547, 39841853, 39842035, 39842247, 39876181, 39876631, 39876692,
+ 39876873, 39926875, 39927633, 39927638, 39927855, 39927855, 39928348, 39928352, 39928544,
+ 39928556, 39929000, 39929004, 39929180, 39929190, 39929585, 39929586, 39929769, 39929818,
+ 39930320, 39930332, 39930554, 39930555, 39931050, 39931054, 39931239, 39943172, 39943198,
+ 39943694, 39951329, 39951392, 39951567, 39951582, 39951620, 39951647, 39954204, 39954211,
+ 39954252, 39954258, 39954338, 39954341, 39954434, 39954434, 39968586, 39968627, 39973179,
+ 39974588, 39974593, 39974762, 39974781, 39974830, 39974860, 39978344, 39978347, 39978432,
+ 39978442, 39978460, 39978533, 39978535, 39978544, 39978547, 39978548, 39978549, 39978554,
+ 39990839, 39990866, 42328859, 42329061, 42389190, 42389398, 46820825, 46823083, 46823153,
+ 46823365, 46823381, 46823415, 46823416, 46823458, 46823532, 46823536, 46823538, 46823566,
+ 46823618, 46823635, 46823641, 46824527, 46824639, 46824648, 46824724, 46824733, 46824744,
+ 46824825, 46825020, 46825034, 46825040, 46825055, 46825055, 46825132, 46825138, 46825150,
+ 46825231, 46825233, 46825235, 46825248, 46825248, 46825251, 46825318, 46825332, 46825332,
+ 46825335, 46825336, 46825337, 46825338, 46825339, 46825340, 46825340, 46825346, 46838433,
+ 46838495, 46838755, 46838765, 46838807, 46838850, 46838903, 46838939, 46838956, 46843472,
+ 46849532, 46850448, 46850720, 46851040, 46851045, 46851341, 46852570, 46897357, 46999250,
+ 46999550, 47160842, 47162764, 47162961, 47163124, 47163153, 47163157, 47163220, 47163242,
+ 47164222, 47164239, 47164241, 47164250, 47164335, 47164345, 47164350, 47164358, 47164359,
+ 47164435, 47164443, 47164459, 47164463, 47164536, 47164537, 47164566, 47164670, 47164731,
+ 47164753, 47164755, 47164755, 47164847, 47164961, 47164962, 47165042, 47165046, 47165048,
+ 47165122, 47165138, 47165235, 47165241, 47165348, 47165430, 47165446, 47165532, 47165553,
+ 47165554, 47165560, 47165561, 47165631, 47165643, 47165645, 47165752, 47165756, 47165831,
+ 47165836, 47165837, 47165838, 47165919, 47165950, 47165971, 47165971, 47166058, 47166059,
+ 47166060, 47166062, 47166144, 47166145, 47166154, 47166155, 47166164, 47166167, 47166218,
+ 47166221, 47166230, 47166231, 47166231, 47166232, 47166232, 47166233, 47166239, 47166240,
+ 47166241, 47214664, 47216483, 47216561, 47216638, 47216730, 47216773, 47218444, 47218651,
+ 47219154, 47219155, 47219155, 47219156, 47219238, 47219339, 47219423, 47219432, 47219435,
+ 47219435, 47219436, 47219441, 47219443, 47219445, 47219534, 47219536, 47219539, 47219540,
+ 47219545, 47219633, 47219634, 47219635, 47219637, 47219638, 47219642, 47219642, 47219643,
+ 47219644, 47219648, 47219648, 47219732, 47219732, 47219735, 47219735, 47219736, 47219737,
+ 47226123, 47226151, 47226152, 47228824, 47240622, 47240625, 47240636, 47240638, 47240643,
+ 47240645, 47240658, 47240722, 47240736, 47240754, 47240820, 47240820, 47240840, 47240843,
+ 47240848, 47240854, 47240931, 47240942, 47240948, 47241033, 47241033, 47241037, 47241049,
+ 47241052, 47241054, 47241130, 47241132, 47241133, 47241138, 47241138, 47241142, 47241142,
+ 47241144, 47241218, 47241218, 47241219, 47241219, 47243584, 47281871, 47281890, 47780562,
+ 47782106, 47782119, 47782134, 47782150, 47782170, 47783844, 47783844, 47783852, 47783855,
+ 47783874, 47783931, 47783934, 47783937, 47783946, 47783949, 47783953, 47784154, 47784222,
+ 47784242, 47784330, 47784339, 47784341, 47784343, 47784350, 47784354, 47784418, 47784431,
+ 47784431, 47784434, 47784436, 47784436, 47784437, 47784438, 47784440, 47784444, 47784444,
+ 47784445, 47784445, 47784446, 47784519, 47784530, 47784539, 47789702, 51470079, 51475056,
+ 51475116, 51475294, 51475324, 51475495, 51475536, 51475671, 51475697, 51476754, 51476754,
+ 51476780, 51476782, 51476804, 51476834, 51476951, 51476955, 51476959, 51476962, 51476968,
+ 51477031, 51477133, 51477138, 51477139, 51477139, 51477143, 51477144, 51477145, 51477150,
+ 51477262, 51477442, 51477472, 51477532, 51477560, 51477561, 51477573, 51477740, 51477744,
+ 51477764, 51477764, 51477833, 51477844, 51477932, 51477963, 51477964, 51478044, 51478046,
+ 51478046, 51478049, 51478049, 51478062, 51478133, 51478136, 51478165, 51478249, 51478261,
+ 51478262, 51478262, 51478335, 51478336, 51478354, 51478356, 51478357, 51478435, 51478438,
+ 51478439, 51478442, 51478443, 51478445, 51478449, 51478532, 51478533, 51478534, 51478535,
+ 51478662, 51478663, 51478665, 51478665, 51478725, 51478726, 51478727, 51478728, 51478738,
+ 51478740, 51481830, 51481854, 51483632, 51494864, 51494891, 51494913, 51494916, 51494921,
+ 51494930, 51494948, 51494959, 51495003, 51495017, 51495103, 51495222, 51495324, 51495537,
+ 51495644, 51495648, 51495719, 51495721, 51495730, 51495731, 51495733, 51495733, 51495734,
+ 51495734, 51495735, 51495736, 51495738, 51495739, 51495740, 51495743, 51495743, 51495743,
+ 51495743, 51495744, 51495745, 51495746, 51495747, 51495748, 51495748, 51495749, 51498023,
+ 51502484, 51516336, 51516376, 51516404, 51527076, 51559203, 51580622, 51582210, 51582298,
+ 51582509, 51582512, 51582576, 51582607, 51582618, 51582632, 51584712, 51584717, 51584739,
+ 51584740, 51584740, 51584747, 51584932, 51585119, 51585139, 51585155, 51585173, 51585241,
+ 51585245, 51585432, 51585545, 51585574, 51585631, 51585645, 51585651, 51585651, 51585721,
+ 51585731, 51585734, 51585821, 51585852, 51585852, 51585862, 51585927, 51585928, 51585939,
+ 51585950, 51585960, 51585961, 51586024, 51586035, 51586036, 51586060, 51586064, 51586064,
+ 51586064, 51586066, 51586066, 51586120, 51586153, 51586154, 51586154, 51586218, 51586219,
+ 51586232, 51586237, 51586239, 51586239, 51586241, 51586243, 51586319, 51586319, 51586331,
+ 51586331, 51586331, 51586332, 51586333, 51586337, 51586337, 51586371, 51586372, 51586469,
+ 51586485, 51586534, 51586534, 51586549, 51586552, 51587029, 51693736, 51693823, 51693919,
+ 51693922, 51693945, 51870259, 51870300, 51870466, 51870780, 51870804, 51870848, 51870920,
+ 51870936, 51870937, 51870938, 51871033, 51871143, 51871163, 51871232, 51871235, 51871246,
+ 51871257, 51871334, 51871341, 51871341, 51871439, 51871533, 51871645, 51871646, 51871834,
+ 51871837, 51871837, 51871840, 51871846, 51871944, 51871948, 51871952, 51871952, 51871953,
+ 51871956, 51871956, 51871957, 51872034, 51872051, 51875057, 51875061, 51875067, 51875114,
+ 51875119, 51876231, 51876239, 51876242, 51876338, 51876357, 51876439, 51876540, 51876553,
+ 51876565, 51876619, 51876635, 51876637, 51876643, 51876649, 51876651, 51876653, 51876656,
+ 51876720, 51876735, 51876741, 51876749, 51876752, 51876755, 51876832, 51876834, 51876839,
+ 51876848, 51876853, 51876932, 51876938, 51876952, 51876963, 51876965, 51877018, 51877031,
+ 51877031, 51877034, 51877036, 51877044, 51877130, 51878827, 51878831, 51898432, 51898456,
+ 54509512, 54511704, 54511756, 54511936, 54511972, 54512151, 54512151, 54512233, 54512250,
+ 54512251, 54512252, 54512332, 54512332, 54512338, 54512342, 54512344, 54513688, 54706490,
+ 54872243, 54920061, 54969804, 54980346, 54980444, 54980444, 54980447, 54980532, 54980536,
+ 54980537, 54980542, 54980544, 54980545, 54980546, 54980546, 54980547, 54980551, 54981511,
+ 55024926, 55090758, 55091009, 55091205, 55091409, 55091432, 55091439, 55091448, 55091450,
+ 55091452, 55091533, 55091554, 55091618, 55091635, 55091642, 55091646, 55091654, 55203226,
+ 55203229, 55203542, 55209731, 55209847, 55210060, 55210251, 55210254, 55210260, 55210446,
+ 55210450, 55210532, 55210543, 55210549, 55210632, 55210633, 55210635, 55210637, 55210639,
+ 55212527, 56233931, 56233943, 56477010, 56477034, 56484707, 56484904, 56485527, 56485697,
+ 56485703, 56485703, 56485732, 56485746, 56485772, 56485856, 56485861, 56485870, 56485874,
+ 56485940, 56485940, 56486002, 56487135, 56487137, 56487320, 56487511, 56487517, 56487531,
+ 56487535, 56487539, 56487542, 56487546, 56487548, 56487549, 56487562, 56487631, 56487633,
+ 56487637, 56500577, 56500620, 56513324, 56513399, 56513580, 56513832, 56513931, 56514220,
+ 56514264, 56514321, 56514334, 56514335, 56514341, 56514345, 56514345, 56514348, 56514348,
+ 56514348, 56517007, 56517040, 56517219, 56517394, 56517398, 56517406, 56517431, 56517438,
+ 56517440, 56517451, 56517454, 56517456, 56517540, 56517540, 56517547, 56517550, 56518684,
+ 56518692, 56518872, 56519062, 56519134, 56519138, 56519143, 56519146, 56519231, 56519233,
+ 56519235, 56519239, 56519239, 56519240, 56519241, 56519243, 56529141, 56529235, 56529464,
+ 56529643, 56529647, 56529739, 56529944, 56530021, 56530033, 56530039, 56530041, 56530046,
+ 56530120, 56530121, 56530130, 56530131, 60503686, 60507124, 60507167, 60507644, 60507645,
+ 60507861, 60507862, 60507867, 60507919, 60507934, 60507935, 60507948, 60507950, 60507951,
+ 60507991, 60513617, 60514810, 60514944, 60514974, 60515342, 60515346, 60515418, 60515423,
+ 60515425, 60515431, 60515434, 60515434, 60515437, 60515440, 60515442, 60515444, 60516483,
+ 60517366, 60517401, 60517553, 60517633, 60517635, 60517639, 60517643, 60517720, 60517735,
+ 60517739, 60517739, 60517740, 60517741, 60517742, 60527758, 60527965, 60528118, 60528159,
+ 60528394, 60528426, 60529858, 60533363, 60543166, 60543187, 60550767, 60551124, 60551379,
+ 60551507, 60554040, 60554046, 60566285, 60604219, 60604236, 60611044, 60611055, 60997202,
+ 60997546, 61148893, 61148964, 61149150, 61149348, 61149350, 61149547, 61149569, 61149621,
+ 61149700, 61149739, 61149837, 61149838, 61149846, 61149936, 61149942, 61149950, 61150023,
+ 61150039, 74804559, 74813322, 81525220, 81527024, 81529443, 81541659, 81541742, 81551606,
+ 81551649, 81563812, 81563834, 81566324, 81571381, 81606295, 81616390, 81698194, 81698197,
+ 81699419, 81830256, 82411751, 82481563, 82971880, 82973542, 82973586, 82973709, 82973878,
+ 82973879, 82973901, 82974037, 82974038, 82974116, 82974125, 82974136, 82974150, 82974157,
+ 82974166, 82974221, 82978322, 82978329, 82978503, 82978673, 82978842, 82978844, 82979072,
+ 82979077, 82979090, 82979101, 82980428, 82980431, 82980604, 82980782, 82980785, 82980787,
+ 82980789, 82980811, 82980813, 82980938, 82980941, 82980947, 82980950, 82981016, 82983256,
+ 82983258, 83004435, 83004454, 83004628, 83004796, 83004959, 83004961, 83004969, 83005105,
+ 83005109, 83005127, 83007671, 83007672, 83013567, 83013609, 83013778, 83013946, 83014106,
+ 83014109, 83014269, 83014271, 83014349, 83014350, 83015915, 83015923, 83016090, 83016268,
+ 83016269, 83016302, 83016307, 83016312, 83016331, 83016432, 83016436, 83016438, 83016451,
+ 83016520, 83017289, 83017290, 83018610, 83018611, 83020557, 83020566, 83071804, 83608529,
+ 83637588, 83639108, 83644094, 83644130, 83648784, 83648818, 83651953, 83651965, 83686748,
+ 83701427, 83701450, 83703328, 83703343, 83816580, 83823972, 83825581, 83992973, 84003229,
+ 84023427, 84023429, 84049930, 84052160, 84103813
+ ]
+}
diff --git a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts
index e9ead490a..fc7fef64d 100644
--- a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts
+++ b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts
@@ -1,23 +1,37 @@
-import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'
import { randomBytes, randomUUID } from 'node:crypto'
+import { readFileSync } from 'node:fs'
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
import { Pool } from 'pg'
-import request from 'supertest'
// Concurrent users at the external GFS rate limits, through the production
// Express app against real PostgreSQL. Config, the core and limiter pools, both limiters,
// the session lifecycle query, authority resolution, the prom registry and the
-// logger are real; the ONLY mock is the session-token verifier (identity),
-// which decodes a JSON test token so each scenario controls users and sessions.
+// logger are real; the ONLY mocks are the session-token verifier (identity),
+// which decodes a JSON test token so each scenario controls users and sessions,
+// and, in E3 only, the fetch to gfsc (the service boundary).
//
-// Denial source (plan Addendum A3), for status 429 only:
+// The pools run at their production bounds: every CORE_POOL_* and
+// RATE_LIMIT_POOL_* variable is deleted before db.js is imported, so a 503
+// from either pool fails the exact tallies below. Each scenario serves its app
+// from one listening server and sends through one keep-alive agent.
+//
+// Denial source, for status 429 only, and only on the
+// resource read and resource mutation routes. Those routes have no
+// rateLimitMiddleware, so their headers identify the limiter:
// - route backstop: RateLimit-Policy === "<route L>;w=60";
// - Postgres bucket: RateLimit-Policy === "<ingress L>;w=60" AND
// X-RateLimit-Remaining === "0" AND X-RateLimit-Limit === "<bucket L>";
// - anything else fails the test.
+// GET /grants and GET /shares also run rateLimitMiddleware, whose 429 carries
+// the same headers as a Postgres bucket's, so the classifier refuses those
+// routes and S2b attributes their denials through the ledger and the metric.
// Headers cannot say WHICH Postgres bucket denied, so every scenario also reads
// the rate_limit_buckets ledger: each hit increments its row once and a bucket
// only sees requests that passed the buckets before it, so per (key, window)
-// denials = max(0, count - limit). The key prefix names the phase.
+// denials = max(0, count - limit). The key prefix names the phase. Nothing
+// prunes the ledger here: cleanupExpiredBuckets runs only from main.ts.
//
// Both limiters read Date.now(), so faking only Date pins their windows. Time
// only moves forward, each scenario deletes the ledger, uses fresh identities,
@@ -42,6 +56,33 @@ vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({
const INGRESS_L = 1_800
const IP_L_DEFAULT = 1_200
const MINUTE_MS = 60_000
+const DAY_MS = 24 * 60 * MINUTE_MS
+
+/** The shipped defaults (config.ts); beforeAll asserts config still has them. */
+const PRODUCTION_BUDGETS: Budgets = {
+ resource: 480,
+ proxy: 480,
+ grants: 120,
+ shares: 120,
+ operation: 90,
+ ip: IP_L_DEFAULT,
+}
+
+const RESOURCE_READ_ROUTE = /^\/api\/v1\/external\/gfs\/resources\/[0-9a-f-]{36}\/affordances\?/
+const RESOURCE_MUTATION_ROUTE = /^\/api\/v1\/external\/gfs\/resources\/[0-9a-f-]{36}\/children\?/
+
+/**
+ * E4 fixture: every Desktop resource-class read on cluster clerum on
+ * 2026-09-18 (UTC), the day of the Desktop 429 incident, as ms since 00:00 UTC.
+ */
+const DESKTOP_READS_2026_09_18 = (
+ JSON.parse(
+ readFileSync(
+ new URL('./fixtures/external-gfs-desktop-reads-2026-09-18.json', import.meta.url),
+ 'utf-8'
+ )
+ ) as { offsetsMs: number[] }
+).offsetsMs
/** Every read class at `read`: for scenarios that exercise one read class. */
function uniformBudgets(read: number, operation: number, ip: number): Budgets {
@@ -82,7 +123,15 @@ const describeRealPostgres = adminUrl ? describe : describe.skip
describeRealPostgres('external GFS rate limits under concurrent load (real PostgreSQL)', () => {
const database = `control_api_gfs_rl_stress_${randomBytes(6).toString('hex')}`
- const envKeys = ['CONTROL_API_PG_CONNECTION_STRING', 'CORE_POOL_CONNECTION_TIMEOUT_MS'] as const
+ const envKeys = [
+ 'CONTROL_API_PG_CONNECTION_STRING',
+ 'CORE_POOL_MAX',
+ 'CORE_POOL_CONNECTION_TIMEOUT_MS',
+ 'CORE_POOL_STATEMENT_TIMEOUT_MS',
+ 'RATE_LIMIT_POOL_MAX',
+ 'RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS',
+ 'RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS',
+ ] as const
const previousEnv = new Map<string, string | undefined>()
let adminPool: Pool
@@ -105,19 +154,26 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
throw new Error('CONTROL_API_REAL_PG_ADMIN_URL is required by this suite')
}
const connectionString = databaseUrl(adminUrl, database)
- for (const key of envKeys) previousEnv.set(key, process.env[key])
+ for (const key of envKeys) {
+ previousEnv.set(key, process.env[key])
+ delete process.env[key]
+ }
adminPool = new Pool({ connectionString: adminUrl })
await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString
- // A 200-request burst queues its session and authority queries on the 10
- // core connections; the production 2 s acquire timeout would turn some of
- // them into 500s by accident (see
- // services.rateLimiterPoolSaturation.realPostgres.integration.test.ts).
- // The limiter's own pool runs at its production bounds.
- process.env.CORE_POOL_CONNECTION_TIMEOUT_MS = '10000'
const dbMod = await import('../src/db.js')
corePool = dbMod.pool as unknown as Pool
+ // Witness that nothing raised the bounds: core 10 / 2000 ms, limiter 6 / 5000 ms.
+ type PoolOptions = { options: { max: number; connectionTimeoutMillis: number } }
+ expect((corePool as unknown as PoolOptions).options).toMatchObject({
+ max: 10,
+ connectionTimeoutMillis: 2_000,
+ })
+ expect((dbMod.rateLimitPool as unknown as PoolOptions).options).toMatchObject({
+ max: 6,
+ connectionTimeoutMillis: 5_000,
+ })
corePool.on('error', () => {})
limiterPool = dbMod.rateLimitPool as unknown as Pool
limiterPool.on('error', () => {})
@@ -140,11 +196,27 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
MockGateway,
}
expect(mod.config.externalGfsIngressRlPerMin).toBe(INGRESS_L)
- expect(mod.config.externalGfsIpRlPerMin).toBe(IP_L_DEFAULT)
+ // E1-E4 run at PRODUCTION_BUDGETS; they must be the shipped defaults.
+ expect({
+ resource: mod.config.externalGfsResourceReadRlPerMin,
+ proxy: mod.config.externalGfsProxyReadRlPerMin,
+ grants: mod.config.externalGfsGrantsReadRlPerMin,
+ shares: mod.config.externalGfsSharesReadRlPerMin,
+ operation: mod.config.externalGfsOperationRlPerMin,
+ ip: mod.config.externalGfsIpRlPerMin,
+ }).toEqual(PRODUCTION_BUDGETS)
vi.useFakeTimers({ toFake: ['Date'] })
nextScenarioMinuteMs = Math.floor(Date.now() / MINUTE_MS) * MINUTE_MS + 10 * MINUTE_MS
}, 60_000)
+ // The server of the running scenario; closed after every test.
+ let openServer: { close: () => Promise<void> } | null = null
+ afterEach(async () => {
+ const server = openServer
+ openServer = null
+ await server?.close()
+ })
+
afterAll(async () => {
vi.useRealTimers()
for (const key of envKeys) {
@@ -164,8 +236,13 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
await adminPool.end()
})
- /** Sets a boot-valid budget, clears the ledger, returns a fresh app and the scenario minute. */
- async function startScenario(budgets: Budgets) {
+ /**
+ * Sets a boot-valid budget, clears the ledger, serves a fresh app and
+ * returns its `send` and the scenario start. The start is a multiple of
+ * `alignMs` (a minute, or a UTC day for the E4 replay), and the next
+ * scenario starts at least `spanMs` later.
+ */
+ async function startScenario(budgets: Budgets, spanMs = 5 * MINUTE_MS, alignMs = MINUTE_MS) {
mod.assertExternalGfsBudgetInvariants({
resourceReadPerMin: budgets.resource,
proxyReadPerMin: budgets.proxy,
@@ -181,10 +258,87 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
mod.config.externalGfsOperationRlPerMin = budgets.operation
mod.config.externalGfsIpRlPerMin = budgets.ip
await mod.pool.query('DELETE FROM rate_limit_buckets')
- const minuteMs = nextScenarioMinuteMs
- nextScenarioMinuteMs += 5 * MINUTE_MS
+ const minuteMs = Math.ceil(nextScenarioMinuteMs / alignMs) * alignMs
+ nextScenarioMinuteMs = minuteMs + spanMs
vi.setSystemTime(minuteMs + 10_000)
- return { app: mod.createApp(new mod.MockGateway()), minuteMs }
+ const send = await serve(mod.createApp(new mod.MockGateway()))
+ return { send, minuteMs }
+ }
+
+ /** Serves `app` on one listening server; `send` reuses one keep-alive agent. */
+ async function serve(app: import('express').Express) {
+ const internalToken = mod.config.internalServiceTokens['external-rest-api']
+ if (!internalToken) throw new Error('config has no external-rest-api internal service token')
+ const server = http.createServer(app)
+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve))
+ const { port } = server.address() as AddressInfo
+ const agent = new http.Agent({ keepAlive: true })
+ if (openServer) throw new Error('a scenario server is already open')
+ openServer = {
+ close: () => {
+ agent.destroy()
+ server.closeAllConnections()
+ return new Promise<void>((resolve, reject) =>
+ server.close(error => (error ? reject(error) : resolve()))
+ )
+ },
+ }
+
+ /** GET, or POST with `post.body` as JSON. */
+ return function send(
+ path: string,
+ token: string,
+ ip: string,
+ post?: { body: unknown }
+ ): Promise<Sent> {
+ const method = post === undefined ? 'GET' : 'POST'
+ const payload = post === undefined ? undefined : JSON.stringify(post.body)
+ const headers: Record<string, string> = {
+ authorization: `Bearer ${internalToken}`,
+ 'x-service-token': 'external-rest-api',
+ 'x-user-session-token': token,
+ 'x-forwarded-for': ip,
+ }
+ if (payload !== undefined) {
+ headers['content-type'] = 'application/json'
+ headers['content-length'] = String(Buffer.byteLength(payload))
+ }
+ return new Promise<Sent>((resolve, reject) => {
+ const req = http.request({ host: '127.0.0.1', port, method, path, agent, headers }, res => {
+ const chunks: Buffer[] = []
+ res.on('data', (chunk: Buffer) => chunks.push(chunk))
+ res.on('error', reject)
+ res.on('end', () => {
+ const text = Buffer.concat(chunks).toString('utf-8')
+ try {
+ resolve({
+ path,
+ status: res.statusCode ?? 0,
+ body: (text === '' ? {} : JSON.parse(text)) as { error?: unknown },
+ policy: singleHeader(res, 'ratelimit-policy'),
+ xLimit: singleHeader(res, 'x-ratelimit-limit'),
+ xRemaining: singleHeader(res, 'x-ratelimit-remaining'),
+ })
+ } catch (error) {
+ reject(
+ new Error(
+ `${method} ${path} answered ${res.statusCode} with a non-JSON body ` +
+ `(${String(error)}): ${text}`
+ )
+ )
+ }
+ })
+ })
+ req.on('error', reject)
+ req.end(payload)
+ })
+ }
+ }
+
+ function singleHeader(res: http.IncomingMessage, name: string): string | undefined {
+ const value = res.headers[name]
+ if (Array.isArray(value)) throw new Error(`header ${name} repeated: ${value.join(', ')}`)
+ return value
}
async function seedUser(): Promise<string> {
@@ -203,32 +357,26 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
const grantsPath = (resourceId: string) =>
`/api/v1/external/gfs/grants?drive=main&resourceId=${resourceId}`
- async function send(
- app: import('express').Express,
- path: string,
- token: string,
- ip: string
- ): Promise<Sent> {
- const internalToken = mod.config.internalServiceTokens['external-rest-api']
- if (!internalToken) throw new Error('config has no external-rest-api internal service token')
- const res = await request(app)
- .get(path)
- .set('Authorization', `Bearer ${internalToken}`)
- .set('x-service-token', 'external-rest-api')
- .set('x-user-session-token', token)
- .set('x-forwarded-for', ip)
- return {
- path,
- status: res.status,
- body: res.body as { error?: unknown },
- policy: res.headers['ratelimit-policy'],
- xLimit: res.headers['x-ratelimit-limit'],
- xRemaining: res.headers['x-ratelimit-remaining'],
+ const childrenPath = (resourceId: string) =>
+ `/api/v1/external/gfs/resources/${resourceId}/children?drive=main`
+
+ /**
+ * Denial-source classifier, for the two routes whose 429 headers identify the
+ * limiter (see the header comment). Any other route throws. Allowed
+ * responses keep their status.
+ */
+ function classify(
+ response: Sent,
+ route: 'resource-read' | 'resource-mutation',
+ budgets: Budgets
+ ): string {
+ const [pattern, routeL] =
+ route === 'resource-read'
+ ? [RESOURCE_READ_ROUTE, budgets.resource]
+ : [RESOURCE_MUTATION_ROUTE, budgets.operation]
+ if (!pattern.test(response.path)) {
+ throw new Error(`classify(${route}) received ${response.path}`)
}
- }
-
- /** Addendum A3 classifier. Allowed responses keep their status. */
- function classify(response: Sent, routeL: number): string {
if (response.status !== 429) return `allowed:${response.status}`
if (response.policy === `${routeL};w=60`) return 'route-backstop'
if (response.policy === `${INGRESS_L};w=60` && response.xRemaining === '0')
@@ -274,6 +422,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
function limitOf(key: string, budgets: Budgets): number {
if (key.startsWith('gfs-ext:pre:ip:')) return budgets.ip
if (/^gfs-ext:(pre|resolved):resource:/.test(key)) return budgets.resource
+ if (/^gfs-ext:(pre|resolved):resource-mutation:/.test(key)) return budgets.operation
if (/^gfs-ext:(pre|resolved):grants-read:/.test(key)) return budgets.grants
// GET /grants and GET /shares share this bucket; its limit is the sum.
if (key.startsWith('gfsgrants-ext-read:user:')) return budgets.grants + budgets.shares
@@ -332,7 +481,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
it('S1: 20 users with distinct sessions and IPs each get exactly the read budget', async () => {
const budgets = uniformBudgets(6, 6, IP_L_DEFAULT)
- const { app } = await startScenario(budgets)
+ const { send } = await startScenario(budgets)
const users = await Promise.all(Array.from({ length: 20 }, () => seedUser()))
const resourceId = randomUUID()
@@ -340,12 +489,12 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
Promise.all(
users.flatMap((userId, index) =>
Array.from({ length: 10 }, () =>
- send(app, affordancesPath(resourceId), sessionToken(userId), `198.51.100.${index + 1}`)
+ send(affordancesPath(resourceId), sessionToken(userId), `198.51.100.${index + 1}`)
)
)
)
)
- const labels = run.result.map(response => classify(response, budgets.resource))
+ const labels = run.result.map(response => classify(response, 'resource-read', budgets))
expect(run.result).toHaveLength(200)
for (let user = 0; user < 20; user += 1) {
@@ -375,7 +524,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
it('S2a: 20 users behind one IP share the (class, IP) read bucket', async () => {
const budgets = uniformBudgets(30, 30, IP_L_DEFAULT)
- const { app } = await startScenario(budgets)
+ const { send } = await startScenario(budgets)
const users = await Promise.all(Array.from({ length: 20 }, () => seedUser()))
const resourceId = randomUUID()
@@ -383,16 +532,18 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
Promise.all(
users.flatMap(userId =>
Array.from({ length: 5 }, () =>
- send(app, affordancesPath(resourceId), sessionToken(userId), '198.51.100.30')
+ send(affordancesPath(resourceId), sessionToken(userId), '198.51.100.30')
)
)
)
)
- expect(tally(run.result.map(response => classify(response, budgets.resource)))).toEqual({
- 'allowed:200': 30,
- 'postgres:30': 70,
- })
+ expect(tally(run.result.map(response => classify(response, 'resource-read', budgets)))).toEqual(
+ {
+ 'allowed:200': 30,
+ 'postgres:30': 70,
+ }
+ )
const ledger = await readLedger()
expect(counts(ledger, 'gfs-ext:pre:resource:session:')).toEqual(Array(20).fill(5))
expect(counts(ledger, 'gfs-ext:pre:resource:ip:')).toEqual([100])
@@ -414,7 +565,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
// the IP, because each class alone is capped at its own L first. The two
// classes carry different budgets, so each bucket must use its own class's.
const budgets = { resource: 30, proxy: 30, grants: 20, shares: 20, operation: 20, ip: 40 }
- const { app } = await startScenario(budgets)
+ const { send } = await startScenario(budgets)
const users = await Promise.all(Array.from({ length: 10 }, () => seedUser()))
const resourceId = randomUUID()
@@ -423,7 +574,6 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
users.flatMap((userId, index) =>
Array.from({ length: 5 }, () =>
send(
- app,
index < 5 ? affordancesPath(resourceId) : grantsPath(resourceId),
sessionToken(userId),
'198.51.100.40'
@@ -432,31 +582,33 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
)
)
)
- const labels = run.result.map(response => classify(response, budgets.resource))
- const tallied = tally(labels)
+ const affordances = run.result.filter(response => response.path.includes('/affordances'))
+ const grants = run.result.filter(response => response.path.includes('/grants'))
+ expect(affordances).toHaveLength(25)
+ expect(grants).toHaveLength(25)
+ const affordanceLabels = tally(
+ affordances.map(response => classify(response, 'resource-read', budgets))
+ )
+ const grantStatuses = tally(grants.map(response => String(response.status)))
// The (grants-read, IP) bucket sees 25 against 20 and denies 5; pre:ip
- // then sees 25 resource + 20 grants = 45 against 40 and denies 5.
- expect(tallied['postgres:20']).toBe(5)
- expect(tallied['postgres:40']).toBe(5)
- expect((tallied['allowed:200'] ?? 0) + (tallied['allowed:403'] ?? 0)).toBe(40)
- expect(Object.keys(tallied).sort()).toEqual(
- ['allowed:200', 'allowed:403', 'postgres:20', 'postgres:40'].filter(
- label => tallied[label] !== undefined
- )
+ // then sees 25 resource + 20 grants = 45 against 40 and denies 5, on
+ // either route. On the affordances route only pre:ip can deny.
+ const affordances429 = affordanceLabels['postgres:40'] ?? 0
+ const grants429 = grantStatuses['429'] ?? 0
+ expect(Object.keys(affordanceLabels).sort()).toEqual(
+ ['allowed:200', 'postgres:40'].filter(label => affordanceLabels[label] !== undefined)
)
- // Only the grants class is capped at 20.
- run.result.forEach((response, index) => {
- if (labels[index] === 'postgres:20') expect(response.path).toContain('/grants')
- })
- // "Allowed" is defined per route: affordances answer 200, the grants list
- // answers 403 manage_acl_required for a caller without manage_acl.
- for (const response of run.result) {
- if (response.status === 200) expect(response.path).toContain('/affordances')
- if (response.status === 403) {
- expect(response.path).toContain('/grants')
- expect(response.body.error).toBe('manage_acl_required')
- }
+ expect(Object.keys(grantStatuses).sort()).toEqual(
+ ['403', '429'].filter(status => grantStatuses[status] !== undefined)
+ )
+ expect(affordances429 + grants429).toBe(10)
+ expect(grants429).toBeGreaterThanOrEqual(5)
+ expect((affordanceLabels['allowed:200'] ?? 0) + (grantStatuses['403'] ?? 0)).toBe(40)
+ // "Allowed" is defined per route: the grants list answers 403
+ // manage_acl_required for a caller without manage_acl.
+ for (const response of grants) {
+ if (response.status === 403) expect(response.body.error).toBe('manage_acl_required')
}
const ledger = await readLedger()
expect(counts(ledger, 'gfs-ext:pre:resource:session:')).toEqual(Array(5).fill(5))
@@ -465,7 +617,11 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
expect(counts(ledger, 'gfs-ext:pre:grants-read:ip:')).toEqual([25])
expect(counts(ledger, 'gfs-ext:pre:ip:')).toEqual([45])
expect(sum(counts(ledger, 'gfs-ext:resolved:'))).toBe(40)
- expect(sum(counts(ledger, 'gfsgrants-ext-read:user:'))).toBe(tallied['allowed:403'] ?? 0)
+ // Every grants request that reached the rateLimitMiddleware bucket got a
+ // 403: that limiter denied none, and its row is under its limit.
+ expect(sum(counts(ledger, 'gfsgrants-ext-read:user:'))).toBe(grantStatuses['403'] ?? 0)
+ // The 10 denials are exactly the two Postgres buckets' overflow, and the
+ // metric below shows no backstop denial: this is the grants attribution.
expect(ledgerDenials(ledger, budgets)).toBe(10)
expect(run.dbErrors).toBe(0)
expect(run.metricDelta).toEqual({
@@ -478,25 +634,27 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
it('S3: two sessions of one user share the resolved actor bucket', async () => {
const budgets = uniformBudgets(10, 10, IP_L_DEFAULT)
- const { app } = await startScenario(budgets)
+ const { send } = await startScenario(budgets)
const userId = await seedUser()
const resourceId = randomUUID()
const run = await observed(() =>
Promise.all([
...Array.from({ length: 8 }, () =>
- send(app, affordancesPath(resourceId), sessionToken(userId, 1), '198.51.100.51')
+ send(affordancesPath(resourceId), sessionToken(userId, 1), '198.51.100.51')
),
...Array.from({ length: 8 }, () =>
- send(app, affordancesPath(resourceId), sessionToken(userId, 2), '198.51.100.52')
+ send(affordancesPath(resourceId), sessionToken(userId, 2), '198.51.100.52')
),
])
)
- expect(tally(run.result.map(response => classify(response, budgets.resource)))).toEqual({
- 'allowed:200': 10,
- 'postgres:10': 6,
- })
+ expect(tally(run.result.map(response => classify(response, 'resource-read', budgets)))).toEqual(
+ {
+ 'allowed:200': 10,
+ 'postgres:10': 6,
+ }
+ )
const ledger = await readLedger()
// Each pre-resolution bucket saw 8 <= 10 and denied nothing; the resolved
// actor bucket saw all 16: the phase witness, independent of telemetry.
@@ -516,19 +674,19 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
it('S4: a burst straddling a minute boundary is denied only by the route backstop', async () => {
const budgets = uniformBudgets(8, 8, IP_L_DEFAULT)
- const { app, minuteMs } = await startScenario(budgets)
+ const { send, minuteMs } = await startScenario(budgets)
const userId = await seedUser()
const resourceId = randomUUID()
const burst = () =>
observed(() =>
Promise.all(
Array.from({ length: 8 }, () =>
- send(app, affordancesPath(resourceId), sessionToken(userId), '198.51.100.60')
+ send(affordancesPath(resourceId), sessionToken(userId), '198.51.100.60')
)
)
)
const labelsOf = (responses: Sent[]) =>
- tally(responses.map(response => classify(response, budgets.resource)))
+ tally(responses.map(response => classify(response, 'resource-read', budgets)))
// A: 50 s into window 1. Opens the backstop window, fills the Postgres buckets.
vi.setSystemTime(minuteMs + 50_000)
@@ -585,4 +743,253 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
'edge-backstop': 1,
})
}, 120_000)
+
+ it('E1: 481 concurrent reads by one actor at the production budget: 480 allowed, one Postgres 429, no 503', async () => {
+ const budgets = PRODUCTION_BUDGETS
+ const { send } = await startScenario(budgets)
+ const userId = await seedUser()
+ const resourceId = randomUUID()
+
+ const run = await observed(() =>
+ Promise.all(
+ Array.from({ length: 481 }, () =>
+ send(affordancesPath(resourceId), sessionToken(userId), '198.51.100.70')
+ )
+ )
+ )
+
+ // Exact: a 503 from either pool or any other status fails here.
+ expect(tally(run.result.map(response => classify(response, 'resource-read', budgets)))).toEqual(
+ { 'allowed:200': 480, 'postgres:480': 1 }
+ )
+ const ledger = await readLedger()
+ expect(counts(ledger, 'gfs-ext:pre:resource:session:')).toEqual([481])
+ expect(counts(ledger, 'gfs-ext:pre:resource:ip:')).toEqual([480])
+ expect(counts(ledger, 'gfs-ext:pre:ip:')).toEqual([480])
+ expect(counts(ledger, `gfs-ext:resolved:resource:actor:user-session:${userId}`)).toEqual([480])
+ expect(ledgerDenials(ledger, budgets)).toBe(1)
+ expect(run.dbErrors).toBe(0)
+ expect(run.metricDelta).toEqual({
+ 'pre-resolution': 1,
+ 'resolved-operation': 0,
+ 'edge-backstop': 0,
+ })
+ expect(run.logged).toEqual(run.metricDelta)
+ }, 120_000)
+
+ it('E2: a burst straddling a minute boundary passes 2L-1 = 959 reads in 60 s; the next is a Postgres 429', async () => {
+ const budgets = PRODUCTION_BUDGETS
+ const { send, minuteMs } = await startScenario(budgets)
+ const userId = await seedUser()
+ const resourceId = randomUUID()
+ const boundary = minuteMs + 2 * MINUTE_MS
+ const burstAt = async (atMs: number, size: number) => {
+ vi.setSystemTime(atMs)
+ const responses = await Promise.all(
+ Array.from({ length: size }, () =>
+ send(affordancesPath(resourceId), sessionToken(userId), '198.51.100.71')
+ )
+ )
+ return responses.map(response => ({
+ atMs,
+ label: classify(response, 'resource-read', budgets),
+ }))
+ }
+
+ const run = await observed(async () => [
+ // Opens the backstop window at M:00.0, so it resets at M+1:00.0.
+ ...(await burstAt(boundary - MINUTE_MS, 1)),
+ ...(await burstAt(boundary - 100, 479)),
+ ...(await burstAt(boundary, 480)),
+ ...(await burstAt(boundary, 1)),
+ ])
+
+ expect(tally(run.result.map(sent => sent.label))).toEqual({
+ 'allowed:200': 960,
+ 'postgres:480': 1,
+ })
+ // The sliding bound: in the 60 s ending at M+1:00.0, 959 reads passed.
+ expect(
+ run.result.filter(
+ sent =>
+ sent.label === 'allowed:200' && sent.atMs > boundary - MINUTE_MS && sent.atMs <= boundary
+ )
+ ).toHaveLength(959)
+ expect(run.result[run.result.length - 1]?.label).toBe('postgres:480')
+ const ledger = await readLedger()
+ const before = boundary - MINUTE_MS
+ expect(counts(ledger, 'gfs-ext:pre:resource:session:', before)).toEqual([480])
+ expect(counts(ledger, 'gfs-ext:pre:resource:session:', boundary)).toEqual([481])
+ expect(counts(ledger, 'gfs-ext:resolved:resource:actor:', before)).toEqual([480])
+ expect(counts(ledger, 'gfs-ext:resolved:resource:actor:', boundary)).toEqual([480])
+ expect(ledgerDenials(ledger, budgets)).toBe(1)
+ expect(run.dbErrors).toBe(0)
+ expect(run.metricDelta).toEqual({
+ 'pre-resolution': 1,
+ 'resolved-operation': 0,
+ 'edge-backstop': 0,
+ })
+ expect(run.logged).toEqual(run.metricDelta)
+ }, 120_000)
+
+ it('E3: 91 mutations by one actor at the production operation budget: 90 reach gfsc, the 91st is a Postgres 429', async () => {
+ const budgets = PRODUCTION_BUDGETS
+ const { send } = await startScenario(budgets)
+ const userId = await seedUser()
+ const resourceId = randomUUID()
+ // The only boundary mock: the fetch from control-api to gfsc.
+ const gfscFetch = vi.fn(
+ async (_url: string | URL | Request, _init?: RequestInit) =>
+ new Response(JSON.stringify({ id: randomUUID() }), {
+ status: 201,
+ headers: { 'content-type': 'application/json' },
+ })
+ )
+ vi.stubGlobal('fetch', gfscFetch)
+ try {
+ const run = await observed(() =>
+ Promise.all(
+ Array.from({ length: 91 }, () =>
+ send(childrenPath(resourceId), sessionToken(userId), '198.51.100.72', {
+ body: { name: 'stress', kind: 'folder' },
+ })
+ )
+ )
+ )
+
+ expect(
+ tally(run.result.map(response => classify(response, 'resource-mutation', budgets)))
+ ).toEqual({ 'allowed:201': 90, 'postgres:90': 1 })
+ // Witness: exactly the 90 admitted mutations reached gfsc, each as the
+ // POST the route proxies.
+ expect(gfscFetch).toHaveBeenCalledTimes(90)
+ const target =
+ `${mod.config.gfscWriteBaseUrl.replace(/\/+$/, '')}` +
+ `/v1/resources/${resourceId.replace(/-/g, '')}/children`
+ for (const [url, init] of gfscFetch.mock.calls) {
+ expect(String(url)).toBe(target)
+ expect(init?.method).toBe('POST')
+ }
+ const ledger = await readLedger()
+ expect(counts(ledger, 'gfs-ext:pre:resource-mutation:session:')).toEqual([91])
+ expect(counts(ledger, 'gfs-ext:pre:resource-mutation:ip:')).toEqual([90])
+ expect(counts(ledger, 'gfs-ext:pre:ip:')).toEqual([90])
+ expect(
+ counts(ledger, `gfs-ext:resolved:resource-mutation:actor:user-session:${userId}`)
+ ).toEqual([90])
+ expect(ledgerDenials(ledger, budgets)).toBe(1)
+ expect(run.dbErrors).toBe(0)
+ expect(run.metricDelta).toEqual({
+ 'pre-resolution': 1,
+ 'resolved-operation': 0,
+ 'edge-backstop': 0,
+ })
+ expect(run.logged).toEqual(run.metricDelta)
+ } finally {
+ vi.unstubAllGlobals()
+ }
+ }, 120_000)
+
+ /**
+ * Replays DESKTOP_READS_2026_09_18 serially, one actor, one session, one IP
+ * (the upper bound on any one actor's count), each read at its offset on a
+ * fresh fake UTC day. Denials are labelled with their UTC second.
+ */
+ async function replayDesktopDay(budgets: Budgets) {
+ const { send, minuteMs: dayStartMs } = await startScenario(
+ budgets,
+ DAY_MS + 5 * MINUTE_MS,
+ DAY_MS
+ )
+ const userId = await seedUser()
+ const resourceId = randomUUID()
+ const denials: string[] = []
+ const run = await observed(async () => {
+ const labels: string[] = []
+ for (const offsetMs of DESKTOP_READS_2026_09_18) {
+ vi.setSystemTime(dayStartMs + offsetMs)
+ const response = await send(
+ affordancesPath(resourceId),
+ sessionToken(userId),
+ '198.51.100.80'
+ )
+ const label = classify(response, 'resource-read', budgets)
+ labels.push(label)
+ if (!label.startsWith('allowed:')) {
+ denials.push(`${label}@${new Date(offsetMs).toISOString().slice(11, 19)}`)
+ }
+ }
+ return labels
+ })
+ return { ...run, denials, userId }
+ }
+
+ it('E4: the 1965 Desktop reads of 2026-09-18 replayed at the production budget get no 429 and no 503', async () => {
+ expect(DESKTOP_READS_2026_09_18).toHaveLength(1965)
+ expect(
+ DESKTOP_READS_2026_09_18.every((offset, i, all) => i === 0 || all[i - 1]! <= offset)
+ ).toBe(true)
+ const budgets = PRODUCTION_BUDGETS
+
+ const run = await replayDesktopDay(budgets)
+
+ expect(tally(run.result)).toEqual({ 'allowed:200': 1965 })
+ expect(run.denials).toEqual([])
+ // Witness that the limiter counted the whole day, including its busiest
+ // calendar minute (125 reads).
+ const ledger = await readLedger()
+ expect(sum(counts(ledger, 'gfs-ext:pre:resource:session:'))).toBe(1965)
+ expect(Math.max(...counts(ledger, 'gfs-ext:pre:resource:session:'))).toBe(125)
+ expect(sum(counts(ledger, `gfs-ext:resolved:resource:actor:user-session:${run.userId}`))).toBe(
+ 1965
+ )
+ expect(ledgerDenials(ledger, budgets)).toBe(0)
+ expect(run.dbErrors).toBe(0)
+ expect(run.metricDelta).toEqual({
+ 'pre-resolution': 0,
+ 'resolved-operation': 0,
+ 'edge-backstop': 0,
+ })
+ expect(run.logged).toEqual(run.metricDelta)
+ }, 300_000)
+
+ it('E4 control: the same replay at the old 120 budget reproduces the 28 production denials at their seconds', async () => {
+ // The 28 × 429 served on cluster clerum on 2026-09-18, by UTC second and
+ // limiter: nginx statuses give the seconds, the external_gfs_rate_limit
+ // events give the Postgres phase of the five at 13:06:59.
+ const observedDenials = [
+ ...Array<string>(10).fill('route-backstop@04:46:20'),
+ ...Array<string>(5).fill('postgres:120@13:06:59'),
+ ...Array<string>(7).fill('route-backstop@14:18:15'),
+ 'route-backstop@14:18:18',
+ 'route-backstop@14:18:22',
+ ...Array<string>(3).fill('route-backstop@14:18:36'),
+ 'route-backstop@14:18:47',
+ ]
+ const budgets = { ...PRODUCTION_BUDGETS, resource: 120 }
+
+ const run = await replayDesktopDay(budgets)
+
+ expect(run.denials).toEqual(observedDenials)
+ expect(tally(run.result)).toEqual({
+ 'allowed:200': 1937,
+ 'postgres:120': 5,
+ 'route-backstop': 23,
+ })
+ const ledger = await readLedger()
+ expect(ledgerDenials(ledger, budgets)).toBe(5)
+ expect(run.dbErrors).toBe(0)
+ expect(run.metricDelta).toEqual({
+ 'pre-resolution': 5,
+ 'resolved-operation': 0,
+ 'edge-backstop': 23,
+ })
+ // The backstop logs once per key per window: its 23 denials fall in two
+ // windows (04:46 and 14:18).
+ expect(run.logged).toEqual({
+ 'pre-resolution': 5,
+ 'resolved-operation': 0,
+ 'edge-backstop': 2,
+ })
+ }, 300_000)
})
diff --git a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
index d1086edb3..518903300 100644
--- a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
+++ b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
@@ -184,12 +184,15 @@ describeRealPostgres('rate limiter under pool saturation', () => {
let released = false
try {
const startedAt = performance.now()
+ const settledAfterMs: number[] = []
const results = await Promise.all(
Array.from({ length: CONCURRENT_CHECKS }, () =>
- mod.checkAndIncrement(bucketKey, LIMIT, nowMs)
+ mod.checkAndIncrement(bucketKey, LIMIT, nowMs).then(result => {
+ settledAfterMs.push(performance.now() - startedAt)
+ return result
+ })
)
)
- const elapsedMs = performance.now() - startedAt
expect(results).toHaveLength(CONCURRENT_CHECKS)
// The service reports allowed:true; the admission decision belongs to
@@ -209,9 +212,13 @@ describeRealPostgres('rate limiter under pool saturation', () => {
expect(payload.err).toMatch(/timeout exceeded when trying to connect/)
}
- // Every call waited for the acquire timeout, in parallel, not in series.
- expect(elapsedMs).toBeGreaterThanOrEqual(LIMITER_ACQUIRE_TIMEOUT_MS - 100)
- expect(elapsedMs).toBeLessThan(LIMITER_ACQUIRE_TIMEOUT_MS + 3_000)
+ // Every call waited for the acquire timeout: none settled before it,
+ // which is a property of the pool, not of machine speed. They waited in
+ // parallel: in series the second call alone would settle after two
+ // timeouts, so the bound is structural rather than a latency budget.
+ expect(settledAfterMs).toHaveLength(CONCURRENT_CHECKS)
+ expect(Math.min(...settledAfterMs)).toBeGreaterThanOrEqual(LIMITER_ACQUIRE_TIMEOUT_MS - 100)
+ expect(Math.max(...settledAfterMs)).toBeLessThan(2 * LIMITER_ACQUIRE_TIMEOUT_MS)
// Read through a held client: this also proves the database and the
// table are reachable, so "no row" is not an absent table.
diff --git a/control-api/vitest.config.ts b/control-api/vitest.config.ts
index eb9bc2a77..995a382d7 100644
--- a/control-api/vitest.config.ts
+++ b/control-api/vitest.config.ts
@@ -28,9 +28,10 @@ export default defineConfig({
fileParallelism: false,
maxWorkers: 1,
pool: 'threads',
- // Real-Postgres tests mutate shared database state and cannot be retried safely
- // after a timeout because the abandoned async work may still commit.
- retry: process.env.CONTROL_API_REAL_PG_ADMIN_URL ? 0 : 2,
+ // No retries: a retry turns a flaky test green and hides the flake. A
+ // failing test is fixed in the test. Real-Postgres tests could not be
+ // retried safely anyway, because abandoned async work may still commit.
+ retry: 0,
testTimeout: 10_000,
// Real-Postgres beforeAll runs initDb across the full migration list.
// Default hookTimeout follows testTimeout (10s) and turns a slow CREATE