Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(control-api): stress the external GFS limiter at the real budgets
Alfredo Lopez91885e1f645a
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex c0f7b6769..7c2a1ed14 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.611",+ "version": "0.1.612","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.611",+ "version": "0.1.612","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex b9757d8d4..7480cba97 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.611",+ "version": "0.1.612","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/test/config.externalGfsBudgets.test.ts b/control-api/test/config.externalGfsBudgets.test.tsindex f13d73d90..33432d113 100644--- a/control-api/test/config.externalGfsBudgets.test.ts+++ b/control-api/test/config.externalGfsBudgets.test.ts@@ -131,9 +131,11 @@ describe('external GFS actor budgets', () => {expect(Number(preferredPartMib)).toBe(UPLOAD_PREFERRED_PART_MIB)expect(MAX_UPLOAD_MUTATIONS).toBe(27)- // 27 is a floor: larger parts, retried 429s and reconciliation add more.+ // 27 is a floor for one upload at the preferred part size: a smaller part+ // size, retried 429s and reconciliation add mutations (a larger part size+ // removes some). The budget is sized for three such uploads in one minute.const { config } = await loadConfigModuleWith({})- expect(config.externalGfsOperationRlPerMin).toBeGreaterThanOrEqual(MAX_UPLOAD_MUTATIONS)+ expect(3 * MAX_UPLOAD_MUTATIONS).toBeLessThanOrEqual(config.externalGfsOperationRlPerMin)})it('L12: defaults to 480/480/120/120 reads and 90 operations per minute', async () => {diff --git a/control-api/test/fixtures/external-gfs-desktop-reads-2026-09-18.json b/control-api/test/fixtures/external-gfs-desktop-reads-2026-09-18.jsonnew file mode 100644index 000000000..beb813b60--- /dev/null+++ b/control-api/test/fixtures/external-gfs-desktop-reads-2026-09-18.json@@ -0,0 +1,223 @@+{+ "description": "Desktop external GFS resource-class reads on cluster clerum, 2026-09-18 (UTC), the day of the Desktop 429 incident. Each entry is the request's log time in milliseconds since 00:00 UTC; nothing else about the request is kept.",+ "offsetsMs": [+ 1978538, 1986761, 1990501, 1994750, 1999585, 2028160, 2029717, 2029749, 2029891, 2029894,+ 2030047, 2030047, 2030137, 2030138, 2030235, 2030236, 2030419, 2030437, 2030438, 2030532,+ 2030537, 2034222, 2035454, 2035458, 2035462, 2035520, 2035532, 2035533, 2035632, 2035634,+ 2035638, 2035642, 2035643, 2035645, 2035647, 2035719, 2037866, 2525622, 2525721, 2540070,+ 2540155, 2602460, 2618647, 2629123, 2743548, 2927492, 3006574, 3042130, 3107792, 3123288,+ 3129202, 3135569, 3140778, 3244659, 6437148, 6509820, 6511749, 6516534, 6582257, 6598528,+ 9264229, 9353239, 9422163, 16802775, 16885324, 16888864, 16888922, 16889076, 16889115, 16889209,+ 16889212, 16889232, 16889236, 16889241, 16889339, 16889419, 16889435, 16889519, 16889533,+ 16889630, 16890445, 16895285, 16895329, 16895477, 16895630, 16895635, 16895783, 16895784,+ 16895806, 16895814, 16895815, 16895824, 16895826, 16895838, 16895842, 16895844, 16895846,+ 16898055, 16899486, 16899522, 16899669, 16899820, 16899837, 16899843, 16899843, 16899849,+ 16899924, 16899931, 16899931, 16899931, 16899932, 16899932, 16899936, 16899938, 16903966,+ 16907386, 16952938, 16952979, 16953167, 16953652, 16953731, 16953737, 16953831, 16953933,+ 16953941, 16954031, 16954036, 16954037, 16954120, 16954130, 16954133, 16954134, 16955452,+ 16955461, 16955609, 16956130, 16956139, 16956232, 16956233, 16956239, 16956242, 16956245,+ 16956332, 16956335, 16956345, 16956348, 16956431, 16956433, 16958160, 16958168, 16958337,+ 16958645, 16958647, 16958732, 16958733, 16958735, 16958741, 16958741, 16958746, 16958831,+ 16958832, 16958832, 16958833, 16958835, 16965578, 16965607, 16965748, 16965886, 16965890,+ 16965893, 16965897, 16966028, 16966041, 16966047, 16966051, 16966052, 16966057, 16966057,+ 16966069, 16966121, 16970621, 16973328, 16974763, 16974785, 16974937, 16975091, 16975095,+ 16975099, 16975103, 16975106, 16975107, 16975118, 16975244, 16975248, 16975252, 16975255,+ 16975260, 16975294, 16977275, 16977281, 16977421, 16977572, 16977580, 16977584, 16977588,+ 16977590, 16977620, 16977631, 16977632, 16977634, 16977643, 16977646, 16977648, 16977731,+ 16996392, 16996403, 17004893, 17004913, 17052354, 17052372, 17052516, 17052672, 17052685,+ 17052707, 17052818, 17052818, 17052845, 17053038, 17053038, 17053041, 17053130, 17053136,+ 17053137, 17053139, 17077142, 17077169, 17077281, 17077418, 17077422, 17077427, 17077447,+ 17077561, 17077577, 17077583, 17077586, 17077589, 17077600, 17077606, 17077613, 17077631,+ 17094232, 17094262, 17130362, 17130379, 17130510, 17130663, 17130665, 17130850, 17130930,+ 17130944, 17131030, 17131041, 17131118, 17131334, 17131334, 17131335, 17131336, 17131338,+ 17148321, 17148376, 17148994, 17148998, 17149144, 17149531, 17149536, 17149537, 17149603,+ 17149603, 17149623, 17149630, 17149640, 17149640, 17149641, 17149642, 17149646, 17149646,+ 17150812, 17150829, 17150967, 17151130, 17151133, 17151136, 17151147, 17151154, 17151218,+ 17151231, 17151232, 17151233, 17151234, 17151238, 17151238, 17151240, 17152530, 17152530,+ 17152671, 17152810, 17152817, 17152820, 17152822, 17152827, 17152831, 17152833, 17152835,+ 17152846, 17152848, 17152849, 17152851, 17152852, 17158827, 17158858, 17158998, 17159151,+ 17159152, 17159155, 17159159, 17159160, 17159171, 17159323, 17159338, 17159341, 17159344,+ 17159344, 17159347, 17159365, 17160517, 17160523, 17160667, 17160813, 17160821, 17160822,+ 17160828, 17160828, 17160831, 17160835, 17160845, 17160851, 17160853, 17160918, 17160922,+ 17160931, 17176445, 17176466, 17176610, 17176763, 17176771, 17176779, 17176832, 17176834,+ 17176836, 17176940, 17176941, 17176945, 17176956, 17176958, 17176961, 17176978, 17179979,+ 17180005, 17180143, 17180287, 17180289, 17180296, 17180298, 17180299, 17180300, 17180304,+ 17180308, 17180334, 17180339, 17180341, 17180344, 17180345, 17212431, 17212563, 17241535,+ 17241539, 17365431, 17375739, 17375850, 17375879, 17376031, 17376051, 17376052, 17376135,+ 17376136, 17376219, 17376230, 17376233, 17376236, 17376335, 17376337, 17376338, 17377205,+ 17378599, 17378606, 17378745, 17378761, 17378832, 17378833, 17378837, 17378838, 17378851,+ 17378854, 17378857, 17378858, 17378859, 17378939, 17380795, 17387632, 17391552, 17391633,+ 17391779, 17391940, 17391955, 17392133, 17392142, 17392152, 17392156, 17392232, 17392234,+ 17392236, 17392240, 17392243, 17392243, 17392245, 17394546, 17394556, 17394743, 17394945,+ 17395036, 17395036, 17395039, 17395043, 17395046, 17395048, 17395051, 17395131, 17395135,+ 17439461, 17439476, 17439618, 17439774, 17439778, 17439794, 17439830, 17439843, 17439847,+ 17439929, 17439944, 17439948, 17440018, 17440018, 17440030, 17440030, 17440945, 17440955,+ 17441092, 17441238, 17441240, 17441246, 17441250, 17441254, 17441333, 17441334, 17441336,+ 17441361, 17441435, 17441928, 17546226, 17546257, 17546402, 17546555, 17546624, 17546634,+ 17546637, 17546639, 17546643, 17546739, 17546835, 17546837, 17546844, 17546845, 17546920,+ 17546933, 17546951, 17547023, 17547168, 17547305, 17547308, 17547315, 17547319, 17547320,+ 17547334, 17547339, 17547339, 17547344, 17547344, 17547971, 17549355, 17549356, 17549488,+ 17549642, 17549644, 17549650, 17549652, 17549731, 17549733, 17549736, 17549740, 17549742,+ 17549819, 17549829, 17549830, 17549834, 17552454, 17552464, 17552605, 17552748, 17552755,+ 17552759, 17552761, 17552763, 17552831, 17552833, 17552839, 17552839, 17552841, 17552844,+ 17552845, 17552940, 17679956, 17683828, 17683885, 17684028, 17684035, 17684035, 17684040,+ 17684119, 17684130, 17684131, 17684134, 17684141, 17684222, 17684226, 17684230, 17684301,+ 18518715, 18520166, 18520261, 18520310, 18520470, 18520494, 18520496, 18520531, 18520533,+ 18520539, 18520543, 18520547, 18520631, 18520636, 18520640, 18520640, 18521965, 18524534,+ 18524720, 18524866, 18525009, 18525029, 18525162, 18525177, 18525231, 18525237, 18525245,+ 18525331, 18525333, 18525336, 18525338, 18525339, 18525341, 18526082, 18588292, 18588306,+ 18588621, 18588779, 18588781, 18588837, 18589021, 18589032, 18589036, 18589036, 18589039,+ 18589137, 18589218, 18589232, 18589333, 18589337, 18589420, 19066795, 19066957, 19066981,+ 19067115, 19070667, 19070691, 19070840, 19071015, 19071018, 19071018, 19071034, 19071045,+ 19071118, 19071131, 19071160, 19071165, 19071169, 19071173, 19071242, 19071244, 19071784,+ 20488089, 22460772, 22467355, 22467383, 22467521, 22467678, 22467679, 22467703, 22467736,+ 22467737, 22467738, 22467744, 22467831, 22467834, 22467841, 22467842, 22467848, 22468994,+ 22482461, 22484332, 22484584, 22484605, 22484849, 22484866, 22484872, 22484877, 22484899,+ 22484902, 22484908, 22484930, 22484940, 22484943, 22484946, 22484948, 22488723, 22488756,+ 30961421, 30964363, 30964368, 30964546, 30964550, 30976683, 30976778, 30977006, 30977012,+ 30978681, 31068568, 31072392, 31072684, 31072864, 31072892, 31081456, 31081578, 31081848,+ 31084660, 31084685, 31091763, 31100796, 31110955, 31111052, 31116102, 31116126, 31116130,+ 31129425, 31129445, 31129496, 32879160, 32879204, 32879669, 36023762, 36027274, 36027340,+ 36027527, 36027532, 36027560, 36027631, 36029341, 36029367, 36030032, 36030035, 36030062,+ 36049448, 36049632, 36049638, 36050537, 36050538, 36050555, 36050636, 36052444, 36052446,+ 36052456, 36052538, 36053394, 36053395, 36053402, 36053408, 36054625, 36077541, 36077559,+ 36077561, 36079498, 36079504, 36079513, 36079535, 36081602, 36081605, 36081610, 36081645,+ 36081648, 36091457, 36091506, 36091508, 36113741, 36113754, 36115370, 36115402, 36115403,+ 36115449, 36115449, 36154290, 36154330, 36154336, 37306056, 37306058, 37308123, 37311549,+ 37311570, 37311733, 37311764, 37311802, 37311833, 37318770, 37335330, 37338414, 37339699,+ 37339735, 37339933, 37339938, 37339948, 37339963, 37341561, 37341567, 37341567, 37341568,+ 37341648, 37342874, 37342890, 37342892, 37342895, 37345150, 37345158, 37345159, 37354808,+ 37570479, 37570614, 38093708, 38096207, 38096235, 38096424, 38096427, 38096471, 38096707,+ 38549235, 38551057, 38551109, 38551133, 38551136, 38551316, 38551329, 38552527, 38552534,+ 38552551, 38552561, 38552653, 38552654, 38552723, 38552828, 38553110, 38553132, 38553232,+ 38553563, 38553651, 38553742, 38553756, 38553840, 38553926, 38553927, 38553928, 38554047,+ 38554058, 38554142, 38554143, 38554144, 38554232, 38554260, 38554266, 38554266, 38554338,+ 38554338, 38554345, 38554418, 38554432, 38554432, 38554440, 38554441, 38554441, 38554441,+ 38554442, 38554518, 38554633, 38554635, 38554640, 38554734, 38554833, 38554934, 38554940,+ 38555031, 38555032, 38555034, 38555038, 38555040, 38555041, 38555045, 39540820, 39542565,+ 39542617, 39542797, 39542920, 39543021, 39543202, 39544778, 39607643, 39607670, 39614637,+ 39615167, 39615451, 39615629, 39619207, 39619646, 39619744, 39619915, 39694859, 39694860,+ 39695358, 39695423, 39695591, 39695617, 39697189, 39697190, 39697216, 39697258, 39697262,+ 39709679, 39710122, 39710190, 39710372, 39710432, 39710434, 39710775, 39710789, 39710971,+ 39710981, 39714716, 39715307, 39715454, 39715636, 39715647, 39737399, 39737450, 39822665,+ 39822700, 39826382, 39828365, 39828395, 39828567, 39828625, 39828640, 39828662, 39830466,+ 39830473, 39830534, 39840547, 39841853, 39842035, 39842247, 39876181, 39876631, 39876692,+ 39876873, 39926875, 39927633, 39927638, 39927855, 39927855, 39928348, 39928352, 39928544,+ 39928556, 39929000, 39929004, 39929180, 39929190, 39929585, 39929586, 39929769, 39929818,+ 39930320, 39930332, 39930554, 39930555, 39931050, 39931054, 39931239, 39943172, 39943198,+ 39943694, 39951329, 39951392, 39951567, 39951582, 39951620, 39951647, 39954204, 39954211,+ 39954252, 39954258, 39954338, 39954341, 39954434, 39954434, 39968586, 39968627, 39973179,+ 39974588, 39974593, 39974762, 39974781, 39974830, 39974860, 39978344, 39978347, 39978432,+ 39978442, 39978460, 39978533, 39978535, 39978544, 39978547, 39978548, 39978549, 39978554,+ 39990839, 39990866, 42328859, 42329061, 42389190, 42389398, 46820825, 46823083, 46823153,+ 46823365, 46823381, 46823415, 46823416, 46823458, 46823532, 46823536, 46823538, 46823566,+ 46823618, 46823635, 46823641, 46824527, 46824639, 46824648, 46824724, 46824733, 46824744,+ 46824825, 46825020, 46825034, 46825040, 46825055, 46825055, 46825132, 46825138, 46825150,+ 46825231, 46825233, 46825235, 46825248, 46825248, 46825251, 46825318, 46825332, 46825332,+ 46825335, 46825336, 46825337, 46825338, 46825339, 46825340, 46825340, 46825346, 46838433,+ 46838495, 46838755, 46838765, 46838807, 46838850, 46838903, 46838939, 46838956, 46843472,+ 46849532, 46850448, 46850720, 46851040, 46851045, 46851341, 46852570, 46897357, 46999250,+ 46999550, 47160842, 47162764, 47162961, 47163124, 47163153, 47163157, 47163220, 47163242,+ 47164222, 47164239, 47164241, 47164250, 47164335, 47164345, 47164350, 47164358, 47164359,+ 47164435, 47164443, 47164459, 47164463, 47164536, 47164537, 47164566, 47164670, 47164731,+ 47164753, 47164755, 47164755, 47164847, 47164961, 47164962, 47165042, 47165046, 47165048,+ 47165122, 47165138, 47165235, 47165241, 47165348, 47165430, 47165446, 47165532, 47165553,+ 47165554, 47165560, 47165561, 47165631, 47165643, 47165645, 47165752, 47165756, 47165831,+ 47165836, 47165837, 47165838, 47165919, 47165950, 47165971, 47165971, 47166058, 47166059,+ 47166060, 47166062, 47166144, 47166145, 47166154, 47166155, 47166164, 47166167, 47166218,+ 47166221, 47166230, 47166231, 47166231, 47166232, 47166232, 47166233, 47166239, 47166240,+ 47166241, 47214664, 47216483, 47216561, 47216638, 47216730, 47216773, 47218444, 47218651,+ 47219154, 47219155, 47219155, 47219156, 47219238, 47219339, 47219423, 47219432, 47219435,+ 47219435, 47219436, 47219441, 47219443, 47219445, 47219534, 47219536, 47219539, 47219540,+ 47219545, 47219633, 47219634, 47219635, 47219637, 47219638, 47219642, 47219642, 47219643,+ 47219644, 47219648, 47219648, 47219732, 47219732, 47219735, 47219735, 47219736, 47219737,+ 47226123, 47226151, 47226152, 47228824, 47240622, 47240625, 47240636, 47240638, 47240643,+ 47240645, 47240658, 47240722, 47240736, 47240754, 47240820, 47240820, 47240840, 47240843,+ 47240848, 47240854, 47240931, 47240942, 47240948, 47241033, 47241033, 47241037, 47241049,+ 47241052, 47241054, 47241130, 47241132, 47241133, 47241138, 47241138, 47241142, 47241142,+ 47241144, 47241218, 47241218, 47241219, 47241219, 47243584, 47281871, 47281890, 47780562,+ 47782106, 47782119, 47782134, 47782150, 47782170, 47783844, 47783844, 47783852, 47783855,+ 47783874, 47783931, 47783934, 47783937, 47783946, 47783949, 47783953, 47784154, 47784222,+ 47784242, 47784330, 47784339, 47784341, 47784343, 47784350, 47784354, 47784418, 47784431,+ 47784431, 47784434, 47784436, 47784436, 47784437, 47784438, 47784440, 47784444, 47784444,+ 47784445, 47784445, 47784446, 47784519, 47784530, 47784539, 47789702, 51470079, 51475056,+ 51475116, 51475294, 51475324, 51475495, 51475536, 51475671, 51475697, 51476754, 51476754,+ 51476780, 51476782, 51476804, 51476834, 51476951, 51476955, 51476959, 51476962, 51476968,+ 51477031, 51477133, 51477138, 51477139, 51477139, 51477143, 51477144, 51477145, 51477150,+ 51477262, 51477442, 51477472, 51477532, 51477560, 51477561, 51477573, 51477740, 51477744,+ 51477764, 51477764, 51477833, 51477844, 51477932, 51477963, 51477964, 51478044, 51478046,+ 51478046, 51478049, 51478049, 51478062, 51478133, 51478136, 51478165, 51478249, 51478261,+ 51478262, 51478262, 51478335, 51478336, 51478354, 51478356, 51478357, 51478435, 51478438,+ 51478439, 51478442, 51478443, 51478445, 51478449, 51478532, 51478533, 51478534, 51478535,+ 51478662, 51478663, 51478665, 51478665, 51478725, 51478726, 51478727, 51478728, 51478738,+ 51478740, 51481830, 51481854, 51483632, 51494864, 51494891, 51494913, 51494916, 51494921,+ 51494930, 51494948, 51494959, 51495003, 51495017, 51495103, 51495222, 51495324, 51495537,+ 51495644, 51495648, 51495719, 51495721, 51495730, 51495731, 51495733, 51495733, 51495734,+ 51495734, 51495735, 51495736, 51495738, 51495739, 51495740, 51495743, 51495743, 51495743,+ 51495743, 51495744, 51495745, 51495746, 51495747, 51495748, 51495748, 51495749, 51498023,+ 51502484, 51516336, 51516376, 51516404, 51527076, 51559203, 51580622, 51582210, 51582298,+ 51582509, 51582512, 51582576, 51582607, 51582618, 51582632, 51584712, 51584717, 51584739,+ 51584740, 51584740, 51584747, 51584932, 51585119, 51585139, 51585155, 51585173, 51585241,+ 51585245, 51585432, 51585545, 51585574, 51585631, 51585645, 51585651, 51585651, 51585721,+ 51585731, 51585734, 51585821, 51585852, 51585852, 51585862, 51585927, 51585928, 51585939,+ 51585950, 51585960, 51585961, 51586024, 51586035, 51586036, 51586060, 51586064, 51586064,+ 51586064, 51586066, 51586066, 51586120, 51586153, 51586154, 51586154, 51586218, 51586219,+ 51586232, 51586237, 51586239, 51586239, 51586241, 51586243, 51586319, 51586319, 51586331,+ 51586331, 51586331, 51586332, 51586333, 51586337, 51586337, 51586371, 51586372, 51586469,+ 51586485, 51586534, 51586534, 51586549, 51586552, 51587029, 51693736, 51693823, 51693919,+ 51693922, 51693945, 51870259, 51870300, 51870466, 51870780, 51870804, 51870848, 51870920,+ 51870936, 51870937, 51870938, 51871033, 51871143, 51871163, 51871232, 51871235, 51871246,+ 51871257, 51871334, 51871341, 51871341, 51871439, 51871533, 51871645, 51871646, 51871834,+ 51871837, 51871837, 51871840, 51871846, 51871944, 51871948, 51871952, 51871952, 51871953,+ 51871956, 51871956, 51871957, 51872034, 51872051, 51875057, 51875061, 51875067, 51875114,+ 51875119, 51876231, 51876239, 51876242, 51876338, 51876357, 51876439, 51876540, 51876553,+ 51876565, 51876619, 51876635, 51876637, 51876643, 51876649, 51876651, 51876653, 51876656,+ 51876720, 51876735, 51876741, 51876749, 51876752, 51876755, 51876832, 51876834, 51876839,+ 51876848, 51876853, 51876932, 51876938, 51876952, 51876963, 51876965, 51877018, 51877031,+ 51877031, 51877034, 51877036, 51877044, 51877130, 51878827, 51878831, 51898432, 51898456,+ 54509512, 54511704, 54511756, 54511936, 54511972, 54512151, 54512151, 54512233, 54512250,+ 54512251, 54512252, 54512332, 54512332, 54512338, 54512342, 54512344, 54513688, 54706490,+ 54872243, 54920061, 54969804, 54980346, 54980444, 54980444, 54980447, 54980532, 54980536,+ 54980537, 54980542, 54980544, 54980545, 54980546, 54980546, 54980547, 54980551, 54981511,+ 55024926, 55090758, 55091009, 55091205, 55091409, 55091432, 55091439, 55091448, 55091450,+ 55091452, 55091533, 55091554, 55091618, 55091635, 55091642, 55091646, 55091654, 55203226,+ 55203229, 55203542, 55209731, 55209847, 55210060, 55210251, 55210254, 55210260, 55210446,+ 55210450, 55210532, 55210543, 55210549, 55210632, 55210633, 55210635, 55210637, 55210639,+ 55212527, 56233931, 56233943, 56477010, 56477034, 56484707, 56484904, 56485527, 56485697,+ 56485703, 56485703, 56485732, 56485746, 56485772, 56485856, 56485861, 56485870, 56485874,+ 56485940, 56485940, 56486002, 56487135, 56487137, 56487320, 56487511, 56487517, 56487531,+ 56487535, 56487539, 56487542, 56487546, 56487548, 56487549, 56487562, 56487631, 56487633,+ 56487637, 56500577, 56500620, 56513324, 56513399, 56513580, 56513832, 56513931, 56514220,+ 56514264, 56514321, 56514334, 56514335, 56514341, 56514345, 56514345, 56514348, 56514348,+ 56514348, 56517007, 56517040, 56517219, 56517394, 56517398, 56517406, 56517431, 56517438,+ 56517440, 56517451, 56517454, 56517456, 56517540, 56517540, 56517547, 56517550, 56518684,+ 56518692, 56518872, 56519062, 56519134, 56519138, 56519143, 56519146, 56519231, 56519233,+ 56519235, 56519239, 56519239, 56519240, 56519241, 56519243, 56529141, 56529235, 56529464,+ 56529643, 56529647, 56529739, 56529944, 56530021, 56530033, 56530039, 56530041, 56530046,+ 56530120, 56530121, 56530130, 56530131, 60503686, 60507124, 60507167, 60507644, 60507645,+ 60507861, 60507862, 60507867, 60507919, 60507934, 60507935, 60507948, 60507950, 60507951,+ 60507991, 60513617, 60514810, 60514944, 60514974, 60515342, 60515346, 60515418, 60515423,+ 60515425, 60515431, 60515434, 60515434, 60515437, 60515440, 60515442, 60515444, 60516483,+ 60517366, 60517401, 60517553, 60517633, 60517635, 60517639, 60517643, 60517720, 60517735,+ 60517739, 60517739, 60517740, 60517741, 60517742, 60527758, 60527965, 60528118, 60528159,+ 60528394, 60528426, 60529858, 60533363, 60543166, 60543187, 60550767, 60551124, 60551379,+ 60551507, 60554040, 60554046, 60566285, 60604219, 60604236, 60611044, 60611055, 60997202,+ 60997546, 61148893, 61148964, 61149150, 61149348, 61149350, 61149547, 61149569, 61149621,+ 61149700, 61149739, 61149837, 61149838, 61149846, 61149936, 61149942, 61149950, 61150023,+ 61150039, 74804559, 74813322, 81525220, 81527024, 81529443, 81541659, 81541742, 81551606,+ 81551649, 81563812, 81563834, 81566324, 81571381, 81606295, 81616390, 81698194, 81698197,+ 81699419, 81830256, 82411751, 82481563, 82971880, 82973542, 82973586, 82973709, 82973878,+ 82973879, 82973901, 82974037, 82974038, 82974116, 82974125, 82974136, 82974150, 82974157,+ 82974166, 82974221, 82978322, 82978329, 82978503, 82978673, 82978842, 82978844, 82979072,+ 82979077, 82979090, 82979101, 82980428, 82980431, 82980604, 82980782, 82980785, 82980787,+ 82980789, 82980811, 82980813, 82980938, 82980941, 82980947, 82980950, 82981016, 82983256,+ 82983258, 83004435, 83004454, 83004628, 83004796, 83004959, 83004961, 83004969, 83005105,+ 83005109, 83005127, 83007671, 83007672, 83013567, 83013609, 83013778, 83013946, 83014106,+ 83014109, 83014269, 83014271, 83014349, 83014350, 83015915, 83015923, 83016090, 83016268,+ 83016269, 83016302, 83016307, 83016312, 83016331, 83016432, 83016436, 83016438, 83016451,+ 83016520, 83017289, 83017290, 83018610, 83018611, 83020557, 83020566, 83071804, 83608529,+ 83637588, 83639108, 83644094, 83644130, 83648784, 83648818, 83651953, 83651965, 83686748,+ 83701427, 83701450, 83703328, 83703343, 83816580, 83823972, 83825581, 83992973, 84003229,+ 84023427, 84023429, 84049930, 84052160, 84103813+ ]+}diff --git a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.tsindex e9ead490a..fc7fef64d 100644--- a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts+++ b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts@@ -1,23 +1,37 @@-import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'import { randomBytes, randomUUID } from 'node:crypto'+import { readFileSync } from 'node:fs'+import http from 'node:http'+import type { AddressInfo } from 'node:net'import { Pool } from 'pg'-import request from 'supertest'// Concurrent users at the external GFS rate limits, through the production// Express app against real PostgreSQL. Config, the core and limiter pools, both limiters,// the session lifecycle query, authority resolution, the prom registry and the-// logger are real; the ONLY mock is the session-token verifier (identity),-// which decodes a JSON test token so each scenario controls users and sessions.+// logger are real; the ONLY mocks are the session-token verifier (identity),+// which decodes a JSON test token so each scenario controls users and sessions,+// and, in E3 only, the fetch to gfsc (the service boundary).//-// Denial source (plan Addendum A3), for status 429 only:+// The pools run at their production bounds: every CORE_POOL_* and+// RATE_LIMIT_POOL_* variable is deleted before db.js is imported, so a 503+// from either pool fails the exact tallies below. Each scenario serves its app+// from one listening server and sends through one keep-alive agent.+//+// Denial source, for status 429 only, and only on the+// resource read and resource mutation routes. Those routes have no+// rateLimitMiddleware, so their headers identify the limiter:// - route backstop: RateLimit-Policy === "<route L>;w=60";// - Postgres bucket: RateLimit-Policy === "<ingress L>;w=60" AND// X-RateLimit-Remaining === "0" AND X-RateLimit-Limit === "<bucket L>";// - anything else fails the test.+// GET /grants and GET /shares also run rateLimitMiddleware, whose 429 carries+// the same headers as a Postgres bucket's, so the classifier refuses those+// routes and S2b attributes their denials through the ledger and the metric.// Headers cannot say WHICH Postgres bucket denied, so every scenario also reads// the rate_limit_buckets ledger: each hit increments its row once and a bucket// only sees requests that passed the buckets before it, so per (key, window)-// denials = max(0, count - limit). The key prefix names the phase.+// denials = max(0, count - limit). The key prefix names the phase. Nothing+// prunes the ledger here: cleanupExpiredBuckets runs only from main.ts.//// Both limiters read Date.now(), so faking only Date pins their windows. Time// only moves forward, each scenario deletes the ledger, uses fresh identities,@@ -42,6 +56,33 @@ vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({const INGRESS_L = 1_800const IP_L_DEFAULT = 1_200const MINUTE_MS = 60_000+const DAY_MS = 24 * 60 * MINUTE_MS++/** The shipped defaults (config.ts); beforeAll asserts config still has them. */+const PRODUCTION_BUDGETS: Budgets = {+ resource: 480,+ proxy: 480,+ grants: 120,+ shares: 120,+ operation: 90,+ ip: IP_L_DEFAULT,+}++const RESOURCE_READ_ROUTE = /^\/api\/v1\/external\/gfs\/resources\/[0-9a-f-]{36}\/affordances\?/+const RESOURCE_MUTATION_ROUTE = /^\/api\/v1\/external\/gfs\/resources\/[0-9a-f-]{36}\/children\?/++/**+ * E4 fixture: every Desktop resource-class read on cluster clerum on+ * 2026-09-18 (UTC), the day of the Desktop 429 incident, as ms since 00:00 UTC.+ */+const DESKTOP_READS_2026_09_18 = (+ JSON.parse(+ readFileSync(+ new URL('./fixtures/external-gfs-desktop-reads-2026-09-18.json', import.meta.url),+ 'utf-8'+ )+ ) as { offsetsMs: number[] }+).offsetsMs/** Every read class at `read`: for scenarios that exercise one read class. */function uniformBudgets(read: number, operation: number, ip: number): Budgets {@@ -82,7 +123,15 @@ const describeRealPostgres = adminUrl ? describe : describe.skipdescribeRealPostgres('external GFS rate limits under concurrent load (real PostgreSQL)', () => {const database = `control_api_gfs_rl_stress_${randomBytes(6).toString('hex')}`- const envKeys = ['CONTROL_API_PG_CONNECTION_STRING', 'CORE_POOL_CONNECTION_TIMEOUT_MS'] as const+ const envKeys = [+ 'CONTROL_API_PG_CONNECTION_STRING',+ 'CORE_POOL_MAX',+ 'CORE_POOL_CONNECTION_TIMEOUT_MS',+ 'CORE_POOL_STATEMENT_TIMEOUT_MS',+ 'RATE_LIMIT_POOL_MAX',+ 'RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS',+ 'RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS',+ ] as constconst previousEnv = new Map<string, string | undefined>()let adminPool: Pool@@ -105,19 +154,26 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgthrow new Error('CONTROL_API_REAL_PG_ADMIN_URL is required by this suite')}const connectionString = databaseUrl(adminUrl, database)- for (const key of envKeys) previousEnv.set(key, process.env[key])+ for (const key of envKeys) {+ previousEnv.set(key, process.env[key])+ delete process.env[key]+ }adminPool = new Pool({ connectionString: adminUrl })await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString- // A 200-request burst queues its session and authority queries on the 10- // core connections; the production 2 s acquire timeout would turn some of- // them into 500s by accident (see- // services.rateLimiterPoolSaturation.realPostgres.integration.test.ts).- // The limiter's own pool runs at its production bounds.- process.env.CORE_POOL_CONNECTION_TIMEOUT_MS = '10000'const dbMod = await import('../src/db.js')corePool = dbMod.pool as unknown as Pool+ // Witness that nothing raised the bounds: core 10 / 2000 ms, limiter 6 / 5000 ms.+ type PoolOptions = { options: { max: number; connectionTimeoutMillis: number } }+ expect((corePool as unknown as PoolOptions).options).toMatchObject({+ max: 10,+ connectionTimeoutMillis: 2_000,+ })+ expect((dbMod.rateLimitPool as unknown as PoolOptions).options).toMatchObject({+ max: 6,+ connectionTimeoutMillis: 5_000,+ })corePool.on('error', () => {})limiterPool = dbMod.rateLimitPool as unknown as PoollimiterPool.on('error', () => {})@@ -140,11 +196,27 @@ describeRealPostgres('external GFS rate limits under concurrent load (real PostgMockGateway,}expect(mod.config.externalGfsIngressRlPerMin).toBe(INGRESS_L)- expect(mod.config.externalGfsIpRlPerMin).toBe(IP_L_DEFAULT)+ // E1-E4 run at PRODUCTION_BUDGETS; they must be the shipped defaults.+ expect({+ resource: mod.config.externalGfsResourceReadRlPerMin,+ proxy: mod.config.externalGfsProxyReadRlPerMin,+ grants: mod.config.externalGfsGrantsReadRlPerMin,+ shares: mod.config.externalGfsSharesReadRlPerMin,+ operation: mod.config.externalGfsOperationRlPerMin,+ ip: mod.config.externalGfsIpRlPerMin,+ }).toEqual(PRODUCTION_BUDGETS)vi.useFakeTimers({ toFake: ['Date'] })nextScenarioMinuteMs = Math.floor(Date.now() / MINUTE_MS) * MINUTE_MS + 10 * MINUTE_MS}, 60_000)+ // The server of the running scenario; closed after every test.+ let openServer: { close: () => Promise<void> } | null = null+ afterEach(async () => {+ const server = openServer+ openServer = null+ await server?.close()+ })+afterAll(async () => {vi.useRealTimers()for (const key of envKeys) {@@ -164,8 +236,13 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgawait adminPool.end()})- /** Sets a boot-valid budget, clears the ledger, returns a fresh app and the scenario minute. */- async function startScenario(budgets: Budgets) {+ /**+ * Sets a boot-valid budget, clears the ledger, serves a fresh app and+ * returns its `send` and the scenario start. The start is a multiple of+ * `alignMs` (a minute, or a UTC day for the E4 replay), and the next+ * scenario starts at least `spanMs` later.+ */+ async function startScenario(budgets: Budgets, spanMs = 5 * MINUTE_MS, alignMs = MINUTE_MS) {mod.assertExternalGfsBudgetInvariants({resourceReadPerMin: budgets.resource,proxyReadPerMin: budgets.proxy,@@ -181,10 +258,87 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgmod.config.externalGfsOperationRlPerMin = budgets.operationmod.config.externalGfsIpRlPerMin = budgets.ipawait mod.pool.query('DELETE FROM rate_limit_buckets')- const minuteMs = nextScenarioMinuteMs- nextScenarioMinuteMs += 5 * MINUTE_MS+ const minuteMs = Math.ceil(nextScenarioMinuteMs / alignMs) * alignMs+ nextScenarioMinuteMs = minuteMs + spanMsvi.setSystemTime(minuteMs + 10_000)- return { app: mod.createApp(new mod.MockGateway()), minuteMs }+ const send = await serve(mod.createApp(new mod.MockGateway()))+ return { send, minuteMs }+ }++ /** Serves `app` on one listening server; `send` reuses one keep-alive agent. */+ async function serve(app: import('express').Express) {+ const internalToken = mod.config.internalServiceTokens['external-rest-api']+ if (!internalToken) throw new Error('config has no external-rest-api internal service token')+ const server = http.createServer(app)+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve))+ const { port } = server.address() as AddressInfo+ const agent = new http.Agent({ keepAlive: true })+ if (openServer) throw new Error('a scenario server is already open')+ openServer = {+ close: () => {+ agent.destroy()+ server.closeAllConnections()+ return new Promise<void>((resolve, reject) =>+ server.close(error => (error ? reject(error) : resolve()))+ )+ },+ }++ /** GET, or POST with `post.body` as JSON. */+ return function send(+ path: string,+ token: string,+ ip: string,+ post?: { body: unknown }+ ): Promise<Sent> {+ const method = post === undefined ? 'GET' : 'POST'+ const payload = post === undefined ? undefined : JSON.stringify(post.body)+ const headers: Record<string, string> = {+ authorization: `Bearer ${internalToken}`,+ 'x-service-token': 'external-rest-api',+ 'x-user-session-token': token,+ 'x-forwarded-for': ip,+ }+ if (payload !== undefined) {+ headers['content-type'] = 'application/json'+ headers['content-length'] = String(Buffer.byteLength(payload))+ }+ return new Promise<Sent>((resolve, reject) => {+ const req = http.request({ host: '127.0.0.1', port, method, path, agent, headers }, res => {+ const chunks: Buffer[] = []+ res.on('data', (chunk: Buffer) => chunks.push(chunk))+ res.on('error', reject)+ res.on('end', () => {+ const text = Buffer.concat(chunks).toString('utf-8')+ try {+ resolve({+ path,+ status: res.statusCode ?? 0,+ body: (text === '' ? {} : JSON.parse(text)) as { error?: unknown },+ policy: singleHeader(res, 'ratelimit-policy'),+ xLimit: singleHeader(res, 'x-ratelimit-limit'),+ xRemaining: singleHeader(res, 'x-ratelimit-remaining'),+ })+ } catch (error) {+ reject(+ new Error(+ `${method} ${path} answered ${res.statusCode} with a non-JSON body ` ++ `(${String(error)}): ${text}`+ )+ )+ }+ })+ })+ req.on('error', reject)+ req.end(payload)+ })+ }+ }++ function singleHeader(res: http.IncomingMessage, name: string): string | undefined {+ const value = res.headers[name]+ if (Array.isArray(value)) throw new Error(`header ${name} repeated: ${value.join(', ')}`)+ return value}async function seedUser(): Promise<string> {@@ -203,32 +357,26 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgconst grantsPath = (resourceId: string) =>`/api/v1/external/gfs/grants?drive=main&resourceId=${resourceId}`- async function send(- app: import('express').Express,- path: string,- token: string,- ip: string- ): Promise<Sent> {- const internalToken = mod.config.internalServiceTokens['external-rest-api']- if (!internalToken) throw new Error('config has no external-rest-api internal service token')- const res = await request(app)- .get(path)- .set('Authorization', `Bearer ${internalToken}`)- .set('x-service-token', 'external-rest-api')- .set('x-user-session-token', token)- .set('x-forwarded-for', ip)- return {- path,- status: res.status,- body: res.body as { error?: unknown },- policy: res.headers['ratelimit-policy'],- xLimit: res.headers['x-ratelimit-limit'],- xRemaining: res.headers['x-ratelimit-remaining'],+ const childrenPath = (resourceId: string) =>+ `/api/v1/external/gfs/resources/${resourceId}/children?drive=main`++ /**+ * Denial-source classifier, for the two routes whose 429 headers identify the+ * limiter (see the header comment). Any other route throws. Allowed+ * responses keep their status.+ */+ function classify(+ response: Sent,+ route: 'resource-read' | 'resource-mutation',+ budgets: Budgets+ ): string {+ const [pattern, routeL] =+ route === 'resource-read'+ ? [RESOURCE_READ_ROUTE, budgets.resource]+ : [RESOURCE_MUTATION_ROUTE, budgets.operation]+ if (!pattern.test(response.path)) {+ throw new Error(`classify(${route}) received ${response.path}`)}- }-- /** Addendum A3 classifier. Allowed responses keep their status. */- function classify(response: Sent, routeL: number): string {if (response.status !== 429) return `allowed:${response.status}`if (response.policy === `${routeL};w=60`) return 'route-backstop'if (response.policy === `${INGRESS_L};w=60` && response.xRemaining === '0')@@ -274,6 +422,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgfunction limitOf(key: string, budgets: Budgets): number {if (key.startsWith('gfs-ext:pre:ip:')) return budgets.ipif (/^gfs-ext:(pre|resolved):resource:/.test(key)) return budgets.resource+ if (/^gfs-ext:(pre|resolved):resource-mutation:/.test(key)) return budgets.operationif (/^gfs-ext:(pre|resolved):grants-read:/.test(key)) return budgets.grants// GET /grants and GET /shares share this bucket; its limit is the sum.if (key.startsWith('gfsgrants-ext-read:user:')) return budgets.grants + budgets.shares@@ -332,7 +481,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgit('S1: 20 users with distinct sessions and IPs each get exactly the read budget', async () => {const budgets = uniformBudgets(6, 6, IP_L_DEFAULT)- const { app } = await startScenario(budgets)+ const { send } = await startScenario(budgets)const users = await Promise.all(Array.from({ length: 20 }, () => seedUser()))const resourceId = randomUUID()@@ -340,12 +489,12 @@ describeRealPostgres('external GFS rate limits under concurrent load (real PostgPromise.all(users.flatMap((userId, index) =>Array.from({ length: 10 }, () =>- send(app, affordancesPath(resourceId), sessionToken(userId), `198.51.100.${index + 1}`)+ send(affordancesPath(resourceId), sessionToken(userId), `198.51.100.${index + 1}`)))))- const labels = run.result.map(response => classify(response, budgets.resource))+ const labels = run.result.map(response => classify(response, 'resource-read', budgets))expect(run.result).toHaveLength(200)for (let user = 0; user < 20; user += 1) {@@ -375,7 +524,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgit('S2a: 20 users behind one IP share the (class, IP) read bucket', async () => {const budgets = uniformBudgets(30, 30, IP_L_DEFAULT)- const { app } = await startScenario(budgets)+ const { send } = await startScenario(budgets)const users = await Promise.all(Array.from({ length: 20 }, () => seedUser()))const resourceId = randomUUID()@@ -383,16 +532,18 @@ describeRealPostgres('external GFS rate limits under concurrent load (real PostgPromise.all(users.flatMap(userId =>Array.from({ length: 5 }, () =>- send(app, affordancesPath(resourceId), sessionToken(userId), '198.51.100.30')+ send(affordancesPath(resourceId), sessionToken(userId), '198.51.100.30')))))- expect(tally(run.result.map(response => classify(response, budgets.resource)))).toEqual({- 'allowed:200': 30,- 'postgres:30': 70,- })+ expect(tally(run.result.map(response => classify(response, 'resource-read', budgets)))).toEqual(+ {+ 'allowed:200': 30,+ 'postgres:30': 70,+ }+ )const ledger = await readLedger()expect(counts(ledger, 'gfs-ext:pre:resource:session:')).toEqual(Array(20).fill(5))expect(counts(ledger, 'gfs-ext:pre:resource:ip:')).toEqual([100])@@ -414,7 +565,7 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg// the IP, because each class alone is capped at its own L first. The two// classes carry different budgets, so each bucket must use its own class's.const budgets = { resource: 30, proxy: 30, grants: 20, shares: 20, operation: 20, ip: 40 }- const { app } = await startScenario(budgets)+ const { send } = await startScenario(budgets)const users = await Promise.all(Array.from({ length: 10 }, () => seedUser()))const resourceId = randomUUID()@@ -423,7 +574,6 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgusers.flatMap((userId, index) =>Array.from({ length: 5 }, () =>send(- app,index < 5 ? affordancesPath(resourceId) : grantsPath(resourceId),sessionToken(userId),'198.51.100.40'@@ -432,31 +582,33 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg)))- const labels = run.result.map(response => classify(response, budgets.resource))- const tallied = tally(labels)+ const affordances = run.result.filter(response => response.path.includes('/affordances'))+ const grants = run.result.filter(response => response.path.includes('/grants'))+ expect(affordances).toHaveLength(25)+ expect(grants).toHaveLength(25)+ const affordanceLabels = tally(+ affordances.map(response => classify(response, 'resource-read', budgets))+ )+ const grantStatuses = tally(grants.map(response => String(response.status)))// The (grants-read, IP) bucket sees 25 against 20 and denies 5; pre:ip- // then sees 25 resource + 20 grants = 45 against 40 and denies 5.- expect(tallied['postgres:20']).toBe(5)- expect(tallied['postgres:40']).toBe(5)- expect((tallied['allowed:200'] ?? 0) + (tallied['allowed:403'] ?? 0)).toBe(40)- expect(Object.keys(tallied).sort()).toEqual(- ['allowed:200', 'allowed:403', 'postgres:20', 'postgres:40'].filter(- label => tallied[label] !== undefined- )+ // then sees 25 resource + 20 grants = 45 against 40 and denies 5, on+ // either route. On the affordances route only pre:ip can deny.+ const affordances429 = affordanceLabels['postgres:40'] ?? 0+ const grants429 = grantStatuses['429'] ?? 0+ expect(Object.keys(affordanceLabels).sort()).toEqual(+ ['allowed:200', 'postgres:40'].filter(label => affordanceLabels[label] !== undefined))- // Only the grants class is capped at 20.- run.result.forEach((response, index) => {- if (labels[index] === 'postgres:20') expect(response.path).toContain('/grants')- })- // "Allowed" is defined per route: affordances answer 200, the grants list- // answers 403 manage_acl_required for a caller without manage_acl.- for (const response of run.result) {- if (response.status === 200) expect(response.path).toContain('/affordances')- if (response.status === 403) {- expect(response.path).toContain('/grants')- expect(response.body.error).toBe('manage_acl_required')- }+ expect(Object.keys(grantStatuses).sort()).toEqual(+ ['403', '429'].filter(status => grantStatuses[status] !== undefined)+ )+ expect(affordances429 + grants429).toBe(10)+ expect(grants429).toBeGreaterThanOrEqual(5)+ expect((affordanceLabels['allowed:200'] ?? 0) + (grantStatuses['403'] ?? 0)).toBe(40)+ // "Allowed" is defined per route: the grants list answers 403+ // manage_acl_required for a caller without manage_acl.+ for (const response of grants) {+ if (response.status === 403) expect(response.body.error).toBe('manage_acl_required')}const ledger = await readLedger()expect(counts(ledger, 'gfs-ext:pre:resource:session:')).toEqual(Array(5).fill(5))@@ -465,7 +617,11 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgexpect(counts(ledger, 'gfs-ext:pre:grants-read:ip:')).toEqual([25])expect(counts(ledger, 'gfs-ext:pre:ip:')).toEqual([45])expect(sum(counts(ledger, 'gfs-ext:resolved:'))).toBe(40)- expect(sum(counts(ledger, 'gfsgrants-ext-read:user:'))).toBe(tallied['allowed:403'] ?? 0)+ // Every grants request that reached the rateLimitMiddleware bucket got a+ // 403: that limiter denied none, and its row is under its limit.+ expect(sum(counts(ledger, 'gfsgrants-ext-read:user:'))).toBe(grantStatuses['403'] ?? 0)+ // The 10 denials are exactly the two Postgres buckets' overflow, and the+ // metric below shows no backstop denial: this is the grants attribution.expect(ledgerDenials(ledger, budgets)).toBe(10)expect(run.dbErrors).toBe(0)expect(run.metricDelta).toEqual({@@ -478,25 +634,27 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgit('S3: two sessions of one user share the resolved actor bucket', async () => {const budgets = uniformBudgets(10, 10, IP_L_DEFAULT)- const { app } = await startScenario(budgets)+ const { send } = await startScenario(budgets)const userId = await seedUser()const resourceId = randomUUID()const run = await observed(() =>Promise.all([...Array.from({ length: 8 }, () =>- send(app, affordancesPath(resourceId), sessionToken(userId, 1), '198.51.100.51')+ send(affordancesPath(resourceId), sessionToken(userId, 1), '198.51.100.51')),...Array.from({ length: 8 }, () =>- send(app, affordancesPath(resourceId), sessionToken(userId, 2), '198.51.100.52')+ send(affordancesPath(resourceId), sessionToken(userId, 2), '198.51.100.52')),]))- expect(tally(run.result.map(response => classify(response, budgets.resource)))).toEqual({- 'allowed:200': 10,- 'postgres:10': 6,- })+ expect(tally(run.result.map(response => classify(response, 'resource-read', budgets)))).toEqual(+ {+ 'allowed:200': 10,+ 'postgres:10': 6,+ }+ )const ledger = await readLedger()// Each pre-resolution bucket saw 8 <= 10 and denied nothing; the resolved// actor bucket saw all 16: the phase witness, independent of telemetry.@@ -516,19 +674,19 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postgit('S4: a burst straddling a minute boundary is denied only by the route backstop', async () => {const budgets = uniformBudgets(8, 8, IP_L_DEFAULT)- const { app, minuteMs } = await startScenario(budgets)+ const { send, minuteMs } = await startScenario(budgets)const userId = await seedUser()const resourceId = randomUUID()const burst = () =>observed(() =>Promise.all(Array.from({ length: 8 }, () =>- send(app, affordancesPath(resourceId), sessionToken(userId), '198.51.100.60')+ send(affordancesPath(resourceId), sessionToken(userId), '198.51.100.60'))))const labelsOf = (responses: Sent[]) =>- tally(responses.map(response => classify(response, budgets.resource)))+ tally(responses.map(response => classify(response, 'resource-read', budgets)))// A: 50 s into window 1. Opens the backstop window, fills the Postgres buckets.vi.setSystemTime(minuteMs + 50_000)@@ -585,4 +743,253 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg'edge-backstop': 1,})}, 120_000)++ it('E1: 481 concurrent reads by one actor at the production budget: 480 allowed, one Postgres 429, no 503', async () => {+ const budgets = PRODUCTION_BUDGETS+ const { send } = await startScenario(budgets)+ const userId = await seedUser()+ const resourceId = randomUUID()++ const run = await observed(() =>+ Promise.all(+ Array.from({ length: 481 }, () =>+ send(affordancesPath(resourceId), sessionToken(userId), '198.51.100.70')+ )+ )+ )++ // Exact: a 503 from either pool or any other status fails here.+ expect(tally(run.result.map(response => classify(response, 'resource-read', budgets)))).toEqual(+ { 'allowed:200': 480, 'postgres:480': 1 }+ )+ const ledger = await readLedger()+ expect(counts(ledger, 'gfs-ext:pre:resource:session:')).toEqual([481])+ expect(counts(ledger, 'gfs-ext:pre:resource:ip:')).toEqual([480])+ expect(counts(ledger, 'gfs-ext:pre:ip:')).toEqual([480])+ expect(counts(ledger, `gfs-ext:resolved:resource:actor:user-session:${userId}`)).toEqual([480])+ expect(ledgerDenials(ledger, budgets)).toBe(1)+ expect(run.dbErrors).toBe(0)+ expect(run.metricDelta).toEqual({+ 'pre-resolution': 1,+ 'resolved-operation': 0,+ 'edge-backstop': 0,+ })+ expect(run.logged).toEqual(run.metricDelta)+ }, 120_000)++ it('E2: a burst straddling a minute boundary passes 2L-1 = 959 reads in 60 s; the next is a Postgres 429', async () => {+ const budgets = PRODUCTION_BUDGETS+ const { send, minuteMs } = await startScenario(budgets)+ const userId = await seedUser()+ const resourceId = randomUUID()+ const boundary = minuteMs + 2 * MINUTE_MS+ const burstAt = async (atMs: number, size: number) => {+ vi.setSystemTime(atMs)+ const responses = await Promise.all(+ Array.from({ length: size }, () =>+ send(affordancesPath(resourceId), sessionToken(userId), '198.51.100.71')+ )+ )+ return responses.map(response => ({+ atMs,+ label: classify(response, 'resource-read', budgets),+ }))+ }++ const run = await observed(async () => [+ // Opens the backstop window at M:00.0, so it resets at M+1:00.0.+ ...(await burstAt(boundary - MINUTE_MS, 1)),+ ...(await burstAt(boundary - 100, 479)),+ ...(await burstAt(boundary, 480)),+ ...(await burstAt(boundary, 1)),+ ])++ expect(tally(run.result.map(sent => sent.label))).toEqual({+ 'allowed:200': 960,+ 'postgres:480': 1,+ })+ // The sliding bound: in the 60 s ending at M+1:00.0, 959 reads passed.+ expect(+ run.result.filter(+ sent =>+ sent.label === 'allowed:200' && sent.atMs > boundary - MINUTE_MS && sent.atMs <= boundary+ )+ ).toHaveLength(959)+ expect(run.result[run.result.length - 1]?.label).toBe('postgres:480')+ const ledger = await readLedger()+ const before = boundary - MINUTE_MS+ expect(counts(ledger, 'gfs-ext:pre:resource:session:', before)).toEqual([480])+ expect(counts(ledger, 'gfs-ext:pre:resource:session:', boundary)).toEqual([481])+ expect(counts(ledger, 'gfs-ext:resolved:resource:actor:', before)).toEqual([480])+ expect(counts(ledger, 'gfs-ext:resolved:resource:actor:', boundary)).toEqual([480])+ expect(ledgerDenials(ledger, budgets)).toBe(1)+ expect(run.dbErrors).toBe(0)+ expect(run.metricDelta).toEqual({+ 'pre-resolution': 1,+ 'resolved-operation': 0,+ 'edge-backstop': 0,+ })+ expect(run.logged).toEqual(run.metricDelta)+ }, 120_000)++ it('E3: 91 mutations by one actor at the production operation budget: 90 reach gfsc, the 91st is a Postgres 429', async () => {+ const budgets = PRODUCTION_BUDGETS+ const { send } = await startScenario(budgets)+ const userId = await seedUser()+ const resourceId = randomUUID()+ // The only boundary mock: the fetch from control-api to gfsc.+ const gfscFetch = vi.fn(+ async (_url: string | URL | Request, _init?: RequestInit) =>+ new Response(JSON.stringify({ id: randomUUID() }), {+ status: 201,+ headers: { 'content-type': 'application/json' },+ })+ )+ vi.stubGlobal('fetch', gfscFetch)+ try {+ const run = await observed(() =>+ Promise.all(+ Array.from({ length: 91 }, () =>+ send(childrenPath(resourceId), sessionToken(userId), '198.51.100.72', {+ body: { name: 'stress', kind: 'folder' },+ })+ )+ )+ )++ expect(+ tally(run.result.map(response => classify(response, 'resource-mutation', budgets)))+ ).toEqual({ 'allowed:201': 90, 'postgres:90': 1 })+ // Witness: exactly the 90 admitted mutations reached gfsc, each as the+ // POST the route proxies.+ expect(gfscFetch).toHaveBeenCalledTimes(90)+ const target =+ `${mod.config.gfscWriteBaseUrl.replace(/\/+$/, '')}` ++ `/v1/resources/${resourceId.replace(/-/g, '')}/children`+ for (const [url, init] of gfscFetch.mock.calls) {+ expect(String(url)).toBe(target)+ expect(init?.method).toBe('POST')+ }+ const ledger = await readLedger()+ expect(counts(ledger, 'gfs-ext:pre:resource-mutation:session:')).toEqual([91])+ expect(counts(ledger, 'gfs-ext:pre:resource-mutation:ip:')).toEqual([90])+ expect(counts(ledger, 'gfs-ext:pre:ip:')).toEqual([90])+ expect(+ counts(ledger, `gfs-ext:resolved:resource-mutation:actor:user-session:${userId}`)+ ).toEqual([90])+ expect(ledgerDenials(ledger, budgets)).toBe(1)+ expect(run.dbErrors).toBe(0)+ expect(run.metricDelta).toEqual({+ 'pre-resolution': 1,+ 'resolved-operation': 0,+ 'edge-backstop': 0,+ })+ expect(run.logged).toEqual(run.metricDelta)+ } finally {+ vi.unstubAllGlobals()+ }+ }, 120_000)++ /**+ * Replays DESKTOP_READS_2026_09_18 serially, one actor, one session, one IP+ * (the upper bound on any one actor's count), each read at its offset on a+ * fresh fake UTC day. Denials are labelled with their UTC second.+ */+ async function replayDesktopDay(budgets: Budgets) {+ const { send, minuteMs: dayStartMs } = await startScenario(+ budgets,+ DAY_MS + 5 * MINUTE_MS,+ DAY_MS+ )+ const userId = await seedUser()+ const resourceId = randomUUID()+ const denials: string[] = []+ const run = await observed(async () => {+ const labels: string[] = []+ for (const offsetMs of DESKTOP_READS_2026_09_18) {+ vi.setSystemTime(dayStartMs + offsetMs)+ const response = await send(+ affordancesPath(resourceId),+ sessionToken(userId),+ '198.51.100.80'+ )+ const label = classify(response, 'resource-read', budgets)+ labels.push(label)+ if (!label.startsWith('allowed:')) {+ denials.push(`${label}@${new Date(offsetMs).toISOString().slice(11, 19)}`)+ }+ }+ return labels+ })+ return { ...run, denials, userId }+ }++ it('E4: the 1965 Desktop reads of 2026-09-18 replayed at the production budget get no 429 and no 503', async () => {+ expect(DESKTOP_READS_2026_09_18).toHaveLength(1965)+ expect(+ DESKTOP_READS_2026_09_18.every((offset, i, all) => i === 0 || all[i - 1]! <= offset)+ ).toBe(true)+ const budgets = PRODUCTION_BUDGETS++ const run = await replayDesktopDay(budgets)++ expect(tally(run.result)).toEqual({ 'allowed:200': 1965 })+ expect(run.denials).toEqual([])+ // Witness that the limiter counted the whole day, including its busiest+ // calendar minute (125 reads).+ const ledger = await readLedger()+ expect(sum(counts(ledger, 'gfs-ext:pre:resource:session:'))).toBe(1965)+ expect(Math.max(...counts(ledger, 'gfs-ext:pre:resource:session:'))).toBe(125)+ expect(sum(counts(ledger, `gfs-ext:resolved:resource:actor:user-session:${run.userId}`))).toBe(+ 1965+ )+ expect(ledgerDenials(ledger, budgets)).toBe(0)+ expect(run.dbErrors).toBe(0)+ expect(run.metricDelta).toEqual({+ 'pre-resolution': 0,+ 'resolved-operation': 0,+ 'edge-backstop': 0,+ })+ expect(run.logged).toEqual(run.metricDelta)+ }, 300_000)++ it('E4 control: the same replay at the old 120 budget reproduces the 28 production denials at their seconds', async () => {+ // The 28 × 429 served on cluster clerum on 2026-09-18, by UTC second and+ // limiter: nginx statuses give the seconds, the external_gfs_rate_limit+ // events give the Postgres phase of the five at 13:06:59.+ const observedDenials = [+ ...Array<string>(10).fill('route-backstop@04:46:20'),+ ...Array<string>(5).fill('postgres:120@13:06:59'),+ ...Array<string>(7).fill('route-backstop@14:18:15'),+ 'route-backstop@14:18:18',+ 'route-backstop@14:18:22',+ ...Array<string>(3).fill('route-backstop@14:18:36'),+ 'route-backstop@14:18:47',+ ]+ const budgets = { ...PRODUCTION_BUDGETS, resource: 120 }++ const run = await replayDesktopDay(budgets)++ expect(run.denials).toEqual(observedDenials)+ expect(tally(run.result)).toEqual({+ 'allowed:200': 1937,+ 'postgres:120': 5,+ 'route-backstop': 23,+ })+ const ledger = await readLedger()+ expect(ledgerDenials(ledger, budgets)).toBe(5)+ expect(run.dbErrors).toBe(0)+ expect(run.metricDelta).toEqual({+ 'pre-resolution': 5,+ 'resolved-operation': 0,+ 'edge-backstop': 23,+ })+ // The backstop logs once per key per window: its 23 denials fall in two+ // windows (04:46 and 14:18).+ expect(run.logged).toEqual({+ 'pre-resolution': 5,+ 'resolved-operation': 0,+ 'edge-backstop': 2,+ })+ }, 300_000)})diff --git a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.tsindex d1086edb3..518903300 100644--- a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts+++ b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts@@ -184,12 +184,15 @@ describeRealPostgres('rate limiter under pool saturation', () => {let released = falsetry {const startedAt = performance.now()+ const settledAfterMs: number[] = []const results = await Promise.all(Array.from({ length: CONCURRENT_CHECKS }, () =>- mod.checkAndIncrement(bucketKey, LIMIT, nowMs)+ mod.checkAndIncrement(bucketKey, LIMIT, nowMs).then(result => {+ settledAfterMs.push(performance.now() - startedAt)+ return result+ })))- const elapsedMs = performance.now() - startedAtexpect(results).toHaveLength(CONCURRENT_CHECKS)// The service reports allowed:true; the admission decision belongs to@@ -209,9 +212,13 @@ describeRealPostgres('rate limiter under pool saturation', () => {expect(payload.err).toMatch(/timeout exceeded when trying to connect/)}- // Every call waited for the acquire timeout, in parallel, not in series.- expect(elapsedMs).toBeGreaterThanOrEqual(LIMITER_ACQUIRE_TIMEOUT_MS - 100)- expect(elapsedMs).toBeLessThan(LIMITER_ACQUIRE_TIMEOUT_MS + 3_000)+ // Every call waited for the acquire timeout: none settled before it,+ // which is a property of the pool, not of machine speed. They waited in+ // parallel: in series the second call alone would settle after two+ // timeouts, so the bound is structural rather than a latency budget.+ expect(settledAfterMs).toHaveLength(CONCURRENT_CHECKS)+ expect(Math.min(...settledAfterMs)).toBeGreaterThanOrEqual(LIMITER_ACQUIRE_TIMEOUT_MS - 100)+ expect(Math.max(...settledAfterMs)).toBeLessThan(2 * LIMITER_ACQUIRE_TIMEOUT_MS)// Read through a held client: this also proves the database and the// table are reachable, so "no row" is not an absent table.diff --git a/control-api/vitest.config.ts b/control-api/vitest.config.tsindex eb9bc2a77..995a382d7 100644--- a/control-api/vitest.config.ts+++ b/control-api/vitest.config.ts@@ -28,9 +28,10 @@ export default defineConfig({fileParallelism: false,maxWorkers: 1,pool: 'threads',- // Real-Postgres tests mutate shared database state and cannot be retried safely- // after a timeout because the abandoned async work may still commit.- retry: process.env.CONTROL_API_REAL_PG_ADMIN_URL ? 0 : 2,+ // No retries: a retry turns a flaky test green and hides the flake. A+ // failing test is fixed in the test. Real-Postgres tests could not be+ // retried safely anyway, because abandoned async work may still commit.+ retry: 0,testTimeout: 10_000,// Real-Postgres beforeAll runs initDb across the full migration list.// Default hookTimeout follows testTimeout (10s) and turns a slow CREATE