Skip to content

Agent Mesh

scale: a thousand agents on one host

Antonio Ojea8893269816d9
diff --git a/scripts/launch-microvms.sh b/scripts/launch-microvms.sh
index f109fca9..f46a6f17 100755
--- a/scripts/launch-microvms.sh
+++ b/scripts/launch-microvms.sh
@@ -34,6 +34,12 @@ AGENT_DOMAIN="${AGENT_DOMAIN:-scale.sam-mesh.dev}"
VM_MEM_MIB="${VM_MEM_MIB:-160}"
VM_VCPUS="${VM_VCPUS:-1}"
+# How long a sandbox stays up after its agent finishes. A density measurement
+# needs the population resident at the same time, and left alone a sandbox
+# powers off the moment it is done -- so a thousand started in sequence are
+# never a thousand at once. Zero is the right default for real work.
+SANDBOX_LINGER="${SANDBOX_LINGER:-0}"
+
if [ ! -d "$WORKDIR" ]; then
echo "Error: $WORKDIR does not exist. Did cloud-init finish?" >&2
exit 1
@@ -66,8 +72,12 @@ command -v "$SAM_BOX" >/dev/null 2>&1 || [ -x "$SAM_BOX" ] || fail "sam-box not
# An agent sandbox has no network device and reaches the mesh through a tun, so
# a kernel without the driver gives every sandbox no route at all.
+#
+# grep reads the binary directly rather than piping strings, which is not
+# installed everywhere: a check that cannot run must not report that the check
+# failed, and this one did exactly that on a host without binutils.
if [ -f "$WORKDIR/vmlinux.bin" ]; then
- tun_driver="$(strings "$WORKDIR/vmlinux.bin" 2>/dev/null | grep -c "Universal TUN/TAP" || true)"
+ tun_driver="$(grep -ac "Universal TUN/TAP" "$WORKDIR/vmlinux.bin" 2>/dev/null || true)"
[ "${tun_driver:-0}" -gt 0 ] || fail "guest kernel has no TUN driver; sandboxes need CONFIG_TUN"
fi
@@ -192,9 +202,12 @@ EOF
sleep 0.05
done
+ # The kernel passes cmdline KEY=VALUE pairs it does not recognise to init
+ # as environment variables, which is the only way to tell PID 1 anything:
+ # it is started with an empty environment.
fc_put "$API_SOCKET" /boot-source "{
\"kernel_image_path\": \"$WORKDIR/vmlinux.bin\",
- \"boot_args\": \"console=ttyS0 reboot=k panic=1 pci=off ro\"
+ \"boot_args\": \"console=ttyS0 reboot=k panic=1 pci=off ro SANDBOX_LINGER=${SANDBOX_LINGER}\"
}"
fc_put "$API_SOCKET" /machine-config "{
diff --git a/scripts/microvm-init.sh b/scripts/microvm-init.sh
index e12d5229..38ce8bd1 100755
--- a/scripts/microvm-init.sh
+++ b/scripts/microvm-init.sh
@@ -44,5 +44,14 @@ if ! /usr/local/bin/nano-init run vsock://2:1080 python3 /app/agent/agent.py "${
echo "sandbox init failed; see above" > /dev/console
fi
+# A density measurement needs the population to exist at the same time. Left to
+# itself a sandbox powers off the moment its agent is done, so a thousand of
+# them started in sequence are never a thousand at once, and the memory figure
+# would describe how many passed through rather than how many fit.
+if [ "${SANDBOX_LINGER:-0}" -gt 0 ] 2>/dev/null; then
+ echo "Agent finished; holding the sandbox for ${SANDBOX_LINGER}s." > /dev/console
+ sleep "${SANDBOX_LINGER}"
+fi
+
sleep 3
reboot -f
diff --git a/scripts/provision-scale-vm.sh b/scripts/provision-scale-vm.sh
index 174765a4..888705b1 100755
--- a/scripts/provision-scale-vm.sh
+++ b/scripts/provision-scale-vm.sh
@@ -16,11 +16,21 @@ while [[ "$#" -gt 0 ]]; do
--local-binaries) GCS_URL="$2"; shift ;;
--count) COUNT="$2"; shift ;;
--machine-type) MACHINE_TYPE="$2"; shift ;;
+ # Spot is the right default for a fleet of minions, and the wrong one for
+ # a measurement: a run reclaimed halfway through produces no result and
+ # costs the same as one that finished.
+ --no-spot) SPOT=0 ;;
*) echo "Unknown parameter passed: $1"; exit 1 ;;
esac
shift
done
+if [ "${SPOT:-1}" -eq 1 ]; then
+ SCHEDULING="--provisioning-model=SPOT --preemption-notice-duration=0s --instance-termination-action=STOP"
+else
+ SCHEDULING="--provisioning-model=STANDARD"
+fi
+
METADATA="enable-osconfig=TRUE"
if [ -n "$GCS_URL" ]; then
@@ -61,9 +71,7 @@ if [ "$COUNT" -eq 1 ]; then
--metadata-from-file=startup-script=scripts/startup-script.sh \
--no-restart-on-failure \
--maintenance-policy=TERMINATE \
- --provisioning-model=SPOT \
- --preemption-notice-duration=0s \
- --instance-termination-action=STOP \
+ ${SCHEDULING} \
--service-account=628944397724-compute@developer.gserviceaccount.com \
--scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/trace.append \
--tags=http-server,https-server,lb-health-check \
@@ -88,9 +96,7 @@ else
--metadata-from-file=startup-script=scripts/startup-script.sh \
--no-restart-on-failure \
--maintenance-policy=TERMINATE \
- --provisioning-model=SPOT \
- --preemption-notice-duration=0s \
- --instance-termination-action=STOP \
+ ${SCHEDULING} \
--service-account=628944397724-compute@developer.gserviceaccount.com \
--scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/trace.append \
--tags=http-server,https-server,lb-health-check \
diff --git a/site/content/docs/scale-report.md b/site/content/docs/scale-report.md
index e853eee4..500baeb3 100644
--- a/site/content/docs/scale-report.md
+++ b/site/content/docs/scale-report.md
@@ -227,13 +227,20 @@ Written down because a result without them is advocacy.
## Reproducing this
```bash
-# Requires docker and kind; stands up a real mesh, then tears it down.
+# The boundary measurements: requires docker and kind, stands up a real mesh.
./tests/scale/run-density.sh --steps 1,2,4,8,16,32,64 --requests 500
-# Results, including environment.json and the raw per-step observations,
-# land in tests/scale/results/<timestamp>/ with a rendered table.md.
+# The thousand-agent run: needs a host with KVM, a guest kernel with
+# CONFIG_TUN, and a bootstrap token for a control plane.
+./scripts/provision-scale-vm.sh --no-spot --local-binaries gs://your-bucket/sam
+# then, on the host:
+SANDBOX_LINGER=2400 /opt/microvm/launch-microvms.sh 1000
+/opt/microvm/collect-fleet.sh --node-socket /var/run/sam-node.sock --duration 1500
```
+Results land in `tests/scale/results/`, including `environment.json` and the
+raw per-step observations, with a rendered `table.md`.
+
Individual measurements can be taken by hand:
```bash
@@ -255,5 +262,86 @@ from wherever it is bound.
This measures a boundary on a host. It says nothing yet about a thousand agents
in real sandboxes against a public testnet, where microVM memory, control-plane
enrolment and DHT behaviour all start to matter and none of them are exercised
-above. That experiment is the next one, and it is described in
-[the scale experiment guide](../scale-experiment/).
+above. That experiment is [below](#a-thousand-agents-on-one-host).
+
+## A thousand agents on one host
+
+The measurements above are of a boundary, not of a sandbox: host processes, no
+kernel per agent, no mesh beyond the machine. The obvious objection is that
+none of it survives contact with the real arrangement, where every agent is a
+Firecracker microVM with its own kernel and the node has to be a real member of
+a real mesh.
+
+So: one `n2-standard-64`, one `sam-node` enrolled in the public `bananas`
+testnet, and a thousand agents. Each agent is a microVM with **no network
+device at all**, reaching the mesh through its own `sam-box` over vsock.
+
+_Recorded 2026-08-21. Raw samples in
+[`tests/scale/results/fleet-1k`](https://github.com/google/sam/tree/main/tests/scale/results/fleet-1k)._
+
+### What a thousand agents cost
+
+| | total | per agent |
+| --- | --- | --- |
+| Guest microVMs | 162.7 GiB | **167 MiB** |
+| Boundaries (`sam-box`) | 24.6 GiB | **25 MiB** |
+| Node (one, for all of them) | 386 MiB | 0.4 MiB |
+| **Together** | **187 GiB of 251 GiB** | **192 MiB** |
+
+The node is the number worth pausing on. One `sam-node` served a thousand
+distinct agent principals in 386 MB, because an agent is not a peer: it has no
+enrolment, no key and no place in the DHT. The mesh gained a thousand
+principals and no members.
+
+### How fast they came up
+
+A thousand sandboxes launched in **149 seconds — 6.7 per second**, median
+153 ms each, p95 245 ms. The population became reachable as fast as it was
+started:
+
+| elapsed | agents serving | microVMs | guest memory |
+| --- | --- | --- | --- |
+| 0 s | 71 | 120 | 17.2 GiB |
+| 44 s | 328 | 383 | 62.1 GiB |
+| 99 s | 691 | 734 | 119.7 GiB |
+| 160 s | **1000** | 1000 | 162.7 GiB |
+| 323 s | 1000 | 1000 | 162.7 GiB |
+
+Agents trail microVMs by a few seconds and never by more than about fifty —
+that gap is a guest booting and its agent resolving a mesh name, not a queue
+forming. Nothing degraded as the population grew, and the last hundred came up
+as quickly as the first.
+
+`sam_node_agents_untracked_total` was **0**: every agent that ran was counted,
+so the thousand is a thousand and not a ceiling.
+
+### What this run says
+
+**An agent costs about 192 MiB and 150 ms**, nearly all of it the microVM. The
+boundary is 25 MB of that and the node's share is under half a megabyte. A
+host's agent count is decided by how much memory the agents themselves want.
+
+**The mesh does not grow when the agent population does.** A thousand agents
+arrived as one member with one enrolment. That is the whole reason for
+separating the principal from the peer, and it is the difference between a
+mesh that can hold this population and one that cannot.
+
+**It ran against a real, shared control plane** — not a fixture — with one
+bootstrap enrolment.
+
+### Threats to validity
+
+- **One host, one run.** No repetitions, no confidence intervals.
+- **The agents are the example harness**, which discovers tools, calls a model
+ and stops. A heavier agent needs more guest memory, and 167 MiB is a
+ statement about this workload rather than about agents in general.
+- **The population is idle at peak.** Every agent reached the mesh and was
+ counted; they were not all issuing requests when memory was measured, so this
+ is a density result and not a throughput one.
+- **A thousand is not a limit.** It is what fits comfortably in 251 GiB with
+ this agent. Nothing here found a ceiling, which also means nothing here
+ located one.
+- **Spot instances were preempted twice** before this run, mid-measurement. The
+ final run used a standard instance. Worth knowing if you reproduce it: an
+ experiment that takes ten minutes and a machine that lasts ten minutes do not
+ mix.
diff --git a/site/content/docs/user/running-agents.md b/site/content/docs/user/running-agents.md
index 22b74d92..d06e5e22 100644
--- a/site/content/docs/user/running-agents.md
+++ b/site/content/docs/user/running-agents.md
@@ -140,8 +140,9 @@ Two separations do the work.
member: it enrols, holds the mesh identity and maintains peer connections. The
box holds no identity of its own — it consumes the node's API socket on the
agent's behalf and names the agent on every request. Adding an agent therefore
-costs a `sam-box`, not a mesh member: [measured at about 18 MB and 59 ms](../../scale-report/),
-rather than a new enrolment and a new peer in the DHT.
+costs a `sam-box`, not a mesh member: [measured at about 25 MB and 150 ms](../../scale-report/),
+rather than a new enrolment and a new peer in the DHT. One node has been run
+with a thousand agents behind it.
**The node's API stays on the node's side of the boundary.** An agent never
reaches it. What the agent gets is the curated surface the gateway builds on
diff --git a/tests/e2e/microvm_sandbox.bats b/tests/e2e/microvm_sandbox.bats
index 28b10541..6926e720 100644
--- a/tests/e2e/microvm_sandbox.bats
+++ b/tests/e2e/microvm_sandbox.bats
@@ -36,11 +36,10 @@ setup() {
# virtio and nothing else. Checked here because the alternative is a guest
# that panics with its explanation truncated by the panic itself.
#
- # grep -c rather than grep -q: where pipefail is set, a -q that matches exits
- # early, strings takes SIGPIPE, and the pipeline reports failure for having
- # succeeded.
+ # grep reads the binary directly rather than piping strings, which is not
+ # installed everywhere; a check that cannot run must not report a failure.
local tun_driver
- tun_driver="$(strings "${kernel}" 2>/dev/null | grep -c "Universal TUN/TAP" || true)"
+ tun_driver="$(grep -ac "Universal TUN/TAP" "${kernel}" 2>/dev/null || true)"
[[ "${tun_driver}" -gt 0 ]] \
|| skip "guest kernel has no TUN driver; agent sandboxes need CONFIG_TUN"
diff --git a/tests/scale/measure-guest.sh b/tests/scale/measure-guest.sh
index c754e056..b6438180 100755
--- a/tests/scale/measure-guest.sh
+++ b/tests/scale/measure-guest.sh
@@ -48,9 +48,9 @@ command -v "${FC}" >/dev/null 2>&1 || { echo "firecracker not found; set FC_BIN"
# An agent sandbox has no network device and reaches the mesh through a tun, so
# a kernel without the driver gives it no route and this measures nothing.
#
-# grep -c rather than grep -q: under pipefail, a -q that matches exits early,
-# strings takes SIGPIPE, and the pipeline reports failure for having succeeded.
-tun_driver="$(strings "${KERNEL}" 2>/dev/null | grep -c "Universal TUN/TAP" || true)"
+# grep reads the binary directly rather than piping strings, which is not
+# installed everywhere; a check that cannot run must not report a failure.
+tun_driver="$(grep -ac "Universal TUN/TAP" "${KERNEL}" 2>/dev/null || true)"
if [[ "${tun_driver}" -eq 0 ]]; then
echo "guest kernel has no TUN driver; agent sandboxes need CONFIG_TUN" >&2
exit 1
diff --git a/tests/scale/results/fleet-1k/environment.json b/tests/scale/results/fleet-1k/environment.json
new file mode 100644
index 00000000..be822a95
--- /dev/null
+++ b/tests/scale/results/fleet-1k/environment.json
@@ -0,0 +1 @@
+{"cpus":64,"memory_kb":264112392,"kernel":"6.12.101+deb13-cloud-amd64","machine":"n2-standard-64","zone":"us-central1-c","control_plane":"bananas.sam-mesh.dev"}
diff --git a/tests/scale/results/fleet-1k/fleet.jsonl b/tests/scale/results/fleet-1k/fleet.jsonl
new file mode 100644
index 00000000..5164fd81
--- /dev/null
+++ b/tests/scale/results/fleet-1k/fleet.jsonl
@@ -0,0 +1,14 @@
+{"elapsed_s":0,"agents_seen":71,"microvms":120,"boundaries":120,"guest_rss_kb":18084472,"boundary_rss_kb":3210524,"node_rss_kb":90924,"mem_available_kb":239769864,"load1":9.14,"requests_in_flight":141}
+{"elapsed_s":21,"agents_seen":190,"microvms":240,"boundaries":240,"guest_rss_kb":39666876,"boundary_rss_kb":6523124,"node_rss_kb":141940,"mem_available_kb":214940756,"load1":50.64,"requests_in_flight":34}
+{"elapsed_s":44,"agents_seen":328,"microvms":383,"boundaries":384,"guest_rss_kb":65083260,"boundary_rss_kb":10493304,"node_rss_kb":195820,"mem_available_kb":184662356,"load1":74.14,"requests_in_flight":26}
+{"elapsed_s":71,"agents_seen":503,"microvms":553,"boundaries":554,"guest_rss_kb":94647980,"boundary_rss_kb":15146248,"node_rss_kb":255572,"mem_available_kb":150693864,"load1":86.04,"requests_in_flight":41}
+{"elapsed_s":99,"agents_seen":691,"microvms":734,"boundaries":735,"guest_rss_kb":125481936,"boundary_rss_kb":19656624,"node_rss_kb":316572,"mem_available_kb":117163088,"load1":92.43,"requests_in_flight":9}
+{"elapsed_s":126,"agents_seen":880,"microvms":930,"boundaries":933,"guest_rss_kb":159709888,"boundary_rss_kb":25319452,"node_rss_kb":388896,"mem_available_kb":75279728,"load1":101.08,"requests_in_flight":18}
+{"elapsed_s":160,"agents_seen":1000,"microvms":1000,"boundaries":1000,"guest_rss_kb":170637612,"boundary_rss_kb":25838892,"node_rss_kb":395488,"mem_available_kb":74554096,"load1":68.57,"requests_in_flight":0}
+{"elapsed_s":184,"agents_seen":1000,"microvms":1000,"boundaries":1000,"guest_rss_kb":170637612,"boundary_rss_kb":25739804,"node_rss_kb":395504,"mem_available_kb":74580948,"load1":49.30,"requests_in_flight":0}
+{"elapsed_s":207,"agents_seen":1000,"microvms":1000,"boundaries":1000,"guest_rss_kb":170637612,"boundary_rss_kb":25643792,"node_rss_kb":395504,"mem_available_kb":74652972,"load1":32.68,"requests_in_flight":0}
+{"elapsed_s":230,"agents_seen":1000,"microvms":1000,"boundaries":1000,"guest_rss_kb":170637612,"boundary_rss_kb":25531624,"node_rss_kb":395508,"mem_available_kb":74783932,"load1":23.47,"requests_in_flight":0}
+{"elapsed_s":254,"agents_seen":1000,"microvms":1000,"boundaries":1000,"guest_rss_kb":170637612,"boundary_rss_kb":25395664,"node_rss_kb":395516,"mem_available_kb":74857396,"load1":15.97,"requests_in_flight":0}
+{"elapsed_s":277,"agents_seen":1000,"microvms":1000,"boundaries":1000,"guest_rss_kb":170637612,"boundary_rss_kb":25324920,"node_rss_kb":285464,"mem_available_kb":74909588,"load1":10.52,"requests_in_flight":0}
+{"elapsed_s":300,"agents_seen":1000,"microvms":1000,"boundaries":1000,"guest_rss_kb":170637612,"boundary_rss_kb":25279044,"node_rss_kb":307168,"mem_available_kb":74927616,"load1":7.60,"requests_in_flight":0}
+{"elapsed_s":323,"agents_seen":1000,"microvms":1000,"boundaries":1000,"guest_rss_kb":170637612,"boundary_rss_kb":25139408,"node_rss_kb":319016,"mem_available_kb":74979432,"load1":5.17,"requests_in_flight":0}
diff --git a/tests/scale/results/fleet-1k/launch-timings.csv b/tests/scale/results/fleet-1k/launch-timings.csv
new file mode 100644
index 00000000..46342c7c
--- /dev/null
+++ b/tests/scale/results/fleet-1k/launch-timings.csv
@@ -0,0 +1,1001 @@
+index,launched_epoch_ms
+1,1787332008192
+2,1787332008244
+3,1787332008295
+4,1787332008346
+5,1787332008397
+6,1787332008447
+7,1787332008498
+8,1787332008550
+9,1787332008598
+10,1787332008646
+11,1787332008699
+12,1787332008749
+13,1787332008801
+14,1787332008854
+15,1787332008907
+16,1787332008957
+17,1787332009010
+18,1787332009063
+19,1787332009116
+20,1787332009169
+21,1787332009221
+22,1787332009273
+23,1787332009326
+24,1787332009376
+25,1787332009427
+26,1787332009478
+27,1787332009532
+28,1787332009586
+29,1787332009640
+30,1787332009692
+31,1787332009747
+32,1787332009799
+33,1787332009855
+34,1787332009909
+35,1787332009962
+36,1787332010018
+37,1787332010073
+38,1787332010128
+39,1787332010187
+40,1787332010249
+41,1787332010312
+42,1787332010369
+43,1787332010432
+44,1787332010501
+45,1787332010564
+46,1787332010621
+47,1787332010686
+48,1787332010753
+49,1787332010816
+50,1787332010880
+51,1787332010948
+52,1787332011021
+53,1787332011089
+54,1787332011156
+55,1787332011224
+56,1787332011295
+57,1787332011365
+58,1787332011441
+59,1787332011515
+60,1787332011589
+61,1787332011669
+62,1787332011745
+63,1787332011839
+64,1787332011918
+65,1787332012000
+66,1787332012104
+67,1787332012247
+68,1787332012377
+69,1787332012514
+70,1787332012668
+71,1787332012819
+72,1787332012981
+73,1787332013138
+74,1787332013255
+75,1787332013440
+76,1787332013656
+77,1787332013818
+78,1787332013949
+79,1787332014132
+80,1787332014288
+81,1787332014428
+82,1787332014557
+83,1787332014650
+84,1787332014801
+85,1787332014897
+86,1787332014985
+87,1787332015079
+88,1787332015173
+89,1787332015269
+90,1787332015360
+91,1787332015446
+92,1787332015531
+93,1787332015667
+94,1787332015762
+95,1787332015846
+96,1787332015927
+97,1787332016008
+98,1787332016092
+99,1787332016175
+100,1787332016260
+101,1787332016330
+102,1787332016412
+103,1787332016495
+104,1787332016578
+105,1787332016657
+106,1787332016742
+107,1787332016820
+108,1787332016889
+109,1787332016968
+110,1787332017040
+111,1787332017113
+112,1787332017193
+113,1787332017278
+114,1787332017353
+115,1787332017416
+116,1787332017495
+117,1787332017561
+118,1787332017639
+119,1787332017711
+120,1787332017791
+121,1787332017860
+122,1787332017933
+123,1787332018003
+124,1787332018083
+125,1787332018163
+126,1787332018241
+127,1787332018316
+128,1787332018395
+129,1787332018468
+130,1787332018560
+131,1787332018665
+132,1787332018889
+133,1787332019037
+134,1787332019269
+135,1787332019520
+136,1787332019767
+137,1787332020047
+138,1787332020368
+139,1787332020692
+140,1787332020968
+141,1787332021277
+142,1787332021593
+143,1787332021844
+144,1787332022164
+145,1787332022529
+146,1787332022849
+147,1787332023093
+148,1787332023354
+149,1787332023627
+150,1787332023898
+151,1787332024203
+152,1787332024496
+153,1787332024747
+154,1787332024997
+155,1787332025173
+156,1787332025392
+157,1787332025622
+158,1787332025848
+159,1787332026070
+160,1787332026292
+161,1787332026480
+162,1787332026695
+163,1787332026876
+164,1787332027069
+165,1787332027320
+166,1787332027499
+167,1787332027736
+168,1787332027929
+169,1787332028080
+170,1787332028204
+171,1787332028321
+172,1787332028404
+173,1787332028498
+174,1787332028584
+175,1787332028733
+176,1787332028828
+177,1787332028987
+178,1787332029215
+179,1787332029382
+180,1787332029635
+181,1787332029788
+182,1787332029967
+183,1787332030125
+184,1787332030366
+185,1787332030600
+186,1787332030789
+187,1787332031008
+188,1787332031222
+189,1787332031434
+190,1787332031638
+191,1787332031828
+192,1787332031964
+193,1787332032178
+194,1787332032362
+195,1787332032566
+196,1787332032777
+197,1787332033016
+198,1787332033235
+199,1787332033369
+200,1787332033552
+201,1787332033740
+202,1787332033992
+203,1787332034204
+204,1787332034438
+205,1787332034625
+206,1787332034780
+207,1787332034878
+208,1787332034988
+209,1787332035148
+210,1787332035272
+211,1787332035437
+212,1787332035576
+213,1787332035685
+214,1787332035828
+215,1787332035980
+216,1787332036109
+217,1787332036193
+218,1787332036290
+219,1787332036377
+220,1787332036459
+221,1787332036555
+222,1787332036645
+223,1787332036734
+224,1787332036878
+225,1787332036965
+226,1787332037081
+227,1787332037162
+228,1787332037283
+229,1787332037433
+230,1787332037564
+231,1787332037662
+232,1787332037810
+233,1787332037929
+234,1787332038061
+235,1787332038170
+236,1787332038277
+237,1787332038360
+238,1787332038470
+239,1787332038582
+240,1787332038746
+241,1787332038852
+242,1787332038941
+243,1787332039045
+244,1787332039161
+245,1787332039243
+246,1787332039328
+247,1787332039426
+248,1787332039525
+249,1787332039641
+250,1787332039774
+251,1787332039893
+252,1787332040021
+253,1787332040156
+254,1787332040339
+255,1787332040580
+256,1787332040753
+257,1787332040940
+258,1787332041127
+259,1787332041332
+260,1787332041504
+261,1787332041692
+262,1787332041900
+263,1787332042051
+264,1787332042250
+265,1787332042382
+266,1787332042561
+267,1787332042732
+268,1787332042916
+269,1787332043086
+270,1787332043280
+271,1787332043465
+272,1787332043633
+273,1787332043840
+274,1787332044026
+275,1787332044232
+276,1787332044420
+277,1787332044564
+278,1787332044732
+279,1787332044928
+280,1787332045154
+281,1787332045304
+282,1787332045487
+283,1787332045691
+284,1787332045900
+285,1787332046090
+286,1787332046260
+287,1787332046444
+288,1787332046612
+289,1787332046785
+290,1787332046972
+291,1787332047158
+292,1787332047327
+293,1787332047496
+294,1787332047643
+295,1787332047860
+296,1787332048008
+297,1787332048178
+298,1787332048316
+299,1787332048452
+300,1787332048597
+301,1787332048776
+302,1787332048954
+303,1787332049077
+304,1787332049292
+305,1787332049460
+306,1787332049614
+307,1787332049863
+308,1787332050072
+309,1787332050254
+310,1787332050444
+311,1787332050630
+312,1787332050825
+313,1787332051061
+314,1787332051228
+315,1787332051368
+316,1787332051465
+317,1787332051605
+318,1787332051701
+319,1787332051785
+320,1787332051870
+321,1787332051960
+322,1787332052043
+323,1787332052129
+324,1787332052208
+325,1787332052290
+326,1787332052388
+327,1787332052473
+328,1787332052547
+329,1787332052635
+330,1787332052706
+331,1787332052779
+332,1787332052864
+333,1787332052948
+334,1787332053027
+335,1787332053108
+336,1787332053179
+337,1787332053257
+338,1787332053338
+339,1787332053420
+340,1787332053499
+341,1787332053578
+342,1787332053663
+343,1787332053746
+344,1787332053830
+345,1787332053906
+346,1787332053990
+347,1787332054085
+348,1787332054232
+349,1787332054476
+350,1787332054674
+351,1787332054930
+352,1787332055208
+353,1787332055416
+354,1787332055633
+355,1787332055910
+356,1787332056109
+357,1787332056388
+358,1787332056604
+359,1787332056844
+360,1787332057025
+361,1787332057228
+362,1787332057494
+363,1787332057696
+364,1787332057892
+365,1787332058074
+366,1787332058296
+367,1787332058499
+368,1787332058711
+369,1787332058936
+370,1787332059181
+371,1787332059344
+372,1787332059547
+373,1787332059788
+374,1787332059944
+375,1787332060120
+376,1787332060324
+377,1787332060528
+378,1787332060704
+379,1787332060884
+380,1787332061120
+381,1787332061312
+382,1787332061476
+383,1787332061663
+384,1787332061836
+385,1787332062016
+386,1787332062192
+387,1787332062339
+388,1787332062513
+389,1787332062702
+390,1787332062860
+391,1787332063088
+392,1787332063333
+393,1787332063516
+394,1787332063672
+395,1787332063809
+396,1787332063988
+397,1787332064175
+398,1787332064346
+399,1787332064568
+400,1787332064749
+401,1787332064976
+402,1787332065140
+403,1787332065307
+404,1787332065534
+405,1787332065670
+406,1787332065872
+407,1787332066053
+408,1787332066212
+409,1787332066380
+410,1787332066552
+411,1787332066731
+412,1787332066912
+413,1787332067015
+414,1787332067148
+415,1787332067257
+416,1787332067412
+417,1787332067580
+418,1787332067739
+419,1787332067900
+420,1787332068060
+421,1787332068205
+422,1787332068316
+423,1787332068452
+424,1787332068605
+425,1787332068796
+426,1787332068998
+427,1787332069154
+428,1787332069261
+429,1787332069345
+430,1787332069490
+431,1787332069580
+432,1787332069690
+433,1787332069798
+434,1787332069938
+435,1787332070032
+436,1787332070116
+437,1787332070203
+438,1787332070337
+439,1787332070422
+440,1787332070517
+441,1787332070640
+442,1787332070738
+443,1787332070826
+444,1787332070965
+445,1787332071069
+446,1787332071147
+447,1787332071261
+448,1787332071358
+449,1787332071447
+450,1787332071530
+451,1787332071616
+452,1787332071763
+453,1787332071853
+454,1787332071946
+455,1787332072031
+456,1787332072118
+457,1787332072210
+458,1787332072293
+459,1787332072368
+460,1787332072458
+461,1787332072550
+462,1787332072633
+463,1787332072721
+464,1787332072814
+465,1787332072895
+466,1787332072981
+467,1787332073061
+468,1787332073145
+469,1787332073234
+470,1787332073332
+471,1787332073472
+472,1787332073621
+473,1787332073806
+474,1787332074060
+475,1787332074220
+476,1787332074432
+477,1787332074632
+478,1787332074850
+479,1787332075077
+480,1787332075246
+481,1787332075380
+482,1787332075612
+483,1787332075792
+484,1787332076076
+485,1787332076303
+486,1787332076537
+487,1787332076736
+488,1787332076920
+489,1787332077198
+490,1787332077385
+491,1787332077599
+492,1787332077804
+493,1787332078000
+494,1787332078180
+495,1787332078320
+496,1787332078476
+497,1787332078676
+498,1787332078845
+499,1787332079092
+500,1787332079344
+501,1787332079562
+502,1787332079781
+503,1787332079924
+504,1787332080170
+505,1787332080336
+506,1787332080600
+507,1787332080788
+508,1787332080947
+509,1787332081146
+510,1787332081267
+511,1787332081454
+512,1787332081648
+513,1787332081814
+514,1787332081991
+515,1787332082183
+516,1787332082325
+517,1787332082477
+518,1787332082704
+519,1787332082928
+520,1787332083113
+521,1787332083303
+522,1787332083436
+523,1787332083543
+524,1787332083724
+525,1787332083893
+526,1787332084048
+527,1787332084279
+528,1787332084444
+529,1787332084640
+530,1787332084836
+531,1787332085028
+532,1787332085209
+533,1787332085400
+534,1787332085553
+535,1787332085692
+536,1787332085837
+537,1787332085966
+538,1787332086160
+539,1787332086339
+540,1787332086520
+541,1787332086704
+542,1787332086870
+543,1787332086984
+544,1787332087112
+545,1787332087296
+546,1787332087469
+547,1787332087645
+548,1787332087840
+549,1787332088016
+550,1787332088163
+551,1787332088354
+552,1787332088511
+553,1787332088736
+554,1787332088898
+555,1787332089084
+556,1787332089248
+557,1787332089408
+558,1787332089576
+559,1787332089788
+560,1787332089924
+561,1787332090028
+562,1787332090157
+563,1787332090337
+564,1787332090485
+565,1787332090594
+566,1787332090748
+567,1787332090865
+568,1787332091056
+569,1787332091272
+570,1787332091391
+571,1787332091536
+572,1787332091688
+573,1787332091844
+574,1787332091997
+575,1787332092140
+576,1787332092356
+577,1787332092492
+578,1787332092623
+579,1787332092787
+580,1787332092965
+581,1787332093155
+582,1787332093355
+583,1787332093505
+584,1787332093652
+585,1787332093811
+586,1787332093976
+587,1787332094153
+588,1787332094294
+589,1787332094415
+590,1787332094589
+591,1787332094750
+592,1787332094920
+593,1787332095100
+594,1787332095244
+595,1787332095413
+596,1787332095571
+597,1787332095706
+598,1787332095817
+599,1787332095908
+600,1787332095997
+601,1787332096176
+602,1787332096279
+603,1787332096360
+604,1787332096481
+605,1787332096554
+606,1787332096637
+607,1787332096716
+608,1787332096804
+609,1787332096888
+610,1787332097003
+611,1787332097089
+612,1787332097176
+613,1787332097262
+614,1787332097349
+615,1787332097424
+616,1787332097510
+617,1787332097600
+618,1787332097685
+619,1787332097762
+620,1787332097846
+621,1787332097931
+622,1787332098012
+623,1787332098090
+624,1787332098178
+625,1787332098255
+626,1787332098329
+627,1787332098411
+628,1787332098491
+629,1787332098565
+630,1787332098642
+631,1787332098714
+632,1787332098790
+633,1787332098868
+634,1787332098941
+635,1787332099018
+636,1787332099102
+637,1787332099184
+638,1787332099272
+639,1787332099371
+640,1787332099560
+641,1787332099769
+642,1787332099968
+643,1787332100164
+644,1787332100381
+645,1787332100640
+646,1787332100865
+647,1787332101042
+648,1787332101245
+649,1787332101444
+650,1787332101656
+651,1787332101864
+652,1787332102123
+653,1787332102357
+654,1787332102625
+655,1787332102860
+656,1787332103082
+657,1787332103311
+658,1787332103523
+659,1787332103723
+660,1787332103922
+661,1787332104104
+662,1787332104292
+663,1787332104504
+664,1787332104696
+665,1787332104865
+666,1787332105056
+667,1787332105226
+668,1787332105383
+669,1787332105555
+670,1787332105804
+671,1787332105983
+672,1787332106169
+673,1787332106340
+674,1787332106500
+675,1787332106633
+676,1787332106725
+677,1787332106833
+678,1787332106986
+679,1787332107131
+680,1787332107283
+681,1787332107483
+682,1787332107644
+683,1787332107838
+684,1787332108013
+685,1787332108217
+686,1787332108457
+687,1787332108636
+688,1787332108837
+689,1787332109038
+690,1787332109231
+691,1787332109461
+692,1787332109715
+693,1787332109924
+694,1787332110132
+695,1787332110312
+696,1787332110514
+697,1787332110712
+698,1787332110895
+699,1787332111076
+700,1787332111265
+701,1787332111393
+702,1787332111548
+703,1787332111722
+704,1787332111904
+705,1787332112067
+706,1787332112222
+707,1787332112412
+708,1787332112536
+709,1787332112716
+710,1787332112868
+711,1787332113100
+712,1787332113257
+713,1787332113380
+714,1787332113485
+715,1787332113601
+716,1787332113729
+717,1787332113972
+718,1787332114131
+719,1787332114272
+720,1787332114461
+721,1787332114641
+722,1787332114837
+723,1787332115019
+724,1787332115156
+725,1787332115336
+726,1787332115474
+727,1787332115606
+728,1787332115733
+729,1787332115900
+730,1787332116063
+731,1787332116182
+732,1787332116311
+733,1787332116442
+734,1787332116579
+735,1787332116711
+736,1787332116910
+737,1787332117084
+738,1787332117242
+739,1787332117444
+740,1787332117593
+741,1787332117765
+742,1787332117891
+743,1787332118023
+744,1787332118206
+745,1787332118384
+746,1787332118525
+747,1787332118638
+748,1787332118768
+749,1787332118952
+750,1787332119151
+751,1787332119336
+752,1787332119462
+753,1787332119632
+754,1787332119801
+755,1787332119996
+756,1787332120108
+757,1787332120272
+758,1787332120472
+759,1787332120626
+760,1787332120788
+761,1787332120885
+762,1787332121032
+763,1787332121254
+764,1787332121422
+765,1787332121560
+766,1787332121716
+767,1787332121904
+768,1787332122122
+769,1787332122300
+770,1787332122464
+771,1787332122605
+772,1787332122724
+773,1787332122832
+774,1787332122942
+775,1787332123112
+776,1787332123247
+777,1787332123451
+778,1787332123639
+779,1787332123844
+780,1787332123976
+781,1787332124153
+782,1787332124272
+783,1787332124373
+784,1787332124471
+785,1787332124559
+786,1787332124652
+787,1787332124735
+788,1787332124840
+789,1787332124926
+790,1787332125014
+791,1787332125103
+792,1787332125212
+793,1787332125396
+794,1787332125488
+795,1787332125593
+796,1787332125683
+797,1787332125785
+798,1787332125885
+799,1787332125984
+800,1787332126127
+801,1787332126231
+802,1787332126344
+803,1787332126492
+804,1787332126617
+805,1787332126774
+806,1787332126885
+807,1787332126999
+808,1787332127104
+809,1787332127255
+810,1787332127371
+811,1787332127539
+812,1787332127657
+813,1787332127807
+814,1787332127901
+815,1787332127999
+816,1787332128162
+817,1787332128264
+818,1787332128377
+819,1787332128529
+820,1787332128684
+821,1787332128845
+822,1787332129000
+823,1787332129146
+824,1787332129317
+825,1787332129548
+826,1787332129654
+827,1787332129808
+828,1787332129971
+829,1787332130085
+830,1787332130180
+831,1787332130296
+832,1787332130392
+833,1787332130493
+834,1787332130598
+835,1787332130762
+836,1787332130881
+837,1787332131023
+838,1787332131125
+839,1787332131235
+840,1787332131380
+841,1787332131469
+842,1787332131558
+843,1787332131648
+844,1787332131736
+845,1787332131825
+846,1787332131904
+847,1787332131991
+848,1787332132084
+849,1787332132171
+850,1787332132257
+851,1787332132341
+852,1787332132426
+853,1787332132502
+854,1787332132592
+855,1787332132673
+856,1787332132754
+857,1787332132826
+858,1787332132906
+859,1787332132986
+860,1787332133071
+861,1787332133151
+862,1787332133237
+863,1787332133316
+864,1787332133399
+865,1787332133479
+866,1787332133570
+867,1787332133658
+868,1787332133744
+869,1787332133834
+870,1787332133915
+871,1787332133990
+872,1787332134065
+873,1787332134150
+874,1787332134234
+875,1787332134312
+876,1787332134401
+877,1787332134488
+878,1787332134570
+879,1787332134675
+880,1787332134854
+881,1787332135024
+882,1787332135224
+883,1787332135461
+884,1787332135701
+885,1787332135864
+886,1787332136060
+887,1787332136240
+888,1787332136425
+889,1787332136636
+890,1787332136842
+891,1787332137077
+892,1787332137286
+893,1787332137512
+894,1787332137769
+895,1787332138014
+896,1787332138214
+897,1787332138402
+898,1787332138592
+899,1787332138803
+900,1787332138957
+901,1787332139160
+902,1787332139334
+903,1787332139596
+904,1787332139828
+905,1787332140041
+906,1787332140238
+907,1787332140440
+908,1787332140654
+909,1787332140853
+910,1787332141044
+911,1787332141212
+912,1787332141460
+913,1787332141654
+914,1787332141906
+915,1787332142092
+916,1787332142277
+917,1787332142522
+918,1787332142721
+919,1787332142882
+920,1787332143073
+921,1787332143197
+922,1787332143318
+923,1787332143503
+924,1787332143682
+925,1787332143830
+926,1787332143981
+927,1787332144107
+928,1787332144300
+929,1787332144494
+930,1787332144668
+931,1787332144840
+932,1787332145024
+933,1787332145254
+934,1787332145401
+935,1787332145588
+936,1787332145720
+937,1787332145909
+938,1787332146101
+939,1787332146251
+940,1787332146435
+941,1787332146615
+942,1787332146798
+943,1787332147037
+944,1787332147196
+945,1787332147334
+946,1787332147491
+947,1787332147607
+948,1787332147830
+949,1787332148008
+950,1787332148131
+951,1787332148282
+952,1787332148407
+953,1787332148549
+954,1787332148677
+955,1787332148836
+956,1787332149013
+957,1787332149192
+958,1787332149369
+959,1787332149529
+960,1787332149693
+961,1787332149884
+962,1787332150023
+963,1787332150176
+964,1787332150348
+965,1787332150466
+966,1787332150579
+967,1787332150754
+968,1787332150926
+969,1787332151064
+970,1787332151214
+971,1787332151349
+972,1787332151481
+973,1787332151648
+974,1787332151794
+975,1787332151980
+976,1787332152169
+977,1787332152344
+978,1787332152513
+979,1787332152673
+980,1787332152836
+981,1787332153028
+982,1787332153165
+983,1787332153356
+984,1787332153548
+985,1787332153808
+986,1787332154003
+987,1787332154156
+988,1787332154349
+989,1787332154527
+990,1787332154717
+991,1787332154980
+992,1787332155233
+993,1787332155429
+994,1787332155596
+995,1787332155849
+996,1787332156048
+997,1787332156241
+998,1787332156448
+999,1787332156622
+1000,1787332156780