Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
1.4 KB
Raw Blame
{"cancelled": false, "chat_turns": 1, "config": {"compute_threads": 1, "io_threads": 4, "max_context_tokens": 4096, "pool_budget_bytes": 100663296, "scrub_percent": 0}, "decode_ms": 1908, "decode_tok_s": 4.1928721174004195, "error": "", "first_token_ms": 2725, "generate_seq": 1, "gpu": {"active": true, "cache_hit": true, "cus": 12, "device": "QUALCOMM Adreno(TM) 830", "driver": "OpenCL 3.0 QUALCOMM build: 0800.72 Compiler E031.47.18.49", "mem_mib": 7576, "ms": 1415.627034187317, "n_submit": 7944, "n_sync": 207, "ok": true, "policy": 0, "probed": true, "why": "ok"}, "int8_effective": false, "layer_track": 0, "new_tokens": 8, "params": {"max_new_tokens": 8, "seed": 539363605, "temperature": 0.0, "top_k": 0, "top_p": 1.0}, "peak_rss_bytes": 1366167552, "pool": {"claim_ns": 0, "claim_ns_per_tok": 0.0, "evictions": 4029, "flash_bytes": 5429182464, "hits": 635, "io_failures": 0, "lookup_ns": 0, "lookup_ns_per_tok": 0.0, "misses": 2095, "prefetch_dropped": 0, "prefetch_issued": 4216, "prefetch_wasted": 2220, "resident_bytes": 100564992, "slots_total": 62, "slots_used": 62, "stall_ns": 637740209, "sync_rescues": 0}, "prefill_cold_s": 8.443465491923641, "prefill_ms": 2724, "prefill_tok_s": 8.443465491923641, "prefill_warm_s": 0.0, "prompt_tokens": 23, "route": {"agree": null, "dec_steps": 8, "feature": "executed", "mode": "predicted", "pred_used": 128}, "schema": 1, "thinking_on": true}