Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Windows: fix non-ASCII chat prompt corruption (ANSI codepage vs UTF-8)

volognamoref97f3cf2b0a
diff --git a/c/compat.h b/c/compat.h
index 82de8b6c..06779e0a 100644
--- a/c/compat.h
+++ b/c/compat.h
@@ -304,8 +304,39 @@ static inline int compat_setenv(const char *name, const char *value, int overwri
}
#define setenv(name,value,overwrite) compat_setenv(name,value,overwrite)
+/* --- getenv_utf8: read an env var as UTF-8, not through the ANSI codepage ---
+ * Plain getenv()/_environ are populated by the CRT from the ANSI-codepage view
+ * of the process environment block, not UTF-8. A parent that hands the child a
+ * Unicode value via CreateProcessW's wide env block (e.g. Python's subprocess
+ * module, which coli uses to pass the chat prompt) round-trips correctly only
+ * through GetEnvironmentVariableW; going through narrow getenv() re-encodes it
+ * via CP_ACP first, so any non-ASCII prompt text (Cyrillic, CJK, ...) comes out
+ * corrupted before the byte-level tokenizer ever sees it. Read the wide value
+ * directly and convert straight to UTF-8, bypassing the ANSI codepage entirely.
+ * Returned buffer is intentionally leaked: called a handful of times at
+ * startup, lives for the process. */
+static inline const char *compat_getenv_utf8(const char *name){
+ wchar_t wname[64];
+ if(MultiByteToWideChar(CP_UTF8, 0, name, -1, wname, 64) <= 0) return getenv(name);
+ DWORD need = GetEnvironmentVariableW(wname, NULL, 0);
+ if(!need) return NULL;
+ wchar_t *wval = (wchar_t*)malloc(need * sizeof(wchar_t));
+ if(!wval) return NULL;
+ GetEnvironmentVariableW(wname, wval, need);
+ int blen = WideCharToMultiByte(CP_UTF8, 0, wval, -1, NULL, 0, NULL, NULL);
+ char *val = blen>0 ? (char*)malloc((size_t)blen) : NULL;
+ if(val) WideCharToMultiByte(CP_UTF8, 0, wval, -1, val, blen, NULL, NULL);
+ free(wval);
+ return val;
+}
+#define getenv_utf8(name) compat_getenv_utf8(name)
+
#endif /* _WIN32 */
+#ifndef getenv_utf8
+#define getenv_utf8(name) getenv(name)
+#endif
+
/* --- compat_aligned_free su piattaforme diverse da Windows ---
* Su Linux/macOS, posix_memalign usa free() normale. */
#ifndef compat_aligned_free
diff --git a/c/glm.c b/c/glm.c
index 00c70768..b0e419c0 100644
--- a/c/glm.c
+++ b/c/glm.c
@@ -5701,9 +5701,9 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){
* self-test, and would "generate" from "$P$G". So on Windows a PROMPT carrying
* cmd's $-metacodes is ignored; set COLI_PROMPT to pass a real prompt from cmd. */
static const char *coli_user_prompt(void){
- const char *p = getenv("COLI_PROMPT");
+ const char *p = getenv_utf8("COLI_PROMPT");
if(p) return p;
- p = getenv("PROMPT");
+ p = getenv_utf8("PROMPT");
#ifdef _WIN32
if(p) for(const char *q=p; q[0]; q++)
if(q[0]=='$' && q[1] && strchr("ABCDEFGHLNPQSTV_+|$", q[1]&~0x20)){ p=NULL; break; }