Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
5.1 KB
Raw Blame
"""Real HTTP chat validation, with emitted token IDs captured at the engine boundary."""
import argparse, json, sys, time, urllib.error, urllib.request
from pathlib import Path
ap=argparse.ArgumentParser(description=__doc__)
for name in ['source','config','baseline','candidate','mtp','output']:
 ap.add_argument('--'+name,type=Path,required=True)
a=ap.parse_args()
source=a.source.resolve()
sys.path[:0]=[str(source),str(source/'tools')]
from serve.server import StrataEngine, Service, engine_args, child_env, serve
from serve.frontend import ChatTemplate
import strata_tokenizer as ST

cfg=json.loads(a.config.read_text())
cfg={k:v for k,v in cfg.items() if k in ['args','tokenizer','gpu','lib_dirs','env']}
args=list(cfg['args'])
for flag in ['--conversation-cache-spill-dir','--conversation-cache-disk-mib','--mtp']:
 if flag in args:i=args.index(flag);del args[i:i+2]
args=[a for a in args if a!='--conversation-cache-disk-only']
tp=Path(cfg['tokenizer']);vocab=json.loads((tp/'vocab.json').read_text());tokens=[None]*len(vocab)
for t,i in vocab.items():tokens[i]=t
tok=ST.Tokenizer(tokens,(tp/'merges.txt').read_text().split('\n'),json.loads((tp/'token_type.json').read_text()))
template=ChatTemplate(tp/'chat_template.jinja')
rows=[]
out=a.output;out.mkdir(parents=True,exist_ok=True)
class AuditEngine(StrataEngine):
 def generate(self,*a,**kw):
  self.emitted=[]
  for token in super().generate(*a,**kw):
   if token is not None:self.emitted.append(token)
   yield token

for mode in ['baseline','off','mtp']:
 exe=(a.baseline if mode=='baseline' else a.candidate).resolve()
 current=list(args)
 if mode=='mtp':current+=['--mtp',str(a.mtp.resolve())];current[current.index('--spec')+1]='4'
 local=dict(cfg,args=current,exe=str(exe),cwd=str(source),log=str(out/(mode+'.log')))
 engine=AuditEngine(local['exe'],engine_args(local),cwd=local['cwd'],log=local['log'],env=child_env(local))
 service=Service(engine,tok,template,model_name='bias-validation')
 server=serve(service,port=0)
 def call(label,prompt='Write the Chinese word for hello. Only the word.',expect=200,**kw):
  req={'model':'bias-validation','messages':[{'role':'user','content':prompt}],'max_tokens':8,'temperature':0,'reasoning_effort':'none',**kw}
  data=json.dumps(req).encode();started=time.perf_counter()
  http=urllib.request.Request(f'http://127.0.0.1:{server.server_address[1]}/v1/chat/completions',data=data,headers={'Content-Type':'application/json'})
  try:
   with urllib.request.urlopen(http,timeout=180) as response:status=response.status;raw=response.read().decode()
  except urllib.error.HTTPError as e:
   with e:status=e.code;raw=e.read().decode()
  row={'mode':mode,'label':label,'status':status,'seconds':time.perf_counter()-started,'request_bytes':len(data),
       'token_ids':list(engine.emitted) if status==200 else [],'response':raw,'engine':dict(engine.last) if status==200 else {}}
  rows.append(row);(out/'rows.json').write_text(json.dumps(rows,indent=2,ensure_ascii=False))
  print(mode,label,status,row['token_ids'],flush=True)
  assert status==expect,(label,status,raw)
  return row
 try:
  first=call('unchanged-greedy')
  sampled=call('unchanged-sampled',prompt='Write one sentence about Python.',temperature=0.7,top_k=20,seed=42)
  if mode=='baseline':continue
  assert engine.info.get('logit_bias')==1
  banned=first['token_ids'][0]
  one=call('ban-object',logit_bias={str(banned):-100})
  two=call('ban-pairs',logit_bias=[[banned,False]])
  assert banned not in one['token_ids'] and banned not in two['token_ids']
  assert one['token_ids']==two['token_ids']
  boosted=tok.encode(' banana',parse_special=False)[0]
  positive=call('positive-bias',logit_bias={str(boosted):100},max_tokens=3)
  assert positive['token_ids'][0]==boosted
  # Changing requests must not inherit a prior mask, including after a rejected request.
  call('invalid-id',expect=400,logit_bias={str(len(tokens)):1},stream=True)
  call('invalid-value',expect=400,logit_bias={'0':101},stream=True)
  again=call('cleared-bias')
  assert again['token_ids']==first['token_ids']
  empty=call('empty-bias',logit_bias={})
  assert empty['token_ids']==first['token_ids']
  large=set(range(103214));large.add(banned)
  if len(large)<103215:large.add(103214)
  bulk=call('103215-bans',logit_bias=[[i,False] for i in sorted(large)],max_tokens=8)
  assert not set(bulk['token_ids']) & large
  streamed=call('stream-ban',logit_bias={str(banned):-100},stream=True)
  assert banned not in streamed['token_ids']
  sampled_ban=call('sampled-ban',logit_bias={str(banned):-100},temperature=0.8,seed=123)
  assert banned not in sampled_ban['token_ids']
 finally:
  server.shutdown();server.server_close();engine.close()

for label in ['unchanged-greedy','unchanged-sampled']:
 before=next(x for x in rows if x['mode']=='baseline' and x['label']==label)
 after=next(x for x in rows if x['mode']=='off' and x['label']==label)
 assert before['token_ids']==after['token_ids'],('default-output-changed',label)
(out/'complete.json').write_text(json.dumps({'status':'passed','requests':len(rows),'token_109266':tok.decode([109266]),
 'baseline':str(a.baseline),'candidate':str(a.candidate),'config':str(a.config)},indent=2,ensure_ascii=False))