Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

11.3 KB
Raw Blame
from copy import deepcopy
import hashlib
import json
import pytest

from model_mtp import mtp_metadata, parse_runtime_capability, recommend_mtp, qualify_memory_fit


@pytest.mark.parametrize('declared,expected', [
    ('--load-mode MODE\n', ['--load-mode','mmap']),
    ('--mmap, --no-mmap\n', ['--mmap']),
    ('--load-mode MODE\n--mmap, --no-mmap\n', ['--load-mode','mmap']),
    ('Description mentions --mmap and --no-mmap but neither is a flag.\n', []),
])
def test_runtime_probe_reads_only_declared_load_mode_flags(tmp_path, monkeypatch, declared, expected):
    import model_mtp
    import subprocess
    runtime = tmp_path/'runtime'
    runtime.write_bytes(b'fixture')
    calls = []
    def run(command, **kwargs):
        calls.append((command,kwargs))
        return subprocess.CompletedProcess(command,0,stdout=declared+'--spec-type none,draft-mtp\n--spec-draft-n-max N\n')
    monkeypatch.setattr(model_mtp.subprocess, 'run', run)
    result = model_mtp.probe_runtime(runtime)
    assert result['mtp'] is True
    assert result['loadModeArguments'] == expected
    # No router compatibility verdict: ODS launches llama-server itself.
    assert 'lemonade10MmapCompatible' not in result
    assert calls[0][0] == [str(runtime.resolve()), '--help']
    assert calls[0][1]['creationflags'] == getattr(subprocess,'CREATE_NO_WINDOW',0)


def test_complete_launch_arguments_are_parsed_without_loading_or_exposing_output(monkeypatch):
    import model_mtp
    import subprocess
    calls = []
    def run(command, **kwargs):
        calls.append((command,kwargs))
        if '--removed-option' in command:
            return subprocess.CompletedProcess(command,1,stdout='invalid argument with private-local-path')
        return subprocess.CompletedProcess(command,0,stdout='--ctx-size N\n--model FILE\n')
    monkeypatch.setattr(model_mtp.subprocess,'run',run)
    launch = ['llama-server','--model','private-local-path','--load-mode','mmap']
    model_mtp.validate_runtime_command(launch)
    assert calls[0][0] == [*launch,'--help']
    assert calls[0][1]['stdin'] == subprocess.DEVNULL
    assert calls[0][1].get('shell',False) is False
    assert calls[0][1]['creationflags'] == getattr(subprocess,'CREATE_NO_WINDOW',0)
    with pytest.raises(ValueError,match='rejected its launch arguments') as failure:
        model_mtp.validate_runtime_command([*launch,'--removed-option'])
    assert 'private-local-path' not in str(failure.value)


def test_ngram_only_runtime_is_not_mtp_capable():
    old = "--spec-type [none|ngram-cache|ngram-simple]\n type of speculative decoding\n--draft-max N\n"
    assert parse_runtime_capability(old)["mtp"] is False
    current = "--spec-type none,draft-simple,draft-mtp,ngram-mod\n type of speculative decoding\n--spec-draft-n-max N\n"
    assert parse_runtime_capability(current)["mtp"] is True
    assert parse_runtime_capability("Use draft-mtp models.\n--spec-type none,ngram-mod\n--spec-draft-n-max N")["mtp"] is False


def test_real_metadata_not_filename_decides_embedded_support():
    assert mtp_metadata({"name":"Amazing MTP model"}, {"readable":True,"metadata":{}}) is None
    declared = {"mtp":{"source_url":"https://unsloth.ai/docs/models/mtp"}}
    assert mtp_metadata(declared, {})["modelSupport"] == "publisher-declared"
    actual = mtp_metadata(declared, {"metadata":{"qwen35.nextn_predict_layers":1}})
    assert actual["modelSupport"] == "embedded"
    assert actual["defaultEnabled"] is False
    assert actual["recommendation"] == "benchmark-required"
    assert mtp_metadata({}, {"metadata":{"qwen35.nextn_predict_layers":True}}) is None


def evidence():
    return {"signature":"same-model-runtime-hardware-context",
            "baseline":[{"valid":True,"tokens":256,"milliseconds":10000} for _ in range(3)],
            "mtp":[{"valid":True,"tokens":256,"milliseconds":7500,"acceptedDraftTokens":100} for _ in range(3)]}


def test_mtp_is_recommended_only_for_matching_valid_paired_measurements():
    measured=evidence()
    assert recommend_mtp(measured,measured["signature"]) == {"recommendation":"mtp","speedup":1.333}
    assert recommend_mtp(measured,"different-hardware-or-context")["recommendation"] == "benchmark-required"
    for bad in (None, {}, {**measured,"mtp":measured["mtp"][:1]}):
        assert recommend_mtp(bad,measured["signature"])["recommendation"] == "benchmark-required"
    for key,value in (("valid",False),("tokens",10),("acceptedDraftTokens",0),("milliseconds",float("nan"))):
        invalid=deepcopy(measured)
        invalid["mtp"][0][key]=value
        assert recommend_mtp(invalid,measured["signature"])["recommendation"] == "benchmark-required"


def test_slower_or_negligible_mtp_gain_keeps_baseline():
    measured=evidence()
    for duration in (12000,9900):
        for sample in measured["mtp"]: sample["milliseconds"]=duration
        assert recommend_mtp(measured,measured["signature"])["recommendation"] == "baseline"


def test_catalog_mtp_contract_survives_model_payload(tmp_path):
    from performance_oracle import build_models_payload
    from models import ModelLibraryResponse
    raw={"id":"qwen3.8-27b-iq4-xs","name":"Qwen 3.8 27B","gguf_file":"qwen.gguf",
         "size_mb":14253,"vram_required_gb":18,"context_length":32768,"max_context_length":262144,
         "mtp":{"mode":"embedded","source_url":"https://unsloth.ai/docs/models/mtp"}}
    payload=build_models_payload(None,None,0,tmp_path,catalog=[raw],evidence=[])
    encoded=ModelLibraryResponse(**payload).model_dump()
    value=encoded["models"][0]["metadata"]["mtp"]
    assert value["modelSupport"] == "publisher-declared"
    assert value["runtimeCheckRequired"] is True
    assert value["defaultEnabled"] is False


def memory_evidence(launch_mode='native', gpu_layers='auto', isolation='isolatedFromRuntime'):
    profile = {'modelSha256':'a'*64,'runtimeSha256':'b'*64,'hardware':'AMD-Radeon-RX-9070-XT-Vulkan',
        'context':16384,'draftTokens':2,'cacheType':'q4_0','parallel':1,'gpuLayers':gpu_layers,'flashAttention':'on',
        'launchMode':launch_mode,'visionProjectorSha256':'c'*64}
    signature = hashlib.sha256(json.dumps(profile,sort_keys=True).encode()).hexdigest()
    qualified = {'profile':profile,'signature':signature}
    execution = {'qualificationSignature':signature,'runtimeMode':launch_mode,'backend':'vulkan','gpuLayers':gpu_layers,
        'context':16384,'draftTokens':2,'cacheType':'q4_0','visionProjectorSha256':'c'*64,'visionProjectorFile':'mmproj-F16.gguf'}
    execution['signature'] = hashlib.sha256(json.dumps(execution,sort_keys=True).encode()).hexdigest()
    hardware = {'name':'AMD Radeon RX 9070 XT','backend':'amd','memoryTotalMB':16188,'systemRamGB':61}
    measured = {**evidence(), 'signature':signature,'profile':profile,'status':'completed','execution':execution,
        'conditions':{isolation:True,'visionProjectorLoaded':True},'hardware':hardware,
        'memorySnapshots':[{'mode':mode,**hardware,'memoryUsedMB':15000} for mode in ('baseline','mtp')]}
    return qualified, measured


def test_only_actual_native_memory_configuration_qualifies():
    qualified, measured = memory_evidence()
    fit = qualify_memory_fit(measured, qualified)
    assert fit['contextLength'] == 16384
    assert fit['observedGpuMemoryMB'] == 15000
    assert fit['runtimeBackend'] == 'vulkan'
    assert fit['runtimeMode'] == 'native'
    assert fit['gpuLayers'] == 'auto'
    for change in ('missing-projector','changed-hardware','failed','text-only','other-offload','missing-drafts',
                   'shared-runtime'):
        invalid = deepcopy(measured)
        if change == 'missing-projector': invalid['execution']['visionProjectorSha256'] = None
        if change == 'changed-hardware': invalid['hardware']['name'] = 'Unrelated GPU'
        if change == 'failed': invalid['status'] = 'failed'
        if change == 'text-only': invalid['conditions']['visionProjectorLoaded'] = False
        if change == 'other-offload': invalid['execution']['gpuLayers'] = '99'
        if change == 'missing-drafts': invalid['mtp'][0]['acceptedDraftTokens'] = 0
        if change == 'shared-runtime': invalid['conditions']['isolatedFromRuntime'] = False
        with pytest.raises(ValueError):
            qualify_memory_fit(invalid, qualified)


def test_memory_evidence_from_the_lemonade_launch_never_qualifies():
    """R6: a complete, self-consistent Lemonade-era record is refused."""
    qualified, measured = memory_evidence(launch_mode='lemonade', gpu_layers='99', isolation='isolatedFromLemonade')
    with pytest.raises(ValueError, match='native launch'):
        qualify_memory_fit(measured, qualified)


def test_verified_profile_exposes_separate_availability_without_claiming_gpu_fit(tmp_path, monkeypatch):
    from performance_oracle import build_models_payload
    from models import GPUInfo, ModelLibraryResponse
    qualified, measured = memory_evidence()
    fit = qualify_memory_fit(measured, qualified)
    data, external = tmp_path/'data', tmp_path/'external'
    data.mkdir()
    external.mkdir()
    model = external/'model.gguf'
    model.write_bytes(b'fixture')
    projector = external/'mmproj-F16.gguf'
    projector.write_bytes(b'vision')
    fit.update(visionProjectorSize=projector.stat().st_size, visionProjectorMtimeNs=projector.stat().st_mtime_ns)
    profile = {'modelSha256':'a'*64,'runtimeSha256':'b'*64,'qualificationSignature':qualified['signature'],
        'contextLength':16384,'draftTokens':2,'backend':'vulkan','mtp':True,'memoryQualification':fit}
    (data/'model-stores.json').write_text(json.dumps({'schemaVersion':1,'stores':[{'id':'ssd','hostPath':str(external),
        'containerPath':'/model-stores/ssd','profiles':{'model.gguf':profile}}]}))
    env = ('LLM_BACKEND=llama-server\nAMD_INFERENCE_LOCATION=host\n'
           'AMD_INFERENCE_RUNTIME_MODE=windows-native-llama-server\nSYSTEM_RAM_GB=61\n')
    (tmp_path/'.env').write_text(env)
    monkeypatch.setattr('performance_oracle.inspect_gguf', lambda path:{'readable':True,'metadata':{},'context_length':262144})
    gpu = GPUInfo(name='AMD Radeon RX 9070 XT',memory_total_mb=16188,memory_used_mb=1000,memory_percent=6,
        utilization_percent=0,temperature_c=0,gpu_backend='amd')
    catalog = [{'id':'candidate','name':'27B','gguf_file':'model.gguf','gguf_sha256':'a'*64,'size_mb':14253,
        'vram_required_gb':18,'context_length':32768,'max_context_length':262144}]
    def result():
        return ModelLibraryResponse(**build_models_payload(gpu,None,0,tmp_path,catalog=catalog,evidence=[])).model_dump()['models'][0]
    value = result()
    assert value['fitsVram'] is False
    assert value['activationSupport']['available'] is True
    assert value['activationSupport']['contextLength'] == value['contextLength'] == 16384
    (tmp_path/'.env').write_text(env.replace('SYSTEM_RAM_GB=61','SYSTEM_RAM_GB=16'))
    assert result()['activationSupport'] is None
    (tmp_path/'.env').write_text(env.replace('windows-native-llama-server','linux-container'))
    assert result()['activationSupport'] is None
    # The fit was measured with automatic offload; a pinned layer count is
    # a different launch.
    (tmp_path/'.env').write_text(env + 'N_GPU_LAYERS=99\n')
    assert result()['activationSupport'] is None
    # A Lemonade-era host never earns native activation support.
    (tmp_path/'.env').write_text(env.replace('LLM_BACKEND=llama-server','LLM_BACKEND=lemonade'))
    assert result()['activationSupport'] is None
    (tmp_path/'.env').write_text(env)
    projector.write_bytes(b'different vision artifact')
    assert result()['activationSupport'] is None