Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: turn train on completions back on when leaving CPT (#12308)

Nilay340e1d2e0a9b
diff --git a/studio/frontend/src/features/studio/wizard/advanced-settings-summary.ts b/studio/frontend/src/features/studio/wizard/advanced-settings-summary.ts
index 13b542dc5ab..b3689dfd6e3 100644
--- a/studio/frontend/src/features/studio/wizard/advanced-settings-summary.ts
+++ b/studio/frontend/src/features/studio/wizard/advanced-settings-summary.ts
@@ -62,7 +62,10 @@ function countNonDefaultScalarSettings(
): number {
return Object.entries(SCALAR_DEFAULTS).reduce((total, [rawKey, value]) => {
const key = rawKey as keyof typeof SCALAR_DEFAULTS;
- const expected = baselineValue(baseline, key, value);
+ const expected =
+ key === "trainOnCompletions" && state.trainingMethod === "cpt"
+ ? false
+ : baselineValue(baseline, key, value);
return state[key] === expected ? total : total + 1;
}, 0);
}
diff --git a/studio/frontend/src/features/training/stores/training-config-persistence.ts b/studio/frontend/src/features/training/stores/training-config-persistence.ts
index 6f7f7caaac4..6879a3b736e 100644
--- a/studio/frontend/src/features/training/stores/training-config-persistence.ts
+++ b/studio/frontend/src/features/training/stores/training-config-persistence.ts
@@ -228,6 +228,7 @@ function migrateThroughVersion19(
loraRankBeforeCpt: null,
loraAlphaBeforeCpt: null,
loraVariantBeforeCpt: null,
+ trainOnCompletionsBeforeCpt: null,
} satisfies TrainingMethodProvenance;
}
}
@@ -358,6 +359,10 @@ function normalizeTrainingMethodProvenance(
wasCpt && isLoraVariant(provenance.loraVariantBeforeCpt)
? provenance.loraVariantBeforeCpt
: null,
+ trainOnCompletionsBeforeCpt:
+ wasCpt && typeof provenance.trainOnCompletionsBeforeCpt === "boolean"
+ ? provenance.trainOnCompletionsBeforeCpt
+ : null,
};
}
diff --git a/studio/frontend/src/features/training/stores/training-config-policy.ts b/studio/frontend/src/features/training/stores/training-config-policy.ts
index 816f9366001..57e756df1e2 100644
--- a/studio/frontend/src/features/training/stores/training-config-policy.ts
+++ b/studio/frontend/src/features/training/stores/training-config-policy.ts
@@ -69,6 +69,7 @@ export const initialTrainingConfigState: TrainingConfigState = {
loraRankBeforeCpt: null,
loraAlphaBeforeCpt: null,
loraVariantBeforeCpt: null,
+ trainOnCompletionsBeforeCpt: null,
},
datasetSource: "huggingface",
browseDatasetSelection: createHfBrowseDatasetSelection(null),
diff --git a/studio/frontend/src/features/training/stores/training-config-store.ts b/studio/frontend/src/features/training/stores/training-config-store.ts
index cf410cfa61f..445ebde7f5b 100644
--- a/studio/frontend/src/features/training/stores/training-config-store.ts
+++ b/studio/frontend/src/features/training/stores/training-config-store.ts
@@ -78,6 +78,9 @@ let _modelConfigController: AbortController | null = null;
// Has the user manually toggled trainOnCompletions since the last auto-set?
let _trainOnCompletionsManuallySet = false;
+// Model whose completions value came from the user (toggle or config import), not
+// its defaults; CPT entry captures that value even while the defaults are pending.
+let _trainOnCompletionsExplicitModel: string | null = null;
let _trainingMethodEditGeneration = 0;
let _modelDefaultsEditGeneration = 0;
@@ -228,6 +231,7 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
requestedTargetModulesEditGeneration;
if (shouldApplyTrainingDefaults) {
_trainOnCompletionsManuallySet = false;
+ _trainOnCompletionsExplicitModel = null;
}
if (modelDetails.is_lora) {
@@ -322,12 +326,10 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
const cptTargetOverrides = shouldApplyCptTargetDefaults
? { targetModules: cptDefaultsPatch.targetModules }
: {};
- // Only trainOnCompletions: CPT's forced adapter values are not the model's.
// Targets are pinned to what cptDefaultsPatch resolved FROM, so the summary's
// resolveCptTargetModules(baseline) reproduces the live set even when the model
// config carries none and cptTargetModules falls back to live state.
const cptBaselineOverride = {
- trainOnCompletions: cptDefaultsPatch.trainOnCompletions,
targetModules: [...cptTargetModules],
};
const modelDefaultsBaseline = {
@@ -383,9 +385,17 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
: {}),
}
: {};
+ const cptCompletionProvenanceRefresh =
+ inCpt && modelDefaultsPatch.trainOnCompletions !== undefined
+ ? {
+ trainOnCompletionsBeforeCpt:
+ modelDefaultsPatch.trainOnCompletions,
+ }
+ : {};
const cptProvenanceRefresh = {
...cptTargetProvenanceRefresh,
...cptLoraProvenanceRefresh,
+ ...cptCompletionProvenanceRefresh,
};
const cptFallbackProvenanceRefresh = {
...(shouldApplyCptTargetDefaults
@@ -394,6 +404,16 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
...(requestedSelectionOwnsLoraSnapshot
? cptLoraProvenanceRefresh
: {}),
+ // Fill only: a value captured on entering CPT is the user's, not a default.
+ ...(requestedSelectionOwnsLoraSnapshot &&
+ !(
+ _trainOnCompletionsManuallySet &&
+ _trainOnCompletionsExplicitModel === modelName
+ ) &&
+ get().trainingMethodProvenance.trainOnCompletionsBeforeCpt ===
+ null
+ ? cptCompletionProvenanceRefresh
+ : {}),
};
set({
@@ -593,6 +613,12 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
if (isAudioModel && isVisionModel && isAudio) {
updates.trainOnCompletions = false;
}
+ if (updates.trainOnCompletions === false) {
+ updates.trainingMethodProvenance = {
+ ...current.trainingMethodProvenance,
+ trainOnCompletionsBeforeCpt: false,
+ };
+ }
}
set(updates);
if (streamingDisabled) {
@@ -852,6 +878,7 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
modelDefaultsAppliedFor?: string | null;
advancedSettingsBaseline?: null;
trainOnCompletionsDefaultPendingFor?: null;
+ trainingMethodProvenance?: TrainingConfigState["trainingMethodProvenance"];
} = {
selectedModel,
modelDefaultsError: null,
@@ -878,6 +905,10 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
patch.modelDefaultsAppliedFor = null;
patch.advancedSettingsBaseline = null;
patch.trainOnCompletionsDefaultPendingFor = null;
+ patch.trainingMethodProvenance = {
+ ...currentState.trainingMethodProvenance,
+ trainOnCompletionsBeforeCpt: null,
+ };
}
setUserEdit(patch);
@@ -1023,6 +1054,16 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
_trainingMethodEditGeneration += 1;
const state = get();
const patch = buildTrainingMethodPatch(state, trainingMethod);
+ if (
+ state.trainingMethod !== "cpt" &&
+ trainingMethod === "cpt" &&
+ state.selectedModel !== null &&
+ _trainOnCompletionsExplicitModel === state.selectedModel &&
+ patch.trainingMethodProvenance
+ ) {
+ patch.trainingMethodProvenance.trainOnCompletionsBeforeCpt =
+ state.trainOnCompletions;
+ }
setUserEdit({
...patch,
...(patch.trainOnCompletions !== undefined
@@ -1345,6 +1386,7 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
setPacking: (packing) => setUserEdit({ packing }),
setTrainOnCompletions: (trainOnCompletions) => {
_trainOnCompletionsManuallySet = true;
+ _trainOnCompletionsExplicitModel = get().selectedModel;
setUserEdit({
trainOnCompletions,
trainOnCompletionsDefaultPendingFor: null,
@@ -1377,6 +1419,7 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
reset: () => {
trainingDatasetCacheRejections.reset();
_trainOnCompletionsManuallySet = false;
+ _trainOnCompletionsExplicitModel = null;
_targetModulesEditGeneration += 1;
for (const key of LORA_PARAM_KEYS) {
_loraParamEditGenerations[key] += 1;
@@ -1402,20 +1445,36 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
for (const key of LORA_PARAM_KEYS) {
if (patch[key] !== undefined) _loraParamEditGenerations[key] += 1;
}
- setUserEdit((state) => ({
- ...patch,
- ...(patch.trainOnCompletions !== undefined
- ? { trainOnCompletionsDefaultPendingFor: null }
- : {}),
- ...(patch.learningRate !== undefined
- ? {
- trainingMethodProvenance: {
- ...state.trainingMethodProvenance,
- learningRateManuallySet: false,
- },
- }
- : {}),
- }));
+ if (patch.trainOnCompletions !== undefined) {
+ _trainOnCompletionsExplicitModel = get().selectedModel;
+ }
+ setUserEdit((state) => {
+ const importsCompletionsInCpt =
+ state.trainingMethod === "cpt" &&
+ patch.trainOnCompletions !== undefined;
+ return {
+ ...patch,
+ ...(patch.trainOnCompletions !== undefined
+ ? { trainOnCompletionsDefaultPendingFor: null }
+ : {}),
+ ...(patch.learningRate !== undefined || importsCompletionsInCpt
+ ? {
+ trainingMethodProvenance: {
+ ...state.trainingMethodProvenance,
+ ...(patch.learningRate !== undefined
+ ? { learningRateManuallySet: false }
+ : {}),
+ ...(importsCompletionsInCpt
+ ? {
+ trainOnCompletionsBeforeCpt:
+ patch.trainOnCompletions,
+ }
+ : {}),
+ },
+ }
+ : {}),
+ };
+ });
},
};
},
diff --git a/studio/frontend/src/features/training/stores/training-method-transition.ts b/studio/frontend/src/features/training/stores/training-method-transition.ts
index 8bfc77e8ec0..ae4c5e472fa 100644
--- a/studio/frontend/src/features/training/stores/training-method-transition.ts
+++ b/studio/frontend/src/features/training/stores/training-method-transition.ts
@@ -93,6 +93,26 @@ function resolveTrainingMethodLearningRate(
: LR_DEFAULT_FULL;
}
+// Re-checked at exit: modality or streaming learned inside CPT still vetoes a saved true.
+function completionsAllowedOnExit(
+ state: Pick<
+ TrainingConfigState,
+ | "datasetStreaming"
+ | "isEmbeddingModel"
+ | "isVisionModel"
+ | "isAudioModel"
+ | "isDatasetImage"
+ | "isDatasetAudio"
+ >,
+): boolean {
+ if (state.datasetStreaming || state.isEmbeddingModel) return false;
+ if (state.isVisionModel && state.isDatasetImage === true) return false;
+ if (state.isAudioModel && (!state.isVisionModel || state.isDatasetAudio)) {
+ return false;
+ }
+ return true;
+}
+
export function buildTrainingMethodPatch(
state: Pick<
TrainingConfigState,
@@ -103,6 +123,15 @@ export function buildTrainingMethodPatch(
| "loraRank"
| "loraAlpha"
| "loraVariant"
+ | "trainOnCompletions"
+ | "datasetStreaming"
+ | "selectedModel"
+ | "modelDefaultsAppliedFor"
+ | "isEmbeddingModel"
+ | "isVisionModel"
+ | "isAudioModel"
+ | "isDatasetImage"
+ | "isDatasetAudio"
>,
nextMethod: TrainingMethod,
): TrainingMethodStatePatch {
@@ -120,18 +149,29 @@ export function buildTrainingMethodPatch(
provenance.loraRankBeforeCpt = state.loraRank;
provenance.loraAlphaBeforeCpt = state.loraAlpha;
provenance.loraVariantBeforeCpt = state.loraVariant;
+ provenance.trainOnCompletionsBeforeCpt =
+ state.modelDefaultsAppliedFor === state.selectedModel
+ ? state.trainOnCompletions
+ : null;
Object.assign(patch, getCptTrainingPatch(state.targetModules));
}
if (prevMethod === "cpt" && nextMethod !== "cpt") {
Object.assign(patch, getRestoreFromCptPatch(provenance));
if (provenance.datasetFormatBeforeCpt !== null) {
patch.datasetFormat = provenance.datasetFormatBeforeCpt;
+ if (
+ provenance.trainOnCompletionsBeforeCpt &&
+ completionsAllowedOnExit(state)
+ ) {
+ patch.trainOnCompletions = true;
+ }
}
provenance.datasetFormatBeforeCpt = null;
provenance.targetModulesBeforeCpt = null;
provenance.loraRankBeforeCpt = null;
provenance.loraAlphaBeforeCpt = null;
provenance.loraVariantBeforeCpt = null;
+ provenance.trainOnCompletionsBeforeCpt = null;
}
const learningRate = resolveTrainingMethodLearningRate(
diff --git a/studio/frontend/src/features/training/types/config.ts b/studio/frontend/src/features/training/types/config.ts
index 24327702711..21f6f150cc5 100644
--- a/studio/frontend/src/features/training/types/config.ts
+++ b/studio/frontend/src/features/training/types/config.ts
@@ -53,6 +53,7 @@ export interface TrainingMethodProvenance {
loraRankBeforeCpt: number | null;
loraAlphaBeforeCpt: number | null;
loraVariantBeforeCpt: LoraVariant | null;
+ trainOnCompletionsBeforeCpt: boolean | null;
}
/** Column-to-role mapping, e.g. { "problem": "user", "solution": "assistant", "context": "system" } */
diff --git a/studio/frontend/tests/training-config-persistence.test.ts b/studio/frontend/tests/training-config-persistence.test.ts
index f16d0ae4dbe..e4c343dd41a 100644
--- a/studio/frontend/tests/training-config-persistence.test.ts
+++ b/studio/frontend/tests/training-config-persistence.test.ts
@@ -81,6 +81,7 @@ test("migration preserves tuned values while protecting them from model defaults
loraRankBeforeCpt: null,
loraAlphaBeforeCpt: null,
loraVariantBeforeCpt: null,
+ trainOnCompletionsBeforeCpt: null,
});
assert.equal("wandbToken" in migrated, false);
});
diff --git a/studio/frontend/tests/training-cpt-model-switch.test.ts b/studio/frontend/tests/training-cpt-model-switch.test.ts
index 0d85d241437..279a74dabac 100644
--- a/studio/frontend/tests/training-cpt-model-switch.test.ts
+++ b/studio/frontend/tests/training-cpt-model-switch.test.ts
@@ -86,6 +86,7 @@ test("leaving CPT after a model switch restores the new model targets", async ()
loraRankBeforeCpt: null,
loraAlphaBeforeCpt: null,
loraVariantBeforeCpt: null,
+ trainOnCompletionsBeforeCpt: null,
},
});
@@ -798,6 +799,7 @@ test("a reload inside CPT keeps the pre-CPT LoRA params the session saved", asyn
loraRankBeforeCpt: 64,
loraAlphaBeforeCpt: 64,
loraVariantBeforeCpt: "dora",
+ trainOnCompletionsBeforeCpt: null,
},
});
@@ -824,3 +826,468 @@ test("a reload inside CPT keeps the pre-CPT LoRA params the session saved", asyn
assert.equal(state.loraAlpha, 64);
assert.equal(state.loraVariant, "dora");
});
+
+test("leaving CPT after a model switch does not restore the old model's train on completions", async () => {
+ useTrainingConfigStore.getState().reset();
+ const defaultsWith = (id: string, trainOnCompletions: boolean) => () =>
+ Promise.resolve(
+ Response.json({
+ id,
+ config: { training: { train_on_completions: trainOnCompletions } },
+ is_vision: false,
+ is_embedding: false,
+ is_audio: false,
+ audio_type_known: true,
+ is_lora: false,
+ model_type: "text",
+ model_size_bytes: null,
+ max_position_embeddings: 32768,
+ }),
+ );
+ setAuthFetchHandler(defaultsWith("old/llama", true));
+ useTrainingConfigStore.getState().selectTrainingModel("old/llama", "text");
+ await waitForModelDefaults("old/llama");
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, true);
+
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ setAuthFetchHandler(defaultsWith("new/model", false));
+ useTrainingConfigStore.getState().selectTrainingModel("new/model", "text");
+ await waitForModelDefaults("new/model");
+
+ useTrainingConfigStore.getState().setTrainingMethod("qlora");
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, false);
+});
+
+test("leaving CPT after switching to a model whose default is on turns train on completions back on", async () => {
+ const defaultsWith = (id: string, trainOnCompletions: boolean) => () =>
+ Promise.resolve(
+ Response.json({
+ id,
+ config: { training: { train_on_completions: trainOnCompletions } },
+ is_vision: false,
+ is_embedding: false,
+ is_audio: false,
+ audio_type_known: true,
+ is_lora: false,
+ model_type: "text",
+ model_size_bytes: null,
+ max_position_embeddings: 32768,
+ }),
+ );
+ for (const oldDefault of [false, true]) {
+ useTrainingConfigStore.getState().reset();
+ setAuthFetchHandler(defaultsWith("old/llama", oldDefault));
+ useTrainingConfigStore.getState().selectTrainingModel("old/llama", "text");
+ await waitForModelDefaults("old/llama");
+
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ setAuthFetchHandler(defaultsWith("new/model", true));
+ useTrainingConfigStore.getState().selectTrainingModel("new/model", "text");
+ await waitForModelDefaults("new/model");
+
+ useTrainingConfigStore.getState().setTrainingMethod("qlora");
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, true);
+ }
+});
+
+test("leaving CPT while the new model's defaults are loading does not restore the old model's train on completions", async () => {
+ useTrainingConfigStore.getState().reset();
+ setAuthFetchHandler(() =>
+ Response.json({
+ id: "old/llama",
+ config: { training: { train_on_completions: true } },
+ is_vision: false,
+ is_embedding: false,
+ is_audio: false,
+ audio_type_known: true,
+ is_lora: false,
+ model_type: "text",
+ model_size_bytes: null,
+ max_position_embeddings: 32768,
+ }),
+ );
+ useTrainingConfigStore.getState().selectTrainingModel("old/llama", "text");
+ await waitForModelDefaults("old/llama");
+
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ setAuthFetchHandler(() => new Promise<Response>(() => {}));
+ useTrainingConfigStore.getState().selectTrainingModel("new/audio", "audio");
+ useTrainingConfigStore.getState().setTrainingMethod("qlora");
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, false);
+});
+
+test("an image dataset picked inside CPT keeps train on completions off for a vision model", async () => {
+ useTrainingConfigStore.getState().reset();
+ setAuthFetchHandler((input, init) => {
+ if (input === "/api/hub/datasets/check-format") {
+ const { dataset_name } = JSON.parse(String(init?.body));
+ return Response.json({
+ columns: ["messages"],
+ detected_format: "chatml",
+ is_audio: false,
+ is_image: dataset_name === "org/images",
+ requires_manual_mapping: false,
+ });
+ }
+ return Response.json({
+ id: "org/vision",
+ config: { training: { train_on_completions: true } },
+ is_vision: true,
+ is_embedding: false,
+ is_audio: false,
+ audio_type_known: true,
+ is_lora: false,
+ model_type: "vision",
+ model_size_bytes: null,
+ max_position_embeddings: 32768,
+ });
+ });
+ const datasetChecked = async (): Promise<void> => {
+ for (let attempt = 0; attempt < 100; attempt += 1) {
+ if (!useTrainingConfigStore.getState().isCheckingDataset) return;
+ await new Promise((resolve) => setTimeout(resolve, 5));
+ }
+ throw new Error("dataset check did not settle");
+ };
+ useTrainingConfigStore.getState().selectTrainingModel("org/vision", "vision");
+ await waitForModelDefaults("org/vision");
+ useTrainingConfigStore.getState().setDataset("org/text");
+ await datasetChecked();
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, true);
+
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ useTrainingConfigStore.getState().setDataset("org/images");
+ await datasetChecked();
+ assert.equal(useTrainingConfigStore.getState().isDatasetImage, true);
+
+ useTrainingConfigStore.getState().setTrainingMethod("qlora");
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, false);
+});
+
+const completionDefaults = (id: string, isAudio: boolean) =>
+ Response.json({
+ id,
+ config: { training: { train_on_completions: true } },
+ is_vision: false,
+ is_embedding: false,
+ is_audio: isAudio,
+ audio_type_known: true,
+ is_lora: false,
+ model_type: isAudio ? "audio" : "text",
+ model_size_bytes: null,
+ max_position_embeddings: 32768,
+ });
+
+function deferModelDefaults(): (response: Response) => void {
+ let resolveDefaults!: (response: Response) => void;
+ setAuthFetchHandler(
+ () =>
+ new Promise<Response>((resolve) => {
+ resolveDefaults = resolve;
+ }),
+ );
+ return (response) => resolveDefaults(response);
+}
+
+test("entering CPT before the new model's defaults load restores that model's train on completions", async () => {
+ for (const isAudio of [true, false]) {
+ useTrainingConfigStore.getState().reset();
+ setAuthFetchHandler(() => completionDefaults("old/llama", false));
+ useTrainingConfigStore.getState().selectTrainingModel("old/llama", "text");
+ await waitForModelDefaults("old/llama");
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, true);
+
+ const resolveDefaults = deferModelDefaults();
+ useTrainingConfigStore
+ .getState()
+ .selectTrainingModel("new/model", isAudio ? "audio" : "text");
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ resolveDefaults(completionDefaults("new/model", isAudio));
+ await waitForModelDefaults("new/model");
+
+ useTrainingConfigStore.getState().setTrainingMethod("qlora");
+ assert.equal(
+ useTrainingConfigStore.getState().trainOnCompletions,
+ !isAudio,
+ );
+ }
+});
+
+test("an edit while the new model's defaults load inside CPT still restores its train on completions", async () => {
+ useTrainingConfigStore.getState().reset();
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ const resolveDefaults = deferModelDefaults();
+ useTrainingConfigStore.getState().selectTrainingModel("new/model", "text");
+ useTrainingConfigStore.getState().setBatchSize(8);
+ resolveDefaults(completionDefaults("new/model", false));
+ await waitForModelDefaults("new/model");
+
+ useTrainingConfigStore.getState().setTrainingMethod("qlora");
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, true);
+});
+
+test("restored train on completions is not counted as a modified setting", async () => {
+ const defaults = () =>
+ Response.json({
+ id: "org/chat",
+ config: {
+ lora: { target_modules: [...LLAMA_TARGETS] },
+ training: { train_on_completions: true },
+ },
+ is_vision: false,
+ is_embedding: false,
+ is_audio: false,
+ audio_type_known: true,
+ is_lora: false,
+ model_type: "text",
+ model_size_bytes: null,
+ max_position_embeddings: 32768,
+ });
+ const count = () => {
+ const state = useTrainingConfigStore.getState();
+ return countNonDefaultAdvancedSettings(
+ state,
+ state.advancedSettingsBaseline,
+ );
+ };
+ for (const chooseInsideCpt of [false, true]) {
+ useTrainingConfigStore.getState().reset();
+ setAuthFetchHandler(defaults);
+ if (chooseInsideCpt) {
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ }
+ useTrainingConfigStore.getState().selectTrainingModel("org/chat", "text");
+ await waitForModelDefaults("org/chat");
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ assert.equal(count(), 0);
+
+ useTrainingConfigStore.getState().setTrainingMethod("qlora");
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, true);
+ assert.equal(count(), 0);
+ }
+});
+
+test("a cache refetch inside CPT keeps the train on completions the user turned off", async () => {
+ useTrainingConfigStore.getState().reset();
+ setAuthFetchHandler((input) => {
+ if (input === "/api/hub/datasets/check-format") {
+ return Response.json({
+ columns: ["messages"],
+ detected_format: "chatml",
+ is_audio: false,
+ is_image: false,
+ requires_manual_mapping: false,
+ });
+ }
+ return completionDefaults("org/chat", false);
+ });
+ useTrainingConfigStore.getState().selectTrainingModel("org/chat", "text");
+ await waitForModelDefaults("org/chat");
+ useTrainingConfigStore.getState().setTrainOnCompletions(false);
+ // A dataset pick clears the manual-toggle flag.
+ useTrainingConfigStore.getState().setDataset("org/text");
+ for (let attempt = 0; attempt < 100; attempt += 1) {
+ if (!useTrainingConfigStore.getState().isCheckingDataset) break;
+ await new Promise((resolve) => setTimeout(resolve, 5));
+ }
+
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ useTrainingConfigStore.getState().setSelectedModelCacheReference("org/chat", {
+ localPath: "/cache/org/chat",
+ modelFormat: null,
+ });
+ await waitForModelDefaults("org/chat");
+
+ useTrainingConfigStore.getState().setTrainingMethod("qlora");
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, false);
+});
+
+test("a config imported inside CPT decides train on completions after leaving", async () => {
+ for (const imported of [false, true]) {
+ useTrainingConfigStore.getState().reset();
+ setAuthFetchHandler(() => completionDefaults("org/chat", false));
+ useTrainingConfigStore.getState().selectTrainingModel("org/chat", "text");
+ await waitForModelDefaults("org/chat");
+ useTrainingConfigStore.getState().setTrainOnCompletions(!imported);
+
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ useTrainingConfigStore
+ .getState()
+ .applyConfigPatch({ training: { train_on_completions: imported } });
+ useTrainingConfigStore.getState().setTrainingMethod("qlora");
+ assert.equal(
+ useTrainingConfigStore.getState().trainOnCompletions,
+ imported,
+ );
+ }
+});
+
+const completionModel = (id: string, isVision: boolean) =>
+ Response.json({
+ id,
+ config: { training: { train_on_completions: true } },
+ is_vision: isVision,
+ is_embedding: false,
+ is_audio: false,
+ audio_type_known: true,
+ is_lora: false,
+ model_type: isVision ? "vision" : "text",
+ model_size_bytes: null,
+ max_position_embeddings: 32768,
+ });
+
+const datasetCheck = (isImage: boolean) =>
+ Response.json({
+ columns: ["messages"],
+ detected_format: "chatml",
+ is_audio: false,
+ is_image: isImage,
+ requires_manual_mapping: false,
+ });
+
+async function settleStore(): Promise<void> {
+ for (let attempt = 0; attempt < 200; attempt += 1) {
+ const state = useTrainingConfigStore.getState();
+ if (
+ !state.isLoadingModelDefaults &&
+ !state.isCheckingDataset &&
+ state.modelDefaultsAppliedFor === state.selectedModel
+ ) {
+ return;
+ }
+ await new Promise((resolve) => setTimeout(resolve, 5));
+ }
+ throw new Error("store did not settle");
+}
+
+test("a model found to be vision inside CPT keeps completions off for an image dataset", async () => {
+ useTrainingConfigStore.getState().reset();
+ let isVision = false;
+ setAuthFetchHandler((input) =>
+ input === "/api/hub/datasets/check-format"
+ ? datasetCheck(true)
+ : completionModel("org/model", isVision),
+ );
+ useTrainingConfigStore.getState().selectTrainingModel("org/model", "text");
+ await settleStore();
+ useTrainingConfigStore.getState().setDataset("org/images");
+ await settleStore();
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, true);
+
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ isVision = true;
+ useTrainingConfigStore
+ .getState()
+ .setSelectedModelCacheReference("org/model", {
+ localPath: "/cache/org/model",
+ modelFormat: null,
+ });
+ await settleStore();
+ assert.equal(useTrainingConfigStore.getState().isVisionModel, true);
+
+ useTrainingConfigStore.getState().setTrainingMethod("qlora");
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, false);
+});
+
+test("an opt-out made while defaults load survives a CPT round trip", async () => {
+ useTrainingConfigStore.getState().reset();
+ let resolveDefaults!: (response: Response) => void;
+ setAuthFetchHandler((input) =>
+ input === "/api/hub/datasets/check-format"
+ ? datasetCheck(false)
+ : new Promise<Response>((resolve) => {
+ resolveDefaults = resolve;
+ }),
+ );
+ useTrainingConfigStore.getState().selectTrainingModel("org/model", "text");
+ useTrainingConfigStore.getState().setTrainOnCompletions(false);
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ useTrainingConfigStore.getState().setDataset("org/text");
+ await new Promise((resolve) => setTimeout(resolve, 5));
+ resolveDefaults(completionModel("org/model", false));
+ await settleStore();
+
+ useTrainingConfigStore.getState().setTrainingMethod("qlora");
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, false);
+});
+
+test("an image split picked inside CPT keeps completions off after manual toggles", async () => {
+ useTrainingConfigStore.getState().reset();
+ setAuthFetchHandler((input, init) =>
+ input === "/api/hub/datasets/check-format"
+ ? datasetCheck(JSON.parse(String(init?.body)).train_split === "images")
+ : completionModel("org/vision", true),
+ );
+ useTrainingConfigStore.getState().selectTrainingModel("org/vision", "vision");
+ await settleStore();
+ useTrainingConfigStore.getState().setDataset("org/mixed");
+ await settleStore();
+ useTrainingConfigStore.getState().setTrainOnCompletions(false);
+ useTrainingConfigStore.getState().setTrainOnCompletions(true);
+
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ useTrainingConfigStore.getState().setDatasetSplit("images");
+ await settleStore();
+ assert.equal(useTrainingConfigStore.getState().isDatasetImage, true);
+
+ useTrainingConfigStore.getState().setTrainingMethod("qlora");
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, false);
+});
+
+test("a manual toggle on the previous model does not decide the next model's CPT restore", async () => {
+ useTrainingConfigStore.getState().reset();
+ setAuthFetchHandler(() => completionModel("old/model", false));
+ useTrainingConfigStore.getState().selectTrainingModel("old/model", "text");
+ await settleStore();
+ useTrainingConfigStore.getState().setTrainOnCompletions(true);
+
+ let resolveDefaults!: (response: Response) => void;
+ setAuthFetchHandler(
+ () =>
+ new Promise<Response>((resolve) => {
+ resolveDefaults = resolve;
+ }),
+ );
+ useTrainingConfigStore.getState().selectTrainingModel("new/model", "text");
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ useTrainingConfigStore.getState().setBatchSize(8);
+ resolveDefaults(
+ Response.json({
+ id: "new/model",
+ config: { training: { train_on_completions: false } },
+ is_vision: false,
+ is_embedding: false,
+ is_audio: false,
+ audio_type_known: true,
+ is_lora: false,
+ model_type: "text",
+ model_size_bytes: null,
+ max_position_embeddings: 32768,
+ }),
+ );
+ await settleStore();
+
+ useTrainingConfigStore.getState().setTrainingMethod("qlora");
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, false);
+});
+
+test("a config imported while defaults load survives entering and leaving CPT", async () => {
+ useTrainingConfigStore.getState().reset();
+ let resolveDefaults!: (response: Response) => void;
+ setAuthFetchHandler(
+ () =>
+ new Promise<Response>((resolve) => {
+ resolveDefaults = resolve;
+ }),
+ );
+ useTrainingConfigStore.getState().selectTrainingModel("org/model", "text");
+ useTrainingConfigStore
+ .getState()
+ .applyConfigPatch({ training: { train_on_completions: false } });
+ useTrainingConfigStore.getState().setTrainingMethod("cpt");
+ resolveDefaults(completionModel("org/model", false));
+ await settleStore();
+
+ useTrainingConfigStore.getState().setTrainingMethod("qlora");
+ assert.equal(useTrainingConfigStore.getState().trainOnCompletions, false);
+});
diff --git a/studio/frontend/tests/training-method-provenance.test.ts b/studio/frontend/tests/training-method-provenance.test.ts
index a4855b48301..833e5f81fed 100644
--- a/studio/frontend/tests/training-method-provenance.test.ts
+++ b/studio/frontend/tests/training-method-provenance.test.ts
@@ -27,6 +27,7 @@ test("rehydrated CPT provenance survives the next method switch", () => {
modelAdapterLearningRate: 0.00001,
datasetFormatBeforeCpt: "sharegpt",
targetModulesBeforeCpt: null,
+ trainOnCompletionsBeforeCpt: true,
},
});
const rehydrated = mergeTrainingConfig(
@@ -41,6 +42,7 @@ test("rehydrated CPT provenance survives the next method switch", () => {
assert.equal(state.trainingMethod, "qlora");
assert.equal(state.learningRate, 0.000031);
assert.equal(state.datasetFormat, "sharegpt");
+ assert.equal(state.trainOnCompletions, true);
assert.equal(state.trainingMethodProvenance.datasetFormatBeforeCpt, null);
});
@@ -99,6 +101,7 @@ test("switching away from CPT restores pre-CPT target modules", () => {
loraRankBeforeCpt: null,
loraAlphaBeforeCpt: null,
loraVariantBeforeCpt: null,
+ trainOnCompletionsBeforeCpt: null,
},
};
const patch = buildTrainingMethodPatch(state, "qlora");
@@ -131,3 +134,26 @@ test("switching away from CPT restores the pre-CPT LoRA rank, alpha and variant"
assert.equal(restored.loraVariant, "lora");
assert.equal(restored.trainingMethodProvenance.loraRankBeforeCpt, null);
});
+
+test("switching away from CPT turns train on completions back on", () => {
+ const state = {
+ ...initialTrainingConfigState,
+ trainingMethod: "qlora" as const,
+ datasetFormat: "auto" as const,
+ trainOnCompletions: true,
+ };
+ const cptState = { ...state, ...buildTrainingMethodPatch(state, "cpt") };
+
+ assert.equal(cptState.trainOnCompletions, false);
+
+ const restored = {
+ ...cptState,
+ ...buildTrainingMethodPatch(cptState, "qlora"),
+ };
+
+ assert.equal(restored.trainOnCompletions, true);
+ assert.equal(
+ restored.trainingMethodProvenance.trainOnCompletionsBeforeCpt,
+ null,
+ );
+});