Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(e2e): seed workspace defaults and refresh post-merge filters (#44412)

yyh322d049c511b
diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml
index 4b00687fb6e..adb0aa6d581 100644
--- a/.github/workflows/post-merge.yml
+++ b/.github/workflows/post-merge.yml
@@ -1,6 +1,7 @@
name: Post-Merge Checks
on:
+ workflow_dispatch:
push:
branches: ['main']
@@ -52,6 +53,12 @@ jobs:
- 'api/tests/integration_tests/.env.example'
- 'api/clients/agent_backend/**'
- 'api/core/app/apps/agent_app/**'
+ - 'api/core/model_manager.py'
+ - 'api/core/provider_manager.py'
+ - 'api/controllers/console/workspace/models.py'
+ - 'api/controllers/console/workspace/model_providers.py'
+ - 'api/services/model_provider/**'
+ - 'api/tasks/install_default_plugins_task.py'
- 'api/core/workflow/nodes/agent_v2/**'
- 'api/controllers/console/agent/**'
- 'api/controllers/console/app/agent*.py'
@@ -70,6 +77,12 @@ jobs:
- 'api/services/entities/agent_tool_inner.py'
- 'api/services/errors/agent_tool_inner.py'
- 'api/services/agent/**'
+ - 'api/services/app/agent_*.py'
+ - 'api/repositories/app/agent_*.py'
+ - 'api/extensions/application_services/agent.py'
+ - 'api/models/agent.py'
+ - 'api/models/agent_config_entities.py'
+ - 'api/services/entities/agent_entities.py'
- 'api/core/plugin/**'
- 'api/services/plugin/**'
- 'api/core/tools/**'
@@ -112,7 +125,7 @@ jobs:
external-e2e:
name: External Runtime E2E
needs: check-changes
- if: needs.check-changes.outputs.external-e2e-changed == 'true'
+ if: github.event_name == 'workflow_dispatch' || needs.check-changes.outputs.external-e2e-changed == 'true'
permissions:
actions: read
contents: read
diff --git a/e2e/features/agent-v2/configure-persistence.feature b/e2e/features/agent-v2/configure-persistence.feature
index ca17f8ee67c..ab8bf34340e 100644
--- a/e2e/features/agent-v2/configure-persistence.feature
+++ b/e2e/features/agent-v2/configure-persistence.feature
@@ -3,6 +3,7 @@ Feature: Agent v2 configure persistence
@prepared @configure-persistence
Scenario: A new Agent shows the workspace default model in Configure
Given I am signed in as the default E2E admin
+ And the workspace default chat model is configured
And an Agent v2 test agent has been created via API
When I open the Agent v2 configure page
Then I should see the workspace default chat model in the Agent v2 model selector
diff --git a/e2e/features/agent-v2/support/seed.ts b/e2e/features/agent-v2/support/seed.ts
index 25a9be453ba..b5b50f32d8b 100644
--- a/e2e/features/agent-v2/support/seed.ts
+++ b/e2e/features/agent-v2/support/seed.ts
@@ -4,9 +4,10 @@ import type {
AgentSoulDifyToolConfig,
} from '@dify/contracts/api/console/agent/types.gen'
import type { KnowledgeConfig } from '@dify/contracts/api/console/datasets/types.gen'
-import type { ModelType } from '@dify/contracts/api/console/workspaces/types.gen'
+import type { ModelType, ParserPostDefault } from '@dify/contracts/api/console/workspaces/types.gen'
import type { SeedContext, SeedResource, SeedTask } from '../../../support/seed.ts'
import { readFile } from 'node:fs/promises'
+import { zModelType } from '@dify/contracts/api/console/workspaces/zod.gen'
import { createTestApp } from '../../../support/api/apps.ts'
import { bootstrapMarketplacePlugins } from '../../../support/marketplace-plugins.ts'
import { sleep } from '../../../support/process.ts'
@@ -305,67 +306,87 @@ const seedAgentDecisionModel = async (context: SeedContext) =>
title: agentBuilderPreseededResources.agentDecisionChatModel,
})
-const seedSpeechToTextModel = async (context: SeedContext) => {
- const config = speechToTextModelConfig()
- const title = agentBuilderPreseededResources.speechToTextModel
- const modelResult = await seedModel(context, { config, title })
- if (modelResult.status === 'blocked' || modelResult.status === 'skipped') return modelResult
+const seedSpeechToTextModel = async (context: SeedContext) =>
+ seedModel(context, {
+ config: speechToTextModelConfig(),
+ title: agentBuilderPreseededResources.speechToTextModel,
+ })
- const model = await findModel(context.consoleClient, config)
- if (!model || model.status !== activeModelStatus)
- return blocked(title, `${config.provider}/${config.name} is not active after model setup.`)
+const readWorkspaceDefaultModels = async (context: SeedContext) =>
+ new Map(
+ await Promise.all(
+ zModelType.options.map(async (modelType) => {
+ const response = await context.consoleClient.workspaces.current.defaultModel.get({
+ query: { model_type: modelType },
+ })
+ return [modelType, response.data] as const
+ }),
+ ),
+ )
- const resource = {
- id: `${model.provider}/${model.name}`,
- kind: 'model',
- name: title,
- }
- const defaultModelResponse = await context.consoleClient.workspaces.current.defaultModel.get({
- query: { model_type: config.type },
- })
- const defaultModel = defaultModelResponse.data
- const isExpectedDefault =
- defaultModel?.model === model.name &&
- matchesProvider(defaultModel.provider.provider, model.provider)
+const seedWorkspaceDefaultModels = (includeSpeechToText: boolean): SeedTask => ({
+ id: 'workspace-default-models',
+ title: 'Workspace default models',
+ run: async (context) => {
+ const title = 'Workspace default models'
+ const chatModel = stableModelConfig()
+ if (chatModel.type !== 'llm')
+ return blocked(
+ title,
+ 'E2E_STABLE_MODEL_TYPE must be llm for the workspace default chat model.',
+ )
- if (isExpectedDefault)
- return modelResult.status === 'updated' ? modelResult : verified(title, resource)
+ const configs = includeSpeechToText ? [chatModel, speechToTextModelConfig()] : [chatModel]
+ const defaults = await readWorkspaceDefaultModels(context)
+ const settings = new Map<ModelType, ParserPostDefault['model_settings'][number]>()
+ for (const [modelType, model] of defaults) {
+ if (model)
+ settings.set(modelType, {
+ model: model.model,
+ model_type: modelType,
+ provider: model.provider.provider,
+ })
+ }
- if (context.dryRun)
- return skipped(
- title,
- `Would set ${model.provider}/${model.name} as the workspace default Speech-to-Text model.`,
- )
+ let needsUpdate = false
+ for (const config of configs) {
+ const model = await findModel(context.consoleClient, config)
+ if (!model || model.status !== activeModelStatus) {
+ const reason = `${config.provider}/${config.name} is not active after model setup.`
+ return context.dryRun ? skipped(title, reason) : blocked(title, reason)
+ }
+
+ const current = settings.get(config.type)
+ if (current?.model !== model.name || current.provider !== model.provider) needsUpdate = true
+ settings.set(config.type, {
+ model: model.name,
+ model_type: config.type,
+ provider: model.provider,
+ })
+ }
- await context.consoleClient.workspaces.current.defaultModel.post({
- body: {
- model_settings: [
- {
- model: model.name,
- model_type: config.type,
- provider: model.provider,
- },
- ],
- },
- })
+ if (!needsUpdate) return verified(title)
+ if (context.dryRun)
+ return skipped(title, 'Would configure the seeded workspace default models.')
- const updatedDefaultModelResponse =
- await context.consoleClient.workspaces.current.defaultModel.get({
- query: { model_type: config.type },
+ // Saving defaults replaces every type, so retain defaults outside this seed profile.
+ await context.consoleClient.workspaces.current.defaultModel.post({
+ body: { model_settings: [...settings.values()] },
})
- const updatedDefaultModel = updatedDefaultModelResponse.data
- if (
- updatedDefaultModel?.model !== model.name ||
- !matchesProvider(updatedDefaultModel.provider.provider, model.provider)
- ) {
- return blocked(
- title,
- `${model.provider}/${model.name} was not selected as the workspace default Speech-to-Text model.`,
- )
- }
- return updated(title, resource)
-}
+ const savedDefaults = await readWorkspaceDefaultModels(context)
+ for (const [modelType, setting] of settings) {
+ const saved = savedDefaults.get(modelType)
+ if (saved?.model !== setting.model || saved?.provider.provider !== setting.provider)
+ return blocked(
+ title,
+ `Workspace default ${modelType} was not saved as ${setting.provider}/${setting.model}.`,
+ )
+ }
+
+ return updated(title)
+ },
+})
const findBuiltinTool = async (client: SeedContext['consoleClient'], displayName: string) => {
const parsed = splitToolDisplayName(displayName)
@@ -849,6 +870,7 @@ const agentV2PreparedFixtureSeedTasks = (): SeedTask[] => [
const agentV2PreparedSeedTasks = (): SeedTask[] => [
...agentV2BaseSeedTasks(),
+ seedWorkspaceDefaultModels(false),
...agentV2PreparedFixtureSeedTasks(),
]
@@ -859,6 +881,7 @@ const agentV2ExternalRuntimeSeedTasks = (): SeedTask[] => [
title: agentBuilderPreseededResources.speechToTextModel,
run: seedSpeechToTextModel,
},
+ seedWorkspaceDefaultModels(true),
]
const agentV2PostMergeSeedTasks = (): SeedTask[] => [
diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts
index bce268be4f0..df8bb7a3b8f 100644
--- a/e2e/features/step-definitions/agent-v2/configure.steps.ts
+++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts
@@ -299,12 +299,11 @@ Then(
Then(
'I should see the workspace default chat model in the Agent v2 model selector',
async function (this: DifyWorld) {
- const response = await this.getConsoleClient().workspaces.current.defaultModel.get({
- query: { model_type: 'llm' },
- })
- if (!response.data) throw new Error('The workspace default chat model is not configured.')
+ const defaultModel = this.agentBuilder.fixtures.defaultChatModel
+ if (!defaultModel)
+ throw new Error('Resolve the workspace default chat model before creating the Agent.')
- await expect(getAgentModelSelector(this.getPage())).toContainText(response.data.model, {
+ await expect(getAgentModelSelector(this.getPage())).toContainText(defaultModel.model, {
timeout: 30_000,
})
},
diff --git a/e2e/features/step-definitions/agent-v2/fixtures.steps.ts b/e2e/features/step-definitions/agent-v2/fixtures.steps.ts
index 56a1b5d3b7b..026f68e5bd0 100644
--- a/e2e/features/step-definitions/agent-v2/fixtures.steps.ts
+++ b/e2e/features/step-definitions/agent-v2/fixtures.steps.ts
@@ -23,6 +23,18 @@ Given('the Agent Builder stable chat model is available', async function (this:
this.agentBuilder.fixtures.stableModel = stableModel
})
+Given('the workspace default chat model is configured', async function (this: DifyWorld) {
+ const response = await this.getConsoleClient().workspaces.current.defaultModel.get({
+ query: { model_type: 'llm' },
+ })
+ if (!response.data)
+ throw new Error(
+ 'The workspace default chat model is not configured. Run the prepared seed first.',
+ )
+
+ this.agentBuilder.fixtures.defaultChatModel = response.data
+})
+
Given('the workspace default speech-to-text model is active', async function (this: DifyWorld) {
const speechToTextModel = await requireAgentBuilderSpeechToTextModel(
this,
diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts
index 664aec6883b..5aebbac292b 100644
--- a/e2e/features/support/world.ts
+++ b/e2e/features/support/world.ts
@@ -1,4 +1,5 @@
import type { IWorldOptions } from '@cucumber/cucumber'
+import type { DefaultModelResponse } from '@dify/contracts/api/console/workspaces/types.gen'
import type { APIRequestContext, Browser, BrowserContext, Download, Page } from '@playwright/test'
import type { AuthSessionMetadata } from '../../fixtures/auth.ts'
import type { ConsoleClient } from '../../support/api/console-client.ts'
@@ -51,6 +52,7 @@ export type WorkflowPreviewFixture = {
export const createAgentBuilderWorldState = () => ({
fixtures: {
agentDecisionModel: undefined as AgentBuilderChatModel | undefined,
+ defaultChatModel: undefined as DefaultModelResponse | undefined,
preseededResources: {} as Record<string, AgentBuilderPreseededResource>,
speechToTextModel: undefined as AgentBuilderChatModel | undefined,
stableModel: undefined as AgentBuilderChatModel | undefined,
diff --git a/e2e/tests/agent-v2-seed.test.ts b/e2e/tests/agent-v2-seed.test.ts
index 8cadc1f9d79..f76c8c2e547 100644
--- a/e2e/tests/agent-v2-seed.test.ts
+++ b/e2e/tests/agent-v2-seed.test.ts
@@ -21,13 +21,18 @@ const taskIds = (profile: string) => createAgentV2SeedTasks(profile).map((task)
describe('Agent v2 seed profiles', () => {
it('prepares every fixture required by prepared behavior', () => {
- expect(taskIds('prepared')).toEqual([...baseTaskIds, ...preparedFixtureTaskIds])
+ expect(taskIds('prepared')).toEqual([
+ ...baseTaskIds,
+ 'workspace-default-models',
+ ...preparedFixtureTaskIds,
+ ])
})
it('prepares external runtime and prepared behavior in post-merge', () => {
expect(taskIds('post-merge')).toEqual([
...baseTaskIds,
'speech-to-text-model',
+ 'workspace-default-models',
...preparedFixtureTaskIds,
])
})
diff --git a/e2e/tests/workspace-default-model-seed.test.ts b/e2e/tests/workspace-default-model-seed.test.ts
new file mode 100644
index 00000000000..d602999852f
--- /dev/null
+++ b/e2e/tests/workspace-default-model-seed.test.ts
@@ -0,0 +1,212 @@
+import type {
+ AvailableModelListResponse,
+ DefaultModelResponse,
+ ModelType,
+ ParserPostDefault,
+ ProviderModelWithStatusEntity,
+} from '@dify/contracts/api/console/workspaces/types.gen'
+import type { ConsoleClient } from '../support/api/console-client.ts'
+import { consoleRouterContract } from '@dify/contracts/api/console/router.gen'
+import {
+ zModelType,
+ zPostWorkspacesCurrentDefaultModelBody,
+} from '@dify/contracts/api/console/workspaces/zod.gen'
+import { createORPCClient } from '@orpc/client'
+import { RequestValidationPlugin, ResponseValidationPlugin } from '@orpc/contract/plugins'
+import { OpenAPILink } from '@orpc/openapi-client/fetch'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'
+import { createAgentV2SeedTasks } from '../features/agent-v2/support/seed.ts'
+
+const provider = 'langgenius/openai/openai'
+const tenantId = '00000000-0000-4000-8000-000000000001'
+
+const defaultModel = (modelType: ModelType, model: string): DefaultModelResponse => ({
+ model,
+ model_type: modelType,
+ provider: {
+ label: { en_US: 'OpenAI', zh_Hans: 'OpenAI' },
+ provider,
+ supported_model_types: [modelType],
+ tenant_id: tenantId,
+ },
+})
+
+const availableModel = (modelType: ModelType, model: string): ProviderModelWithStatusEntity => ({
+ fetch_from: 'predefined-model',
+ label: { en_US: model, zh_Hans: model },
+ model,
+ model_properties: {},
+ model_type: modelType,
+ status: 'active',
+})
+
+const createSeedApi = ({
+ initialDefaults = [],
+ models = [
+ availableModel('llm', 'gpt-5-nano'),
+ availableModel('speech2text', 'gpt-4o-mini-transcribe'),
+ ],
+ persist = true,
+}: {
+ initialDefaults?: DefaultModelResponse[]
+ models?: ProviderModelWithStatusEntity[]
+ persist?: boolean
+} = {}) => {
+ const defaults = new Map(initialDefaults.map((model) => [model.model_type, model]))
+ const writes: ParserPostDefault[] = []
+ const link = new OpenAPILink(consoleRouterContract, {
+ url: 'http://seed.test/console/api',
+ plugins: [
+ new RequestValidationPlugin(consoleRouterContract),
+ new ResponseValidationPlugin(consoleRouterContract),
+ ],
+ fetch: async (request) => {
+ const url = new URL(request.url)
+ if (url.pathname === '/console/api/workspaces/current/default-model') {
+ if (request.method === 'GET') {
+ const modelType = zModelType.parse(url.searchParams.get('model_type'))
+ return Response.json({ data: defaults.get(modelType) ?? null })
+ }
+ if (request.method === 'POST') {
+ const body = zPostWorkspacesCurrentDefaultModelBody.parse(await request.json())
+ writes.push(body)
+ if (persist) {
+ // Match the API's replacement semantics, including deletion of omitted types.
+ defaults.clear()
+ for (const setting of body.model_settings) {
+ if (!setting.model || !setting.provider) continue
+ const model = defaultModel(setting.model_type, setting.model)
+ model.provider.provider = setting.provider
+ defaults.set(setting.model_type, model)
+ }
+ }
+ return Response.json({ result: 'success' })
+ }
+ }
+ const modelTypesPath = '/console/api/workspaces/current/models/model-types/'
+ if (request.method === 'GET' && url.pathname.startsWith(modelTypesPath)) {
+ const modelType = zModelType.parse(url.pathname.slice(modelTypesPath.length))
+ const response: AvailableModelListResponse = {
+ data: [
+ {
+ label: { en_US: 'OpenAI', zh_Hans: 'OpenAI' },
+ models: models.filter((model) => model.model_type === modelType),
+ provider,
+ status: 'active',
+ tenant_id: tenantId,
+ },
+ ],
+ }
+ return Response.json(response)
+ }
+ throw new Error(`Unexpected seed request: ${request.method} ${url.pathname}`)
+ },
+ })
+ const consoleClient = createORPCClient<ConsoleClient>(link)
+ const run = (profile = 'post-merge', dryRun = false) => {
+ const task = createAgentV2SeedTasks(profile).find(
+ (task) => task.id === 'workspace-default-models',
+ )
+ if (!task) throw new Error(`No workspace default model seed in ${profile}`)
+ return task.run({ consoleClient, dryRun, resources: new Map() })
+ }
+ return { defaults, run, writes }
+}
+
+describe('workspace default model seed', () => {
+ beforeEach(() => {
+ vi.stubEnv('E2E_STABLE_MODEL_NAME', 'gpt-5-nano')
+ vi.stubEnv('E2E_STABLE_MODEL_PROVIDER', 'openai')
+ vi.stubEnv('E2E_STABLE_MODEL_TYPE', 'llm')
+ vi.stubEnv('E2E_SPEECH_TO_TEXT_MODEL_NAME', 'gpt-4o-mini-transcribe')
+ vi.stubEnv('E2E_SPEECH_TO_TEXT_MODEL_PROVIDER', 'openai')
+ })
+
+ afterEach(() => vi.unstubAllEnvs())
+
+ it.each(['post-merge', 'external-runtime'])(
+ 'saves chat and speech defaults together for %s and is idempotent',
+ async (profile) => {
+ const api = createSeedApi({
+ initialDefaults: [defaultModel('text-embedding', 'text-embedding-3-small')],
+ })
+
+ expect((await api.run(profile)).status).toBe('updated')
+ expect(api.writes).toHaveLength(1)
+ expect(api.defaults.get('llm')?.model).toBe('gpt-5-nano')
+ expect(api.defaults.get('speech2text')?.model).toBe('gpt-4o-mini-transcribe')
+ expect(api.defaults.get('text-embedding')?.model).toBe('text-embedding-3-small')
+ expect((await api.run(profile)).status).toBe('verified')
+ expect(api.writes).toHaveLength(1)
+ },
+ )
+
+ it('configures the prepared chat default without changing an existing speech default', async () => {
+ const api = createSeedApi({
+ initialDefaults: [
+ defaultModel('llm', 'old-chat-model'),
+ defaultModel('speech2text', 'whisper-1'),
+ ],
+ models: [availableModel('llm', 'gpt-5-nano')],
+ })
+
+ expect((await api.run('prepared')).status).toBe('updated')
+ expect(api.defaults.get('llm')?.model).toBe('gpt-5-nano')
+ expect(api.defaults.get('speech2text')?.model).toBe('whisper-1')
+ })
+
+ it('honors the configured stable model instead of assuming the first available model', async () => {
+ vi.stubEnv('E2E_STABLE_MODEL_NAME', 'gpt-5.5')
+ const api = createSeedApi({
+ models: [availableModel('llm', 'gpt-5-nano'), availableModel('llm', 'gpt-5.5')],
+ })
+
+ expect((await api.run('prepared')).status).toBe('updated')
+ expect(api.defaults.get('llm')?.model).toBe('gpt-5.5')
+ })
+
+ it.each([undefined, 'no-configure'] as const)(
+ 'blocks missing or inactive speech models before writing any defaults (%s)',
+ async (status) => {
+ const models = [availableModel('llm', 'gpt-5-nano')]
+ if (status)
+ models.push({ ...availableModel('speech2text', 'gpt-4o-mini-transcribe'), status })
+ const api = createSeedApi({ models })
+
+ expect((await api.run()).status).toBe('blocked')
+ expect(api.writes).toHaveLength(0)
+ },
+ )
+
+ it('does not mutate workspace defaults during a dry run', async () => {
+ const api = createSeedApi()
+
+ expect((await api.run('post-merge', true)).status).toBe('skipped')
+ expect(api.writes).toHaveLength(0)
+ expect(api.defaults.size).toBe(0)
+ })
+
+ it('fails readiness when a successful save does not persist the requested defaults', async () => {
+ const api = createSeedApi({ persist: false })
+
+ expect(await api.run()).toEqual(
+ expect.objectContaining({
+ reason: expect.stringContaining('was not saved'),
+ status: 'blocked',
+ }),
+ )
+ })
+
+ it('rejects a non-LLM stable model configuration', async () => {
+ vi.stubEnv('E2E_STABLE_MODEL_TYPE', 'text-embedding')
+ const api = createSeedApi()
+
+ expect(await api.run('prepared')).toEqual(
+ expect.objectContaining({
+ reason: expect.stringContaining('must be llm'),
+ status: 'blocked',
+ }),
+ )
+ expect(api.writes).toHaveLength(0)
+ })
+})