Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(cli): serialize token file updates (#44329)

mika3adc33981460
diff --git a/cli/src/store/local-file.ts b/cli/src/store/local-file.ts
index 1b250f609e5..c39152f1d07 100644
--- a/cli/src/store/local-file.ts
+++ b/cli/src/store/local-file.ts
@@ -1,7 +1,7 @@
import type { z } from 'zod'
import { isRecord } from '@/util/is-record'
import { BadYamlFormatError } from './errors'
-import { YamlStore } from './store'
+import { loadYaml, YamlStore } from './store'
export type Notice = (text: string) => void
@@ -31,20 +31,30 @@ function remove(doc: Record<string, unknown>, path: Path): void {
else if (isRecord(parent)) delete parent[last as string]
}
-async function readTolerant<S extends z.ZodType>(
- store: YamlStore,
+function readTolerant<S extends z.ZodType>(
+ content: string | undefined,
schema: S,
+ path: string,
notice: Notice,
-): Promise<z.output<S> | null> {
- const path = store.filePath
+): z.output<S> | null {
let raw: unknown
try {
- raw = await store.getTyped<unknown>()
+ raw = loadYaml(content, path)
} catch (err) {
if (!(err instanceof BadYamlFormatError)) throw err
notice(`${path} is not valid YAML; ignoring it`)
return null
}
+ if (raw === null) return null
+ return parseTolerant(raw, path, schema, notice)
+}
+
+function parseTolerant<S extends z.ZodType>(
+ raw: unknown,
+ path: string,
+ schema: S,
+ notice: Notice,
+): z.output<S> | null {
if (raw === null) return null
if (!isRecord(raw)) {
notice(`${path} is not a YAML mapping; ignoring it`)
@@ -70,6 +80,7 @@ export type LocalFile<T> = Readonly<{
path: string
read: () => Promise<T | null>
write: (doc: T) => Promise<void>
+ update: (transform: (doc: T | null) => T | undefined | Promise<T | undefined>) => Promise<void>
remove: () => Promise<void>
/** Remove the file only while it still holds a document `matches` accepts. */
removeIf: (matches: (doc: T) => boolean) => Promise<void>
@@ -84,8 +95,14 @@ export function localFile<S extends z.ZodType>(
const store = new YamlStore(path)
return {
path,
- read: () => readTolerant(store, schema, notice),
+ read: async () => readTolerant(await store.getRaw(), schema, path, notice),
write: async (doc) => store.setTyped(schema.parse(doc)),
+ update: async (transform) => {
+ await store.update(async (content) => {
+ const next = await transform(readTolerant(content, schema, path, notice))
+ return next === undefined ? undefined : schema.parse(next)
+ })
+ },
remove: () => store.rm(),
removeIf: (matches) =>
store.rmIf((doc) => {
diff --git a/cli/src/store/store.ts b/cli/src/store/store.ts
index 172783d4653..76da5315a39 100644
--- a/cli/src/store/store.ts
+++ b/cli/src/store/store.ts
@@ -132,6 +132,26 @@ export class YamlStore {
return this.rawContent
}
+ /** Read, transform, and replace a document under one lock for read-modify-write operations. */
+ async update<T>(
+ update: (rawContent: string | undefined) => T | undefined | Promise<T | undefined>,
+ ): Promise<void> {
+ await this.withLock(async () => {
+ await this.load()
+ const next = await update(this.rawContent)
+ if (next === undefined) return
+ this.setRawContent(dump(next, { lineWidth: -1, noRefs: true }))
+ await this.flush()
+ })
+ }
+
+ async getRaw(): Promise<string | undefined> {
+ return this.withLock(async () => {
+ await this.load()
+ return this.rawContent
+ })
+ }
+
private async withLock<R>(body: () => R | Promise<R>): Promise<R> {
await this.lock()
try {
@@ -179,7 +199,10 @@ export class YamlStore {
}
}
-function loadYaml(raw: string | undefined, file_path: string): Record<string, unknown> | null {
+export function loadYaml(
+ raw: string | undefined,
+ file_path: string,
+): Record<string, unknown> | null {
if (raw === undefined) return null
try {
const documents = loadAll(raw, { schema: YAML_LOAD_SCHEMA })
diff --git a/cli/src/store/token-store.test.ts b/cli/src/store/token-store.test.ts
index 6efd2ed0dc6..ea3dbb9bee4 100644
--- a/cli/src/store/token-store.test.ts
+++ b/cli/src/store/token-store.test.ts
@@ -48,6 +48,60 @@ describe('FileTokenStore', () => {
expect(await s.read('https://self.example.com', '[email protected]')).toBe('C')
})
+ it('preserves concurrent writes from separate stores', async () => {
+ let updateCount = 0
+ let releaseFirstUpdate!: () => void
+ let signalFirstUpdate!: () => void
+ let signalSecondRequest!: () => void
+ const firstUpdateStarted = new Promise<void>((resolve) => {
+ signalFirstUpdate = resolve
+ })
+ const secondUpdateRequested = new Promise<void>((resolve) => {
+ signalSecondRequest = resolve
+ })
+ const firstUpdateGate = new Promise<void>((resolve) => {
+ releaseFirstUpdate = resolve
+ })
+ let secondTransformEntered = false
+
+ function openWithSynchronizedUpdate(): FileTokenStore {
+ const source = localFile(file, TOKENS_SCHEMA, ioService(streams).notice)
+ return new FileTokenStore({
+ ...source,
+ update: async (transform) => {
+ updateCount += 1
+ if (updateCount === 2) signalSecondRequest()
+ await source.update(async (doc) => {
+ if (updateCount === 1) {
+ signalFirstUpdate()
+ await firstUpdateGate
+ } else {
+ secondTransformEntered = true
+ }
+ return transform(doc)
+ })
+ },
+ })
+ }
+
+ const first = openWithSynchronizedUpdate()
+ const second = openWithSynchronizedUpdate()
+ const firstWrite = first.write('https://one.example.com', '[email protected]', 'A')
+ await firstUpdateStarted
+ const secondWrite = second.write('https://two.example.com', '[email protected]', 'B')
+ await secondUpdateRequested
+ await new Promise<void>((resolve) => setImmediate(resolve))
+ const secondWaitedForLock = !secondTransformEntered
+ releaseFirstUpdate()
+ await Promise.all([firstWrite, secondWrite])
+
+ expect(secondWaitedForLock).toBe(true)
+
+ const result = open()
+ expect(await result.read('https://one.example.com', '[email protected]')).toBe('A')
+ expect(await result.read('https://two.example.com', '[email protected]')).toBe('B')
+ })
+
it('persists the nested shape on disk', async () => {
const s = open()
await s.write('https://cloud.dify.ai', '[email protected]', 'A')
diff --git a/cli/src/store/token-store.ts b/cli/src/store/token-store.ts
index 2728a02e895..0fe1fcfd69c 100644
--- a/cli/src/store/token-store.ts
+++ b/cli/src/store/token-store.ts
@@ -36,18 +36,26 @@ export class FileTokenStore implements TokenStore {
}
async write(host: string, email: string, bearer: string): Promise<void> {
- const doc = (await this.file.read()) ?? { tokens: {} }
- doc.tokens[host] = { ...doc.tokens[host], [email]: bearer }
- await this.file.write(doc)
+ await this.file.update((doc) => {
+ const tokens = doc?.tokens ?? {}
+ return {
+ tokens: {
+ ...tokens,
+ [host]: { ...tokens[host], [email]: bearer },
+ },
+ }
+ })
}
async remove(host: string, email: string): Promise<void> {
- const doc = await this.file.read()
- const hostMap = doc?.tokens[host]
- if (doc === null || hostMap === undefined || !(email in hostMap)) return
- delete hostMap[email]
- if (Object.keys(hostMap).length === 0) delete doc.tokens[host]
- await this.file.write(doc)
+ await this.file.update((doc) => {
+ if (doc === null) return undefined
+ const hostMap = doc.tokens[host]
+ if (hostMap === undefined || !(email in hostMap)) return undefined
+ delete hostMap[email]
+ if (Object.keys(hostMap).length === 0) delete doc.tokens[host]
+ return doc
+ })
}
}