Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🌐 fix: Preserve Unicode Filenames (#12977)

Danny Avilaf2de3a219cf0
diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.js
index 285f783909..0e3273efe2 100644
--- a/api/server/routes/files/files.js
+++ b/api/server/routes/files/files.js
@@ -29,7 +29,7 @@ const { getOpenAIClient } = require('~/server/controllers/assistants/helpers');
const { hasCapability } = require('~/server/middleware/roles/capabilities');
const { checkPermission } = require('~/server/services/PermissionService');
const { hasAccessToFilesViaAgent } = require('~/server/services/Files');
-const { cleanFileName } = require('~/server/utils/files');
+const { getContentDisposition } = require('~/server/utils/files');
const { getLogStores } = require('~/cache');
const { Readable } = require('stream');
const db = require('~/models');
@@ -397,8 +397,7 @@ router.get('/download/:userId/:file_id', fileAccess, async (req, res) => {
}
const setHeaders = () => {
- const cleanedFilename = cleanFileName(file.filename);
- res.setHeader('Content-Disposition', `attachment; filename="${cleanedFilename}"`);
+ res.setHeader('Content-Disposition', getContentDisposition(file.filename));
res.setHeader('Content-Type', 'application/octet-stream');
res.setHeader('X-File-Metadata', JSON.stringify(file));
};
diff --git a/api/server/routes/files/preview.spec.js b/api/server/routes/files/preview.spec.js
index f86c128d4b..84790c1e5f 100644
--- a/api/server/routes/files/preview.spec.js
+++ b/api/server/routes/files/preview.spec.js
@@ -62,6 +62,7 @@ jest.mock('~/server/services/Files', () => ({
jest.mock('~/server/utils/files', () => ({
cleanFileName: (name) => name,
+ getContentDisposition: (name) => `attachment; filename="${name}"`,
}));
jest.mock('~/cache', () => ({
diff --git a/api/server/utils/files.js b/api/server/utils/files.js
index e4825b5ac8..3129c8ef6a 100644
--- a/api/server/utils/files.js
+++ b/api/server/utils/files.js
@@ -64,4 +64,32 @@ const cleanFileName = (fileName) => {
return cleaned;
};
-module.exports = { determineFileType, getBufferMetadata, cleanFileName };
+const encodeRFC5987ValueChars = (value) =>
+ encodeURIComponent(value).replace(
+ /['()*]/g,
+ (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`,
+ );
+
+const getAsciiFilenameFallback = (fileName) => {
+ const fallback = fileName
+ .normalize('NFKD')
+ .replace(/[^\x20-\x7e]/g, '_')
+ .replace(/["\\\r\n]/g, '_');
+
+ return fallback || 'download';
+};
+
+const getContentDisposition = (fileName, disposition = 'attachment') => {
+ const cleanedFilename = cleanFileName(fileName) || 'download';
+ const asciiFallback = getAsciiFilenameFallback(cleanedFilename);
+ const encodedFilename = encodeRFC5987ValueChars(cleanedFilename);
+
+ return `${disposition}; filename="${asciiFallback}"; filename*=UTF-8''${encodedFilename}`;
+};
+
+module.exports = {
+ determineFileType,
+ getBufferMetadata,
+ cleanFileName,
+ getContentDisposition,
+};
diff --git a/api/server/utils/files.spec.js b/api/server/utils/files.spec.js
new file mode 100644
index 0000000000..41332c682e
--- /dev/null
+++ b/api/server/utils/files.spec.js
@@ -0,0 +1,31 @@
+jest.mock('sharp', () => jest.fn(), { virtual: true });
+
+const { cleanFileName, getContentDisposition } = require('./files');
+
+describe('file utilities', () => {
+ describe('cleanFileName', () => {
+ it('removes storage UUID prefixes', () => {
+ expect(cleanFileName('123e4567-e89b-12d3-a456-426614174000__report.txt')).toBe('report.txt');
+ });
+ });
+
+ describe('getContentDisposition', () => {
+ it('adds RFC 8187 encoding for Unicode filenames', () => {
+ const filename = '日本語レポート.xlsx';
+ const header = getContentDisposition(`123e4567-e89b-12d3-a456-426614174000__${filename}`);
+
+ expect(header).toMatch(/^attachment; filename=".*"; filename\*=UTF-8''/);
+ expect(header).not.toContain('123e4567-e89b-12d3-a456-426614174000__');
+ expect(header).toContain(`filename*=UTF-8''${encodeURIComponent(filename)}`);
+ });
+
+ it('escapes the ASCII fallback without dropping the encoded filename', () => {
+ const filename = 'bad"name\r\n.txt';
+ const header = getContentDisposition(filename);
+
+ expect(header).toContain('filename="bad_name__.txt"');
+ expect(header).toContain("filename*=UTF-8''bad%22name%0D%0A.txt");
+ expect(header).not.toMatch(/[\r\n]/);
+ });
+ });
+});
diff --git a/packages/api/src/files/code/form.spec.ts b/packages/api/src/files/code/form.spec.ts
index 14f8d577b9..da8d2d815e 100644
--- a/packages/api/src/files/code/form.spec.ts
+++ b/packages/api/src/files/code/form.spec.ts
@@ -38,6 +38,13 @@ describe('code env FormData filenames', () => {
});
});
+ it('uses filepath for nested Unicode filenames', () => {
+ expect(getCodeEnvFileOptions('分析/結果📊.csv')).toEqual({
+ filename: '結果📊.csv',
+ filepath: '分析/結果📊.csv',
+ });
+ });
+
it('documents the form-data string overload regression', async () => {
const disposition = await renderMultipartDisposition((form) => {
form.append('file', Readable.from(['x']), 'pptx/pptx.py');
diff --git a/packages/api/src/files/code/form.ts b/packages/api/src/files/code/form.ts
index 1565721b64..6f9abab7d4 100644
--- a/packages/api/src/files/code/form.ts
+++ b/packages/api/src/files/code/form.ts
@@ -7,11 +7,30 @@ export interface CodeEnvFileOptions {
filepath?: string;
}
-const CODE_ENV_SAFE_FILEPATH_PATTERN = /^[a-zA-Z0-9._\-/]+$/;
+const CODE_ENV_SAFE_ASCII_FILEPATH_CHAR_PATTERN = /^[a-zA-Z0-9._\-/]$/;
+const CODE_ENV_UNSAFE_UNICODE_FILEPATH_CHAR_PATTERN =
+ /[^\p{L}\p{M}\p{N}\p{Emoji}\u200d._\-/]/u;
const CODE_ENV_FILENAME_CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
+function hasUnsafeCodeEnvFilepathChar(filepath: string): boolean {
+ for (const char of filepath) {
+ if (char.charCodeAt(0) <= 0x7f) {
+ if (!CODE_ENV_SAFE_ASCII_FILEPATH_CHAR_PATTERN.test(char)) {
+ return true;
+ }
+ continue;
+ }
+
+ if (CODE_ENV_UNSAFE_UNICODE_FILEPATH_CHAR_PATTERN.test(char)) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
function isSafeCodeEnvFilepath(filepath: string): boolean {
- if (!filepath || filepath.startsWith('/') || !CODE_ENV_SAFE_FILEPATH_PATTERN.test(filepath)) {
+ if (!filepath || filepath.startsWith('/') || hasUnsafeCodeEnvFilepathChar(filepath)) {
return false;
}
diff --git a/packages/api/src/utils/files.spec.ts b/packages/api/src/utils/files.spec.ts
index fc41b00a11..89fd3a35c2 100644
--- a/packages/api/src/utils/files.spec.ts
+++ b/packages/api/src/utils/files.spec.ts
@@ -22,6 +22,10 @@ function expectedHexSuffix(input: string): string {
return createHash('sha256').update(input).digest('hex').slice(0, 6);
}
+function utf8ByteLength(input: string): number {
+ return Buffer.byteLength(input, 'utf8');
+}
+
describe('sanitizeFilename', () => {
test('removes directory components (1/2)', () => {
expect(sanitizeFilename('/path/to/file.txt')).toBe('file.txt');
@@ -35,6 +39,17 @@ describe('sanitizeFilename', () => {
expect(sanitizeFilename('file name@#$.txt')).toBe('file_name___.txt');
});
+ test('preserves Unicode filenames', () => {
+ expect(sanitizeFilename('日本語レポート.xlsx')).toBe('日本語レポート.xlsx');
+ expect(sanitizeFilename('résumé-данные-تقرير-보고서📊.csv')).toBe(
+ 'résumé-данные-تقرير-보고서📊.csv',
+ );
+ });
+
+ test('normalizes decomposed Unicode marks before sanitizing', () => {
+ expect(sanitizeFilename('Cafe\u0301.txt')).toBe('Café.txt');
+ });
+
test('preserves dots and hyphens', () => {
expect(sanitizeFilename('file-name.with.dots.txt')).toBe('file-name.with.dots.txt');
});
@@ -50,6 +65,13 @@ describe('sanitizeFilename', () => {
expect(result).toMatch(/^a+-abc123\.txt$/);
});
+ test('truncates Unicode filenames by UTF-8 bytes, preserving the extension', () => {
+ const longName = '界'.repeat(100) + '.txt';
+ const result = sanitizeFilename(longName);
+ expect(utf8ByteLength(result)).toBeLessThanOrEqual(255);
+ expect(result.endsWith('-abc123.txt')).toBe(true);
+ });
+
test('handles filenames with no extension', () => {
const longName = 'a'.repeat(300);
const result = sanitizeFilename(longName);
@@ -75,6 +97,10 @@ describe('sanitizeArtifactPath', () => {
expect(sanitizeArtifactPath('a/b/c/file.txt')).toBe('a/b/c/file.txt');
});
+ test('preserves Unicode path segments', () => {
+ expect(sanitizeArtifactPath('分析/結果📊.csv')).toBe('分析/結果📊.csv');
+ });
+
test('replaces non-alphanumeric characters per segment + adds raw-input disambiguator', () => {
/* Different raw inputs that sanitize to the same form (`out 1.csv`
* vs `out_1.csv`, `[email protected]` vs `out#1.csv`) would otherwise share
@@ -146,6 +172,13 @@ describe('sanitizeArtifactPath', () => {
expect(result).toMatch(new RegExp(`^a+-${expectedHexSuffix(longName)}\\.txt$`));
});
+ test('caps Unicode leaf segments at 255 UTF-8 bytes with extension-preserving truncation', () => {
+ const longName = '界'.repeat(100) + '.txt';
+ const result = sanitizeArtifactPath(longName);
+ expect(utf8ByteLength(result)).toBeLessThanOrEqual(255);
+ expect(result.endsWith(`-${expectedHexSuffix(longName)}.txt`)).toBe(true);
+ });
+
test('caps the leaf when nested under a directory, preserving the directory verbatim', () => {
const longLeaf = 'b'.repeat(300) + '.csv';
const result = sanitizeArtifactPath(`reports/${longLeaf}`);
@@ -167,6 +200,15 @@ describe('sanitizeArtifactPath', () => {
expect(leaf).toBe('notes.txt');
});
+ test('caps Unicode non-leaf directory segments at 255 UTF-8 bytes', () => {
+ const longDir = '界'.repeat(100);
+ const result = sanitizeArtifactPath(`${longDir}/notes.txt`);
+ const [dir, leaf] = result.split('/');
+ expect(utf8ByteLength(dir)).toBeLessThanOrEqual(255);
+ expect(dir.endsWith(`-${expectedHexSuffix(longDir)}`)).toBe(true);
+ expect(leaf).toBe('notes.txt');
+ });
+
test('produces deterministic output across calls (no orphaned uploads on re-truncation)', () => {
/* Codex review P2: `sanitizeFilename`'s `crypto.randomBytes(3)` made
* the truncated form non-deterministic — re-uploading the same long
@@ -221,6 +263,14 @@ describe('sanitizeArtifactPath', () => {
expect(result).toBe('file.txt');
});
+ test('falls back to leaf-only when Unicode path bytes exceed the DB-index cap', () => {
+ const segA = '界'.repeat(80);
+ const segB = '分'.repeat(80);
+ const segC = '析'.repeat(80);
+ const result = sanitizeArtifactPath(`${segA}/${segB}/${segC}/file.txt`);
+ expect(result).toBe('file.txt');
+ });
+
test('keeps the nested path when total length is within the DB-index cap', () => {
/* The cap doesn't fire for realistic outputs — typical artifact
* depth is ≤ 3 segments × short names. */
@@ -276,6 +326,16 @@ describe('sanitizeArtifactPath', () => {
expect(a).toBe(`_.hidden-${expectedHexSuffix('.hidden')}`);
});
+ test('normalization-only collisions get distinct safe forms', () => {
+ const composed = 'reports/Café.csv';
+ const decomposed = 'reports/Cafe\u0301.csv';
+ const normalizedDecomposed = `reports/Café-${expectedHexSuffix(decomposed)}.csv`;
+
+ expect(sanitizeArtifactPath(composed)).toBe(composed);
+ expect(sanitizeArtifactPath(decomposed)).toBe(normalizedDecomposed);
+ expect(normalizedDecomposed).not.toBe(composed);
+ });
+
test('idempotent: same raw input always produces the same safe form', () => {
/* Disambiguator is deterministic (SHA-256 prefix of raw input),
* so re-uploading the same long-or-mutated name lands at the
@@ -372,6 +432,14 @@ describe('flattenArtifactPath', () => {
expect(result).toMatch(new RegExp(`-${expectedHexSuffix(safePath)}\\.txt$`));
});
+ test('truncates Unicode flat forms by UTF-8 bytes', () => {
+ const safePath = `${'界'.repeat(80)}/結果.csv`;
+ const result = flattenArtifactPath(safePath, 100);
+ expect(utf8ByteLength(result)).toBeLessThanOrEqual(100);
+ expect(result.endsWith('.csv')).toBe(true);
+ expect(result).toMatch(new RegExp(`-${expectedHexSuffix(safePath)}\\.csv$`));
+ });
+
test('preserves the extension even when only the leaf overflows', () => {
const longLeaf = 'L'.repeat(300);
const result = flattenArtifactPath(`${longLeaf}.json`, 200);
diff --git a/packages/api/src/utils/files.ts b/packages/api/src/utils/files.ts
index 37c993a3f3..e98797b97b 100644
--- a/packages/api/src/utils/files.ts
+++ b/packages/api/src/utils/files.ts
@@ -9,6 +9,56 @@ const USER_FACING_UPLOAD_ERRORS = [
'Unable to extract text from',
] as const;
+const ASCII_FILENAME_SAFE_PATTERN = /^[a-zA-Z0-9._-]$/;
+const UNSAFE_UNICODE_FILENAME_PATTERN = /[^\p{L}\p{M}\p{N}\p{Emoji}\u200d._-]/gu;
+const FILENAME_SEGMENT_MAX_BYTES = 255;
+
+function sanitizeFilenameSegment(segment: string): string {
+ return segment
+ .normalize('NFC')
+ .replace(/[\u0000-\u007f]/g, (char) => (ASCII_FILENAME_SAFE_PATTERN.test(char) ? char : '_'))
+ .replace(UNSAFE_UNICODE_FILENAME_PATTERN, '_');
+}
+
+function utf8ByteLength(value: string): number {
+ return Buffer.byteLength(value, 'utf8');
+}
+
+function truncateUtf8Bytes(value: string, maxBytes: number): string {
+ if (maxBytes <= 0) return '';
+ let bytes = 0;
+ let result = '';
+
+ for (const char of value) {
+ const charBytes = utf8ByteLength(char);
+ if (bytes + charBytes > maxBytes) break;
+ result += char;
+ bytes += charBytes;
+ }
+
+ return result;
+}
+
+function truncateWithSuffix(value: string, suffix: string, maxBytes: number): string {
+ const suffixBytes = utf8ByteLength(suffix);
+ const stemBudget = Math.max(0, maxBytes - suffixBytes);
+ const result = truncateUtf8Bytes(value, stemBudget) + suffix;
+ return utf8ByteLength(result) <= maxBytes ? result : truncateUtf8Bytes(result, maxBytes);
+}
+
+function truncateLeafWithSuffix(leaf: string, suffix: string, maxBytes: number): string {
+ if (utf8ByteLength(leaf) <= maxBytes) return leaf;
+ const ext = path.extname(leaf);
+ const stem = path.basename(leaf, ext);
+ const suffixBytes = utf8ByteLength(suffix);
+ const extBytes = utf8ByteLength(ext);
+ if (extBytes > maxBytes - suffixBytes - 1) {
+ return truncateWithSuffix(leaf, suffix, maxBytes);
+ }
+ const stemBudget = maxBytes - extBytes - suffixBytes;
+ return truncateUtf8Bytes(stem, stemBudget) + suffix + ext;
+}
+
/**
* Resolves a user-facing error message from a file upload error.
* Returns the error's own message if it matches a known user-facing pattern,
@@ -37,42 +87,37 @@ export function resolveUploadErrorMessage(
}
/**
- * Sanitize a filename by removing any directory components, replacing non-alphanumeric characters
+ * Sanitize a filename by removing any directory components, replacing unsafe characters
* @param inputName
*/
export function sanitizeFilename(inputName: string): string {
// Remove any directory components
let name = path.basename(inputName);
- // Replace any non-alphanumeric characters except for '.' and '-'
- name = name.replace(/[^a-zA-Z0-9.-]/g, '_');
+ // Preserve Unicode word characters and emoji while replacing unsafe ASCII punctuation.
+ name = sanitizeFilenameSegment(name);
// Ensure the name doesn't start with a dot (hidden file in Unix-like systems)
if (name.startsWith('.') || name === '') {
name = '_' + name;
}
- // Limit the length of the filename
- const MAX_LENGTH = 255;
- if (name.length > MAX_LENGTH) {
- const ext = path.extname(name);
- const nameWithoutExt = path.basename(name, ext);
- name =
- nameWithoutExt.slice(0, MAX_LENGTH - ext.length - 7) +
- '-' +
- crypto.randomBytes(3).toString('hex') +
- ext;
- }
+ // Limit the filename to filesystem NAME_MAX, which is byte-based on Linux/APFS.
+ name = truncateLeafWithSuffix(
+ name,
+ '-' + crypto.randomBytes(3).toString('hex'),
+ FILENAME_SEGMENT_MAX_BYTES,
+ );
return name;
}
-/** Per-path-component length cap. Mirrors `sanitizeFilename`'s 255-char
+/** Per-path-component byte cap. Mirrors `sanitizeFilename`'s 255-byte
* basename cap and matches filesystem `NAME_MAX` (255 bytes on Linux/ext4,
* 255 chars on Windows/NTFS) — without it, `saveBuffer` writes
* `${file_id}__${flatName}` and a long artifact name surfaces as
* `ENAMETOOLONG` and falls back to a download URL instead of persisting. */
-const ARTIFACT_PATH_SEGMENT_MAX = 255;
+const ARTIFACT_PATH_SEGMENT_MAX_BYTES = FILENAME_SEGMENT_MAX_BYTES;
/** Whole-path length cap for the path-preserving form. The DB stores this
* value in `filename`, which participates in a compound unique index on
@@ -81,7 +126,7 @@ const ARTIFACT_PATH_SEGMENT_MAX = 255;
* (`packages/data-schemas/src/schema/file.ts`). MongoDB 4.0 and earlier
* reject indexed values past 1024 bytes; even on 4.2+ where the limit
* is configurable, runaway nested paths bloat the index for no real
- * benefit. 512 chars is plenty for realistic code-execution outputs
+ * benefit. 512 bytes is plenty for realistic code-execution outputs
* (typical depth ≤ 3 segments × short names) and gives headroom for
* BSON / index-overhead encoding.
*
@@ -92,7 +137,7 @@ const ARTIFACT_PATH_SEGMENT_MAX = 255;
* with a missing artifact is strictly worse than a flat-name fallback.
* Pre-PR every artifact got this treatment regardless of depth, so the
* cap is monotonically better than the prior behavior. */
-const ARTIFACT_PATH_TOTAL_MAX = 512;
+const ARTIFACT_PATH_TOTAL_MAX_BYTES = 512;
/**
* Deterministic disambiguator suffix for truncated names. The original
@@ -126,23 +171,23 @@ function deterministicHexSuffix(input: string): string {
* still blow past the cap.
*/
function truncateLeafSegment(leaf: string): string {
- if (leaf.length <= ARTIFACT_PATH_SEGMENT_MAX) return leaf;
- const ext = path.extname(leaf);
- const stem = path.basename(leaf, ext);
- // 8 = 1 (`-`) + 6 (hex disambiguator) + 1 (minimum 1-char stem)
- if (ext.length > ARTIFACT_PATH_SEGMENT_MAX - 8) {
- return truncateDirSegment(leaf);
- }
- const stemBudget = ARTIFACT_PATH_SEGMENT_MAX - ext.length - 7;
- return stem.slice(0, stemBudget) + '-' + deterministicHexSuffix(leaf) + ext;
+ return truncateLeafWithSuffix(
+ leaf,
+ '-' + deterministicHexSuffix(leaf),
+ ARTIFACT_PATH_SEGMENT_MAX_BYTES,
+ );
}
/** Truncates a non-leaf (directory) segment. Directory segments don't
* carry semantic extensions, so we just slice and append the same 6-hex
* disambiguation suffix. */
function truncateDirSegment(seg: string): string {
- if (seg.length <= ARTIFACT_PATH_SEGMENT_MAX) return seg;
- return seg.slice(0, ARTIFACT_PATH_SEGMENT_MAX - 7) + '-' + deterministicHexSuffix(seg);
+ if (utf8ByteLength(seg) <= ARTIFACT_PATH_SEGMENT_MAX_BYTES) return seg;
+ return truncateWithSuffix(
+ seg,
+ '-' + deterministicHexSuffix(seg),
+ ARTIFACT_PATH_SEGMENT_MAX_BYTES,
+ );
}
/**
@@ -154,7 +199,7 @@ function truncateDirSegment(seg: string): string {
* first record and overwrite the first artifact's bytes.
*
* The disambiguator survives length capping: if appending pushes the
- * segment past `ARTIFACT_PATH_SEGMENT_MAX`, the stem is trimmed to make
+ * segment past `ARTIFACT_PATH_SEGMENT_MAX_BYTES`, the stem is trimmed to make
* room (the hash + extension are load-bearing for collision avoidance
* and MIME inference, the stem is just the human-readable prefix).
*/
@@ -172,30 +217,33 @@ function embedDisambiguatorInLeaf(segment: string, hashSource: string): string {
if (dot <= 1) {
const proposed = segment + suffix;
result =
- proposed.length <= ARTIFACT_PATH_SEGMENT_MAX
+ utf8ByteLength(proposed) <= ARTIFACT_PATH_SEGMENT_MAX_BYTES
? proposed
- : segment.slice(0, ARTIFACT_PATH_SEGMENT_MAX - suffix.length) + suffix;
+ : truncateWithSuffix(segment, suffix, ARTIFACT_PATH_SEGMENT_MAX_BYTES);
} else {
const stem = segment.slice(0, dot);
const ext = segment.slice(dot);
const proposed = stem + suffix + ext;
- if (proposed.length <= ARTIFACT_PATH_SEGMENT_MAX) {
+ if (utf8ByteLength(proposed) <= ARTIFACT_PATH_SEGMENT_MAX_BYTES) {
result = proposed;
} else {
// Trim stem to make room while preserving disambiguator + extension.
- const stemBudget = Math.max(0, ARTIFACT_PATH_SEGMENT_MAX - suffix.length - ext.length);
- result = stem.slice(0, stemBudget) + suffix + ext;
+ const stemBudget = Math.max(
+ 0,
+ ARTIFACT_PATH_SEGMENT_MAX_BYTES - utf8ByteLength(suffix) - utf8ByteLength(ext),
+ );
+ result = truncateUtf8Bytes(stem, stemBudget) + suffix + ext;
}
}
/* Defensive final clamp. The branches above already keep the result
- * within `ARTIFACT_PATH_SEGMENT_MAX` for any input where the
+ * within `ARTIFACT_PATH_SEGMENT_MAX_BYTES` for any input where the
* extension fits the segment, but a pathological extension (e.g. the
* `.aaaaa…` shape `path.extname` returns for contrived inputs) could
* still produce a longer result. Hard-clamp so the segment cap holds
* unconditionally. */
- return result.length <= ARTIFACT_PATH_SEGMENT_MAX
+ return utf8ByteLength(result) <= ARTIFACT_PATH_SEGMENT_MAX_BYTES
? result
- : result.slice(0, ARTIFACT_PATH_SEGMENT_MAX);
+ : truncateUtf8Bytes(result, ARTIFACT_PATH_SEGMENT_MAX_BYTES);
}
/**
@@ -205,7 +253,7 @@ function embedDisambiguatorInLeaf(segment: string, hashSource: string): string {
* same rules as `sanitizeFilename`. Falls back to the basename for absolute
* paths or names containing `..` traversal.
*
- * Each path component is capped at `ARTIFACT_PATH_SEGMENT_MAX` (255) chars
+ * Each path component is capped at `ARTIFACT_PATH_SEGMENT_MAX_BYTES` (255 bytes)
* — the leaf with extension preservation (matching `sanitizeFilename`),
* non-leaf segments with a plain truncate-and-disambiguate. Without the
* cap, long artifact names flow into `saveBuffer`'s storage key
@@ -226,7 +274,7 @@ export function sanitizeArtifactPath(inputName: string): string {
}
const segments = normalized
.split('/')
- .map((seg) => seg.replace(/[^a-zA-Z0-9.-]/g, '_'))
+ .map(sanitizeFilenameSegment)
.filter((seg) => seg.length > 0 && seg !== '.');
if (segments.length === 0) return '_';
const leafIdx = segments.length - 1;
@@ -256,7 +304,7 @@ export function sanitizeArtifactPath(inputName: string): string {
* input → same safe form (idempotent for re-uploads); different raw
* inputs that would have collided → different safe forms.
*
- * Clean inputs (where the regex/normalize pass was a no-op) skip
+ * Clean inputs (where the regex/normalize pass was a raw-string no-op) skip
* the disambiguator — no collision is possible because they're
* already distinct strings, and we don't want to clutter human-
* readable filenames with a hash when nothing was at risk. */
@@ -271,21 +319,21 @@ export function sanitizeArtifactPath(inputName: string): string {
* the (filename, conversationId, context, tenantId) compound unique
* index never sees an oversized key. The leaf is already capped by
* `truncateLeafSegment` + `embedDisambiguatorInLeaf`, so this
- * guarantees ≤ ARTIFACT_PATH_SEGMENT_MAX (255) chars. Same shape as
+ * guarantees ≤ ARTIFACT_PATH_SEGMENT_MAX_BYTES (255 bytes). Same shape as
* the absolute-path / `..`-traversal fallback above. */
- if (joined.length > ARTIFACT_PATH_TOTAL_MAX) {
+ if (utf8ByteLength(joined) > ARTIFACT_PATH_TOTAL_MAX_BYTES) {
return capped[leafIdx];
}
return joined;
}
-/** Limit on `path.extname`'s "extension" length we'll honor when
+/** Limit on `path.extname`'s "extension" byte length we'll honor when
* truncating a flat key. Real file extensions cap out around 8 chars
* (`.parquet`, `.tsv`, `.html`); a 16-char ceiling keeps us tolerant of
* legitimate edge cases (`.openshift`) while ignoring pathological
* outputs from `path.extname` on contrived inputs. Above that we treat
* the trailing `.foo` as part of the stem and just hard-truncate. */
-const FLAT_KEY_MAX_EXT_LENGTH = 16;
+const FLAT_KEY_MAX_EXT_BYTES = 16;
/**
* Map a (sanitized) artifact path to a flat storage-safe key. The local
@@ -293,7 +341,7 @@ const FLAT_KEY_MAX_EXT_LENGTH = 16;
* unintended subdirectories on disk while the DB record retains the
* nested path for the next prime().
*
- * Optionally caps the result at `maxLength` characters. Per-segment caps
+ * Optionally caps the result at `maxLength` UTF-8 bytes. Per-segment caps
* applied by `sanitizeArtifactPath` aren't enough on their own —
* `${file_id}__${flatName}` has to fit in one filesystem path component
* (NAME_MAX = 255 on most filesystems), so a deeply-nested path whose
@@ -308,32 +356,37 @@ const FLAT_KEY_MAX_EXT_LENGTH = 16;
*/
export function flattenArtifactPath(safePath: string, maxLength?: number): string {
const flat = safePath.replace(/\//g, '__');
- if (maxLength == null || flat.length <= maxLength) return flat;
+ if (maxLength == null || utf8ByteLength(flat) <= maxLength) return flat;
if (maxLength <= 0) return '';
/* Find the leaf's extension (last `.`) — segment separators are `__`,
* never `.`, so the last dot is always inside the leaf. Ignore
- * "extensions" longer than `FLAT_KEY_MAX_EXT_LENGTH` so a pathological
+ * "extensions" longer than `FLAT_KEY_MAX_EXT_BYTES` so a pathological
* input doesn't leave us with no stem budget. */
const lastDot = flat.lastIndexOf('.');
const candidateExt = lastDot >= 0 ? flat.slice(lastDot) : '';
const ext =
- candidateExt.length > 0 && candidateExt.length <= FLAT_KEY_MAX_EXT_LENGTH ? candidateExt : '';
+ candidateExt.length > 0 && utf8ByteLength(candidateExt) <= FLAT_KEY_MAX_EXT_BYTES
+ ? candidateExt
+ : '';
const stem = ext ? flat.slice(0, lastDot) : flat;
// 7 = '-' + 6 hex disambiguator. Stem budget can collapse to 0 when
- // `ext.length > maxLength - 7` — that's fine; the stem just disappears
+ // `ext` byte length > maxLength - 7 — that's fine; the stem just disappears
// and we fall back to `-<hash><ext>` (still bounded). The hash is a
// SHA-256 prefix of `safePath` so re-flattening the same input
// produces the same key (same storage location across re-uploads).
- const stemBudget = Math.max(0, maxLength - ext.length - 7);
- const truncated = stem.slice(0, stemBudget) + '-' + deterministicHexSuffix(safePath) + ext;
+ const stemBudget = Math.max(0, maxLength - utf8ByteLength(ext) - 7);
+ const truncated =
+ truncateUtf8Bytes(stem, stemBudget) + '-' + deterministicHexSuffix(safePath) + ext;
/* Final clamp. The stemBudget formula above keeps `truncated.length`
- * exactly at `maxLength` for any maxLength ≥ ext.length + 7, and at
- * `7 + ext.length` otherwise. The latter can still exceed maxLength
- * for absurdly small budgets (maxLength < ext.length + 7) — clamp
+ * exactly at `maxLength` for any maxLength ≥ ext byte length + 7, and at
+ * `7 + ext byte length` otherwise. The latter can still exceed maxLength
+ * for absurdly small budgets (maxLength < ext byte length + 7) — clamp
* defensively so callers always get a key ≤ maxLength regardless of
* what they passed in. */
- return truncated.length <= maxLength ? truncated : truncated.slice(0, maxLength);
+ return utf8ByteLength(truncated) <= maxLength
+ ? truncated
+ : truncateUtf8Bytes(truncated, maxLength);
}
/**