Skip to content

armgit / Orkas

Public Imported from GitHub
0

Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.

Fetch pinned embedding files from the public upstream

wyt4a68ae92c17b
diff --git a/bin/packaged-resource-gate.cjs b/bin/packaged-resource-gate.cjs
index 64fb6f8a..2d389f9a 100644
--- a/bin/packaged-resource-gate.cjs
+++ b/bin/packaged-resource-gate.cjs
@@ -44,6 +44,7 @@ const EMBEDDING_MODEL_CONTRACT = Object.freeze({
kind: 'embedding-model',
id: 'fast-bge-small-zh-v1.5',
source: 'https://storage.googleapis.com/qdrant-fastembed/fast-bge-small-zh-v1.5.tar.gz',
+ filesSource: 'https://huggingface.co/Qdrant/bge-small-zh-v1.5/resolve/46fbe35fd4374a00fee7de77dfddaeb6dd6a2c59/',
archive: Object.freeze({
bytes: 54_584_282,
sha256: 'bf023219b6029148fddf764d248808816c0ca1f107f058231bb1ae0fa526f83f',
@@ -435,6 +436,7 @@ function verifyExtraResourcesConfig(extraResources) {
}
module.exports = {
+ requirePinnedFile,
XLS_READER_CONTRACT,
verifyXlsReaderRoot,
EMBEDDING_MODEL_CONTRACT,
diff --git a/scripts/fetch-embedding-model.mjs b/scripts/fetch-embedding-model.mjs
index a4c3c8a4..13c47548 100644
--- a/scripts/fetch-embedding-model.mjs
+++ b/scripts/fetch-embedding-model.mjs
@@ -15,6 +15,7 @@ const pcRoot = path.resolve(here, '..');
const require = createRequire(import.meta.url);
const {
EMBEDDING_MODEL_CONTRACT,
+ requirePinnedFile,
verifyEmbeddingModelArchive,
verifyEmbeddingModelRoot,
} = require('../bin/packaged-resource-gate.cjs');
@@ -254,10 +255,12 @@ export function installVerifiedModel({
export async function ensureEmbeddingModel({
destination = destDir,
downloadArchive = download,
+ downloadFile = download,
extractArchive = extract,
log = console.log,
model = MODEL,
sourceUrl = tarballUrl,
+ filesSource = sourceUrl === tarballUrl ? EMBEDDING_MODEL_CONTRACT.filesSource : null,
verifyArchive = verifyEmbeddingModelArchive,
verifyRoot = verifyEmbeddingModelRoot,
warn = console.warn,
@@ -279,11 +282,27 @@ export async function ensureEmbeddingModel({
const extractionRoot = path.join(stagingRoot, 'extract');
fs.mkdirSync(extractionRoot);
- log(`[embedding-model] fetching ${sourceUrl}`);
+ log(`[embedding-model] fetching ${filesSource || sourceUrl}`);
try {
- await downloadArchive(sourceUrl, archivePath);
- verifyArchive(archivePath);
- await extractArchive(archivePath, extractionRoot);
+ if (filesSource) {
+ const stagedModel = path.join(extractionRoot, model);
+ fs.mkdirSync(stagedModel);
+ const deadline = Date.now() + EMBEDDING_DOWNLOAD_TOTAL_TIMEOUT_MS;
+ for (const file of EMBEDDING_MODEL_CONTRACT.files) {
+ const remaining = deadline - Date.now();
+ if (remaining <= 0) throw new Error('embedding model download exceeded its total time limit');
+ const output = path.join(stagedModel, file.name);
+ await downloadFile(new URL(file.name, filesSource).toString(), output, {
+ expectedBytes: file.bytes,
+ totalTimeoutMs: remaining,
+ });
+ requirePinnedFile(`embedding-model ${file.name}`, output, file);
+ }
+ } else {
+ await downloadArchive(sourceUrl, archivePath);
+ verifyArchive(archivePath);
+ await extractArchive(archivePath, extractionRoot);
+ }
verifyRoot(extractionRoot);
// A concurrent installer may have completed while this process fetched.
diff --git a/test/main/util/fetch-embedding-model.test.ts b/test/main/util/fetch-embedding-model.test.ts
index 4b7ddf2e..386bd8ab 100644
--- a/test/main/util/fetch-embedding-model.test.ts
+++ b/test/main/util/fetch-embedding-model.test.ts
@@ -2,6 +2,7 @@ import * as fs from 'node:fs';
import * as http from 'node:http';
import * as os from 'node:os';
import * as path from 'node:path';
+import { createRequire } from 'node:module';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
@@ -11,6 +12,9 @@ import {
installVerifiedModel,
} from '../../../scripts/fetch-embedding-model.mjs';
+const require = createRequire(import.meta.url);
+const { EMBEDDING_MODEL_CONTRACT, verifyEmbeddingModelRoot } = require('../../../bin/packaged-resource-gate.cjs');
+
let root = '';
const servers: http.Server[] = [];
@@ -105,6 +109,49 @@ describe('embedding model archive download', () => {
});
describe('embedding model staged installation', () => {
+ it('installs the exact existing model from pinned public files without the retired archive service', async () => {
+ const destination = path.join(root, 'resources', 'embedding-model');
+ const source = path.join(process.cwd(), 'resources', 'embedding-model', EMBEDDING_MODEL_CONTRACT.id);
+ const downloadArchive = vi.fn(async () => { throw new Error('HTTP 403 from retired archive service'); });
+ const downloadFile = vi.fn(async (_url: string, output: string) => {
+ fs.copyFileSync(path.join(source, path.basename(output)), output);
+ });
+
+ await ensureEmbeddingModel({ destination, downloadArchive, downloadFile, log: vi.fn(), warn: vi.fn() });
+
+ expect(verifyEmbeddingModelRoot(destination)).toBe('resource:embedding-model:fast-bge-small-zh-v1.5');
+ expect(downloadArchive).not.toHaveBeenCalled();
+ expect(downloadFile.mock.calls.map(([url]) => url)).toEqual(
+ ['config.json', 'model_optimized.onnx', 'ort_config.json', 'special_tokens_map.json',
+ 'tokenizer.json', 'tokenizer_config.json', 'vocab.txt'].map(name =>
+ `https://huggingface.co/Qdrant/bge-small-zh-v1.5/resolve/46fbe35fd4374a00fee7de77dfddaeb6dd6a2c59/${name}`),
+ );
+ for (const call of downloadFile.mock.calls as unknown as Array<[string, string, { expectedBytes: number; totalTimeoutMs: number }]>) {
+ expect(call[2].expectedBytes).toBeGreaterThan(0);
+ expect(call[2].totalTimeoutMs).toBeGreaterThan(0);
+ expect(call[2].totalTimeoutMs).toBeLessThanOrEqual(600_000);
+ }
+ });
+
+ it('rejects a same-size changed public file before replacing the old payload or downloading later files', async () => {
+ const destination = path.join(root, 'resources', 'embedding-model');
+ const old = path.join(destination, EMBEDDING_MODEL_CONTRACT.id, 'old');
+ fs.mkdirSync(path.dirname(old), { recursive: true });
+ fs.writeFileSync(old, 'preserve');
+ const downloadFile = vi.fn(async (_url: string, output: string) => {
+ fs.writeFileSync(output, Buffer.alloc(739, 0));
+ });
+
+ await expect(ensureEmbeddingModel({
+ destination, downloadFile, downloadArchive: async () => { throw new Error('retired archive'); },
+ log: vi.fn(), warn: vi.fn(),
+ })).rejects.toThrow(/sha256 mismatch/);
+
+ expect(downloadFile).toHaveBeenCalledOnce();
+ expect(fs.readFileSync(old, 'utf8')).toBe('preserve');
+ expect(fs.readdirSync(path.dirname(destination)).filter(name => name.startsWith('.embedding-model-fetch-'))).toEqual([]);
+ });
+
it('uses a verified installed model without downloading and cleans a legacy partial archive', async () => {
const destination = path.join(root, 'resources', 'embedding-model');
const modelDir = path.join(destination, 'model');