A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Forward group chat images to member Sessions with their read-only path
Fei2411bd8b7449cc80
diff --git a/ds-bot/src/host/delivery.js b/ds-bot/src/host/delivery.jsindex d8fdd0f1..d356a554 100644--- a/ds-bot/src/host/delivery.js+++ b/ds-bot/src/host/delivery.js@@ -50,12 +50,12 @@ export function install(rt) {// a message a shutdown cancelled; its first send may not have been recorded.// `askedBy` is the member whose @mention called a group turn (groups.js).// `fromRoomId`: the group whose Session the sender wrote from; the reply goes back- // to that Session (turns.js). `files` go along as file blocks, which DSH turns into- // a read-only path the recipient's tools can read. `length`: the whole message's+ // to that Session (turns.js). `images` and `files` go along as image and file blocks,+ // which DSH projects for the recipient's model and tools. `length`: the whole message's// length when it went as a file (attachLong). `work`: the work (handoffs.js) a// result that came back goes on with. `merged`: a request that takes in an earlier// one its recipient had not read, under that one's exchange.- async function deliver({ fromId, toId, sessionId, text, role, replyTo, roomId, audience, note, exchangeId = randomUUID(), steer = false, deferred = false, resumed = false, retried = false, resent = false, merged = false, askedBy, fromRoomId, files = [], length, work }) {+ async function deliver({ fromId, toId, sessionId, text, role, replyTo, roomId, audience, note, exchangeId = randomUUID(), steer = false, deferred = false, resumed = false, retried = false, resent = false, merged = false, askedBy, fromRoomId, images = [], files = [], length, work }) {const from = rt.botOf(fromId)const to = rt.botOf(toId)if (to === undefined) throw new Error(`Unknown Bot ${toId}`)@@ -88,6 +88,7 @@ export function install(rt) {const message = createUserMessage({content: [{ type: 'text', text: `${headerFor(role, fromName, room?.name, audience, note, deferred, { resumed, retried, resent, asker })}\n${text}` },+ ...images.map(attachment => ({ type: 'image', attachment })),...files.map(attachment => ({ type: 'file', attachment })),],source,diff --git a/ds-bot/src/host/groups.js b/ds-bot/src/host/groups.jsindex 9c6d5651..ce8cbab7 100644--- a/ds-bot/src/host/groups.js+++ b/ds-bot/src/host/groups.js@@ -323,7 +323,7 @@ export function install(rt) {const spokeThisRound = (room.seen[member.id] ?? 0) >= round.startSeqconst members = `Members: the user${peers.length ? `, ${peers.join(', ')}` : ''} and you${leads ? ' (admin)' : ''}. ${ROOM_CALLS}`const notice = room.notice ? `Group notice (shared instructions for every member):\n${room.notice}` : ''- if (round.kickoff) return { text: [members, notice, leads ? adminNote(room) : '', roomKickoff(rt.uiLanguage?.())].filter(Boolean).join('\n\n'), files: [] }+ if (round.kickoff) return { text: [members, notice, leads ? adminNote(room) : '', roomKickoff(rt.uiLanguage?.())].filter(Boolean).join('\n\n'), images: [], files: [] }const goal = rt.goalBlock?.(room, member, { nudge: round.goalNudge === true, resume: round.goalResume === true }) ?? ''// Members answering in this round are named with the round below.const answering = new Set(round.slots.filter(other => other.status === 'speaking').map(other => other.id))@@ -343,7 +343,8 @@ export function install(rt) {...placeLines(round, slot),ROOM_REPLY,].filter(Boolean).join('\n\n')- return { text, files: [...earlier, ...current].flatMap(line => line.files ?? []) }+ const shown = [...earlier, ...current]+ return { text, images: shown.flatMap(line => line.images ?? []), files: shown.flatMap(line => line.files ?? []) }}// Where a turn stands in its round: its place in the order when members take turns,@@ -473,7 +474,7 @@ export function install(rt) {// called it, which answers what the turn posts after the round moved on. What the// member writes before its last step is posted at once and collected in `said`, and// `heard` takes each such line once it is posted.- async function roomTurn(room, member, sessionId, { text, files }, note, signal, round, by, said = [], heard = () => {}, onWork = () => {}) {+ async function roomTurn(room, member, sessionId, { text, images = [], files = [] }, note, signal, round, by, said = [], heard = () => {}, onWork = () => {}) {if (signal.aborted) return nullconst startedAt = Date.now()const exchangeId = randomUUID()@@ -514,7 +515,7 @@ export function install(rt) {signal.addEventListener('abort', () => finish(null), { once: true })})try {- await rt.deliver({ fromId: undefined, toId: member.id, sessionId, role: 'room', roomId: room.id, exchangeId, note, text, files, steer: busy, ...(by ? { askedBy: by } : {}) })+ await rt.deliver({ fromId: undefined, toId: member.id, sessionId, role: 'room', roomId: room.id, exchangeId, note, text, images, files, steer: busy, ...(by ? { askedBy: by } : {}) })// The prompt reached the member: its read position moves on.room.seen[member.id] = seenAt// Still held by an earlier request: this turn cannot start until the user answers.@@ -806,7 +807,7 @@ export function install(rt) {markBatchWorked(room.id, member.id)}const reply = await roomTurn(- room, member, session.id, { text, files: prompt.files }, slot.note ?? 'your turn', signal, round, slot.by, said,+ room, member, session.id, { text, images: prompt.images, files: prompt.files }, slot.note ?? 'your turn', signal, round, slot.by, said,line => callFrom(room, round, member, line, called, { worked, byBot: slot.by !== undefined }),onWork,)@@ -915,12 +916,13 @@ export function install(rt) {const stoppedOnly = posts.length > 0 && posts.every(item => event(item) && item.stopped === true)for (const post of posts.filter(item => !item.kickoff && !event(item) && item.logged === undefined)) {// Image readings go into the log only; `@` names are picked from what the user typed.- // A file stays with its line and reaches each member the line is shown to.- const readings = post.images === undefined ? [] : await rt.imagesForGroup(post.images, room.id)+ // Images and files stay with their line and reach each member the line is shown to.+ const readings = post.images === undefined ? [] : await rt.imagesForGroup(post.images, rt.sessionOfRoom(room))const text = [post.text, ...readings, ...(post.files ?? []).map(fileLine)].filter(Boolean).join('\n')appendRoom(room, {...(post.botId ? { who: rt.botOf(post.botId)?.name ?? post.name ?? 'Bot', botId: post.botId } : { who: 'User' }),text,+ ...(post.images === undefined ? {} : { images: post.images }),...(post.files === undefined ? {} : { files: post.files }),...(post.via === undefined ? {} : { via: post.via }),})diff --git a/ds-bot/src/host/images.js b/ds-bot/src/host/images.jsindex df263088..eba9ebd0 100644--- a/ds-bot/src/host/images.js+++ b/ds-bot/src/host/images.js@@ -53,9 +53,9 @@ export function install(rt) {}// The configured `ocrModel` first, then served models that declare image input, the- // DSH default among them first.+ // DSH default among them first. A provider with no key set is skipped.async function ocrRoute() {- const models = await rt.catalog()+ const models = (await rt.catalog()).filter(entry => entry.needsKey !== true)const served = new Set(models.map(entry => entry.ref))const declared = models.filter(entry => entry.input?.includes('image')).map(entry => entry.ref)const fallback = rt.defaultRef()@@ -133,26 +133,41 @@ export function install(rt) {? `"${attachment.name}"`: `sha256:${String(attachment.attachmentId).slice('sha256:'.length, 'sha256:'.length + 8)}`)+ function imagePath(attachment) {+ try {+ const hostPath = ctx.get?.('attachments')?.imageHostPath?.(attachment)+ if (hostPath === undefined) return undefined+ return ctx.get?.('fs')?.processPathFromHostPath?.(hostPath) ?? hostPath+ } catch {+ return undefined+ }+ }+function imageAsText(attachment, reading) {const label = imageLabel(attachment)+ const path = imagePath(attachment)+ const saved = path === undefined ? '' : ` Read-only copy saved at ${JSON.stringify(path)}.`return {type: 'text',text: reading === null- ? `[Image ${label}: you cannot see images and it could not be read right now. Ask the user what it shows if it matters.]`- : `[Image ${label}. You cannot see images, so ${reading.model} read it for you:]\n${reading.text}\n[End of image ${label}]`,+ ? `[Image ${label}: you cannot see images and it could not be read right now.${saved} Ask the user what it shows if it matters.]`+ : `[Image ${label}.${saved} You cannot see images, so ${reading.model} read it for you:]\n${reading.text}\n[End of image ${label}]`,}}// Members read the group as text, so a picture posted there reaches every member,- // vision model or not, as its reading.+ // vision model or not, as its reading; the image block itself also rides with the+ // prompt for members whose model or tools can inspect it directly.async function imagesForGroup(attachments, sessionId) {await loadOcr()return Promise.all(attachments.map(async (attachment) => {const reading = await imageReading(attachment, sessionId)const label = imageLabel(attachment)+ const path = imagePath(attachment)+ const saved = path === undefined ? '' : ` (read-only copy saved at ${JSON.stringify(path)})`return reading === null- ? `[Image ${label}: it could not be read right now. Ask the user what it shows if it matters.]`- : `[Image ${label}, read by ${reading.model}:]\n${reading.text}\n[End of image ${label}]`+ ? `[Image ${label}${saved}: it could not be read right now. Ask the user what it shows if it matters.]`+ : `[Image ${label}${saved}, read by ${reading.model}:]\n${reading.text}\n[End of image ${label}]`}))}diff --git a/ds-bot/test/rounds.test.js b/ds-bot/test/rounds.test.jsindex 9fe8ae1b..046e6f20 100644--- a/ds-bot/test/rounds.test.js+++ b/ds-bot/test/rounds.test.js@@ -595,6 +595,40 @@ test('a file the user sends in a group reaches each member the line is shown to,await running})+// #955: an image the user sends in a group reaches each member the line is shown to as+// an image block once, and the log line keeps its read-only copy path even when no+// vision model can read it.+test('an image the user sends in a group reaches each member the line is shown to, once, with its read-only path', async (t) => {+ const { call, emit, relay, answer, stop, sent, ctx, rt } = start()+ t.after(stop)+ const savedPath = '/tmp/dsh-attachments/objects/ab/cd'+ ctx.get = name => (name === 'attachments' ? { imageHostPath: () => savedPath } : undefined)+ const room = await team(call, sent)+ const png = { attachmentId: 'sha256:abcd1234', name: 'image.png', mediaType: 'image/png', bytes: 67_000, width: 800, height: 600 }+ await emit(room.id, { type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'What is in this screenshot?' }, { type: 'image', attachment: png }] } })+ let running = relay(room.id, 'r1')+ const first = await turnFor(sent)+ const images = message => message.content.filter(block => block.type === 'image').map(block => block.attachment)+ assert.deepEqual(images(first.message), [png])+ assert.match(first.message.content[0].text, /read-only copy saved at "\/tmp\/dsh-attachments\/objects\/ab\/cd"/)+ await answer(first.sessionId, first.message.source.exchangeId, 'I see it.', 1)+ const second = await turnFor(sent, first)+ assert.deepEqual(images(second.message), [png])+ await answer(second.sessionId, second.message.source.exchangeId, 'Me too.', 2)+ await running+ assert.deepEqual(rt.roomOf(room.id).log.find(line => line.who === 'User').images, [png])++ sent.length = 0+ await emit(room.id, { type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Thanks' }] } })+ running = relay(room.id, 'r2')+ const next = await turnFor(sent)+ assert.deepEqual(images(next.message), [])+ await answer(next.sessionId, next.message.source.exchangeId, 'OK.', 3)+ const after = await turnFor(sent, next)+ await answer(after.sessionId, after.message.source.exchangeId, 'OK.', 4)+ await running+})+// #206: a Bot's post past 20,000 characters goes as a file; the line keeps the file and// each member it is shown to gets it with its prompt.test('a long post goes to the group as a file, and reaches the member with its prompt', async (t) => {diff --git a/ds-bot/test/session-identity.test.js b/ds-bot/test/session-identity.test.jsindex 76757756..ba1d0988 100644--- a/ds-bot/test/session-identity.test.js+++ b/ds-bot/test/session-identity.test.js@@ -149,6 +149,28 @@ test('an image in a group is read for the group, and the group still relays itsassert.equal(bench.requests.length, 1)})+test('image readings skip providers without a key and include the read-only path when available', async (t) => {+ const { ctx, rt, bench, stop } = start()+ t.after(stop)+ ctx.llm.listProviders = () => [{ id: 'unkeyed', name: 'Unkeyed' }, { id: 'deepseek', name: 'DeepSeek' }]+ ctx.llm.listModels = async provider => (provider === 'unkeyed'+ ? [{ id: 'unkeyed-vision', inputModalities: ['text', 'image'] }]+ : [{ id: 'v4' }, { id: 'vision', inputModalities: ['text', 'image'] }])+ rt.missingKeys = async () => new Set(['unkeyed'])+ rt.dropCatalog()+ const baseGet = ctx.get+ ctx.get = name => (name === 'attachments'+ ? { imageHostPath: () => '/host/attachments/v1/objects/aa/bb' }+ : name === 'fs'+ ? { processPathFromHostPath: p => p.replace('/host/', '/world/') }+ : baseGet?.(name))+ bench.respond = () => 'A blue circle.'+ const [reading] = await rt.imagesForGroup([attachment], 'r1')+ assert.match(reading, /read-only copy saved at "\/world\/attachments\/v1\/objects\/aa\/bb"/)+ assert.match(reading, /read by vision:\]\nA blue circle\./)+ assert.deepEqual(bench.requests.map(request => [request.provider, request.model, request.sessionId]), [['deepseek', 'vision', 'r1']])+})+test('the Session hooks pass DS Bot\'s own requests through, and still act on a Session\'s own rounds', async (t) => {const { rt, hook, stop } = start()t.after(stop)