Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Forward group chat images to member Sessions with their read-only path

Fei2411bd8b7449cc80
diff --git a/ds-bot/src/host/delivery.js b/ds-bot/src/host/delivery.js
index d8fdd0f1..d356a554 100644
--- a/ds-bot/src/host/delivery.js
+++ b/ds-bot/src/host/delivery.js
@@ -50,12 +50,12 @@ export function install(rt) {
// a message a shutdown cancelled; its first send may not have been recorded.
// `askedBy` is the member whose @mention called a group turn (groups.js).
// `fromRoomId`: the group whose Session the sender wrote from; the reply goes back
- // to that Session (turns.js). `files` go along as file blocks, which DSH turns into
- // a read-only path the recipient's tools can read. `length`: the whole message's
+ // to that Session (turns.js). `images` and `files` go along as image and file blocks,
+ // which DSH projects for the recipient's model and tools. `length`: the whole message's
// length when it went as a file (attachLong). `work`: the work (handoffs.js) a
// result that came back goes on with. `merged`: a request that takes in an earlier
// one its recipient had not read, under that one's exchange.
- async function deliver({ fromId, toId, sessionId, text, role, replyTo, roomId, audience, note, exchangeId = randomUUID(), steer = false, deferred = false, resumed = false, retried = false, resent = false, merged = false, askedBy, fromRoomId, files = [], length, work }) {
+ async function deliver({ fromId, toId, sessionId, text, role, replyTo, roomId, audience, note, exchangeId = randomUUID(), steer = false, deferred = false, resumed = false, retried = false, resent = false, merged = false, askedBy, fromRoomId, images = [], files = [], length, work }) {
const from = rt.botOf(fromId)
const to = rt.botOf(toId)
if (to === undefined) throw new Error(`Unknown Bot ${toId}`)
@@ -88,6 +88,7 @@ export function install(rt) {
const message = createUserMessage({
content: [
{ type: 'text', text: `${headerFor(role, fromName, room?.name, audience, note, deferred, { resumed, retried, resent, asker })}\n${text}` },
+ ...images.map(attachment => ({ type: 'image', attachment })),
...files.map(attachment => ({ type: 'file', attachment })),
],
source,
diff --git a/ds-bot/src/host/groups.js b/ds-bot/src/host/groups.js
index 9c6d5651..ce8cbab7 100644
--- a/ds-bot/src/host/groups.js
+++ b/ds-bot/src/host/groups.js
@@ -323,7 +323,7 @@ export function install(rt) {
const spokeThisRound = (room.seen[member.id] ?? 0) >= round.startSeq
const members = `Members: the user${peers.length ? `, ${peers.join(', ')}` : ''} and you${leads ? ' (admin)' : ''}. ${ROOM_CALLS}`
const notice = room.notice ? `Group notice (shared instructions for every member):\n${room.notice}` : ''
- if (round.kickoff) return { text: [members, notice, leads ? adminNote(room) : '', roomKickoff(rt.uiLanguage?.())].filter(Boolean).join('\n\n'), files: [] }
+ if (round.kickoff) return { text: [members, notice, leads ? adminNote(room) : '', roomKickoff(rt.uiLanguage?.())].filter(Boolean).join('\n\n'), images: [], files: [] }
const goal = rt.goalBlock?.(room, member, { nudge: round.goalNudge === true, resume: round.goalResume === true }) ?? ''
// Members answering in this round are named with the round below.
const answering = new Set(round.slots.filter(other => other.status === 'speaking').map(other => other.id))
@@ -343,7 +343,8 @@ export function install(rt) {
...placeLines(round, slot),
ROOM_REPLY,
].filter(Boolean).join('\n\n')
- return { text, files: [...earlier, ...current].flatMap(line => line.files ?? []) }
+ const shown = [...earlier, ...current]
+ return { text, images: shown.flatMap(line => line.images ?? []), files: shown.flatMap(line => line.files ?? []) }
}
// Where a turn stands in its round: its place in the order when members take turns,
@@ -473,7 +474,7 @@ export function install(rt) {
// called it, which answers what the turn posts after the round moved on. What the
// member writes before its last step is posted at once and collected in `said`, and
// `heard` takes each such line once it is posted.
- async function roomTurn(room, member, sessionId, { text, files }, note, signal, round, by, said = [], heard = () => {}, onWork = () => {}) {
+ async function roomTurn(room, member, sessionId, { text, images = [], files = [] }, note, signal, round, by, said = [], heard = () => {}, onWork = () => {}) {
if (signal.aborted) return null
const startedAt = Date.now()
const exchangeId = randomUUID()
@@ -514,7 +515,7 @@ export function install(rt) {
signal.addEventListener('abort', () => finish(null), { once: true })
})
try {
- await rt.deliver({ fromId: undefined, toId: member.id, sessionId, role: 'room', roomId: room.id, exchangeId, note, text, files, steer: busy, ...(by ? { askedBy: by } : {}) })
+ await rt.deliver({ fromId: undefined, toId: member.id, sessionId, role: 'room', roomId: room.id, exchangeId, note, text, images, files, steer: busy, ...(by ? { askedBy: by } : {}) })
// The prompt reached the member: its read position moves on.
room.seen[member.id] = seenAt
// Still held by an earlier request: this turn cannot start until the user answers.
@@ -806,7 +807,7 @@ export function install(rt) {
markBatchWorked(room.id, member.id)
}
const reply = await roomTurn(
- room, member, session.id, { text, files: prompt.files }, slot.note ?? 'your turn', signal, round, slot.by, said,
+ room, member, session.id, { text, images: prompt.images, files: prompt.files }, slot.note ?? 'your turn', signal, round, slot.by, said,
line => callFrom(room, round, member, line, called, { worked, byBot: slot.by !== undefined }),
onWork,
)
@@ -915,12 +916,13 @@ export function install(rt) {
const stoppedOnly = posts.length > 0 && posts.every(item => event(item) && item.stopped === true)
for (const post of posts.filter(item => !item.kickoff && !event(item) && item.logged === undefined)) {
// Image readings go into the log only; `@` names are picked from what the user typed.
- // A file stays with its line and reaches each member the line is shown to.
- const readings = post.images === undefined ? [] : await rt.imagesForGroup(post.images, room.id)
+ // Images and files stay with their line and reach each member the line is shown to.
+ const readings = post.images === undefined ? [] : await rt.imagesForGroup(post.images, rt.sessionOfRoom(room))
const text = [post.text, ...readings, ...(post.files ?? []).map(fileLine)].filter(Boolean).join('\n')
appendRoom(room, {
...(post.botId ? { who: rt.botOf(post.botId)?.name ?? post.name ?? 'Bot', botId: post.botId } : { who: 'User' }),
text,
+ ...(post.images === undefined ? {} : { images: post.images }),
...(post.files === undefined ? {} : { files: post.files }),
...(post.via === undefined ? {} : { via: post.via }),
})
diff --git a/ds-bot/src/host/images.js b/ds-bot/src/host/images.js
index df263088..eba9ebd0 100644
--- a/ds-bot/src/host/images.js
+++ b/ds-bot/src/host/images.js
@@ -53,9 +53,9 @@ export function install(rt) {
}
// The configured `ocrModel` first, then served models that declare image input, the
- // DSH default among them first.
+ // DSH default among them first. A provider with no key set is skipped.
async function ocrRoute() {
- const models = await rt.catalog()
+ const models = (await rt.catalog()).filter(entry => entry.needsKey !== true)
const served = new Set(models.map(entry => entry.ref))
const declared = models.filter(entry => entry.input?.includes('image')).map(entry => entry.ref)
const fallback = rt.defaultRef()
@@ -133,26 +133,41 @@ export function install(rt) {
? `"${attachment.name}"`
: `sha256:${String(attachment.attachmentId).slice('sha256:'.length, 'sha256:'.length + 8)}`)
+ function imagePath(attachment) {
+ try {
+ const hostPath = ctx.get?.('attachments')?.imageHostPath?.(attachment)
+ if (hostPath === undefined) return undefined
+ return ctx.get?.('fs')?.processPathFromHostPath?.(hostPath) ?? hostPath
+ } catch {
+ return undefined
+ }
+ }
+
function imageAsText(attachment, reading) {
const label = imageLabel(attachment)
+ const path = imagePath(attachment)
+ const saved = path === undefined ? '' : ` Read-only copy saved at ${JSON.stringify(path)}.`
return {
type: 'text',
text: reading === null
- ? `[Image ${label}: you cannot see images and it could not be read right now. Ask the user what it shows if it matters.]`
- : `[Image ${label}. You cannot see images, so ${reading.model} read it for you:]\n${reading.text}\n[End of image ${label}]`,
+ ? `[Image ${label}: you cannot see images and it could not be read right now.${saved} Ask the user what it shows if it matters.]`
+ : `[Image ${label}.${saved} You cannot see images, so ${reading.model} read it for you:]\n${reading.text}\n[End of image ${label}]`,
}
}
// Members read the group as text, so a picture posted there reaches every member,
- // vision model or not, as its reading.
+ // vision model or not, as its reading; the image block itself also rides with the
+ // prompt for members whose model or tools can inspect it directly.
async function imagesForGroup(attachments, sessionId) {
await loadOcr()
return Promise.all(attachments.map(async (attachment) => {
const reading = await imageReading(attachment, sessionId)
const label = imageLabel(attachment)
+ const path = imagePath(attachment)
+ const saved = path === undefined ? '' : ` (read-only copy saved at ${JSON.stringify(path)})`
return reading === null
- ? `[Image ${label}: it could not be read right now. Ask the user what it shows if it matters.]`
- : `[Image ${label}, read by ${reading.model}:]\n${reading.text}\n[End of image ${label}]`
+ ? `[Image ${label}${saved}: it could not be read right now. Ask the user what it shows if it matters.]`
+ : `[Image ${label}${saved}, read by ${reading.model}:]\n${reading.text}\n[End of image ${label}]`
}))
}
diff --git a/ds-bot/test/rounds.test.js b/ds-bot/test/rounds.test.js
index 9fe8ae1b..046e6f20 100644
--- a/ds-bot/test/rounds.test.js
+++ b/ds-bot/test/rounds.test.js
@@ -595,6 +595,40 @@ test('a file the user sends in a group reaches each member the line is shown to,
await running
})
+// #955: an image the user sends in a group reaches each member the line is shown to as
+// an image block once, and the log line keeps its read-only copy path even when no
+// vision model can read it.
+test('an image the user sends in a group reaches each member the line is shown to, once, with its read-only path', async (t) => {
+ const { call, emit, relay, answer, stop, sent, ctx, rt } = start()
+ t.after(stop)
+ const savedPath = '/tmp/dsh-attachments/objects/ab/cd'
+ ctx.get = name => (name === 'attachments' ? { imageHostPath: () => savedPath } : undefined)
+ const room = await team(call, sent)
+ const png = { attachmentId: 'sha256:abcd1234', name: 'image.png', mediaType: 'image/png', bytes: 67_000, width: 800, height: 600 }
+ await emit(room.id, { type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'What is in this screenshot?' }, { type: 'image', attachment: png }] } })
+ let running = relay(room.id, 'r1')
+ const first = await turnFor(sent)
+ const images = message => message.content.filter(block => block.type === 'image').map(block => block.attachment)
+ assert.deepEqual(images(first.message), [png])
+ assert.match(first.message.content[0].text, /read-only copy saved at "\/tmp\/dsh-attachments\/objects\/ab\/cd"/)
+ await answer(first.sessionId, first.message.source.exchangeId, 'I see it.', 1)
+ const second = await turnFor(sent, first)
+ assert.deepEqual(images(second.message), [png])
+ await answer(second.sessionId, second.message.source.exchangeId, 'Me too.', 2)
+ await running
+ assert.deepEqual(rt.roomOf(room.id).log.find(line => line.who === 'User').images, [png])
+
+ sent.length = 0
+ await emit(room.id, { type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Thanks' }] } })
+ running = relay(room.id, 'r2')
+ const next = await turnFor(sent)
+ assert.deepEqual(images(next.message), [])
+ await answer(next.sessionId, next.message.source.exchangeId, 'OK.', 3)
+ const after = await turnFor(sent, next)
+ await answer(after.sessionId, after.message.source.exchangeId, 'OK.', 4)
+ await running
+})
+
// #206: a Bot's post past 20,000 characters goes as a file; the line keeps the file and
// each member it is shown to gets it with its prompt.
test('a long post goes to the group as a file, and reaches the member with its prompt', async (t) => {
diff --git a/ds-bot/test/session-identity.test.js b/ds-bot/test/session-identity.test.js
index 76757756..ba1d0988 100644
--- a/ds-bot/test/session-identity.test.js
+++ b/ds-bot/test/session-identity.test.js
@@ -149,6 +149,28 @@ test('an image in a group is read for the group, and the group still relays its
assert.equal(bench.requests.length, 1)
})
+test('image readings skip providers without a key and include the read-only path when available', async (t) => {
+ const { ctx, rt, bench, stop } = start()
+ t.after(stop)
+ ctx.llm.listProviders = () => [{ id: 'unkeyed', name: 'Unkeyed' }, { id: 'deepseek', name: 'DeepSeek' }]
+ ctx.llm.listModels = async provider => (provider === 'unkeyed'
+ ? [{ id: 'unkeyed-vision', inputModalities: ['text', 'image'] }]
+ : [{ id: 'v4' }, { id: 'vision', inputModalities: ['text', 'image'] }])
+ rt.missingKeys = async () => new Set(['unkeyed'])
+ rt.dropCatalog()
+ const baseGet = ctx.get
+ ctx.get = name => (name === 'attachments'
+ ? { imageHostPath: () => '/host/attachments/v1/objects/aa/bb' }
+ : name === 'fs'
+ ? { processPathFromHostPath: p => p.replace('/host/', '/world/') }
+ : baseGet?.(name))
+ bench.respond = () => 'A blue circle.'
+ const [reading] = await rt.imagesForGroup([attachment], 'r1')
+ assert.match(reading, /read-only copy saved at "\/world\/attachments\/v1\/objects\/aa\/bb"/)
+ assert.match(reading, /read by vision:\]\nA blue circle\./)
+ assert.deepEqual(bench.requests.map(request => [request.provider, request.model, request.sessionId]), [['deepseek', 'vision', 'r1']])
+})
+
test('the Session hooks pass DS Bot\'s own requests through, and still act on a Session\'s own rounds', async (t) => {
const { rt, hook, stop } = start()
t.after(stop)