Open source file-system and knowledge based agent template. Build AI agents that stay up to date with your knowledge base
up
Hugo Richard791ef8f0bf2a
diff --git a/apps/chat/app/components/SourceCard.vue b/apps/chat/app/components/SourceCard.vueindex 55b274b..25f8d45 100644--- a/apps/chat/app/components/SourceCard.vue+++ b/apps/chat/app/components/SourceCard.vue@@ -91,7 +91,7 @@ const sourceIdentifier = computed(() => {</div><UTooltip v-if="source.outputPath" text="Output folder in snapshot" :delay-open="200"><div class="inline-flex items-center gap-1 h-[22px] px-2 rounded-md text-[11px] font-medium text-muted bg-muted">- <UIcon name="i-lucide-folder-output" class="size-3 opacity-60" />+ <UIcon name="i-lucide-package" class="size-3 opacity-60" />{{ source.outputPath }}</div></UTooltip>diff --git a/apps/chat/app/pages/admin/sources/new.vue b/apps/chat/app/pages/admin/sources/new.vueindex 27a5060..c66eb6c 100644--- a/apps/chat/app/pages/admin/sources/new.vue+++ b/apps/chat/app/pages/admin/sources/new.vue@@ -506,8 +506,8 @@ const hasValidSources = computed(() => sources.value.some(s => s.data.label))</template><div class="flex items-center gap-2 text-xs text-muted">- <UIcon name="i-lucide-folder-output" class="size-3.5" />- <span>Output:</span>+ <UIcon name="i-lucide-package" class="size-3.5" />+ <span>Snapshot:</span><code class="text-highlighted font-mono">{{ getSnapshotPreview(source) }}</code></div></div>diff --git a/apps/chat/server/api/sources/ocr.post.ts b/apps/chat/server/api/sources/ocr.post.tsindex b3f50e4..1310a60 100644--- a/apps/chat/server/api/sources/ocr.post.ts+++ b/apps/chat/server/api/sources/ocr.post.ts@@ -1,5 +1,6 @@import { generateText, Output } from 'ai'import { z } from 'zod'+import type { SourceOcrItem } from '~~/shared/utils/source-ocr'const bodySchema = z.object({images: z.array(z.string()).optional().default([]),@@ -11,20 +12,108 @@ const bodySchema = z.object({const systemPrompt = `Extract ALL documentation source configurations from the provided content.-For each source found, extract:+STRICT FORMAT RULES:**GitHub repositories:**-- repo: owner/repo format (e.g. "nuxt/nuxt", "unjs/h3")-- branch: the git branch (default "main" if not specified)-- contentPath: the folder path containing documentation files (e.g. "docs", "docs/content", "content/docs")-- label: derive from the label field if present, otherwise from repo name (e.g. "Nuxt" from nuxt/nuxt)+- repo: MUST be exactly "owner/repo" format with no extra text (e.g. "nuxt/nuxt", "nuxt/nuxt.com", "unjs/h3")+- branch: git branch name only, lowercase (default "main")+- contentPath: folder path only (e.g. "docs", "docs/content")+- label: short name from repo, keep dots if present (e.g. "nuxt", "nuxt.com", "h3")**YouTube channels:**-- channelId: starts with "UC" (e.g. "UCxxxxxx")-- handle: the @handle format (e.g. "@TheAlexLichter")-- label: the channel name+- channelId: MUST start with "UC" followed by 22 characters+- handle: @username format (e.g. "@TheAlexLichter")+- label: short channel name++IMPORTANT:+- label should match the repo name (e.g. repo "nuxt/nuxt.com" → label "nuxt.com")+- repo must be owner/repo format only (no URLs, no status codes, no extra text)+- Only extract clearly defined sources, ignore partial or unclear data++Return ALL valid sources found.`++const YOUTUBE_CHANNEL_PATTERN = /^UC[a-zA-Z0-9_-]{22}$/++function sanitizeLabel(label: string): string {+ return label+ .toLowerCase()+ .replace(/[^a-z0-9.-]/g, '') // Keep dots+ .replace(/\.+/g, '.') // Collapse multiple dots+ .replace(/-+/g, '-')+ .replace(/^[.-]+|[.-]+$/g, '') // Trim dots and dashes+}++function sanitizeRepo(repo: string): string | null {+ // Clean the input first+ let cleaned = repo.trim().toLowerCase()++ // Remove common garbage patterns+ cleaned = cleaned+ .replace(/status:\s*\d+.*/i, '') // Remove "Status: 200 OK" etc+ .replace(/https?:\/\/.*/i, '') // Remove URLs+ .replace(/[^a-z0-9_.\/-]/g, '') // Remove invalid characters+ .replace(/\/+/g, '/') // Collapse multiple slashes+ .replace(/^\/|\/$/g, '') // Trim slashes++ // Must be owner/repo format+ const parts = cleaned.split('/')+ if (parts.length !== 2) return null++ const [owner, name] = parts+ if (!owner || !name) return null-Return ALL sources found. Set confidence to 1.0 for clearly defined sources, 0.8 for sources with some missing fields.`+ // Validate each part (GitHub rules: 1-39 chars, alphanumeric and hyphens)+ const validPart = /^[a-z0-9][a-z0-9._-]*$/+ if (!validPart.test(owner) || !validPart.test(name)) return null++ // Reject suspiciously long names (probably garbage)+ if (owner.length > 39 || name.length > 100) return null++ return `${owner}/${name}`+}++function sanitizeSource(source: SourceOcrItem): SourceOcrItem | null {+ let label = sanitizeLabel(source.label || '')+ if (!label) return null++ if (source.type === 'github') {+ const repo = sanitizeRepo(source.repo || '')+ if (!repo) return null++ // Derive label from repo name if model lost the dot+ // e.g. repo "nuxt/nuxt.com" + label "nuxtcom" → label "nuxt.com"+ const [, repoName] = repo.split('/')+ if (repoName && repoName.includes('.')) {+ const labelWithoutDots = label.replace(/\./g, '')+ const repoNameWithoutDots = repoName.replace(/\./g, '')+ if (labelWithoutDots === repoNameWithoutDots) {+ label = repoName+ }+ }++ return {+ ...source,+ label,+ repo,+ branch: source.branch?.toLowerCase().replace(/[^a-z0-9._-]/g, '') || 'main',+ contentPath: source.contentPath?.replace(/^\/+|\/+$/g, '') || '',+ }+ }++ if (source.type === 'youtube') {+ const channelId = source.channelId?.trim()+ if (!channelId || !YOUTUBE_CHANNEL_PATTERN.test(channelId)) return null++ return {+ ...source,+ label,+ channelId,+ handle: source.handle?.startsWith('@') ? source.handle : source.handle ? `@${source.handle}` : '',+ }+ }++ return null+}async function extractFromImage(image: string) {const { output } = await generateText({@@ -73,16 +162,22 @@ export default defineEventHandler(async (event) => {const allSources = results.flat()+ // Sanitize and deduplicateconst seen = new Set<string>()- const uniqueSources = allSources.filter((source) => {- const key = source.type === 'github'- ? source.repo?.toLowerCase()- : source.channelId?.toLowerCase()+ const uniqueSources: SourceOcrItem[] = []- if (!key || seen.has(key)) return false+ for (const source of allSources) {+ const sanitized = sanitizeSource(source)+ if (!sanitized) continue++ const key = sanitized.type === 'github'+ ? sanitized.repo+ : sanitized.channelId++ if (!key || seen.has(key)) continueseen.add(key)- return true- })+ uniqueSources.push(sanitized)+ }return { sources: uniqueSources }})diff --git a/apps/chat/shared/utils/source-ocr.ts b/apps/chat/shared/utils/source-ocr.tsindex 6387805..a10a6ce 100644--- a/apps/chat/shared/utils/source-ocr.ts+++ b/apps/chat/shared/utils/source-ocr.ts@@ -1,14 +1,34 @@import { z } from 'zod'export const sourceOcrItemSchema = z.object({- type: z.enum(['github', 'youtube']),- label: z.string(),- repo: z.string().optional(),- branch: z.string().optional(),- contentPath: z.string().optional(),- channelId: z.string().optional(),- handle: z.string().optional(),- confidence: z.number().min(0).max(1),+ type: z.enum(['github', 'youtube']).describe('Source type'),+ label: z.string()+ .max(50)+ .describe('Short lowercase name (e.g. "nuxt", "nuxt.com", "vue", "h3")'),+ repo: z.string()+ .max(80)+ .describe('GitHub repo in owner/repo format only (e.g. "nuxt/nuxt", "unjs/h3")')+ .optional(),+ branch: z.string()+ .max(50)+ .describe('Git branch name (e.g. "main", "v4")')+ .optional(),+ contentPath: z.string()+ .max(100)+ .describe('Folder path to docs (e.g. "docs", "docs/content")')+ .optional(),+ channelId: z.string()+ .length(24)+ .describe('YouTube channel ID starting with UC (e.g. "UCxxxxxxxxxxxxxxxx")')+ .optional(),+ handle: z.string()+ .max(50)+ .describe('YouTube handle with @ (e.g. "@TheAlexLichter")')+ .optional(),+ confidence: z.number()+ .min(0)+ .max(1)+ .describe('Confidence score: 1.0 for complete data, 0.8 for partial'),})export const sourceOcrSchema = z.object({