Skip to content

An AI-powered search engine with a generative UI

feat(genui): present found items as linked source cards (#1090)

Yoshiki Miurad938a76f7c8b
diff --git a/evals/README.md b/evals/README.md
index 55a78d7c..8138bdd7 100644
--- a/evals/README.md
+++ b/evals/README.md
@@ -82,6 +82,29 @@ question draws out a structured answer, and the block that follows it is
arguably correct, which inflates `falsePositiveRate` with disagreement rather
than defects. Questions whose answers expand belong in the substantive set.
+## source-cards
+
+```bash
+bun run eval:source-cards # adaptive, 1 trial per item
+bun run eval:source-cards -- --mode quick --trials 3
+```
+
+Answers 16 fixed queries (8 listing, 8 other) and reports:
+
+- `emissionRate`: listing answers ("find", "recommend", "where can I get")
+ that presented the items as linked cards (`Card` with `href` inside a
+ `Grid`)
+- `misuseRate`: other answers (explanation, steps, comparison, single fact,
+ writing) that carried cards anyway
+- `wellFormedRate`: emitted blocks with 2 to 8 cards, each with a title `Text`
+ and a unique web `href`
+- `groundedRate`: cards whose URL is a fixture search result URL, verbatim
+- `duplicatedRate`: answers that also repeated the cards as a markdown list
+
+Fixture URLs use real-looking hosts. With `.example` hosts the adaptive model
+correctly judged the results to be placeholders and declined to present them,
+which measured the fixture instead of the prompt.
+
## Thresholds
`thresholds.json` gates CI. A `null` value means the metric is reported but not
diff --git a/evals/datasets/source-cards.json b/evals/datasets/source-cards.json
new file mode 100644
index 00000000..99a85e07
--- /dev/null
+++ b/evals/datasets/source-cards.json
@@ -0,0 +1,512 @@
+{
+ "name": "source-cards",
+ "description": "Does the answer present a set of real, linkable items as a SourceCards spec block when the user asks to find or collect them, and skip it otherwise? Search results are fixed. A listing item asks for several real things that each have their own page in the fixture. An other item asks for an explanation, steps, a comparison, a single fact, or writing, where cards would be misuse. Fixture URLs use real-looking hosts because a model reasonably refuses to present placeholder .example links as genuine items; job and clinic entries use fictional organizations. Queries are written for the eval in the shape of common production requests, not copied from user data. A roundup item has only articles that mention items, never an item's own page, so cards would link to articles: the expected answer has none. nonItemUrls lists every result that is not an item's own page; it is used for scoring and never shown to the model.",
+ "fixtures": {
+ "sql-courses": [
+ {
+ "title": "SQLBolt - Learn SQL with simple, interactive exercises",
+ "url": "https://sqlbolt.com/",
+ "content": "SQLBolt is a free series of interactive lessons and exercises that teach SQL directly in the browser, from basic SELECT queries to joins and aggregates. No account is required."
+ },
+ {
+ "title": "SQL Tutorial | Mode Analytics",
+ "url": "https://mode.com/sql-tutorial/",
+ "content": "Mode's free SQL tutorial is aimed at data analysts. Lessons use real datasets and progress from basic to advanced SQL, including window functions. Practice runs in a hosted editor."
+ },
+ {
+ "title": "Khan Academy: Intro to SQL - Querying and managing data",
+ "url": "https://www.khanacademy.org/computing/sql",
+ "content": "A free beginner course with short video talk-throughs and coding challenges covering tables, queries, joins, and modifying databases."
+ },
+ {
+ "title": "Introduction to Databases (Stanford Online)",
+ "url": "https://online.stanford.edu/courses/databases",
+ "content": "A self-paced university course on relational databases, SQL, and database design. Free to audit; a certificate costs extra."
+ },
+ {
+ "title": "Why learning SQL still matters in 2026",
+ "url": "https://www.dataweekly.com/why-sql-matters",
+ "content": "An opinion piece arguing that SQL remains the most transferable data skill, with survey figures on employer demand."
+ }
+ ],
+ "orthodontists-shibuya": [
+ {
+ "title": "Shibuya Smile Orthodontics | English-speaking orthodontist",
+ "url": "https://shibuya-smile-ortho.jp/en/",
+ "content": "Orthodontic clinic a 3-minute walk from Shibuya Station. Offers clear aligners and ceramic braces. English-speaking staff. Open Saturdays."
+ },
+ {
+ "title": "Aoyama Orthodontic Clinic - Omotesando",
+ "url": "https://aoyama-ortho.jp/",
+ "content": "Specialist orthodontic practice near Omotesando Station, between Shibuya and Aoyama. Treats children and adults; initial consultation 3,300 yen."
+ },
+ {
+ "title": "Ebisu Dental & Orthodontics",
+ "url": "https://ebisu-dental-clinic.jp/orthodontics",
+ "content": "General dentistry clinic in Ebisu, one stop from Shibuya, with an orthodontics department offering lingual braces and invisible aligners."
+ },
+ {
+ "title": "How much do braces cost in Japan?",
+ "url": "https://japanhealthguide.org/braces-cost",
+ "content": "Orthodontic treatment in Japan is usually not covered by national health insurance for adults. Total costs range from about 600,000 to 1,200,000 yen."
+ }
+ ],
+ "tunnel-spalling-papers": [
+ {
+ "title": "Spalling failure around deep tunnels in brittle rock: a field and numerical study",
+ "url": "https://www.sciencedirect.com/tunnelling/2025/spalling-deep-tunnels",
+ "content": "Tunnelling and Underground Space Technology, 2025. Combines field observations from a 1,500 m deep tunnel with discrete element modelling to predict spalling depth in granite."
+ },
+ {
+ "title": "Stress-driven spalling in hard rock: a review of prediction criteria",
+ "url": "https://www.sciencedirect.com/rmre/2024/spalling-review",
+ "content": "Rock Mechanics and Rock Engineering, 2024. Reviews empirical and numerical criteria for spalling onset and compares their accuracy across 40 case histories."
+ },
+ {
+ "title": "Acoustic emission monitoring of spalling in a brittle rock excavation",
+ "url": "https://link.springer.com/ijrmms/2026/acoustic-emission-spalling",
+ "content": "International Journal of Rock Mechanics and Mining Sciences, 2026. Uses acoustic emission sensors to detect micro-cracking before visible spalling."
+ },
+ {
+ "title": "What is rock spalling?",
+ "url": "https://en.wikipedia.org/rock-spalling",
+ "content": "Spalling is the breaking off of rock slabs from an excavation surface under high stress, common in hard, brittle rocks such as granite."
+ }
+ ],
+ "canon-drivers": [
+ {
+ "title": "Canon Support - imageCLASS MF5650 drivers and downloads",
+ "url": "https://www.usa.canon.com/mf5650/drivers",
+ "content": "Official Canon support page for the imageCLASS MF5650 with printer and scanner drivers for Windows and macOS, plus firmware updates."
+ },
+ {
+ "title": "Canon Europe: i-SENSYS MF5650 software",
+ "url": "https://www.canon-europe.com/support/mf5650",
+ "content": "Official Canon Europe download centre listing the UFRII printer driver, MF scan utility, and user manuals for the MF5650."
+ },
+ {
+ "title": "Windows Update Catalog: Canon MF5600 series driver",
+ "url": "https://www.catalog.update.microsoft.com/canon-mf5600",
+ "content": "Microsoft's update catalog hosts a signed Canon MF5600 series driver package that also covers the MF5650."
+ },
+ {
+ "title": "Fix: Canon printer offline on Windows 11",
+ "url": "https://www.techfixes.net/canon-printer-offline",
+ "content": "Troubleshooting steps for a Canon printer that shows as offline, including restarting the print spooler."
+ }
+ ],
+ "ecommerce-cms": [
+ {
+ "title": "GitHub - opencart/opencart: A free shopping cart system",
+ "url": "https://github.com/opencart/opencart",
+ "content": "OpenCart is a free, open-source PHP e-commerce platform. Runs on standard PHP/MySQL shared hosting. GPLv3. 7k stars."
+ },
+ {
+ "title": "GitHub - PrestaShop/PrestaShop: open source e-commerce",
+ "url": "https://github.com/PrestaShop/PrestaShop",
+ "content": "PrestaShop is an open-source e-commerce application written in PHP and Symfony. Works on shared hosting with PHP 8.1 and MySQL. OSL-3.0."
+ },
+ {
+ "title": "GitHub - woocommerce/woocommerce: A customizable eCommerce platform for WordPress",
+ "url": "https://github.com/woocommerce/woocommerce",
+ "content": "WooCommerce turns a WordPress site into an online store. Installs on any host that runs WordPress. GPLv3."
+ },
+ {
+ "title": "GitHub - medusajs/medusa: headless commerce engine",
+ "url": "https://github.com/medusajs/medusa",
+ "content": "Medusa is a Node.js headless commerce platform. Requires a Node server and PostgreSQL, so it does not run on typical PHP shared hosting."
+ }
+ ],
+ "remote-frontend-jobs": [
+ {
+ "title": "Senior Frontend Engineer (React) - Remote, Africa - Lumora Pay",
+ "url": "https://careers.lumorapay.com/frontend-engineer",
+ "content": "Fully remote role open to candidates in Nigeria, Ghana, and Kenya. React and TypeScript. Posted 3 days ago."
+ },
+ {
+ "title": "Frontend Developer - Remote (EMEA) | Talentbridge Africa",
+ "url": "https://talentbridge.africa/careers/frontend-developer",
+ "content": "Remote frontend developer position for engineers based in Africa, including Nigeria. Vue or React experience required. Posted 1 week ago."
+ },
+ {
+ "title": "Remote React Developer jobs in Nigeria | We Work Remotely",
+ "url": "https://weworkremotely.com/remote-react-jobs-nigeria",
+ "content": "Job board listing 24 remote React developer roles that accept applicants from Nigeria, updated daily."
+ },
+ {
+ "title": "How to land a remote tech job from Nigeria",
+ "url": "https://techcabal.com/remote-jobs-guide",
+ "content": "Advice article on building a portfolio and preparing for interviews with international remote employers."
+ }
+ ],
+ "books-like-science-of-evil": [
+ {
+ "title": "Zero Degrees of Empathy by Simon Baron-Cohen",
+ "url": "https://bookshop.org/zero-degrees-of-empathy",
+ "content": "The UK edition of The Science of Evil, arguing that cruelty is best explained by low empathy rather than evil."
+ },
+ {
+ "title": "The Psychopath Test by Jon Ronson",
+ "url": "https://bookshop.org/the-psychopath-test",
+ "content": "A journalistic exploration of psychopathy and how it is diagnosed, often recommended to readers of Baron-Cohen."
+ },
+ {
+ "title": "Behave: The Biology of Humans at Our Best and Worst by Robert Sapolsky",
+ "url": "https://bookshop.org/behave",
+ "content": "A neuroscientist explains the biological roots of violence, empathy, and cooperation across timescales."
+ },
+ {
+ "title": "Against Empathy by Paul Bloom",
+ "url": "https://bookshop.org/against-empathy",
+ "content": "A psychologist argues that empathy is a poor moral guide and makes the case for rational compassion."
+ }
+ ],
+ "poster-tools": [
+ {
+ "title": "Canva Free - Poster maker",
+ "url": "https://www.canva.com/create/posters/",
+ "content": "Free browser-based poster maker with thousands of templates. Exports high-resolution PDF for print. Some elements require a paid plan."
+ },
+ {
+ "title": "Adobe Express - Free online poster maker",
+ "url": "https://www.adobe.com/create/poster",
+ "content": "Adobe's free design tool with poster templates, stock photos, and print-quality PDF download."
+ },
+ {
+ "title": "PosterMyWall - Free poster maker",
+ "url": "https://www.postermywall.com/",
+ "content": "Template-based poster maker. Free downloads are limited to standard resolution; high-resolution print files cost a small fee."
+ },
+ {
+ "title": "Inkscape - Free and open source vector graphics editor",
+ "url": "https://inkscape.org/",
+ "content": "Desktop vector editor for Windows, macOS, and Linux. Free and open source. Produces print-ready posters at any size."
+ }
+ ],
+ "sky-blue": [
+ {
+ "title": "Why is the sky blue? | NASA Space Place",
+ "url": "https://spaceplace.nasa.gov/blue-sky",
+ "content": "Sunlight is made of many colours. Gas molecules scatter shorter blue wavelengths much more than red ones, a process called Rayleigh scattering, so blue light reaches our eyes from all directions."
+ },
+ {
+ "title": "Rayleigh scattering - Physics explained",
+ "url": "https://www.physicsclassroom.com/rayleigh-scattering",
+ "content": "Scattering intensity is inversely proportional to the fourth power of wavelength, so blue light at 450 nm scatters about 5.5 times more than red light at 700 nm."
+ },
+ {
+ "title": "Why sunsets are red",
+ "url": "https://www.weather.gov/why-sunsets-are-red",
+ "content": "At sunset light travels through more atmosphere, so most blue is scattered away and reds and oranges remain."
+ }
+ ],
+ "python-venv-macos": [
+ {
+ "title": "venv - Creation of virtual environments | Python docs",
+ "url": "https://docs.python.org/3/library/venv.html",
+ "content": "Create a virtual environment with python3 -m venv .venv, then activate it on macOS with source .venv/bin/activate. Run deactivate to leave it."
+ },
+ {
+ "title": "Installing packages using pip and virtual environments",
+ "url": "https://packaging.python.org/guides/venv",
+ "content": "After activating the environment, install packages with python -m pip install requests. Use pip freeze > requirements.txt to record dependencies."
+ },
+ {
+ "title": "Install Python on macOS with Homebrew",
+ "url": "https://formulae.brew.sh/formula/python",
+ "content": "brew install python installs the latest Python 3, which includes the venv module."
+ }
+ ],
+ "phone-cameras": [
+ {
+ "title": "iPhone 17 Pro review: camera and battery",
+ "url": "https://www.gsmarena.com/iphone-17-pro",
+ "content": "The iPhone 17 Pro has a 48 MP main camera, a 5x telephoto, and lasted 14 hours in our video playback battery test."
+ },
+ {
+ "title": "Pixel 11 Pro review",
+ "url": "https://www.gsmarena.com/pixel-11-pro",
+ "content": "Google's Pixel 11 Pro uses a 50 MP main sensor with strong night mode processing and a 5x telephoto. Battery lasted 12.5 hours in the same test."
+ },
+ {
+ "title": "Best camera phones 2026",
+ "url": "https://www.phonearena.com/best-camera-phones",
+ "content": "Our camera ranking puts the Pixel 11 Pro first for low light and the iPhone 17 Pro first for video."
+ }
+ ],
+ "berlin-wall": [
+ {
+ "title": "Fall of the Berlin Wall | History",
+ "url": "https://www.history.com/berlin-wall-falls",
+ "content": "The Berlin Wall fell on 9 November 1989, when East German authorities opened the border crossings."
+ },
+ {
+ "title": "Berlin Wall - Encyclopedia",
+ "url": "https://www.britannica.com/berlin-wall",
+ "content": "Built in 1961, the Berlin Wall divided East and West Berlin until 9 November 1989. Formal demolition began in 1990."
+ }
+ ],
+ "autumn-coffee-caption": [
+ {
+ "title": "50 fall coffee captions for Instagram",
+ "url": "https://www.shutterfly.com/fall-coffee",
+ "content": "Ideas like 'Leaves are falling, autumn is calling' and 'Pumpkin spice and everything nice' are popular for autumn coffee posts."
+ },
+ {
+ "title": "How to write Instagram captions that get engagement",
+ "url": "https://blog.hootsuite.com/instagram-captions",
+ "content": "Short captions with a question or call to action and two to five relevant hashtags tend to perform best."
+ }
+ ],
+ "mrna-vaccines": [
+ {
+ "title": "How mRNA vaccines work | CDC",
+ "url": "https://www.cdc.gov/vaccines/mrna",
+ "content": "mRNA vaccines deliver instructions that teach cells to make a harmless piece of a viral protein. The immune system recognizes the protein and builds antibodies. The mRNA breaks down within days and never enters the cell nucleus."
+ },
+ {
+ "title": "Lipid nanoparticles in mRNA delivery",
+ "url": "https://www.nature.com/lipid-nanoparticles",
+ "content": "Lipid nanoparticles protect fragile mRNA and help it enter cells, where ribosomes translate it into protein."
+ },
+ {
+ "title": "mRNA vaccine history",
+ "url": "https://www.science.org/mrna-history",
+ "content": "Decades of research on modified nucleosides made mRNA vaccines stable enough for clinical use."
+ }
+ ],
+ "fridge-mold": [
+ {
+ "title": "How to clean mold from a refrigerator gasket",
+ "url": "https://www.thespruce.com/fridge-gasket-mold",
+ "content": "Use a solution of one part white vinegar to one part water or diluted dish soap. Avoid undiluted bleach on rubber seals because it can dry and crack them."
+ },
+ {
+ "title": "Mold cleanup in your home | EPA",
+ "url": "https://www.epa.gov/mold-cleanup",
+ "content": "Scrub mold off hard surfaces with detergent and water and dry completely. Never mix bleach with ammonia or other cleaners."
+ },
+ {
+ "title": "Is 70% isopropyl alcohol effective against mold?",
+ "url": "https://www.cleaninginstitute.org/alcohol-mold",
+ "content": "70% isopropyl alcohol kills surface mold on non-porous materials but does not remove stains."
+ }
+ ],
+ "ai-news-explainer": [
+ {
+ "title": "What is retrieval-augmented generation?",
+ "url": "https://www.ibm.com/rag",
+ "content": "Retrieval-augmented generation (RAG) retrieves relevant documents at query time and passes them to a language model, so answers can cite current sources instead of relying only on training data."
+ },
+ {
+ "title": "RAG vs fine-tuning",
+ "url": "https://huggingface.co/rag-vs-fine-tuning",
+ "content": "RAG updates knowledge without retraining; fine-tuning changes model behaviour and style. Many systems combine both."
+ },
+ {
+ "title": "Vector databases explained",
+ "url": "https://www.ibm.com/vector-databases",
+ "content": "Vector databases store embeddings so a RAG system can find passages similar in meaning to the query."
+ }
+ ],
+ "budget-keyboards": [
+ {
+ "title": "The 5 Best Budget Mechanical Keyboards of 2026 - RTINGS.com",
+ "url": "https://www.rtings.com/keyboard/reviews/best/budget-mechanical",
+ "content": "Our picks under $60: the Keychron C3 Pro is the best overall budget board, the Redragon K552 is the best value tenkeyless, and the Royal Kludge RK61 is the best compact wireless option."
+ },
+ {
+ "title": "Best cheap mechanical keyboards in 2026 | Tom's Guide",
+ "url": "https://www.tomsguide.com/best-picks/best-cheap-mechanical-keyboards",
+ "content": "We tested a dozen boards under $60. Standouts include the Keychron C3 Pro, the Royal Kludge RK61, and the Akko 5075B Plus for typing feel."
+ },
+ {
+ "title": "Keychron C3 Pro QMK/VIA Wired Mechanical Keyboard",
+ "url": "https://www.keychron.com/products/keychron-c3-pro-qmk-via-wired-mechanical-keyboard",
+ "content": "Official product page. Tenkeyless wired mechanical keyboard with hot-swappable switches and QMK/VIA support. $39.99."
+ },
+ {
+ "title": "Redragon K552 KUMARA Mechanical Gaming Keyboard",
+ "url": "https://www.redragonshop.com/products/redragon-k552-kumara",
+ "content": "Official store page. 87-key tenkeyless mechanical keyboard with red LED backlight and metal frame. $34.99."
+ }
+ ],
+ "free-vpns": [
+ {
+ "title": "Free VPN - Proton VPN",
+ "url": "https://protonvpn.com/free-vpn",
+ "content": "Proton VPN's free plan has no data limit, no ads, and a strict no-logs policy. Free users can connect one device to servers in several countries."
+ },
+ {
+ "title": "Windscribe - Free VPN and Ad Block",
+ "url": "https://windscribe.com/features/free",
+ "content": "Windscribe's free plan includes 10 GB of data per month with a confirmed email and access to servers in 10 countries."
+ },
+ {
+ "title": "The best free VPNs in 2026 | CNET",
+ "url": "https://www.cnet.com/tech/services-and-software/best-free-vpn/",
+ "content": "Our ranking puts Proton VPN first for its unlimited free data. TunnelBear's free tier gives 2 GB a month, and hide.me offers 10 GB free with servers in a few locations."
+ }
+ ],
+ "roman-history-podcasts": [
+ {
+ "title": "15 Best Roman History Podcasts You Should Listen To in 2026",
+ "url": "https://www.historyhit.com/best-roman-history-podcasts/",
+ "content": "Our favourites include The History of Rome by Mike Duncan, Emperors of Rome from La Trobe University, and The Rest Is History's Roman episodes."
+ },
+ {
+ "title": "Podcasts for ancient history fans - The Guardian",
+ "url": "https://www.theguardian.com/culture/2026/mar/ancient-history-podcasts",
+ "content": "Mike Duncan's The History of Rome remains the gateway series, while Fall of Rome by Patrick Wyman covers the late empire in depth."
+ }
+ ],
+ "hike-planning-apps": [
+ {
+ "title": "The best hiking apps of 2026 | Outdoor Gear Lab",
+ "url": "https://www.outdoorgearlab.com/topics/camping-and-hiking/best-hiking-apps",
+ "content": "AllTrails is our top pick for trail discovery, Gaia GPS for offline topographic maps, and FarOut for long-distance trails like the Appalachian Trail."
+ },
+ {
+ "title": "9 apps every hiker should download - REI Co-op Journal",
+ "url": "https://www.rei.com/blog/hike/hiking-apps",
+ "content": "Download AllTrails for reviews, Gaia GPS or CalTopo for route planning, and PeakVisor to identify mountains."
+ }
+ ]
+ },
+ "items": [
+ {
+ "id": "sql-courses",
+ "category": "listing",
+ "fixture": "sql-courses",
+ "query": "Recommend some free online courses to learn SQL."
+ },
+ {
+ "id": "orthodontists",
+ "category": "listing",
+ "fixture": "orthodontists-shibuya",
+ "query": "Find orthodontic clinics near Shibuya that speak English."
+ },
+ {
+ "id": "spalling-papers",
+ "category": "listing",
+ "fixture": "tunnel-spalling-papers",
+ "query": "Find recent research papers on spalling around tunnels in brittle rock."
+ },
+ {
+ "id": "canon-drivers",
+ "category": "listing",
+ "fixture": "canon-drivers",
+ "query": "Where can I download drivers for the Canon MF5650?"
+ },
+ {
+ "id": "ecommerce-cms",
+ "category": "listing",
+ "fixture": "ecommerce-cms",
+ "query": "Find open-source e-commerce CMS projects on GitHub that run on ordinary shared hosting."
+ },
+ {
+ "id": "remote-jobs",
+ "category": "listing",
+ "fixture": "remote-frontend-jobs",
+ "query": "Find remote frontend developer jobs open to applicants in Nigeria."
+ },
+ {
+ "id": "similar-books",
+ "category": "listing",
+ "fixture": "books-like-science-of-evil",
+ "query": "Suggest books similar to The Science of Evil by Simon Baron-Cohen."
+ },
+ {
+ "id": "poster-tools",
+ "category": "listing",
+ "fixture": "poster-tools",
+ "query": "Find me free tools to make high-quality posters."
+ },
+ {
+ "id": "sky-blue",
+ "category": "other",
+ "fixture": "sky-blue",
+ "query": "Why is the sky blue?"
+ },
+ {
+ "id": "python-venv",
+ "category": "other",
+ "fixture": "python-venv-macos",
+ "query": "How do I set up a Python virtual environment on macOS?"
+ },
+ {
+ "id": "phone-compare",
+ "category": "other",
+ "fixture": "phone-cameras",
+ "query": "Compare the iPhone 17 Pro and Pixel 11 Pro cameras and battery life."
+ },
+ {
+ "id": "berlin-wall",
+ "category": "other",
+ "fixture": "berlin-wall",
+ "query": "What year did the Berlin Wall fall?"
+ },
+ {
+ "id": "coffee-caption",
+ "category": "other",
+ "fixture": "autumn-coffee-caption",
+ "query": "Write a short Instagram caption for an autumn coffee photo."
+ },
+ {
+ "id": "mrna",
+ "category": "other",
+ "fixture": "mrna-vaccines",
+ "query": "Explain how mRNA vaccines work."
+ },
+ {
+ "id": "fridge-mold",
+ "category": "other",
+ "fixture": "fridge-mold",
+ "query": "There is mold on my fridge door seal. Should I use bleach, 70% alcohol, or soap?"
+ },
+ {
+ "id": "rag-explainer",
+ "category": "other",
+ "fixture": "ai-news-explainer",
+ "query": "What is retrieval-augmented generation and how is it different from fine-tuning?"
+ },
+ {
+ "id": "budget-keyboards",
+ "category": "listing",
+ "fixture": "budget-keyboards",
+ "query": "Find good budget mechanical keyboards under $60."
+ },
+ {
+ "id": "free-vpns",
+ "category": "listing",
+ "fixture": "free-vpns",
+ "query": "Find VPNs that have a genuinely free plan."
+ },
+ {
+ "id": "roman-podcasts",
+ "category": "roundup",
+ "fixture": "roman-history-podcasts",
+ "query": "Recommend some podcasts about Roman history."
+ },
+ {
+ "id": "hiking-apps",
+ "category": "roundup",
+ "fixture": "hike-planning-apps",
+ "query": "Suggest apps for planning hikes."
+ }
+ ],
+ "nonItemUrls": [
+ "https://www.dataweekly.com/why-sql-matters",
+ "https://japanhealthguide.org/braces-cost",
+ "https://en.wikipedia.org/rock-spalling",
+ "https://www.techfixes.net/canon-printer-offline",
+ "https://techcabal.com/remote-jobs-guide",
+ "https://www.rtings.com/keyboard/reviews/best/budget-mechanical",
+ "https://www.tomsguide.com/best-picks/best-cheap-mechanical-keyboards",
+ "https://www.cnet.com/tech/services-and-software/best-free-vpn/",
+ "https://www.historyhit.com/best-roman-history-podcasts/",
+ "https://www.theguardian.com/culture/2026/mar/ancient-history-podcasts",
+ "https://www.outdoorgearlab.com/topics/camping-and-hiking/best-hiking-apps",
+ "https://www.rei.com/blog/hike/hiking-apps"
+ ]
+}
diff --git a/evals/evaluators/source-cards.ts b/evals/evaluators/source-cards.ts
new file mode 100644
index 00000000..08a84e6a
--- /dev/null
+++ b/evals/evaluators/source-cards.ts
@@ -0,0 +1,229 @@
+import type { Evaluation, Evaluator, RunEvaluator } from '@langfuse/client'
+
+import {
+ analyzeSourceCards,
+ isSourceCardsCategory,
+ type SourceCardsExpected,
+ type SourceCardsInput,
+ type SourceCardsMetadata
+} from '../lib/source-cards'
+
+type ItemResult = {
+ input?: unknown
+ output?: unknown
+ metadata?: unknown
+ item?: { input?: unknown; metadata?: unknown }
+}
+
+export type SourceCardsSummary = {
+ listingCount: number
+ otherCount: number
+ roundupCount: number
+ emittedCount: number
+ /** Share of listing answers that carried cards. */
+ emissionRate: number
+ /** Share of other answers that carried cards anyway. */
+ misuseRate: number
+ /** Share of roundup answers that carried cards, which can only link articles. */
+ roundupMisuseRate: number
+ /** Share of emitted answers whose cards satisfy the contract. */
+ wellFormedRate: number
+ /** Share of all emitted cards whose URL is a search result URL. */
+ groundedRate: number
+ /** Share of all emitted cards linking an article instead of the item's page. */
+ misattributedRate: number
+ /** Share of emitted answers that also repeated the cards as a list. */
+ duplicatedRate: number
+ cardCount: number
+}
+
+function answerText(output: unknown): string {
+ if (typeof output === 'string') return output
+ if (output && typeof output === 'object' && 'text' in output) {
+ const text = (output as { text?: unknown }).text
+ if (typeof text === 'string') return text
+ }
+ return ''
+}
+
+function resultUrls(input: unknown): string[] {
+ const results = (input as SourceCardsInput | undefined)?.results ?? []
+ return results.map(result => result.url)
+}
+
+function itemCategory(result: ItemResult) {
+ const metadata = (result.metadata ?? result.item?.metadata) as
+ | { category?: unknown }
+ | undefined
+ const category = metadata?.category
+ return isSourceCardsCategory(category) ? category : undefined
+}
+
+export const sourceCardsEvaluator: Evaluator<
+ SourceCardsInput,
+ SourceCardsExpected,
+ SourceCardsMetadata
+> = async ({ input, output, expectedOutput }) => {
+ const analysis = analyzeSourceCards(answerText(output), resultUrls(input))
+ const expected = expectedOutput?.emitsCards ?? true
+
+ const evaluations: Evaluation[] = [
+ {
+ name: 'source_cards_emitted',
+ value: analysis.emitted ? 1 : 0,
+ dataType: 'BOOLEAN',
+ comment: analysis.emitted
+ ? `${analysis.cards.length} card(s)`
+ : 'no cards'
+ },
+ {
+ name: 'source_cards_matches_expectation',
+ value: analysis.emitted === expected ? 1 : 0,
+ dataType: 'BOOLEAN'
+ }
+ ]
+
+ // Contract and grounding are undefined when nothing was emitted.
+ if (analysis.emitted) {
+ evaluations.push(
+ {
+ name: 'source_cards_wellformed',
+ value: analysis.wellFormed ? 1 : 0,
+ dataType: 'BOOLEAN',
+ comment: analysis.wellFormed ? 'ok' : analysis.issues.join('; ')
+ },
+ {
+ name: 'source_cards_grounded',
+ value: analysis.ungroundedUrls.length === 0 ? 1 : 0,
+ dataType: 'BOOLEAN',
+ comment:
+ analysis.ungroundedUrls.length === 0
+ ? 'all URLs from search results'
+ : `ungrounded: ${analysis.ungroundedUrls.join(', ')}`
+ },
+ {
+ name: 'source_cards_item_pages',
+ value: analysis.misattributedUrls.length === 0 ? 1 : 0,
+ dataType: 'BOOLEAN',
+ comment:
+ analysis.misattributedUrls.length === 0
+ ? 'every card links an item page'
+ : `article links: ${analysis.misattributedUrls.join(', ')}`
+ }
+ )
+ }
+
+ return evaluations
+}
+
+export function summarizeSourceCards(
+ itemResults: ItemResult[]
+): SourceCardsSummary {
+ let listingCount = 0
+ let otherCount = 0
+ let roundupCount = 0
+ let listingEmitted = 0
+ let otherEmitted = 0
+ let roundupEmitted = 0
+ let misattributedCards = 0
+ let emittedCount = 0
+ let wellFormed = 0
+ let duplicated = 0
+ let cardCount = 0
+ let groundedCards = 0
+
+ for (const result of itemResults) {
+ const analysis = analyzeSourceCards(
+ answerText(result.output),
+ resultUrls(result.input ?? result.item?.input)
+ )
+ const category = itemCategory(result)
+
+ if (analysis.emitted) {
+ emittedCount++
+ if (analysis.wellFormed) wellFormed++
+ if (analysis.duplicatedAsList) duplicated++
+ cardCount += analysis.cards.length
+ groundedCards += analysis.cards.length - analysis.ungroundedUrls.length
+ misattributedCards += analysis.misattributedUrls.length
+ }
+
+ if (category === 'listing') {
+ listingCount++
+ if (analysis.emitted) listingEmitted++
+ } else if (category === 'other') {
+ otherCount++
+ if (analysis.emitted) otherEmitted++
+ } else if (category === 'roundup') {
+ roundupCount++
+ if (analysis.emitted) roundupEmitted++
+ }
+ }
+
+ const rate = (n: number, d: number) => (d === 0 ? 0 : n / d)
+
+ return {
+ listingCount,
+ otherCount,
+ roundupCount,
+ emittedCount,
+ emissionRate: rate(listingEmitted, listingCount),
+ misuseRate: rate(otherEmitted, otherCount),
+ roundupMisuseRate: rate(roundupEmitted, roundupCount),
+ wellFormedRate: rate(wellFormed, emittedCount),
+ groundedRate: rate(groundedCards, cardCount),
+ misattributedRate: rate(misattributedCards, cardCount),
+ duplicatedRate: rate(duplicated, emittedCount),
+ cardCount
+ }
+}
+
+export const sourceCardsRunEvaluator: RunEvaluator<
+ SourceCardsInput,
+ SourceCardsExpected,
+ SourceCardsMetadata
+> = async ({ itemResults }) => {
+ const summary = summarizeSourceCards(itemResults as ItemResult[])
+
+ return [
+ {
+ name: 'emission_rate',
+ value: summary.emissionRate,
+ dataType: 'NUMERIC',
+ comment: `listing items: ${summary.listingCount}`
+ },
+ {
+ name: 'misuse_rate',
+ value: summary.misuseRate,
+ dataType: 'NUMERIC',
+ comment: `other items: ${summary.otherCount}`
+ },
+ {
+ name: 'wellformed_rate',
+ value: summary.wellFormedRate,
+ dataType: 'NUMERIC'
+ },
+ {
+ name: 'grounded_rate',
+ value: summary.groundedRate,
+ dataType: 'NUMERIC',
+ comment: `cards: ${summary.cardCount}`
+ },
+ {
+ name: 'roundup_misuse_rate',
+ value: summary.roundupMisuseRate,
+ dataType: 'NUMERIC',
+ comment: `roundup items: ${summary.roundupCount}`
+ },
+ {
+ name: 'misattributed_rate',
+ value: summary.misattributedRate,
+ dataType: 'NUMERIC'
+ },
+ {
+ name: 'duplicated_rate',
+ value: summary.duplicatedRate,
+ dataType: 'NUMERIC'
+ }
+ ]
+}
diff --git a/evals/lib/__tests__/source-cards.test.ts b/evals/lib/__tests__/source-cards.test.ts
new file mode 100644
index 00000000..0b32b174
--- /dev/null
+++ b/evals/lib/__tests__/source-cards.test.ts
@@ -0,0 +1,113 @@
+import { describe, expect, test } from 'vitest'
+
+import { analyzeSourceCards, loadSourceCardsDataset } from '../source-cards'
+
+const URLS = ['https://a.example/', 'https://b.example/', 'https://c.example/']
+
+function cardsBlock(items: { title: string; url: string }[]): string {
+ const keys = items.map((_, i) => `c${i}`)
+ return [
+ '```spec',
+ '{"op":"add","path":"/root","value":"grid"}',
+ `{"op":"add","path":"/elements/grid","value":{"type":"Grid","props":{"columns":2},"children":${JSON.stringify(keys)}}}`,
+ ...items.flatMap((item, i) => [
+ `{"op":"add","path":"/elements/c${i}","value":{"type":"Card","props":{"href":${JSON.stringify(item.url)}},"children":["t${i}"]}}`,
+ `{"op":"add","path":"/elements/t${i}","value":{"type":"Text","props":{"text":${JSON.stringify(item.title)},"weight":"semibold"},"children":[]}}`
+ ]),
+ '```'
+ ].join('\n')
+}
+
+describe('analyzeSourceCards', () => {
+ test('accepts grounded, well-formed cards', () => {
+ const answer = `Lead-in.\n\n${cardsBlock([
+ { title: 'Alpha', url: URLS[0] },
+ { title: 'Beta', url: URLS[1] }
+ ])}\n\nClosing.`
+ const analysis = analyzeSourceCards(answer, URLS)
+
+ expect(analysis.emitted).toBe(true)
+ expect(analysis.wellFormed).toBe(true)
+ expect(analysis.ungroundedUrls).toEqual([])
+ expect(analysis.duplicatedAsList).toBe(false)
+ })
+
+ test('reports URLs that are not search results', () => {
+ const answer = cardsBlock([
+ { title: 'Alpha', url: URLS[0] },
+ { title: 'Made up', url: 'https://invented.example/' }
+ ])
+ const analysis = analyzeSourceCards(answer, URLS)
+
+ expect(analysis.ungroundedUrls).toEqual(['https://invented.example/'])
+ })
+
+ test('reports cards that link an article instead of the item page', () => {
+ const answer = cardsBlock([
+ { title: 'Alpha', url: URLS[0] },
+ { title: 'Gamma', url: URLS[2] }
+ ])
+ const analysis = analyzeSourceCards(answer, URLS, new Set([URLS[2]]))
+
+ expect(analysis.ungroundedUrls).toEqual([])
+ expect(analysis.misattributedUrls).toEqual([URLS[2]])
+ })
+
+ test('does not count cards the root does not reach', () => {
+ const answer = [
+ '```spec',
+ '{"op":"add","path":"/root","value":"note"}',
+ '{"op":"add","path":"/elements/note","value":{"type":"Text","props":{"text":"Nothing linked here"},"children":[]}}',
+ `{"op":"add","path":"/elements/c1","value":{"type":"Card","props":{"href":"${URLS[0]}"},"children":[]}}`,
+ `{"op":"add","path":"/elements/c2","value":{"type":"Card","props":{"href":"${URLS[1]}"},"children":[]}}`,
+ '```'
+ ].join('\n')
+ const analysis = analyzeSourceCards(answer, URLS)
+
+ expect(analysis.cards).toEqual([])
+ expect(analysis.wellFormed).toBe(false)
+ })
+
+ test('flags a single card as malformed', () => {
+ const analysis = analyzeSourceCards(
+ cardsBlock([{ title: 'Alpha', url: URLS[0] }]),
+ URLS
+ )
+
+ expect(analysis.emitted).toBe(true)
+ expect(analysis.wellFormed).toBe(false)
+ })
+
+ test('detects the same items repeated as a list', () => {
+ const answer = `${cardsBlock([
+ { title: 'Alpha', url: URLS[0] },
+ { title: 'Beta', url: URLS[1] }
+ ])}\n\n- **Alpha:** good\n- **Beta:** also good`
+ expect(analyzeSourceCards(answer, URLS).duplicatedAsList).toBe(true)
+ })
+
+ test('ignores answers without cards, including other spec blocks', () => {
+ const answer = [
+ 'Plain answer.',
+ '',
+ '```spec',
+ '{"op":"add","path":"/root","value":"main"}',
+ '```'
+ ].join('\n')
+ const analysis = analyzeSourceCards(answer, URLS)
+
+ expect(analysis.emitted).toBe(false)
+ expect(analysis.wellFormed).toBe(false)
+ })
+})
+
+describe('loadSourceCardsDataset', () => {
+ test('expands trials and labels categories', () => {
+ const items = loadSourceCardsDataset({ trials: 2 })
+ const listing = items.filter(item => item.metadata.category === 'listing')
+
+ expect(items.length % 2).toBe(0)
+ expect(listing.length).toBeGreaterThan(0)
+ expect(listing.every(item => item.expectedOutput.emitsCards)).toBe(true)
+ })
+})
diff --git a/evals/lib/otel.ts b/evals/lib/otel.ts
index b14b2f83..c34c45de 100644
--- a/evals/lib/otel.ts
+++ b/evals/lib/otel.ts
@@ -25,7 +25,11 @@ export function setupTracing(): Tracing {
const processor = new LangfuseSpanProcessor()
registerOTel({
serviceName: 'morphic-evals',
- spanProcessors: [processor]
+ spanProcessors: [processor],
+ // The default fetch instrumentation corrupts non-streaming provider
+ // responses outside Next.js ("Invalid JSON response" on most multi-step
+ // generations). The AI SDK emits its own spans, so evals do not need it.
+ instrumentations: []
})
registerTelemetry(new LangfuseVercelAiSdkIntegration())
diff --git a/evals/lib/source-cards.ts b/evals/lib/source-cards.ts
new file mode 100644
index 00000000..1a7947f4
--- /dev/null
+++ b/evals/lib/source-cards.ts
@@ -0,0 +1,220 @@
+import {
+ reachableElements,
+ type SpecElement
+} from '@/lib/render/linked-cards-markdown'
+import { parseSpecBlock } from '@/lib/render/parse-spec-block'
+import { stripSpecBlocks } from '@/lib/render/strip-spec-blocks'
+
+import datasetJson from '../datasets/source-cards.json'
+
+import type { RelatedQuestionsInput, SearchFixture } from './dataset'
+
+export type SourceCardsInput = RelatedQuestionsInput
+
+export type SourceCardsExpected = {
+ emitsCards: boolean
+}
+
+export type SourceCardsCategory = 'listing' | 'other' | 'roundup'
+
+export type SourceCardsMetadata = {
+ itemId: string
+ category: SourceCardsCategory
+ trial: number
+}
+
+export type SourceCardsItem = {
+ input: SourceCardsInput
+ expectedOutput: SourceCardsExpected
+ metadata: SourceCardsMetadata
+}
+
+type RawItem = {
+ id: string
+ category: string
+ fixture: string
+ query: string
+}
+
+export function isSourceCardsCategory(
+ value: unknown
+): value is SourceCardsCategory {
+ return value === 'listing' || value === 'other' || value === 'roundup'
+}
+
+export function loadSourceCardsDataset({
+ trials = 1,
+ ids
+}: { trials?: number; ids?: string[] } = {}): SourceCardsItem[] {
+ const fixtures = datasetJson.fixtures as Record<string, SearchFixture[]>
+ const wanted = ids && ids.length > 0 ? new Set(ids) : null
+ const items: SourceCardsItem[] = []
+
+ for (const raw of datasetJson.items as RawItem[]) {
+ if (wanted && !wanted.has(raw.id)) continue
+
+ const results = fixtures[raw.fixture]
+ if (!results) {
+ throw new Error(
+ `Item "${raw.id}" references unknown fixture "${raw.fixture}"`
+ )
+ }
+ if (!isSourceCardsCategory(raw.category)) {
+ throw new Error(`Item "${raw.id}" has unknown category "${raw.category}"`)
+ }
+
+ for (let trial = 1; trial <= trials; trial++) {
+ items.push({
+ input: { query: raw.query, results },
+ expectedOutput: { emitsCards: raw.category === 'listing' },
+ metadata: { itemId: raw.id, category: raw.category, trial }
+ })
+ }
+ }
+
+ if (wanted) {
+ const found = new Set(items.map(item => item.metadata.itemId))
+ const missing = [...wanted].filter(id => !found.has(id))
+ if (missing.length > 0) {
+ throw new Error(`Unknown item id(s): ${missing.join(', ')}`)
+ }
+ }
+
+ return items
+}
+
+export const sourceCardsDatasetDescription = datasetJson.description
+
+/** Results that are not the page of any item the user asked for. */
+const NON_ITEM_URLS = new Set<string>(datasetJson.nonItemUrls)
+
+export type SourceCard = {
+ title: string
+ url: string
+}
+
+export type SourceCardsAnalysis = {
+ /** A spec block holding a linked Card was present. */
+ emitted: boolean
+ cards: SourceCard[]
+ /** Card URLs that are not a search result URL. */
+ ungroundedUrls: string[]
+ /**
+ * Card URLs that are a search result, but an article or roundup rather than
+ * the page of the item the card names.
+ */
+ misattributedUrls: string[]
+ /** The cards satisfy the contract the prompt asks for. */
+ wellFormed: boolean
+ issues: string[]
+ /** The same items also appear as a markdown list outside the block. */
+ duplicatedAsList: boolean
+}
+
+const SPEC_BLOCK_PATTERN = /```spec\n([\s\S]*?)```/g
+const MIN_ITEMS = 2
+const MAX_ITEMS = 8
+
+function extractCards(markdown: string): {
+ found: boolean
+ cards: SourceCard[]
+ parseFailures: number
+} {
+ const pattern = new RegExp(SPEC_BLOCK_PATTERN.source, 'g')
+ const cards: SourceCard[] = []
+ let found = false
+ let parseFailures = 0
+ let match: RegExpExecArray | null
+
+ while ((match = pattern.exec(markdown)) !== null) {
+ const source = match[1]
+ // A source card is a Card with href; other Cards are not counted.
+ if (!source.includes('"Card"') || !source.includes('"href"')) continue
+ found = true
+
+ let root: string
+ let elements: Record<string, SpecElement>
+ try {
+ const spec = parseSpecBlock(source)
+ root = spec.root
+ elements = spec.elements as typeof elements
+ } catch {
+ parseFailures++
+ continue
+ }
+
+ // Only cards reachable from the root render, so only those count.
+ for (const element of reachableElements(root, elements)) {
+ const href = element.props?.href
+ if (element.type !== 'Card' || typeof href !== 'string') continue
+ // The card's name is its first Text child, per the prompt.
+ const title = (element.children ?? [])
+ .map(key => elements[key])
+ .find(child => child?.type === 'Text')?.props?.text
+ cards.push({ title: typeof title === 'string' ? title : '', url: href })
+ }
+ }
+
+ return { found, cards, parseFailures }
+}
+
+function isWebUrl(url: string): boolean {
+ try {
+ const { protocol } = new URL(url)
+ return protocol === 'http:' || protocol === 'https:'
+ } catch {
+ return false
+ }
+}
+
+export function analyzeSourceCards(
+ markdown: string,
+ resultUrls: string[],
+ nonItemUrls: ReadonlySet<string> = NON_ITEM_URLS
+): SourceCardsAnalysis {
+ const { found, cards, parseFailures } = extractCards(markdown)
+ const allowed = new Set(resultUrls)
+ const ungroundedUrls = cards
+ .map(card => card.url)
+ .filter(url => !allowed.has(url))
+ const misattributedUrls = cards
+ .map(card => card.url)
+ .filter(url => nonItemUrls.has(url))
+
+ const issues: string[] = []
+ if (parseFailures > 0)
+ issues.push(`${parseFailures} block(s) failed to parse`)
+ if (found && cards.length < MIN_ITEMS) {
+ issues.push(`${cards.length} card(s), expected at least ${MIN_ITEMS}`)
+ }
+ if (cards.length > MAX_ITEMS) {
+ issues.push(`${cards.length} cards, expected at most ${MAX_ITEMS}`)
+ }
+ if (cards.some(card => !card.title?.trim())) issues.push('card without title')
+ if (cards.some(card => !isWebUrl(card.url))) issues.push('non-web URL')
+ if (new Set(cards.map(card => card.url)).size !== cards.length) {
+ issues.push('duplicate URL')
+ }
+
+ // A markdown list item naming a card's title means the answer repeated the
+ // cards as a list, which the prompt forbids.
+ const prose = markdown.replace(new RegExp(SPEC_BLOCK_PATTERN.source, 'g'), '')
+ const listLines = stripSpecBlocks(prose)
+ .split('\n')
+ .filter(line => /^\s*(?:[-*]|\d+\.)\s/.test(line))
+ .map(line => line.toLowerCase())
+ const repeated = cards.filter(card => {
+ const title = card.title?.toLowerCase().trim()
+ return title && listLines.some(line => line.includes(title))
+ })
+
+ return {
+ emitted: found,
+ cards,
+ ungroundedUrls,
+ misattributedUrls,
+ wellFormed: found && issues.length === 0,
+ issues,
+ duplicatedAsList: repeated.length >= MIN_ITEMS
+ }
+}
diff --git a/evals/run-source-cards.ts b/evals/run-source-cards.ts
new file mode 100644
index 00000000..1ce00805
--- /dev/null
+++ b/evals/run-source-cards.ts
@@ -0,0 +1,226 @@
+#!/usr/bin/env bun
+/**
+ * Measure how often a deployed search mode configuration presents found items
+ * as SourceCards, and how often it uses them where it should not.
+ *
+ * bun run eval:source-cards
+ * bun run eval:source-cards -- --mode quick --trials 3
+ *
+ * See evals/README.md for what this measures and why.
+ */
+import { config as dotenvConfig } from 'dotenv'
+
+dotenvConfig({ path: '.env.local' })
+
+// The search mode prompts branch on whether a general search provider is
+// configured, so an unset key would silently measure a different prompt than
+// the deployed one. Pinned before anything reads it. The fixture search tool
+// means no request ever reaches Brave.
+process.env.BRAVE_SEARCH_API_KEY ||= 'eval-pinned-general-provider'
+
+type Mode = 'quick' | 'adaptive'
+
+type Args = {
+ model?: string
+ mode: Mode
+ trials: number
+ concurrency: number
+ runName?: string
+ gate: boolean
+ ids?: string[]
+}
+
+function parseArgs(argv: string[]): Args {
+ const args: Args = {
+ trials: 1,
+ concurrency: 5,
+ gate: true,
+ mode: 'adaptive'
+ }
+
+ for (let i = 0; i < argv.length; i++) {
+ const flag = argv[i]
+ const next = () => {
+ const value = argv[++i]
+ if (value === undefined) throw new Error(`${flag} requires a value`)
+ return value
+ }
+
+ switch (flag) {
+ case '--model':
+ args.model = next()
+ break
+ case '--mode': {
+ const mode = next()
+ if (mode !== 'quick' && mode !== 'adaptive') {
+ throw new Error('--mode must be quick or adaptive')
+ }
+ args.mode = mode
+ break
+ }
+ case '--trials':
+ args.trials = Number(next())
+ break
+ case '--concurrency':
+ args.concurrency = Number(next())
+ break
+ case '--run-name':
+ args.runName = next()
+ break
+ case '--items':
+ args.ids = next()
+ .split(',')
+ .map(id => id.trim())
+ .filter(Boolean)
+ break
+ case '--no-gate':
+ args.gate = false
+ break
+ case '--help':
+ console.log(
+ [
+ 'Usage: bun run eval:source-cards [options]',
+ '',
+ ' --mode <quick|adaptive> search mode to evaluate (default: adaptive)',
+ ' --model <id> provider:model to evaluate (default: the selected mode model)',
+ ' --trials <n> generations per dataset item (default: 1)',
+ ' --concurrency <n> parallel generations (default: 5)',
+ ' --run-name <name> exact Langfuse run name (default: name + timestamp)',
+ ' --items <a,b> run only these dataset item ids',
+ ' --no-gate report metrics without failing on threshold violations'
+ ].join('\n')
+ )
+ process.exit(0)
+ break
+ default:
+ throw new Error(`Unknown argument: ${flag}`)
+ }
+ }
+
+ if (!Number.isFinite(args.trials) || args.trials < 1) {
+ throw new Error('--trials must be a positive number')
+ }
+ if (!Number.isFinite(args.concurrency) || args.concurrency < 1) {
+ throw new Error('--concurrency must be a positive number')
+ }
+
+ return args
+}
+
+async function main() {
+ const args = parseArgs(process.argv.slice(2))
+
+ // Imported after dotenv: the provider registry reads API keys at module load.
+ const { LangfuseClient } = await import('@langfuse/client')
+ const { loadSourceCardsDataset, sourceCardsDatasetDescription } =
+ await import('./lib/source-cards')
+ const { applyGate } = await import('./lib/gate')
+ type Threshold = import('./lib/gate').Threshold
+ const { setupTracing } = await import('./lib/otel')
+ const {
+ sourceCardsEvaluator,
+ sourceCardsRunEvaluator,
+ summarizeSourceCards
+ } = await import('./evaluators/source-cards')
+ const adaptiveTask = await import('./tasks/adaptive-answer')
+ const quickTask = await import('./tasks/quick-answer')
+ const thresholds = (await import('./thresholds.json')).default
+
+ const taskConfig =
+ args.mode === 'quick'
+ ? {
+ createTask: quickTask.createQuickAnswerTask,
+ defaultModel: quickTask.DEFAULT_MODEL
+ }
+ : {
+ createTask: adaptiveTask.createAdaptiveAnswerTask,
+ defaultModel: adaptiveTask.DEFAULT_MODEL
+ }
+ const model = args.model ?? taskConfig.defaultModel
+ const data = loadSourceCardsDataset({ trials: args.trials, ids: args.ids })
+ const tracing = setupTracing()
+
+ console.log(`mode: ${args.mode}`)
+ console.log(`model: ${model}`)
+ console.log(`items: ${data.length} (${args.trials} trial(s) per query)`)
+ console.log(`concurrency: ${args.concurrency}\n`)
+
+ const langfuse = new LangfuseClient()
+
+ const result = await langfuse.experiment.run({
+ name: 'source-cards',
+ runName: args.runName,
+ description: sourceCardsDatasetDescription,
+ metadata: { mode: args.mode, model, trials: args.trials },
+ data,
+ task: taskConfig.createTask({ model, tracingEnabled: true }),
+ evaluators: [sourceCardsEvaluator],
+ runEvaluators: [sourceCardsRunEvaluator],
+ maxConcurrency: args.concurrency
+ })
+
+ console.log(await result.format({ includeItemResults: true }))
+
+ const completionRate = result.itemResults.length / data.length
+ const summary = summarizeSourceCards(result.itemResults)
+ console.log('\nsummary:', JSON.stringify(summary, null, 2))
+
+ const config = thresholds['source-cards'] as Record<string, Threshold>
+
+ try {
+ if (args.gate) {
+ applyGate(result, [
+ {
+ metric: 'completionRate',
+ value: completionRate,
+ samples: data.length,
+ threshold: config.completionRate
+ },
+ {
+ metric: 'emissionRate',
+ value: summary.emissionRate,
+ samples: summary.listingCount,
+ threshold: config.emissionRate
+ },
+ {
+ metric: 'misuseRate',
+ value: summary.misuseRate,
+ samples: summary.otherCount,
+ threshold: config.misuseRate
+ },
+ {
+ metric: 'wellFormedRate',
+ value: summary.wellFormedRate,
+ samples: summary.emittedCount,
+ threshold: config.wellFormedRate
+ },
+ {
+ metric: 'groundedRate',
+ value: summary.groundedRate,
+ samples: summary.cardCount,
+ threshold: config.groundedRate
+ },
+ {
+ metric: 'misattributedRate',
+ value: summary.misattributedRate,
+ samples: summary.cardCount,
+ threshold: config.misattributedRate
+ },
+ {
+ metric: 'roundupMisuseRate',
+ value: summary.roundupMisuseRate,
+ samples: summary.roundupCount,
+ threshold: config.roundupMisuseRate
+ }
+ ])
+ }
+ } finally {
+ await tracing.flush()
+ if (result.datasetRunUrl) console.log(`\n${result.datasetRunUrl}`)
+ }
+}
+
+main().catch(error => {
+ console.error(error instanceof Error ? error.message : error)
+ process.exitCode = 1
+})
diff --git a/evals/thresholds.json b/evals/thresholds.json
index dd59e6aa..9233bb98 100644
--- a/evals/thresholds.json
+++ b/evals/thresholds.json
@@ -20,5 +20,35 @@
"value": 0.9,
"minSamples": 50
}
+ },
+ "source-cards": {
+ "completionRate": {
+ "direction": "min",
+ "value": 0.9
+ },
+ "emissionRate": {
+ "direction": "min",
+ "value": null
+ },
+ "misuseRate": {
+ "direction": "max",
+ "value": null
+ },
+ "wellFormedRate": {
+ "direction": "min",
+ "value": null
+ },
+ "groundedRate": {
+ "direction": "min",
+ "value": null
+ },
+ "misattributedRate": {
+ "direction": "max",
+ "value": null
+ },
+ "roundupMisuseRate": {
+ "direction": "max",
+ "value": null
+ }
}
}
diff --git a/lib/agents/prompts/search-mode-prompts.ts b/lib/agents/prompts/search-mode-prompts.ts
index d0e386c9..37249133 100644
--- a/lib/agents/prompts/search-mode-prompts.ts
+++ b/lib/agents/prompts/search-mode-prompts.ts
@@ -1,6 +1,7 @@
import {
getImageSpecPrompt,
- getRelatedQuestionsSpecPrompt
+ getRelatedQuestionsSpecPrompt,
+ getSourceCardsSpecPrompt
} from '@/lib/render/prompt'
import {
getContentTypesGuidance,
@@ -140,7 +141,7 @@ OUTPUT FORMAT (MANDATORY):
- Use bullets with bolded keywords for key points: \`- **Point:** concise explanation\`.
- **Use tables for comparisons** (pricing, specs, features, pros/cons) - they're clearer than bullets for side-by-side data
- Focus on delivering clear information with natural flow, avoiding rigid templates.
-- Only use fenced code blocks if the user explicitly asks for code or commands (optional \`\`\`spec blocks for images or valuable related questions are exceptions).
+- Only use fenced code blocks if the user explicitly asks for code or commands (optional \`\`\`spec blocks for images, source cards, or valuable related questions are exceptions).
- Prefer natural, conversational tone while maintaining informativeness.
- Always end with a brief conclusion that synthesizes the main points into a cohesive summary.
- Response length guidance:
@@ -173,6 +174,8 @@ End with a synthesizing conclusion that ties the main points together into a cle
${getImageSpecPrompt()}
+${getSourceCardsSpecPrompt()}
+
${getRelatedQuestionsSpecPrompt()}
`
}
@@ -348,6 +351,8 @@ Conclude with a brief synthesis that ties together the main insights into a clea
${getImageSpecPrompt()}
+${getSourceCardsSpecPrompt()}
+
${getRelatedQuestionsSpecPrompt()}
`
}
diff --git a/lib/render/__tests__/linked-cards-markdown.test.ts b/lib/render/__tests__/linked-cards-markdown.test.ts
new file mode 100644
index 00000000..de5ef909
--- /dev/null
+++ b/lib/render/__tests__/linked-cards-markdown.test.ts
@@ -0,0 +1,130 @@
+import { describe, expect, test } from 'vitest'
+
+import { linkedCardsFenceToMarkdown } from '../linked-cards-markdown'
+import { parseSpecBlock } from '../parse-spec-block'
+import { stripSpecBlocks } from '../strip-spec-blocks'
+
+const CARDS_SOURCE = [
+ '{"op":"add","path":"/root","value":"cards"}',
+ '{"op":"add","path":"/elements/cards","value":{"type":"Grid","props":{"columns":2},"children":["c1","c2"]}}',
+ '{"op":"add","path":"/elements/c1","value":{"type":"Card","props":{"href":"https://sqlbolt.com/"},"children":["c1t","c1d","c1b"]}}',
+ '{"op":"add","path":"/elements/c1t","value":{"type":"Text","props":{"text":"SQLBolt","weight":"semibold"},"children":[]}}',
+ '{"op":"add","path":"/elements/c1d","value":{"type":"Text","props":{"text":"Interactive lessons.","tone":"muted"},"children":[]}}',
+ '{"op":"add","path":"/elements/c1b","value":{"type":"Badge","props":{"text":"Free"},"children":[]}}',
+ '{"op":"add","path":"/elements/c2","value":{"type":"Card","props":{"href":"https://mode.com/sql-tutorial/"},"children":["c2t"]}}',
+ '{"op":"add","path":"/elements/c2t","value":{"type":"Text","props":{"text":"Mode SQL Tutorial","weight":"semibold"},"children":[]}}'
+].join('\n')
+
+const CARDS_FENCE = ['```spec', CARDS_SOURCE, '```'].join('\n')
+
+describe('linked card spec', () => {
+ test('validates against the catalog', () => {
+ const spec = parseSpecBlock(CARDS_SOURCE)
+ expect(spec.elements.c1.type).toBe('Card')
+ expect(spec.elements.c1t.type).toBe('Text')
+ expect(spec.elements.c1b.type).toBe('Badge')
+ })
+})
+
+describe('linkedCardsFenceToMarkdown', () => {
+ test('renders each linked card as a markdown link', () => {
+ expect(linkedCardsFenceToMarkdown(CARDS_FENCE)).toBe(
+ [
+ '- [SQLBolt](https://sqlbolt.com/): Interactive lessons. (Free)',
+ '- [Mode SQL Tutorial](https://mode.com/sql-tutorial/)'
+ ].join('\n')
+ )
+ })
+
+ test('follows the rendered order, not the definition order', () => {
+ const fence = [
+ '```spec',
+ '{"op":"add","path":"/root","value":"g"}',
+ '{"op":"add","path":"/elements/g","value":{"type":"Grid","props":{"columns":2},"children":["c2","c1"]}}',
+ '{"op":"add","path":"/elements/c1","value":{"type":"Card","props":{"href":"https://one.example/"},"children":["t1"]}}',
+ '{"op":"add","path":"/elements/t1","value":{"type":"Text","props":{"text":"One"},"children":[]}}',
+ '{"op":"add","path":"/elements/c2","value":{"type":"Card","props":{"href":"https://two.example/"},"children":["t2"]}}',
+ '{"op":"add","path":"/elements/t2","value":{"type":"Text","props":{"text":"Two"},"children":[]}}',
+ '```'
+ ].join('\n')
+ expect(linkedCardsFenceToMarkdown(fence)).toBe(
+ '- [Two](https://two.example/)\n- [One](https://one.example/)'
+ )
+ })
+
+ test('skips cards the root does not reach', () => {
+ const fence = [
+ '```spec',
+ '{"op":"add","path":"/root","value":"g"}',
+ '{"op":"add","path":"/elements/g","value":{"type":"Grid","props":{"columns":1},"children":["c1"]}}',
+ '{"op":"add","path":"/elements/c1","value":{"type":"Card","props":{"href":"https://one.example/"},"children":[]}}',
+ '{"op":"add","path":"/elements/orphan","value":{"type":"Card","props":{"href":"https://orphan.example/"},"children":[]}}',
+ '```'
+ ].join('\n')
+ expect(linkedCardsFenceToMarkdown(fence)).toBe(
+ '- [https://one.example/](https://one.example/)'
+ )
+ })
+
+ test('keeps the cards that render when one line is malformed', () => {
+ const fence = [
+ '```spec',
+ '{"op":"add","path":"/root","value":"g"}',
+ '{"op":"add","path":"/elements/g","value":{"type":"Grid","props":{"columns":2},"children":["c1","c2"]}}',
+ '{"op":"add","path":"/elements/c1","value":{"type":"Card","props":{"href":"https://one.example/"},"children":["t1"]}}',
+ '{"op":"add","path":"/elements/t1","value":{"type":"Text","props":{"text":"One"},"children":[]}}',
+ '{"op":"add","path":"/elements/c2","value":{"type":"Card","props":{"href":',
+ '```'
+ ].join('\n')
+ expect(linkedCardsFenceToMarkdown(fence)).toBe(
+ '- [One](https://one.example/)'
+ )
+ })
+
+ test('returns null instead of throwing for malformed children', () => {
+ const fence = [
+ '```spec',
+ '{"op":"add","path":"/root","value":"g"}',
+ '{"op":"add","path":"/elements/g","value":{"type":"Grid","props":{"columns":2},"children":"c1"}}',
+ '{"op":"add","path":"/elements/c1","value":{"type":"Card","props":{"href":"https://one.example/"},"children":"t1"}}',
+ '```'
+ ].join('\n')
+ expect(() => linkedCardsFenceToMarkdown(fence)).not.toThrow()
+ expect(linkedCardsFenceToMarkdown(fence)).toBeNull()
+ expect(() => stripSpecBlocks(`Answer.\n\n${fence}`)).not.toThrow()
+ })
+
+ test('returns null for cards without href', () => {
+ const fence = [
+ '```spec',
+ '{"op":"add","path":"/root","value":"c"}',
+ '{"op":"add","path":"/elements/c","value":{"type":"Card","props":{},"children":["t"]}}',
+ '{"op":"add","path":"/elements/t","value":{"type":"Text","props":{"text":"Note"},"children":[]}}',
+ '```'
+ ].join('\n')
+ expect(linkedCardsFenceToMarkdown(fence)).toBeNull()
+ })
+
+ test('returns null for a fence without cards', () => {
+ const fence = [
+ '```spec',
+ '{"op":"add","path":"/root","value":"x"}',
+ '```'
+ ].join('\n')
+ expect(linkedCardsFenceToMarkdown(fence)).toBeNull()
+ })
+
+ test('returns null for an unparseable fence', () => {
+ const fence = ['```spec', '{"type":"Card" broken', '```'].join('\n')
+ expect(linkedCardsFenceToMarkdown(fence)).toBeNull()
+ })
+})
+
+describe('stripSpecBlocks with linked cards', () => {
+ test('keeps the cards as a list when copying', () => {
+ const result = stripSpecBlocks(`Lead-in.\n\n${CARDS_FENCE}\n\nClosing.`)
+ expect(result).not.toContain('```spec')
+ expect(result).toContain('- [SQLBolt](https://sqlbolt.com/)')
+ expect(result).toContain('Closing.')
+ })
+})
diff --git a/lib/render/catalog.ts b/lib/render/catalog.ts
index 4afdf0a4..fb3eb152 100644
--- a/lib/render/catalog.ts
+++ b/lib/render/catalog.ts
@@ -42,10 +42,11 @@ export const catalog = schema.createCatalog({
z.literal(3),
z.literal(4)
]),
- gap: z.enum(['xs', 'sm', 'md', 'lg']).optional()
+ gap: z.enum(['xs', 'sm', 'md', 'lg']).optional(),
+ responsive: z.boolean().optional()
}),
description:
- 'A fixed-column CSS grid container. Used to lay out images so that cell widths are determined upfront.'
+ 'A fixed-column CSS grid container, so cell widths are determined upfront while children stream in. With responsive, it collapses to one column on narrow screens.'
},
Image: {
props: z.object({
@@ -57,6 +58,29 @@ export const catalog = schema.createCatalog({
}),
description:
'An inline image referencing a web source; click to expand with credit'
+ },
+ Card: {
+ props: z
+ .object({
+ href: z.string().optional()
+ })
+ .partial(),
+ description:
+ 'A bordered container for its children. With href, the whole card links to that page and shows its domain.'
+ },
+ Text: {
+ props: z.object({
+ text: z.string(),
+ tone: z.enum(['default', 'muted']).optional(),
+ weight: z.enum(['normal', 'semibold']).optional()
+ }),
+ description: 'A short run of text'
+ },
+ Badge: {
+ props: z.object({
+ text: z.string()
+ }),
+ description: 'A small label such as a price, location, date, or type'
}
},
actions: {
diff --git a/lib/render/components/badge.tsx b/lib/render/components/badge.tsx
new file mode 100644
index 00000000..c397f231
--- /dev/null
+++ b/lib/render/components/badge.tsx
@@ -0,0 +1,13 @@
+'use client'
+
+import type { ComponentFn } from '@json-render/react'
+
+import type { CatalogType } from './shared'
+
+export const Badge: ComponentFn<CatalogType, 'Badge'> = ({ props }) => {
+ return (
+ <span className="w-fit rounded bg-muted px-1.5 py-0.5 text-[11px] text-muted-foreground">
+ {props.text}
+ </span>
+ )
+}
diff --git a/lib/render/components/card.tsx b/lib/render/components/card.tsx
new file mode 100644
index 00000000..42bd1962
--- /dev/null
+++ b/lib/render/components/card.tsx
@@ -0,0 +1,60 @@
+'use client'
+
+import type { ComponentFn } from '@json-render/react'
+
+import { captureClient, chatIdFromPath } from '@/lib/analytics/posthog-client'
+import { displayUrlName } from '@/lib/utils/domain'
+import { faviconUrl } from '@/lib/utils/favicon'
+
+import type { CatalogType } from './shared'
+
+function isWebUrl(url: string): boolean {
+ try {
+ const { protocol } = new URL(url)
+ return protocol === 'http:' || protocol === 'https:'
+ } catch {
+ return false
+ }
+}
+
+const cardClass = 'flex h-full min-w-0 flex-col gap-1 rounded-lg border p-3'
+
+export const Card: ComponentFn<CatalogType, 'Card'> = ({ props, children }) => {
+ const { href } = props
+
+ // Only real web links make a card clickable. A javascript: or relative URL,
+ // or the prompt's EXAMPLE_ placeholders if a model copies them, renders as a
+ // plain card instead.
+ if (!href || !isWebUrl(href)) {
+ return <div className={cardClass}>{children}</div>
+ }
+
+ return (
+ <a
+ href={href}
+ target="_blank"
+ rel="noopener noreferrer"
+ onClick={() =>
+ captureClient('genui_component_clicked', {
+ type: 'card',
+ chatId:
+ typeof window === 'undefined'
+ ? undefined
+ : chatIdFromPath(window.location.pathname)
+ })
+ }
+ className={`${cardClass} transition-colors hover:bg-muted/50`}
+ >
+ {children}
+ <span className="mt-auto flex min-w-0 items-center gap-1.5 pt-1 text-xs text-muted-foreground">
+ {/* eslint-disable-next-line @next/next/no-img-element */}
+ <img
+ src={faviconUrl(href)}
+ alt=""
+ className="size-3.5 shrink-0 rounded-sm"
+ />
+ <span className="truncate">{displayUrlName(href)}</span>
+ </span>
+ </a>
+ )
+}
diff --git a/lib/render/components/grid.tsx b/lib/render/components/grid.tsx
index 56170013..7fe53f81 100644
--- a/lib/render/components/grid.tsx
+++ b/lib/render/components/grid.tsx
@@ -13,10 +13,24 @@ const gridCols = {
4: 'grid-cols-4'
} as const
+// Same column counts from the sm breakpoint up, one column below it.
+const responsiveGridCols = {
+ 1: 'grid-cols-1',
+ 2: 'grid-cols-1 sm:grid-cols-2',
+ 3: 'grid-cols-1 sm:grid-cols-3',
+ 4: 'grid-cols-1 sm:grid-cols-4'
+} as const
+
export const Grid: ComponentFn<CatalogType, 'Grid'> = ({ props, children }) => {
- const { columns, gap = 'sm' } = props
+ const { columns, gap = 'sm', responsive = false } = props
return (
- <div className={cn('grid w-full', gridCols[columns], stackGap[gap])}>
+ <div
+ className={cn(
+ 'grid w-full',
+ (responsive ? responsiveGridCols : gridCols)[columns],
+ stackGap[gap]
+ )}
+ >
{children}
</div>
)
diff --git a/lib/render/components/text.tsx b/lib/render/components/text.tsx
new file mode 100644
index 00000000..acfcb96c
--- /dev/null
+++ b/lib/render/components/text.tsx
@@ -0,0 +1,23 @@
+'use client'
+
+import type { ComponentFn } from '@json-render/react'
+
+import { cn } from '@/lib/utils'
+
+import type { CatalogType } from './shared'
+
+export const Text: ComponentFn<CatalogType, 'Text'> = ({ props }) => {
+ const { text, tone = 'default', weight = 'normal' } = props
+ return (
+ <span
+ className={cn(
+ tone === 'muted'
+ ? 'text-xs text-muted-foreground line-clamp-3'
+ : 'text-sm leading-snug text-foreground line-clamp-2',
+ weight === 'semibold' && 'font-medium'
+ )}
+ >
+ {text}
+ </span>
+ )
+}
diff --git a/lib/render/linked-cards-markdown.ts b/lib/render/linked-cards-markdown.ts
new file mode 100644
index 00000000..35cbbe0a
--- /dev/null
+++ b/lib/render/linked-cards-markdown.ts
@@ -0,0 +1,88 @@
+import { createPartialSpecParser } from './parse-spec-block'
+
+const FENCE_BODY = /^```spec\s*\n([\s\S]*?)```$/
+
+export type SpecElement = {
+ type: string
+ props?: Record<string, unknown>
+ children?: string[]
+}
+
+/**
+ * Elements reachable from the root, in the order they render. Elements the
+ * root does not reach are never shown, so they are left out.
+ */
+export function reachableElements(
+ root: string,
+ elements: Record<string, SpecElement>
+): SpecElement[] {
+ const ordered: SpecElement[] = []
+ const seen = new Set<string>()
+ const walk = (key: string) => {
+ if (seen.has(key)) return
+ seen.add(key)
+ const element = elements[key]
+ if (!element) return
+ ordered.push(element)
+ // Generated specs can carry a malformed children value.
+ if (!Array.isArray(element.children)) return
+ for (const child of element.children) walk(child)
+ }
+ walk(root)
+ return ordered
+}
+
+/**
+ * Turns a ```spec fence holding linked Cards into a markdown list, or returns
+ * null when the fence holds none. Linked cards stand in for the list the answer
+ * would otherwise contain, so dropping them from history or a copied answer
+ * would lose the items themselves. Must depend only on the fence text: the
+ * history path runs on the prompt-cache prefix.
+ */
+export function linkedCardsFenceToMarkdown(fence: string): string | null {
+ if (!fence.includes('"Card"')) return null
+
+ const body = FENCE_BODY.exec(fence.trim())?.[1]
+ if (!body) return null
+
+ // This runs on every later turn's history, so a malformed generated block
+ // must fall back to being dropped rather than fail the request.
+ try {
+ return cardsToMarkdown(body)
+ } catch {
+ return null
+ }
+}
+
+function cardsToMarkdown(body: string): string | null {
+ // The same lenient parser the renderer uses, so a block that renders some
+ // cards despite a malformed line keeps those cards here too.
+ const spec = createPartialSpecParser().parse(body)
+ if (!spec) return null
+ const elements = spec.elements as Record<string, SpecElement>
+
+ const lines: string[] = []
+ // Rendered order, so a follow-up like "the second one" points at the same
+ // item the user sees.
+ for (const element of reachableElements(spec.root, elements)) {
+ const href = element.props?.href
+ if (element.type !== 'Card' || typeof href !== 'string') continue
+
+ const children = (Array.isArray(element.children) ? element.children : [])
+ .map(key => elements[key])
+ .filter(Boolean)
+ const texts = children
+ .filter(child => child.type === 'Text')
+ .map(child => String(child.props?.text ?? ''))
+ .filter(Boolean)
+ const badges = children
+ .filter(child => child.type === 'Badge')
+ .map(child => `(${String(child.props?.text ?? '')})`)
+
+ const [title = href, ...rest] = texts
+ const detail = [...rest, ...badges].join(' ')
+ lines.push(`- [${title}](${href})${detail ? `: ${detail}` : ''}`)
+ }
+
+ return lines.length > 0 ? lines.join('\n') : null
+}
diff --git a/lib/render/prompt.ts b/lib/render/prompt.ts
index 86c75b20..34679f29 100644
--- a/lib/render/prompt.ts
+++ b/lib/render/prompt.ts
@@ -59,6 +59,67 @@ SPEC RULES:
`
}
+export const EXAMPLE_SOURCE_CARDS_SPEC_BLOCK = `\`\`\`spec
+{"op":"add","path":"/root","value":"cards"}
+{"op":"add","path":"/elements/cards","value":{"type":"Grid","props":{"columns":2,"gap":"sm","responsive":true},"children":["c1","c2"]}}
+{"op":"add","path":"/elements/c1","value":{"type":"Card","props":{"href":"EXAMPLE_URL_1"},"children":["c1t","c1d","c1b"]}}
+{"op":"add","path":"/elements/c1t","value":{"type":"Text","props":{"text":"SQLBolt","weight":"semibold"},"children":[]}}
+{"op":"add","path":"/elements/c1d","value":{"type":"Text","props":{"text":"Free interactive lessons that run SQL in the browser.","tone":"muted"},"children":[]}}
+{"op":"add","path":"/elements/c1b","value":{"type":"Badge","props":{"text":"Free"},"children":[]}}
+{"op":"add","path":"/elements/c2","value":{"type":"Card","props":{"href":"EXAMPLE_URL_2"},"children":["c2t","c2d"]}}
+{"op":"add","path":"/elements/c2t","value":{"type":"Text","props":{"text":"Mode SQL Tutorial","weight":"semibold"},"children":[]}}
+{"op":"add","path":"/elements/c2d","value":{"type":"Text","props":{"text":"Analytics-focused tutorial using real datasets.","tone":"muted"},"children":[]}}
+\`\`\``
+
+/**
+ * Returns the prompt section that instructs the LLM to present a set of real,
+ * linkable items found in search results as linked cards in a ```spec block.
+ */
+export function getSourceCardsSpecPrompt(): string {
+ return `
+SOURCE CARDS:
+When the user asks you to find, recommend, or collect a set of real things that each have their own web page
+(resources, websites, tools, apps, courses, papers, books, clinics, shops, places, jobs, articles), and the search
+results contain the item's own page for at least 2 of them, present those items as linked cards in a \`\`\`spec block
+instead of a bullet list.
+
+Do NOT use source cards for explanations, single facts, how-to steps, side-by-side comparisons of attributes (use a
+table), writing, translation, or code tasks. Also do NOT use them when the search results are only roundups, reviews,
+or articles that mention the items: answer with a bullet list citing those articles instead.
+
+AVAILABLE COMPONENTS FOR SOURCE CARDS:
+- Grid: { columns: 1 | 2 | 3 | 4, gap?: "xs" | "sm" | "md" | "lg", responsive?: boolean } - responsive collapses to one column on narrow screens.
+- Card: { href?: string } - A bordered container; with href the whole card links to that page and shows its domain.
+- Text: { text: string, tone?: "default" | "muted", weight?: "normal" | "semibold" }
+- Badge: { text: string } - A small label.
+
+SOURCE CARDS RULES:
+1. Wrap the cards in a Grid with columns=2 and responsive=true. Each item is one Card whose children are, in order:
+ a Text with the item's name (weight "semibold"), an optional Text with one short sentence from that result on why it
+ fits the request (tone "muted"), and an optional Badge with a short tag such as a price, location, date, or type,
+ only when the result states it.
+2. "href" is the item's OWN page: its official site, product or store page, listing, or paper page, copied verbatim
+ from the "url" field of a search result. NEVER fabricate, guess, or shorten URLs, and never add an item's page from
+ memory when it is not in the search results.
+3. A roundup, review, listicle, or news article that only mentions an item is NOT that item's page. Make a card only
+ for items whose own page is in the results; mention the others in prose without a card. If fewer than 2 items have
+ their own page, use no cards at all.
+4. Include 2–8 cards, most relevant first. Do not repeat the same items as a bullet list.
+5. Write a short lead-in sentence before the block and your analysis or recommendation after it.
+6. Place the block where the list belongs in the answer. The block must contain ONLY JSONL patches.
+
+Example:
+
+## Free ways to learn SQL
+
+These free courses let you practice queries as you go.
+
+${EXAMPLE_SOURCE_CARDS_SPEC_BLOCK}
+
+Start with SQLBolt for the basics, then move to Mode for analysis practice.
+`
+}
+
export const EXAMPLE_IMAGE_SPEC_BLOCK = `\`\`\`spec
{"op":"add","path":"/root","value":"grid"}
{"op":"add","path":"/elements/grid","value":{"type":"Grid","props":{"columns":2,"gap":"sm"},"children":["img1","img2"]}}
@@ -85,7 +146,7 @@ Skip images only for genuinely abstract or text-only topics where no picture hel
Never include irrelevant or decorative images, and never fabricate URLs.
AVAILABLE COMPONENTS FOR IMAGES:
-- Grid: { columns: 1 | 2 | 3 | 4, gap?: "xs" | "sm" | "md" | "lg" } - A fixed-column container that reserves cell widths upfront so streaming images don't reflow.
+- Grid: { columns: 1 | 2 | 3 | 4, gap?: "xs" | "sm" | "md" | "lg", responsive?: boolean } - responsive collapses to one column on narrow screens. - A fixed-column container that reserves cell widths upfront so streaming images don't reflow.
- Image: { src: string, sourceUrl?: string, title?: string, description?: string, aspectRatio?: "1:1" | "16:9" | "4:3" }
IMAGE SPEC RULES:
diff --git a/lib/render/registry.tsx b/lib/render/registry.tsx
index 79c3f680..44c71c9c 100644
--- a/lib/render/registry.tsx
+++ b/lib/render/registry.tsx
@@ -2,11 +2,14 @@
import { defineRegistry } from '@json-render/react'
+import { Badge } from './components/badge'
import { Button } from './components/button'
+import { Card } from './components/card'
import { Grid } from './components/grid'
import { Heading } from './components/heading'
import { Image } from './components/image'
import { Stack } from './components/stack'
+import { Text } from './components/text'
import { catalog } from './catalog'
export const { registry } = defineRegistry(catalog, {
@@ -15,7 +18,10 @@ export const { registry } = defineRegistry(catalog, {
Stack,
Button,
Grid,
- Image
+ Image,
+ Card,
+ Text,
+ Badge
},
actions: {
submitQuery: async () => {
diff --git a/lib/render/strip-spec-blocks.ts b/lib/render/strip-spec-blocks.ts
index dfdee32a..5d65500c 100644
--- a/lib/render/strip-spec-blocks.ts
+++ b/lib/render/strip-spec-blocks.ts
@@ -1,10 +1,15 @@
+import { linkedCardsFenceToMarkdown } from './linked-cards-markdown'
+
/**
- * Remove ```spec fenced blocks from text content.
+ * Remove ```spec fenced blocks from text content, keeping linked Card items as
+ * a markdown list.
* Used when copying message text to clipboard.
*/
export function stripSpecBlocks(text: string): string {
return text
- .replace(/```spec[\s\S]*?```/g, '')
+ .replace(/```spec[\s\S]*?```/g, block => {
+ return linkedCardsFenceToMarkdown(block) ?? ''
+ })
.replace(/\n{3,}/g, '\n\n')
.trim()
}
diff --git a/lib/streaming/helpers/__tests__/strip-spec-from-messages.test.ts b/lib/streaming/helpers/__tests__/strip-spec-from-messages.test.ts
index 4185a777..ae3b7b7f 100644
--- a/lib/streaming/helpers/__tests__/strip-spec-from-messages.test.ts
+++ b/lib/streaming/helpers/__tests__/strip-spec-from-messages.test.ts
@@ -41,6 +41,23 @@ describe('stripSpecFromMessages', () => {
expect(textOf(result)).toBe(input)
})
+ it('replaces a linked card block with a markdown list', () => {
+ const cards = [
+ '```spec',
+ '{"op":"add","path":"/root","value":"g"}',
+ '{"op":"add","path":"/elements/g","value":{"type":"Grid","props":{"columns":2},"children":["c"]}}',
+ '{"op":"add","path":"/elements/c","value":{"type":"Card","props":{"href":"https://sqlbolt.com/"},"children":["t"]}}',
+ '{"op":"add","path":"/elements/t","value":{"type":"Text","props":{"text":"SQLBolt","weight":"semibold"},"children":[]}}',
+ '```'
+ ].join('\n')
+ const input = `Lead-in.\n\n${cards}\n\nClosing.`
+ const [result] = stripSpecFromMessages([assistant(input)])
+
+ expect(textOf(result)).toBe(
+ 'Lead-in.\n\n- [SQLBolt](https://sqlbolt.com/)\n\nClosing.'
+ )
+ })
+
it('strips a spec block that is not the related-questions block', () => {
const input = `Here is the answer.\n\n${IMAGE_BLOCK}`
const [result] = stripSpecFromMessages([assistant(input)])
diff --git a/lib/streaming/helpers/strip-spec-from-messages.ts b/lib/streaming/helpers/strip-spec-from-messages.ts
index 423c5088..edff7df2 100644
--- a/lib/streaming/helpers/strip-spec-from-messages.ts
+++ b/lib/streaming/helpers/strip-spec-from-messages.ts
@@ -1,5 +1,7 @@
import { UIMessage } from 'ai'
+import { linkedCardsFenceToMarkdown } from '@/lib/render/linked-cards-markdown'
+
const SPEC_FENCE = /```spec[\s\S]*?```/g
// The related-questions block is the only spec block whose JSONL carries a
@@ -11,6 +13,8 @@ const RELATED_QUESTIONS_MARKER = '"icon":"related"'
* related-questions block. Dropping the rest stops prior spec payloads from
* spending context window budget on every later turn; keeping the related one
* leaves the model an example of the format it is asked to keep producing.
+ * Linked Card blocks become a markdown list instead, because they replace the
+ * list the answer would otherwise contain and a follow-up may refer to it.
*
* The result must depend only on the text passed in, because this runs on the
* prompt-cache prefix and a message has to replay identically on every turn.
@@ -25,7 +29,7 @@ function stripNonRelatedSpecBlocks(text: string): string {
return block
}
removed = true
- return ''
+ return linkedCardsFenceToMarkdown(block) ?? ''
})
if (!removed) {
diff --git a/package.json b/package.json
index 6b6c72f2..2898069a 100644
--- a/package.json
+++ b/package.json
@@ -16,7 +16,8 @@
"test": "NODE_ENV=test vitest run",
"test:watch": "vitest --watch",
"chat": "bun run scripts/chat-cli.ts",
- "eval:related-questions": "bun run evals/run-related-questions.ts"
+ "eval:related-questions": "bun run evals/run-related-questions.ts",
+ "eval:source-cards": "bun run evals/run-source-cards.ts"
},
"dependencies": {
"@ai-sdk/anthropic": "^4.0.87",