Skip to content

An AI-powered search engine with a generative UI

Update Use Ollama sub model

Yoshiki Miura5df77b973ea3
diff --git a/.env.local.example b/.env.local.example
index f98ba209..25d12d91 100644
--- a/.env.local.example
+++ b/.env.local.example
@@ -1,10 +1,6 @@
# Required
# The settings below are essential for the basic functionality of the system.
-# Ollama variables
-OLLAMA_MODEL=[YOUR_OLLAMA_MODEL]
-OLLAMA_BASE_URL=[YOUR_OLLAMA_URL]
-
# OpenAI API key retrieved here: https://platform.openai.com/api-keys
OPENAI_API_KEY=[YOUR_OPENAI_API_KEY]
@@ -31,6 +27,11 @@ UPSTASH_REDIS_REST_TOKEN=[YOUR_UPSTASH_REDIS_REST_TOKEN]
# Google Generative AI API key retrieved here: https://aistudio.google.com/app/apikey
# GOOGLE_GENERATIVE_AI_API_KEY=[YOUR_GOOGLE_GENERATIVE_AI_API_KEY]
+# If you want to use Ollama, enable the following variables.
+# OLLAMA_MODEL=[YOUR_OLLAMA_MODEL] # The main model to use. Recommended: mistral or openherms
+# OLLAMA_SUB_MODEL=[YOUR_OLLAMA_SUB_MODEL] # The sub model to use. Recommended: phi3 or llama3
+# OLLAMA_BASE_URL=[YOUR_OLLAMA_URL] # The base URL to use. e.g. http://localhost:11434
+
# Only writers can set a specific model. It must be compatible with the OpenAI API.
# USE_SPECIFIC_API_FOR_WRITER=true
# SPECIFIC_API_BASE=
diff --git a/app/actions.tsx b/app/actions.tsx
index bda8accc..8eca38ca 100644
--- a/app/actions.tsx
+++ b/app/actions.tsx
@@ -49,7 +49,10 @@ async function submit(formData?: FormData, skip?: boolean) {
const groupeId = nanoid()
const useSpecificAPI = process.env.USE_SPECIFIC_API_FOR_WRITER === 'true'
- const maxMessages = useSpecificAPI ? 5 : 10
+ const useOllamaProvider = !!(
+ process.env.OLLAMA_MODEL && process.env.OLLAMA_BASE_URL
+ )
+ const maxMessages = useSpecificAPI ? 5 : useOllamaProvider ? 1 : 10
// Limit the number of messages to the maximum
messages.splice(0, Math.max(messages.length - maxMessages, 0))
// Get the user input from the form data
@@ -180,10 +183,16 @@ async function submit(formData?: FormData, skip?: boolean) {
if (!errorOccurred) {
const useGoogleProvider = process.env.GOOGLE_GENERATIVE_AI_API_KEY
+ const useOllamaProvider = !!(
+ process.env.OLLAMA_MODEL && process.env.OLLAMA_BASE_URL
+ )
let processedMessages = messages
// If using Google provider, we need to modify the messages
if (useGoogleProvider) {
- processedMessages = transformToolMessages(aiState.get().messages)
+ processedMessages = transformToolMessages(messages)
+ }
+ if (useOllamaProvider) {
+ processedMessages = [{ role: 'assistant', content: answer }]
}
aiState.update({
diff --git a/lib/agents/researcher.tsx b/lib/agents/researcher.tsx
index 92590828..a680b756 100644
--- a/lib/agents/researcher.tsx
+++ b/lib/agents/researcher.tsx
@@ -27,10 +27,12 @@ export async function researcher(
if (useOllamaProvider) {
processedMessages = transformToolMessages(messages)
}
+ const includeToolResponses = messages.some(message => message.role === 'tool')
+ const useSubModel = useOllamaProvider && includeToolResponses
const currentDate = new Date().toLocaleString()
const result = await streamText({
- model: getModel(),
+ model: getModel(useSubModel),
maxTokens: 2500,
system: `As a professional search expert, you possess the ability to search for any information on the web.
or any information on the web.
diff --git a/lib/utils/index.ts b/lib/utils/index.ts
index 2fa6a1f5..07ee81af 100644
--- a/lib/utils/index.ts
+++ b/lib/utils/index.ts
@@ -10,9 +10,10 @@ export function cn(...inputs: ClassValue[]) {
return twMerge(clsx(inputs))
}
-export function getModel() {
+export function getModel(useSubModel = false) {
const ollamaBaseUrl = process.env.OLLAMA_BASE_URL + '/api'
const ollamaModel = process.env.OLLAMA_MODEL
+ const ollamaSubModel = process.env.OLLAMA_SUB_MODEL
const openaiApiBase = process.env.OPENAI_API_BASE
const openaiApiKey = process.env.OPENAI_API_KEY
let openaiApiModel = process.env.OPENAI_API_MODEL || 'gpt-4o'
@@ -26,6 +27,10 @@ export function getModel() {
if (ollamaBaseUrl && ollamaModel) {
const ollama = createOllama({ baseURL: ollamaBaseUrl })
+ if (useSubModel && ollamaSubModel) {
+ return ollama(ollamaSubModel)
+ }
+
return ollama(ollamaModel)
}