diff --git a/src/libs/agent-runtime/ollama/index.ts b/src/libs/agent-runtime/ollama/index.ts index c902c5806620f..d7cfab121892c 100644 --- a/src/libs/agent-runtime/ollama/index.ts +++ b/src/libs/agent-runtime/ollama/index.ts @@ -3,6 +3,7 @@ import { ClientOptions } from 'openai'; import { OpenAIChatMessage } from '@/libs/agent-runtime'; import { ChatModelCard } from '@/types/llm'; +import { imageUrlToBase64 } from '@/utils/imageToBase64'; import { LobeRuntimeAI } from '../BaseAI'; import { AgentRuntimeErrorType } from '../error'; @@ -40,15 +41,12 @@ export class LobeOllamaAI implements LobeRuntimeAI { options?.signal?.addEventListener('abort', abort); const response = await this.client.chat({ - messages: this.buildOllamaMessages(payload.messages), + messages: await this.buildOllamaMessages(payload.messages), model: payload.model, options: { frequency_penalty: payload.frequency_penalty, presence_penalty: payload.presence_penalty, - temperature: - payload.temperature !== undefined - ? payload.temperature / 2 - : undefined, + temperature: payload.temperature !== undefined ? payload.temperature / 2 : undefined, top_p: payload.top_p, }, stream: true, @@ -75,11 +73,15 @@ export class LobeOllamaAI implements LobeRuntimeAI { })); } - private buildOllamaMessages(messages: OpenAIChatMessage[]) { - return messages.map((message) => this.convertContentToOllamaMessage(message)); + private async buildOllamaMessages(messages: OpenAIChatMessage[]): Promise { + return await Promise.all( + messages.map(async (message) => this.convertContentToOllamaMessage(message)), + ); } - private convertContentToOllamaMessage = (message: OpenAIChatMessage): OllamaMessage => { + private convertContentToOllamaMessage = async ( + message: OpenAIChatMessage, + ): Promise => { if (typeof message.content === 'string') { return { content: message.content, role: message.role }; } @@ -97,10 +99,14 @@ export class LobeOllamaAI implements LobeRuntimeAI { break; } case 'image_url': { - const { base64 } = parseDataUri(content.image_url.url); + const { base64, type } = parseDataUri(content.image_url.url); if (base64) { ollamaMessage.images ??= []; ollamaMessage.images.push(base64); + } else if (type === 'url') { + const { base64 } = await imageUrlToBase64(content.image_url.url); + ollamaMessage.images ??= []; + ollamaMessage.images.push(base64); } break; }