From b316c3ae64a8bcc3e3565b8961b7c1c170b4142a Mon Sep 17 00:00:00 2001
From: Vaayne <liu.vaayne@gmail.com>
Date: Mon, 17 Mar 2025 14:36:50 +0800
Subject: [PATCH] feat: enhance OpenAIProvider to handle diverse content types
 in tool responses

---
 src/renderer/src/providers/OpenAIProvider.ts | 22 +++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/renderer/src/providers/OpenAIProvider.ts b/src/renderer/src/providers/OpenAIProvider.ts
index 3b134419..5a6c0830 100644
--- a/src/renderer/src/providers/OpenAIProvider.ts
+++ b/src/renderer/src/providers/OpenAIProvider.ts
@@ -513,10 +513,30 @@ export default class OpenAIProvider extends BaseProvider {
             upsertMCPToolResponse(toolResponses, { tool: mcpTool, status: 'invoking', id: toolCall.id }, onChunk)
 
             const toolCallResponse = await callMCPTool(mcpTool)
+            const toolResponsContent: { type: string; text?: string; image_url?: { url: string } }[] = []
+            for (const content of toolCallResponse.content) {
+              if (content.type === 'text') {
+                toolResponsContent.push({
+                  type: 'text',
+                  text: content.text
+                })
+              } else if (content.type === 'image') {
+                toolResponsContent.push({
+                  type: 'image_url',
+                  image_url: { url: `data:${content.mimeType};base64,${content.data}` }
+                })
+              } else {
+                console.warn('Unsupported content type:', content.type)
+                toolResponsContent.push({
+                  type: 'text',
+                  text: 'unsupported content type: ' + content.type
+                })
+              }
+            }
 
             reqMessages.push({
               role: 'tool',
-              content: toolCallResponse.content,
+              content: toolResponsContent,
               tool_call_id: toolCall.id
             } as ChatCompletionToolMessageParam)