feat(search): support using google as default search provider (#4569)

* feat(websearch): implement search window functionality and enhance search service

* feat(DefaultProvider): integrate @mozilla/readability for improved content parsing

* Add LocalSearchProvider for web page scraping

AI: Change `provider` from private to protected in BaseWebSearchProvider and implement LocalSearchProvider for web searching with browser-based content extraction.

* Add web search provider management features

Implement addWebSearchProvider function to prevent duplicates,
automatically load default providers on initialization, fix
LocalSearchProvider implementation, and update local provider
identification logic.

* Improve web search with specialized search engine parsers

Add dedicated parsers for Google, Bing, and Baidu search results,
replacing the generic URL extraction approach. Enhance page loading
with proper wait mechanisms and window cleanup. Remove DuckDuckGo
provider as it's no longer supported.

* Simplify DefaultProvider to unimplemented placeholder

* Remove default search engine from initial state

* Improve web search providers config and display

Add configuration for local search providers, remove empty apiKey fields,
and enhance the UI by sorting providers alphabetically and showing
whether they require an API key.

* Add stderr logging for MCP servers

* Make search window initially hidden

package.json

@@ -66,6 +66,7 @@
     "@electron/notarize": "^2.5.0",
     "@google/generative-ai": "^0.24.0",
     "@langchain/community": "^0.3.36",
+    "@mozilla/readability": "^0.6.0",
     "@notionhq/client": "^2.2.15",
     "@strongtz/win32-arm64-msvc": "^0.4.7",
     "@tryfabric/martian": "^1.2.4",

packages/shared/IpcChannel.ts

@@ -146,5 +146,10 @@ export enum IpcChannel {
   MiniWindowReload = 'miniwindow-reload',
 
   ReduxStateChange = 'redux-state-change',
-  ReduxStoreReady = 'redux-store-ready'
+  ReduxStoreReady = 'redux-store-ready',
+
+  // Search Window
+  SearchWindow_Open = 'search-window:open',
+  SearchWindow_Close = 'search-window:close',
+  SearchWindow_OpenUrl = 'search-window:open-url'
 }

src/main/ipc.ts

@@ -21,6 +21,7 @@ import mcpService from './services/MCPService'
 import * as NutstoreService from './services/NutstoreService'
 import ObsidianVaultService from './services/ObsidianVaultService'
 import { ProxyConfig, proxyManager } from './services/ProxyManager'
+import { searchService } from './services/SearchService'
 import { registerShortcuts, unregisterAllShortcuts } from './services/ShortcutService'
 import { TrayService } from './services/TrayService'
 import { windowService } from './services/WindowService'
@@ -291,4 +292,15 @@ export function registerIpc(mainWindow: BrowserWindow, app: Electron.App) {
   ipcMain.handle(IpcChannel.Nutstore_GetDirectoryContents, (_, token: string, path: string) =>
     NutstoreService.getDirectoryContents(token, path)
   )
+
+  // search window
+  ipcMain.handle(IpcChannel.SearchWindow_Open, async (_, uid: string) => {
+    await searchService.openSearchWindow(uid)
+  })
+  ipcMain.handle(IpcChannel.SearchWindow_Close, async (_, uid: string) => {
+    await searchService.closeSearchWindow(uid)
+  })
+  ipcMain.handle(IpcChannel.SearchWindow_OpenUrl, async (_, uid: string, url: string) => {
+    return await searchService.openUrlInSearchWindow(uid, url)
+  })
 }

src/main/services/MCPService.ts

@@ -147,8 +147,12 @@ class McpService {
             ...getDefaultEnvironment(),
             PATH: this.getEnhancedPath(process.env.PATH || ''),
             ...server.env
-          }
+          },
+          stderr: 'pipe'
         })
+        transport.stderr?.on('data', (data) =>
+          Logger.info(`[MCP] Stdio stderr for server: ${server.name} `, data.toString())
+        )
       } else {
         throw new Error('Either baseUrl or command must be provided')
       }

src/main/services/SearchService.ts

@@ -0,0 +1,82 @@
+import { is } from '@electron-toolkit/utils'
+import { BrowserWindow } from 'electron'
+
+export class SearchService {
+  private static instance: SearchService | null = null
+  private searchWindows: Record<string, BrowserWindow> = {}
+  public static getInstance(): SearchService {
+    if (!SearchService.instance) {
+      SearchService.instance = new SearchService()
+    }
+    return SearchService.instance
+  }
+
+  constructor() {
+    // Initialize the service
+  }
+
+  private async createNewSearchWindow(uid: string): Promise<BrowserWindow> {
+    const newWindow = new BrowserWindow({
+      width: 800,
+      height: 600,
+      show: false,
+      webPreferences: {
+        nodeIntegration: true,
+        contextIsolation: false,
+        devTools: is.dev
+      }
+    })
+    newWindow.webContents.session.webRequest.onBeforeSendHeaders({ urls: ['*://*/*'] }, (details, callback) => {
+      const headers = {
+        ...details.requestHeaders,
+        'User-Agent':
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+      }
+      callback({ requestHeaders: headers })
+    })
+    this.searchWindows[uid] = newWindow
+    newWindow.on('closed', () => {
+      delete this.searchWindows[uid]
+    })
+    return newWindow
+  }
+
+  public async openSearchWindow(uid: string): Promise<void> {
+    await this.createNewSearchWindow(uid)
+  }
+
+  public async closeSearchWindow(uid: string): Promise<void> {
+    const window = this.searchWindows[uid]
+    if (window) {
+      window.close()
+      delete this.searchWindows[uid]
+    }
+  }
+
+  public async openUrlInSearchWindow(uid: string, url: string): Promise<any> {
+    let window = this.searchWindows[uid]
+    if (window) {
+      await window.loadURL(url)
+    } else {
+      window = await this.createNewSearchWindow(uid)
+      await window.loadURL(url)
+    }
+
+    // Get the page content after loading the URL
+    // Wait for the page to fully load before getting the content
+    await new Promise<void>((resolve) => {
+      const loadTimeout = setTimeout(() => resolve(), 10000) // 10 second timeout
+      window.webContents.once('did-finish-load', () => {
+        clearTimeout(loadTimeout)
+        // Small delay to ensure JavaScript has executed
+        setTimeout(resolve, 500)
+      })
+    })
+
+    // Get the page content after ensuring it's fully loaded
+    const content = await window.webContents.executeJavaScript('document.documentElement.outerHTML')
+    return content
+  }
+}
+
+export const searchService = SearchService.getInstance()

src/preload/index.d.ts

@@ -175,6 +175,11 @@ declare global {
         decryptToken: (token: string) => Promise<{ username: string; access_token: string }>
         getDirectoryContents: (token: string, path: string) => Promise<any>
       }
+      searchService: {
+        openSearchWindow: (uid: string) => Promise<string>
+        closeSearchWindow: (uid: string) => Promise<string>
+        openUrlInSearchWindow: (uid: string, url: string) => Promise<string>
+      }
     }
   }
 }

src/preload/index.ts

@@ -169,6 +169,11 @@ const api = {
     decryptToken: (token: string) => ipcRenderer.invoke(IpcChannel.Nutstore_DecryptToken, token),
     getDirectoryContents: (token: string, path: string) =>
       ipcRenderer.invoke(IpcChannel.Nutstore_GetDirectoryContents, token, path)
+  },
+  searchService: {
+    openSearchWindow: (uid: string) => ipcRenderer.invoke(IpcChannel.SearchWindow_Open, uid),
+    closeSearchWindow: (uid: string) => ipcRenderer.invoke(IpcChannel.SearchWindow_Close, uid),
+    openUrlInSearchWindow: (uid: string, url: string) => ipcRenderer.invoke(IpcChannel.SearchWindow_OpenUrl, uid, url)
   }
 }
 

src/renderer/src/config/webSearchProviders.ts

@@ -31,5 +31,20 @@ export const WEB_SEARCH_PROVIDER_CONFIG = {
       official: 'https://exa.ai',
       apiKey: 'https://dashboard.exa.ai/api-keys'
     }
+  },
+  'local-google': {
+    websites: {
+      official: 'https://www.google.com'
+    }
+  },
+  'local-bing': {
+    websites: {
+      official: 'https://www.bing.com'
+    }
+  },
+  'local-baidu': {
+    websites: {
+      official: 'https://www.baidu.com'
+    }
   }
 }

src/renderer/src/hooks/useWebSearchProviders.ts

@@ -29,7 +29,15 @@ export const useWebSearchProviders = () => {
 
   return {
     providers,
-    updateWebSearchProviders: (providers: WebSearchProvider[]) => dispatch(updateWebSearchProviders(providers))
+    updateWebSearchProviders: (providers: WebSearchProvider[]) => dispatch(updateWebSearchProviders(providers)),
+    addWebSearchProvider: (provider: WebSearchProvider) => {
+      // Check if provider exists
+      const exists = providers.some((p) => p.id === provider.id)
+      if (!exists) {
+        // Use the existing update action to add the new provider
+        dispatch(updateWebSearchProviders([...providers, provider]))
+      }
+    }
   }
 }
 

src/renderer/src/pages/settings/WebSearchSettings/index.tsx

@@ -1,8 +1,10 @@
 import { useTheme } from '@renderer/context/ThemeProvider'
 import { useDefaultWebSearchProvider, useWebSearchProviders } from '@renderer/hooks/useWebSearchProviders'
+import { defaultWebSearchProviders } from '@renderer/store/websearch'
 import { WebSearchProvider } from '@renderer/types'
+import { hasObjectKey } from '@renderer/utils'
 import { Select } from 'antd'
-import { FC, useState } from 'react'
+import { FC, useEffect, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 
 import { SettingContainer, SettingDivider, SettingGroup, SettingRow, SettingRowTitle, SettingTitle } from '..'
@@ -11,12 +13,16 @@ import BlacklistSettings from './BlacklistSettings'
 import WebSearchProviderSetting from './WebSearchProviderSetting'
 
 const WebSearchSettings: FC = () => {
-  const { providers } = useWebSearchProviders()
+  const { providers, addWebSearchProvider } = useWebSearchProviders()
   const { provider: defaultProvider, setDefaultProvider } = useDefaultWebSearchProvider()
   const { t } = useTranslation()
   const [selectedProvider, setSelectedProvider] = useState<WebSearchProvider | undefined>(defaultProvider)
   const { theme: themeMode } = useTheme()
 
+  useEffect(() => {
+    defaultWebSearchProviders.map((p) => addWebSearchProvider(p))
+  })
+
   function updateSelectedWebSearchProvider(providerId: string) {
     const provider = providers.find((p) => p.id === providerId)
     if (!provider) {
@@ -39,7 +45,12 @@ const WebSearchSettings: FC = () => {
               style={{ width: '200px' }}
               onChange={(value: string) => updateSelectedWebSearchProvider(value)}
               placeholder={t('settings.websearch.search_provider_placeholder')}
-              options={providers.map((p) => ({ value: p.id, label: p.name }))}
+              options={providers
+                .toSorted((p1, p2) => p1.name.localeCompare(p2.name))
+                .map((p) => ({
+                  value: p.id,
+                  label: `${p.name} (${hasObjectKey(p, 'apiKey') ? 'ApiKey' : 'Free'})`
+                }))}
             />
           </div>
         </SettingRow>

src/renderer/src/providers/WebSearchProvider/BaseWebSearchProvider.ts

@@ -2,7 +2,7 @@ import { WebSearchProvider, WebSearchResponse } from '@renderer/types'
 
 export default abstract class BaseWebSearchProvider {
   // @ts-ignore this
-  private provider: WebSearchProvider
+  protected provider: WebSearchProvider
   protected apiKey: string
 
   constructor(provider: WebSearchProvider) {

src/renderer/src/providers/WebSearchProvider/LocalBaiduProvider.ts

@@ -0,0 +1,28 @@
+import LocalSearchProvider, { SearchItem } from './LocalSearchProvider'
+
+export default class LocalBaiduProvider extends LocalSearchProvider {
+  protected parseValidUrls(htmlContent: string): SearchItem[] {
+    const results: SearchItem[] = []
+
+    try {
+      // Parse HTML string into a DOM document
+      const parser = new DOMParser()
+      const doc = parser.parseFromString(htmlContent, 'text/html')
+
+      const items = doc.querySelectorAll('#content_left .result h3')
+      items.forEach((item) => {
+        const node = item.querySelector('a')
+        if (node) {
+          results.push({
+            title: node.textContent || '',
+            url: node.href
+          })
+        }
+      })
+    } catch (error) {
+      console.error('Failed to parse Baidu search HTML:', error)
+    }
+    console.log('Parsed Baidu search results:', results)
+    return results
+  }
+}

src/renderer/src/providers/WebSearchProvider/LocalBingProvider.ts

@@ -0,0 +1,27 @@
+import LocalSearchProvider, { SearchItem } from './LocalSearchProvider'
+
+export default class LocalBingProvider extends LocalSearchProvider {
+  protected parseValidUrls(htmlContent: string): SearchItem[] {
+    const results: SearchItem[] = []
+
+    try {
+      // Parse HTML string into a DOM document
+      const parser = new DOMParser()
+      const doc = parser.parseFromString(htmlContent, 'text/html')
+
+      const items = doc.querySelectorAll('#b_results h2')
+      items.forEach((item) => {
+        const node = item.querySelector('a')
+        if (node) {
+          results.push({
+            title: node.textContent || '',
+            url: node.href
+          })
+        }
+      })
+    } catch (error) {
+      console.error('Failed to parse Bing search HTML:', error)
+    }
+    return results
+  }
+}

src/renderer/src/providers/WebSearchProvider/LocalGoogleProvider.ts

@@ -0,0 +1,28 @@
+import LocalSearchProvider, { SearchItem } from './LocalSearchProvider'
+
+export default class LocalGoogleProvider extends LocalSearchProvider {
+  protected parseValidUrls(htmlContent: string): SearchItem[] {
+    const results: SearchItem[] = []
+
+    try {
+      // Parse HTML string into a DOM document
+      const parser = new DOMParser()
+      const doc = parser.parseFromString(htmlContent, 'text/html')
+
+      const items = doc.querySelectorAll('#search .MjjYud')
+      items.forEach((item) => {
+        const title = item.querySelector('h3')
+        const link = item.querySelector('a')
+        if (title && link) {
+          results.push({
+            title: title.textContent || '',
+            url: link.href
+          })
+        }
+      })
+    } catch (error) {
+      console.error('Failed to parse Google search HTML:', error)
+    }
+    return results
+  }
+}

src/renderer/src/providers/WebSearchProvider/LocalSearchProvider.ts

@@ -0,0 +1,132 @@
+import { Readability } from '@mozilla/readability'
+import { nanoid } from '@reduxjs/toolkit'
+import { WebSearchProvider, WebSearchResponse, WebSearchResult } from '@renderer/types'
+import TurndownService from 'turndown'
+
+import BaseWebSearchProvider from './BaseWebSearchProvider'
+
+export interface SearchItem {
+  title: string
+  url: string
+}
+
+const noContent = 'No content found'
+
+export default class LocalSearchProvider extends BaseWebSearchProvider {
+  private turndownService: TurndownService = new TurndownService()
+
+  constructor(provider: WebSearchProvider) {
+    if (!provider || !provider.url) {
+      throw new Error('Provider URL is required')
+    }
+    super(provider)
+  }
+
+  public async search(
+    query: string,
+    maxResults: number = 15,
+    excludeDomains: string[] = []
+  ): Promise<WebSearchResponse> {
+    const uid = nanoid()
+    try {
+      if (!query.trim()) {
+        throw new Error('Search query cannot be empty')
+      }
+      if (!this.provider.url) {
+        throw new Error('Provider URL is required')
+      }
+
+      const cleanedQuery = query.split('\r\n')[1] ?? query
+      const url = this.provider.url.replace('%s', encodeURIComponent(cleanedQuery))
+      const content = await window.api.searchService.openUrlInSearchWindow(uid, url)
+
+      // Parse the content to extract URLs and metadata
+      const searchItems = this.parseValidUrls(content).slice(0, maxResults)
+      console.log('Total search items:', searchItems)
+
+      const validItems = searchItems
+        .filter(
+          (item) =>
+            (item.url.startsWith('http') || item.url.startsWith('https')) &&
+            excludeDomains.includes(new URL(item.url).host) === false
+        )
+        .slice(0, maxResults)
+      // console.log('Valid search items:', validItems)
+
+      // Fetch content for each URL concurrently
+      const fetchPromises = validItems.map(async (item) => {
+        // console.log(`Fetching content for ${item.url}...`)
+        const result = await this.fetchPageContent(item.url, this.provider.usingBrowser)
+        if (
+          this.provider.contentLimit &&
+          this.provider.contentLimit != -1 &&
+          result.content.length > this.provider.contentLimit
+        ) {
+          result.content = result.content.slice(0, this.provider.contentLimit) + '...'
+        }
+        return result
+      })
+
+      // Wait for all fetches to complete
+      const results: WebSearchResult[] = await Promise.all(fetchPromises)
+
+      return {
+        query: query,
+        results: results.filter((result) => result.content != noContent)
+      }
+    } catch (error) {
+      console.error('Local search failed:', error)
+      throw new Error(`Search failed: ${error instanceof Error ? error.message : 'Unknown error'}`)
+    } finally {
+      await window.api.searchService.closeSearchWindow(uid)
+    }
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  protected parseValidUrls(htmlContent: string): SearchItem[] {
+    throw new Error('Not implemented')
+  }
+
+  private async fetchPageContent(url: string, usingBrowser: boolean = false): Promise<WebSearchResult> {
+    try {
+      const controller = new AbortController()
+      const timeoutId = setTimeout(() => controller.abort(), 30000) // 30 second timeout
+
+      let html: string
+      if (usingBrowser) {
+        html = await window.api.searchService.openUrlInSearchWindow(`search-window-${nanoid()}`, url)
+      } else {
+        const response = await fetch(url, {
+          headers: {
+            'User-Agent':
+              'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+          },
+          signal: controller.signal
+        })
+        if (!response.ok) {
+          throw new Error(`HTTP error: ${response.status}`)
+        }
+        html = await response.text()
+      }
+
+      clearTimeout(timeoutId) // Clear the timeout if fetch completes successfully
+      const parser = new DOMParser()
+      const doc = parser.parseFromString(html, 'text/html')
+      const article = new Readability(doc).parse()
+      // console.log('Parsed article:', article)
+      const markdown = this.turndownService.turndown(article?.content || '')
+      return {
+        title: article?.title || url,
+        url: url,
+        content: markdown || noContent
+      }
+    } catch (e: unknown) {
+      console.error(`Failed to fetch ${url}`, e)
+      return {
+        title: url,
+        url: url,
+        content: noContent
+      }
+    }
+  }
+}

src/renderer/src/providers/WebSearchProvider/WebSearchProviderFactory.ts

@@ -3,6 +3,9 @@ import { WebSearchProvider } from '@renderer/types'
 import BaseWebSearchProvider from './BaseWebSearchProvider'
 import DefaultProvider from './DefaultProvider'
 import ExaProvider from './ExaProvider'
+import LocalBaiduProvider from './LocalBaiduProvider'
+import LocalBingProvider from './LocalBingProvider'
+import LocalGoogleProvider from './LocalGoogleProvider'
 import SearxngProvider from './SearxngProvider'
 import TavilyProvider from './TavilyProvider'
 
@@ -15,7 +18,12 @@ export default class WebSearchProviderFactory {
         return new SearxngProvider(provider)
       case 'exa':
         return new ExaProvider(provider)
-
+      case 'local-google':
+        return new LocalGoogleProvider(provider)
+      case 'local-baidu':
+        return new LocalBaiduProvider(provider)
+      case 'local-bing':
+        return new LocalBingProvider(provider)
       default:
         return new DefaultProvider(provider)
     }

src/renderer/src/services/WebSearchService.ts

@@ -31,6 +31,10 @@ class WebSearchService {
       return false
     }
 
+    if (provider.id.startsWith('local-')) {
+      return true
+    }
+
     if (hasObjectKey(provider, 'apiKey')) {
       return provider.apiKey !== ''
     }

src/renderer/src/store/websearch.ts

@@ -35,6 +35,21 @@ const initialState: WebSearchState = {
       id: 'exa',
       name: 'Exa',
       apiKey: ''
+    },
+    {
+      id: 'local-google',
+      name: 'Google',
+      url: 'https://www.google.com/search?q=%s'
+    },
+    {
+      id: 'local-bing',
+      name: 'Bing',
+      url: 'https://cn.bing.com/search?q=%s&ensearch=1'
+    },
+    {
+      id: 'local-baidu',
+      name: 'Baidu',
+      url: 'https://www.baidu.com/s?wd=%s'
     }
   ],
   searchWithTime: true,
@@ -44,6 +59,8 @@ const initialState: WebSearchState = {
   overwrite: false
 }
 
+export const defaultWebSearchProviders = initialState.providers
+
 const websearchSlice = createSlice({
   name: 'websearch',
   initialState,
@@ -77,6 +94,15 @@ const websearchSlice = createSlice({
     },
     setOverwrite: (state, action: PayloadAction<boolean>) => {
       state.overwrite = action.payload
+    },
+    addWebSearchProvider: (state, action: PayloadAction<WebSearchProvider>) => {
+      // Check if provider with same ID already exists
+      const exists = state.providers.some((provider) => provider.id === action.payload.id)
+
+      if (!exists) {
+        // Add the new provider to the array
+        state.providers.push(action.payload)
+      }
     }
   }
 })
@@ -90,7 +116,8 @@ export const {
   setExcludeDomains,
   setMaxResult,
   setEnhanceMode,
-  setOverwrite
+  setOverwrite,
+  addWebSearchProvider
 } = websearchSlice.actions
 
 export default websearchSlice.reducer

src/renderer/src/types/index.ts

@@ -336,6 +336,9 @@ export type WebSearchProvider = {
   apiKey?: string
   apiHost?: string
   engines?: string[]
+  url?: string
+  contentLimit?: number
+  usingBrowser?: boolean
 }
 
 export type WebSearchResponse = {