feat: refactor web search logic and support searxng (#2543)
* feat: support searxng model and refactor web search provider * feat: basic refactor * stash: web search settings page * chore: refactor general setting and provider page * feat: finish basic refactor and add searxng search * feat: finish refactor * chore(version): 1.0.2 * feat: change blacklist match pattern * Merge branch 'main' into feat-websearch * chore: add migrate * chore: add old version migrate * refactor UI * chore(version): 1.0.5 * fix: update provider enabled: true, when check seach * chore: fix migrate bug --------- Co-authored-by: kangfenmao <kangfenmao@qq.com>
This commit is contained in:
Chen Tao
GitHub
parent
026f88d1b3
commit
40182befe9
@ -86,13 +86,13 @@ Cherry Studio is a desktop client that supports for multiple LLM providers, avai
|
||||
### Install
|
||||
|
||||
```bash
|
||||
$ yarn
|
||||
yarn
|
||||
```
|
||||
|
||||
### Development
|
||||
|
||||
```bash
|
||||
$ yarn dev
|
||||
yarn dev
|
||||
```
|
||||
|
||||
### Build
|
||||
@ -135,6 +135,8 @@ Thank you for your support and contributions!
|
||||
|
||||
- [one-api](https://github.com/songquanpeng/one-api):LLM API management and distribution system, supporting mainstream models like OpenAI, Azure, and Anthropic. Features unified API interface, suitable for key management and secondary distribution.
|
||||
|
||||
- [ublacklist](https://github.com/iorate/ublacklist):Blocks specific sites from appearing in Google search results
|
||||
|
||||
# 🚀 Contributors
|
||||
|
||||
<a href="https://github.com/kangfenmao/cherry-studio/graphs/contributors">
|
||||
|
||||
@ -45,9 +45,12 @@
|
||||
"generate:icons": "electron-icon-builder --input=./build/logo.png --output=build",
|
||||
"analyze:renderer": "VISUALIZER_RENDERER=true yarn build",
|
||||
"analyze:main": "VISUALIZER_MAIN=true yarn build",
|
||||
"check": "node scripts/check-i18n.js"
|
||||
"check": "node scripts/check-i18n.js",
|
||||
"test": "tsx --test src/**/*.test.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@agentic/searxng": "^7.3.3",
|
||||
"@agentic/tavily": "^7.3.3",
|
||||
"@electron-toolkit/preload": "^3.0.0",
|
||||
"@electron-toolkit/utils": "^3.0.0",
|
||||
"@electron/notarize": "^2.5.0",
|
||||
|
||||
1
src/renderer/src/assets/images/search/searxng.svg
Executable file
1
src/renderer/src/assets/images/search/searxng.svg
Executable file
@ -0,0 +1 @@
|
||||
<svg height="92mm" viewBox="0 0 92 92" width="92mm" xmlns="http://www.w3.org/2000/svg"><g transform="translate(-40.921303 -17.416526)"><g fill="none"><circle cx="75" cy="92" r="0" stroke="#000" stroke-width="12"/><circle cx="75.921" cy="53.903" r="30" stroke="#3050ff" stroke-width="10"/><path d="m67.514849 37.91524a18 18 0 0 1 21.051475 3.312407 18 18 0 0 1 3.137312 21.078282" stroke="#3050ff" stroke-width="5"/></g><path d="m3.706 122.09h18.846v39.963h-18.846z" fill="#3050ff" transform="matrix(.69170581 -.72217939 .72217939 .69170581 0 0)"/></g></svg>
|
||||
|
After Width: | Height: | Size: 557 B |
BIN
src/renderer/src/assets/images/search/tavily.png
Normal file
BIN
src/renderer/src/assets/images/search/tavily.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 4.2 KiB |
30
src/renderer/src/config/webSearchProviders.ts
Normal file
30
src/renderer/src/config/webSearchProviders.ts
Normal file
@ -0,0 +1,30 @@
|
||||
import SearxngLogo from '@renderer/assets/images/search/searxng.svg'
|
||||
import TavilyLogo from '@renderer/assets/images/search/tavily.png'
|
||||
import TavilyLogoDark from '@renderer/assets/images/search/tavily-dark.svg'
|
||||
export function getWebSearchProviderLogo(providerId: string) {
|
||||
switch (providerId) {
|
||||
case 'tavily':
|
||||
return TavilyLogo
|
||||
case 'tavily-dark':
|
||||
return TavilyLogoDark
|
||||
case 'searxng':
|
||||
return SearxngLogo
|
||||
|
||||
default:
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
export const WEB_SEARCH_PROVIDER_CONFIG = {
|
||||
tavily: {
|
||||
websites: {
|
||||
official: 'https://tavily.com',
|
||||
apiKey: 'https://app.tavily.com/home'
|
||||
}
|
||||
},
|
||||
searxng: {
|
||||
websites: {
|
||||
official: 'https://docs.searxng.org'
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,17 +1,17 @@
|
||||
import { useAppDispatch, useAppSelector } from '@renderer/store'
|
||||
import { setDefaultProvider as _setDefaultProvider, updateWebSearchProvider } from '@renderer/store/websearch'
|
||||
import {
|
||||
setDefaultProvider as _setDefaultProvider,
|
||||
updateWebSearchProvider,
|
||||
updateWebSearchProviders
|
||||
} from '@renderer/store/websearch'
|
||||
import { WebSearchProvider } from '@renderer/types'
|
||||
|
||||
export const useDefaultWebSearchProvider = () => {
|
||||
const defaultProvider = useAppSelector((state) => state.websearch.defaultProvider)
|
||||
const providers = useWebSearchProviders()
|
||||
const provider = providers.find((provider) => provider.id === defaultProvider)
|
||||
const { providers } = useWebSearchProviders()
|
||||
const provider = defaultProvider ? providers.find((provider) => provider.id === defaultProvider) : undefined
|
||||
const dispatch = useAppDispatch()
|
||||
|
||||
if (!provider) {
|
||||
throw new Error(`Web search provider with id ${defaultProvider} not found`)
|
||||
}
|
||||
|
||||
const setDefaultProvider = (provider: WebSearchProvider) => {
|
||||
dispatch(_setDefaultProvider(provider.id))
|
||||
}
|
||||
@ -25,14 +25,18 @@ export const useDefaultWebSearchProvider = () => {
|
||||
|
||||
export const useWebSearchProviders = () => {
|
||||
const providers = useAppSelector((state) => state.websearch.providers)
|
||||
return providers
|
||||
const dispatch = useAppDispatch()
|
||||
|
||||
return {
|
||||
providers,
|
||||
updateWebSearchProviders: (providers: WebSearchProvider[]) => dispatch(updateWebSearchProviders(providers))
|
||||
}
|
||||
}
|
||||
|
||||
export const useWebSearchProvider = (id: string) => {
|
||||
const providers = useAppSelector((state) => state.websearch.providers)
|
||||
const provider = providers.find((provider) => provider.id === id)
|
||||
const dispatch = useAppDispatch()
|
||||
|
||||
if (!provider) {
|
||||
throw new Error(`Web search provider with id ${id} not found`)
|
||||
}
|
||||
|
||||
@ -104,7 +104,7 @@
|
||||
"input.web_search": "Enable web search",
|
||||
"input.web_search.button.ok": "Go to Settings",
|
||||
"input.web_search.enable": "Enable web search",
|
||||
"input.web_search.enable_content": "Enable web search in Settings",
|
||||
"input.web_search.enable_content": "Need to check web search connectivity in settings first",
|
||||
"input.auto_resize": "Auto resize height",
|
||||
"message.new.branch": "New Branch",
|
||||
"message.new.branch.created": "New Branch Created",
|
||||
@ -863,7 +863,12 @@
|
||||
"blacklist_description": "Results from the following websites will not appear in search results",
|
||||
"blacklist_tooltip": "Please use the following format (separated by line breaks)\nexample.com\nhttps://www.example.com\nhttps://example.com\n*://*.example.com",
|
||||
"search_max_result": "Number of search results",
|
||||
"search_result_default": "Default"
|
||||
"search_result_default": "Default",
|
||||
"check": "Check",
|
||||
"search_provider": "Search service provider",
|
||||
"search_provider_placeholder": "Choose a search service provider.",
|
||||
"no_provider_selected": "Please select a search service provider before checking.",
|
||||
"check_failed": "Verification failed"
|
||||
},
|
||||
"mcp": {
|
||||
"title": "MCP Servers",
|
||||
|
||||
@ -104,7 +104,7 @@
|
||||
"input.web_search": "ウェブ検索を有効にする",
|
||||
"input.web_search.button.ok": "設定に移動",
|
||||
"input.web_search.enable": "ウェブ検索を有効にする",
|
||||
"input.web_search.enable_content": "ウェブ検索を有効にするには、設定でウェブ検索を有効にする必要があります",
|
||||
"input.web_search.enable_content": "ウェブ検索の接続性を先に設定で確認する必要があります",
|
||||
"input.auto_resize": "高さを自動調整",
|
||||
"message.new.branch": "新しいブランチ",
|
||||
"message.new.branch.created": "新しいブランチが作成されました",
|
||||
@ -863,7 +863,12 @@
|
||||
"blacklist_description": "以下のウェブサイトの結果は検索結果に表示されません",
|
||||
"blacklist_tooltip": "以下の形式を使用してください(改行区切り)\nexample.com\nhttps://www.example.com\nhttps://example.com\n*://*.example.com",
|
||||
"search_max_result": "検索結果の数",
|
||||
"search_result_default": "デフォルト"
|
||||
"search_result_default": "デフォルト",
|
||||
"check": "チェック",
|
||||
"search_provider": "検索サービスプロバイダー",
|
||||
"search_provider_placeholder": "検索サービスプロバイダーを選択する",
|
||||
"no_provider_selected": "検索サービスプロバイダーを選択してから再確認してください。",
|
||||
"check_failed": "検証に失敗しました"
|
||||
},
|
||||
"mcp": {
|
||||
"title": "MCP サーバー",
|
||||
|
||||
@ -104,7 +104,7 @@
|
||||
"input.web_search": "Включить веб-поиск",
|
||||
"input.web_search.button.ok": "Перейти в Настройки",
|
||||
"input.web_search.enable": "Включить веб-поиск",
|
||||
"input.web_search.enable_content": "Необходимо включить веб-поиск в Настройки",
|
||||
"input.web_search.enable_content": "Необходимо предварительно проверить подключение к веб-поиску в настройках",
|
||||
"input.auto_resize": "Автоматическая высота",
|
||||
"message.new.branch": "Новая ветка",
|
||||
"message.new.branch.created": "Новая ветка создана",
|
||||
@ -863,7 +863,12 @@
|
||||
"blacklist_description": "Результаты из следующих веб-сайтов не будут отображаться в результатах поиска",
|
||||
"blacklist_tooltip": "Пожалуйста, используйте следующий формат (разделенный переносами строк)\nexample.com\nhttps://www.example.com\nhttps://example.com\n*://*.example.com",
|
||||
"search_max_result": "Количество результатов поиска",
|
||||
"search_result_default": "По умолчанию"
|
||||
"search_result_default": "По умолчанию",
|
||||
"check": "проверка",
|
||||
"search_provider": "поиск сервисного провайдера",
|
||||
"search_provider_placeholder": "Выберите поставщика поисковых услуг",
|
||||
"no_provider_selected": "Пожалуйста, выберите поставщика поисковых услуг, затем проверьте.",
|
||||
"check_failed": "Проверка не прошла"
|
||||
},
|
||||
"mcp": {
|
||||
"title": "Серверы MCP",
|
||||
@ -964,3 +969,4 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -104,7 +104,7 @@
|
||||
"input.web_search": "开启网络搜索",
|
||||
"input.web_search.button.ok": "去设置",
|
||||
"input.web_search.enable": "开启网络搜索",
|
||||
"input.web_search.enable_content": "需要先在设置中开启网络搜索",
|
||||
"input.web_search.enable_content": "需要先在设置中检查网络搜索连通性",
|
||||
"input.auto_resize": "自动调整高度",
|
||||
"message.new.branch": "分支",
|
||||
"message.new.branch.created": "新分支已创建",
|
||||
@ -850,6 +850,8 @@
|
||||
"assistant.show.icon": "显示模型图标",
|
||||
"tray.title": "启用系统托盘图标",
|
||||
"websearch": {
|
||||
"check": "检查",
|
||||
"check_failed": "验证失败",
|
||||
"blacklist": "黑名单",
|
||||
"blacklist_description": "在搜索结果中不会出现以下网站的结果",
|
||||
"blacklist_tooltip": "请使用以下格式(换行分隔)\nexample.com\nhttps://www.example.com\nhttps://example.com\n*://*.example.com",
|
||||
@ -857,6 +859,9 @@
|
||||
"search_max_result": "搜索结果个数",
|
||||
"search_result_default": "默认",
|
||||
"search_with_time": "搜索包含日期",
|
||||
"search_provider": "搜索服务商",
|
||||
"search_provider_placeholder": "选择一个搜索服务商",
|
||||
"no_provider_selected": "请选择搜索服务商后再检查",
|
||||
"tavily": {
|
||||
"api_key": "Tavily API 密钥",
|
||||
"api_key.placeholder": "请输入 Tavily API 密钥",
|
||||
|
||||
@ -859,10 +859,15 @@
|
||||
},
|
||||
"title": "網路搜尋",
|
||||
"blacklist": "黑名單",
|
||||
"blacklist_description": "以下網站不會出現在搜尋結果中",
|
||||
"blacklist_description": "以下網站不會出現在搜索結果中",
|
||||
"blacklist_tooltip": "請使用以下格式(換行分隔)\nexample.com\nhttps://www.example.com\nhttps://example.com\n*://*.example.com",
|
||||
"search_max_result": "搜尋結果個數",
|
||||
"search_result_default": "預設"
|
||||
"search_result_default": "預設",
|
||||
"check": "檢查",
|
||||
"search_provider": "搜尋服務商",
|
||||
"search_provider_placeholder": "選擇一個搜尋服務商",
|
||||
"no_provider_selected": "請選擇搜索服務商後再檢查",
|
||||
"check_failed": "驗證失敗"
|
||||
},
|
||||
"display.assistant.title": "助手設定",
|
||||
"mcp": {
|
||||
|
||||
@ -29,13 +29,13 @@ const MessageContent: React.FC<Props> = ({ message: _message, model }) => {
|
||||
|
||||
// Process content to make citation numbers clickable
|
||||
const processedContent = useMemo(() => {
|
||||
if (!(message.metadata?.citations || message.metadata?.tavily)) {
|
||||
if (!(message.metadata?.citations || message.metadata?.webSearch)) {
|
||||
return message.content
|
||||
}
|
||||
|
||||
let content = message.content
|
||||
|
||||
const searchResultsCitations = message?.metadata?.tavily?.results?.map((result) => result.url) || []
|
||||
const searchResultsCitations = message?.metadata?.webSearch?.results?.map((result) => result.url) || []
|
||||
|
||||
const citations = message?.metadata?.citations || searchResultsCitations
|
||||
|
||||
@ -127,13 +127,13 @@ const MessageContent: React.FC<Props> = ({ message: _message, model }) => {
|
||||
))}
|
||||
</CitationsContainer>
|
||||
)}
|
||||
{message?.metadata?.tavily && message.status === 'success' && (
|
||||
{message?.metadata?.webSearch && message.status === 'success' && (
|
||||
<CitationsContainer className="footnotes">
|
||||
<CitationsTitle>
|
||||
{t('message.citations')}
|
||||
<InfoCircleOutlined style={{ fontSize: '14px', marginLeft: '4px', opacity: 0.6 }} />
|
||||
</CitationsTitle>
|
||||
{message.metadata.tavily.results.map((result, index) => (
|
||||
{message.metadata.webSearch.results.map((result, index) => (
|
||||
<HStack key={result.url} style={{ alignItems: 'center', gap: 8 }}>
|
||||
<span style={{ fontSize: 13, color: 'var(--color-text-2)' }}>{index + 1}.</span>
|
||||
<Favicon hostname={new URL(result.url).hostname} alt={result.title} />
|
||||
|
||||
@ -268,5 +268,4 @@ const AddButtonWrapper = styled.div`
|
||||
align-items: center;
|
||||
padding: 10px 8px;
|
||||
`
|
||||
|
||||
export default ProvidersList
|
||||
|
||||
@ -1,133 +0,0 @@
|
||||
import tavilyLogo from '@renderer/assets/images/search/tavily.svg'
|
||||
import tavilyLogoDark from '@renderer/assets/images/search/tavily-dark.svg'
|
||||
import { HStack } from '@renderer/components/Layout'
|
||||
import { useTheme } from '@renderer/context/ThemeProvider'
|
||||
import { useWebSearchProvider } from '@renderer/hooks/useWebSearchProviders'
|
||||
import { useAppDispatch, useAppSelector } from '@renderer/store'
|
||||
import { setExcludeDomains, setMaxResult, setSearchWithTime } from '@renderer/store/websearch'
|
||||
import { formatDomains } from '@renderer/utils/blacklist'
|
||||
import { Alert, Button, Input, Slider, Switch, Typography } from 'antd'
|
||||
import TextArea from 'antd/es/input/TextArea'
|
||||
import { FC, useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import styled from 'styled-components'
|
||||
|
||||
import {
|
||||
SettingContainer,
|
||||
SettingDivider,
|
||||
SettingGroup,
|
||||
SettingHelpLink,
|
||||
SettingHelpTextRow,
|
||||
SettingRow,
|
||||
SettingRowTitle,
|
||||
SettingTitle
|
||||
} from '.'
|
||||
|
||||
const WebSearchSettings: FC = () => {
|
||||
const { t } = useTranslation()
|
||||
const { Paragraph } = Typography
|
||||
const { theme } = useTheme()
|
||||
const { provider, updateProvider } = useWebSearchProvider('tavily')
|
||||
const [apiKey, setApiKey] = useState(provider.apiKey)
|
||||
const logo = theme === 'dark' ? tavilyLogoDark : tavilyLogo
|
||||
const searchWithTime = useAppSelector((state) => state.websearch.searchWithTime)
|
||||
const maxResults = useAppSelector((state) => state.websearch.maxResults)
|
||||
const excludeDomains = useAppSelector((state) => state.websearch.excludeDomains)
|
||||
const [errFormat, setErrFormat] = useState(false)
|
||||
const [blacklistInput, setBlacklistInput] = useState('')
|
||||
|
||||
const dispatch = useAppDispatch()
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (apiKey && apiKey !== provider.apiKey) {
|
||||
updateProvider({ ...provider, apiKey })
|
||||
}
|
||||
}
|
||||
}, [apiKey, provider, updateProvider])
|
||||
|
||||
useEffect(() => {
|
||||
if (excludeDomains) {
|
||||
setBlacklistInput(excludeDomains.join('\n'))
|
||||
}
|
||||
}, [excludeDomains])
|
||||
|
||||
function updateManualBlacklist(blacklist: string) {
|
||||
const blacklistDomains = blacklist.split('\n').filter((url) => url.trim() !== '')
|
||||
const { formattedDomains, hasError } = formatDomains(blacklistDomains)
|
||||
setErrFormat(hasError)
|
||||
if (hasError) return
|
||||
dispatch(setExcludeDomains(formattedDomains))
|
||||
}
|
||||
|
||||
return (
|
||||
<SettingContainer theme={theme}>
|
||||
<SettingGroup theme={theme}>
|
||||
<HStack alignItems="center" gap={10}>
|
||||
<TavilyLogo src={logo} alt="web-search" style={{ width: '60px' }} />
|
||||
</HStack>
|
||||
<SettingDivider />
|
||||
<Paragraph type="secondary" style={{ margin: '10px 0' }}>
|
||||
{t('settings.websearch.tavily.description')}
|
||||
</Paragraph>
|
||||
<Input.Password
|
||||
style={{ width: '100%' }}
|
||||
placeholder={t('settings.websearch.tavily.api_key.placeholder')}
|
||||
value={apiKey}
|
||||
onChange={(e) => setApiKey(e.target.value)}
|
||||
onBlur={() => updateProvider({ ...provider, apiKey })}
|
||||
/>
|
||||
<SettingHelpTextRow style={{ justifyContent: 'space-between', marginTop: 5 }}>
|
||||
<SettingHelpLink target="_blank" href="https://app.tavily.com/home">
|
||||
{t('settings.websearch.get_api_key')}
|
||||
</SettingHelpLink>
|
||||
</SettingHelpTextRow>
|
||||
</SettingGroup>
|
||||
<SettingGroup theme={theme}>
|
||||
<SettingTitle>{t('settings.general.title')}</SettingTitle>
|
||||
<SettingDivider />
|
||||
<SettingRow>
|
||||
<SettingRowTitle>{t('settings.websearch.search_with_time')}</SettingRowTitle>
|
||||
<Switch checked={searchWithTime} onChange={(checked) => dispatch(setSearchWithTime(checked))} />
|
||||
</SettingRow>
|
||||
<SettingDivider style={{ marginTop: 15, marginBottom: 5 }} />
|
||||
<SettingRow style={{ marginBottom: -10 }}>
|
||||
<SettingRowTitle>{t('settings.websearch.search_max_result')}</SettingRowTitle>
|
||||
<Slider
|
||||
defaultValue={maxResults}
|
||||
style={{ width: '200px' }}
|
||||
min={1}
|
||||
max={20}
|
||||
step={1}
|
||||
marks={{ 1: '1', 5: t('settings.websearch.search_result_default'), 20: '20' }}
|
||||
onChangeComplete={(value) => dispatch(setMaxResult(value))}
|
||||
/>
|
||||
</SettingRow>
|
||||
</SettingGroup>
|
||||
<SettingGroup theme={theme}>
|
||||
<SettingTitle>{t('settings.websearch.blacklist')}</SettingTitle>
|
||||
<SettingDivider />
|
||||
<SettingRow style={{ marginBottom: 10 }}>
|
||||
<SettingRowTitle>{t('settings.websearch.blacklist_description')}</SettingRowTitle>
|
||||
</SettingRow>
|
||||
<TextArea
|
||||
value={blacklistInput}
|
||||
onChange={(e) => setBlacklistInput(e.target.value)}
|
||||
placeholder={t('settings.websearch.blacklist_tooltip')}
|
||||
autoSize={{ minRows: 4, maxRows: 8 }}
|
||||
rows={4}
|
||||
/>
|
||||
<Button onClick={() => updateManualBlacklist(blacklistInput)} style={{ marginTop: 10 }}>
|
||||
{t('common.save')}
|
||||
</Button>
|
||||
{errFormat && <Alert message={t('settings.websearch.blacklist_tooltip')} type="error" />}
|
||||
</SettingGroup>
|
||||
</SettingContainer>
|
||||
)
|
||||
}
|
||||
|
||||
const TavilyLogo = styled.img`
|
||||
width: 80px;
|
||||
`
|
||||
|
||||
export default WebSearchSettings
|
||||
@ -0,0 +1,43 @@
|
||||
import { useTheme } from '@renderer/context/ThemeProvider'
|
||||
import { useAppDispatch, useAppSelector } from '@renderer/store'
|
||||
import { setMaxResult, setSearchWithTime } from '@renderer/store/websearch'
|
||||
import { Slider, Switch } from 'antd'
|
||||
import { t } from 'i18next'
|
||||
import { FC } from 'react'
|
||||
|
||||
import { SettingDivider, SettingGroup, SettingRow, SettingRowTitle, SettingTitle } from '..'
|
||||
|
||||
const BasicSettings: FC = () => {
|
||||
const { theme } = useTheme()
|
||||
const searchWithTime = useAppSelector((state) => state.websearch.searchWithTime)
|
||||
const maxResults = useAppSelector((state) => state.websearch.maxResults)
|
||||
|
||||
const dispatch = useAppDispatch()
|
||||
|
||||
return (
|
||||
<>
|
||||
<SettingGroup theme={theme}>
|
||||
<SettingTitle>{t('settings.general.title')}</SettingTitle>
|
||||
<SettingDivider />
|
||||
<SettingRow>
|
||||
<SettingRowTitle>{t('settings.websearch.search_with_time')}</SettingRowTitle>
|
||||
<Switch checked={searchWithTime} onChange={(checked) => dispatch(setSearchWithTime(checked))} />
|
||||
</SettingRow>
|
||||
<SettingDivider style={{ marginTop: 15, marginBottom: 5 }} />
|
||||
<SettingRow style={{ marginBottom: -10 }}>
|
||||
<SettingRowTitle>{t('settings.websearch.search_max_result')}</SettingRowTitle>
|
||||
<Slider
|
||||
defaultValue={maxResults}
|
||||
style={{ width: '200px' }}
|
||||
min={1}
|
||||
max={20}
|
||||
step={1}
|
||||
marks={{ 1: '1', 5: t('settings.websearch.search_result_default'), 20: '20' }}
|
||||
onChangeComplete={(value) => dispatch(setMaxResult(value))}
|
||||
/>
|
||||
</SettingRow>
|
||||
</SettingGroup>
|
||||
</>
|
||||
)
|
||||
}
|
||||
export default BasicSettings
|
||||
@ -0,0 +1,68 @@
|
||||
import { useTheme } from '@renderer/context/ThemeProvider'
|
||||
import { useAppDispatch, useAppSelector } from '@renderer/store'
|
||||
import { setExcludeDomains } from '@renderer/store/websearch'
|
||||
import { parseMatchPattern } from '@renderer/utils/blacklistMatchPattern'
|
||||
import { Alert, Button } from 'antd'
|
||||
import TextArea from 'antd/es/input/TextArea'
|
||||
import { t } from 'i18next'
|
||||
import { FC, useEffect, useState } from 'react'
|
||||
|
||||
import { SettingDivider, SettingGroup, SettingRow, SettingRowTitle, SettingTitle } from '..'
|
||||
|
||||
const BlacklistSettings: FC = () => {
|
||||
const [errFormat, setErrFormat] = useState(false)
|
||||
const [blacklistInput, setBlacklistInput] = useState('')
|
||||
const excludeDomains = useAppSelector((state) => state.websearch.excludeDomains)
|
||||
const { theme } = useTheme()
|
||||
|
||||
const dispatch = useAppDispatch()
|
||||
|
||||
useEffect(() => {
|
||||
if (excludeDomains) {
|
||||
setBlacklistInput(excludeDomains.join('\n'))
|
||||
}
|
||||
}, [excludeDomains])
|
||||
|
||||
function updateManualBlacklist(blacklist: string) {
|
||||
const blacklistDomains = blacklist.split('\n').filter((url) => url.trim() !== '')
|
||||
|
||||
const validDomains: string[] = []
|
||||
const hasError = blacklistDomains.some((domain) => {
|
||||
const parsed = parseMatchPattern(domain.trim())
|
||||
if (parsed === null) {
|
||||
return true // 有错误
|
||||
}
|
||||
validDomains.push(domain.trim())
|
||||
return false
|
||||
})
|
||||
|
||||
setErrFormat(hasError)
|
||||
if (hasError) return
|
||||
|
||||
dispatch(setExcludeDomains(validDomains))
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<SettingGroup theme={theme}>
|
||||
<SettingTitle>{t('settings.websearch.blacklist')}</SettingTitle>
|
||||
<SettingDivider />
|
||||
<SettingRow style={{ marginBottom: 10 }}>
|
||||
<SettingRowTitle>{t('settings.websearch.blacklist_description')}</SettingRowTitle>
|
||||
</SettingRow>
|
||||
<TextArea
|
||||
value={blacklistInput}
|
||||
onChange={(e) => setBlacklistInput(e.target.value)}
|
||||
placeholder={t('settings.websearch.blacklist_tooltip')}
|
||||
autoSize={{ minRows: 4, maxRows: 8 }}
|
||||
rows={4}
|
||||
/>
|
||||
<Button onClick={() => updateManualBlacklist(blacklistInput)} style={{ marginTop: 10 }}>
|
||||
{t('common.save')}
|
||||
</Button>
|
||||
{errFormat && <Alert message={t('settings.websearch.blacklist_tooltip')} type="error" />}
|
||||
</SettingGroup>
|
||||
</>
|
||||
)
|
||||
}
|
||||
export default BlacklistSettings
|
||||
@ -0,0 +1,97 @@
|
||||
import { ExportOutlined } from '@ant-design/icons'
|
||||
import { WEB_SEARCH_PROVIDER_CONFIG } from '@renderer/config/webSearchProviders'
|
||||
import { useWebSearchProvider } from '@renderer/hooks/useWebSearchProviders'
|
||||
import { WebSearchProvider } from '@renderer/types'
|
||||
import { Divider, Flex, Input } from 'antd'
|
||||
import Link from 'antd/es/typography/Link'
|
||||
import { FC, useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import styled from 'styled-components'
|
||||
|
||||
import { SettingHelpLink, SettingHelpTextRow, SettingSubtitle, SettingTitle } from '..'
|
||||
|
||||
interface Props {
|
||||
provider: WebSearchProvider
|
||||
}
|
||||
const WebSearchProviderSetting: FC<Props> = ({ provider: _provider }) => {
|
||||
const { provider, updateProvider } = useWebSearchProvider(_provider.id)
|
||||
const { t } = useTranslation()
|
||||
const [apiKey, setApiKey] = useState(provider.apiKey)
|
||||
const [apiHost, setApiHost] = useState(provider.apiHost)
|
||||
|
||||
const webSearchProviderConfig = WEB_SEARCH_PROVIDER_CONFIG[provider.id]
|
||||
const apiKeyWebsite = webSearchProviderConfig?.websites?.apiKey
|
||||
const officialWebsite = webSearchProviderConfig?.websites?.official
|
||||
const onUpdateApiKey = () => {
|
||||
if (apiKey !== provider.apiKey) {
|
||||
updateProvider({ ...provider, apiKey })
|
||||
}
|
||||
}
|
||||
|
||||
const onUpdateApiHost = () => {
|
||||
const trimmedHost = apiHost?.trim() || ''
|
||||
if (trimmedHost !== provider.apiHost) {
|
||||
updateProvider({ ...provider, apiHost: trimmedHost })
|
||||
} else {
|
||||
setApiHost(provider.apiHost || '')
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
setApiKey(provider.apiKey ?? '')
|
||||
setApiHost(provider.apiHost ?? '')
|
||||
}, [provider.apiKey, provider.apiHost])
|
||||
|
||||
return (
|
||||
<>
|
||||
<SettingTitle>
|
||||
<Flex align="center" gap={8}>
|
||||
<ProviderName> {provider.name}</ProviderName>
|
||||
{officialWebsite && webSearchProviderConfig?.websites && (
|
||||
<Link target="_blank" href={webSearchProviderConfig.websites.official}>
|
||||
<ExportOutlined style={{ color: 'var(--color-text)', fontSize: '12px' }} />
|
||||
</Link>
|
||||
)}
|
||||
</Flex>
|
||||
</SettingTitle>
|
||||
<Divider style={{ width: '100%', margin: '10px 0' }} />
|
||||
{provider.apiKey !== undefined && (
|
||||
<>
|
||||
<SettingSubtitle style={{ marginTop: 5 }}>{t('settings.provider.api_key')}</SettingSubtitle>
|
||||
<Input.Password
|
||||
value={apiKey}
|
||||
placeholder={t('settings.provider.api_key')}
|
||||
onChange={(e) => setApiKey(e.target.value)}
|
||||
onBlur={onUpdateApiKey}
|
||||
spellCheck={false}
|
||||
type="password"
|
||||
autoFocus={apiKey === ''}
|
||||
/>
|
||||
<SettingHelpTextRow style={{ justifyContent: 'space-between', marginTop: 5 }}>
|
||||
<SettingHelpLink target="_blank" href={apiKeyWebsite}>
|
||||
{t('settings.websearch.get_api_key')}
|
||||
</SettingHelpLink>
|
||||
</SettingHelpTextRow>
|
||||
</>
|
||||
)}
|
||||
{provider.apiHost !== undefined && (
|
||||
<>
|
||||
<SettingSubtitle style={{ marginTop: 5 }}>{t('settings.provider.api_host')}</SettingSubtitle>
|
||||
<Input
|
||||
value={apiHost}
|
||||
placeholder={t('settings.provider.api_host')}
|
||||
onChange={(e) => setApiHost(e.target.value)}
|
||||
onBlur={onUpdateApiHost}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
const ProviderName = styled.span`
|
||||
font-size: 14px;
|
||||
font-weight: 500;
|
||||
`
|
||||
|
||||
export default WebSearchProviderSetting
|
||||
108
src/renderer/src/pages/settings/WebSearchSettings/index.tsx
Normal file
108
src/renderer/src/pages/settings/WebSearchSettings/index.tsx
Normal file
@ -0,0 +1,108 @@
|
||||
import { CheckOutlined, InfoCircleOutlined, LoadingOutlined } from '@ant-design/icons'
|
||||
import { useTheme } from '@renderer/context/ThemeProvider'
|
||||
import { useDefaultWebSearchProvider, useWebSearchProviders } from '@renderer/hooks/useWebSearchProviders'
|
||||
import WebSearchService from '@renderer/services/WebSearchService'
|
||||
import { WebSearchProvider } from '@renderer/types'
|
||||
import { Button, Select } from 'antd'
|
||||
import { FC, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
import { SettingContainer, SettingDivider, SettingGroup, SettingRow, SettingRowTitle, SettingTitle } from '..'
|
||||
import BasicSettings from './BasicSettings'
|
||||
import BlacklistSettings from './BlacklistSettings'
|
||||
import WebSearchProviderSetting from './WebSearchProviderSetting'
|
||||
|
||||
const WebSearchSettings: FC = () => {
|
||||
const { providers } = useWebSearchProviders()
|
||||
const { provider: defaultProvider, setDefaultProvider, updateDefaultProvider } = useDefaultWebSearchProvider()
|
||||
|
||||
const { t } = useTranslation()
|
||||
const [selectedProvider, setSelectedProvider] = useState<WebSearchProvider | undefined>(defaultProvider)
|
||||
const { theme: themeMode } = useTheme()
|
||||
|
||||
const [apiChecking, setApiChecking] = useState(false)
|
||||
const [apiValid, setApiValid] = useState(false)
|
||||
|
||||
function updateSelectedWebSearchProvider(providerId: string) {
|
||||
const provider = providers.find((p) => p.id === providerId)
|
||||
if (!provider) {
|
||||
return
|
||||
}
|
||||
setApiValid(false)
|
||||
setSelectedProvider(provider)
|
||||
setDefaultProvider(provider)
|
||||
}
|
||||
async function checkSearch() {
|
||||
// 检查是否选择了提供商
|
||||
if (!selectedProvider || !selectedProvider.id) {
|
||||
window.message.error({
|
||||
content: t('settings.websearch.no_provider_selected'),
|
||||
duration: 3,
|
||||
icon: <InfoCircleOutlined />,
|
||||
key: 'no-provider-selected'
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
setApiChecking(true)
|
||||
const { valid, error } = await WebSearchService.checkSearch(selectedProvider)
|
||||
|
||||
setApiValid(valid)
|
||||
|
||||
// 如果验证失败且有错误信息，显示错误
|
||||
if (!valid && error) {
|
||||
const errorMessage = error.message ? ' ' + error.message : ''
|
||||
window.message.error({
|
||||
content: errorMessage,
|
||||
duration: 4,
|
||||
key: 'search-check-error'
|
||||
})
|
||||
}
|
||||
updateDefaultProvider({ ...selectedProvider, enabled: true })
|
||||
} catch (err) {
|
||||
console.error('Check search error:', err)
|
||||
setApiValid(false)
|
||||
window.message.error({
|
||||
content: t('settings.websearch.check_failed'),
|
||||
duration: 3,
|
||||
key: 'check-search-error'
|
||||
})
|
||||
} finally {
|
||||
setApiChecking(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<SettingContainer theme={themeMode}>
|
||||
<SettingGroup theme={themeMode}>
|
||||
<SettingTitle>{t('settings.websearch.title')}</SettingTitle>
|
||||
<SettingDivider />
|
||||
<SettingRow>
|
||||
<SettingRowTitle>{t('settings.websearch.search_provider')}</SettingRowTitle>
|
||||
<div style={{ display: 'flex', gap: '8px' }}>
|
||||
<Select
|
||||
value={selectedProvider?.id}
|
||||
style={{ width: '200px' }}
|
||||
onChange={(value: string) => updateSelectedWebSearchProvider(value)}
|
||||
placeholder={t('settings.websearch.search_provider_placeholder')}
|
||||
options={providers.map((p) => ({ value: p.id, label: p.name }))}
|
||||
/>
|
||||
<Button
|
||||
ghost={apiValid}
|
||||
type={apiValid ? 'primary' : 'default'}
|
||||
onClick={async () => await checkSearch()}
|
||||
disabled={apiChecking}>
|
||||
{apiChecking ? <LoadingOutlined spin /> : apiValid ? <CheckOutlined /> : t('settings.websearch.check')}
|
||||
</Button>
|
||||
</div>
|
||||
</SettingRow>
|
||||
<SettingDivider />
|
||||
{selectedProvider && <WebSearchProviderSetting provider={selectedProvider} />}
|
||||
</SettingGroup>
|
||||
<BasicSettings />
|
||||
<BlacklistSettings />
|
||||
</SettingContainer>
|
||||
)
|
||||
}
|
||||
export default WebSearchSettings
|
||||
@ -33,6 +33,7 @@ export async function fetchChatCompletion({
|
||||
window.keyv.set(EVENT_NAMES.CHAT_COMPLETION_PAUSED, false)
|
||||
|
||||
const provider = getAssistantProvider(assistant)
|
||||
const webSearchProvider = WebSearchService.getWebSearchProvider()
|
||||
const AI = new AiProvider(provider)
|
||||
|
||||
store.dispatch(setGenerating(true))
|
||||
@ -67,11 +68,14 @@ export async function fetchChatCompletion({
|
||||
})
|
||||
}
|
||||
onResponse({ ...message, status: 'searching' })
|
||||
const webSearch = await WebSearchService.search(lastMessage.content)
|
||||
console.log('webSearchProvider', webSearchProvider)
|
||||
const webSearch = await WebSearchService.search(webSearchProvider, lastMessage.content)
|
||||
console.log('webSearch', webSearch)
|
||||
message.metadata = {
|
||||
...message.metadata,
|
||||
tavily: webSearch
|
||||
webSearch: webSearch
|
||||
}
|
||||
console.log('message', message)
|
||||
window.keyv.set(`web-search-${lastMessage?.id}`, webSearch)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,44 +1,110 @@
|
||||
import store from '@renderer/store'
|
||||
import { WebSearchProvider } from '@renderer/types'
|
||||
import { tavily } from '@tavily/core'
|
||||
import { setDefaultProvider } from '@renderer/store/websearch'
|
||||
import { WebSearchProvider, WebSearchResponse } from '@renderer/types'
|
||||
import WebSearchEngineProvider from '@renderer/webSearchProvider/WebSearchEngineProvider'
|
||||
import dayjs from 'dayjs'
|
||||
|
||||
class WebSearchService {
|
||||
public isWebSearchEnabled(): boolean {
|
||||
const defaultProvider = store.getState().websearch.defaultProvider
|
||||
const providers = store.getState().websearch.providers
|
||||
const provider = providers.find((provider) => provider.id === defaultProvider)
|
||||
return provider?.apiKey ? true : false
|
||||
interface WebSearchState {
|
||||
// 默认搜索提供商的ID
|
||||
defaultProvider: string
|
||||
// 所有可用的搜索提供商列表
|
||||
providers: WebSearchProvider[]
|
||||
// 是否在搜索查询中添加当前日期
|
||||
searchWithTime: boolean
|
||||
// 搜索结果的最大数量
|
||||
maxResults: number
|
||||
// 要排除的域名列表
|
||||
excludeDomains: string[]
|
||||
}
|
||||
|
||||
public getWebSearchProvider(): WebSearchProvider {
|
||||
const defaultProvider = store.getState().websearch.defaultProvider
|
||||
const providers = store.getState().websearch.providers
|
||||
/**
|
||||
* 提供网络搜索相关功能的服务类
|
||||
*/
|
||||
class WebSearchService {
|
||||
/**
|
||||
* 获取当前存储的网络搜索状态
|
||||
* @private
|
||||
* @returns 网络搜索状态
|
||||
*/
|
||||
private getWebSearchState(): WebSearchState {
|
||||
return store.getState().websearch
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查网络搜索功能是否启用
|
||||
* @public
|
||||
* @returns 如果默认搜索提供商已启用则返回true，否则返回false
|
||||
*/
|
||||
public isWebSearchEnabled(): boolean {
|
||||
const { defaultProvider, providers } = this.getWebSearchState()
|
||||
const provider = providers.find((provider) => provider.id === defaultProvider)
|
||||
return provider?.enabled ?? false
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前默认的网络搜索提供商
|
||||
* @public
|
||||
* @returns 网络搜索提供商
|
||||
* @throws 如果找不到默认提供商则抛出错误
|
||||
*/
|
||||
public getWebSearchProvider(): WebSearchProvider {
|
||||
const { defaultProvider, providers } = this.getWebSearchState()
|
||||
let provider = providers.find((provider) => provider.id === defaultProvider)
|
||||
|
||||
if (!provider) {
|
||||
throw new Error(`Web search provider with id ${defaultProvider} not found`)
|
||||
provider = providers.find((p) => p.enabled) || providers[0]
|
||||
if (provider) {
|
||||
// 可选：自动更新默认提供商
|
||||
store.dispatch(setDefaultProvider(provider.id))
|
||||
} else {
|
||||
throw new Error(`No web search providers available`)
|
||||
}
|
||||
}
|
||||
|
||||
return provider
|
||||
}
|
||||
|
||||
public async search(query: string) {
|
||||
const searchWithTime = store.getState().websearch.searchWithTime
|
||||
const maxResults = store.getState().websearch.maxResults
|
||||
const excludeDomains = store.getState().websearch.excludeDomains
|
||||
let formatted_query = query
|
||||
if (searchWithTime) {
|
||||
formatted_query = `today is ${dayjs().format('YYYY-MM-DD')} \r\n ${query}`
|
||||
}
|
||||
const provider = this.getWebSearchProvider()
|
||||
const tvly = tavily({ apiKey: provider.apiKey })
|
||||
const result = await tvly.search(formatted_query, {
|
||||
maxResults: maxResults,
|
||||
excludeDomains: excludeDomains
|
||||
})
|
||||
/**
|
||||
* 使用指定的提供商执行网络搜索
|
||||
* @public
|
||||
* @param provider 搜索提供商
|
||||
* @param query 搜索查询
|
||||
* @returns 搜索响应
|
||||
*/
|
||||
public async search(provider: WebSearchProvider, query: string): Promise<WebSearchResponse> {
|
||||
const { searchWithTime, maxResults, excludeDomains } = this.getWebSearchState()
|
||||
const webSearchEngine = new WebSearchEngineProvider(provider)
|
||||
|
||||
return result
|
||||
let formattedQuery = query
|
||||
if (searchWithTime) {
|
||||
formattedQuery = `today is ${dayjs().format('YYYY-MM-DD')} \r\n ${query}`
|
||||
}
|
||||
|
||||
try {
|
||||
return await webSearchEngine.search(formattedQuery, maxResults, excludeDomains)
|
||||
} catch (error) {
|
||||
console.error('Search failed:', error)
|
||||
return {
|
||||
results: []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查搜索提供商是否正常工作
|
||||
* @public
|
||||
* @param provider 要检查的搜索提供商
|
||||
* @returns 如果提供商可用返回true，否则返回false
|
||||
*/
|
||||
public async checkSearch(provider: WebSearchProvider): Promise<{ valid: boolean; error?: any }> {
|
||||
try {
|
||||
const response = await this.search(provider, 'test query')
|
||||
|
||||
// 优化的判断条件：检查结果是否有效且没有错误
|
||||
return { valid: response.results.length > 0, error: undefined }
|
||||
} catch (error) {
|
||||
return { valid: false, error }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1214,6 +1214,16 @@ const migrateConfig = {
|
||||
isSystem: true,
|
||||
enabled: false
|
||||
})
|
||||
const existWebsearchProvider = state.websearch.providers.find((p) => p.id === 'tavily')
|
||||
if (existWebsearchProvider && existWebsearchProvider.apiKey !== '') {
|
||||
existWebsearchProvider.enabled = true
|
||||
}
|
||||
state.websearch.providers.push({
|
||||
id: 'searxng',
|
||||
name: 'Searxng',
|
||||
enabled: false,
|
||||
apiHost: ''
|
||||
})
|
||||
return state
|
||||
}
|
||||
}
|
||||
|
||||
@ -9,12 +9,19 @@ export interface WebSearchState {
|
||||
}
|
||||
|
||||
const initialState: WebSearchState = {
|
||||
defaultProvider: 'tavily',
|
||||
defaultProvider: '',
|
||||
providers: [
|
||||
{
|
||||
id: 'tavily',
|
||||
name: 'Tavily',
|
||||
enabled: false,
|
||||
apiKey: ''
|
||||
},
|
||||
{
|
||||
id: 'searxng',
|
||||
name: 'Searxng',
|
||||
enabled: false,
|
||||
apiHost: ''
|
||||
}
|
||||
],
|
||||
searchWithTime: true,
|
||||
@ -32,6 +39,10 @@ const websearchSlice = createSlice({
|
||||
setWebSearchProviders: (state, action: PayloadAction<WebSearchProvider[]>) => {
|
||||
state.providers = action.payload
|
||||
},
|
||||
updateWebSearchProviders: (state, action: PayloadAction<WebSearchProvider[]>) => {
|
||||
state.providers = action.payload
|
||||
},
|
||||
|
||||
updateWebSearchProvider: (state, action: PayloadAction<WebSearchProvider>) => {
|
||||
const index = state.providers.findIndex((provider) => provider.id === action.payload.id)
|
||||
if (index !== -1) {
|
||||
@ -53,6 +64,7 @@ const websearchSlice = createSlice({
|
||||
export const {
|
||||
setWebSearchProviders,
|
||||
updateWebSearchProvider,
|
||||
updateWebSearchProviders,
|
||||
setDefaultProvider,
|
||||
setSearchWithTime,
|
||||
setExcludeDomains,
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
import type { TavilySearchResponse } from '@tavily/core'
|
||||
import OpenAI from 'openai'
|
||||
import React from 'react'
|
||||
import { BuiltinTheme } from 'shiki'
|
||||
@ -74,7 +73,7 @@ export type Message = {
|
||||
// Perplexity
|
||||
citations?: string[]
|
||||
// Web search
|
||||
tavily?: TavilySearchResponse
|
||||
webSearch?: WebSearchResponse
|
||||
}
|
||||
}
|
||||
|
||||
@ -291,7 +290,20 @@ export type SidebarIcon = 'assistants' | 'agents' | 'paintings' | 'translate' |
|
||||
export type WebSearchProvider = {
|
||||
id: string
|
||||
name: string
|
||||
apiKey: string
|
||||
enabled: boolean
|
||||
apiKey?: string
|
||||
apiHost?: string
|
||||
engines?: string[]
|
||||
}
|
||||
|
||||
export type WebSearchResponse = {
|
||||
query?: string
|
||||
results: WebSearchResult[]
|
||||
}
|
||||
export type WebSearchResult = {
|
||||
title: string
|
||||
content: string
|
||||
url: string
|
||||
}
|
||||
|
||||
export type KnowledgeReference = {
|
||||
|
||||
@ -1,59 +0,0 @@
|
||||
interface FormatDomainsResult {
|
||||
formattedDomains: string[]
|
||||
hasError: boolean
|
||||
}
|
||||
export function formatDomains(urls: string[]): FormatDomainsResult {
|
||||
let hasError = false
|
||||
const formattedDomains: string[] = []
|
||||
|
||||
for (const urlString of urls) {
|
||||
try {
|
||||
let modifiedUrlString = urlString
|
||||
|
||||
// 1. 处理通配符协议 (*://)
|
||||
if (modifiedUrlString.startsWith('*://')) {
|
||||
modifiedUrlString = modifiedUrlString.substring(4)
|
||||
}
|
||||
|
||||
// 2. 处理域名通配符 (*.example.com)
|
||||
let domain = modifiedUrlString
|
||||
if (domain.includes('://')) {
|
||||
const parts = domain.split('://')
|
||||
const domainPart = parts[1]
|
||||
if (domainPart.startsWith('*.')) {
|
||||
domain = parts[0] + '://' + domainPart.substring(2)
|
||||
} else {
|
||||
domain = modifiedUrlString
|
||||
}
|
||||
} else if (domain.startsWith('*.')) {
|
||||
domain = domain.substring(2)
|
||||
} else {
|
||||
domain = modifiedUrlString
|
||||
}
|
||||
|
||||
// 3. 检查并添加协议前缀
|
||||
if (!domain.match(/^[a-zA-Z]+:\/\//)) {
|
||||
domain = 'https://' + domain
|
||||
}
|
||||
|
||||
// 4. URL 解析和验证
|
||||
const url = new URL(domain)
|
||||
if (url.protocol !== 'https:') {
|
||||
if (url.protocol !== 'http:') {
|
||||
hasError = true
|
||||
} else {
|
||||
url.protocol = 'https:'
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 格式化
|
||||
const formattedDomain = `https://${url.hostname}`
|
||||
formattedDomains.push(formattedDomain)
|
||||
} catch (error) {
|
||||
hasError = true
|
||||
console.error('Error formatting URL:', urlString, error)
|
||||
}
|
||||
}
|
||||
|
||||
return { formattedDomains, hasError }
|
||||
}
|
||||
165
src/renderer/src/utils/blacklistMatchPattern.test.ts
Normal file
165
src/renderer/src/utils/blacklistMatchPattern.test.ts
Normal file
@ -0,0 +1,165 @@
|
||||
/*
|
||||
* MIT License
|
||||
*
|
||||
* Copyright (c) 2018 iorate
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*
|
||||
* https://github.com/iorate/ublacklist
|
||||
*/
|
||||
|
||||
import assert from 'node:assert'
|
||||
import { test } from 'node:test'
|
||||
|
||||
import { MatchPatternMap } from './blacklistMatchPattern'
|
||||
|
||||
function get(map: MatchPatternMap<number>, url: string) {
|
||||
return map.get(url).sort()
|
||||
}
|
||||
|
||||
test('MatchPatternMap', async (t) => {
|
||||
// https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Match_patterns
|
||||
await t.test('MDN Examples', () => {
|
||||
const map = new MatchPatternMap<number>()
|
||||
map.set('<all_urls>', 0)
|
||||
map.set('*://*/*', 1)
|
||||
map.set('*://*.mozilla.org/*', 2)
|
||||
map.set('*://mozilla.org/', 3)
|
||||
assert.throws(() => map.set('ftp://mozilla.org/', 4))
|
||||
map.set('https://*/path', 5)
|
||||
map.set('https://*/path/', 6)
|
||||
map.set('https://mozilla.org/*', 7)
|
||||
map.set('https://mozilla.org/a/b/c/', 8)
|
||||
map.set('https://mozilla.org/*/b/*/', 9)
|
||||
assert.throws(() => map.set('file:///blah/*', 10))
|
||||
// <all_urls>
|
||||
assert.deepStrictEqual(get(map, 'http://example.org/'), [0, 1])
|
||||
assert.deepStrictEqual(get(map, 'https://a.org/some/path/'), [0, 1])
|
||||
assert.deepStrictEqual(get(map, 'ws://sockets.somewhere.org/'), [])
|
||||
assert.deepStrictEqual(get(map, 'wss://ws.example.com/stuff/'), [])
|
||||
assert.deepStrictEqual(get(map, 'ftp://files.somewhere.org/'), [])
|
||||
assert.deepStrictEqual(get(map, 'resource://a/b/c/'), [])
|
||||
assert.deepStrictEqual(get(map, 'ftps://files.somewhere.org/'), [])
|
||||
// *://*/*
|
||||
assert.deepStrictEqual(get(map, 'http://example.org/'), [0, 1])
|
||||
assert.deepStrictEqual(get(map, 'https://a.org/some/path/'), [0, 1])
|
||||
assert.deepStrictEqual(get(map, 'ws://sockets.somewhere.org/'), [])
|
||||
assert.deepStrictEqual(get(map, 'wss://ws.example.com/stuff/'), [])
|
||||
assert.deepStrictEqual(get(map, 'ftp://ftp.example.org/'), [])
|
||||
assert.deepStrictEqual(get(map, 'file:///a/'), [])
|
||||
// *://*.mozilla.org/*
|
||||
assert.deepStrictEqual(get(map, 'http://mozilla.org/'), [0, 1, 2, 3])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/'), [0, 1, 2, 3, 7])
|
||||
assert.deepStrictEqual(get(map, 'http://a.mozilla.org/'), [0, 1, 2])
|
||||
assert.deepStrictEqual(get(map, 'http://a.b.mozilla.org/'), [0, 1, 2])
|
||||
assert.deepStrictEqual(get(map, 'https://b.mozilla.org/path/'), [0, 1, 2, 6])
|
||||
assert.deepStrictEqual(get(map, 'ws://ws.mozilla.org/'), [])
|
||||
assert.deepStrictEqual(get(map, 'wss://secure.mozilla.org/something'), [])
|
||||
assert.deepStrictEqual(get(map, 'ftp://mozilla.org/'), [])
|
||||
assert.deepStrictEqual(get(map, 'http://mozilla.com/'), [0, 1])
|
||||
assert.deepStrictEqual(get(map, 'http://firefox.org/'), [0, 1])
|
||||
// *://mozilla.org/
|
||||
assert.deepStrictEqual(get(map, 'http://mozilla.org/'), [0, 1, 2, 3])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/'), [0, 1, 2, 3, 7])
|
||||
assert.deepStrictEqual(get(map, 'ws://mozilla.org/'), [])
|
||||
assert.deepStrictEqual(get(map, 'wss://mozilla.org/'), [])
|
||||
assert.deepStrictEqual(get(map, 'ftp://mozilla.org/'), [])
|
||||
assert.deepStrictEqual(get(map, 'http://a.mozilla.org/'), [0, 1, 2])
|
||||
assert.deepStrictEqual(get(map, 'http://mozilla.org/a'), [0, 1, 2])
|
||||
// ftp://mozilla.org/
|
||||
assert.deepStrictEqual(get(map, 'ftp://mozilla.org/'), [])
|
||||
assert.deepStrictEqual(get(map, 'http://mozilla.org/'), [0, 1, 2, 3])
|
||||
assert.deepStrictEqual(get(map, 'ftp://sub.mozilla.org/'), [])
|
||||
assert.deepStrictEqual(get(map, 'ftp://mozilla.org/path'), [])
|
||||
// https://*/path
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/path'), [0, 1, 2, 5, 7])
|
||||
assert.deepStrictEqual(get(map, 'https://a.mozilla.org/path'), [0, 1, 2, 5])
|
||||
assert.deepStrictEqual(get(map, 'https://something.com/path'), [0, 1, 5])
|
||||
assert.deepStrictEqual(get(map, 'http://mozilla.org/path'), [0, 1, 2])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/path/'), [0, 1, 2, 6, 7])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/a'), [0, 1, 2, 7])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/'), [0, 1, 2, 3, 7])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/path?foo=1'), [0, 1, 2, 7])
|
||||
// https://*/path/
|
||||
assert.deepStrictEqual(get(map, 'http://mozilla.org/path/'), [0, 1, 2])
|
||||
assert.deepStrictEqual(get(map, 'https://a.mozilla.org/path/'), [0, 1, 2, 6])
|
||||
assert.deepStrictEqual(get(map, 'https://something.com/path/'), [0, 1, 6])
|
||||
assert.deepStrictEqual(get(map, 'http://mozilla.org/path/'), [0, 1, 2])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/path'), [0, 1, 2, 5, 7])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/a'), [0, 1, 2, 7])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/'), [0, 1, 2, 3, 7])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/path?foo=1'), [0, 1, 2, 7])
|
||||
// https://mozilla.org/*
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/'), [0, 1, 2, 3, 7])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/path'), [0, 1, 2, 5, 7])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/another'), [0, 1, 2, 7])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/path/to/doc'), [0, 1, 2, 7])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/path/to/doc?foo=1'), [0, 1, 2, 7])
|
||||
// https://mozilla.org/a/b/c/
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/a/b/c/'), [0, 1, 2, 7, 8, 9])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/a/b/c/#section1'), [0, 1, 2, 7, 8, 9])
|
||||
// https://mozilla.org/*/b/*/
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/a/b/c/'), [0, 1, 2, 7, 8, 9])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/d/b/f/'), [0, 1, 2, 7, 9])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/a/b/c/d/'), [0, 1, 2, 7, 9])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/a/b/c/d/#section1'), [0, 1, 2, 7, 9])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/a/b/c/d/?foo=/'), [0, 1, 2, 7, 9])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/a?foo=21314&bar=/b/&extra=c/'), [0, 1, 2, 7, 9])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/b/*/'), [0, 1, 2, 7])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/a/b/'), [0, 1, 2, 7])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/a/b/c/d/?foo=bar'), [0, 1, 2, 7])
|
||||
// file:///blah/*
|
||||
assert.deepStrictEqual(get(map, 'file:///blah/'), [])
|
||||
assert.deepStrictEqual(get(map, 'file:///blah/bleh'), [])
|
||||
assert.deepStrictEqual(get(map, 'file:///bleh/'), [])
|
||||
// Invalid match patterns
|
||||
assert.throws(() => map.set('resource://path/', 11))
|
||||
assert.throws(() => map.set('https://mozilla.org', 12))
|
||||
assert.throws(() => map.set('https://mozilla.*.org/', 13))
|
||||
assert.throws(() => map.set('https://*zilla.org', 14))
|
||||
assert.throws(() => map.set('http*://mozilla.org/', 15))
|
||||
assert.throws(() => map.set('https://mozilla.org:80/', 16))
|
||||
assert.throws(() => map.set('*//*', 17))
|
||||
assert.throws(() => map.set('file://*', 18))
|
||||
})
|
||||
|
||||
await t.test('Serialization', () => {
|
||||
let map = new MatchPatternMap<number>()
|
||||
map.set('<all_urls>', 0)
|
||||
map.set('*://*/*', 1)
|
||||
map.set('*://*.mozilla.org/*', 2)
|
||||
map.set('*://mozilla.org/', 3)
|
||||
map.set('https://*/path', 5)
|
||||
map.set('https://*/path/', 6)
|
||||
map.set('https://mozilla.org/*', 7)
|
||||
map.set('https://mozilla.org/a/b/c/', 8)
|
||||
map.set('https://mozilla.org/*/b/*/', 9)
|
||||
const json = map.toJSON()
|
||||
assert.strictEqual(
|
||||
JSON.stringify(json),
|
||||
'[[0],[[],[[1],[5,"https","/path"],[6,"https","/path/"]],{"org":[[],[],{"mozilla":[[[3,"*","/"],[7,"https"],[8,"https","/a/b/c/"],[9,"https","/*/b/*/"]],[[2]]]}]}]]'
|
||||
)
|
||||
map = new MatchPatternMap(json)
|
||||
assert.deepStrictEqual(get(map, 'http://mozilla.org/'), [0, 1, 2, 3])
|
||||
assert.deepStrictEqual(get(map, 'https://mozilla.org/'), [0, 1, 2, 3, 7])
|
||||
assert.deepStrictEqual(get(map, 'http://a.mozilla.org/'), [0, 1, 2])
|
||||
assert.deepStrictEqual(get(map, 'http://a.b.mozilla.org/'), [0, 1, 2])
|
||||
assert.deepStrictEqual(get(map, 'https://b.mozilla.org/path/'), [0, 1, 2, 6])
|
||||
})
|
||||
})
|
||||
172
src/renderer/src/utils/blacklistMatchPattern.ts
Normal file
172
src/renderer/src/utils/blacklistMatchPattern.ts
Normal file
@ -0,0 +1,172 @@
|
||||
/*
|
||||
* MIT License
|
||||
*
|
||||
* Copyright (c) 2018 iorate
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*
|
||||
* https://github.com/iorate/ublacklist
|
||||
*/
|
||||
export type ParsedMatchPattern =
|
||||
| {
|
||||
allURLs: true
|
||||
}
|
||||
| {
|
||||
allURLs: false
|
||||
scheme: string
|
||||
host: string
|
||||
path: string
|
||||
}
|
||||
|
||||
export function parseMatchPattern(pattern: string): ParsedMatchPattern | null {
|
||||
const execResult = matchPatternRegExp.exec(pattern)
|
||||
if (!execResult) {
|
||||
return null
|
||||
}
|
||||
const groups = execResult.groups as
|
||||
| { allURLs: string }
|
||||
| { allURLs?: never; scheme: string; host: string; path: string }
|
||||
return groups.allURLs != null
|
||||
? { allURLs: true }
|
||||
: {
|
||||
allURLs: false,
|
||||
scheme: groups.scheme.toLowerCase(),
|
||||
host: groups.host.toLowerCase(),
|
||||
path: groups.path
|
||||
}
|
||||
}
|
||||
|
||||
const matchPatternRegExp = (() => {
|
||||
const allURLs = String.raw`(?<allURLs><all_urls>)`
|
||||
const scheme = String.raw`(?<scheme>\*|[A-Za-z][0-9A-Za-z+.-]*)`
|
||||
const label = String.raw`(?:[0-9A-Za-z](?:[0-9A-Za-z-]*[0-9A-Za-z])?)`
|
||||
const host = String.raw`(?<host>(?:\*|${label})(?:\.${label})*)`
|
||||
const path = String.raw`(?<path>/(?:\*|[0-9A-Za-z._~:/?[\]@!$&'()+,;=-]|%[0-9A-Fa-f]{2})*)`
|
||||
return new RegExp(String.raw`^(?:${allURLs}|${scheme}://${host}${path})$`)
|
||||
})()
|
||||
|
||||
export type MatchPatternMapJSON<T> = [allURLs: T[], hostMap: HostMap<T>]
|
||||
|
||||
export class MatchPatternMap<T> {
|
||||
static supportedSchemes: string[] = ['http', 'https']
|
||||
|
||||
private allURLs: T[]
|
||||
private hostMap: HostMap<T>
|
||||
|
||||
constructor(json?: Readonly<MatchPatternMapJSON<T>>) {
|
||||
if (json) {
|
||||
this.allURLs = json[0]
|
||||
this.hostMap = json[1]
|
||||
} else {
|
||||
this.allURLs = []
|
||||
this.hostMap = [[], []]
|
||||
}
|
||||
}
|
||||
|
||||
toJSON(): MatchPatternMapJSON<T> {
|
||||
return [this.allURLs, this.hostMap]
|
||||
}
|
||||
|
||||
get(url: string): T[] {
|
||||
const { protocol, hostname: host, pathname, search } = new URL(url)
|
||||
const scheme = protocol.slice(0, -1)
|
||||
const path = `${pathname}${search}`
|
||||
if (!MatchPatternMap.supportedSchemes.includes(scheme)) {
|
||||
return []
|
||||
}
|
||||
const values: T[] = [...this.allURLs]
|
||||
let node = this.hostMap
|
||||
for (const label of host.split('.').reverse()) {
|
||||
collectBucket(node[1], scheme, path, values)
|
||||
if (!node[2]?.[label]) {
|
||||
return values
|
||||
}
|
||||
node = node[2][label]
|
||||
}
|
||||
collectBucket(node[1], scheme, path, values)
|
||||
collectBucket(node[0], scheme, path, values)
|
||||
return values
|
||||
}
|
||||
|
||||
set(pattern: string, value: T) {
|
||||
const parseResult = parseMatchPattern(pattern)
|
||||
if (!parseResult) {
|
||||
throw new Error(`Invalid match pattern: ${pattern}`)
|
||||
}
|
||||
if (parseResult.allURLs) {
|
||||
this.allURLs.push(value)
|
||||
return
|
||||
}
|
||||
const { scheme, host, path } = parseResult
|
||||
if (scheme !== '*' && !MatchPatternMap.supportedSchemes.includes(scheme)) {
|
||||
throw new Error(`Unsupported scheme: ${scheme}`)
|
||||
}
|
||||
const labels = host.split('.').reverse()
|
||||
const anySubdomain = labels[labels.length - 1] === '*'
|
||||
if (anySubdomain) {
|
||||
labels.pop()
|
||||
}
|
||||
let node = this.hostMap
|
||||
for (const label of labels) {
|
||||
node[2] ||= {}
|
||||
node = node[2][label] ||= [[], []]
|
||||
}
|
||||
node[anySubdomain ? 1 : 0].push(
|
||||
path === '/*' ? (scheme === '*' ? [value] : [value, scheme]) : [value, scheme, path]
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
type HostMap<T> = [self: HostMapBucket<T>, anySubdomain: HostMapBucket<T>, subdomains?: Record<string, HostMap<T>>]
|
||||
|
||||
type HostMapBucket<T> = [value: T, scheme?: string, path?: string][]
|
||||
|
||||
function collectBucket<T>(bucket: HostMapBucket<T>, scheme: string, path: string, values: T[]): void {
|
||||
for (const [value, schemePattern = '*', pathPattern = '/*'] of bucket) {
|
||||
if (testScheme(schemePattern, scheme) && testPath(pathPattern, path)) {
|
||||
values.push(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function testScheme(schemePattern: string, scheme: string): boolean {
|
||||
return schemePattern === '*' ? scheme === 'http' || scheme === 'https' : scheme === schemePattern
|
||||
}
|
||||
|
||||
function testPath(pathPattern: string, path: string): boolean {
|
||||
if (pathPattern === '/*') {
|
||||
return true
|
||||
}
|
||||
const [first, ...rest] = pathPattern.split('*')
|
||||
if (rest.length === 0) {
|
||||
return path === first
|
||||
}
|
||||
if (!path.startsWith(first)) {
|
||||
return false
|
||||
}
|
||||
let pos = first.length
|
||||
for (const part of rest.slice(0, -1)) {
|
||||
const partPos = path.indexOf(part, pos)
|
||||
if (partPos === -1) {
|
||||
return false
|
||||
}
|
||||
pos = partPos + part.length
|
||||
}
|
||||
return path.slice(pos).endsWith(rest[rest.length - 1])
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
import { WebSearchProvider, WebSearchResponse } from '@renderer/types'
|
||||
|
||||
export default abstract class BaseWebSearchProvider {
|
||||
private provider: WebSearchProvider
|
||||
constructor(provider: WebSearchProvider) {
|
||||
this.provider = provider
|
||||
}
|
||||
abstract search(query: string, maxResult: number, excludeDomains: string[]): Promise<WebSearchResponse>
|
||||
}
|
||||
9
src/renderer/src/webSearchProvider/DefaultProvider.ts
Normal file
9
src/renderer/src/webSearchProvider/DefaultProvider.ts
Normal file
@ -0,0 +1,9 @@
|
||||
import { WebSearchResponse } from '@renderer/types'
|
||||
|
||||
import BaseWebSearchProvider from './BaseWebSearchProvider'
|
||||
|
||||
export default class DefaultProvider extends BaseWebSearchProvider {
|
||||
search(): Promise<WebSearchResponse> {
|
||||
throw new Error('Method not implemented.')
|
||||
}
|
||||
}
|
||||
97
src/renderer/src/webSearchProvider/SearxngProvider.ts
Normal file
97
src/renderer/src/webSearchProvider/SearxngProvider.ts
Normal file
@ -0,0 +1,97 @@
|
||||
import { SearxngClient } from '@agentic/searxng'
|
||||
import { WebSearchProvider, WebSearchResponse } from '@renderer/types'
|
||||
import axios from 'axios'
|
||||
|
||||
import BaseWebSearchProvider from './BaseWebSearchProvider'
|
||||
|
||||
export default class SearxngProvider extends BaseWebSearchProvider {
|
||||
private searxng: SearxngClient
|
||||
private engines: string[] = []
|
||||
private readonly apiHost: string
|
||||
private isInitialized = false
|
||||
|
||||
constructor(provider: WebSearchProvider) {
|
||||
super(provider)
|
||||
if (!provider.apiHost) {
|
||||
throw new Error('API host is required for SearxNG provider')
|
||||
}
|
||||
this.apiHost = provider.apiHost
|
||||
this.searxng = new SearxngClient({ apiBaseUrl: this.apiHost })
|
||||
this.initEngines().catch((err) => console.error('Failed to initialize SearxNG engines:', err))
|
||||
}
|
||||
|
||||
private async initEngines(): Promise<void> {
|
||||
try {
|
||||
const response = await axios.get(`${this.apiHost}/config`, { timeout: 5000 })
|
||||
|
||||
if (!response.data || !Array.isArray(response.data.engines)) {
|
||||
throw new Error('Invalid response format from SearxNG config endpoint')
|
||||
}
|
||||
|
||||
this.engines = response.data.engines
|
||||
.filter(
|
||||
(engine: { enabled: boolean; categories: string[]; name: string }) =>
|
||||
engine.enabled &&
|
||||
Array.isArray(engine.categories) &&
|
||||
engine.categories.includes('general') &&
|
||||
engine.categories.includes('web')
|
||||
)
|
||||
.map((engine) => engine.name)
|
||||
|
||||
this.isInitialized = true
|
||||
console.log(`SearxNG initialized with ${this.engines.length} engines`)
|
||||
} catch (err) {
|
||||
console.error('Failed to fetch SearxNG engine configuration:', err)
|
||||
this.engines = []
|
||||
}
|
||||
}
|
||||
|
||||
public async search(query: string, maxResults: number): Promise<WebSearchResponse> {
|
||||
try {
|
||||
if (!query) {
|
||||
throw new Error('Search query cannot be empty')
|
||||
}
|
||||
|
||||
// Wait for initialization if it's the first search
|
||||
if (!this.isInitialized) {
|
||||
await this.initEngines().catch(() => {}) // Ignore errors
|
||||
}
|
||||
|
||||
// 如果engines为空，直接返回空结果
|
||||
if (this.engines.length === 0) {
|
||||
return {
|
||||
query: query,
|
||||
results: []
|
||||
}
|
||||
}
|
||||
|
||||
const result = await this.searxng.search({
|
||||
query: query,
|
||||
engines: this.engines as any,
|
||||
language: 'auto'
|
||||
})
|
||||
|
||||
if (!result || !Array.isArray(result.results)) {
|
||||
throw new Error('Invalid search results from SearxNG')
|
||||
}
|
||||
|
||||
return {
|
||||
query: result.query,
|
||||
results: result.results.slice(0, maxResults).map((result) => {
|
||||
return {
|
||||
title: result.title || 'No title',
|
||||
content: result.content || '',
|
||||
url: result.url || ''
|
||||
}
|
||||
})
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Search failed:', err)
|
||||
// Return empty results instead of throwing to prevent UI crashes
|
||||
return {
|
||||
query: query,
|
||||
results: []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
42
src/renderer/src/webSearchProvider/TavilyProvider.ts
Normal file
42
src/renderer/src/webSearchProvider/TavilyProvider.ts
Normal file
@ -0,0 +1,42 @@
|
||||
import { TavilyClient } from '@agentic/tavily'
|
||||
import { WebSearchProvider, WebSearchResponse } from '@renderer/types'
|
||||
|
||||
import BaseWebSearchProvider from './BaseWebSearchProvider'
|
||||
|
||||
export default class TavilyProvider extends BaseWebSearchProvider {
|
||||
private tvly: TavilyClient
|
||||
|
||||
constructor(provider: WebSearchProvider) {
|
||||
super(provider)
|
||||
if (!provider.apiKey) {
|
||||
throw new Error('API key is required for Tavily provider')
|
||||
}
|
||||
this.tvly = new TavilyClient({ apiKey: provider.apiKey })
|
||||
}
|
||||
|
||||
public async search(query: string, maxResults: number, excludeDomains: string[]): Promise<WebSearchResponse> {
|
||||
try {
|
||||
if (!query.trim()) {
|
||||
throw new Error('Search query cannot be empty')
|
||||
}
|
||||
|
||||
const result = await this.tvly.search({
|
||||
query,
|
||||
max_results: Math.max(1, maxResults),
|
||||
exclude_domains: excludeDomains || []
|
||||
})
|
||||
|
||||
return {
|
||||
query: result.query,
|
||||
results: result.results.map((result) => ({
|
||||
title: result.title || 'No title',
|
||||
content: result.content || '',
|
||||
url: result.url || ''
|
||||
}))
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Tavily search failed:', error)
|
||||
throw new Error(`Search failed: ${error instanceof Error ? error.message : 'Unknown error'}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,14 @@
|
||||
import { WebSearchProvider, WebSearchResponse } from '@renderer/types'
|
||||
|
||||
import BaseWebSearchProvider from './BaseWebSearchProvider'
|
||||
import WebSearchProviderFactory from './WebSearchProviderFactory'
|
||||
|
||||
export default class WebSearchEngineProvider {
|
||||
private sdk: BaseWebSearchProvider
|
||||
constructor(provider: WebSearchProvider) {
|
||||
this.sdk = WebSearchProviderFactory.create(provider)
|
||||
}
|
||||
public async search(query: string, maxResult: number, excludeDomains: string[]): Promise<WebSearchResponse> {
|
||||
return await this.sdk.search(query, maxResult, excludeDomains)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
import { WebSearchProvider } from '@renderer/types'
|
||||
|
||||
import BaseWebSearchProvider from './BaseWebSearchProvider'
|
||||
import DefaultProvider from './DefaultProvider'
|
||||
import SearxngProvider from './SearxngProvider'
|
||||
import TavilyProvider from './TavilyProvider'
|
||||
|
||||
export default class WebSearchProviderFactory {
|
||||
static create(provider: WebSearchProvider): BaseWebSearchProvider {
|
||||
switch (provider.id) {
|
||||
case 'tavily':
|
||||
return new TavilyProvider(provider)
|
||||
case 'searxng':
|
||||
return new SearxngProvider(provider)
|
||||
default:
|
||||
return new DefaultProvider(provider)
|
||||
}
|
||||
}
|
||||
}
|
||||
113
yarn.lock
113
yarn.lock
@ -12,6 +12,50 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@agentic/core@npm:7.3.3":
|
||||
version: 7.3.3
|
||||
resolution: "@agentic/core@npm:7.3.3"
|
||||
dependencies:
|
||||
dedent: "npm:^1.5.3"
|
||||
delay: "npm:^6.0.0"
|
||||
jsonrepair: "npm:^3.9.0"
|
||||
ky: "npm:^1.7.5"
|
||||
openai-zod-to-json-schema: "npm:^1.0.3"
|
||||
p-map: "npm:^7.0.2"
|
||||
p-throttle: "npm:^6.2.0"
|
||||
type-fest: "npm:^4.35.0"
|
||||
zod-validation-error: "npm:^3.4.0"
|
||||
peerDependencies:
|
||||
zod: ^3.24.2
|
||||
checksum: 10c0/5f9736cdfa72da5093a8889b168cce38107be1df60cade62df82236b96fdf6970a5fcdf1f12a17630190f75f5b431fff2e70ca3cb33bb1e550ad5488e03699e7
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@agentic/searxng@npm:^7.3.3":
|
||||
version: 7.3.3
|
||||
resolution: "@agentic/searxng@npm:7.3.3"
|
||||
dependencies:
|
||||
"@agentic/core": "npm:7.3.3"
|
||||
ky: "npm:^1.7.5"
|
||||
peerDependencies:
|
||||
zod: ^3.24.2
|
||||
checksum: 10c0/474e53f232c6ad46315db9a9f2381f8d51623ba6f1eb514c92f328762a350565ff4bfa75d5cb5fb1454ce0d426bf033565167b708b21e8d3488ea1454461f713
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@agentic/tavily@npm:^7.3.3":
|
||||
version: 7.3.3
|
||||
resolution: "@agentic/tavily@npm:7.3.3"
|
||||
dependencies:
|
||||
"@agentic/core": "npm:7.3.3"
|
||||
ky: "npm:^1.7.5"
|
||||
p-throttle: "npm:^6.2.0"
|
||||
peerDependencies:
|
||||
zod: ^3.24.2
|
||||
checksum: 10c0/604e9a1a15cd8a6b70754dcc919c843988376d7ed65f83ba24d8add7c8febfae782184cf804611d5baacc5249d50ac8e19fa85390d8a40086c52c352a9bbb87d
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@ampproject/remapping@npm:^2.2.0":
|
||||
version: 2.3.0
|
||||
resolution: "@ampproject/remapping@npm:2.3.0"
|
||||
@ -3050,6 +3094,8 @@ __metadata:
|
||||
version: 0.0.0-use.local
|
||||
resolution: "CherryStudio@workspace:."
|
||||
dependencies:
|
||||
"@agentic/searxng": "npm:^7.3.3"
|
||||
"@agentic/tavily": "npm:^7.3.3"
|
||||
"@anthropic-ai/sdk": "npm:^0.38.0"
|
||||
"@electron-toolkit/eslint-config-prettier": "npm:^2.0.0"
|
||||
"@electron-toolkit/eslint-config-ts": "npm:^1.0.1"
|
||||
@ -4997,6 +5043,18 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"dedent@npm:^1.5.3":
|
||||
version: 1.5.3
|
||||
resolution: "dedent@npm:1.5.3"
|
||||
peerDependencies:
|
||||
babel-plugin-macros: ^3.1.0
|
||||
peerDependenciesMeta:
|
||||
babel-plugin-macros:
|
||||
optional: true
|
||||
checksum: 10c0/d94bde6e6f780be4da4fd760288fcf755ec368872f4ac5218197200d86430aeb8d90a003a840bff1c20221188e3f23adced0119cb811c6873c70d0ac66d12832
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"deep-extend@npm:^0.6.0":
|
||||
version: 0.6.0
|
||||
resolution: "deep-extend@npm:0.6.0"
|
||||
@ -5070,6 +5128,13 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"delay@npm:^6.0.0":
|
||||
version: 6.0.0
|
||||
resolution: "delay@npm:6.0.0"
|
||||
checksum: 10c0/5175e887512d65b2bfe9e1168b5ce7a488de99c1d0af52cb4f799bb13dd7cb0bbbba8a4f5c500a5b03fb42bec8621d6ab59244bd8dfbe9a2bf7b173f25621a10
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"delayed-stream@npm:~1.0.0":
|
||||
version: 1.0.0
|
||||
resolution: "delayed-stream@npm:1.0.0"
|
||||
@ -8575,6 +8640,15 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"jsonrepair@npm:^3.9.0":
|
||||
version: 3.12.0
|
||||
resolution: "jsonrepair@npm:3.12.0"
|
||||
bin:
|
||||
jsonrepair: bin/cli.js
|
||||
checksum: 10c0/f61bea017e9675c888dc8087bec6f868595ab0103a211827f9fc6a327abd9c70fa4d3241ee2846b06e229a628d51e5a5448c516d7be3040862863a2f2c3f78cd
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"jsprim@npm:^1.2.2":
|
||||
version: 1.4.2
|
||||
resolution: "jsprim@npm:1.4.2"
|
||||
@ -8650,6 +8724,13 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"ky@npm:^1.7.5":
|
||||
version: 1.7.5
|
||||
resolution: "ky@npm:1.7.5"
|
||||
checksum: 10c0/9f9c70a4916592f728c90e38ecbe2ed468eb7161b7525a4561a861e457edd5cb706751e2aba615d350380231d021f535147f9ed3ca07271af836465ecc725761
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"langchain@npm:^0.3.8":
|
||||
version: 0.3.19
|
||||
resolution: "langchain@npm:0.3.19"
|
||||
@ -10677,6 +10758,15 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"openai-zod-to-json-schema@npm:^1.0.3":
|
||||
version: 1.0.3
|
||||
resolution: "openai-zod-to-json-schema@npm:1.0.3"
|
||||
peerDependencies:
|
||||
zod: ^3.23.8
|
||||
checksum: 10c0/1fd4afb01a3e82955ba4f7bed4e8f12ad0b67d33329484ddf36025cada117faa65486a89286cce1b4b5890078f9de14cc9ccc9f00be140e6dd68b77988239e64
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"openai@npm:4.77.3":
|
||||
version: 4.77.3
|
||||
resolution: "openai@npm:4.77.3"
|
||||
@ -10886,6 +10976,13 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"p-throttle@npm:^6.2.0":
|
||||
version: 6.2.0
|
||||
resolution: "p-throttle@npm:6.2.0"
|
||||
checksum: 10c0/3be65f66eb21137be78b8d18a5240117312b942e3aa788f838ac4be785ab3c40b64ee34b2c393cd948ec7845c0a00241f446395b98ff4754e718fe54fdee0b00
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"p-timeout@npm:^3.2.0":
|
||||
version: 3.2.0
|
||||
resolution: "p-timeout@npm:3.2.0"
|
||||
@ -14316,6 +14413,13 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"type-fest@npm:^4.35.0":
|
||||
version: 4.37.0
|
||||
resolution: "type-fest@npm:4.37.0"
|
||||
checksum: 10c0/5bad189f66fbe3431e5d36befa08cab6010e56be68b7467530b7ef94c3cf81ef775a8ac3047c8bbda4dd3159929285870357498d7bc1df062714f9c5c3a84926
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"type-is@npm:^2.0.0":
|
||||
version: 2.0.0
|
||||
resolution: "type-is@npm:2.0.0"
|
||||
@ -15383,6 +15487,15 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"zod-validation-error@npm:^3.4.0":
|
||||
version: 3.4.0
|
||||
resolution: "zod-validation-error@npm:3.4.0"
|
||||
peerDependencies:
|
||||
zod: ^3.18.0
|
||||
checksum: 10c0/aaadb0e65c834aacb12fa088663d52d9f4224b5fe6958f09b039f4ab74145fda381c8a7d470bfddf7ddd9bbb5fdfbb52739cd66958ce6d388c256a44094d1fba
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"zod@npm:^3.22.4, zod@npm:^3.23.8":
|
||||
version: 3.24.2
|
||||
resolution: "zod@npm:3.24.2"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user