From 2e0251aed7f49243caa6fcdc95eb7f40c0a1f4e1 Mon Sep 17 00:00:00 2001 From: Chen Tao <70054568+eeee0717@users.noreply.github.com> Date: Thu, 10 Apr 2025 17:25:38 +0800 Subject: [PATCH] feat: support ublacklist subscribe (#2974) * feat: support ublacklist subscribe * Merge branch 'main' into feat-ublacklist * chore * chore --- .../src/hooks/useWebSearchProviders.ts | 33 +++ src/renderer/src/i18n/locales/en-us.json | 11 +- src/renderer/src/i18n/locales/ja-jp.json | 11 +- src/renderer/src/i18n/locales/ru-ru.json | 11 +- src/renderer/src/i18n/locales/zh-cn.json | 13 +- src/renderer/src/i18n/locales/zh-tw.json | 29 ++- .../WebSearchSettings/AddSubscribePopup.tsx | 120 ++++++++++ .../WebSearchSettings/BlacklistSettings.tsx | 213 +++++++++++++++++- .../BaseWebSearchProvider.ts | 4 +- .../WebSearchProvider/ExaProvider.ts | 5 +- .../WebSearchProvider/LocalSearchProvider.ts | 18 +- .../WebSearchProvider/SearxngProvider.ts | 6 +- .../WebSearchProvider/TavilyProvider.ts | 7 +- .../src/providers/WebSearchProvider/index.ts | 9 +- src/renderer/src/services/WebSearchService.ts | 7 +- src/renderer/src/store/websearch.ts | 39 ++++ .../src/utils/blacklistMatchPattern.ts | 108 +++++++++ 17 files changed, 599 insertions(+), 45 deletions(-) create mode 100644 src/renderer/src/pages/settings/WebSearchSettings/AddSubscribePopup.tsx diff --git a/src/renderer/src/hooks/useWebSearchProviders.ts b/src/renderer/src/hooks/useWebSearchProviders.ts index dc9c0ee0..b018d2a6 100644 --- a/src/renderer/src/hooks/useWebSearchProviders.ts +++ b/src/renderer/src/hooks/useWebSearchProviders.ts @@ -1,6 +1,10 @@ import { useAppDispatch, useAppSelector } from '@renderer/store' import { + addSubscribeSource as _addSubscribeSource, + removeSubscribeSource as _removeSubscribeSource, setDefaultProvider as _setDefaultProvider, + setSubscribeSources as _setSubscribeSources, + updateSubscribeBlacklist as _updateSubscribeBlacklist, updateWebSearchProvider, updateWebSearchProviders } from '@renderer/store/websearch' @@ -57,3 +61,32 @@ export const useWebSearchProvider = (id: string) => { return { provider, updateProvider } } + +export const useBlacklist = () => { + const dispatch = useAppDispatch() + const websearch = useAppSelector((state) => state.websearch) + + const addSubscribeSource = ({ url, name, blacklist }) => { + dispatch(_addSubscribeSource({ url, name, blacklist })) + } + + const removeSubscribeSource = (key: number) => { + dispatch(_removeSubscribeSource(key)) + } + + const updateSubscribeBlacklist = (key: number, blacklist: string[]) => { + dispatch(_updateSubscribeBlacklist({ key, blacklist })) + } + + const setSubscribeSources = (sources: { key: number; url: string; name: string; blacklist?: string[] }[]) => { + dispatch(_setSubscribeSources(sources)) + } + + return { + websearch, + addSubscribeSource, + removeSubscribeSource, + updateSubscribeBlacklist, + setSubscribeSources + } +} diff --git a/src/renderer/src/i18n/locales/en-us.json b/src/renderer/src/i18n/locales/en-us.json index fca3b4e1..bcd980da 100644 --- a/src/renderer/src/i18n/locales/en-us.json +++ b/src/renderer/src/i18n/locales/en-us.json @@ -1299,7 +1299,16 @@ "description": "Tavily is a search engine tailored for AI agents, delivering real-time, accurate results, intelligent query suggestions, and in-depth research capabilities.", "title": "Tavily" }, - "title": "Web Search", + "title": "Web Search", + "subscribe": "Subscribe", + "subscribe_tooltip": "Subscribe to the blacklist.", + "subscribe_update": "Update now", + "subscribe_add": "Add Subscription", + "subscribe_url": "Subscription feed address", + "subscribe_name": "Alternative name", + "subscribe_name.placeholder": "Alternative name used when the downloaded subscription feed has no name.", + "subscribe_add_success": "Subscription feed added successfully!", + "subscribe_delete": "Delete subscription source", "overwrite": "Override search service", "overwrite_tooltip": "Force use search service instead of LLM", "apikey": "API key", diff --git a/src/renderer/src/i18n/locales/ja-jp.json b/src/renderer/src/i18n/locales/ja-jp.json index 8b71523b..d1a39138 100644 --- a/src/renderer/src/i18n/locales/ja-jp.json +++ b/src/renderer/src/i18n/locales/ja-jp.json @@ -1279,7 +1279,6 @@ "websearch": { "blacklist": "ブラックリスト", "blacklist_description": "以下のウェブサイトの結果は検索結果に表示されません", - "blacklist_tooltip": "以下の形式を使用してください(改行区切り)\nexample.com\nhttps://www.example.com\nhttps://example.com\n*://*.example.com", "check": "チェック", "check_failed": "検証に失敗しました", "check_success": "検証に成功しました", @@ -1299,6 +1298,16 @@ "title": "Tavily" }, "title": "ウェブ検索", + "blacklist_tooltip": "マッチパターン: *://*.example.com/*\n正規表現: /example\\.(net|org)/", + "subscribe": "購読", + "subscribe_tooltip": "ブラックリストに登録する", + "subscribe_update": "今すぐ更新", + "subscribe_add": "サブスクリプションを追加", + "subscribe_url": "フィードのURL", + "subscribe_name": "代替名", + "subscribe_name.placeholder": "ダウンロードしたフィードに名前がない場合に使用される代替名", + "subscribe_add_success": "フィードの追加が成功しました！", + "subscribe_delete": "フィードの削除", "overwrite": "サービス検索を上書き", "overwrite_tooltip": "大規模言語モデルではなく、サービス検索を使用する", "apikey": "API キー", diff --git a/src/renderer/src/i18n/locales/ru-ru.json b/src/renderer/src/i18n/locales/ru-ru.json index b22f009d..c07c0c81 100644 --- a/src/renderer/src/i18n/locales/ru-ru.json +++ b/src/renderer/src/i18n/locales/ru-ru.json @@ -1279,7 +1279,6 @@ "websearch": { "blacklist": "Черный список", "blacklist_description": "Результаты из следующих веб-сайтов не будут отображаться в результатах поиска", - "blacklist_tooltip": "Пожалуйста, используйте следующий формат (разделенный переносами строк)\nexample.com\nhttps://www.example.com\nhttps://example.com\n*://*.example.com", "check": "проверка", "check_failed": "Проверка не прошла", "check_success": "Проверка успешна", @@ -1299,6 +1298,16 @@ "title": "Tavily" }, "title": "Поиск в Интернете", + "blacklist_tooltip": "Соответствующий шаблон: *://*.example.com/*\nРегулярное выражение: /example\\.(net|org)/", + "subscribe": "Подписка", + "subscribe_tooltip": "Подписаться на черный список", + "subscribe_update": "Обновить сейчас", + "subscribe_add": "Добавить подписку", + "subscribe_url": "Адрес источника подписки", + "subscribe_name": "альтернативное имя", + "subscribe_name.placeholder": "替代名称, используемый, когда загружаемый подписочный источник не имеет названия", + "subscribe_add_success": "Подписка добавлена успешно!", + "subscribe_delete": "Удалить источник подписки", "overwrite": "Переопределить поставщика поиска", "overwrite_tooltip": "Использовать поставщика поиска вместо LLM", "apikey": "Ключ API", diff --git a/src/renderer/src/i18n/locales/zh-cn.json b/src/renderer/src/i18n/locales/zh-cn.json index 6a585dd6..43359bf9 100644 --- a/src/renderer/src/i18n/locales/zh-cn.json +++ b/src/renderer/src/i18n/locales/zh-cn.json @@ -1280,7 +1280,7 @@ "websearch": { "blacklist": "黑名单", "blacklist_description": "在搜索结果中不会出现以下网站的结果", - "blacklist_tooltip": "请使用以下格式(换行分隔)\nexample.com\nhttps://www.example.com\nhttps://example.com\n*://*.example.com", + "blacklist_tooltip": "请使用以下格式(换行分隔)\n匹配模式: *://*.example.com/*\n正则表达式: /example\\.(net|org)/", "check": "检查", "check_failed": "验证失败", "check_success": "验证成功", @@ -1293,6 +1293,15 @@ "search_max_result": "搜索结果个数", "search_provider": "搜索服务商", "search_provider_placeholder": "选择一个搜索服务商", + "subscribe": "订阅", + "subscribe_tooltip": "订阅黑名单列表", + "subscribe_update": "立即更新", + "subscribe_add": "添加订阅", + "subscribe_url": "订阅源地址", + "subscribe_name": "替代名字", + "subscribe_name.placeholder": "当下载的订阅源没有名称时所使用的替代名称", + "subscribe_add_success": "订阅源添加成功!", + "subscribe_delete": "删除订阅源", "search_result_default": "默认", "search_with_time": "搜索包含日期", "tavily": { @@ -1372,3 +1381,5 @@ } } } + + diff --git a/src/renderer/src/i18n/locales/zh-tw.json b/src/renderer/src/i18n/locales/zh-tw.json index b814472e..41a25f78 100644 --- a/src/renderer/src/i18n/locales/zh-tw.json +++ b/src/renderer/src/i18n/locales/zh-tw.json @@ -1277,20 +1277,10 @@ "tray.show": "顯示系统匣圖示", "tray.title": "系统匣", "websearch": { - "blacklist": "黑名單", - "blacklist_description": "以下網站不會出現在搜尋結果中", - "blacklist_tooltip": "請使用以下格式 (換行符號分隔)\nexample.com\nhttps://www.example.com\nhttps://example.com\n*://*.example.com", - "check": "檢查", - "check_failed": "驗證失敗", "check_success": "驗證成功", "enhance_mode": "搜索增強模式", "enhance_mode_tooltip": "使用預設模型提取關鍵詞後搜索", "get_api_key": "點選這裡取得金鑰", - "no_provider_selected": "請選擇搜尋服務商後再檢查", - "search_max_result": "搜尋結果個數", - "search_provider": "搜尋服務商", - "search_provider_placeholder": "選擇一個搜尋服務商", - "search_result_default": "預設", "search_with_time": "搜尋包含日期", "tavily": { "api_key": "Tavily API 金鑰", @@ -1298,6 +1288,25 @@ "description": "Tavily 是一個為 AI 代理量身訂製的搜尋引擎，提供即時、準確的結果、智慧查詢建議和深入的研究能力", "title": "Tavily" }, + "blacklist": "黑名單", + "blacklist_description": "以下網站不會出現在搜索結果中", + "search_max_result": "搜尋結果個數", + "search_result_default": "預設", + "check": "檢查", + "search_provider": "搜尋服務商", + "search_provider_placeholder": "選擇一個搜尋服務商", + "no_provider_selected": "請選擇搜索服務商後再檢查", + "check_failed": "驗證失敗", + "blacklist_tooltip": "匹配模式: *://*.example.com/*\n正则表达式: /example\\.(net|org)/", + "subscribe": "訂閱", + "subscribe_tooltip": "訂閱黑名單列表", + "subscribe_update": "立即更新", + "subscribe_add": "添加訂閱", + "subscribe_url": "訂閱源地址", + "subscribe_name": "替代名稱", + "subscribe_name.placeholder": "當下載的訂閱源沒有名稱時所使用的替代名稱", + "subscribe_add_success": "訂閱源添加成功!", + "subscribe_delete": "刪除訂閱源", "title": "網路搜尋", "overwrite": "覆蓋搜尋服務商", "overwrite_tooltip": "強制使用搜尋服務商而不是大語言模型進行搜尋", diff --git a/src/renderer/src/pages/settings/WebSearchSettings/AddSubscribePopup.tsx b/src/renderer/src/pages/settings/WebSearchSettings/AddSubscribePopup.tsx new file mode 100644 index 00000000..9ce19e2e --- /dev/null +++ b/src/renderer/src/pages/settings/WebSearchSettings/AddSubscribePopup.tsx @@ -0,0 +1,120 @@ +import { TopView } from '@renderer/components/TopView' +import { Button, Form, FormProps, Input, Modal } from 'antd' +import { useState } from 'react' +import { useTranslation } from 'react-i18next' + +interface ShowParams { + title: string +} + +interface Props extends ShowParams { + resolve: (data: any) => void +} + +type FieldType = { + url: string + name?: string +} + +const PopupContainer: React.FC = ({ title, resolve }) => { + const [open, setOpen] = useState(true) + const [form] = Form.useForm() + const { t } = useTranslation() + + const onOk = () => { + setOpen(false) + } + + const onCancel = () => { + setOpen(false) + } + + const onClose = () => { + resolve({}) + } + + const onFinish: FormProps['onFinish'] = (values) => { + const url = values.url.trim() + const name = values.name?.trim() || url + + if (!url) { + window.message.error(t('settings.websearch.url_required')) + return + } + + // 验证URL格式 + try { + new URL(url) + } catch (e) { + window.message.error(t('settings.websearch.url_invalid')) + return + } + + resolve({ url, name }) + } + + return ( + +
+ + { + try { + const url = new URL(e.target.value) + form.setFieldValue('name', url.hostname) + } catch (e) { + // URL不合法，忽略 + } + }} + /> + + + + + + + +
+ + ) +} + +export default class AddSubscribePopup { + static topviewId = 0 + static hide() { + TopView.hide('AddSubscribePopup') + } + static show(props: ShowParams) { + return new Promise((resolve) => { + TopView.show( + { + resolve(v) + this.hide() + }} + />, + 'AddSubscribePopup' + ) + }) + } +} diff --git a/src/renderer/src/pages/settings/WebSearchSettings/BlacklistSettings.tsx b/src/renderer/src/pages/settings/WebSearchSettings/BlacklistSettings.tsx index 8441c0e3..23d9ffab 100644 --- a/src/renderer/src/pages/settings/WebSearchSettings/BlacklistSettings.tsx +++ b/src/renderer/src/pages/settings/WebSearchSettings/BlacklistSettings.tsx @@ -1,22 +1,61 @@ +import { CheckOutlined, InfoCircleOutlined, LoadingOutlined } from '@ant-design/icons' import { useTheme } from '@renderer/context/ThemeProvider' +import { useBlacklist } from '@renderer/hooks/useWebSearchProviders' import { useAppDispatch, useAppSelector } from '@renderer/store' import { setExcludeDomains } from '@renderer/store/websearch' -import { parseMatchPattern } from '@renderer/utils/blacklistMatchPattern' -import { Alert, Button } from 'antd' +import { parseMatchPattern, parseSubscribeContent } from '@renderer/utils/blacklistMatchPattern' +import { Alert, Button, Table, TableProps } from 'antd' import TextArea from 'antd/es/input/TextArea' import { t } from 'i18next' import { FC, useEffect, useState } from 'react' import { SettingDivider, SettingGroup, SettingRow, SettingRowTitle, SettingTitle } from '..' +import AddSubscribePopup from './AddSubscribePopup' +type TableRowSelection = TableProps['rowSelection'] +interface DataType { + key: React.Key + url: string + name: string +} +const columns: TableProps['columns'] = [ + { title: t('common.name'), dataIndex: 'name', key: 'name' }, + { + title: 'URL', + dataIndex: 'url', + key: 'url' + } +] const BlacklistSettings: FC = () => { const [errFormat, setErrFormat] = useState(false) const [blacklistInput, setBlacklistInput] = useState('') const excludeDomains = useAppSelector((state) => state.websearch.excludeDomains) + const { websearch, setSubscribeSources, addSubscribeSource } = useBlacklist() const { theme } = useTheme() + const [subscribeChecking, setSubscribeChecking] = useState(false) + const [subscribeValid, setSubscribeValid] = useState(false) + const [selectedRowKeys, setSelectedRowKeys] = useState([]) + const [dataSource, setDataSource] = useState( + websearch.subscribeSources?.map((source) => ({ + key: source.key, + url: source.url, + name: source.name + })) || [] + ) const dispatch = useAppDispatch() + useEffect(() => { + setDataSource( + (websearch.subscribeSources || []).map((source) => ({ + key: source.key, + url: source.url, + name: source.name + })) + ) + console.log('subscribeSources', websearch.subscribeSources) + }, [websearch.subscribeSources]) + useEffect(() => { if (excludeDomains) { setBlacklistInput(excludeDomains.join('\n')) @@ -40,6 +79,137 @@ const BlacklistSettings: FC = () => { if (hasError) return dispatch(setExcludeDomains(validDomains)) + window.message.info({ + content: t('message.save.success.title'), + duration: 4, + icon: , + key: 'save-blacklist-info' + }) + } + const onSelectChange = (newSelectedRowKeys: React.Key[]) => { + console.log('selectedRowKeys changed: ', newSelectedRowKeys) + setSelectedRowKeys(newSelectedRowKeys) + } + + const rowSelection: TableRowSelection = { + selectedRowKeys, + onChange: onSelectChange + } + async function updateSubscribe() { + setSubscribeChecking(true) + + try { + // 获取选中的订阅源 + const selectedSources = dataSource.filter((item) => selectedRowKeys.includes(item.key)) + + // 用于存储所有成功解析的订阅源数据 + const updatedSources: { + key: number + url: string + name: string + blacklist: string[] + }[] = [] + + // 为每个选中的订阅源获取并解析内容 + for (const source of selectedSources) { + try { + // 获取并解析订阅源内容 + const blacklist = await parseSubscribeContent(source.url) + + if (blacklist.length > 0) { + updatedSources.push({ + key: Number(source.key), + url: source.url, + name: source.name, + blacklist + }) + } + } catch (error) { + console.error(`Error updating subscribe source ${source.url}:`, error) + // 显示具体源更新失败的消息 + window.message.warning({ + content: t('settings.websearch.subscribe_source_update_failed', { url: source.url }), + duration: 3 + }) + } + } + + if (updatedSources.length > 0) { + // 更新 Redux store + setSubscribeSources(updatedSources) + setSubscribeValid(true) + // 显示成功消息 + window.message.success({ + content: t('settings.websearch.subscribe_update_success'), + duration: 2 + }) + setTimeout(() => setSubscribeValid(false), 3000) + } else { + setSubscribeValid(false) + throw new Error('No valid sources updated') + } + } catch (error) { + console.error('Error updating subscribes:', error) + window.message.error({ + content: t('settings.websearch.subscribe_update_failed'), + duration: 2 + }) + } + setSubscribeChecking(false) + } + + // 修改 handleAddSubscribe 函数 + async function handleAddSubscribe() { + setSubscribeChecking(true) + const result = await AddSubscribePopup.show({ + title: t('settings.websearch.subscribe_add') + }) + + if (result && result.url) { + try { + // 获取并解析订阅源内容 + const blacklist = await parseSubscribeContent(result.url) + + if (blacklist.length === 0) { + throw new Error('No valid patterns found in subscribe content') + } + // 添加到 Redux store + addSubscribeSource({ + url: result.url, + name: result.name || result.url, + blacklist + }) + setSubscribeValid(true) + // 显示成功消息 + window.message.success({ + content: t('settings.websearch.subscribe_add_success'), + duration: 2 + }) + setTimeout(() => setSubscribeValid(false), 3000) + } catch (error) { + setSubscribeValid(false) + window.message.error({ + content: t('settings.websearch.subscribe_add_failed'), + duration: 2 + }) + } + } + setSubscribeChecking(false) + } + function handleDeleteSubscribe() { + try { + // 过滤掉被选中要删除的项目 + const remainingSources = + websearch.subscribeSources?.filter((source) => !selectedRowKeys.includes(source.key)) || [] + + // 更新 Redux store + setSubscribeSources(remainingSources) + + // 清空选中状态 + setSelectedRowKeys([]) + } catch (error) { + console.error('Error deleting subscribes:', error) + } } return ( @@ -61,6 +231,45 @@ const BlacklistSettings: FC = () => { {t('common.save')} {errFormat && } + + {t('settings.websearch.subscribe')} +
+ + {t('settings.websearch.subscribe_tooltip')} + + + + rowSelection={{ type: 'checkbox', ...rowSelection }} + columns={columns} + dataSource={dataSource} + pagination={{ position: ['none'] }} + /> + + + + +
) diff --git a/src/renderer/src/providers/WebSearchProvider/BaseWebSearchProvider.ts b/src/renderer/src/providers/WebSearchProvider/BaseWebSearchProvider.ts index 8359e037..ca7e33fa 100644 --- a/src/renderer/src/providers/WebSearchProvider/BaseWebSearchProvider.ts +++ b/src/renderer/src/providers/WebSearchProvider/BaseWebSearchProvider.ts @@ -1,3 +1,4 @@ +import { WebSearchState } from '@renderer/store/websearch' import { WebSearchProvider, WebSearchResponse } from '@renderer/types' export default abstract class BaseWebSearchProvider { @@ -9,8 +10,7 @@ export default abstract class BaseWebSearchProvider { this.provider = provider this.apiKey = this.getApiKey() } - - abstract search(query: string, maxResult: number, excludeDomains: string[]): Promise + abstract search(query: string, websearch: WebSearchState): Promise public getApiKey() { const keys = this.provider.apiKey?.split(',').map((key) => key.trim()) || [] diff --git a/src/renderer/src/providers/WebSearchProvider/ExaProvider.ts b/src/renderer/src/providers/WebSearchProvider/ExaProvider.ts index 6f918907..79140a02 100644 --- a/src/renderer/src/providers/WebSearchProvider/ExaProvider.ts +++ b/src/renderer/src/providers/WebSearchProvider/ExaProvider.ts @@ -1,4 +1,5 @@ import { ExaClient } from '@agentic/exa' +import { WebSearchState } from '@renderer/store/websearch' import { WebSearchProvider, WebSearchResponse } from '@renderer/types' import BaseWebSearchProvider from './BaseWebSearchProvider' @@ -14,7 +15,7 @@ export default class ExaProvider extends BaseWebSearchProvider { this.exa = new ExaClient({ apiKey: this.apiKey }) } - public async search(query: string, maxResults: number): Promise { + public async search(query: string, websearch: WebSearchState): Promise { try { if (!query.trim()) { throw new Error('Search query cannot be empty') @@ -22,7 +23,7 @@ export default class ExaProvider extends BaseWebSearchProvider { const response = await this.exa.search({ query, - numResults: Math.max(1, maxResults), + numResults: Math.max(1, websearch.maxResults), contents: { text: true } diff --git a/src/renderer/src/providers/WebSearchProvider/LocalSearchProvider.ts b/src/renderer/src/providers/WebSearchProvider/LocalSearchProvider.ts index 2bb11f7f..b5a6e595 100644 --- a/src/renderer/src/providers/WebSearchProvider/LocalSearchProvider.ts +++ b/src/renderer/src/providers/WebSearchProvider/LocalSearchProvider.ts @@ -1,5 +1,6 @@ import { Readability } from '@mozilla/readability' import { nanoid } from '@reduxjs/toolkit' +import { WebSearchState } from '@renderer/store/websearch' import { WebSearchProvider, WebSearchResponse, WebSearchResult } from '@renderer/types' import TurndownService from 'turndown' @@ -22,11 +23,7 @@ export default class LocalSearchProvider extends BaseWebSearchProvider { super(provider) } - public async search( - query: string, - maxResults: number = 15, - excludeDomains: string[] = [] - ): Promise { + public async search(query: string, websearch: WebSearchState): Promise { const uid = nanoid() try { if (!query.trim()) { @@ -41,16 +38,11 @@ export default class LocalSearchProvider extends BaseWebSearchProvider { const content = await window.api.searchService.openUrlInSearchWindow(uid, url) // Parse the content to extract URLs and metadata - const searchItems = this.parseValidUrls(content).slice(0, maxResults) - console.log('Total search items:', searchItems) + const searchItems = this.parseValidUrls(content).slice(0, websearch.maxResults) const validItems = searchItems - .filter( - (item) => - (item.url.startsWith('http') || item.url.startsWith('https')) && - excludeDomains.includes(new URL(item.url).host) === false - ) - .slice(0, maxResults) + .filter((item) => item.url.startsWith('http') || item.url.startsWith('https')) + .slice(0, websearch.maxResults) // console.log('Valid search items:', validItems) // Fetch content for each URL concurrently diff --git a/src/renderer/src/providers/WebSearchProvider/SearxngProvider.ts b/src/renderer/src/providers/WebSearchProvider/SearxngProvider.ts index 0b0a8119..3fd7993b 100644 --- a/src/renderer/src/providers/WebSearchProvider/SearxngProvider.ts +++ b/src/renderer/src/providers/WebSearchProvider/SearxngProvider.ts @@ -1,4 +1,5 @@ import { SearxngClient } from '@agentic/searxng' +import { WebSearchState } from '@renderer/store/websearch' import { WebSearchProvider, WebSearchResponse } from '@renderer/types' import axios from 'axios' @@ -68,7 +69,7 @@ export default class SearxngProvider extends BaseWebSearchProvider { } } - public async search(query: string, maxResults: number): Promise { + public async search(query: string, websearch: WebSearchState): Promise { try { if (!query) { throw new Error('Search query cannot be empty') @@ -88,10 +89,9 @@ export default class SearxngProvider extends BaseWebSearchProvider { if (!result || !Array.isArray(result.results)) { throw new Error('Invalid search results from SearxNG') } - return { query: result.query, - results: result.results.slice(0, maxResults).map((result) => { + results: result.results.slice(0, websearch.maxResults).map((result) => { return { title: result.title || 'No title', content: result.content || '', diff --git a/src/renderer/src/providers/WebSearchProvider/TavilyProvider.ts b/src/renderer/src/providers/WebSearchProvider/TavilyProvider.ts index f45747aa..c1d7a528 100644 --- a/src/renderer/src/providers/WebSearchProvider/TavilyProvider.ts +++ b/src/renderer/src/providers/WebSearchProvider/TavilyProvider.ts @@ -1,4 +1,5 @@ import { TavilyClient } from '@agentic/tavily' +import { WebSearchState } from '@renderer/store/websearch' import { WebSearchProvider, WebSearchResponse } from '@renderer/types' import BaseWebSearchProvider from './BaseWebSearchProvider' @@ -14,7 +15,7 @@ export default class TavilyProvider extends BaseWebSearchProvider { this.tvly = new TavilyClient({ apiKey: this.apiKey }) } - public async search(query: string, maxResults: number, excludeDomains: string[]): Promise { + public async search(query: string, websearch: WebSearchState): Promise { try { if (!query.trim()) { throw new Error('Search query cannot be empty') @@ -22,10 +23,8 @@ export default class TavilyProvider extends BaseWebSearchProvider { const result = await this.tvly.search({ query, - max_results: Math.max(1, maxResults), - exclude_domains: excludeDomains || [] + max_results: Math.max(1, websearch.maxResults) }) - return { query: result.query, results: result.results.map((result) => ({ diff --git a/src/renderer/src/providers/WebSearchProvider/index.ts b/src/renderer/src/providers/WebSearchProvider/index.ts index 1839960a..999e9e0a 100644 --- a/src/renderer/src/providers/WebSearchProvider/index.ts +++ b/src/renderer/src/providers/WebSearchProvider/index.ts @@ -1,4 +1,6 @@ +import { WebSearchState } from '@renderer/store/websearch' import { WebSearchProvider, WebSearchResponse } from '@renderer/types' +import { filterResultWithBlacklist } from '@renderer/utils/blacklistMatchPattern' import BaseWebSearchProvider from './BaseWebSearchProvider' import WebSearchProviderFactory from './WebSearchProviderFactory' @@ -8,7 +10,10 @@ export default class WebSearchEngineProvider { constructor(provider: WebSearchProvider) { this.sdk = WebSearchProviderFactory.create(provider) } - public async search(query: string, maxResult: number, excludeDomains: string[]): Promise { - return await this.sdk.search(query, maxResult, excludeDomains) + public async search(query: string, websearch: WebSearchState): Promise { + const result = await this.sdk.search(query, websearch) + const filteredResult = await filterResultWithBlacklist(result, websearch) + + return filteredResult } } diff --git a/src/renderer/src/services/WebSearchService.ts b/src/renderer/src/services/WebSearchService.ts index 3c05a0fe..883cd2f3 100644 --- a/src/renderer/src/services/WebSearchService.ts +++ b/src/renderer/src/services/WebSearchService.ts @@ -97,16 +97,17 @@ class WebSearchService { * @returns 搜索响应 */ public async search(provider: WebSearchProvider, query: string): Promise { - const { searchWithTime, maxResults, excludeDomains } = this.getWebSearchState() + const websearch = this.getWebSearchState() const webSearchEngine = new WebSearchEngineProvider(provider) let formattedQuery = query - if (searchWithTime) { + // 有待商榷，效果一般 + if (websearch.searchWithTime) { formattedQuery = `today is ${dayjs().format('YYYY-MM-DD')} \r\n ${query}` } try { - return await webSearchEngine.search(formattedQuery, maxResults, excludeDomains) + return await webSearchEngine.search(formattedQuery, websearch) } catch (error) { console.error('Search failed:', error) throw new Error(`Search failed: ${error instanceof Error ? error.message : 'Unknown error'}`) diff --git a/src/renderer/src/store/websearch.ts b/src/renderer/src/store/websearch.ts index 48a06d84..82e471f9 100644 --- a/src/renderer/src/store/websearch.ts +++ b/src/renderer/src/store/websearch.ts @@ -1,5 +1,11 @@ import { createSlice, PayloadAction } from '@reduxjs/toolkit' import type { WebSearchProvider } from '@renderer/types' +export interface SubscribeSource { + key: number + url: string + name: string + blacklist?: string[] // 存储从该订阅源获取的黑名单 +} export interface WebSearchState { // 默认搜索提供商的ID @@ -12,6 +18,7 @@ export interface WebSearchState { maxResults: number // 要排除的域名列表 excludeDomains: string[] + subscribeSources: SubscribeSource[] // 是否启用搜索增强模式 enhanceMode: boolean // 是否覆盖服务商搜索 @@ -55,6 +62,7 @@ const initialState: WebSearchState = { searchWithTime: true, maxResults: 5, excludeDomains: [], + subscribeSources: [], enhanceMode: false, overwrite: false } @@ -89,6 +97,33 @@ const websearchSlice = createSlice({ setExcludeDomains: (state, action: PayloadAction) => { state.excludeDomains = action.payload }, + // 添加订阅源 + addSubscribeSource: (state, action: PayloadAction>) => { + state.subscribeSources = state.subscribeSources || [] + const newKey = + state.subscribeSources.length > 0 ? Math.max(...state.subscribeSources.map((item) => item.key)) + 1 : 0 + state.subscribeSources.push({ + key: newKey, + url: action.payload.url, + name: action.payload.name, + blacklist: action.payload.blacklist + }) + }, + // 删除订阅源 + removeSubscribeSource: (state, action: PayloadAction) => { + state.subscribeSources = state.subscribeSources.filter((source) => source.key !== action.payload) + }, + // 更新订阅源的黑名单 + updateSubscribeBlacklist: (state, action: PayloadAction<{ key: number; blacklist: string[] }>) => { + const source = state.subscribeSources.find((s) => s.key === action.payload.key) + if (source) { + source.blacklist = action.payload.blacklist + } + }, + // 更新订阅源列表 + setSubscribeSources: (state, action: PayloadAction) => { + state.subscribeSources = action.payload + }, setEnhanceMode: (state, action: PayloadAction) => { state.enhanceMode = action.payload }, @@ -115,6 +150,10 @@ export const { setSearchWithTime, setExcludeDomains, setMaxResult, + addSubscribeSource, + removeSubscribeSource, + updateSubscribeBlacklist, + setSubscribeSources, setEnhanceMode, setOverwrite, addWebSearchProvider diff --git a/src/renderer/src/utils/blacklistMatchPattern.ts b/src/renderer/src/utils/blacklistMatchPattern.ts index 38e3ddfd..d9a11d09 100644 --- a/src/renderer/src/utils/blacklistMatchPattern.ts +++ b/src/renderer/src/utils/blacklistMatchPattern.ts @@ -1,3 +1,6 @@ +import { WebSearchState } from '@renderer/store/websearch' +import { WebSearchResponse } from '@renderer/types' + /* * MIT License * @@ -170,3 +173,108 @@ function testPath(pathPattern: string, path: string): boolean { } return path.slice(pos).endsWith(rest[rest.length - 1]) } + +// 添加新的解析函数 +export async function parseSubscribeContent(url: string): Promise { + try { + // 获取订阅源内容 + const response = await fetch(url) + console.log('response', response) + if (!response.ok) { + throw new Error('Failed to fetch subscribe content') + } + + const content = await response.text() + + // 按行分割内容 + const lines = content.split('\n') + + // 过滤出有效的匹配模式 + const patterns = lines + .filter((line) => line.trim() !== '' && !line.startsWith('#')) + .map((line) => line.trim()) + .filter((pattern) => parseMatchPattern(pattern) !== null) + + return patterns + } catch (error) { + console.error('Error parsing subscribe content:', error) + throw error + } +} +export async function filterResultWithBlacklist( + response: WebSearchResponse, + websearch: WebSearchState +): Promise { + console.log('filterResultWithBlacklist', response) + // 没有结果或者没有黑名单规则时，直接返回原始结果 + if (!response.results?.length || (!websearch.excludeDomains.length && !websearch.subscribeSources.length)) { + return response + } + + // 创建匹配模式映射实例 + const patternMap = new MatchPatternMap() + + // 合并所有黑名单规则 + const blacklistPatterns: string[] = [ + ...websearch.excludeDomains, + ...(websearch.subscribeSources?.length + ? websearch.subscribeSources.reduce((acc, source) => { + return acc.concat(source.blacklist || []) + }, []) + : []) + ] + + // 正则表达式规则集合 + const regexPatterns: RegExp[] = [] + + // 分类处理黑名单规则 + blacklistPatterns.forEach((pattern) => { + if (pattern.startsWith('/') && pattern.endsWith('/')) { + // 处理正则表达式格式 + try { + const regexPattern = pattern.slice(1, -1) + regexPatterns.push(new RegExp(regexPattern, 'i')) + } catch (error) { + console.error('Invalid regex pattern:', pattern, error) + } + } else { + // 处理匹配模式格式 + try { + patternMap.set(pattern, pattern) + } catch (error) { + console.error('Invalid match pattern:', pattern, error) + } + } + }) + + // 过滤搜索结果 + const filteredResults = response.results.filter((result) => { + try { + const url = new URL(result.url) + + // 检查URL是否匹配任何正则表达式规则 + const matchesRegex = regexPatterns.some((regex) => regex.test(url.hostname)) + if (matchesRegex) { + return false + } + + // 检查URL是否匹配任何匹配模式规则 + const matchesPattern = patternMap.get(result.url).length > 0 + if (matchesPattern) { + return false + } + + return true + } catch (error) { + console.error('Error processing URL:', result.url, error) + return true // 如果URL解析失败，保留该结果 + } + }) + + console.log('filterResultWithBlacklist filtered results:', filteredResults) + + return { + ...response, + results: filteredResults + } +}