feat: support ublacklist subscribe (#2974)
* feat: support ublacklist subscribe * Merge branch 'main' into feat-ublacklist * chore * chore
This commit is contained in:
Chen Tao
GitHub
parent
afd1381d7f
commit
2e0251aed7
@ -1,6 +1,10 @@
|
||||
import { useAppDispatch, useAppSelector } from '@renderer/store'
|
||||
import {
|
||||
addSubscribeSource as _addSubscribeSource,
|
||||
removeSubscribeSource as _removeSubscribeSource,
|
||||
setDefaultProvider as _setDefaultProvider,
|
||||
setSubscribeSources as _setSubscribeSources,
|
||||
updateSubscribeBlacklist as _updateSubscribeBlacklist,
|
||||
updateWebSearchProvider,
|
||||
updateWebSearchProviders
|
||||
} from '@renderer/store/websearch'
|
||||
@ -57,3 +61,32 @@ export const useWebSearchProvider = (id: string) => {
|
||||
|
||||
return { provider, updateProvider }
|
||||
}
|
||||
|
||||
export const useBlacklist = () => {
|
||||
const dispatch = useAppDispatch()
|
||||
const websearch = useAppSelector((state) => state.websearch)
|
||||
|
||||
const addSubscribeSource = ({ url, name, blacklist }) => {
|
||||
dispatch(_addSubscribeSource({ url, name, blacklist }))
|
||||
}
|
||||
|
||||
const removeSubscribeSource = (key: number) => {
|
||||
dispatch(_removeSubscribeSource(key))
|
||||
}
|
||||
|
||||
const updateSubscribeBlacklist = (key: number, blacklist: string[]) => {
|
||||
dispatch(_updateSubscribeBlacklist({ key, blacklist }))
|
||||
}
|
||||
|
||||
const setSubscribeSources = (sources: { key: number; url: string; name: string; blacklist?: string[] }[]) => {
|
||||
dispatch(_setSubscribeSources(sources))
|
||||
}
|
||||
|
||||
return {
|
||||
websearch,
|
||||
addSubscribeSource,
|
||||
removeSubscribeSource,
|
||||
updateSubscribeBlacklist,
|
||||
setSubscribeSources
|
||||
}
|
||||
}
|
||||
|
||||
@ -1300,6 +1300,15 @@
|
||||
"title": "Tavily"
|
||||
},
|
||||
"title": "Web Search",
|
||||
"subscribe": "Subscribe",
|
||||
"subscribe_tooltip": "Subscribe to the blacklist.",
|
||||
"subscribe_update": "Update now",
|
||||
"subscribe_add": "Add Subscription",
|
||||
"subscribe_url": "Subscription feed address",
|
||||
"subscribe_name": "Alternative name",
|
||||
"subscribe_name.placeholder": "Alternative name used when the downloaded subscription feed has no name.",
|
||||
"subscribe_add_success": "Subscription feed added successfully!",
|
||||
"subscribe_delete": "Delete subscription source",
|
||||
"overwrite": "Override search service",
|
||||
"overwrite_tooltip": "Force use search service instead of LLM",
|
||||
"apikey": "API key",
|
||||
|
||||
@ -1279,7 +1279,6 @@
|
||||
"websearch": {
|
||||
"blacklist": "ブラックリスト",
|
||||
"blacklist_description": "以下のウェブサイトの結果は検索結果に表示されません",
|
||||
"blacklist_tooltip": "以下の形式を使用してください(改行区切り)\nexample.com\nhttps://www.example.com\nhttps://example.com\n*://*.example.com",
|
||||
"check": "チェック",
|
||||
"check_failed": "検証に失敗しました",
|
||||
"check_success": "検証に成功しました",
|
||||
@ -1299,6 +1298,16 @@
|
||||
"title": "Tavily"
|
||||
},
|
||||
"title": "ウェブ検索",
|
||||
"blacklist_tooltip": "マッチパターン: *://*.example.com/*\n正規表現: /example\\.(net|org)/",
|
||||
"subscribe": "購読",
|
||||
"subscribe_tooltip": "ブラックリストに登録する",
|
||||
"subscribe_update": "今すぐ更新",
|
||||
"subscribe_add": "サブスクリプションを追加",
|
||||
"subscribe_url": "フィードのURL",
|
||||
"subscribe_name": "代替名",
|
||||
"subscribe_name.placeholder": "ダウンロードしたフィードに名前がない場合に使用される代替名",
|
||||
"subscribe_add_success": "フィードの追加が成功しました！",
|
||||
"subscribe_delete": "フィードの削除",
|
||||
"overwrite": "サービス検索を上書き",
|
||||
"overwrite_tooltip": "大規模言語モデルではなく、サービス検索を使用する",
|
||||
"apikey": "API キー",
|
||||
|
||||
@ -1279,7 +1279,6 @@
|
||||
"websearch": {
|
||||
"blacklist": "Черный список",
|
||||
"blacklist_description": "Результаты из следующих веб-сайтов не будут отображаться в результатах поиска",
|
||||
"blacklist_tooltip": "Пожалуйста, используйте следующий формат (разделенный переносами строк)\nexample.com\nhttps://www.example.com\nhttps://example.com\n*://*.example.com",
|
||||
"check": "проверка",
|
||||
"check_failed": "Проверка не прошла",
|
||||
"check_success": "Проверка успешна",
|
||||
@ -1299,6 +1298,16 @@
|
||||
"title": "Tavily"
|
||||
},
|
||||
"title": "Поиск в Интернете",
|
||||
"blacklist_tooltip": "Соответствующий шаблон: *://*.example.com/*\nРегулярное выражение: /example\\.(net|org)/",
|
||||
"subscribe": "Подписка",
|
||||
"subscribe_tooltip": "Подписаться на черный список",
|
||||
"subscribe_update": "Обновить сейчас",
|
||||
"subscribe_add": "Добавить подписку",
|
||||
"subscribe_url": "Адрес источника подписки",
|
||||
"subscribe_name": "альтернативное имя",
|
||||
"subscribe_name.placeholder": "替代名称, используемый, когда загружаемый подписочный источник не имеет названия",
|
||||
"subscribe_add_success": "Подписка добавлена успешно!",
|
||||
"subscribe_delete": "Удалить источник подписки",
|
||||
"overwrite": "Переопределить поставщика поиска",
|
||||
"overwrite_tooltip": "Использовать поставщика поиска вместо LLM",
|
||||
"apikey": "Ключ API",
|
||||
|
||||
@ -1280,7 +1280,7 @@
|
||||
"websearch": {
|
||||
"blacklist": "黑名单",
|
||||
"blacklist_description": "在搜索结果中不会出现以下网站的结果",
|
||||
"blacklist_tooltip": "请使用以下格式(换行分隔)\nexample.com\nhttps://www.example.com\nhttps://example.com\n*://*.example.com",
|
||||
"blacklist_tooltip": "请使用以下格式(换行分隔)\n匹配模式: *://*.example.com/*\n正则表达式: /example\\.(net|org)/",
|
||||
"check": "检查",
|
||||
"check_failed": "验证失败",
|
||||
"check_success": "验证成功",
|
||||
@ -1293,6 +1293,15 @@
|
||||
"search_max_result": "搜索结果个数",
|
||||
"search_provider": "搜索服务商",
|
||||
"search_provider_placeholder": "选择一个搜索服务商",
|
||||
"subscribe": "订阅",
|
||||
"subscribe_tooltip": "订阅黑名单列表",
|
||||
"subscribe_update": "立即更新",
|
||||
"subscribe_add": "添加订阅",
|
||||
"subscribe_url": "订阅源地址",
|
||||
"subscribe_name": "替代名字",
|
||||
"subscribe_name.placeholder": "当下载的订阅源没有名称时所使用的替代名称",
|
||||
"subscribe_add_success": "订阅源添加成功!",
|
||||
"subscribe_delete": "删除订阅源",
|
||||
"search_result_default": "默认",
|
||||
"search_with_time": "搜索包含日期",
|
||||
"tavily": {
|
||||
@ -1372,3 +1381,5 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1277,20 +1277,10 @@
|
||||
"tray.show": "顯示系统匣圖示",
|
||||
"tray.title": "系统匣",
|
||||
"websearch": {
|
||||
"blacklist": "黑名單",
|
||||
"blacklist_description": "以下網站不會出現在搜尋結果中",
|
||||
"blacklist_tooltip": "請使用以下格式 (換行符號分隔)\nexample.com\nhttps://www.example.com\nhttps://example.com\n*://*.example.com",
|
||||
"check": "檢查",
|
||||
"check_failed": "驗證失敗",
|
||||
"check_success": "驗證成功",
|
||||
"enhance_mode": "搜索增強模式",
|
||||
"enhance_mode_tooltip": "使用預設模型提取關鍵詞後搜索",
|
||||
"get_api_key": "點選這裡取得金鑰",
|
||||
"no_provider_selected": "請選擇搜尋服務商後再檢查",
|
||||
"search_max_result": "搜尋結果個數",
|
||||
"search_provider": "搜尋服務商",
|
||||
"search_provider_placeholder": "選擇一個搜尋服務商",
|
||||
"search_result_default": "預設",
|
||||
"search_with_time": "搜尋包含日期",
|
||||
"tavily": {
|
||||
"api_key": "Tavily API 金鑰",
|
||||
@ -1298,6 +1288,25 @@
|
||||
"description": "Tavily 是一個為 AI 代理量身訂製的搜尋引擎，提供即時、準確的結果、智慧查詢建議和深入的研究能力",
|
||||
"title": "Tavily"
|
||||
},
|
||||
"blacklist": "黑名單",
|
||||
"blacklist_description": "以下網站不會出現在搜索結果中",
|
||||
"search_max_result": "搜尋結果個數",
|
||||
"search_result_default": "預設",
|
||||
"check": "檢查",
|
||||
"search_provider": "搜尋服務商",
|
||||
"search_provider_placeholder": "選擇一個搜尋服務商",
|
||||
"no_provider_selected": "請選擇搜索服務商後再檢查",
|
||||
"check_failed": "驗證失敗",
|
||||
"blacklist_tooltip": "匹配模式: *://*.example.com/*\n正则表达式: /example\\.(net|org)/",
|
||||
"subscribe": "訂閱",
|
||||
"subscribe_tooltip": "訂閱黑名單列表",
|
||||
"subscribe_update": "立即更新",
|
||||
"subscribe_add": "添加訂閱",
|
||||
"subscribe_url": "訂閱源地址",
|
||||
"subscribe_name": "替代名稱",
|
||||
"subscribe_name.placeholder": "當下載的訂閱源沒有名稱時所使用的替代名稱",
|
||||
"subscribe_add_success": "訂閱源添加成功!",
|
||||
"subscribe_delete": "刪除訂閱源",
|
||||
"title": "網路搜尋",
|
||||
"overwrite": "覆蓋搜尋服務商",
|
||||
"overwrite_tooltip": "強制使用搜尋服務商而不是大語言模型進行搜尋",
|
||||
|
||||
@ -0,0 +1,120 @@
|
||||
import { TopView } from '@renderer/components/TopView'
|
||||
import { Button, Form, FormProps, Input, Modal } from 'antd'
|
||||
import { useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
interface ShowParams {
|
||||
title: string
|
||||
}
|
||||
|
||||
interface Props extends ShowParams {
|
||||
resolve: (data: any) => void
|
||||
}
|
||||
|
||||
type FieldType = {
|
||||
url: string
|
||||
name?: string
|
||||
}
|
||||
|
||||
const PopupContainer: React.FC<Props> = ({ title, resolve }) => {
|
||||
const [open, setOpen] = useState(true)
|
||||
const [form] = Form.useForm()
|
||||
const { t } = useTranslation()
|
||||
|
||||
const onOk = () => {
|
||||
setOpen(false)
|
||||
}
|
||||
|
||||
const onCancel = () => {
|
||||
setOpen(false)
|
||||
}
|
||||
|
||||
const onClose = () => {
|
||||
resolve({})
|
||||
}
|
||||
|
||||
const onFinish: FormProps<FieldType>['onFinish'] = (values) => {
|
||||
const url = values.url.trim()
|
||||
const name = values.name?.trim() || url
|
||||
|
||||
if (!url) {
|
||||
window.message.error(t('settings.websearch.url_required'))
|
||||
return
|
||||
}
|
||||
|
||||
// 验证URL格式
|
||||
try {
|
||||
new URL(url)
|
||||
} catch (e) {
|
||||
window.message.error(t('settings.websearch.url_invalid'))
|
||||
return
|
||||
}
|
||||
|
||||
resolve({ url, name })
|
||||
}
|
||||
|
||||
return (
|
||||
<Modal
|
||||
title={title}
|
||||
open={open}
|
||||
onOk={onOk}
|
||||
onCancel={onCancel}
|
||||
maskClosable={false}
|
||||
afterClose={onClose}
|
||||
footer={null}
|
||||
centered>
|
||||
<Form
|
||||
form={form}
|
||||
labelCol={{ flex: '110px' }}
|
||||
labelAlign="left"
|
||||
colon={false}
|
||||
style={{ marginTop: 25 }}
|
||||
onFinish={onFinish}>
|
||||
<Form.Item name="url" label={t('settings.websearch.subscribe_url')} rules={[{ required: true }]}>
|
||||
<Input
|
||||
placeholder="https://git.io/ublacklist"
|
||||
spellCheck={false}
|
||||
maxLength={500}
|
||||
onChange={(e) => {
|
||||
try {
|
||||
const url = new URL(e.target.value)
|
||||
form.setFieldValue('name', url.hostname)
|
||||
} catch (e) {
|
||||
// URL不合法，忽略
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Form.Item>
|
||||
<Form.Item name="name" label={t('settings.websearch.subscribe_name')}>
|
||||
<Input placeholder={t('settings.websearch.subscribe_name.placeholder')} spellCheck={false} />
|
||||
</Form.Item>
|
||||
<Form.Item label=" ">
|
||||
<Button type="primary" htmlType="submit">
|
||||
{t('settings.websearch.subscribe_add')}
|
||||
</Button>
|
||||
</Form.Item>
|
||||
</Form>
|
||||
</Modal>
|
||||
)
|
||||
}
|
||||
|
||||
export default class AddSubscribePopup {
|
||||
static topviewId = 0
|
||||
static hide() {
|
||||
TopView.hide('AddSubscribePopup')
|
||||
}
|
||||
static show(props: ShowParams) {
|
||||
return new Promise<any>((resolve) => {
|
||||
TopView.show(
|
||||
<PopupContainer
|
||||
{...props}
|
||||
resolve={(v) => {
|
||||
resolve(v)
|
||||
this.hide()
|
||||
}}
|
||||
/>,
|
||||
'AddSubscribePopup'
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -1,22 +1,61 @@
|
||||
import { CheckOutlined, InfoCircleOutlined, LoadingOutlined } from '@ant-design/icons'
|
||||
import { useTheme } from '@renderer/context/ThemeProvider'
|
||||
import { useBlacklist } from '@renderer/hooks/useWebSearchProviders'
|
||||
import { useAppDispatch, useAppSelector } from '@renderer/store'
|
||||
import { setExcludeDomains } from '@renderer/store/websearch'
|
||||
import { parseMatchPattern } from '@renderer/utils/blacklistMatchPattern'
|
||||
import { Alert, Button } from 'antd'
|
||||
import { parseMatchPattern, parseSubscribeContent } from '@renderer/utils/blacklistMatchPattern'
|
||||
import { Alert, Button, Table, TableProps } from 'antd'
|
||||
import TextArea from 'antd/es/input/TextArea'
|
||||
import { t } from 'i18next'
|
||||
import { FC, useEffect, useState } from 'react'
|
||||
|
||||
import { SettingDivider, SettingGroup, SettingRow, SettingRowTitle, SettingTitle } from '..'
|
||||
import AddSubscribePopup from './AddSubscribePopup'
|
||||
|
||||
type TableRowSelection<T extends object = object> = TableProps<T>['rowSelection']
|
||||
interface DataType {
|
||||
key: React.Key
|
||||
url: string
|
||||
name: string
|
||||
}
|
||||
const columns: TableProps<DataType>['columns'] = [
|
||||
{ title: t('common.name'), dataIndex: 'name', key: 'name' },
|
||||
{
|
||||
title: 'URL',
|
||||
dataIndex: 'url',
|
||||
key: 'url'
|
||||
}
|
||||
]
|
||||
const BlacklistSettings: FC = () => {
|
||||
const [errFormat, setErrFormat] = useState(false)
|
||||
const [blacklistInput, setBlacklistInput] = useState('')
|
||||
const excludeDomains = useAppSelector((state) => state.websearch.excludeDomains)
|
||||
const { websearch, setSubscribeSources, addSubscribeSource } = useBlacklist()
|
||||
const { theme } = useTheme()
|
||||
const [subscribeChecking, setSubscribeChecking] = useState(false)
|
||||
const [subscribeValid, setSubscribeValid] = useState(false)
|
||||
const [selectedRowKeys, setSelectedRowKeys] = useState<React.Key[]>([])
|
||||
const [dataSource, setDataSource] = useState<DataType[]>(
|
||||
websearch.subscribeSources?.map((source) => ({
|
||||
key: source.key,
|
||||
url: source.url,
|
||||
name: source.name
|
||||
})) || []
|
||||
)
|
||||
|
||||
const dispatch = useAppDispatch()
|
||||
|
||||
useEffect(() => {
|
||||
setDataSource(
|
||||
(websearch.subscribeSources || []).map((source) => ({
|
||||
key: source.key,
|
||||
url: source.url,
|
||||
name: source.name
|
||||
}))
|
||||
)
|
||||
console.log('subscribeSources', websearch.subscribeSources)
|
||||
}, [websearch.subscribeSources])
|
||||
|
||||
useEffect(() => {
|
||||
if (excludeDomains) {
|
||||
setBlacklistInput(excludeDomains.join('\n'))
|
||||
@ -40,6 +79,137 @@ const BlacklistSettings: FC = () => {
|
||||
if (hasError) return
|
||||
|
||||
dispatch(setExcludeDomains(validDomains))
|
||||
window.message.info({
|
||||
content: t('message.save.success.title'),
|
||||
duration: 4,
|
||||
icon: <InfoCircleOutlined />,
|
||||
key: 'save-blacklist-info'
|
||||
})
|
||||
}
|
||||
const onSelectChange = (newSelectedRowKeys: React.Key[]) => {
|
||||
console.log('selectedRowKeys changed: ', newSelectedRowKeys)
|
||||
setSelectedRowKeys(newSelectedRowKeys)
|
||||
}
|
||||
|
||||
const rowSelection: TableRowSelection<DataType> = {
|
||||
selectedRowKeys,
|
||||
onChange: onSelectChange
|
||||
}
|
||||
async function updateSubscribe() {
|
||||
setSubscribeChecking(true)
|
||||
|
||||
try {
|
||||
// 获取选中的订阅源
|
||||
const selectedSources = dataSource.filter((item) => selectedRowKeys.includes(item.key))
|
||||
|
||||
// 用于存储所有成功解析的订阅源数据
|
||||
const updatedSources: {
|
||||
key: number
|
||||
url: string
|
||||
name: string
|
||||
blacklist: string[]
|
||||
}[] = []
|
||||
|
||||
// 为每个选中的订阅源获取并解析内容
|
||||
for (const source of selectedSources) {
|
||||
try {
|
||||
// 获取并解析订阅源内容
|
||||
const blacklist = await parseSubscribeContent(source.url)
|
||||
|
||||
if (blacklist.length > 0) {
|
||||
updatedSources.push({
|
||||
key: Number(source.key),
|
||||
url: source.url,
|
||||
name: source.name,
|
||||
blacklist
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error updating subscribe source ${source.url}:`, error)
|
||||
// 显示具体源更新失败的消息
|
||||
window.message.warning({
|
||||
content: t('settings.websearch.subscribe_source_update_failed', { url: source.url }),
|
||||
duration: 3
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (updatedSources.length > 0) {
|
||||
// 更新 Redux store
|
||||
setSubscribeSources(updatedSources)
|
||||
setSubscribeValid(true)
|
||||
// 显示成功消息
|
||||
window.message.success({
|
||||
content: t('settings.websearch.subscribe_update_success'),
|
||||
duration: 2
|
||||
})
|
||||
setTimeout(() => setSubscribeValid(false), 3000)
|
||||
} else {
|
||||
setSubscribeValid(false)
|
||||
throw new Error('No valid sources updated')
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error updating subscribes:', error)
|
||||
window.message.error({
|
||||
content: t('settings.websearch.subscribe_update_failed'),
|
||||
duration: 2
|
||||
})
|
||||
}
|
||||
setSubscribeChecking(false)
|
||||
}
|
||||
|
||||
// 修改 handleAddSubscribe 函数
|
||||
async function handleAddSubscribe() {
|
||||
setSubscribeChecking(true)
|
||||
const result = await AddSubscribePopup.show({
|
||||
title: t('settings.websearch.subscribe_add')
|
||||
})
|
||||
|
||||
if (result && result.url) {
|
||||
try {
|
||||
// 获取并解析订阅源内容
|
||||
const blacklist = await parseSubscribeContent(result.url)
|
||||
|
||||
if (blacklist.length === 0) {
|
||||
throw new Error('No valid patterns found in subscribe content')
|
||||
}
|
||||
// 添加到 Redux store
|
||||
addSubscribeSource({
|
||||
url: result.url,
|
||||
name: result.name || result.url,
|
||||
blacklist
|
||||
})
|
||||
setSubscribeValid(true)
|
||||
// 显示成功消息
|
||||
window.message.success({
|
||||
content: t('settings.websearch.subscribe_add_success'),
|
||||
duration: 2
|
||||
})
|
||||
setTimeout(() => setSubscribeValid(false), 3000)
|
||||
} catch (error) {
|
||||
setSubscribeValid(false)
|
||||
window.message.error({
|
||||
content: t('settings.websearch.subscribe_add_failed'),
|
||||
duration: 2
|
||||
})
|
||||
}
|
||||
}
|
||||
setSubscribeChecking(false)
|
||||
}
|
||||
function handleDeleteSubscribe() {
|
||||
try {
|
||||
// 过滤掉被选中要删除的项目
|
||||
const remainingSources =
|
||||
websearch.subscribeSources?.filter((source) => !selectedRowKeys.includes(source.key)) || []
|
||||
|
||||
// 更新 Redux store
|
||||
setSubscribeSources(remainingSources)
|
||||
|
||||
// 清空选中状态
|
||||
setSelectedRowKeys([])
|
||||
} catch (error) {
|
||||
console.error('Error deleting subscribes:', error)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
@ -61,6 +231,45 @@ const BlacklistSettings: FC = () => {
|
||||
{t('common.save')}
|
||||
</Button>
|
||||
{errFormat && <Alert message={t('settings.websearch.blacklist_tooltip')} type="error" />}
|
||||
<SettingDivider />
|
||||
<SettingTitle>{t('settings.websearch.subscribe')}</SettingTitle>
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: '5px' }}>
|
||||
<SettingRow>
|
||||
{t('settings.websearch.subscribe_tooltip')}
|
||||
<Button
|
||||
type={subscribeValid ? 'primary' : 'default'}
|
||||
ghost={subscribeValid}
|
||||
disabled={subscribeChecking}
|
||||
onClick={handleAddSubscribe}>
|
||||
{t('settings.websearch.subscribe_add')}
|
||||
</Button>
|
||||
</SettingRow>
|
||||
<Table<DataType>
|
||||
rowSelection={{ type: 'checkbox', ...rowSelection }}
|
||||
columns={columns}
|
||||
dataSource={dataSource}
|
||||
pagination={{ position: ['none'] }}
|
||||
/>
|
||||
<SettingRow>
|
||||
<Button
|
||||
type={subscribeValid ? 'primary' : 'default'}
|
||||
ghost={subscribeValid}
|
||||
disabled={subscribeChecking || selectedRowKeys.length === 0}
|
||||
style={{ width: 100 }}
|
||||
onClick={updateSubscribe}>
|
||||
{subscribeChecking ? (
|
||||
<LoadingOutlined spin />
|
||||
) : subscribeValid ? (
|
||||
<CheckOutlined />
|
||||
) : (
|
||||
t('settings.websearch.subscribe_update')
|
||||
)}
|
||||
</Button>
|
||||
<Button style={{ width: 100 }} disabled={selectedRowKeys.length === 0} onClick={handleDeleteSubscribe}>
|
||||
{t('settings.websearch.subscribe_delete')}
|
||||
</Button>
|
||||
</SettingRow>
|
||||
</div>
|
||||
</SettingGroup>
|
||||
</>
|
||||
)
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import { WebSearchState } from '@renderer/store/websearch'
|
||||
import { WebSearchProvider, WebSearchResponse } from '@renderer/types'
|
||||
|
||||
export default abstract class BaseWebSearchProvider {
|
||||
@ -9,8 +10,7 @@ export default abstract class BaseWebSearchProvider {
|
||||
this.provider = provider
|
||||
this.apiKey = this.getApiKey()
|
||||
}
|
||||
|
||||
abstract search(query: string, maxResult: number, excludeDomains: string[]): Promise<WebSearchResponse>
|
||||
abstract search(query: string, websearch: WebSearchState): Promise<WebSearchResponse>
|
||||
|
||||
public getApiKey() {
|
||||
const keys = this.provider.apiKey?.split(',').map((key) => key.trim()) || []
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { ExaClient } from '@agentic/exa'
|
||||
import { WebSearchState } from '@renderer/store/websearch'
|
||||
import { WebSearchProvider, WebSearchResponse } from '@renderer/types'
|
||||
|
||||
import BaseWebSearchProvider from './BaseWebSearchProvider'
|
||||
@ -14,7 +15,7 @@ export default class ExaProvider extends BaseWebSearchProvider {
|
||||
this.exa = new ExaClient({ apiKey: this.apiKey })
|
||||
}
|
||||
|
||||
public async search(query: string, maxResults: number): Promise<WebSearchResponse> {
|
||||
public async search(query: string, websearch: WebSearchState): Promise<WebSearchResponse> {
|
||||
try {
|
||||
if (!query.trim()) {
|
||||
throw new Error('Search query cannot be empty')
|
||||
@ -22,7 +23,7 @@ export default class ExaProvider extends BaseWebSearchProvider {
|
||||
|
||||
const response = await this.exa.search({
|
||||
query,
|
||||
numResults: Math.max(1, maxResults),
|
||||
numResults: Math.max(1, websearch.maxResults),
|
||||
contents: {
|
||||
text: true
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { Readability } from '@mozilla/readability'
|
||||
import { nanoid } from '@reduxjs/toolkit'
|
||||
import { WebSearchState } from '@renderer/store/websearch'
|
||||
import { WebSearchProvider, WebSearchResponse, WebSearchResult } from '@renderer/types'
|
||||
import TurndownService from 'turndown'
|
||||
|
||||
@ -22,11 +23,7 @@ export default class LocalSearchProvider extends BaseWebSearchProvider {
|
||||
super(provider)
|
||||
}
|
||||
|
||||
public async search(
|
||||
query: string,
|
||||
maxResults: number = 15,
|
||||
excludeDomains: string[] = []
|
||||
): Promise<WebSearchResponse> {
|
||||
public async search(query: string, websearch: WebSearchState): Promise<WebSearchResponse> {
|
||||
const uid = nanoid()
|
||||
try {
|
||||
if (!query.trim()) {
|
||||
@ -41,16 +38,11 @@ export default class LocalSearchProvider extends BaseWebSearchProvider {
|
||||
const content = await window.api.searchService.openUrlInSearchWindow(uid, url)
|
||||
|
||||
// Parse the content to extract URLs and metadata
|
||||
const searchItems = this.parseValidUrls(content).slice(0, maxResults)
|
||||
console.log('Total search items:', searchItems)
|
||||
const searchItems = this.parseValidUrls(content).slice(0, websearch.maxResults)
|
||||
|
||||
const validItems = searchItems
|
||||
.filter(
|
||||
(item) =>
|
||||
(item.url.startsWith('http') || item.url.startsWith('https')) &&
|
||||
excludeDomains.includes(new URL(item.url).host) === false
|
||||
)
|
||||
.slice(0, maxResults)
|
||||
.filter((item) => item.url.startsWith('http') || item.url.startsWith('https'))
|
||||
.slice(0, websearch.maxResults)
|
||||
// console.log('Valid search items:', validItems)
|
||||
|
||||
// Fetch content for each URL concurrently
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { SearxngClient } from '@agentic/searxng'
|
||||
import { WebSearchState } from '@renderer/store/websearch'
|
||||
import { WebSearchProvider, WebSearchResponse } from '@renderer/types'
|
||||
import axios from 'axios'
|
||||
|
||||
@ -68,7 +69,7 @@ export default class SearxngProvider extends BaseWebSearchProvider {
|
||||
}
|
||||
}
|
||||
|
||||
public async search(query: string, maxResults: number): Promise<WebSearchResponse> {
|
||||
public async search(query: string, websearch: WebSearchState): Promise<WebSearchResponse> {
|
||||
try {
|
||||
if (!query) {
|
||||
throw new Error('Search query cannot be empty')
|
||||
@ -88,10 +89,9 @@ export default class SearxngProvider extends BaseWebSearchProvider {
|
||||
if (!result || !Array.isArray(result.results)) {
|
||||
throw new Error('Invalid search results from SearxNG')
|
||||
}
|
||||
|
||||
return {
|
||||
query: result.query,
|
||||
results: result.results.slice(0, maxResults).map((result) => {
|
||||
results: result.results.slice(0, websearch.maxResults).map((result) => {
|
||||
return {
|
||||
title: result.title || 'No title',
|
||||
content: result.content || '',
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { TavilyClient } from '@agentic/tavily'
|
||||
import { WebSearchState } from '@renderer/store/websearch'
|
||||
import { WebSearchProvider, WebSearchResponse } from '@renderer/types'
|
||||
|
||||
import BaseWebSearchProvider from './BaseWebSearchProvider'
|
||||
@ -14,7 +15,7 @@ export default class TavilyProvider extends BaseWebSearchProvider {
|
||||
this.tvly = new TavilyClient({ apiKey: this.apiKey })
|
||||
}
|
||||
|
||||
public async search(query: string, maxResults: number, excludeDomains: string[]): Promise<WebSearchResponse> {
|
||||
public async search(query: string, websearch: WebSearchState): Promise<WebSearchResponse> {
|
||||
try {
|
||||
if (!query.trim()) {
|
||||
throw new Error('Search query cannot be empty')
|
||||
@ -22,10 +23,8 @@ export default class TavilyProvider extends BaseWebSearchProvider {
|
||||
|
||||
const result = await this.tvly.search({
|
||||
query,
|
||||
max_results: Math.max(1, maxResults),
|
||||
exclude_domains: excludeDomains || []
|
||||
max_results: Math.max(1, websearch.maxResults)
|
||||
})
|
||||
|
||||
return {
|
||||
query: result.query,
|
||||
results: result.results.map((result) => ({
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
import { WebSearchState } from '@renderer/store/websearch'
|
||||
import { WebSearchProvider, WebSearchResponse } from '@renderer/types'
|
||||
import { filterResultWithBlacklist } from '@renderer/utils/blacklistMatchPattern'
|
||||
|
||||
import BaseWebSearchProvider from './BaseWebSearchProvider'
|
||||
import WebSearchProviderFactory from './WebSearchProviderFactory'
|
||||
@ -8,7 +10,10 @@ export default class WebSearchEngineProvider {
|
||||
constructor(provider: WebSearchProvider) {
|
||||
this.sdk = WebSearchProviderFactory.create(provider)
|
||||
}
|
||||
public async search(query: string, maxResult: number, excludeDomains: string[]): Promise<WebSearchResponse> {
|
||||
return await this.sdk.search(query, maxResult, excludeDomains)
|
||||
public async search(query: string, websearch: WebSearchState): Promise<WebSearchResponse> {
|
||||
const result = await this.sdk.search(query, websearch)
|
||||
const filteredResult = await filterResultWithBlacklist(result, websearch)
|
||||
|
||||
return filteredResult
|
||||
}
|
||||
}
|
||||
|
||||
@ -97,16 +97,17 @@ class WebSearchService {
|
||||
* @returns 搜索响应
|
||||
*/
|
||||
public async search(provider: WebSearchProvider, query: string): Promise<WebSearchResponse> {
|
||||
const { searchWithTime, maxResults, excludeDomains } = this.getWebSearchState()
|
||||
const websearch = this.getWebSearchState()
|
||||
const webSearchEngine = new WebSearchEngineProvider(provider)
|
||||
|
||||
let formattedQuery = query
|
||||
if (searchWithTime) {
|
||||
// 有待商榷，效果一般
|
||||
if (websearch.searchWithTime) {
|
||||
formattedQuery = `today is ${dayjs().format('YYYY-MM-DD')} \r\n ${query}`
|
||||
}
|
||||
|
||||
try {
|
||||
return await webSearchEngine.search(formattedQuery, maxResults, excludeDomains)
|
||||
return await webSearchEngine.search(formattedQuery, websearch)
|
||||
} catch (error) {
|
||||
console.error('Search failed:', error)
|
||||
throw new Error(`Search failed: ${error instanceof Error ? error.message : 'Unknown error'}`)
|
||||
|
||||
@ -1,5 +1,11 @@
|
||||
import { createSlice, PayloadAction } from '@reduxjs/toolkit'
|
||||
import type { WebSearchProvider } from '@renderer/types'
|
||||
export interface SubscribeSource {
|
||||
key: number
|
||||
url: string
|
||||
name: string
|
||||
blacklist?: string[] // 存储从该订阅源获取的黑名单
|
||||
}
|
||||
|
||||
export interface WebSearchState {
|
||||
// 默认搜索提供商的ID
|
||||
@ -12,6 +18,7 @@ export interface WebSearchState {
|
||||
maxResults: number
|
||||
// 要排除的域名列表
|
||||
excludeDomains: string[]
|
||||
subscribeSources: SubscribeSource[]
|
||||
// 是否启用搜索增强模式
|
||||
enhanceMode: boolean
|
||||
// 是否覆盖服务商搜索
|
||||
@ -55,6 +62,7 @@ const initialState: WebSearchState = {
|
||||
searchWithTime: true,
|
||||
maxResults: 5,
|
||||
excludeDomains: [],
|
||||
subscribeSources: [],
|
||||
enhanceMode: false,
|
||||
overwrite: false
|
||||
}
|
||||
@ -89,6 +97,33 @@ const websearchSlice = createSlice({
|
||||
setExcludeDomains: (state, action: PayloadAction<string[]>) => {
|
||||
state.excludeDomains = action.payload
|
||||
},
|
||||
// 添加订阅源
|
||||
addSubscribeSource: (state, action: PayloadAction<Omit<SubscribeSource, 'key'>>) => {
|
||||
state.subscribeSources = state.subscribeSources || []
|
||||
const newKey =
|
||||
state.subscribeSources.length > 0 ? Math.max(...state.subscribeSources.map((item) => item.key)) + 1 : 0
|
||||
state.subscribeSources.push({
|
||||
key: newKey,
|
||||
url: action.payload.url,
|
||||
name: action.payload.name,
|
||||
blacklist: action.payload.blacklist
|
||||
})
|
||||
},
|
||||
// 删除订阅源
|
||||
removeSubscribeSource: (state, action: PayloadAction<number>) => {
|
||||
state.subscribeSources = state.subscribeSources.filter((source) => source.key !== action.payload)
|
||||
},
|
||||
// 更新订阅源的黑名单
|
||||
updateSubscribeBlacklist: (state, action: PayloadAction<{ key: number; blacklist: string[] }>) => {
|
||||
const source = state.subscribeSources.find((s) => s.key === action.payload.key)
|
||||
if (source) {
|
||||
source.blacklist = action.payload.blacklist
|
||||
}
|
||||
},
|
||||
// 更新订阅源列表
|
||||
setSubscribeSources: (state, action: PayloadAction<SubscribeSource[]>) => {
|
||||
state.subscribeSources = action.payload
|
||||
},
|
||||
setEnhanceMode: (state, action: PayloadAction<boolean>) => {
|
||||
state.enhanceMode = action.payload
|
||||
},
|
||||
@ -115,6 +150,10 @@ export const {
|
||||
setSearchWithTime,
|
||||
setExcludeDomains,
|
||||
setMaxResult,
|
||||
addSubscribeSource,
|
||||
removeSubscribeSource,
|
||||
updateSubscribeBlacklist,
|
||||
setSubscribeSources,
|
||||
setEnhanceMode,
|
||||
setOverwrite,
|
||||
addWebSearchProvider
|
||||
|
||||
@ -1,3 +1,6 @@
|
||||
import { WebSearchState } from '@renderer/store/websearch'
|
||||
import { WebSearchResponse } from '@renderer/types'
|
||||
|
||||
/*
|
||||
* MIT License
|
||||
*
|
||||
@ -170,3 +173,108 @@ function testPath(pathPattern: string, path: string): boolean {
|
||||
}
|
||||
return path.slice(pos).endsWith(rest[rest.length - 1])
|
||||
}
|
||||
|
||||
// 添加新的解析函数
|
||||
export async function parseSubscribeContent(url: string): Promise<string[]> {
|
||||
try {
|
||||
// 获取订阅源内容
|
||||
const response = await fetch(url)
|
||||
console.log('response', response)
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to fetch subscribe content')
|
||||
}
|
||||
|
||||
const content = await response.text()
|
||||
|
||||
// 按行分割内容
|
||||
const lines = content.split('\n')
|
||||
|
||||
// 过滤出有效的匹配模式
|
||||
const patterns = lines
|
||||
.filter((line) => line.trim() !== '' && !line.startsWith('#'))
|
||||
.map((line) => line.trim())
|
||||
.filter((pattern) => parseMatchPattern(pattern) !== null)
|
||||
|
||||
return patterns
|
||||
} catch (error) {
|
||||
console.error('Error parsing subscribe content:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
export async function filterResultWithBlacklist(
|
||||
response: WebSearchResponse,
|
||||
websearch: WebSearchState
|
||||
): Promise<WebSearchResponse> {
|
||||
console.log('filterResultWithBlacklist', response)
|
||||
// 没有结果或者没有黑名单规则时，直接返回原始结果
|
||||
if (!response.results?.length || (!websearch.excludeDomains.length && !websearch.subscribeSources.length)) {
|
||||
return response
|
||||
}
|
||||
|
||||
// 创建匹配模式映射实例
|
||||
const patternMap = new MatchPatternMap<string>()
|
||||
|
||||
// 合并所有黑名单规则
|
||||
const blacklistPatterns: string[] = [
|
||||
...websearch.excludeDomains,
|
||||
...(websearch.subscribeSources?.length
|
||||
? websearch.subscribeSources.reduce<string[]>((acc, source) => {
|
||||
return acc.concat(source.blacklist || [])
|
||||
}, [])
|
||||
: [])
|
||||
]
|
||||
|
||||
// 正则表达式规则集合
|
||||
const regexPatterns: RegExp[] = []
|
||||
|
||||
// 分类处理黑名单规则
|
||||
blacklistPatterns.forEach((pattern) => {
|
||||
if (pattern.startsWith('/') && pattern.endsWith('/')) {
|
||||
// 处理正则表达式格式
|
||||
try {
|
||||
const regexPattern = pattern.slice(1, -1)
|
||||
regexPatterns.push(new RegExp(regexPattern, 'i'))
|
||||
} catch (error) {
|
||||
console.error('Invalid regex pattern:', pattern, error)
|
||||
}
|
||||
} else {
|
||||
// 处理匹配模式格式
|
||||
try {
|
||||
patternMap.set(pattern, pattern)
|
||||
} catch (error) {
|
||||
console.error('Invalid match pattern:', pattern, error)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// 过滤搜索结果
|
||||
const filteredResults = response.results.filter((result) => {
|
||||
try {
|
||||
const url = new URL(result.url)
|
||||
|
||||
// 检查URL是否匹配任何正则表达式规则
|
||||
const matchesRegex = regexPatterns.some((regex) => regex.test(url.hostname))
|
||||
if (matchesRegex) {
|
||||
return false
|
||||
}
|
||||
|
||||
// 检查URL是否匹配任何匹配模式规则
|
||||
const matchesPattern = patternMap.get(result.url).length > 0
|
||||
if (matchesPattern) {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
} catch (error) {
|
||||
console.error('Error processing URL:', result.url, error)
|
||||
return true // 如果URL解析失败，保留该结果
|
||||
}
|
||||
})
|
||||
|
||||
console.log('filterResultWithBlacklist filtered results:', filteredResults)
|
||||
|
||||
return {
|
||||
...response,
|
||||
results: filteredResults
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user