feat: knowledge base

This commit is contained in:
kangfenmao 2024-11-13 20:56:41 +08:00
parent 0739758469
commit c2462fd51c
39 changed files with 3864 additions and 90 deletions

View File

@ -0,0 +1,29 @@
diff --git a/lib/pdf-parse.js b/lib/pdf-parse.js
index 96bfbc705dcb4fb73cb077a75f02c115371b3477..6d02d2bb426063c3a31cb740c3d86841de162a22 100644
--- a/lib/pdf-parse.js
+++ b/lib/pdf-parse.js
@@ -21,12 +21,12 @@ function render_page(pageData) {
for (let item of textContent.items) {
if (lastY == item.transform[5] || !lastY){
text += item.str;
- }
+ }
else{
text += '\n' + item.str;
- }
+ }
lastY = item.transform[5];
- }
+ }
//let strings = textContent.items.map(item => item.str);
//let text = strings.join("\n");
//text = text.replace(/[ ]+/ig," ");
@@ -60,7 +60,7 @@ async function PDF(dataBuffer, options) {
if (typeof options.version != 'string') options.version = DEFAULT_OPTIONS.version;
if (options.version == 'default') options.version = DEFAULT_OPTIONS.version;
- PDFJS = PDFJS ? PDFJS : require(`./pdf.js/${options.version}/build/pdf.js`);
+ PDFJS = PDFJS ? PDFJS : require(`./pdf.js/v1.10.100/build/pdf.js`);
ret.version = PDFJS.version;

View File

@ -4,13 +4,33 @@ import { resolve } from 'path'
export default defineConfig({ export default defineConfig({
main: { main: {
plugins: [externalizeDepsPlugin()], plugins: [
externalizeDepsPlugin({
exclude: [
'@llm-tools/embedjs',
'@llm-tools/embedjs-lancedb',
'@llm-tools/embedjs-ollama',
'@llm-tools/embedjs-openai',
'@llm-tools/embedjs-loader-web',
'@llm-tools/embedjs-loader-markdown',
'@llm-tools/embedjs-loader-msoffice',
'@llm-tools/embedjs-loader-xml',
'@llm-tools/embedjs-loader-pdf',
'@lancedb/lancedb'
]
})
],
resolve: { resolve: {
alias: { alias: {
'@main': resolve('src/main'), '@main': resolve('src/main'),
'@types': resolve('src/renderer/src/types'), '@types': resolve('src/renderer/src/types'),
'@shared': resolve('packages/shared') '@shared': resolve('packages/shared')
} }
},
build: {
rollupOptions: {
external: ['@lancedb/lancedb', '@llm-tools/embedjs-loader-sitemap']
}
} }
}, },
preload: { preload: {

View File

@ -11,9 +11,11 @@
"local", "local",
"packages/*" "packages/*"
], ],
"nohoist": [ "installConfig": {
"hoistingLimits": [
"packages/database" "packages/database"
] ]
}
}, },
"scripts": { "scripts": {
"format": "prettier --write .", "format": "prettier --write .",
@ -38,7 +40,19 @@
"dependencies": { "dependencies": {
"@electron-toolkit/preload": "^3.0.0", "@electron-toolkit/preload": "^3.0.0",
"@electron-toolkit/utils": "^3.0.0", "@electron-toolkit/utils": "^3.0.0",
"@llm-tools/embedjs": "^0.1.24",
"@llm-tools/embedjs-lancedb": "^0.1.24",
"@llm-tools/embedjs-loader-csv": "^0.1.24",
"@llm-tools/embedjs-loader-markdown": "^0.1.24",
"@llm-tools/embedjs-loader-msoffice": "^0.1.24",
"@llm-tools/embedjs-loader-pdf": "^0.1.24",
"@llm-tools/embedjs-loader-web": "^0.1.24",
"@llm-tools/embedjs-loader-xml": "^0.1.24",
"@llm-tools/embedjs-ollama": "^0.1.24",
"@llm-tools/embedjs-openai": "^0.1.24",
"@types/react-infinite-scroll-component": "^5.0.0",
"adm-zip": "^0.5.16", "adm-zip": "^0.5.16",
"apache-arrow": "^18.0.0",
"docx": "^9.0.2", "docx": "^9.0.2",
"electron-log": "^5.1.5", "electron-log": "^5.1.5",
"electron-store": "^8.2.0", "electron-store": "^8.2.0",
@ -124,7 +138,8 @@
"react-dom": "^17.0.0 || ^18.0.0" "react-dom": "^17.0.0 || ^18.0.0"
}, },
"resolutions": { "resolutions": {
"@electron/notarize@npm:2.2.1": "patch:@electron/notarize@npm%3A2.3.2#~/.yarn/patches/@electron-notarize-npm-2.3.2-535908a4bd.patch" "@electron/notarize@npm:2.2.1": "patch:@electron/notarize@npm%3A2.3.2#~/.yarn/patches/@electron-notarize-npm-2.3.2-535908a4bd.patch",
"pdf-parse@npm:1.1.1": "patch:pdf-parse@npm%3A1.1.1#~/.yarn/patches/pdf-parse-npm-1.1.1-04a6109b2a.patch"
}, },
"packageManager": "yarn@4.5.0" "packageManager": "yarn@4.5.0"
} }

View File

@ -11,6 +11,7 @@ import BackupManager from './services/BackupManager'
import { configManager } from './services/ConfigManager' import { configManager } from './services/ConfigManager'
import { ExportService } from './services/ExportService' import { ExportService } from './services/ExportService'
import FileStorage from './services/FileStorage' import FileStorage from './services/FileStorage'
import KnowledgeService from './services/KnowledgeService'
import { registerShortcuts, unregisterAllShortcuts } from './services/ShortcutService' import { registerShortcuts, unregisterAllShortcuts } from './services/ShortcutService'
import { windowService } from './services/WindowService' import { windowService } from './services/WindowService'
import { compress, decompress } from './utils/zip' import { compress, decompress } from './utils/zip'
@ -144,4 +145,12 @@ export function registerIpc(mainWindow: BrowserWindow, app: Electron.App) {
registerShortcuts(mainWindow) registerShortcuts(mainWindow)
} }
}) })
// knowledge base
ipcMain.handle('knowledge-base:create', KnowledgeService.create)
ipcMain.handle('knowledge-base:reset', KnowledgeService.reset)
ipcMain.handle('knowledge-base:delete', KnowledgeService.delete)
ipcMain.handle('knowledge-base:add', KnowledgeService.add)
ipcMain.handle('knowledge-base:remove', KnowledgeService.remove)
ipcMain.handle('knowledge-base:search', KnowledgeService.search)
} }

View File

@ -0,0 +1,109 @@
import * as fs from 'node:fs'
import path from 'node:path'
import { RAGApplication, RAGApplicationBuilder, TextLoader } from '@llm-tools/embedjs'
import { AddLoaderReturn, ExtractChunkData } from '@llm-tools/embedjs-interfaces'
import { LanceDb } from '@llm-tools/embedjs-lancedb'
import { MarkdownLoader } from '@llm-tools/embedjs-loader-markdown'
import { DocxLoader } from '@llm-tools/embedjs-loader-msoffice'
import { PdfLoader } from '@llm-tools/embedjs-loader-pdf'
import { WebLoader } from '@llm-tools/embedjs-loader-web'
import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'
import { FileType, RagAppRequestParams } from '@types'
import { app } from 'electron'
import Logger from 'electron-log'
class KnowledgeService {
private storageDir = path.join(app.getPath('userData'), 'Data', 'KnowledgeBase')
constructor() {
this.initStorageDir()
}
private initStorageDir = (): void => {
if (!fs.existsSync(this.storageDir)) {
fs.mkdirSync(this.storageDir, { recursive: true })
}
}
private getRagApplication = async ({ id, model, apiKey, baseURL }: RagAppRequestParams): Promise<RAGApplication> => {
Logger.log('getRagApplication', { id, model, apiKey, baseURL })
return new RAGApplicationBuilder()
.setModel('NO_MODEL')
.setEmbeddingModel(
new OpenAiEmbeddings({
model,
apiKey,
configuration: { baseURL },
dimensions: 1024
})
)
.setVectorDatabase(new LanceDb({ path: path.join(this.storageDir, id) }))
.build()
}
public create = async (
_: Electron.IpcMainInvokeEvent,
{ id, model, apiKey, baseURL }: RagAppRequestParams
): Promise<void> => {
this.getRagApplication({ id, model, apiKey, baseURL })
}
public reset = async (_: Electron.IpcMainInvokeEvent, { config }: { config: RagAppRequestParams }): Promise<void> => {
const ragApplication = await this.getRagApplication(config)
await ragApplication.reset()
}
public delete = async (_: Electron.IpcMainInvokeEvent, id: string): Promise<void> => {
const dbPath = path.join(this.storageDir, id)
if (fs.existsSync(dbPath)) {
fs.rmSync(dbPath, { recursive: true })
}
}
public add = async (
_: Electron.IpcMainInvokeEvent,
{ data, config }: { data: string | FileType; config: RagAppRequestParams }
): Promise<AddLoaderReturn> => {
const ragApplication = await this.getRagApplication(config)
if (typeof data === 'string') {
if (data.startsWith('http')) {
return await ragApplication.addLoader(new WebLoader({ urlOrContent: data }))
}
return await ragApplication.addLoader(new TextLoader({ text: data }))
}
if (data.ext === '.pdf') {
return await ragApplication.addLoader(new PdfLoader({ filePathOrUrl: data.path }) as any)
}
if (data.ext === '.docx') {
return await ragApplication.addLoader(new DocxLoader({ filePathOrUrl: data.path }) as any)
}
if (data.ext === '.md') {
return await ragApplication.addLoader(new MarkdownLoader({ filePathOrUrl: data.path }) as any)
}
return { entriesAdded: 0, uniqueId: '', loaderType: '' }
}
public remove = async (
_: Electron.IpcMainInvokeEvent,
{ uniqueId, config }: { uniqueId: string; config: RagAppRequestParams }
): Promise<void> => {
const ragApplication = await this.getRagApplication(config)
await ragApplication.deleteLoader(uniqueId)
}
public search = async (
_: Electron.IpcMainInvokeEvent,
{ search, config }: { search: string; config: RagAppRequestParams }
): Promise<ExtractChunkData[]> => {
const ragApplication = await this.getRagApplication(config)
return await ragApplication.search(search)
}
}
export default new KnowledgeService()

View File

@ -1,7 +1,8 @@
import { ElectronAPI } from '@electron-toolkit/preload' import { ElectronAPI } from '@electron-toolkit/preload'
import { AddLoaderReturn, ExtractChunkData } from '@llm-tools/embedjs-interfaces'
import { FileType } from '@renderer/types' import { FileType } from '@renderer/types'
import { WebDavConfig } from '@renderer/types' import { WebDavConfig } from '@renderer/types'
import { AppInfo, LanguageVarious } from '@renderer/types' import { AppInfo, LanguageVarious, RagAppRequestParams } from '@renderer/types'
import type { OpenDialogOptions } from 'electron' import type { OpenDialogOptions } from 'electron'
import type { UpdateInfo } from 'electron-updater' import type { UpdateInfo } from 'electron-updater'
import { Readable } from 'stream' import { Readable } from 'stream'
@ -58,6 +59,14 @@ declare global {
shortcuts: { shortcuts: {
update: (shortcuts: Shortcut[]) => Promise<void> update: (shortcuts: Shortcut[]) => Promise<void>
} }
knowledgeBase: {
create: ({ id, model, apiKey, baseURL }: RagAppRequestParams) => Promise<void>
reset: ({ config }: { config: RagAppRequestParams }) => Promise<void>
delete: (id: string) => Promise<void>
add: ({ data, config }: { data: string | FileType; config: RagAppRequestParams }) => Promise<AddLoaderReturn>
remove: ({ uniqueId, config }: { uniqueId: string; config: RagAppRequestParams }) => Promise<void>
search: ({ search, config }: { search: string; config: RagAppRequestParams }) => Promise<ExtractChunkData[]>
}
} }
} }
} }

View File

@ -1,5 +1,5 @@
import { electronAPI } from '@electron-toolkit/preload' import { electronAPI } from '@electron-toolkit/preload'
import { Shortcut, WebDavConfig } from '@types' import { FileType, RagAppRequestParams, Shortcut, WebDavConfig } from '@types'
import { contextBridge, ipcRenderer, OpenDialogOptions } from 'electron' import { contextBridge, ipcRenderer, OpenDialogOptions } from 'electron'
// Custom APIs for renderer // Custom APIs for renderer
@ -50,6 +50,18 @@ const api = {
openPath: (path: string) => ipcRenderer.invoke('open:path', path), openPath: (path: string) => ipcRenderer.invoke('open:path', path),
shortcuts: { shortcuts: {
update: (shortcuts: Shortcut[]) => ipcRenderer.invoke('shortcuts:update', shortcuts) update: (shortcuts: Shortcut[]) => ipcRenderer.invoke('shortcuts:update', shortcuts)
},
knowledgeBase: {
create: ({ id, model, apiKey, baseURL }: RagAppRequestParams) =>
ipcRenderer.invoke('knowledge-base:create', { id, model, apiKey, baseURL }),
reset: ({ config }: { config: RagAppRequestParams }) => ipcRenderer.invoke('knowledge-base:reset', { config }),
delete: (id: string) => ipcRenderer.invoke('knowledge-base:delete', id),
add: ({ data, config }: { data: string | FileType; config: RagAppRequestParams }) =>
ipcRenderer.invoke('knowledge-base:add', { data, config }),
remove: ({ uniqueId, config }: { uniqueId: string; config: RagAppRequestParams }) =>
ipcRenderer.invoke('knowledge-base:remove', { uniqueId, config }),
search: ({ search, config }: { search: string; config: RagAppRequestParams }) =>
ipcRenderer.invoke('knowledge-base:search', { search, config })
} }
} }

View File

@ -14,6 +14,7 @@ import AgentsPage from './pages/agents/AgentsPage'
import AppsPage from './pages/apps/AppsPage' import AppsPage from './pages/apps/AppsPage'
import FilesPage from './pages/files/FilesPage' import FilesPage from './pages/files/FilesPage'
import HomePage from './pages/home/HomePage' import HomePage from './pages/home/HomePage'
import KnowledgePage from './pages/knowledge/KnowledgePage'
import PaintingsPage from './pages/paintings/PaintingsPage' import PaintingsPage from './pages/paintings/PaintingsPage'
import SettingsPage from './pages/settings/SettingsPage' import SettingsPage from './pages/settings/SettingsPage'
import TranslatePage from './pages/translate/TranslatePage' import TranslatePage from './pages/translate/TranslatePage'
@ -30,10 +31,11 @@ function App(): JSX.Element {
<Sidebar /> <Sidebar />
<Routes> <Routes>
<Route path="/" element={<HomePage />} /> <Route path="/" element={<HomePage />} />
<Route path="/files" element={<FilesPage />} />
<Route path="/agents" element={<AgentsPage />} /> <Route path="/agents" element={<AgentsPage />} />
<Route path="/paintings" element={<PaintingsPage />} /> <Route path="/paintings" element={<PaintingsPage />} />
<Route path="/translate" element={<TranslatePage />} /> <Route path="/translate" element={<TranslatePage />} />
<Route path="/files" element={<FilesPage />} />
<Route path="/knowledge" element={<KnowledgePage />} />
<Route path="/apps" element={<AppsPage />} /> <Route path="/apps" element={<AppsPage />} />
<Route path="/settings/*" element={<SettingsPage />} /> <Route path="/settings/*" element={<SettingsPage />} />
</Routes> </Routes>

View File

@ -0,0 +1,52 @@
import { ReactNode } from 'react'
import styled from 'styled-components'
interface ListItemProps {
active?: boolean
icon?: ReactNode
title: string
onClick?: () => void
}
const ListItem = ({ active, icon, title, onClick }: ListItemProps) => {
return (
<ListItemContainer className={active ? 'active' : ''} onClick={onClick}>
<ListItemContent>
{icon && <span style={{ marginRight: '8px' }}>{icon}</span>}
{title}
</ListItemContent>
</ListItemContainer>
)
}
const ListItemContainer = styled.div`
padding: 7px 12px;
border-radius: 16px;
font-size: 13px;
display: flex;
flex-direction: column;
justify-content: space-between;
position: relative;
font-family: Ubuntu;
cursor: pointer;
border: 1px solid transparent;
&:hover {
background-color: var(--color-background-soft);
}
&.active {
background-color: var(--color-background-soft);
border: 1px solid var(--color-border-soft);
}
`
const ListItemContent = styled.div`
display: -webkit-box;
-webkit-line-clamp: 1;
-webkit-box-orient: vertical;
overflow: hidden;
font-size: 13px;
`
export default ListItem

View File

@ -1,4 +1,4 @@
import { FolderOutlined, PictureOutlined, TranslationOutlined } from '@ant-design/icons' import { BookOutlined, FolderOutlined, PictureOutlined, TranslationOutlined } from '@ant-design/icons'
import { isMac } from '@renderer/config/constant' import { isMac } from '@renderer/config/constant'
import { isLocalAi, UserAvatar } from '@renderer/config/env' import { isLocalAi, UserAvatar } from '@renderer/config/env'
import { useTheme } from '@renderer/context/ThemeProvider' import { useTheme } from '@renderer/context/ThemeProvider'
@ -88,6 +88,13 @@ const Sidebar: FC = () => {
</StyledLink> </StyledLink>
</Tooltip> </Tooltip>
)} )}
<Tooltip title={t('knowledge_base.title')} mouseEnterDelay={0.5} placement="right">
<StyledLink onClick={() => to('/knowledge')}>
<Icon className={isRoute('/knowledge')}>
<BookOutlined />
</Icon>
</StyledLink>
</Tooltip>
{showFilesIcon && ( {showFilesIcon && (
<Tooltip title={t('files.title')} mouseEnterDelay={0.8} placement="right"> <Tooltip title={t('files.title')} mouseEnterDelay={0.8} placement="right">
<StyledLink onClick={() => to('/files')}> <StyledLink onClick={() => to('/files')}>

View File

@ -153,7 +153,7 @@ export const VISION_REGEX = new RegExp(
const TEXT_TO_IMAGE_REGEX = /flux|diffusion|stabilityai|sd-|dall|cogview/i const TEXT_TO_IMAGE_REGEX = /flux|diffusion|stabilityai|sd-|dall|cogview/i
const EMBEDDING_REGEX = /(?:^text-|embed|rerank|davinci|babbage|bge-|base|retrieval|uae-)/i const EMBEDDING_REGEX = /(?:^text-|embed|rerank|davinci|babbage|bge-|base|retrieval|uae-)/i
const NOT_SUPPORTED_REGEX = /(?:^text-|embed|tts|rerank|whisper|speech|davinci|babbage|bge-|base|retrieval|uae-)/i const NOT_SUPPORTED_REGEX = /(?:^tts|rerank|whisper|speech)/i
export function getModelLogo(modelId: string) { export function getModelLogo(modelId: string) {
const isLight = true const isLight = true

View File

@ -31,6 +31,13 @@ const AntdProvider: FC<PropsWithChildren> = ({ children }) => {
Menu: { Menu: {
activeBarBorderWidth: 0, activeBarBorderWidth: 0,
darkItemBg: 'transparent' darkItemBg: 'transparent'
},
Button: {
boxShadow: 'none',
boxShadowSecondary: 'none',
defaultShadow: 'none',
dangerShadow: 'none',
primaryShadow: 'none'
} }
}, },
token: { token: {

View File

@ -1,4 +1,4 @@
import { FileType, Topic } from '@renderer/types' import { FileType, KnowledgeItem, Topic } from '@renderer/types'
import { Dexie, type EntityTable } from 'dexie' import { Dexie, type EntityTable } from 'dexie'
// Database declaration (move this to its own module also) // Database declaration (move this to its own module also)
@ -6,6 +6,7 @@ export const db = new Dexie('CherryStudio') as Dexie & {
files: EntityTable<FileType, 'id'> files: EntityTable<FileType, 'id'>
topics: EntityTable<Pick<Topic, 'id' | 'messages'>, 'id'> topics: EntityTable<Pick<Topic, 'id' | 'messages'>, 'id'>
settings: EntityTable<{ id: string; value: any }, 'id'> settings: EntityTable<{ id: string; value: any }, 'id'>
knowledge_notes: EntityTable<KnowledgeItem, 'id'>
} }
db.version(1).stores({ db.version(1).stores({
@ -18,4 +19,11 @@ db.version(2).stores({
settings: '&id, value' settings: '&id, value'
}) })
db.version(3).stores({
files: 'id, name, origin_name, path, size, ext, type, created_at, count',
topics: '&id, messages',
settings: '&id, value',
knowledge_notes: '&id, baseId, type, content, created_at, updated_at'
})
export default db export default db

View File

@ -73,4 +73,8 @@ export function useAppInit() {
dispatch(setFilesPath(info.filesPath)) dispatch(setFilesPath(info.filesPath))
}) })
}, [dispatch]) }, [dispatch])
useEffect(() => {
import('@renderer/queue/KnowledgeQueue')
}, [])
} }

View File

@ -0,0 +1,247 @@
/* eslint-disable react-hooks/rules-of-hooks */
import { db } from '@renderer/databases/index'
import { RootState, useAppSelector } from '@renderer/store'
import {
addItem,
addProcessingItem,
clearAllItems,
clearCompletedItems,
removeItem as removeItemAction,
removeProcessingItem,
renameBase,
selectProcessingItemBySource,
selectProcessingItemsByType,
updateBase,
updateFiles as updateFilesAction,
updateNotes,
updateProcessingStatus
} from '@renderer/store/knowledge'
import { FileType, KnowledgeBase, ProcessingStatus } from '@renderer/types'
import { KnowledgeItem } from '@renderer/types'
import { runAsyncFunction } from '@renderer/utils'
import { useEffect, useState } from 'react'
import { useDispatch, useSelector } from 'react-redux'
import { v4 as uuidv4 } from 'uuid'
export const useKnowledge = (baseId: string) => {
const dispatch = useDispatch()
const base = useSelector((state: RootState) => state.knowledge.bases.find((b) => b.id === baseId))
const knowledgeState = useAppSelector((state: RootState) => state.knowledge)
// 重命名知识库
const renameKnowledgeBase = (name: string) => {
dispatch(renameBase({ baseId, name }))
}
// 更新知识库
const updateKnowledgeBase = (base: KnowledgeBase) => {
dispatch(updateBase(base))
}
// 添加文件列表
const addFiles = (files: FileType[]) => {
for (const file of files) {
const newItem = {
id: uuidv4(),
type: 'file' as const,
content: file,
created_at: Date.now(),
updated_at: Date.now()
}
dispatch(addItem({ baseId, item: newItem }))
}
}
// 添加URL
const addUrl = (url: string) => {
const newUrlItem = {
id: uuidv4(),
type: 'url' as const,
content: url,
created_at: Date.now(),
updated_at: Date.now()
}
dispatch(addItem({ baseId, item: newUrlItem }))
}
// 添加笔记
const addNote = async (content: string) => {
const noteId = uuidv4()
const note: KnowledgeItem = {
id: noteId,
type: 'note',
content,
created_at: Date.now(),
updated_at: Date.now()
}
// 存储完整笔记到数据库
await db.knowledge_notes.add(note)
// 在 store 中只存储引用
const noteRef: KnowledgeItem = {
id: noteId,
baseId,
type: 'note',
content: '', // store中不需要存储实际内容
created_at: Date.now(),
updated_at: Date.now()
}
dispatch(updateNotes({ baseId, item: noteRef }))
}
// 更新文件列表
const updateFiles = (files: FileType[]) => {
const newItems = files.map((file) => ({
id: uuidv4(),
type: 'file' as const,
content: file,
created_at: Date.now(),
updated_at: Date.now()
}))
dispatch(updateFilesAction({ baseId, items: newItems }))
}
// 更新笔记内容
const updateNoteContent = async (noteId: string, content: string) => {
const note = await db.knowledge_notes.get(noteId)
if (note) {
const updatedNote = {
...note,
content,
updated_at: Date.now()
}
await db.knowledge_notes.put(updatedNote)
dispatch(updateNotes({ baseId, item: updatedNote }))
}
}
// 获取笔记内容
const getNoteContent = async (noteId: string) => {
return await db.knowledge_notes.get(noteId)
}
// 移除项目
const removeItem = (item: KnowledgeItem) => {
dispatch(removeItemAction({ baseId, item }))
}
// 添加文件到处理队列
const addFileToQueue = (itemId: string) => {
dispatch(
addProcessingItem({
baseId,
type: 'file',
sourceId: itemId
})
)
}
// 添加URL到处理队列
const addUrlToQueue = (itemId: string) => {
dispatch(
addProcessingItem({
baseId,
type: 'url',
sourceId: itemId
})
)
}
// 添加笔记到处理队列
const addNoteToQueue = (itemId: string) => {
dispatch(
addProcessingItem({
baseId,
type: 'note',
sourceId: itemId
})
)
}
// 更新处理状态
const updateItemStatus = (itemId: string, status: ProcessingStatus, progress?: number, error?: string) => {
dispatch(
updateProcessingStatus({
baseId,
itemId,
status,
progress,
error
})
)
}
// 获取特定源的处理状态
const getProcessingStatus = (sourceId: string) => {
return selectProcessingItemBySource(knowledgeState, baseId, sourceId)
}
// 获取特定类型的所有处理项
const getProcessingItemsByType = (type: 'file' | 'url' | 'note') => {
return selectProcessingItemsByType(knowledgeState, baseId, type)
}
// 从队列中移除项目
const removeFromQueue = (itemId: string) => {
dispatch(
removeProcessingItem({
baseId,
itemId
})
)
}
// 清除已完成的项目
const clearCompleted = () => {
dispatch(clearCompletedItems({ baseId }))
}
// 清除所有队列项目
const clearAll = () => {
dispatch(clearAllItems({ baseId }))
}
const fileItems = base?.items.filter((item) => item.type === 'file') || []
const urlItems = base?.items.filter((item) => item.type === 'url') || []
const [noteItems, setNoteItems] = useState<KnowledgeItem[]>([])
useEffect(() => {
const notes = base?.items.filter((item) => item.type === 'note') || []
runAsyncFunction(async () => {
const newNoteItems = await Promise.all(
notes.map(async (item) => {
const note = await db.knowledge_notes.get(item.id)
return { ...item, content: note?.content || '' }
})
)
setNoteItems(newNoteItems.filter((note) => note !== undefined) as KnowledgeItem[])
})
}, [base?.items])
return {
base,
fileItems,
urlItems,
noteItems,
renameKnowledgeBase,
updateKnowledgeBase,
addFiles,
addUrl,
addNote,
updateFiles,
updateNoteContent,
getNoteContent,
addFileToQueue,
addUrlToQueue,
addNoteToQueue,
updateItemStatus,
getProcessingStatus,
getProcessingItemsByType,
removeFromQueue,
clearCompleted,
clearAll,
removeItem
}
}

View File

@ -81,6 +81,7 @@
"input.translate": "Translate to English", "input.translate": "Translate to English",
"input.upload": "Upload image or document file", "input.upload": "Upload image or document file",
"input.web_search": "Enable web search", "input.web_search": "Enable web search",
"input.knowledge_base": "Knowledge Base",
"message.new.branch": "New Branch", "message.new.branch": "New Branch",
"message.new.branch.created": "New Branch Created", "message.new.branch.created": "New Branch Created",
"message.regenerate.model": "Switch Model", "message.regenerate.model": "Switch Model",
@ -390,7 +391,7 @@
"messages.input.paste_long_text_as_file": "Paste long text as file", "messages.input.paste_long_text_as_file": "Paste long text as file",
"messages.input.send_shortcuts": "Send shortcuts", "messages.input.send_shortcuts": "Send shortcuts",
"messages.input.show_estimated_tokens": "Show estimated tokens", "messages.input.show_estimated_tokens": "Show estimated tokens",
"messages.metrics": "{{time_first_token_millsec}}ms to first token • {{token_speed}} tok/sec • ", "messages.metrics": "{{time_first_token_millsec}}ms to first token | {{token_speed}} tok/sec",
"messages.input.title": "Input Settings", "messages.input.title": "Input Settings",
"messages.markdown_rendering_input_message": "Markdown render input msg", "messages.markdown_rendering_input_message": "Markdown render input msg",
"messages.math_engine": "Math render engine", "messages.math_engine": "Math render engine",
@ -522,7 +523,43 @@
}, },
"words": { "words": {
"knowledgeGraph": "Knowledge Graph", "knowledgeGraph": "Knowledge Graph",
"visualization": "Visualization" "visualization": "Visualization",
"show_window": "Show Window",
"quit": "Quit"
},
"knowledge_base": {
"title": "Knowledge Base",
"search": "Search knowledge base",
"empty": "No knowledge base found",
"drag_file": "Drag file here",
"file_hint": "Support pdf, docx, txt and md",
"add": {
"title": "Add Knowledge Base"
},
"notes": "Notes",
"notes_placeholder": "Enter additional information or context for this knowledge base...",
"delete": "Delete",
"rename": "Rename",
"urls": "URLs",
"add_url": "Add URL",
"url_placeholder": "Enter URL",
"invalid_url": "Invalid URL",
"add_file": "Add File",
"status": "Status",
"index_all": "Index All",
"index_started": "Indexing started",
"cancel_index": "Cancel Indexing",
"index_cancelled": "Indexing cancelled",
"status_pending": "Pending",
"status_processing": "Processing",
"status_completed": "Completed",
"status_failed": "Failed",
"url_added": "URL added",
"query": "Query",
"search_placeholder": "Enter text to search",
"add_note": "Add Note",
"no_bases": "No knowledge bases available",
"clear_selection": "Clear selection"
} }
} }
} }

View File

@ -81,6 +81,7 @@
"input.translate": "Перевести на английский", "input.translate": "Перевести на английский",
"input.upload": "Загрузить изображение или документ", "input.upload": "Загрузить изображение или документ",
"input.web_search": "Включить веб-поиск", "input.web_search": "Включить веб-поиск",
"input.knowledge_base": "База знаний",
"message.new.branch": "Новая ветка", "message.new.branch": "Новая ветка",
"message.new.branch.created": "Новая ветка создана", "message.new.branch.created": "Новая ветка создана",
"message.regenerate.model": "Переключить модель", "message.regenerate.model": "Переключить модель",
@ -390,6 +391,7 @@
"messages.input.paste_long_text_as_file": "Вставлять длинный текст как файл", "messages.input.paste_long_text_as_file": "Вставлять длинный текст как файл",
"messages.input.send_shortcuts": "Горячие клавиши для отправки", "messages.input.send_shortcuts": "Горячие клавиши для отправки",
"messages.input.show_estimated_tokens": "Показывать затраты токенов", "messages.input.show_estimated_tokens": "Показывать затраты токенов",
"messages.metrics": "{{time_first_token_millsec}}ms до первого токена | {{token_speed}} tok/sec",
"messages.input.title": "Настройки ввода", "messages.input.title": "Настройки ввода",
"messages.markdown_rendering_input_message": "Отображение ввода в формате Markdown", "messages.markdown_rendering_input_message": "Отображение ввода в формате Markdown",
"messages.math_engine": "Математический движок", "messages.math_engine": "Математический движок",
@ -521,7 +523,43 @@
}, },
"words": { "words": {
"knowledgeGraph": "Граф знаний", "knowledgeGraph": "Граф знаний",
"visualization": "Визуализация" "visualization": "Визуализация",
"show_window": "Показать окно",
"quit": "Выйти"
},
"knowledge_base": {
"title": "База знаний",
"search": "Поиск в базе знаний",
"empty": "База знаний не найдена",
"drag_file": "Перетащите файл сюда",
"file_hint": "Поддерживаются pdf, docx, txt и md",
"add": {
"title": "Добавить базу знаний"
},
"notes": "Заметки",
"notes_placeholder": "Введите дополнительную информацию или контекст для этой базы знаний...",
"delete": "Удалить",
"rename": "Переименовать",
"urls": "URL-адреса",
"add_url": "Добавить URL",
"url_placeholder": "Введите URL",
"invalid_url": "Неверный URL",
"add_file": "Добавить файл",
"status": "Статус",
"index_all": "Индексировать все",
"index_started": "Индексирование началось",
"cancel_index": "Отменить индексирование",
"index_cancelled": "Индексирование отменено",
"status_pending": "Ожидание",
"status_processing": "Обработка",
"status_completed": "Завершено",
"status_failed": "Ошибка",
"url_added": "URL добавлен",
"query": "Поиск",
"search_placeholder": "Введите текст для поиска",
"add_note": "Добавить запись",
"no_bases": "База знаний не найдена",
"clear_selection": "Очистить выбор"
} }
} }
} }

View File

@ -81,6 +81,7 @@
"input.translate": "翻译成英文", "input.translate": "翻译成英文",
"input.upload": "上传图片或文档", "input.upload": "上传图片或文档",
"input.web_search": "开启网络搜索", "input.web_search": "开启网络搜索",
"input.knowledge_base": "知识库",
"message.new.branch": "新分支", "message.new.branch": "新分支",
"message.new.branch.created": "新分支已创建", "message.new.branch.created": "新分支已创建",
"message.regenerate.model": "切换模型", "message.regenerate.model": "切换模型",
@ -148,7 +149,8 @@
"warning": "警告", "warning": "警告",
"you": "用户", "you": "用户",
"clear": "清除", "clear": "清除",
"add": "添加" "add": "添加",
"footnotes": "引用内容"
}, },
"error": { "error": {
"backup.file_format": "备份文件格式错误", "backup.file_format": "备份文件格式错误",
@ -390,7 +392,7 @@
"messages.input.paste_long_text_as_file": "长文本粘贴为文件", "messages.input.paste_long_text_as_file": "长文本粘贴为文件",
"messages.input.send_shortcuts": "发送快捷键", "messages.input.send_shortcuts": "发送快捷键",
"messages.input.show_estimated_tokens": "显示预估 Token 数", "messages.input.show_estimated_tokens": "显示预估 Token 数",
"messages.metrics": "首字时延 {{time_first_token_millsec}}ms • 每秒 {{token_speed}} token • ", "messages.metrics": "首字时延 {{time_first_token_millsec}}ms | 每秒 {{token_speed}} tokens",
"messages.input.title": "输入设置", "messages.input.title": "输入设置",
"messages.markdown_rendering_input_message": "Markdown 渲染输入消息", "messages.markdown_rendering_input_message": "Markdown 渲染输入消息",
"messages.math_engine": "数学公式引擎", "messages.math_engine": "数学公式引擎",
@ -510,7 +512,43 @@
}, },
"words": { "words": {
"knowledgeGraph": "知识图谱", "knowledgeGraph": "知识图谱",
"visualization": "可视化" "visualization": "可视化",
"show_window": "显示窗口",
"quit": "退出"
},
"knowledge_base": {
"title": "知识库",
"search": "搜索知识库",
"empty": "暂无知识库",
"drag_file": "拖拽文件到这里",
"file_hint": "支持 pdf, docx, txt, md 格式",
"add": {
"title": "添加知识库"
},
"notes": "笔记",
"notes_placeholder": "输入此知识库的附加信息或上下文...",
"delete": "删除",
"rename": "重命名",
"urls": "网站",
"add_url": "添加网址",
"url_placeholder": "请输入网址",
"invalid_url": "无效的网址",
"add_file": "添加文件",
"status": "状态",
"index_all": "索引全部",
"index_started": "索引开始",
"cancel_index": "取消索引",
"index_cancelled": "索引已取消",
"status_pending": "等待中",
"status_processing": "处理中",
"status_completed": "已完成",
"status_failed": "失败",
"url_added": "网址已添加",
"query": "查询",
"search_placeholder": "输入查询内容",
"add_note": "添加笔记",
"no_bases": "暂无知识库",
"clear_selection": "清除选择"
} }
} }
} }

View File

@ -81,6 +81,7 @@
"input.translate": "翻譯成英文", "input.translate": "翻譯成英文",
"input.upload": "上傳圖片或文檔", "input.upload": "上傳圖片或文檔",
"input.web_search": "開啟網路搜索", "input.web_search": "開啟網路搜索",
"input.knowledge_base": "知識庫",
"message.new.branch": "新分支", "message.new.branch": "新分支",
"message.new.branch.created": "新分支已建立", "message.new.branch.created": "新分支已建立",
"message.regenerate.model": "切換模型", "message.regenerate.model": "切換模型",
@ -130,7 +131,6 @@
"download": "下載", "download": "下載",
"duplicate": "複製", "duplicate": "複製",
"edit": "編輯", "edit": "編輯",
"footnotes": "引用",
"language": "語言", "language": "語言",
"model": "模型", "model": "模型",
"models": "模型", "models": "模型",
@ -148,7 +148,8 @@
"warning": "警告", "warning": "警告",
"you": "您", "you": "您",
"clear": "清除", "clear": "清除",
"add": "添加" "add": "添加",
"footnotes": "引用"
}, },
"error": { "error": {
"backup.file_format": "備份文件格式錯誤", "backup.file_format": "備份文件格式錯誤",
@ -390,6 +391,7 @@
"messages.input.paste_long_text_as_file": "將長文本貼上為檔案", "messages.input.paste_long_text_as_file": "將長文本貼上為檔案",
"messages.input.send_shortcuts": "發送快捷鍵", "messages.input.send_shortcuts": "發送快捷鍵",
"messages.input.show_estimated_tokens": "顯示預估 Token 數", "messages.input.show_estimated_tokens": "顯示預估 Token 數",
"messages.metrics": "首字時延 {{time_first_token_millsec}}ms | 每秒 {{token_speed}} tokens",
"messages.input.title": "輸入設定", "messages.input.title": "輸入設定",
"messages.math_engine": "Markdown 渲染輸入訊息", "messages.math_engine": "Markdown 渲染輸入訊息",
"messages.math_render_engine": "數學公式引擎", "messages.math_render_engine": "數學公式引擎",
@ -509,7 +511,43 @@
}, },
"words": { "words": {
"knowledgeGraph": "知識圖譜", "knowledgeGraph": "知識圖譜",
"visualization": "可視化" "visualization": "可視化",
"show_window": "顯示視窗",
"quit": "退出"
},
"knowledge_base": {
"title": "知識庫",
"search": "搜尋知識庫",
"empty": "暫無知識庫",
"drag_file": "拖拽文件到這裡",
"file_hint": "支持 pdf, docx, txt, md 格式",
"add": {
"title": "添加知識庫"
},
"notes": "筆記",
"notes_placeholder": "輸入此知識庫的附加資訊或上下文...",
"delete": "刪除",
"rename": "重命名",
"urls": "網址",
"add_url": "添加網址",
"url_placeholder": "請輸入網址",
"invalid_url": "無效的網址",
"add_file": "添加文件",
"status": "狀態",
"index_all": "索引全部",
"index_started": "索引開始",
"cancel_index": "取消索引",
"index_cancelled": "索引已取消",
"status_pending": "等待中",
"status_processing": "處理中",
"status_completed": "已完成",
"status_failed": "失敗",
"url_added": "網址已添加",
"query": "查詢",
"search_placeholder": "輸入查詢內容",
"add_note": "添加筆記",
"no_bases": "暫無知識庫",
"clear_selection": "清除選擇"
} }
} }
} }

View File

@ -25,7 +25,7 @@ import { estimateTextTokens as estimateTxtTokens } from '@renderer/services/Toke
import { translateText } from '@renderer/services/TranslateService' import { translateText } from '@renderer/services/TranslateService'
import store, { useAppDispatch, useAppSelector } from '@renderer/store' import store, { useAppDispatch, useAppSelector } from '@renderer/store'
import { setGenerating, setSearching } from '@renderer/store/runtime' import { setGenerating, setSearching } from '@renderer/store/runtime'
import { Assistant, FileType, Message, Topic } from '@renderer/types' import { Assistant, FileType, KnowledgeBase, Message, Topic } from '@renderer/types'
import { delay, getFileExtension, uuid } from '@renderer/utils' import { delay, getFileExtension, uuid } from '@renderer/utils'
import { documentExts, imageExts, textExts } from '@shared/config/constant' import { documentExts, imageExts, textExts } from '@shared/config/constant'
import { Button, Popconfirm, Tooltip } from 'antd' import { Button, Popconfirm, Tooltip } from 'antd'
@ -38,6 +38,7 @@ import styled from 'styled-components'
import AttachmentButton from './AttachmentButton' import AttachmentButton from './AttachmentButton'
import AttachmentPreview from './AttachmentPreview' import AttachmentPreview from './AttachmentPreview'
import KnowledgeBaseButton from './KnowledgeBaseButton'
import SendMessageButton from './SendMessageButton' import SendMessageButton from './SendMessageButton'
import TokenCount from './TokenCount' import TokenCount from './TokenCount'
@ -48,6 +49,7 @@ interface Props {
let _text = '' let _text = ''
let _files: FileType[] = [] let _files: FileType[] = []
let _base: KnowledgeBase
const Inputbar: FC<Props> = ({ assistant: _assistant, setActiveTopic }) => { const Inputbar: FC<Props> = ({ assistant: _assistant, setActiveTopic }) => {
const [text, setText] = useState(_text) const [text, setText] = useState(_text)
@ -78,6 +80,7 @@ const Inputbar: FC<Props> = ({ assistant: _assistant, setActiveTopic }) => {
const [spaceClickCount, setSpaceClickCount] = useState(0) const [spaceClickCount, setSpaceClickCount] = useState(0)
const spaceClickTimer = useRef<NodeJS.Timeout>() const spaceClickTimer = useRef<NodeJS.Timeout>()
const [isTranslating, setIsTranslating] = useState(false) const [isTranslating, setIsTranslating] = useState(false)
const [selectedKnowledgeBase, setSelectedKnowledgeBase] = useState<KnowledgeBase>(_base)
const isVision = useMemo(() => isVisionModel(model), [model]) const isVision = useMemo(() => isVisionModel(model), [model])
const supportExts = useMemo(() => [...textExts, ...documentExts, ...(isVision ? imageExts : [])], [isVision]) const supportExts = useMemo(() => [...textExts, ...documentExts, ...(isVision ? imageExts : [])], [isVision])
@ -90,6 +93,7 @@ const Inputbar: FC<Props> = ({ assistant: _assistant, setActiveTopic }) => {
_text = text _text = text
_files = files _files = files
_base = selectedKnowledgeBase
const sendMessage = useCallback(async () => { const sendMessage = useCallback(async () => {
if (generating) { if (generating) {
@ -111,6 +115,10 @@ const Inputbar: FC<Props> = ({ assistant: _assistant, setActiveTopic }) => {
status: 'success' status: 'success'
} }
if (selectedKnowledgeBase) {
message.knowledgeBaseIds = [selectedKnowledgeBase.id]
}
if (files.length > 0) { if (files.length > 0) {
message.files = await FileManager.uploadFiles(files) message.files = await FileManager.uploadFiles(files)
} }
@ -123,7 +131,7 @@ const Inputbar: FC<Props> = ({ assistant: _assistant, setActiveTopic }) => {
setTimeout(() => resizeTextArea(), 0) setTimeout(() => resizeTextArea(), 0)
setExpend(false) setExpend(false)
}, [assistant.id, assistant.topics, generating, files, text]) }, [assistant.id, assistant.topics, generating, files, text, selectedKnowledgeBase])
const translate = async () => { const translate = async () => {
if (isTranslating) { if (isTranslating) {
@ -374,6 +382,10 @@ const Inputbar: FC<Props> = ({ assistant: _assistant, setActiveTopic }) => {
const textareaRows = window.innerHeight >= 1000 || isBubbleStyle ? 2 : 1 const textareaRows = window.innerHeight >= 1000 || isBubbleStyle ? 2 : 1
const handleKnowledgeBaseSelect = (base?: KnowledgeBase) => {
setSelectedKnowledgeBase(base)
}
return ( return (
<Container onDragOver={handleDragOver} onDrop={handleDrop}> <Container onDragOver={handleDragOver} onDrop={handleDrop}>
<AttachmentPreview files={files} setFiles={setFiles} /> <AttachmentPreview files={files} setFiles={setFiles} />
@ -438,6 +450,7 @@ const Inputbar: FC<Props> = ({ assistant: _assistant, setActiveTopic }) => {
<ControlOutlined /> <ControlOutlined />
</ToolbarButton> </ToolbarButton>
</Tooltip> </Tooltip>
<KnowledgeBaseButton selectedBase={selectedKnowledgeBase} onSelect={handleKnowledgeBaseSelect} />
<AttachmentButton model={model} files={files} setFiles={setFiles} ToolbarButton={ToolbarButton} /> <AttachmentButton model={model} files={files} setFiles={setFiles} ToolbarButton={ToolbarButton} />
<ToolbarButton type="text" onClick={onNewContext}> <ToolbarButton type="text" onClick={onNewContext}>
<Tooltip placement="top" title={t('chat.input.new.context')}> <Tooltip placement="top" title={t('chat.input.new.context')}>

View File

@ -0,0 +1,117 @@
import { BookOutlined } from '@ant-design/icons'
import { useAppSelector } from '@renderer/store'
import { KnowledgeBase } from '@renderer/types'
import { Button, Popover, Tooltip } from 'antd'
import { FC } from 'react'
import { useTranslation } from 'react-i18next'
import styled from 'styled-components'
interface Props {
selectedBase?: KnowledgeBase
onSelect: (base?: KnowledgeBase) => void
}
const KnowledgeBaseSelector: FC<Props> = ({ selectedBase, onSelect }) => {
const { t } = useTranslation()
const knowledgeState = useAppSelector((state) => state.knowledge)
return (
<SelectorContainer>
{knowledgeState.bases.length === 0 ? (
<EmptyMessage>{t('knowledge.no_bases')}</EmptyMessage>
) : (
<>
{selectedBase && (
<Button type="link" block onClick={() => onSelect(undefined)} style={{ textAlign: 'left' }}>
{t('knowledge.clear_selection')}
</Button>
)}
{knowledgeState.bases.map((base) => (
<Button
key={base.id}
type={selectedBase?.id === base.id ? 'primary' : 'text'}
block
onClick={() => onSelect(base)}
style={{ textAlign: 'left' }}>
{base.name}
</Button>
))}
</>
)}
</SelectorContainer>
)
}
const KnowledgeBaseButton: FC<Props> = ({ selectedBase, onSelect }) => {
const { t } = useTranslation()
if (selectedBase) {
return (
<Tooltip placement="top" title={t('chat.input.knowledge_base')} arrow>
<ToolbarButton type="text" onClick={() => onSelect(undefined)}>
<BookOutlined style={{ color: selectedBase ? 'var(--color-link)' : 'var(--color-icon)' }} />
</ToolbarButton>
</Tooltip>
)
}
return (
<Tooltip placement="top" title={t('chat.input.knowledge_base')} arrow>
<Popover
placement="top"
content={<KnowledgeBaseSelector selectedBase={selectedBase} onSelect={onSelect} />}
trigger="click">
<ToolbarButton type="text" onClick={() => selectedBase && onSelect(undefined)}>
<BookOutlined style={{ color: selectedBase ? 'var(--color-link)' : 'var(--color-icon)' }} />
</ToolbarButton>
</Popover>
</Tooltip>
)
}
const SelectorContainer = styled.div`
max-height: 300px;
overflow-y: auto;
`
const EmptyMessage = styled.div`
padding: 8px;
`
const ToolbarButton = styled(Button)`
width: 30px;
height: 30px;
font-size: 17px;
border-radius: 50%;
transition: all 0.3s ease;
color: var(--color-icon);
display: flex;
flex-direction: row;
justify-content: center;
align-items: center;
padding: 0;
&.anticon,
&.iconfont {
transition: all 0.3s ease;
color: var(--color-icon);
}
&:hover {
background-color: var(--color-background-soft);
.anticon,
.iconfont {
color: var(--color-text-1);
}
}
&.active {
background-color: var(--color-primary) !important;
.anticon,
.iconfont {
color: var(--color-white-soft);
}
&:hover {
background-color: var(--color-primary);
}
}
`
export default KnowledgeBaseButton

View File

@ -29,6 +29,7 @@ const MessgeTokens: React.FC<{ message: Message; isLastMessage: boolean }> = ({
if (message.role === 'assistant') { if (message.role === 'assistant') {
let metrixs = '' let metrixs = ''
if (message?.metrics?.completion_tokens && message?.metrics?.time_completion_millsec) { if (message?.metrics?.completion_tokens && message?.metrics?.time_completion_millsec) {
metrixs = t('settings.messages.metrics', { metrixs = t('settings.messages.metrics', {
time_first_token_millsec: message?.metrics?.time_first_token_millsec, time_first_token_millsec: message?.metrics?.time_first_token_millsec,
@ -37,10 +38,13 @@ const MessgeTokens: React.FC<{ message: Message; isLastMessage: boolean }> = ({
) )
}) })
} }
return ( return (
<MessageMetadata className="message-tokens" onClick={locateMessage}> <MessageMetadata className="message-tokens" onClick={locateMessage}>
{metrixs !== '' ? metrixs : ''} <span className="metrics">{metrixs}</span>
<span className="tokens">
Tokens: {message?.usage?.total_tokens} {message?.usage?.prompt_tokens} {message?.usage?.completion_tokens} Tokens: {message?.usage?.total_tokens} {message?.usage?.prompt_tokens} {message?.usage?.completion_tokens}
</span>
</MessageMetadata> </MessageMetadata>
) )
} }
@ -54,6 +58,25 @@ const MessageMetadata = styled.div`
user-select: text; user-select: text;
margin: 2px 0; margin: 2px 0;
cursor: pointer; cursor: pointer;
text-align: right;
.metrics {
display: none;
}
.tokens {
display: block;
}
&:hover {
.metrics {
display: block;
}
.tokens {
display: none;
}
}
` `
export default MessgeTokens export default MessgeTokens

View File

@ -0,0 +1,128 @@
import { TopView } from '@renderer/components/TopView'
import { isEmbeddingModel } from '@renderer/config/models'
import { useProviders } from '@renderer/hooks/useProvider'
import { getRagAppRequestParams } from '@renderer/services/KnowledgeService'
import { getModelUniqId } from '@renderer/services/ModelService'
import { addBase } from '@renderer/store/knowledge'
import { Model } from '@renderer/types'
import { Form, Input, Modal, Select } from 'antd'
import { find, sortBy } from 'lodash'
import { nanoid } from 'nanoid'
import { useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useDispatch } from 'react-redux'
interface ShowParams {
title: string
}
interface FormData {
name: string
model: string
}
interface Props extends ShowParams {
resolve: (data: any) => void
}
const PopupContainer: React.FC<Props> = ({ title, resolve }) => {
const [open, setOpen] = useState(true)
const [form] = Form.useForm<FormData>()
const { t } = useTranslation()
const dispatch = useDispatch()
const { providers } = useProviders()
const allModels = providers
.map((p) => p.models)
.flat()
.filter((model) => isEmbeddingModel(model))
const selectOptions = providers
.filter((p) => p.models.length > 0)
.map((p) => ({
label: p.isSystem ? t(`provider.${p.id}`) : p.name,
title: p.name,
options: sortBy(p.models, 'name')
.filter((model) => isEmbeddingModel(model))
.map((m) => ({
label: m.name,
value: getModelUniqId(m)
}))
}))
.filter((group) => group.options.length > 0)
const onOk = async () => {
try {
const values = await form.validateFields()
const selectedModel = find(allModels, JSON.parse(values.model)) as Model
if (selectedModel) {
const newBase = {
id: nanoid(),
name: values.name,
model: selectedModel,
items: [],
processingQueue: [],
created_at: Date.now(),
updated_at: Date.now()
}
await window.api.knowledgeBase.create(getRagAppRequestParams(newBase))
dispatch(addBase(newBase as any))
setOpen(false)
resolve(newBase)
}
} catch (error) {
console.error('Validation failed:', error)
}
}
const onCancel = () => {
setOpen(false)
}
const onClose = () => {
resolve(null)
}
return (
<Modal title={title} open={open} onOk={onOk} onCancel={onCancel} afterClose={onClose} destroyOnClose centered>
<Form form={form} layout="vertical">
<Form.Item
name="name"
label={t('common.name')}
rules={[{ required: true, message: t('message.error.enter.name') }]}>
<Input placeholder={t('common.name')} />
</Form.Item>
<Form.Item
name="model"
label={t('common.model')}
rules={[{ required: true, message: t('message.error.enter.model') }]}>
<Select style={{ width: '100%' }} options={selectOptions} placeholder={t('settings.models.empty')} />
</Form.Item>
</Form>
</Modal>
)
}
export default class AddKnowledgePopup {
static hide() {
TopView.hide('AddKnowledgePopup')
}
static show(props: ShowParams) {
return new Promise<any>((resolve) => {
TopView.show(
<PopupContainer
{...props}
resolve={(v) => {
resolve(v)
this.hide()
}}
/>,
'AddKnowledgePopup'
)
})
}
}

View File

@ -0,0 +1,327 @@
import {
DeleteOutlined,
EditOutlined,
FileTextOutlined,
LinkOutlined,
PlusOutlined,
SearchOutlined,
StopOutlined
} from '@ant-design/icons'
import PromptPopup from '@renderer/components/Popups/PromptPopup'
import TextEditPopup from '@renderer/components/Popups/TextEditPopup'
import Scrollbar from '@renderer/components/Scrollbar'
import { useKnowledge } from '@renderer/hooks/useknowledge'
import FileManager from '@renderer/services/FileManager'
import { FileType, FileTypes, KnowledgeBase } from '@renderer/types'
import { Button, Card, message, Typography, Upload } from 'antd'
import { FC } from 'react'
import { useTranslation } from 'react-i18next'
import styled from 'styled-components'
import KnowledgeSearchPopup from './KnowledgeSearchPopup'
import StatusIcon from './StatusIcon'
const { Dragger } = Upload
const { Title } = Typography
interface KnowledgeContentProps {
selectedBase: KnowledgeBase
}
const KnowledgeContent: FC<KnowledgeContentProps> = ({ selectedBase }) => {
const { t } = useTranslation()
const {
base,
noteItems,
fileItems,
urlItems,
addFiles,
updateNoteContent,
addUrl,
removeItem,
getProcessingStatus,
addFileToQueue,
addNoteToQueue,
addUrlToQueue,
clearAll,
addNote
} = useKnowledge(selectedBase.id || '')
if (!base) {
return null
}
const handleAddFile = () => {
const input = document.createElement('input')
input.type = 'file'
input.multiple = true
input.accept = '.pdf,.txt,.docx,.md'
input.onchange = (e) => {
const files = (e.target as HTMLInputElement).files
if (files) {
handleDrop(Array.from(files))
}
}
input.click()
}
const handleDrop = async (files: File[]) => {
if (files) {
const _files: FileType[] = files.map((file) => ({
id: file.name,
name: file.name,
path: file.path,
size: file.size,
ext: `.${file.name.split('.').pop()}`,
count: 1,
origin_name: file.name,
type: file.type as FileTypes,
created_at: new Date()
}))
console.debug('[KnowledgeContent] Uploading files:', _files, files)
const uploadedFiles = await FileManager.uploadFiles(_files)
addFiles(uploadedFiles)
}
}
const handleAddUrl = async () => {
const url = await PromptPopup.show({
title: t('knowledge_base.add_url'),
message: '',
inputPlaceholder: t('knowledge_base.url_placeholder'),
inputProps: {
maxLength: 1000,
rows: 1
}
})
if (url) {
try {
new URL(url)
if (urlItems.find((item) => item.content === url)) {
message.success(t('knowledge_base.url_added'))
return
}
addUrl(url)
} catch (e) {
console.error('Invalid URL:', url)
}
}
}
const handleAddNote = async () => {
const note = await TextEditPopup.show({ text: '', textareaProps: { rows: 20 } })
note && addNote(note)
}
const handleIndexAll = async () => {
fileItems.forEach((item) => !item.uniqueId && addFileToQueue(item.id))
urlItems.forEach((item) => !item.uniqueId && addUrlToQueue(item.id))
noteItems.forEach((note) => !note.uniqueId && addNoteToQueue(note.id))
message.success(t('knowledge_base.index_started'))
}
const handleCancelIndex = () => {
clearAll()
message.success(t('knowledge_base.index_cancelled'))
}
const handleEditNote = async (note: any) => {
const editedText = await TextEditPopup.show({ text: note.content as string, textareaProps: { rows: 20 } })
editedText && updateNoteContent(note.id, editedText)
}
return (
<MainContent>
<FileSection>
<TitleWrapper>
<Title level={5}>{t('files.title')}</Title>
<Button icon={<PlusOutlined />} onClick={handleAddFile}>
{t('knowledge_base.add_file')}
</Button>
</TitleWrapper>
<Dragger
showUploadList={false}
customRequest={({ file }) => handleDrop([file as File])}
multiple={true}
accept=".pdf,.txt,.docx,.md"
style={{ marginTop: 10, background: 'transparent' }}>
<p className="ant-upload-text">{t('knowledge_base.drag_file')}</p>
<p className="ant-upload-hint">{t('knowledge_base.file_hint')}</p>
</Dragger>
</FileSection>
<FileListSection>
{fileItems.map((item) => {
const file = item.content as FileType
return (
<ItemCard key={item.id}>
<ItemContent>
<ItemInfo>
<FileTextOutlined style={{ fontSize: '16px' }} />
<span>{file.origin_name}</span>
</ItemInfo>
<div style={{ display: 'flex', alignItems: 'center', gap: '16px' }}>
<StatusIcon sourceId={item.id} base={base} getProcessingStatus={getProcessingStatus} />
<Button type="text" danger onClick={() => removeItem(item)} icon={<DeleteOutlined />} />
</div>
</ItemContent>
</ItemCard>
)
})}
</FileListSection>
<ContentSection>
<TitleWrapper>
<Title level={5}>{t('knowledge_base.urls')}</Title>
<Button icon={<PlusOutlined />} onClick={handleAddUrl}>
{t('knowledge_base.add_url')}
</Button>
</TitleWrapper>
<div style={{ display: 'flex', flexDirection: 'column', gap: '8px' }}>
{urlItems.map((item) => (
<ItemCard key={item.id}>
<ItemContent>
<ItemInfo>
<LinkOutlined style={{ fontSize: '16px' }} />
<a href={item.content as string} target="_blank" rel="noopener noreferrer">
{item.content as string}
</a>
</ItemInfo>
<div style={{ display: 'flex', alignItems: 'center', gap: '16px' }}>
<StatusIcon sourceId={item.id} base={base} getProcessingStatus={getProcessingStatus} />
<Button type="text" danger onClick={() => removeItem(item)} icon={<DeleteOutlined />} />
</div>
</ItemContent>
</ItemCard>
))}
</div>
</ContentSection>
<ContentSection>
<TitleWrapper>
<Title level={5}>{t('knowledge_base.notes')}</Title>
<Button icon={<PlusOutlined />} onClick={handleAddNote}>
{t('knowledge_base.add_note')}
</Button>
</TitleWrapper>
<div style={{ display: 'flex', flexDirection: 'column', gap: '12px' }}>
{noteItems.map((note) => (
<ItemCard key={note.id}>
<ItemContent>
<ItemInfo onClick={() => handleEditNote(note)} style={{ cursor: 'pointer' }}>
<span>{(note.content as string).slice(0, 50)}...</span>
</ItemInfo>
<div style={{ display: 'flex', alignItems: 'center', gap: '16px' }}>
<Button type="text" onClick={() => handleEditNote(note)} icon={<EditOutlined />} />
<StatusIcon sourceId={note.id} base={base} getProcessingStatus={getProcessingStatus} />
<Button type="text" danger onClick={() => removeItem(note)} icon={<DeleteOutlined />} />
</div>
</ItemContent>
</ItemCard>
))}
</div>
</ContentSection>
<IndexSection>
<Button type="primary" onClick={handleIndexAll} icon={<FileTextOutlined />}>
{t('knowledge_base.index_all')}
</Button>
<Button onClick={handleCancelIndex} icon={<StopOutlined />} style={{ marginLeft: '10px' }}>
{t('knowledge_base.cancel_index')}
</Button>
<Button
onClick={() => KnowledgeSearchPopup.show({ base })}
icon={<SearchOutlined />}
style={{ marginLeft: '10px' }}>
{t('knowledge_base.query')}
</Button>
</IndexSection>
<div style={{ minHeight: '20px' }} />
</MainContent>
)
}
const MainContent = styled(Scrollbar)`
display: flex;
width: 100%;
flex-direction: column;
padding-bottom: 50px;
padding: 15px;
`
const FileSection = styled.div`
display: flex;
flex-direction: column;
`
const ContentSection = styled.div`
margin-top: 20px;
display: flex;
flex-direction: column;
gap: 10px;
.ant-input-textarea {
background: var(--color-background-soft);
border-radius: 8px;
}
`
const TitleWrapper = styled.div`
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 5px;
background-color: var(--color-background-soft);
padding: 5px 20px;
min-height: 45px;
border-radius: 6px;
.ant-typography {
margin-bottom: 0;
}
`
const FileListSection = styled.div`
margin-top: 20px;
width: 100%;
display: flex;
flex-direction: column;
gap: 8px;
`
const ItemCard = styled(Card)`
background-color: transparent;
border: none;
.ant-card-body {
padding: 0 20px;
}
`
const ItemContent = styled.div`
display: flex;
align-items: center;
justify-content: space-between;
gap: 16px;
`
const ItemInfo = styled.div`
display: flex;
align-items: center;
gap: 8px;
flex: 1;
a {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
max-width: 600px;
}
`
const IndexSection = styled.div`
margin-top: 20px;
display: flex;
justify-content: flex-end;
`
export default KnowledgeContent

View File

@ -0,0 +1,202 @@
import { DeleteOutlined, EditOutlined, FileTextOutlined, PlusOutlined } from '@ant-design/icons'
import { Navbar, NavbarCenter } from '@renderer/components/app/Navbar'
import ListItem from '@renderer/components/ListItem'
import PromptPopup from '@renderer/components/Popups/PromptPopup'
import Scrollbar from '@renderer/components/Scrollbar'
import { RootState } from '@renderer/store'
import { deleteBase, renameBase } from '@renderer/store/knowledge'
import { KnowledgeBase } from '@renderer/types'
import { Dropdown, Empty, MenuProps } from 'antd'
import { FC, useCallback, useEffect, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useDispatch, useSelector } from 'react-redux'
import styled from 'styled-components'
import AddKnowledgePopup from './AddKnowledgePopup'
import KnowledgeContent from './KnowledgeContent'
const KnowledgePage: FC = () => {
const { t } = useTranslation()
const { bases } = useSelector((state: RootState) => state.knowledge)
const [selectedBase, setSelectedBase] = useState<KnowledgeBase>()
const dispatch = useDispatch()
const handleAddKnowledge = async () => {
await AddKnowledgePopup.show({
title: t('knowledge_base.add.title')
})
}
useEffect(() => {
if (bases.length > 0) {
setSelectedBase(bases[0])
}
}, [bases])
const getMenuItems = useCallback(
(base: KnowledgeBase) => {
const menus: MenuProps['items'] = [
{
label: t('knowledge_base.rename'),
key: 'rename',
icon: <EditOutlined />,
async onClick() {
const name = await PromptPopup.show({
title: t('knowledge_base.rename'),
message: '',
defaultValue: base.name || ''
})
if (name && base.name !== name) {
dispatch(renameBase({ baseId: base.id, name }))
}
}
},
{ type: 'divider' },
{
label: t('common.delete'),
danger: true,
key: 'delete',
icon: <DeleteOutlined />,
onClick: () => {
dispatch(deleteBase({ baseId: base.id }))
}
}
]
return menus
},
[dispatch, t]
)
return (
<Container>
<Navbar>
<NavbarCenter style={{ borderRight: 'none' }}>{t('knowledge_base.title')}</NavbarCenter>
</Navbar>
<ContentContainer id="content-container">
<SideNav>
<ScrollContainer>
{bases.map((base) => (
<Dropdown menu={{ items: getMenuItems(base) }} trigger={['contextMenu']} key={base.id}>
<div>
<ListItem
active={selectedBase?.id === base.id}
icon={<FileTextOutlined />}
title={base.name}
onClick={() => setSelectedBase(base)}
/>
</div>
</Dropdown>
))}
<AddKnowledgeItem onClick={handleAddKnowledge}>
<AddKnowledgeName>
<PlusOutlined style={{ color: 'var(--color-text-2)', marginRight: 4 }} />
{t('button.add')}
</AddKnowledgeName>
</AddKnowledgeItem>
<div style={{ minHeight: '10px' }}></div>
</ScrollContainer>
</SideNav>
{bases.length === 0 ? (
<MainContent>
<Empty description={t('knowledge_base.empty')} image={Empty.PRESENTED_IMAGE_SIMPLE} />
</MainContent>
) : selectedBase ? (
<KnowledgeContent selectedBase={selectedBase} />
) : null}
</ContentContainer>
</Container>
)
}
const Container = styled.div`
display: flex;
flex: 1;
flex-direction: column;
height: calc(100vh - var(--navbar-height));
`
const ContentContainer = styled.div`
display: flex;
flex: 1;
flex-direction: row;
min-height: 100%;
`
const MainContent = styled(Scrollbar)`
padding: 15px 20px;
display: flex;
width: 100%;
flex-direction: column;
padding-bottom: 50px;
`
const SideNav = styled.div`
width: var(--assistants-width);
border-right: 0.5px solid var(--color-border);
padding: 12px 10px;
display: flex;
flex-direction: column;
.ant-menu {
border-inline-end: none !important;
background: transparent;
flex: 1;
}
.ant-menu-item {
height: 40px;
line-height: 40px;
margin: 4px 0;
width: 100%;
&:hover {
background-color: var(--color-background-soft);
}
&.ant-menu-item-selected {
background-color: var(--color-background-soft);
color: var(--color-primary);
}
}
`
const ScrollContainer = styled(Scrollbar)`
display: flex;
flex-direction: column;
flex: 1;
> div {
margin-bottom: 8px;
&:last-child {
margin-bottom: 0;
}
}
`
const AddKnowledgeItem = styled.div`
display: flex;
flex-direction: row;
justify-content: space-between;
padding: 7px 12px;
position: relative;
font-family: Ubuntu;
border-radius: 16px;
border: 0.5px solid transparent;
cursor: pointer;
&:hover {
background-color: var(--color-background-soft);
}
`
const AddKnowledgeName = styled.div`
color: var(--color-text);
display: -webkit-box;
-webkit-line-clamp: 1;
-webkit-box-orient: vertical;
overflow: hidden;
font-size: 13px;
`
export default KnowledgePage

View File

@ -0,0 +1,182 @@
import { ExtractChunkData } from '@llm-tools/embedjs-interfaces'
import { TopView } from '@renderer/components/TopView'
import { getRagAppRequestParams } from '@renderer/services/KnowledgeService'
import { KnowledgeBase } from '@renderer/types'
import { Input, List, Modal, Spin, Typography } from 'antd'
import { useState } from 'react'
import { useTranslation } from 'react-i18next'
import styled from 'styled-components'
const { Search } = Input
const { Text, Paragraph } = Typography
interface ShowParams {
base: KnowledgeBase
}
interface Props extends ShowParams {
resolve: (data: any) => void
}
const PopupContainer: React.FC<Props> = ({ base, resolve }) => {
const [open, setOpen] = useState(true)
const [loading, setLoading] = useState(false)
const [results, setResults] = useState<ExtractChunkData[]>([])
const [searchKeyword, setSearchKeyword] = useState('')
const { t } = useTranslation()
const handleSearch = async (value: string) => {
if (!value.trim()) {
setResults([])
setSearchKeyword('')
return
}
setSearchKeyword(value.trim())
setLoading(true)
try {
const searchResults = await window.api.knowledgeBase.search({
search: value,
config: getRagAppRequestParams(base)
})
setResults(searchResults)
} catch (error) {
console.error('Search failed:', error)
} finally {
setLoading(false)
}
}
const onOk = () => {
setOpen(false)
}
const onCancel = () => {
setOpen(false)
}
const onClose = () => {
resolve({})
}
KnowledgeSearchPopup.hide = onCancel
const highlightText = (text: string) => {
if (!searchKeyword) return text
const parts = text.split(new RegExp(`(${searchKeyword})`, 'gi'))
return parts.map((part, i) =>
part.toLowerCase() === searchKeyword.toLowerCase() ? <mark key={i}>{part}</mark> : part
)
}
return (
<Modal
title={t('knowledge_base.search')}
open={open}
onOk={onOk}
onCancel={onCancel}
afterClose={onClose}
width={800}
footer={null}
centered
transitionName="ant-move-down">
<SearchContainer>
<Search
placeholder={t('knowledge_base.search_placeholder')}
allowClear
enterButton
size="large"
onSearch={handleSearch}
/>
<ResultsContainer>
{loading ? (
<LoadingContainer>
<Spin size="large" />
</LoadingContainer>
) : (
<List
dataSource={results}
renderItem={(item) => (
<List.Item>
<ResultItem>
<ScoreTag>Score: {(item.score * 100).toFixed(1)}%</ScoreTag>
<Paragraph>{highlightText(item.pageContent)}</Paragraph>
<MetadataContainer>
<Text type="secondary">Source: {item.metadata.source}</Text>
</MetadataContainer>
</ResultItem>
</List.Item>
)}
/>
)}
</ResultsContainer>
</SearchContainer>
</Modal>
)
}
const SearchContainer = styled.div`
display: flex;
flex-direction: column;
gap: 20px;
`
const ResultsContainer = styled.div`
max-height: 60vh;
overflow-y: auto;
`
const LoadingContainer = styled.div`
display: flex;
justify-content: center;
align-items: center;
height: 200px;
`
const ResultItem = styled.div`
width: 100%;
position: relative;
padding: 16px;
background: var(--color-background-soft);
border-radius: 8px;
`
const ScoreTag = styled.div`
position: absolute;
top: 8px;
right: 8px;
padding: 2px 8px;
background: var(--color-primary);
color: white;
border-radius: 4px;
font-size: 12px;
`
const MetadataContainer = styled.div`
margin-top: 8px;
padding-top: 8px;
border-top: 1px solid var(--color-border);
`
const TopViewKey = 'KnowledgeSearchPopup'
export default class KnowledgeSearchPopup {
static topviewId = 0
static hide() {
TopView.hide(TopViewKey)
}
static show(props: ShowParams) {
return new Promise<any>((resolve) => {
TopView.show(
<PopupContainer
{...props}
resolve={(v) => {
resolve(v)
TopView.hide(TopViewKey)
}}
/>,
TopViewKey
)
})
}
}

View File

@ -0,0 +1,89 @@
import { CheckCircleOutlined, CloseCircleOutlined } from '@ant-design/icons'
import { Center } from '@renderer/components/Layout'
import { KnowledgeBase, ProcessingItem } from '@renderer/types'
import { Tooltip } from 'antd'
import { FC } from 'react'
import { useTranslation } from 'react-i18next'
import styled from 'styled-components'
interface StatusIconProps {
sourceId: string
base: KnowledgeBase
getProcessingStatus: (sourceId: string) => ProcessingItem | undefined
}
const StatusIcon: FC<StatusIconProps> = ({ sourceId, base, getProcessingStatus }) => {
const { t } = useTranslation()
const status = getProcessingStatus(sourceId)
const item = base.items.find((item) => item.id === sourceId)
if (!status) {
if (item?.uniqueId) {
return (
<Tooltip title={t('knowledge_base.status_completed')} placement="left">
<CheckCircleOutlined style={{ color: '#52c41a' }} />
</Tooltip>
)
}
return (
<Tooltip title={t('knowledge_base.status_new')} placement="left">
<Center style={{ width: '16px', height: '16px' }}>
<StatusDot $status="new" />
</Center>
</Tooltip>
)
}
switch (status.status) {
case 'pending':
return (
<Tooltip title={t('knowledge_base.status_pending')} placement="left">
<StatusDot $status="pending" />
</Tooltip>
)
case 'processing':
return (
<Tooltip title={t('knowledge_base.status_processing')} placement="left">
<StatusDot $status="processing" />
</Tooltip>
)
case 'completed':
return (
<Tooltip title={t('knowledge_base.status_completed')} placement="left">
<CheckCircleOutlined style={{ color: '#52c41a' }} />
</Tooltip>
)
case 'failed':
return (
<Tooltip title={t('knowledge_base.status_failed')} placement="left">
<CloseCircleOutlined style={{ color: '#ff4d4f' }} />
</Tooltip>
)
default:
return null
}
}
const StatusDot = styled.div<{ $status: 'pending' | 'processing' | 'new' }>`
width: 8px;
height: 8px;
border-radius: 50%;
background-color: ${(props) =>
props.$status === 'pending' ? '#faad14' : props.$status === 'new' ? '#918999' : '#1890ff'};
animation: ${(props) => (props.$status === 'processing' ? 'pulse 2s infinite' : 'none')};
cursor: pointer;
@keyframes pulse {
0% {
opacity: 1;
}
50% {
opacity: 0.4;
}
100% {
opacity: 1;
}
}
`
export default StatusIcon

View File

@ -1,7 +1,7 @@
import { LoadingOutlined, MinusOutlined, PlusOutlined, QuestionCircleOutlined } from '@ant-design/icons' import { LoadingOutlined, MinusOutlined, PlusOutlined, QuestionCircleOutlined } from '@ant-design/icons'
import VisionIcon from '@renderer/components/Icons/VisionIcon' import VisionIcon from '@renderer/components/Icons/VisionIcon'
import WebSearchIcon from '@renderer/components/Icons/WebSearchIcon' import WebSearchIcon from '@renderer/components/Icons/WebSearchIcon'
import { getModelLogo, isVisionModel, isWebSearchModel, SYSTEM_MODELS } from '@renderer/config/models' import { getModelLogo, isEmbeddingModel, isVisionModel, isWebSearchModel, SYSTEM_MODELS } from '@renderer/config/models'
import { useProvider } from '@renderer/hooks/useProvider' import { useProvider } from '@renderer/hooks/useProvider'
import { fetchModels } from '@renderer/services/ApiService' import { fetchModels } from '@renderer/services/ApiService'
import { Model, Provider } from '@renderer/types' import { Model, Provider } from '@renderer/types'
@ -134,6 +134,11 @@ const PopupContainer: React.FC<Props> = ({ provider: _provider, resolve }) => {
Free Free
</Tag> </Tag>
)} )}
{isEmbeddingModel(model) && (
<Tag style={{ marginLeft: 10 }} color="orange">
Embed
</Tag>
)}
{!isEmpty(model.description) && ( {!isEmpty(model.description) && (
<Popover <Popover
trigger="click" trigger="click"

View File

@ -3,6 +3,8 @@ import ProviderFactory from '@renderer/providers/ProviderFactory'
import { Assistant, Message, Provider, Suggestion } from '@renderer/types' import { Assistant, Message, Provider, Suggestion } from '@renderer/types'
import OpenAI from 'openai' import OpenAI from 'openai'
import { CompletionsParams } from '.'
export default class AiProvider { export default class AiProvider {
private sdk: BaseProvider private sdk: BaseProvider
@ -42,6 +44,10 @@ export default class AiProvider {
return this.sdk.models() return this.sdk.models()
} }
public getApiKey(): string {
return this.sdk.getApiKey()
}
public async generateImage(params: { public async generateImage(params: {
prompt: string prompt: string
negativePrompt: string negativePrompt: string

View File

@ -9,6 +9,7 @@ import { Assistant, FileTypes, Message, Provider, Suggestion } from '@renderer/t
import { first, flatten, sum, takeRight } from 'lodash' import { first, flatten, sum, takeRight } from 'lodash'
import OpenAI from 'openai' import OpenAI from 'openai'
import { CompletionsParams } from '.'
import BaseProvider from './BaseProvider' import BaseProvider from './BaseProvider'
export default class AnthropicProvider extends BaseProvider { export default class AnthropicProvider extends BaseProvider {

View File

@ -1,8 +1,13 @@
import { getOllamaKeepAliveTime } from '@renderer/hooks/useOllama' import { getOllamaKeepAliveTime } from '@renderer/hooks/useOllama'
import { Assistant, Message, Provider, Suggestion } from '@renderer/types' import { getRagAppRequestParams } from '@renderer/services/KnowledgeService'
import store from '@renderer/store'
import { Assistant, FileType, Message, Provider, Suggestion } from '@renderer/types'
import { delay } from '@renderer/utils' import { delay } from '@renderer/utils'
import { take } from 'lodash'
import OpenAI from 'openai' import OpenAI from 'openai'
import { CompletionsParams } from '.'
export default abstract class BaseProvider { export default abstract class BaseProvider {
protected provider: Provider protected provider: Provider
protected host: string protected host: string
@ -58,6 +63,66 @@ export default abstract class BaseProvider {
} }
} }
public async getMessageContentWithKnowledgeBase(message: Message) {
if (!message.knowledgeBaseIds) {
return message.content
}
const knowledgeId = message.knowledgeBaseIds[0]
const base = store.getState().knowledge.bases.find((kb) => kb.id === knowledgeId)
console.debug('knowledge', base)
if (!base) {
return message.content
}
const searchResults = await window.api.knowledgeBase.search({
search: message.content,
config: getRagAppRequestParams(base)
})
console.debug('searchResults', searchResults)
const references = take(searchResults, 5)
.map((item, index) => {
let sourceUrl = ''
let sourceName = ''
const baseItem = base.items.find((i) => i.uniqueId === item.metadata.uniqueLoaderId)
if (baseItem) {
switch (baseItem.type) {
case 'file':
sourceUrl = `file://${(baseItem?.content as FileType).path}`
sourceName = (baseItem?.content as FileType).origin_name
break
case 'url':
sourceUrl = baseItem.content as string
sourceName = ''
break
case 'note':
sourceUrl = ''
sourceName = ''
break
}
}
return `
---
id: ${index}
content: ${item.pageContent}
source_type: ${baseItem?.type}
source_name: ${sourceName}
source_url: ${sourceUrl}
`
})
.join('\n\n')
const prompt = `回答问题请参考以下内容,并使用类似 [^1] content [source_name](source_url) 的脚注格式引用数据来源,脚注内容可以点击跳转。当 source_name 为空的时候可以使用 content 作为脚注内容。`
return [message.content, prompt, references].join('\n\n')
}
abstract completions({ messages, assistant, onChunk, onFilterMessages }: CompletionsParams): Promise<void> abstract completions({ messages, assistant, onChunk, onFilterMessages }: CompletionsParams): Promise<void>
abstract translate(message: Message, assistant: Assistant): Promise<string> abstract translate(message: Message, assistant: Assistant): Promise<string>
abstract summaries(messages: Message[], assistant: Assistant): Promise<string> abstract summaries(messages: Message[], assistant: Assistant): Promise<string>

View File

@ -13,6 +13,7 @@ import {
ChatCompletionMessageParam ChatCompletionMessageParam
} from 'openai/resources' } from 'openai/resources'
import { CompletionsParams } from '.'
import BaseProvider from './BaseProvider' import BaseProvider from './BaseProvider'
export default class OpenAIProvider extends BaseProvider { export default class OpenAIProvider extends BaseProvider {
@ -53,7 +54,7 @@ export default class OpenAIProvider extends BaseProvider {
if (!message.files) { if (!message.files) {
return { return {
role: message.role, role: message.role,
content: message.content content: await this.getMessageContentWithKnowledgeBase(message)
} }
} }
@ -149,19 +150,18 @@ export default class OpenAIProvider extends BaseProvider {
}) })
if (!isSupportStreamOutput) { if (!isSupportStreamOutput) {
let time_completion_millsec = new Date().getTime() - start_time_millsec const time_completion_millsec = new Date().getTime() - start_time_millsec
return onChunk({ return onChunk({
text: stream.choices[0].message?.content || '', text: stream.choices[0].message?.content || '',
usage: stream.usage, usage: stream.usage,
metrics: { metrics: {
completion_tokens: stream.usage?.completion_tokens, completion_tokens: stream.usage?.completion_tokens,
time_completion_millsec: time_completion_millsec, time_completion_millsec,
time_first_token_sec: 0, time_first_token_millsec: 0
} }
}) })
} }
for await (const chunk of stream) { for await (const chunk of stream) {
if (window.keyv.get(EVENT_NAMES.CHAT_COMPLETION_PAUSED)) { if (window.keyv.get(EVENT_NAMES.CHAT_COMPLETION_PAUSED)) {
break break
@ -169,14 +169,14 @@ export default class OpenAIProvider extends BaseProvider {
if (time_first_token_millsec == 0) { if (time_first_token_millsec == 0) {
time_first_token_millsec = new Date().getTime() - start_time_millsec time_first_token_millsec = new Date().getTime() - start_time_millsec
} }
let time_completion_millsec = new Date().getTime() - start_time_millsec const time_completion_millsec = new Date().getTime() - start_time_millsec
onChunk({ onChunk({
text: chunk.choices[0]?.delta?.content || '', text: chunk.choices[0]?.delta?.content || '',
usage: chunk.usage, usage: chunk.usage,
metrics: { metrics: {
completion_tokens: chunk.usage?.completion_tokens, completion_tokens: chunk.usage?.completion_tokens,
time_completion_millsec: time_completion_millsec, time_completion_millsec,
time_first_token_millsec: time_first_token_millsec, time_first_token_millsec
} }
}) })
} }
@ -302,13 +302,7 @@ export default class OpenAIProvider extends BaseProvider {
public async models(): Promise<OpenAI.Models.Model[]> { public async models(): Promise<OpenAI.Models.Model[]> {
try { try {
const query: Record<string, any> = {} const response = await this.sdk.models.list()
if (this.provider.id === 'silicon') {
query.type = 'text'
}
const response = await this.sdk.models.list({ query })
if (this.provider.id === 'github') { if (this.provider.id === 'github') {
// @ts-ignore key is not typed // @ts-ignore key is not typed

View File

@ -0,0 +1,225 @@
import { AddLoaderReturn } from '@llm-tools/embedjs-interfaces'
import db from '@renderer/databases'
import { getRagAppRequestParams } from '@renderer/services/KnowledgeService'
import store from '@renderer/store'
import { removeProcessingItem, updateBaseItemUniqueId, updateProcessingStatus } from '@renderer/store/knowledge'
import { ProcessingItem } from '@renderer/types'
class KnowledgeQueue {
private processing: Map<string, boolean> = new Map()
private pollingInterval: NodeJS.Timeout | null = null
private readonly POLLING_INTERVAL = 5000
private readonly MAX_RETRIES = 3
constructor() {
this.checkAllBases().catch(console.error)
this.startPolling()
}
private startPolling(): void {
if (this.pollingInterval) return
const state = store.getState()
state.knowledge.bases.forEach((base) => {
base.processingQueue.forEach((item) => {
if (item.status === 'processing') {
store.dispatch(
updateProcessingStatus({
baseId: base.id,
itemId: item.id,
status: 'pending',
progress: 0
})
)
}
})
})
this.pollingInterval = setInterval(() => {
this.checkAllBases()
}, this.POLLING_INTERVAL)
}
private stopPolling(): void {
if (this.pollingInterval) {
clearInterval(this.pollingInterval)
this.pollingInterval = null
}
}
private async checkAllBases(): Promise<void> {
const state = store.getState()
const bases = state.knowledge.bases
console.log('[KnowledgeQueue] Checking all bases for pending items...')
await Promise.all(
bases.map(async (base) => {
const processableItems = base.processingQueue.filter((item) => {
if (item.status === 'failed') {
return !item.retryCount || item.retryCount < this.MAX_RETRIES
}
return item.status === 'pending'
})
const hasProcessableItems = processableItems.length > 0
console.log(
`[KnowledgeQueue] Base ${base.id}: ${hasProcessableItems ? 'has processable items' : 'no processable items'}`
)
if (hasProcessableItems && !this.processing.get(base.id)) {
await this.processQueue(base.id)
}
})
)
}
async processQueue(baseId: string): Promise<void> {
if (this.processing.get(baseId)) {
console.log(`[KnowledgeQueue] Queue for base ${baseId} is already being processed`)
return
}
this.processing.set(baseId, true)
try {
const state = store.getState()
const base = state.knowledge.bases.find((b) => b.id === baseId)
if (!base) {
throw new Error('Knowledge base not found')
}
const processableItems = base.processingQueue.filter((item) => {
if (item.status === 'failed') {
return !item.retryCount || item.retryCount < this.MAX_RETRIES
}
return item.status === 'pending'
})
for (const item of processableItems) {
if (!this.processing.get(baseId)) {
console.log(`[KnowledgeQueue] Processing interrupted for base ${baseId}`)
break
}
console.log(`[KnowledgeQueue] Processing item ${item.id} (${item.type}) in base ${baseId}`)
await this.processItem(baseId, item)
}
} finally {
console.log(`[KnowledgeQueue] Finished processing queue for base ${baseId}`)
this.processing.set(baseId, false)
}
}
stopProcessing(baseId: string): void {
this.processing.set(baseId, false)
}
stopAllProcessing(): void {
this.stopPolling()
for (const baseId of this.processing.keys()) {
this.processing.set(baseId, false)
}
}
private async processItem(baseId: string, item: ProcessingItem): Promise<void> {
try {
if (item.retryCount && item.retryCount >= this.MAX_RETRIES) {
console.log(`[KnowledgeQueue] Item ${item.id} has reached max retries, skipping`)
return
}
console.log(`[KnowledgeQueue] Starting to process item ${item.id} (${item.type})`)
// Update status to processing
store.dispatch(
updateProcessingStatus({
baseId,
itemId: item.id,
status: 'processing',
retryCount: (item.retryCount || 0) + 1
})
)
const base = store.getState().knowledge.bases.find((b) => b.id === baseId)
if (!base) {
throw new Error(`[KnowledgeQueue] Knowledge base ${baseId} not found`)
}
const requestParams = getRagAppRequestParams(base)
const sourceItem = base.items.find((i) => i.id === item.sourceId)
if (!sourceItem) {
throw new Error(`[KnowledgeQueue] Source item ${item.sourceId} not found in base ${baseId}`)
}
let result: AddLoaderReturn | null = null
let note, content
switch (item.type) {
case 'file':
console.log(`[KnowledgeQueue] Processing file: ${sourceItem.content}`)
result = await window.api.knowledgeBase.add({ data: sourceItem.content, config: requestParams })
console.log(`[KnowledgeQueue] Result: ${JSON.stringify(result)}`)
break
case 'url':
console.log(`[KnowledgeQueue] Processing URL: ${sourceItem.content}`)
result = await window.api.knowledgeBase.add({ data: sourceItem.content, config: requestParams })
console.log(`[KnowledgeQueue] Result: ${JSON.stringify(result)}`)
break
case 'note':
console.log(`[KnowledgeQueue] Processing note: ${sourceItem.content}`)
note = await db.knowledge_notes.get(item.sourceId)
if (!note) throw new Error(`Source note ${item.sourceId} not found`)
content = note.content as string
result = await window.api.knowledgeBase.add({ data: content, config: requestParams })
console.log(`[KnowledgeQueue] Result: ${JSON.stringify(result)}`)
break
}
console.log(`[KnowledgeQueue] Successfully completed processing item ${item.id}`)
// Mark as completed
store.dispatch(
updateProcessingStatus({
baseId,
itemId: item.id,
status: 'completed'
})
)
// Update uniqueId
if (result) {
store.dispatch(
updateBaseItemUniqueId({
baseId,
itemId: item.sourceId,
uniqueId: result.uniqueId
})
)
}
console.debug(`[KnowledgeQueue] Updated uniqueId for item ${item.sourceId} in base ${baseId}`)
// Remove from queue after successful processing
setTimeout(() => {
store.dispatch(removeProcessingItem({ baseId, itemId: item.id }))
}, 1000)
} catch (error) {
console.error(`[KnowledgeQueue] Error processing item ${item.id}:`, error)
store.dispatch(
updateProcessingStatus({
baseId,
itemId: item.id,
status: 'failed',
error: error instanceof Error ? error.message : 'Unknown error',
retryCount: (item.retryCount || 0) + 1
})
)
}
}
}
export default new KnowledgeQueue()

View File

@ -27,6 +27,8 @@ class FileManager {
} }
static async uploadFile(file: FileType): Promise<FileType> { static async uploadFile(file: FileType): Promise<FileType> {
console.debug(`[FileManager] Uploading file: ${JSON.stringify(file)}`)
const uploadFile = await window.api.file.upload(file) const uploadFile = await window.api.file.upload(file)
const fileRecord = await db.files.get(uploadFile.id) const fileRecord = await db.files.get(uploadFile.id)

View File

@ -0,0 +1,16 @@
import AiProvider from '@renderer/providers/AiProvider'
import { KnowledgeBase, RagAppRequestParams } from '@renderer/types'
import { getProviderByModel } from './AssistantService'
export const getRagAppRequestParams = (base: KnowledgeBase): RagAppRequestParams => {
const provider = getProviderByModel(base.model)
const aiProvider = new AiProvider(provider)
return {
id: base.id,
model: base.model.name,
apiKey: aiProvider.getApiKey(),
baseURL: provider.apiHost + '/v1'
}
}

View File

@ -5,6 +5,7 @@ import storage from 'redux-persist/lib/storage'
import agents from './agents' import agents from './agents'
import assistants from './assistants' import assistants from './assistants'
import knowledge from './knowledge'
import llm from './llm' import llm from './llm'
import migrate from './migrate' import migrate from './migrate'
import paintings from './paintings' import paintings from './paintings'
@ -19,7 +20,8 @@ const rootReducer = combineReducers({
llm, llm,
settings, settings,
runtime, runtime,
shortcuts shortcuts,
knowledge
}) })
const persistedReducer = persistReducer( const persistedReducer = persistReducer(

View File

@ -0,0 +1,242 @@
import { createSlice, PayloadAction } from '@reduxjs/toolkit'
import FileManager from '@renderer/services/FileManager'
import { getRagAppRequestParams } from '@renderer/services/KnowledgeService'
import { FileType, KnowledgeBase, KnowledgeItem, ProcessingItem, ProcessingStatus } from '@renderer/types'
export interface KnowledgeState {
bases: KnowledgeBase[]
}
const initialState: KnowledgeState = {
bases: []
}
const knowledgeSlice = createSlice({
name: 'knowledge',
initialState,
reducers: {
// 添加知识库
addBase(state, action: PayloadAction<KnowledgeBase>) {
state.bases.push(action.payload)
},
// 删除知识库
deleteBase(state, action: PayloadAction<{ baseId: string }>) {
const base = state.bases.find((b) => b.id === action.payload.baseId)
if (base) {
state.bases = state.bases.filter((b) => b.id !== action.payload.baseId)
const files = base.items.filter((item) => item.type === 'file')
FileManager.deleteFiles(files.map((item) => item.content) as FileType[])
window.api.knowledgeBase.delete(action.payload.baseId)
}
},
// 重命名知识库
renameBase(state, action: PayloadAction<{ baseId: string; name: string }>) {
const base = state.bases.find((b) => b.id === action.payload.baseId)
if (base) {
base.name = action.payload.name
base.updated_at = Date.now()
}
},
// 更新知识库
updateBase(state, action: PayloadAction<KnowledgeBase>) {
const index = state.bases.findIndex((b) => b.id === action.payload.id)
if (index !== -1) {
state.bases[index] = action.payload
}
},
// 添加条目
addItem(state, action: PayloadAction<{ baseId: string; item: KnowledgeItem }>) {
const base = state.bases.find((b) => b.id === action.payload.baseId)
if (base) {
if (action.payload.item.type === 'note') {
base.items.push(action.payload.item)
} else if (action.payload.item.type === 'url') {
const urlExists = base.items.some((item) => item.content === action.payload.item.content)
if (!urlExists) {
base.items.push(action.payload.item)
}
} else if (action.payload.item.type === 'file') {
action.payload.item.created_at = new Date(action.payload.item.created_at).getTime()
action.payload.item.updated_at = new Date(action.payload.item.updated_at).getTime()
base.items.push(action.payload.item)
}
base.updated_at = Date.now()
}
},
// 删除条目
removeItem(state, action: PayloadAction<{ baseId: string; item: KnowledgeItem }>) {
const { baseId, item } = action.payload
const base = state.bases.find((b) => b.id === baseId)
if (base) {
base.items = base.items.filter((item) => item.id !== action.payload.item.id)
base.updated_at = Date.now()
if (item?.uniqueId) {
window.api.knowledgeBase.remove({
uniqueId: item.uniqueId,
config: getRagAppRequestParams(base)
})
}
if (item.type === 'file' && typeof item.content === 'object') {
FileManager.deleteFile(item.content.id)
}
}
},
// 更新文件
updateFiles(state, action: PayloadAction<{ baseId: string; items: KnowledgeItem[] }>) {
const base = state.bases.find((b) => b.id === action.payload.baseId)
if (base) {
// 保留非文件类型的项目
const nonFileItems = base.items.filter((item) => item.type !== 'file')
base.items = [...nonFileItems, ...action.payload.items]
base.updated_at = Date.now()
}
},
// 更新笔记
updateNotes(state, action: PayloadAction<{ baseId: string; item: KnowledgeItem }>) {
const base = state.bases.find((b) => b.id === action.payload.baseId)
if (base) {
const existingNoteIndex = base.items.findIndex(
(item) => item.type === 'note' && item.id === action.payload.item.id
)
if (existingNoteIndex !== -1) {
base.items[existingNoteIndex] = action.payload.item
} else {
base.items.push(action.payload.item)
}
base.updated_at = Date.now()
}
},
// 添加处理队列项
addProcessingItem(
state,
action: PayloadAction<{ baseId: string; type: 'file' | 'url' | 'note'; sourceId: string }>
) {
const base = state.bases.find((b) => b.id === action.payload.baseId)
if (base) {
const newItem: ProcessingItem = {
id: `${action.payload.type}-${action.payload.sourceId}`,
type: action.payload.type,
sourceId: action.payload.sourceId,
status: 'pending',
createdAt: Date.now(),
updatedAt: Date.now(),
baseId: action.payload.baseId
}
// 避免重复添加
const exists = base.processingQueue.some((item) => item.sourceId === action.payload.sourceId)
if (!exists) {
base.processingQueue.push(newItem)
}
}
},
// 更新处理状态
updateProcessingStatus(
state,
action: PayloadAction<{
baseId: string
itemId: string
status: ProcessingStatus
progress?: number
error?: string
retryCount?: number
}>
) {
const base = state.bases.find((b) => b.id === action.payload.baseId)
if (base) {
const item = base.processingQueue.find((item) => item.id === action.payload.itemId)
if (item) {
item.status = action.payload.status
item.progress = action.payload.progress
item.error = action.payload.error
item.retryCount = action.payload.retryCount
item.updatedAt = Date.now()
}
}
},
// 移除处理队列项
removeProcessingItem(state, action: PayloadAction<{ baseId: string; itemId: string }>) {
const base = state.bases.find((b) => b.id === action.payload.baseId)
if (base) {
base.processingQueue = base.processingQueue.filter((item) => item.id !== action.payload.itemId)
}
},
// 清除已完成的项目
clearCompletedItems(state, action: PayloadAction<{ baseId: string }>) {
const base = state.bases.find((b) => b.id === action.payload.baseId)
if (base) {
base.processingQueue = base.processingQueue.filter(
(item) => item.status !== 'completed' && item.status !== 'failed'
)
}
},
// 清除所有队列项目
clearAllItems(state, action: PayloadAction<{ baseId: string }>) {
const base = state.bases.find((b) => b.id === action.payload.baseId)
if (base) {
base.processingQueue = []
}
},
// 更新知识库单个条目下面的 uniqueId
updateBaseItemUniqueId(state, action: PayloadAction<{ baseId: string; itemId: string; uniqueId: string }>) {
const base = state.bases.find((b) => b.id === action.payload.baseId)
if (base) {
const item = base.items.find((item) => item.id === action.payload.itemId)
if (item) {
item.uniqueId = action.payload.uniqueId
}
}
}
}
})
// Selectors
export const selectProcessingItemBySource = (
state: KnowledgeState,
baseId: string,
sourceId: string
): ProcessingItem | undefined => {
const base = state.bases.find((b) => b.id === baseId)
return base?.processingQueue.find((item) => item.sourceId === sourceId)
}
export const selectProcessingItemsByType = (
state: KnowledgeState,
baseId: string,
type: 'file' | 'url' | 'note'
): ProcessingItem[] => {
const base = state.bases.find((b) => b.id === baseId)
return base?.processingQueue.filter((item) => item.type === type) || []
}
export const {
addBase,
deleteBase,
renameBase,
updateBase,
addItem,
updateFiles,
updateNotes,
removeItem,
addProcessingItem,
updateProcessingStatus,
removeProcessingItem,
clearCompletedItems,
clearAllItems,
updateBaseItemUniqueId
} = knowledgeSlice.actions
export default knowledgeSlice.reducer

View File

@ -48,6 +48,7 @@ export type Message = {
images?: string[] images?: string[]
usage?: OpenAI.Completions.CompletionUsage usage?: OpenAI.Completions.CompletionUsage
metrics?: Metrics metrics?: Metrics
knowledgeBaseIds?: string[]
type: 'text' | '@' | 'clear' type: 'text' | '@' | 'clear'
isPreset?: boolean isPreset?: boolean
} }
@ -179,3 +180,46 @@ export interface Shortcut {
enabled: boolean enabled: boolean
system: boolean system: boolean
} }
export type ProcessingStatus = 'pending' | 'processing' | 'completed' | 'failed'
export type ProcessingItem = {
id: string
type: 'file' | 'url' | 'note'
status: ProcessingStatus
progress?: number
error?: string
createdAt: number
updatedAt: number
sourceId: string // file id, url, or note id
baseId: string
retryCount?: number
}
export type KnowledgeItem = {
id: string
baseId?: string
uniqueId?: string
type: 'file' | 'url' | 'note'
content: string | FileType // for files: FileType, for urls: string, for notes: string
created_at: number
updated_at: number
}
export interface KnowledgeBase {
id: string
name: string
model: Model
description?: string
items: KnowledgeItem[]
created_at: number
updated_at: number
processingQueue: ProcessingItem[]
}
export type RagAppRequestParams = {
id: string
model: string
apiKey: string
baseURL: string
}

1506
yarn.lock

File diff suppressed because it is too large Load Diff