diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7ff29dc4..5aa4d57f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,6 +41,20 @@ jobs: node-version: 20 arch: ${{ matrix.arch }} + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "dir=$(yarn config get cacheFolder)" >> $GITHUB_OUTPUT + + - name: Cache yarn dependencies + uses: actions/cache@v3 + with: + path: | + ${{ steps.yarn-cache-dir-path.outputs.dir }} + node_modules + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + - name: Install corepack run: corepack enable && corepack prepare yarn@4.3.1 --activate diff --git a/.gitignore b/.gitignore index a38c9c13..78bc5b83 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ node_modules dist out build/icons +stats.html # ENV .env diff --git a/electron-builder.yml b/electron-builder.yml index 26189869..8effe102 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -11,6 +11,16 @@ files: - '!src' - '!scripts' - '!local' + - '!docs' + - '!packages' + - '!stats.html' + - '!*.md' + - '!node_modules/rollup-plugin-visualizer' + - '!node_modules/js-tiktoken' + - '!node_modules/node_modules/pdf-parse/lib/pdf.js/v1.9.426' + - '!node_modules/node_modules/pdf-parse/lib/pdf.js/v1.10.88' + - '!node_modules/node_modules/pdf-parse/lib/pdf.js/v2.0.550' + asarUnpack: - resources/** - '**/*.{node,dll,metal,exp,lib}' diff --git a/electron.vite.config.ts b/electron.vite.config.ts index f0646d89..a3f21270 100644 --- a/electron.vite.config.ts +++ b/electron.vite.config.ts @@ -1,6 +1,11 @@ import react from '@vitejs/plugin-react' import { defineConfig, externalizeDepsPlugin } from 'electron-vite' import { resolve } from 'path' +import { visualizer } from 'rollup-plugin-visualizer' + +const visualizerPlugin = (type: 'renderer' | 'main') => { + return process.env[`VISUALIZER_${type.toUpperCase()}`] ? [visualizer({ open: true })] : [] +} export default defineConfig({ main: { @@ -8,8 +13,6 @@ export default defineConfig({ externalizeDepsPlugin({ exclude: [ '@llm-tools/embedjs', - '@llm-tools/embedjs-lancedb', - '@llm-tools/embedjs-ollama', '@llm-tools/embedjs-openai', '@llm-tools/embedjs-loader-web', '@llm-tools/embedjs-loader-markdown', @@ -17,9 +20,10 @@ export default defineConfig({ '@llm-tools/embedjs-loader-xml', '@llm-tools/embedjs-loader-pdf', '@llm-tools/embedjs-loader-sitemap', - '@lancedb/lancedb' + '@llm-tools/embedjs-libsql' ] - }) + }), + ...visualizerPlugin('main') ], resolve: { alias: { @@ -30,7 +34,7 @@ export default defineConfig({ }, build: { rollupOptions: { - external: ['@lancedb/lancedb'] + external: ['@libsql/client'] } } }, @@ -38,7 +42,7 @@ export default defineConfig({ plugins: [externalizeDepsPlugin()] }, renderer: { - plugins: [react()], + plugins: [react(), ...visualizerPlugin('renderer')], resolve: { alias: { '@renderer': resolve('src/renderer/src'), diff --git a/package.json b/package.json index 7c23f243..f44aba2f 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,8 @@ "typecheck": "npm run typecheck:node && npm run typecheck:web", "start": "electron-vite preview", "dev": "electron-vite dev", + "analyze:renderer": "VISUALIZER_RENDERER=true yarn build", + "analyze:main": "VISUALIZER_MAIN=true yarn build", "build": "npm run typecheck && electron-vite build", "postinstall": "electron-builder install-app-deps", "build:unpack": "dotenv npm run build && electron-builder --dir", @@ -46,17 +48,16 @@ "@electron-toolkit/preload": "^3.0.0", "@electron-toolkit/utils": "^3.0.0", "@electron/notarize": "^2.5.0", - "@llm-tools/embedjs": "^0.1.24", - "@llm-tools/embedjs-lancedb": "^0.1.24", - "@llm-tools/embedjs-loader-csv": "^0.1.24", - "@llm-tools/embedjs-loader-markdown": "^0.1.24", - "@llm-tools/embedjs-loader-msoffice": "^0.1.24", - "@llm-tools/embedjs-loader-pdf": "^0.1.24", - "@llm-tools/embedjs-loader-sitemap": "^0.1.24", - "@llm-tools/embedjs-loader-web": "^0.1.24", - "@llm-tools/embedjs-loader-xml": "^0.1.24", - "@llm-tools/embedjs-ollama": "^0.1.24", - "@llm-tools/embedjs-openai": "^0.1.24", + "@llm-tools/embedjs": "^0.1.25", + "@llm-tools/embedjs-libsql": "^0.1.25", + "@llm-tools/embedjs-loader-csv": "^0.1.25", + "@llm-tools/embedjs-loader-markdown": "^0.1.25", + "@llm-tools/embedjs-loader-msoffice": "^0.1.25", + "@llm-tools/embedjs-loader-pdf": "^0.1.25", + "@llm-tools/embedjs-loader-sitemap": "^0.1.25", + "@llm-tools/embedjs-loader-web": "^0.1.25", + "@llm-tools/embedjs-loader-xml": "^0.1.25", + "@llm-tools/embedjs-openai": "^0.1.25", "@types/react-infinite-scroll-component": "^5.0.0", "adm-zip": "^0.5.16", "apache-arrow": "^18.0.0", @@ -69,6 +70,7 @@ "html2canvas": "^1.4.1", "markdown-it": "^14.1.0", "officeparser": "^4.1.1", + "tokenx": "^0.4.1", "webdav": "4.11.4" }, "devDependencies": { @@ -109,7 +111,6 @@ "eslint-plugin-react-hooks": "^4.6.2", "eslint-plugin-simple-import-sort": "^12.1.1", "eslint-plugin-unused-imports": "^4.0.0", - "gpt-tokens": "^1.3.10", "i18next": "^23.11.5", "lodash": "^4.17.21", "mime": "^4.0.4", @@ -132,6 +133,7 @@ "rehype-raw": "^7.0.0", "remark-gfm": "^4.0.0", "remark-math": "^6.0.0", + "rollup-plugin-visualizer": "^5.12.0", "sass": "^1.77.2", "shiki": "^1.22.2", "styled-components": "^6.1.11", diff --git a/src/main/ipc.ts b/src/main/ipc.ts index 8f5c82bf..cebbd766 100644 --- a/src/main/ipc.ts +++ b/src/main/ipc.ts @@ -101,6 +101,7 @@ export function registerIpc(mainWindow: BrowserWindow, app: Electron.App) { // file ipcMain.handle('file:open', fileManager.open) + ipcMain.handle('file:openPath', fileManager.openPath) ipcMain.handle('file:save', fileManager.save) ipcMain.handle('file:select', fileManager.selectFile) ipcMain.handle('file:upload', fileManager.uploadFile) diff --git a/src/main/services/FileStorage.ts b/src/main/services/FileStorage.ts index d7139eeb..dff78863 100644 --- a/src/main/services/FileStorage.ts +++ b/src/main/services/FileStorage.ts @@ -8,7 +8,8 @@ import { OpenDialogOptions, OpenDialogReturnValue, SaveDialogOptions, - SaveDialogReturnValue + SaveDialogReturnValue, + shell } from 'electron' import logger from 'electron-log' import * as fs from 'fs' @@ -300,6 +301,10 @@ class FileStorage { } } + public openPath = async (_: Electron.IpcMainInvokeEvent, path: string): Promise => { + shell.openPath(path).catch((err) => logger.error('[IPC - Error] Failed to open file:', err)) + } + public save = async ( _: Electron.IpcMainInvokeEvent, fileName: string, diff --git a/src/main/services/KnowledgeService.ts b/src/main/services/KnowledgeService.ts index e1b63b7b..5a05acd5 100644 --- a/src/main/services/KnowledgeService.ts +++ b/src/main/services/KnowledgeService.ts @@ -1,11 +1,11 @@ import * as fs from 'node:fs' import path from 'node:path' -import { RAGApplication, RAGApplicationBuilder, TextLoader } from '@llm-tools/embedjs' +import { LocalPathLoader, RAGApplication, RAGApplicationBuilder, TextLoader } from '@llm-tools/embedjs' import { AddLoaderReturn, ExtractChunkData } from '@llm-tools/embedjs-interfaces' -import { LanceDb } from '@llm-tools/embedjs-lancedb' +import { LibSqlDb } from '@llm-tools/embedjs-libsql' import { MarkdownLoader } from '@llm-tools/embedjs-loader-markdown' -import { DocxLoader } from '@llm-tools/embedjs-loader-msoffice' +import { DocxLoader, ExcelLoader, PptLoader } from '@llm-tools/embedjs-loader-msoffice' import { PdfLoader } from '@llm-tools/embedjs-loader-pdf' import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap' import { WebLoader } from '@llm-tools/embedjs-loader-web' @@ -34,10 +34,11 @@ class KnowledgeService { model, apiKey, configuration: { baseURL }, - dimensions: 1024 + dimensions: 1024, + batchSize: 10 }) ) - .setVectorDatabase(new LanceDb({ path: path.join(this.storageDir, id) })) + .setVectorDatabase(new LibSqlDb({ path: path.join(this.storageDir, id) })) .build() } @@ -62,41 +63,58 @@ class KnowledgeService { public add = async ( _: Electron.IpcMainInvokeEvent, - { base, item }: { base: KnowledgeBaseParams; item: KnowledgeItem } + { base, item, forceReload = false }: { base: KnowledgeBaseParams; item: KnowledgeItem; forceReload: boolean } ): Promise => { const ragApplication = await this.getRagApplication(base) + if (item.type === 'directory') { + const directory = item.content as string + return await ragApplication.addLoader(new LocalPathLoader({ path: directory }), forceReload) + } + if (item.type === 'url') { const content = item.content as string if (content.startsWith('http')) { - return await ragApplication.addLoader(new WebLoader({ urlOrContent: content })) + return await ragApplication.addLoader(new WebLoader({ urlOrContent: content }), forceReload) } } if (item.type === 'sitemap') { const content = item.content as string - return await ragApplication.addLoader(new SitemapLoader({ url: content })) + return await ragApplication.addLoader(new SitemapLoader({ url: content }), forceReload) } if (item.type === 'note') { const content = item.content as string - return await ragApplication.addLoader(new TextLoader({ text: content })) + return await ragApplication.addLoader(new TextLoader({ text: content }), forceReload) } if (item.type === 'file') { const file = item.content as FileType if (file.ext === '.pdf') { - return await ragApplication.addLoader(new PdfLoader({ filePathOrUrl: file.path }) as any) + return await ragApplication.addLoader(new PdfLoader({ filePathOrUrl: file.path }) as any, forceReload) } if (file.ext === '.docx') { - return await ragApplication.addLoader(new DocxLoader({ filePathOrUrl: file.path }) as any) + return await ragApplication.addLoader(new DocxLoader({ filePathOrUrl: file.path }) as any, forceReload) } - if (file.ext.startsWith('.md')) { - return await ragApplication.addLoader(new MarkdownLoader({ filePathOrUrl: file.path }) as any) + if (file.ext === '.pptx') { + return await ragApplication.addLoader(new PptLoader({ filePathOrUrl: file.path }) as any, forceReload) } + + if (file.ext === '.xlsx') { + return await ragApplication.addLoader(new ExcelLoader({ filePathOrUrl: file.path }) as any, forceReload) + } + + if (['.md', '.mdx'].includes(file.ext)) { + return await ragApplication.addLoader(new MarkdownLoader({ filePathOrUrl: file.path }) as any, forceReload) + } + + const fileContent = fs.readFileSync(file.path, 'utf-8') + + return await ragApplication.addLoader(new TextLoader({ text: fileContent }), forceReload) } return { entriesAdded: 0, uniqueId: '', loaderType: '' } diff --git a/src/main/services/WindowService.ts b/src/main/services/WindowService.ts index e133505a..ada8b2ad 100644 --- a/src/main/services/WindowService.ts +++ b/src/main/services/WindowService.ts @@ -1,6 +1,7 @@ import { is } from '@electron-toolkit/utils' import { isLinux, isWin } from '@main/constant' import { app, BrowserWindow, Menu, MenuItem, shell } from 'electron' +import Logger from 'electron-log' import windowStateKeeper from 'electron-window-state' import { join } from 'path' @@ -132,7 +133,16 @@ export class WindowService { }) mainWindow.webContents.setWindowOpenHandler((details) => { - shell.openExternal(details.url) + const { url } = details + + if (url.includes('http://file/')) { + const fileUrl = url.replace('http://file/', '') + const filePath = decodeURIComponent(fileUrl) + shell.openPath(filePath).catch((err) => Logger.error('Failed to open file:', err)) + } else { + shell.openExternal(details.url) + } + return { action: 'deny' } }) diff --git a/src/preload/index.d.ts b/src/preload/index.d.ts index 4f34b94d..e9c24b41 100644 --- a/src/preload/index.d.ts +++ b/src/preload/index.d.ts @@ -42,6 +42,7 @@ declare global { create: (fileName: string) => Promise write: (filePath: string, data: Uint8Array | string) => Promise open: (options?: OpenDialogOptions) => Promise<{ fileName: string; filePath: string; content: Buffer } | null> + openPath: (path: string) => Promise save: ( path: string, content: string | NodeJS.ArrayBufferView, @@ -63,7 +64,15 @@ declare global { create: ({ id, model, apiKey, baseURL }: KnowledgeBaseParams) => Promise reset: ({ base }: { base: KnowledgeBaseParams }) => Promise delete: (id: string) => Promise - add: ({ base, item }: { base: KnowledgeBaseParams; item: KnowledgeItem }) => Promise + add: ({ + base, + item, + forceReload = false + }: { + base: KnowledgeBaseParams + item: KnowledgeItem + forceReload?: boolean + }) => Promise remove: ({ uniqueId, base }: { uniqueId: string; base: KnowledgeBaseParams }) => Promise search: ({ search, base }: { search: string; base: KnowledgeBaseParams }) => Promise } diff --git a/src/preload/index.ts b/src/preload/index.ts index 2190f749..2e1b8e99 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -36,6 +36,7 @@ const api = { create: (fileName: string) => ipcRenderer.invoke('file:create', fileName), write: (filePath: string, data: Uint8Array | string) => ipcRenderer.invoke('file:write', filePath, data), open: (options?: { decompress: boolean }) => ipcRenderer.invoke('file:open', options), + openPath: (path: string) => ipcRenderer.invoke('file:openPath', path), save: (path: string, content: string, options?: { compress: boolean }) => ipcRenderer.invoke('file:save', path, content, options), selectFolder: () => ipcRenderer.invoke('file:selectFolder'), @@ -56,8 +57,15 @@ const api = { ipcRenderer.invoke('knowledge-base:create', { id, model, apiKey, baseURL }), reset: ({ base }: { base: KnowledgeBaseParams }) => ipcRenderer.invoke('knowledge-base:reset', { base }), delete: (id: string) => ipcRenderer.invoke('knowledge-base:delete', id), - add: ({ base, item }: { base: KnowledgeBaseParams; item: KnowledgeItem }) => - ipcRenderer.invoke('knowledge-base:add', { base, item }), + add: ({ + base, + item, + forceReload = false + }: { + base: KnowledgeBaseParams + item: KnowledgeItem + forceReload?: boolean + }) => ipcRenderer.invoke('knowledge-base:add', { base, item, forceReload }), remove: ({ uniqueId, base }: { uniqueId: string; base: KnowledgeBaseParams }) => ipcRenderer.invoke('knowledge-base:remove', { uniqueId, base }), search: ({ search, base }: { search: string; base: KnowledgeBaseParams }) => diff --git a/src/renderer/src/assets/styles/index.scss b/src/renderer/src/assets/styles/index.scss index 38559722..2a76db2c 100644 --- a/src/renderer/src/assets/styles/index.scss +++ b/src/renderer/src/assets/styles/index.scss @@ -42,6 +42,7 @@ --color-active: rgba(55, 55, 55, 1); --color-frame-border: #333; --color-group-background: var(--color-background-soft); + --color-reference-background: #0b0e12; --navbar-background-mac: rgba(30, 30, 30, 0.6); --navbar-background: rgba(30, 30, 30); @@ -99,6 +100,7 @@ body[theme-mode='light'] { --color-active: var(--color-white-soft); --color-frame-border: #ddd; --color-group-background: var(--color-white); + --color-reference-background: #f1f7ff; --navbar-background-mac: rgba(255, 255, 255, 0.6); --navbar-background: rgba(255, 255, 255); diff --git a/src/renderer/src/assets/styles/markdown.scss b/src/renderer/src/assets/styles/markdown.scss index 860b2462..4fc4761c 100644 --- a/src/renderer/src/assets/styles/markdown.scss +++ b/src/renderer/src/assets/styles/markdown.scss @@ -229,11 +229,24 @@ .footnotes { margin-top: 1em; + margin-bottom: 1em; padding-top: 1em; - border-top: 1px solid var(--color-border); + + background-color: var(--color-reference-background); + border-radius: 8px; + padding: 8px 12px; + + h4 { + margin-bottom: 5px; + font-size: 12px; + } ol { padding-left: 1em; + margin: 0; + li:last-child { + margin-bottom: 0; + } } li { diff --git a/src/renderer/src/components/AssistantSettings/AssistantModelSettings.tsx b/src/renderer/src/components/AssistantSettings/AssistantModelSettings.tsx index a073aae7..90cfd212 100644 --- a/src/renderer/src/components/AssistantSettings/AssistantModelSettings.tsx +++ b/src/renderer/src/components/AssistantSettings/AssistantModelSettings.tsx @@ -241,7 +241,7 @@ const AssistantModelSettings: FC = ({ assistant, updateAssistant, updateA )} - + { diff --git a/src/renderer/src/components/Popups/SelectModelPopup.tsx b/src/renderer/src/components/Popups/SelectModelPopup.tsx index beb2fbfa..bc444fba 100644 --- a/src/renderer/src/components/Popups/SelectModelPopup.tsx +++ b/src/renderer/src/components/Popups/SelectModelPopup.tsx @@ -1,7 +1,7 @@ import { PushpinOutlined, SearchOutlined } from '@ant-design/icons' import VisionIcon from '@renderer/components/Icons/VisionIcon' import { TopView } from '@renderer/components/TopView' -import { getModelLogo, isVisionModel, isWebSearchModel } from '@renderer/config/models' +import { getModelLogo, isEmbeddingModel, isVisionModel, isWebSearchModel } from '@renderer/config/models' import db from '@renderer/databases' import { useProviders } from '@renderer/hooks/useProvider' import { getModelUniqId } from '@renderer/services/ModelService' @@ -66,6 +66,7 @@ const PopupContainer: React.FC = ({ model, resolve }) => { .filter((p) => p.models && p.models.length > 0) .map((p) => { const filteredModels = sortBy(p.models, ['group', 'name']) + .filter((m) => !isEmbeddingModel(m)) .filter((m) => [m.name + m.provider + t('provider.' + p.id)].join('').toLowerCase().includes(searchText.toLowerCase()) ) @@ -142,7 +143,7 @@ const PopupContainer: React.FC = ({ model, resolve }) => { if (pinnedItems.length > 0) { filteredItems.unshift({ key: 'pinned', - label: t('model.pinned'), + label: t('models.pinned'), type: 'group', children: pinnedItems } as MenuItem) @@ -188,7 +189,7 @@ const PopupContainer: React.FC = ({ model, resolve }) => { } ref={inputRef} - placeholder={t('model.search')} + placeholder={t('models.search')} value={searchText} onChange={(e) => setSearchText(e.target.value)} allowClear diff --git a/src/renderer/src/config/minapps.ts b/src/renderer/src/config/minapps.ts index 223c577a..c20253ec 100644 --- a/src/renderer/src/config/minapps.ts +++ b/src/renderer/src/config/minapps.ts @@ -221,7 +221,8 @@ const _apps: MinAppType[] = [ id: 'thinkany', name: 'ThinkAny', logo: ThinkAnyLogo, - url: 'https://thinkany.ai/' + url: 'https://thinkany.ai/', + bodered: true } ] diff --git a/src/renderer/src/config/models.ts b/src/renderer/src/config/models.ts index 46882134..652f8163 100644 --- a/src/renderer/src/config/models.ts +++ b/src/renderer/src/config/models.ts @@ -151,9 +151,9 @@ export const VISION_REGEX = new RegExp( 'i' ) -const TEXT_TO_IMAGE_REGEX = /flux|diffusion|stabilityai|sd-|dall|cogview/i -const EMBEDDING_REGEX = /(?:^text-|embed|rerank|davinci|babbage|bge-|base|retrieval|uae-)/i -const NOT_SUPPORTED_REGEX = /(?:^tts|rerank|whisper|speech)/i +export const TEXT_TO_IMAGE_REGEX = /flux|diffusion|stabilityai|sd-|dall|cogview/i +export const EMBEDDING_REGEX = /(?:^text-|embed|rerank|davinci|babbage|bge-|base|retrieval|uae-|gte-)/i +export const NOT_SUPPORTED_REGEX = /(?:^tts|rerank|whisper|speech)/i export function getModelLogo(modelId: string) { const isLight = true @@ -1047,18 +1047,38 @@ export function isTextToImageModel(model: Model): boolean { } export function isEmbeddingModel(model: Model): boolean { - return EMBEDDING_REGEX.test(model.id) + if (!model) { + return false + } + + if (['anthropic'].includes(model?.provider)) { + return false + } + + return EMBEDDING_REGEX.test(model.id) || model.type?.includes('embedding') || false } export function isVisionModel(model: Model): boolean { + if (!model) { + return false + } + return VISION_REGEX.test(model.id) || model.type?.includes('vision') || false } export function isSupportedModel(model: OpenAI.Models.Model): boolean { + if (!model) { + return false + } + return !NOT_SUPPORTED_REGEX.test(model.id) } export function isWebSearchModel(model: Model): boolean { + if (!model) { + return false + } + const provider = getProviderByModel(model) if (!provider) { diff --git a/src/renderer/src/config/prompts.ts b/src/renderer/src/config/prompts.ts index 05856be2..2ccd4f73 100644 --- a/src/renderer/src/config/prompts.ts +++ b/src/renderer/src/config/prompts.ts @@ -49,3 +49,30 @@ export const SUMMARIZE_PROMPT = export const TRANSLATE_PROMPT = 'You are a translation expert. Translate from input language to {{target_language}}, provide the translation result directly without any explanation and keep original format. Do not translate if the target language is the same as the source language.' + +export const REFERENCE_PROMPT = `请根据参考资料回答问题，并使用脚注格式引用数据来源。参考资料可能和问题无关，请忽略无关的参考资料。 + +## 脚注格式： + +1. **脚注标记**：在正文中使用 [^数字] 的形式标记脚注，例如 [^1]。 +2. **脚注内容**：在文档末尾使用 [^数字]: 脚注内容 的形式定义脚注的具体内容。 + +## 脚注示例和要求： + +1. type 为 file 时：[^1]: [__name__](http://file/__url__) +2. type 为 directory 时：[^1]: [__name__](http://file/__url__) +3. type 为 url,sitemap 时：[^1]: [__name__](__url__) +4. type 为 note 时：[^1]: __note__ + +__url__ 替换成参考资料的 url +__name__ 请根据参考资料的 url 进行解析和替换 +__note__ 请根据参考资料的 content 进行总结和替换 + +## 我的问题是： + +{question} + +## 参考资料： + +{references} +` diff --git a/src/renderer/src/config/providers.ts b/src/renderer/src/config/providers.ts index 73c9635b..2600e902 100644 --- a/src/renderer/src/config/providers.ts +++ b/src/renderer/src/config/providers.ts @@ -355,11 +355,11 @@ export const PROVIDER_CONFIG = { }, aihubmix: { api: { - url: 'https://aihubmix.com' + url: 'https://aihubmix.com?aff=SJyh' }, websites: { official: 'https://aihubmix.com/', - apiKey: 'https://aihubmix.com/token', + apiKey: 'https://aihubmix.com?aff=SJyh', docs: 'https://doc.aihubmix.com/', models: 'https://aihubmix.com/models' } diff --git a/src/renderer/src/hooks/useKnowledge.ts b/src/renderer/src/hooks/useKnowledge.ts new file mode 100644 index 00000000..3c030cec --- /dev/null +++ b/src/renderer/src/hooks/useKnowledge.ts @@ -0,0 +1,279 @@ +/* eslint-disable react-hooks/rules-of-hooks */ +import { db } from '@renderer/databases/index' +import KnowledgeQueue from '@renderer/queue/KnowledgeQueue' +import FileManager from '@renderer/services/FileManager' +import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService' +import { RootState } from '@renderer/store' +import { + addBase, + addFiles as addFilesAction, + addItem, + clearAllProcessing, + clearCompletedProcessing, + deleteBase, + removeItem as removeItemAction, + renameBase, + updateBase, + updateBases, + updateItemProcessingStatus, + updateNotes +} from '@renderer/store/knowledge' +import { FileType, KnowledgeBase, ProcessingStatus } from '@renderer/types' +import { KnowledgeItem } from '@renderer/types' +import { runAsyncFunction } from '@renderer/utils' +import { useEffect, useState } from 'react' +import { useDispatch, useSelector } from 'react-redux' +import { v4 as uuidv4 } from 'uuid' + +export const useKnowledge = (baseId: string) => { + const dispatch = useDispatch() + const base = useSelector((state: RootState) => state.knowledge.bases.find((b) => b.id === baseId)) + + // 重命名知识库 + const renameKnowledgeBase = (name: string) => { + dispatch(renameBase({ baseId, name })) + } + + // 更新知识库 + const updateKnowledgeBase = (base: KnowledgeBase) => { + dispatch(updateBase(base)) + } + + // 批量添加文件 + const addFiles = (files: FileType[]) => { + const filesItems: KnowledgeItem[] = files.map((file) => ({ + id: uuidv4(), + type: 'file' as const, + content: file, + created_at: Date.now(), + updated_at: Date.now(), + processingStatus: 'pending', + processingProgress: 0, + processingError: '', + retryCount: 0 + })) + dispatch(addFilesAction({ baseId, items: filesItems })) + setTimeout(() => KnowledgeQueue.checkAllBases(), 0) + } + + // 添加URL + const addUrl = (url: string) => { + const newUrlItem: KnowledgeItem = { + id: uuidv4(), + type: 'url' as const, + content: url, + created_at: Date.now(), + updated_at: Date.now(), + processingStatus: 'pending', + processingProgress: 0, + processingError: '', + retryCount: 0 + } + dispatch(addItem({ baseId, item: newUrlItem })) + setTimeout(() => KnowledgeQueue.checkAllBases(), 0) + } + + // 添加笔记 + const addNote = async (content: string) => { + const noteId = uuidv4() + const note: KnowledgeItem = { + id: noteId, + type: 'note', + content, + created_at: Date.now(), + updated_at: Date.now() + } + + // 存储完整笔记到数据库 + await db.knowledge_notes.add(note) + + // 在 store 中只存储引用 + const noteRef: KnowledgeItem = { + id: noteId, + baseId, + type: 'note', + content: '', // store中不需要存储实际内容 + created_at: Date.now(), + updated_at: Date.now(), + processingStatus: 'pending', + processingProgress: 0, + processingError: '', + retryCount: 0 + } + + dispatch(updateNotes({ baseId, item: noteRef })) + setTimeout(() => KnowledgeQueue.checkAllBases(), 0) + } + + // 更新笔记内容 + const updateNoteContent = async (noteId: string, content: string) => { + const note = await db.knowledge_notes.get(noteId) + if (note) { + const updatedNote = { + ...note, + content, + updated_at: Date.now() + } + await db.knowledge_notes.put(updatedNote) + dispatch(updateNotes({ baseId, item: updatedNote })) + } + setTimeout(() => KnowledgeQueue.checkAllBases(), 0) + } + + // 获取笔记内容 + const getNoteContent = async (noteId: string) => { + return await db.knowledge_notes.get(noteId) + } + + // 移除项目 + const removeItem = async (item: KnowledgeItem) => { + dispatch(removeItemAction({ baseId, item })) + if (base) { + if (item?.uniqueId) { + await window.api.knowledgeBase.remove({ uniqueId: item.uniqueId, base: getKnowledgeBaseParams(base) }) + } + if (item.type === 'file' && typeof item.content === 'object') { + await FileManager.deleteFile(item.content.id) + } + } + } + + // 更新处理状态 + const updateItemStatus = (itemId: string, status: ProcessingStatus, progress?: number, error?: string) => { + dispatch( + updateItemProcessingStatus({ + baseId, + itemId, + status, + progress, + error + }) + ) + } + + // 获取特定项目的处理状态 + const getProcessingStatus = (itemId: string) => { + return base?.items.find((item) => item.id === itemId)?.processingStatus + } + + // 获取特定类型的所有处理项 + const getProcessingItemsByType = (type: 'file' | 'url' | 'note') => { + return base?.items.filter((item) => item.type === type && item.processingStatus !== undefined) || [] + } + + // 清除已完成的项目 + const clearCompleted = () => { + dispatch(clearCompletedProcessing({ baseId })) + } + + // 清除所有处理状态 + const clearAll = () => { + dispatch(clearAllProcessing({ baseId })) + } + + // 添加 Sitemap + const addSitemap = (url: string) => { + const newSitemapItem: KnowledgeItem = { + id: uuidv4(), + type: 'sitemap' as const, + content: url, + created_at: Date.now(), + updated_at: Date.now(), + processingStatus: 'pending', + processingProgress: 0, + processingError: '', + retryCount: 0 + } + dispatch(addItem({ baseId, item: newSitemapItem })) + setTimeout(() => KnowledgeQueue.checkAllBases(), 0) + } + + // Add directory support + const addDirectory = (path: string) => { + const newDirectoryItem: KnowledgeItem = { + id: uuidv4(), + type: 'directory', + content: path, + created_at: Date.now(), + updated_at: Date.now(), + processingStatus: 'pending', + processingProgress: 0, + processingError: '', + retryCount: 0 + } + dispatch(addItem({ baseId, item: newDirectoryItem })) + setTimeout(() => KnowledgeQueue.checkAllBases(), 0) + } + + const fileItems = base?.items.filter((item) => item.type === 'file') || [] + const directoryItems = base?.items.filter((item) => item.type === 'directory') || [] + const urlItems = base?.items.filter((item) => item.type === 'url') || [] + const sitemapItems = base?.items.filter((item) => item.type === 'sitemap') || [] + const [noteItems, setNoteItems] = useState([]) + + useEffect(() => { + const notes = base?.items.filter((item) => item.type === 'note') || [] + runAsyncFunction(async () => { + const newNoteItems = await Promise.all( + notes.map(async (item) => { + const note = await db.knowledge_notes.get(item.id) + return { ...item, content: note?.content || '' } + }) + ) + setNoteItems(newNoteItems.filter((note) => note !== undefined) as KnowledgeItem[]) + }) + }, [base?.items]) + + return { + base, + fileItems, + urlItems, + sitemapItems, + noteItems, + renameKnowledgeBase, + updateKnowledgeBase, + addFiles, + addUrl, + addSitemap, + addNote, + updateNoteContent, + getNoteContent, + updateItemStatus, + getProcessingStatus, + getProcessingItemsByType, + clearCompleted, + clearAll, + removeItem, + directoryItems, + addDirectory + } +} + +export const useKnowledgeBases = () => { + const dispatch = useDispatch() + const bases = useSelector((state: RootState) => state.knowledge.bases) + + const addKnowledgeBase = (base: KnowledgeBase) => { + dispatch(addBase(base)) + } + + const renameKnowledgeBase = (baseId: string, name: string) => { + dispatch(renameBase({ baseId, name })) + } + + const deleteKnowledgeBase = (baseId: string) => { + dispatch(deleteBase({ baseId })) + } + + const updateKnowledgeBases = (bases: KnowledgeBase[]) => { + dispatch(updateBases(bases)) + } + + return { + bases, + addKnowledgeBase, + renameKnowledgeBase, + deleteKnowledgeBase, + updateKnowledgeBases + } +} diff --git a/src/renderer/src/hooks/useShortcuts.ts b/src/renderer/src/hooks/useShortcuts.ts index 14283c3b..a9f80147 100644 --- a/src/renderer/src/hooks/useShortcuts.ts +++ b/src/renderer/src/hooks/useShortcuts.ts @@ -37,8 +37,10 @@ export const useShortcut = ( const shortcutConfig = shortcuts.find((s) => s.key === shortcutKey) + console.log(shortcutConfig) + useHotkeys( - shortcutConfig?.enabled ? formatShortcut(shortcutConfig.shortcut) : '', + shortcutConfig?.enabled ? formatShortcut(shortcutConfig.shortcut) : 'none', (e) => { if (options.preventDefault) { e.preventDefault() @@ -49,7 +51,8 @@ export const useShortcut = ( }, { enableOnFormTags: options.enableOnFormTags, - description: options.description || shortcutConfig?.key + description: options.description || shortcutConfig?.key, + enabled: !!shortcutConfig?.enabled } ) } diff --git a/src/renderer/src/hooks/useknowledge.ts b/src/renderer/src/hooks/useknowledge.ts index 0af8b787..3c030cec 100644 --- a/src/renderer/src/hooks/useknowledge.ts +++ b/src/renderer/src/hooks/useknowledge.ts @@ -6,6 +6,7 @@ import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService' import { RootState } from '@renderer/store' import { addBase, + addFiles as addFilesAction, addItem, clearAllProcessing, clearCompletedProcessing, @@ -13,7 +14,7 @@ import { removeItem as removeItemAction, renameBase, updateBase, - updateFiles as updateFilesAction, + updateBases, updateItemProcessingStatus, updateNotes } from '@renderer/store/knowledge' @@ -38,22 +39,20 @@ export const useKnowledge = (baseId: string) => { dispatch(updateBase(base)) } - // 添加文件列表 + // 批量添加文件 const addFiles = (files: FileType[]) => { - for (const file of files) { - const newItem: KnowledgeItem = { - id: uuidv4(), - type: 'file' as const, - content: file, - created_at: Date.now(), - updated_at: Date.now(), - processingStatus: 'pending', - processingProgress: 0, - processingError: '', - retryCount: 0 - } - dispatch(addItem({ baseId, item: newItem })) - } + const filesItems: KnowledgeItem[] = files.map((file) => ({ + id: uuidv4(), + type: 'file' as const, + content: file, + created_at: Date.now(), + updated_at: Date.now(), + processingStatus: 'pending', + processingProgress: 0, + processingError: '', + retryCount: 0 + })) + dispatch(addFilesAction({ baseId, items: filesItems })) setTimeout(() => KnowledgeQueue.checkAllBases(), 0) } @@ -106,19 +105,6 @@ export const useKnowledge = (baseId: string) => { setTimeout(() => KnowledgeQueue.checkAllBases(), 0) } - // 更新文件列表 - const updateFiles = (files: FileType[]) => { - const newItems = files.map((file) => ({ - id: uuidv4(), - type: 'file' as const, - content: file, - created_at: Date.now(), - updated_at: Date.now() - })) - dispatch(updateFilesAction({ baseId, items: newItems })) - setTimeout(() => KnowledgeQueue.checkAllBases(), 0) - } - // 更新笔记内容 const updateNoteContent = async (noteId: string, content: string) => { const note = await db.knowledge_notes.get(noteId) @@ -202,7 +188,25 @@ export const useKnowledge = (baseId: string) => { setTimeout(() => KnowledgeQueue.checkAllBases(), 0) } + // Add directory support + const addDirectory = (path: string) => { + const newDirectoryItem: KnowledgeItem = { + id: uuidv4(), + type: 'directory', + content: path, + created_at: Date.now(), + updated_at: Date.now(), + processingStatus: 'pending', + processingProgress: 0, + processingError: '', + retryCount: 0 + } + dispatch(addItem({ baseId, item: newDirectoryItem })) + setTimeout(() => KnowledgeQueue.checkAllBases(), 0) + } + const fileItems = base?.items.filter((item) => item.type === 'file') || [] + const directoryItems = base?.items.filter((item) => item.type === 'directory') || [] const urlItems = base?.items.filter((item) => item.type === 'url') || [] const sitemapItems = base?.items.filter((item) => item.type === 'sitemap') || [] const [noteItems, setNoteItems] = useState([]) @@ -232,7 +236,6 @@ export const useKnowledge = (baseId: string) => { addUrl, addSitemap, addNote, - updateFiles, updateNoteContent, getNoteContent, updateItemStatus, @@ -240,7 +243,9 @@ export const useKnowledge = (baseId: string) => { getProcessingItemsByType, clearCompleted, clearAll, - removeItem + removeItem, + directoryItems, + addDirectory } } @@ -260,10 +265,15 @@ export const useKnowledgeBases = () => { dispatch(deleteBase({ baseId })) } + const updateKnowledgeBases = (bases: KnowledgeBase[]) => { + dispatch(updateBases(bases)) + } + return { bases, addKnowledgeBase, renameKnowledgeBase, - deleteKnowledgeBase + deleteKnowledgeBase, + updateKnowledgeBases } } diff --git a/src/renderer/src/i18n/locales/en-us.json b/src/renderer/src/i18n/locales/en-us.json index d80a19a1..67be0042 100644 --- a/src/renderer/src/i18n/locales/en-us.json +++ b/src/renderer/src/i18n/locales/en-us.json @@ -252,16 +252,6 @@ "minapp": { "title": "MinApp" }, - "model": { - "pinned": "Pinned", - "search": "Search models...", - "stream_output": "Stream output", - "type": { - "select": "Select Model Types", - "text": "Text", - "vision": "Vision" - } - }, "ollama": { "keep_alive_time.description": "The time in minutes to keep the connection alive, default is 5 minutes.", "keep_alive_time.placeholder": "Minutes", @@ -532,7 +522,7 @@ "search": "Search knowledge base", "empty": "No knowledge base found", "drag_file": "Drag file here", - "file_hint": "Support pdf, docx, txt and md", + "file_hint": "Support {{file_types}}", "add": { "title": "Add Knowledge Base" }, @@ -562,7 +552,26 @@ "delete_confirm": "Are you sure you want to delete this knowledge base?", "sitemaps": "Websites", "add_sitemap": "Website Map", - "sitemap_placeholder": "Enter Website Map URL" + "sitemap_placeholder": "Enter Website Map URL", + "directories": "Directories", + "add_directory": "Add Directory", + "directory_placeholder": "Enter Directory Path" + }, + "models": { + "pinned": "Pinned", + "search": "Search models...", + "stream_output": "Stream output", + "type": { + "select": "Select Model Types", + "text": "Text", + "vision": "Vision", + "embedding": "Embedding" + }, + "all": "All", + "vision": "Vision", + "websearch": "WebSearch", + "free": "Free", + "embedding": "Embedding" } } } diff --git a/src/renderer/src/i18n/locales/ru-ru.json b/src/renderer/src/i18n/locales/ru-ru.json index f286df38..97c9bdb1 100644 --- a/src/renderer/src/i18n/locales/ru-ru.json +++ b/src/renderer/src/i18n/locales/ru-ru.json @@ -252,16 +252,6 @@ "minapp": { "title": "Встроенные приложения" }, - "model": { - "pinned": "Закреплено", - "search": "Поиск моделей...", - "stream_output": "Потоковый вывод", - "type": { - "select": "Выберите тип модели", - "text": "Текст", - "vision": "Изображение" - } - }, "ollama": { "keep_alive_time.description": "Время в минутах, в течение которого модель остается активной, по умолчанию 5 минут.", "keep_alive_time.placeholder": "Минуты", @@ -532,7 +522,7 @@ "search": "Поиск в базе знаний", "empty": "База знаний не найдена", "drag_file": "Перетащите файл сюда", - "file_hint": "Поддерживаются pdf, docx, txt и md", + "file_hint": "Поддерживаются {{file_types}}", "add": { "title": "Добавить базу знаний" }, @@ -562,7 +552,26 @@ "delete_confirm": "Вы уверены, что хотите удалить эту базу знаний?", "sitemaps": "Сайты", "add_sitemap": "Карта сайта", - "sitemap_placeholder": "Введите URL карты сайта" + "sitemap_placeholder": "Введите URL карты сайта", + "directories": "Директории", + "add_directory": "Добавить директорию", + "directory_placeholder": "Введите путь к директории" + }, + "models": { + "pinned": "Закреплено", + "search": "Поиск моделей...", + "stream_output": "Потоковый вывод", + "type": { + "select": "Выберите тип модели", + "text": "Текст", + "vision": "Изображение", + "embedding": "Встраиваемые" + }, + "all": "Все", + "vision": "Визуальные модели", + "websearch": "Веб-поисковые модели", + "free": "Бесплатные модели", + "embedding": "Встраиваемые модели" } } } diff --git a/src/renderer/src/i18n/locales/zh-cn.json b/src/renderer/src/i18n/locales/zh-cn.json index 2cf689f6..3d326447 100644 --- a/src/renderer/src/i18n/locales/zh-cn.json +++ b/src/renderer/src/i18n/locales/zh-cn.json @@ -253,16 +253,6 @@ "minapp": { "title": "小程序" }, - "model": { - "pinned": "已固定", - "search": "搜索模型...", - "stream_output": "流式输出", - "type": { - "select": "选择模型类型", - "text": "文本", - "vision": "图像" - } - }, "ollama": { "keep_alive_time.description": "对话后模型在内存中保持的时间（默认：5分钟）", "keep_alive_time.placeholder": "分钟", @@ -521,7 +511,7 @@ "search": "搜索知识库", "empty": "暂无知识库", "drag_file": "拖拽文件到这里", - "file_hint": "支持 pdf, docx, txt, md 格式", + "file_hint": "支持 {{file_types}} 格式", "add": { "title": "添加知识库" }, @@ -551,7 +541,26 @@ "delete_confirm": "确定要删除此知识库吗?", "sitemaps": "网站", "add_sitemap": "站点地图", - "sitemap_placeholder": "请输入站点地图 URL" + "sitemap_placeholder": "请输入站点地图 URL", + "directories": "目录", + "add_directory": "添加目录", + "directory_placeholder": "请输入目录路径" + }, + "models": { + "pinned": "已固定", + "search": "搜索模型...", + "stream_output": "流式输出", + "type": { + "select": "选择模型类型", + "text": "文本", + "vision": "图像", + "embedding": "嵌入" + }, + "all": "全部", + "vision": "视觉模型", + "websearch": "网络搜索模型", + "free": "免费模型", + "embedding": "嵌入模型" } } } diff --git a/src/renderer/src/i18n/locales/zh-tw.json b/src/renderer/src/i18n/locales/zh-tw.json index dc0f3121..0b520d74 100644 --- a/src/renderer/src/i18n/locales/zh-tw.json +++ b/src/renderer/src/i18n/locales/zh-tw.json @@ -252,16 +252,6 @@ "minapp": { "title": "小程序" }, - "model": { - "pinned": "已固定", - "search": "搜尋模型...", - "stream_output": "串流輸出", - "type": { - "select": "選擇模型類型", - "text": "文字", - "vision": "圖像" - } - }, "ollama": { "keep_alive_time.description": "對話後模型在記憶體中保持的時間（預設為 5 分鐘）。", "keep_alive_time.placeholder": "分鐘", @@ -520,7 +510,7 @@ "search": "搜尋知識庫", "empty": "暫無知識庫", "drag_file": "拖拽文件到這裡", - "file_hint": "支持 pdf, docx, txt, md 格式", + "file_hint": "支持 {{file_types}} 格式", "add": { "title": "添加知識庫" }, @@ -550,7 +540,26 @@ "delete_confirm": "確定要刪除此知識庫嗎?", "sitemaps": "網站", "add_sitemap": "網站地圖", - "sitemap_placeholder": "請輸入網站地圖 URL" + "sitemap_placeholder": "請輸入網站地圖 URL", + "directories": "目錄", + "add_directory": "添加目錄", + "directory_placeholder": "請輸入目錄路徑" + }, + "models": { + "pinned": "已固定", + "search": "搜尋模型...", + "stream_output": "串流輸出", + "type": { + "select": "選擇模型類型", + "text": "文字", + "vision": "圖像", + "embedding": "嵌入" + }, + "all": "全部", + "vision": "視覺模型", + "websearch": "網路搜索模型", + "free": "免費模型", + "embedding": "嵌入模型" } } } diff --git a/src/renderer/src/pages/home/Navbar.tsx b/src/renderer/src/pages/home/Navbar.tsx index 4bd08cfd..8fc8affb 100644 --- a/src/renderer/src/pages/home/Navbar.tsx +++ b/src/renderer/src/pages/home/Navbar.tsx @@ -1,4 +1,4 @@ -import { SearchOutlined } from '@ant-design/icons' +import { FormOutlined, SearchOutlined } from '@ant-design/icons' import { Navbar, NavbarLeft, NavbarRight } from '@renderer/components/app/Navbar' import AssistantSettingsPopup from '@renderer/components/AssistantSettings' import { HStack } from '@renderer/components/Layout' @@ -47,8 +47,8 @@ const HeaderNavbar: FC = ({ activeAssistant }) => { - SearchPopup.show()}> - + EventEmitter.emit(EVENT_NAMES.ADD_NEW_TOPIC)}> + )} @@ -70,6 +70,9 @@ const HeaderNavbar: FC = ({ activeAssistant }) => { + SearchPopup.show()}> + + diff --git a/src/renderer/src/pages/home/Tabs/SettingsTab.tsx b/src/renderer/src/pages/home/Tabs/SettingsTab.tsx index 5be2b38f..1654a093 100644 --- a/src/renderer/src/pages/home/Tabs/SettingsTab.tsx +++ b/src/renderer/src/pages/home/Tabs/SettingsTab.tsx @@ -162,7 +162,7 @@ const SettingsTab: FC = (props) => { - {t('model.stream_output')} + {t('models.stream_output')} = ({ selectedBase }) => { const { t } = useTranslation() const { @@ -36,13 +63,15 @@ const KnowledgeContent: FC = ({ selectedBase }) => { fileItems, urlItems, sitemapItems, + directoryItems, addFiles, updateNoteContent, addUrl, addSitemap, removeItem, getProcessingStatus, - addNote + addNote, + addDirectory } = useKnowledge(selectedBase.id || '') if (!base) { @@ -53,12 +82,10 @@ const KnowledgeContent: FC = ({ selectedBase }) => { const input = document.createElement('input') input.type = 'file' input.multiple = true - input.accept = '.pdf,.txt,.docx,.md' + input.accept = fileTypes.join(',') input.onchange = (e) => { const files = (e.target as HTMLInputElement).files - if (files) { - handleDrop(Array.from(files)) - } + files && handleDrop(Array.from(files)) } input.click() } @@ -142,6 +169,12 @@ const KnowledgeContent: FC = ({ selectedBase }) => { editedText && updateNoteContent(note.id, editedText) } + const handleAddDirectory = async () => { + const path = await window.api.file.selectFolder() + console.log('[KnowledgeContent] Selected directory:', path) + path && addDirectory(path) + } + return ( @@ -155,10 +188,12 @@ const KnowledgeContent: FC = ({ selectedBase }) => { showUploadList={false} customRequest={({ file }) => handleDrop([file as File])} multiple={true} - accept=".pdf,.txt,.docx,.md" + accept={fileTypes.join(',')} style={{ marginTop: 10, background: 'transparent' }}>

{t('knowledge_base.drag_file')}

-

{t('knowledge_base.file_hint')}

+

+ {t('knowledge_base.file_hint', { file_types: fileTypes.join(', ').replaceAll('.', '') })} +

@@ -169,19 +204,46 @@ const KnowledgeContent: FC = ({ selectedBase }) => { - - {file.origin_name} + + window.api.file.openPath(file.path)}>{file.origin_name} -
+
+ ) })} + + + {t('knowledge_base.directories')} + + + + {directoryItems.map((item) => ( + + + + + window.api.file.openPath(item.content as string)}> + {item.content as string} + + + + + -
+ {urlItems.map((item) => ( - + {item.content as string} -
+
+ ))} -
+ @@ -216,24 +278,24 @@ const KnowledgeContent: FC = ({ selectedBase }) => { {t('knowledge_base.add_sitemap')} -
+ {sitemapItems.map((item) => ( - + {item.content as string} -
+
+ ))} -
+ @@ -243,29 +305,31 @@ const KnowledgeContent: FC = ({ selectedBase }) => { {t('knowledge_base.add_note')} -
+ {noteItems.map((note) => ( handleEditNote(note)} style={{ cursor: 'pointer' }}> {(note.content as string).slice(0, 50)}... -
+
+ ))} -
+ + -
+ + ) } diff --git a/src/renderer/src/pages/knowledge/KnowledgePage.tsx b/src/renderer/src/pages/knowledge/KnowledgePage.tsx index 13331a4b..fbb54c59 100644 --- a/src/renderer/src/pages/knowledge/KnowledgePage.tsx +++ b/src/renderer/src/pages/knowledge/KnowledgePage.tsx @@ -1,9 +1,10 @@ import { DeleteOutlined, EditOutlined, FileTextOutlined, PlusOutlined } from '@ant-design/icons' import { Navbar, NavbarCenter } from '@renderer/components/app/Navbar' +import DragableList from '@renderer/components/DragableList' import ListItem from '@renderer/components/ListItem' import PromptPopup from '@renderer/components/Popups/PromptPopup' import Scrollbar from '@renderer/components/Scrollbar' -import { useKnowledgeBases } from '@renderer/hooks/useknowledge' +import { useKnowledgeBases } from '@renderer/hooks/useKnowledge' import { KnowledgeBase } from '@renderer/types' import { Dropdown, Empty, MenuProps } from 'antd' import { FC, useCallback, useEffect, useState } from 'react' @@ -15,8 +16,9 @@ import KnowledgeContent from './KnowledgeContent' const KnowledgePage: FC = () => { const { t } = useTranslation() - const { bases, renameKnowledgeBase, deleteKnowledgeBase } = useKnowledgeBases() + const { bases, renameKnowledgeBase, deleteKnowledgeBase, updateKnowledgeBases } = useKnowledgeBases() const [selectedBase, setSelectedBase] = useState() + const [isDragging, setIsDragging] = useState(false) const handleAddKnowledge = async () => { await AddKnowledgePopup.show({ title: t('knowledge_base.add.title') }) @@ -82,24 +84,33 @@ const KnowledgePage: FC = () => { - {bases.map((base) => ( - -
- } - title={base.name} - onClick={() => setSelectedBase(base)} - /> -
- - ))} - - - - {t('button.add')} - - + setIsDragging(true)} + onDragEnd={() => setIsDragging(false)}> + {(base) => ( + +
+ } + title={base.name} + onClick={() => setSelectedBase(base)} + /> +
+ + )} + + {!isDragging && ( + + + + {t('button.add')} + + + )}
diff --git a/src/renderer/src/pages/knowledge/components/AddKnowledgePopup.tsx b/src/renderer/src/pages/knowledge/components/AddKnowledgePopup.tsx index 51e681d2..7c2fdbcb 100644 --- a/src/renderer/src/pages/knowledge/components/AddKnowledgePopup.tsx +++ b/src/renderer/src/pages/knowledge/components/AddKnowledgePopup.tsx @@ -1,6 +1,6 @@ import { TopView } from '@renderer/components/TopView' import { isEmbeddingModel } from '@renderer/config/models' -import { useKnowledgeBases } from '@renderer/hooks/useknowledge' +import { useKnowledgeBases } from '@renderer/hooks/useKnowledge' import { useProviders } from '@renderer/hooks/useProvider' import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService' import { getModelUniqId } from '@renderer/services/ModelService' diff --git a/src/renderer/src/pages/knowledge/components/KnowledgeSearchPopup.tsx b/src/renderer/src/pages/knowledge/components/KnowledgeSearchPopup.tsx index b5f63140..324272a7 100644 --- a/src/renderer/src/pages/knowledge/components/KnowledgeSearchPopup.tsx +++ b/src/renderer/src/pages/knowledge/components/KnowledgeSearchPopup.tsx @@ -1,4 +1,4 @@ -import { ExtractChunkData } from '@llm-tools/embedjs-interfaces' +import type { ExtractChunkData } from '@llm-tools/embedjs-interfaces' import { TopView } from '@renderer/components/TopView' import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService' import { KnowledgeBase } from '@renderer/types' diff --git a/src/renderer/src/pages/settings/ProviderSettings/EditModelsPopup.tsx b/src/renderer/src/pages/settings/ProviderSettings/EditModelsPopup.tsx index 7e528bcf..754a3aca 100644 --- a/src/renderer/src/pages/settings/ProviderSettings/EditModelsPopup.tsx +++ b/src/renderer/src/pages/settings/ProviderSettings/EditModelsPopup.tsx @@ -1,12 +1,13 @@ import { LoadingOutlined, MinusOutlined, PlusOutlined, QuestionCircleOutlined } from '@ant-design/icons' import VisionIcon from '@renderer/components/Icons/VisionIcon' import WebSearchIcon from '@renderer/components/Icons/WebSearchIcon' +import { Center } from '@renderer/components/Layout' import { getModelLogo, isEmbeddingModel, isVisionModel, isWebSearchModel, SYSTEM_MODELS } from '@renderer/config/models' import { useProvider } from '@renderer/hooks/useProvider' import { fetchModels } from '@renderer/services/ApiService' import { Model, Provider } from '@renderer/types' import { getDefaultGroupName, isFreeModel, runAsyncFunction } from '@renderer/utils' -import { Avatar, Button, Empty, Flex, Modal, Popover, Tag } from 'antd' +import { Avatar, Button, Empty, Flex, Modal, Popover, Radio, Tag } from 'antd' import Search from 'antd/es/input/Search' import { groupBy, isEmpty, uniqBy } from 'lodash' import { useEffect, useState } from 'react' @@ -29,14 +30,29 @@ const PopupContainer: React.FC = ({ provider: _provider, resolve }) => { const [listModels, setListModels] = useState([]) const [loading, setLoading] = useState(false) const [searchText, setSearchText] = useState('') - const { t } = useTranslation() + const [filterType, setFilterType] = useState('all') + const { t, i18n } = useTranslation() const systemModels = SYSTEM_MODELS[_provider.id] || [] const allModels = uniqBy([...systemModels, ...listModels, ...models], 'id') - const list = searchText - ? allModels.filter((model) => model.id.toLocaleLowerCase().includes(searchText.toLocaleLowerCase())) - : allModels + const list = allModels.filter((model) => { + if (searchText && !model.id.toLocaleLowerCase().includes(searchText.toLocaleLowerCase())) { + return false + } + switch (filterType) { + case 'vision': + return isVisionModel(model) + case 'websearch': + return isWebSearchModel(model) + case 'free': + return isFreeModel(model) + case 'embedding': + return isEmbeddingModel(model) + default: + return true + } + }) const modelGroups = groupBy(list, 'group') @@ -89,7 +105,9 @@ const PopupContainer: React.FC = ({ provider: _provider, resolve }) => { return ( - {provider.isSystem ? t(`provider.${provider.id}`) : provider.name} {t('common.models')} + {provider.isSystem ? t(`provider.${provider.id}`) : provider.name} + {i18n.language.startsWith('zh') ? '' : ' '} + {t('common.models')} {loading && } @@ -111,6 +129,15 @@ const PopupContainer: React.FC = ({ provider: _provider, resolve }) => { }} centered> +
+ setFilterType(e.target.value)} buttonStyle="solid"> + {t('models.all')} + {t('models.vision')} + {t('models.websearch')} + {t('models.free')} + {t('models.embedding')} + +
@@ -131,12 +158,12 @@ const PopupContainer: React.FC = ({ provider: _provider, resolve }) => { {isWebSearchModel(model) && } {isFreeModel(model) && ( - Free + {t('models.free')} )} {isEmbeddingModel(model) && ( - Embed + {t('models.embedding')} )} {!isEmpty(model.description) && ( @@ -168,11 +195,16 @@ const PopupContainer: React.FC = ({ provider: _provider, resolve }) => { const SearchContainer = styled.div` display: flex; - flex-direction: row; - align-items: center; - justify-content: space-between; + flex-direction: column; + gap: 12px; padding: 0 22px; padding-bottom: 20px; + margin-top: -10px; + + .ant-radio-group { + display: flex; + flex-wrap: wrap; + } ` const ListContainer = styled.div` diff --git a/src/renderer/src/pages/settings/ProviderSettings/ProviderSetting.tsx b/src/renderer/src/pages/settings/ProviderSettings/ProviderSetting.tsx index 6b061e54..79f8228f 100644 --- a/src/renderer/src/pages/settings/ProviderSettings/ProviderSetting.tsx +++ b/src/renderer/src/pages/settings/ProviderSettings/ProviderSetting.tsx @@ -9,7 +9,7 @@ import { } from '@ant-design/icons' import VisionIcon from '@renderer/components/Icons/VisionIcon' import WebSearchIcon from '@renderer/components/Icons/WebSearchIcon' -import { getModelLogo, isVisionModel, isWebSearchModel, VISION_REGEX } from '@renderer/config/models' +import { EMBEDDING_REGEX, getModelLogo, isVisionModel, isWebSearchModel, VISION_REGEX } from '@renderer/config/models' import { PROVIDER_CONFIG } from '@renderer/config/providers' import { useTheme } from '@renderer/context/ThemeProvider' import { useAssistants, useDefaultModel } from '@renderer/hooks/useAssistant' @@ -165,7 +165,10 @@ const ProviderSetting: FC = ({ provider: _provider }) => { onUpdateModelTypes(model, types as ModelType[])} - options={[{ label: t('model.type.vision'), value: 'vision', disabled: VISION_REGEX.test(model.id) }]} + options={[ + { label: t('models.type.vision'), value: 'vision', disabled: VISION_REGEX.test(model.id) }, + { label: t('models.type.embedding'), value: 'embedding', disabled: EMBEDDING_REGEX.test(model.id) } + ]} />
) @@ -270,7 +273,7 @@ const ProviderSetting: FC = ({ provider: _provider }) => { {model.name} {isVisionModel(model) && } {isWebSearchModel(model) && } - + diff --git a/src/renderer/src/providers/BaseProvider.ts b/src/renderer/src/providers/BaseProvider.ts index c6fdc1b1..d750de97 100644 --- a/src/renderer/src/providers/BaseProvider.ts +++ b/src/renderer/src/providers/BaseProvider.ts @@ -1,7 +1,8 @@ +import { REFERENCE_PROMPT } from '@renderer/config/prompts' import { getOllamaKeepAliveTime } from '@renderer/hooks/useOllama' import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService' import store from '@renderer/store' -import { Assistant, FileType, Message, Provider, Suggestion } from '@renderer/types' +import { Assistant, Message, Provider, Suggestion } from '@renderer/types' import { delay } from '@renderer/utils' import { take } from 'lodash' import OpenAI from 'openai' @@ -88,7 +89,6 @@ export default abstract class BaseProvider { const knowledgeId = message.knowledgeBaseIds[0] const base = store.getState().knowledge.bases.find((kb) => kb.id === knowledgeId) - console.debug('knowledge', base) if (!base) { return message.content @@ -99,43 +99,20 @@ export default abstract class BaseProvider { base: getKnowledgeBaseParams(base) }) - const references = take(searchResults, 5) - .map((item, index) => { - let sourceUrl = '' - let sourceName = '' + const references = take(searchResults, 6).map((item, index) => { + const sourceUrl = item.metadata.source + const baseItem = base.items.find((i) => i.uniqueId === item.metadata.uniqueLoaderId) - const baseItem = base.items.find((i) => i.uniqueId === item.metadata.uniqueLoaderId) + return { + id: index, + content: item.pageContent, + url: sourceUrl, + type: baseItem?.type + } + }) - if (baseItem) { - switch (baseItem.type) { - case 'file': - // sourceUrl = `file://${encodeURIComponent((baseItem?.content as FileType).path)}` - sourceName = (baseItem?.content as FileType).origin_name - break - case 'url': - sourceUrl = baseItem.content as string - sourceName = baseItem.content as string - break - case 'note': - sourceName = baseItem.content as string - break - } - } + const referencesContent = JSON.stringify(references, null, 2) - return ` ---- -id: ${index} -content: ${item.pageContent} -source_type: ${baseItem?.type} -source_name: ${sourceName} -source_url: ${sourceUrl} -` - }) - .join('\n\n') - - const prompt = - '回答问题请参考以下内容，并使用类似 [^1]: source 的脚注格式引用数据来源, source 根据 source_type 决定' - - return [message.content, prompt, references].join('\n\n') + return REFERENCE_PROMPT.replace('{question}', message.content).replace('{references}', referencesContent) } } diff --git a/src/renderer/src/queue/KnowledgeQueue.ts b/src/renderer/src/queue/KnowledgeQueue.ts index 51736ff7..bab5f86e 100644 --- a/src/renderer/src/queue/KnowledgeQueue.ts +++ b/src/renderer/src/queue/KnowledgeQueue.ts @@ -1,4 +1,4 @@ -import { AddLoaderReturn } from '@llm-tools/embedjs-interfaces' +import type { AddLoaderReturn } from '@llm-tools/embedjs-interfaces' import db from '@renderer/databases' import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService' import store from '@renderer/store' @@ -98,8 +98,7 @@ class KnowledgeQueue { break } - console.log(`[KnowledgeQueue] Processing item ${item.id} (${item.type}) in base ${baseId}`) - await this.processItem(baseId, item) + this.processItem(baseId, item) } } finally { console.log(`[KnowledgeQueue] Finished processing queue for base ${baseId}`) @@ -152,25 +151,18 @@ class KnowledgeQueue { let result: AddLoaderReturn | null = null let note, content + console.log(`[KnowledgeQueue] Processing item: ${sourceItem.content}`) + switch (item.type) { - case 'file': - console.log(`[KnowledgeQueue] Processing file: ${sourceItem.content}`) - result = await window.api.knowledgeBase.add({ base: baseParams, item: sourceItem }) - break - case 'url': - console.log(`[KnowledgeQueue] Processing URL: ${sourceItem.content}`) - result = await window.api.knowledgeBase.add({ base: baseParams, item: sourceItem }) - break - case 'sitemap': - console.log(`[KnowledgeQueue] Processing Sitemap: ${sourceItem.content}`) - result = await window.api.knowledgeBase.add({ base: baseParams, item: sourceItem }) - break case 'note': - console.log(`[KnowledgeQueue] Processing note: ${sourceItem.content}`) note = await db.knowledge_notes.get(item.id) - if (!note) throw new Error(`Source note ${item.id} not found`) - content = note.content as string - result = await window.api.knowledgeBase.add({ base: baseParams, item: { ...sourceItem, content } }) + if (note) { + content = note.content as string + result = await window.api.knowledgeBase.add({ base: baseParams, item: { ...sourceItem, content } }) + } + break + default: + result = await window.api.knowledgeBase.add({ base: baseParams, item: sourceItem }) break } diff --git a/src/renderer/src/services/TokenService.ts b/src/renderer/src/services/TokenService.ts index 36dbd677..c895792e 100644 --- a/src/renderer/src/services/TokenService.ts +++ b/src/renderer/src/services/TokenService.ts @@ -1,7 +1,7 @@ import { Assistant, FileType, FileTypes, Message } from '@renderer/types' -import { GPTTokens } from 'gpt-tokens' import { flatten, takeRight } from 'lodash' import { CompletionUsage } from 'openai/resources' +import { approximateTokenSize } from 'tokenx' import { getAssistantSettings } from './AssistantService' import { filterContextMessages, filterMessages } from './MessagesService' @@ -45,12 +45,7 @@ async function getMessageParam(message: Message): Promise { } export function estimateTextTokens(text: string) { - const { usedTokens } = new GPTTokens({ - model: 'gpt-4o', - messages: [{ role: 'user', content: text }] - }) - - return usedTokens - 7 + return approximateTokenSize(text) } export function estimateImageTokens(file: FileType) { @@ -58,11 +53,6 @@ export function estimateImageTokens(file: FileType) { } export async function estimateMessageUsage(message: Message): Promise { - const { usedTokens, promptUsedTokens, completionUsedTokens } = new GPTTokens({ - model: 'gpt-4o', - messages: await getMessageParam(message) - }) - let imageTokens = 0 if (message.files) { @@ -74,10 +64,12 @@ export async function estimateMessageUsage(message: Message): Promise m.content) + .join('\n') - return usedTokens - 7 + uasageTokens + return estimateTextTokens(prompt + input) + uasageTokens } diff --git a/src/renderer/src/store/knowledge.ts b/src/renderer/src/store/knowledge.ts index 48fe5b3e..c8e3880a 100644 --- a/src/renderer/src/store/knowledge.ts +++ b/src/renderer/src/store/knowledge.ts @@ -43,12 +43,24 @@ const knowledgeSlice = createSlice({ } }, + updateBases(state, action: PayloadAction) { + state.bases = action.payload + }, + addItem(state, action: PayloadAction<{ baseId: string; item: KnowledgeItem }>) { const base = state.bases.find((b) => b.id === action.payload.baseId) if (base) { - if (action.payload.item.type === 'note') { + if (action.payload.item.type === 'file') { + action.payload.item.created_at = new Date(action.payload.item.created_at).getTime() + action.payload.item.updated_at = new Date(action.payload.item.updated_at).getTime() base.items.push(action.payload.item) } + if (action.payload.item.type === 'directory') { + const directoryExists = base.items.some((item) => item.content === action.payload.item.content) + if (!directoryExists) { + base.items.push(action.payload.item) + } + } if (action.payload.item.type === 'url') { const urlExists = base.items.some((item) => item.content === action.payload.item.content) if (!urlExists) { @@ -61,9 +73,7 @@ const knowledgeSlice = createSlice({ base.items.push(action.payload.item) } } - if (action.payload.item.type === 'file') { - action.payload.item.created_at = new Date(action.payload.item.created_at).getTime() - action.payload.item.updated_at = new Date(action.payload.item.updated_at).getTime() + if (action.payload.item.type === 'note') { base.items.push(action.payload.item) } base.updated_at = Date.now() @@ -79,12 +89,10 @@ const knowledgeSlice = createSlice({ } }, - updateFiles(state, action: PayloadAction<{ baseId: string; items: KnowledgeItem[] }>) { + addFiles(state, action: PayloadAction<{ baseId: string; items: KnowledgeItem[] }>) { const base = state.bases.find((b) => b.id === action.payload.baseId) if (base) { - // 保留非文件类型的项目 - const nonFileItems = base.items.filter((item) => item.type !== 'file') - base.items = [...nonFileItems, ...action.payload.items] + base.items = [...base.items, ...action.payload.items] base.updated_at = Date.now() } }, @@ -170,8 +178,9 @@ export const { deleteBase, renameBase, updateBase, + updateBases, addItem, - updateFiles, + addFiles, updateNotes, removeItem, updateItemProcessingStatus, diff --git a/src/renderer/src/types/index.ts b/src/renderer/src/types/index.ts index a46409a6..afbbf438 100644 --- a/src/renderer/src/types/index.ts +++ b/src/renderer/src/types/index.ts @@ -89,7 +89,7 @@ export type Provider = { export type ProviderType = 'openai' | 'anthropic' | 'gemini' -export type ModelType = 'text' | 'vision' +export type ModelType = 'text' | 'vision' | 'embedding' export type Model = { id: string @@ -183,11 +183,13 @@ export interface Shortcut { export type ProcessingStatus = 'pending' | 'processing' | 'completed' | 'failed' +export type KnowledgeItemType = 'file' | 'url' | 'note' | 'sitemap' | 'directory' + export type KnowledgeItem = { id: string baseId?: string uniqueId?: string - type: 'file' | 'url' | 'note' | 'sitemap' + type: KnowledgeItemType content: string | FileType created_at: number updated_at: number @@ -197,8 +199,6 @@ export type KnowledgeItem = { retryCount?: number } -export type KnowledgeItemType = 'file' | 'url' | 'note' | 'sitemap' - export interface KnowledgeBase { id: string name: string diff --git a/yarn.lock b/yarn.lock index f0c21ac8..6ef7ac04 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1342,101 +1342,9 @@ __metadata: languageName: node linkType: hard -"@lancedb/lancedb-darwin-arm64@npm:0.14.0": - version: 0.14.0 - resolution: "@lancedb/lancedb-darwin-arm64@npm:0.14.0" - conditions: os=darwin & cpu=arm64 - languageName: node - linkType: hard - -"@lancedb/lancedb-darwin-x64@npm:0.14.0": - version: 0.14.0 - resolution: "@lancedb/lancedb-darwin-x64@npm:0.14.0" - conditions: os=darwin & cpu=x64 - languageName: node - linkType: hard - -"@lancedb/lancedb-linux-arm64-gnu@npm:0.14.0": - version: 0.14.0 - resolution: "@lancedb/lancedb-linux-arm64-gnu@npm:0.14.0" - conditions: os=linux & cpu=arm64 & libc=glibc - languageName: node - linkType: hard - -"@lancedb/lancedb-linux-arm64-musl@npm:0.14.0": - version: 0.14.0 - resolution: "@lancedb/lancedb-linux-arm64-musl@npm:0.14.0" - conditions: os=linux & cpu=arm64 & libc=musl - languageName: node - linkType: hard - -"@lancedb/lancedb-linux-x64-gnu@npm:0.14.0": - version: 0.14.0 - resolution: "@lancedb/lancedb-linux-x64-gnu@npm:0.14.0" - conditions: os=linux & cpu=x64 & libc=glibc - languageName: node - linkType: hard - -"@lancedb/lancedb-linux-x64-musl@npm:0.14.0": - version: 0.14.0 - resolution: "@lancedb/lancedb-linux-x64-musl@npm:0.14.0" - conditions: os=linux & cpu=x64 & libc=musl - languageName: node - linkType: hard - -"@lancedb/lancedb-win32-arm64-msvc@npm:0.14.0": - version: 0.14.0 - resolution: "@lancedb/lancedb-win32-arm64-msvc@npm:0.14.0" - conditions: os=win32 & cpu=arm64 - languageName: node - linkType: hard - -"@lancedb/lancedb-win32-x64-msvc@npm:0.14.0": - version: 0.14.0 - resolution: "@lancedb/lancedb-win32-x64-msvc@npm:0.14.0" - conditions: os=win32 & cpu=x64 - languageName: node - linkType: hard - -"@lancedb/lancedb@npm:^0.14.0": - version: 0.14.0 - resolution: "@lancedb/lancedb@npm:0.14.0" - dependencies: - "@lancedb/lancedb-darwin-arm64": "npm:0.14.0" - "@lancedb/lancedb-darwin-x64": "npm:0.14.0" - "@lancedb/lancedb-linux-arm64-gnu": "npm:0.14.0" - "@lancedb/lancedb-linux-arm64-musl": "npm:0.14.0" - "@lancedb/lancedb-linux-x64-gnu": "npm:0.14.0" - "@lancedb/lancedb-linux-x64-musl": "npm:0.14.0" - "@lancedb/lancedb-win32-arm64-msvc": "npm:0.14.0" - "@lancedb/lancedb-win32-x64-msvc": "npm:0.14.0" - reflect-metadata: "npm:^0.2.2" - peerDependencies: - apache-arrow: ">=13.0.0 <=17.0.0" - dependenciesMeta: - "@lancedb/lancedb-darwin-arm64": - optional: true - "@lancedb/lancedb-darwin-x64": - optional: true - "@lancedb/lancedb-linux-arm64-gnu": - optional: true - "@lancedb/lancedb-linux-arm64-musl": - optional: true - "@lancedb/lancedb-linux-x64-gnu": - optional: true - "@lancedb/lancedb-linux-x64-musl": - optional: true - "@lancedb/lancedb-win32-arm64-msvc": - optional: true - "@lancedb/lancedb-win32-x64-msvc": - optional: true - conditions: (os=darwin | os=linux | os=win32) & (cpu=x64 | cpu=arm64) - languageName: node - linkType: hard - -"@langchain/core@npm:^0.3.23": - version: 0.3.24 - resolution: "@langchain/core@npm:0.3.24" +"@langchain/core@npm:^0.3.25": + version: 0.3.26 + resolution: "@langchain/core@npm:0.3.26" dependencies: "@cfworker/json-schema": "npm:^4.0.2" ansi-styles: "npm:^5.0.0" @@ -1450,23 +1358,11 @@ __metadata: uuid: "npm:^10.0.0" zod: "npm:^3.22.4" zod-to-json-schema: "npm:^3.22.3" - checksum: 10c0/40c66a95eda211b651df41fdb0177befbc9029c9586e458e409f58ea0b40c441599dd27cbf020c3de4b501649ee2542a9c594cf0d4b73b1cb451feaeaaaa5aa2 + checksum: 10c0/9f2621d8461e57c233d2f9ab3a59afaf55e7ac0f8eea527343910c3f2cd0ab63944326b152e2d77692b0efaf65fb57c6d235371ce8cb98ac142547ac119e5b73 languageName: node linkType: hard -"@langchain/ollama@npm:^0.1.3": - version: 0.1.4 - resolution: "@langchain/ollama@npm:0.1.4" - dependencies: - ollama: "npm:^0.5.9" - uuid: "npm:^10.0.0" - peerDependencies: - "@langchain/core": ">=0.2.21 <0.4.0" - checksum: 10c0/713fadb9785255aac8354a92942922294129c590c31a44a843fcf9cd09bd99bef168bbd7a8bfd4bae75af21a2148711b2c7ba0a155864008c6cfd97b31d87185 - languageName: node - linkType: hard - -"@langchain/openai@npm:>=0.1.0 <0.4.0, @langchain/openai@npm:^0.3.14": +"@langchain/openai@npm:>=0.1.0 <0.4.0": version: 0.3.14 resolution: "@langchain/openai@npm:0.3.14" dependencies: @@ -1480,6 +1376,20 @@ __metadata: languageName: node linkType: hard +"@langchain/openai@npm:^0.3.15": + version: 0.3.16 + resolution: "@langchain/openai@npm:0.3.16" + dependencies: + js-tiktoken: "npm:^1.0.12" + openai: "npm:^4.77.0" + zod: "npm:^3.22.4" + zod-to-json-schema: "npm:^3.22.3" + peerDependencies: + "@langchain/core": ">=0.2.26 <0.4.0" + checksum: 10c0/5955a02c09227d8d1d7feef26d3487cf151e2c3d36ec7550c4fe111179eb78de76befd1bd2df6a80ae4fc88676f5ebaa35d5d8788faab62972d82989ca18ec87 + languageName: node + linkType: hard + "@langchain/textsplitters@npm:>=0.0.0 <0.2.0, @langchain/textsplitters@npm:^0.1.0": version: 0.1.0 resolution: "@langchain/textsplitters@npm:0.1.0" @@ -1491,169 +1401,257 @@ __metadata: languageName: node linkType: hard -"@llm-tools/embedjs-interfaces@npm:0.1.24": - version: 0.1.24 - resolution: "@llm-tools/embedjs-interfaces@npm:0.1.24" +"@libsql/client@npm:^0.14.0": + version: 0.14.0 + resolution: "@libsql/client@npm:0.14.0" dependencies: - "@langchain/core": "npm:^0.3.23" + "@libsql/core": "npm:^0.14.0" + "@libsql/hrana-client": "npm:^0.7.0" + js-base64: "npm:^3.7.5" + libsql: "npm:^0.4.4" + promise-limit: "npm:^2.7.0" + checksum: 10c0/9c6bab468453df765f647422c772af3578f1e108b663a80b99063f47ed3542db26ae0fcdba2e153d72e6d5089c5caeba947a167a6c065b0191a0832621539335 + languageName: node + linkType: hard + +"@libsql/core@npm:^0.14.0": + version: 0.14.0 + resolution: "@libsql/core@npm:0.14.0" + dependencies: + js-base64: "npm:^3.7.5" + checksum: 10c0/327bb991cf191d5a9a9fc0cc1a17123f7ca88f222187a3bde845fbad8ceaeaa1f139882080e4b2969da57b83e576c52702572e2838d1743c6bff75f95e6f774a + languageName: node + linkType: hard + +"@libsql/darwin-arm64@npm:0.4.7": + version: 0.4.7 + resolution: "@libsql/darwin-arm64@npm:0.4.7" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@libsql/darwin-x64@npm:0.4.7": + version: 0.4.7 + resolution: "@libsql/darwin-x64@npm:0.4.7" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@libsql/hrana-client@npm:^0.7.0": + version: 0.7.0 + resolution: "@libsql/hrana-client@npm:0.7.0" + dependencies: + "@libsql/isomorphic-fetch": "npm:^0.3.1" + "@libsql/isomorphic-ws": "npm:^0.1.5" + js-base64: "npm:^3.7.5" + node-fetch: "npm:^3.3.2" + checksum: 10c0/85bbe3074e3efcc56fbc33a741afd06c1211f5d7772ea46155ce6fe6578e31ed517c7017510c5bfea3dc28031e95c0e284096154f28cbbab06751fd7d9437e28 + languageName: node + linkType: hard + +"@libsql/isomorphic-fetch@npm:^0.3.1": + version: 0.3.1 + resolution: "@libsql/isomorphic-fetch@npm:0.3.1" + checksum: 10c0/f415f23f0c9921d6aae5133f91194d85a70e555b6c931fb4ff272af94335a46f5893433fbbb8fa24ee45bab8acce8ba5f060a22567ff3fb0577883bfcd1dec9e + languageName: node + linkType: hard + +"@libsql/isomorphic-ws@npm:^0.1.5": + version: 0.1.5 + resolution: "@libsql/isomorphic-ws@npm:0.1.5" + dependencies: + "@types/ws": "npm:^8.5.4" + ws: "npm:^8.13.0" + checksum: 10c0/7028bbc50dd094cdcbe56714dbf52fb646812d1b042c1973e61293f4a1cb5b81d5af670530a2463a2ba485f84f7728daf3eb75d40a7f55316ee4f7015dcc99ae + languageName: node + linkType: hard + +"@libsql/linux-arm64-gnu@npm:0.4.7": + version: 0.4.7 + resolution: "@libsql/linux-arm64-gnu@npm:0.4.7" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@libsql/linux-arm64-musl@npm:0.4.7": + version: 0.4.7 + resolution: "@libsql/linux-arm64-musl@npm:0.4.7" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@libsql/linux-x64-gnu@npm:0.4.7": + version: 0.4.7 + resolution: "@libsql/linux-x64-gnu@npm:0.4.7" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@libsql/linux-x64-musl@npm:0.4.7": + version: 0.4.7 + resolution: "@libsql/linux-x64-musl@npm:0.4.7" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@libsql/win32-x64-msvc@npm:0.4.7": + version: 0.4.7 + resolution: "@libsql/win32-x64-msvc@npm:0.4.7" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"@llm-tools/embedjs-interfaces@npm:0.1.25": + version: 0.1.25 + resolution: "@llm-tools/embedjs-interfaces@npm:0.1.25" + dependencies: + "@langchain/core": "npm:^0.3.25" debug: "npm:^4.4.0" md5: "npm:^2.3.0" uuid: "npm:^11.0.3" - checksum: 10c0/f328f04c83167868cd4c7fbae6733157d480e83f0688fc0babb1c0c65b4f9f5450a038831d604d4341e10c2463ed7b2dbbc5a94f4b113fadf24fe7d1204a4fbe + checksum: 10c0/0aea17be12d659a73425c3c7b879c658fda3f9d4bcc207f3df8a5be06386013a4ebc29fbefb00e6bb0b021f06f7daad4bccb3cfa708a027974fe48739c5d8bf3 languageName: node linkType: hard -"@llm-tools/embedjs-lancedb@npm:^0.1.24": - version: 0.1.24 - resolution: "@llm-tools/embedjs-lancedb@npm:0.1.24" +"@llm-tools/embedjs-libsql@npm:^0.1.25": + version: 0.1.25 + resolution: "@llm-tools/embedjs-libsql@npm:0.1.25" dependencies: - "@lancedb/lancedb": "npm:^0.14.0" - "@llm-tools/embedjs-interfaces": "npm:0.1.24" - compute-cosine-similarity: "npm:^1.1.0" + "@libsql/client": "npm:^0.14.0" + "@llm-tools/embedjs-interfaces": "npm:0.1.25" + "@llm-tools/embedjs-utils": "npm:0.1.25" debug: "npm:^4.4.0" - checksum: 10c0/814ecb27a06fccd61440e4f83461ef1aa791fd7ef3f387c6d3be600a23853fca7fe6724f7e6c76dc24c1271f4202f728515c48d25610806f64cc6c3368284cbb + checksum: 10c0/7a21019ed4d7cb1f8eb21ae9015c31b2e9d5ee517e741b88d5660fb322fc0d6bc163070034036c58e0fc67437dd39aeb3aa30b9874ef1dd553a3c5a2c5bbbe55 languageName: node linkType: hard -"@llm-tools/embedjs-loader-csv@npm:^0.1.24": - version: 0.1.24 - resolution: "@llm-tools/embedjs-loader-csv@npm:0.1.24" +"@llm-tools/embedjs-loader-csv@npm:^0.1.25": + version: 0.1.25 + resolution: "@llm-tools/embedjs-loader-csv@npm:0.1.25" dependencies: - "@llm-tools/embedjs-interfaces": "npm:0.1.24" - "@llm-tools/embedjs-utils": "npm:0.1.24" + "@llm-tools/embedjs-interfaces": "npm:0.1.25" + "@llm-tools/embedjs-utils": "npm:0.1.25" csv-parse: "npm:^5.6.0" debug: "npm:^4.4.0" md5: "npm:^2.3.0" - checksum: 10c0/cd41c873bdb4f1f3ab3b7ea0ef3a592793f469400e5a5fb81e9b17d4cdc87e35674fe2d99266c10d6faba72204dcb0be2c8143f7ae223ec5062939aeb7badd97 + checksum: 10c0/bd876c7c8d48fe5873a198b8746caeb216b3dcb420370343f57d3bab586c88c37f46096f72b3ea1755ec6b093dd2bb8c52ce77002f74c447934fb3ecb816ba0c languageName: node linkType: hard -"@llm-tools/embedjs-loader-markdown@npm:^0.1.24": - version: 0.1.24 - resolution: "@llm-tools/embedjs-loader-markdown@npm:0.1.24" +"@llm-tools/embedjs-loader-markdown@npm:^0.1.25": + version: 0.1.25 + resolution: "@llm-tools/embedjs-loader-markdown@npm:0.1.25" dependencies: - "@llm-tools/embedjs-interfaces": "npm:0.1.24" - "@llm-tools/embedjs-loader-web": "npm:0.1.24" + "@llm-tools/embedjs-interfaces": "npm:0.1.25" + "@llm-tools/embedjs-loader-web": "npm:0.1.25" debug: "npm:^4.4.0" md5: "npm:^2.3.0" micromark: "npm:^4.0.1" micromark-extension-gfm: "npm:^3.0.0" micromark-extension-mdx-jsx: "npm:^3.0.1" - checksum: 10c0/9ad8fe9b41d5f18be853e75d2b1e409c76ca077c0a6d8ff9fee896ba0c987f93bc4eadb77996a42e200891b27491d37b09830fa3caae8afe70a427c899aea3d0 + checksum: 10c0/9cc540a654ece8d108766962017225167d324c40fd1911b1ccd5b749820a3b4e475ca008ef02fc0094bf698de6a87cda672168c4741d2aabb549380b0b64937c languageName: node linkType: hard -"@llm-tools/embedjs-loader-msoffice@npm:^0.1.24": - version: 0.1.24 - resolution: "@llm-tools/embedjs-loader-msoffice@npm:0.1.24" +"@llm-tools/embedjs-loader-msoffice@npm:^0.1.25": + version: 0.1.25 + resolution: "@llm-tools/embedjs-loader-msoffice@npm:0.1.25" dependencies: "@langchain/textsplitters": "npm:^0.1.0" - "@llm-tools/embedjs-interfaces": "npm:0.1.24" - "@llm-tools/embedjs-utils": "npm:0.1.24" + "@llm-tools/embedjs-interfaces": "npm:0.1.25" + "@llm-tools/embedjs-utils": "npm:0.1.25" md5: "npm:^2.3.0" office-text-extractor: "npm:^3.0.3" - checksum: 10c0/d541c2623d2b3c2891c041a83b16e042c1d1d9b7ae8ca80a155e0a86a04389dd49d868443773da470bf337d9dfaa68dcc317db9708c68a034e5960864ad9b66c + checksum: 10c0/0190548a942a35c319fcccc4e82b3bd85a81acd3be043d82df37523355087fbbada35b34320c33ce82b1353f5cd1c555edb7861e4b18d8d037918efdf26f3fff languageName: node linkType: hard -"@llm-tools/embedjs-loader-pdf@npm:^0.1.24": - version: 0.1.24 - resolution: "@llm-tools/embedjs-loader-pdf@npm:0.1.24" +"@llm-tools/embedjs-loader-pdf@npm:^0.1.25": + version: 0.1.25 + resolution: "@llm-tools/embedjs-loader-pdf@npm:0.1.25" dependencies: "@langchain/textsplitters": "npm:^0.1.0" - "@llm-tools/embedjs-interfaces": "npm:0.1.24" - "@llm-tools/embedjs-utils": "npm:0.1.24" + "@llm-tools/embedjs-interfaces": "npm:0.1.25" + "@llm-tools/embedjs-utils": "npm:0.1.25" md5: "npm:^2.3.0" office-text-extractor: "npm:^3.0.3" - checksum: 10c0/6aeb4debd0509aeb1bea16321ab22c2630cf54f26aa8446a4dbbb95a5f963bb05802a7983800ee5740b38deadf77a67c353713b6b366666ee0a04ca48ec0a55c + checksum: 10c0/ef70889166a6a6fc372fbf8a0b640acdb4cc0dea7f18f4729867a6edfd54210b2625dd8ec716d4f701ce3219eddd1d38214247e2b9e433f597779ecc86004972 languageName: node linkType: hard -"@llm-tools/embedjs-loader-sitemap@npm:^0.1.24": - version: 0.1.24 - resolution: "@llm-tools/embedjs-loader-sitemap@npm:0.1.24" +"@llm-tools/embedjs-loader-sitemap@npm:^0.1.25": + version: 0.1.25 + resolution: "@llm-tools/embedjs-loader-sitemap@npm:0.1.25" dependencies: - "@llm-tools/embedjs-interfaces": "npm:0.1.24" - "@llm-tools/embedjs-loader-web": "npm:0.1.24" + "@llm-tools/embedjs-interfaces": "npm:0.1.25" + "@llm-tools/embedjs-loader-web": "npm:0.1.25" debug: "npm:^4.4.0" md5: "npm:^2.3.0" sitemapper: "npm:^3.2.18" - checksum: 10c0/afda59d2f2e60cafd1398bf28b45ded8abc3849e354f85afd1b37c17cdc16d9483627352a85f90d6ecc52ff6aaa5a5c52bc6ebbfafdbc30b8005b26f6a95d5b9 + checksum: 10c0/f415bd130f6a3cf336720c6ea30c942f82e537e1334d990fe35b7ef1f97cc07a6d3a0cf576d891a2ff07a1ef6e71581b82eac10de0ff589393ee4a86313a99bb languageName: node linkType: hard -"@llm-tools/embedjs-loader-web@npm:0.1.24, @llm-tools/embedjs-loader-web@npm:^0.1.24": - version: 0.1.24 - resolution: "@llm-tools/embedjs-loader-web@npm:0.1.24" +"@llm-tools/embedjs-loader-web@npm:0.1.25, @llm-tools/embedjs-loader-web@npm:^0.1.25": + version: 0.1.25 + resolution: "@llm-tools/embedjs-loader-web@npm:0.1.25" dependencies: "@langchain/textsplitters": "npm:^0.1.0" - "@llm-tools/embedjs-interfaces": "npm:0.1.24" - "@llm-tools/embedjs-utils": "npm:0.1.24" + "@llm-tools/embedjs-interfaces": "npm:0.1.25" + "@llm-tools/embedjs-utils": "npm:0.1.25" debug: "npm:^4.4.0" html-to-text: "npm:^9.0.5" md5: "npm:^2.3.0" - checksum: 10c0/82a701b73a60fad0e3a34c53ff909460901f0d3a59fd9935c02b3f4eb3508131367d49a31096a9f1b9a75979f857572af805bcbf17bb2f3579f8a3786cdccd9a + checksum: 10c0/abbf9cc252fb928745a04152fedafc03c07eef03190e10cca830dc41e8a24b35055cb2a9011633311dcf069ffa111582b65c592e1a0ce0ae84f1b4d5e468d8b2 languageName: node linkType: hard -"@llm-tools/embedjs-loader-xml@npm:^0.1.24": - version: 0.1.24 - resolution: "@llm-tools/embedjs-loader-xml@npm:0.1.24" +"@llm-tools/embedjs-loader-xml@npm:^0.1.25": + version: 0.1.25 + resolution: "@llm-tools/embedjs-loader-xml@npm:0.1.25" dependencies: - "@llm-tools/embedjs-interfaces": "npm:0.1.24" + "@llm-tools/embedjs-interfaces": "npm:0.1.25" debug: "npm:^4.4.0" - fast-xml-parser: "npm:^4.5.0" + fast-xml-parser: "npm:^4.5.1" md5: "npm:^2.3.0" - checksum: 10c0/beca4d85d3652c16aeab97d9ef56ad68007db0fcf496afdf602539c44a6d3e35ace17ede3337d25276b4ffcb338bd888ebf7f157a3b9e352c43437c1bf8f691f + checksum: 10c0/6407f0c3651c738093437ed251dfff05135ae5a48d3e079988808a94d13a89160a0b1aac63943635cd57c3082e77408f5947232bec0b489b07829fc272628cd7 languageName: node linkType: hard -"@llm-tools/embedjs-ollama@npm:^0.1.24": - version: 0.1.24 - resolution: "@llm-tools/embedjs-ollama@npm:0.1.24" +"@llm-tools/embedjs-openai@npm:^0.1.25": + version: 0.1.25 + resolution: "@llm-tools/embedjs-openai@npm:0.1.25" dependencies: - "@langchain/core": "npm:^0.3.23" - "@langchain/ollama": "npm:^0.1.3" - "@llm-tools/embedjs-interfaces": "npm:0.1.24" + "@langchain/core": "npm:^0.3.25" + "@langchain/openai": "npm:^0.3.15" + "@llm-tools/embedjs-interfaces": "npm:0.1.25" debug: "npm:^4.4.0" - checksum: 10c0/59017c7d8492f68567e30553bef933408bd05caf0a4f711b6302e3f6ab26a0cdceac7b2a1d02b4426f26d6a16a2154e5d8be3853909a91c5909fc9581d6a452c + checksum: 10c0/38b1b7a9a7893d462d645af5c24634e65d350836b3d3046956d83b2be4365fde629194abdf28b3df97160ac11c10613e318499b827804e603f970cdbfada1092 languageName: node linkType: hard -"@llm-tools/embedjs-openai@npm:^0.1.24": - version: 0.1.24 - resolution: "@llm-tools/embedjs-openai@npm:0.1.24" +"@llm-tools/embedjs-utils@npm:0.1.25": + version: 0.1.25 + resolution: "@llm-tools/embedjs-utils@npm:0.1.25" dependencies: - "@langchain/core": "npm:^0.3.23" - "@langchain/openai": "npm:^0.3.14" - "@llm-tools/embedjs-interfaces": "npm:0.1.24" - debug: "npm:^4.4.0" - checksum: 10c0/447139447886303347a8f20196c0356620a7b0780fe18181730bc41f6754b7b1c910e5c2f87b8c57e2cd1d94a8a64e71aeee5fd118080d478eea32e930b6c171 + "@llm-tools/embedjs-interfaces": "npm:0.1.25" + checksum: 10c0/8220181ac574e1088562f1c7913709806e6558ab02a535b888abd6267a4a03f9a45984998fddf07b4930c91a7f829458754f1da7488f453aee824de08958705a languageName: node linkType: hard -"@llm-tools/embedjs-utils@npm:0.1.24": - version: 0.1.24 - resolution: "@llm-tools/embedjs-utils@npm:0.1.24" - dependencies: - "@llm-tools/embedjs-interfaces": "npm:0.1.24" - checksum: 10c0/cb391b5f9b1f79d89a867a8270ca6f391090ff9a3aa099880c7fff77f22359a4b73bbb9802037cbf848eed1dbf96dd8402ff6893f1faba0f6d40774345207a21 - languageName: node - linkType: hard - -"@llm-tools/embedjs@npm:^0.1.24": - version: 0.1.24 - resolution: "@llm-tools/embedjs@npm:0.1.24" +"@llm-tools/embedjs@npm:^0.1.25": + version: 0.1.25 + resolution: "@llm-tools/embedjs@npm:0.1.25" dependencies: "@langchain/textsplitters": "npm:^0.1.0" - "@llm-tools/embedjs-interfaces": "npm:0.1.24" - "@llm-tools/embedjs-utils": "npm:0.1.24" + "@llm-tools/embedjs-interfaces": "npm:0.1.25" + "@llm-tools/embedjs-utils": "npm:0.1.25" debug: "npm:^4.4.0" langchain: "npm:^0.3.7" md5: "npm:^2.3.0" - mime: "npm:^4.0.4" + mime: "npm:^4.0.6" stream-mime-type: "npm:^2.0.0" - checksum: 10c0/306faad5a9b1096a500a15b3471fabb05a16e7f3b1335a5aebdb39d422e70e1a0486473b531ebc590a77b38f82bfc4d492e840015b651020a6eb721b850fed46 + checksum: 10c0/8a6afefbd8e304c037104eddf0fe35086165eb3a381b866e874bf6901c2da8a6dfc76d3d7eb027f7c694a163133154e3be7be777c362119a48925a6b56cce5db languageName: node linkType: hard @@ -1678,6 +1676,13 @@ __metadata: languageName: node linkType: hard +"@neon-rs/load@npm:^0.0.4": + version: 0.0.4 + resolution: "@neon-rs/load@npm:0.0.4" + checksum: 10c0/546fa4e48aa9cdb402f0a3524b591b1cac863bcfdd0217432323dba42ad37ece24b736019e6196e34326201db6b6deb410d7a983ac3c54f322619c9b6bd568bb + languageName: node + linkType: hard + "@nodelib/fs.scandir@npm:2.1.5": version: 2.1.5 resolution: "@nodelib/fs.scandir@npm:2.1.5" @@ -2582,6 +2587,15 @@ __metadata: languageName: node linkType: hard +"@types/ws@npm:^8.5.4": + version: 8.5.13 + resolution: "@types/ws@npm:8.5.13" + dependencies: + "@types/node": "npm:*" + checksum: 10c0/a5430aa479bde588e69cb9175518d72f9338b6999e3b2ae16fc03d3bdcff8347e486dc031e4ed14601260463c07e1f9a0d7511dfc653712b047c439c680b0b34 + languageName: node + linkType: hard + "@types/yauzl@npm:^2.9.1": version: 2.10.3 resolution: "@types/yauzl@npm:2.10.3" @@ -2757,17 +2771,16 @@ __metadata: "@google/generative-ai": "npm:^0.21.0" "@hello-pangea/dnd": "npm:^16.6.0" "@kangfenmao/keyv-storage": "npm:^0.1.0" - "@llm-tools/embedjs": "npm:^0.1.24" - "@llm-tools/embedjs-lancedb": "npm:^0.1.24" - "@llm-tools/embedjs-loader-csv": "npm:^0.1.24" - "@llm-tools/embedjs-loader-markdown": "npm:^0.1.24" - "@llm-tools/embedjs-loader-msoffice": "npm:^0.1.24" - "@llm-tools/embedjs-loader-pdf": "npm:^0.1.24" - "@llm-tools/embedjs-loader-sitemap": "npm:^0.1.24" - "@llm-tools/embedjs-loader-web": "npm:^0.1.24" - "@llm-tools/embedjs-loader-xml": "npm:^0.1.24" - "@llm-tools/embedjs-ollama": "npm:^0.1.24" - "@llm-tools/embedjs-openai": "npm:^0.1.24" + "@llm-tools/embedjs": "npm:^0.1.25" + "@llm-tools/embedjs-libsql": "npm:^0.1.25" + "@llm-tools/embedjs-loader-csv": "npm:^0.1.25" + "@llm-tools/embedjs-loader-markdown": "npm:^0.1.25" + "@llm-tools/embedjs-loader-msoffice": "npm:^0.1.25" + "@llm-tools/embedjs-loader-pdf": "npm:^0.1.25" + "@llm-tools/embedjs-loader-sitemap": "npm:^0.1.25" + "@llm-tools/embedjs-loader-web": "npm:^0.1.25" + "@llm-tools/embedjs-loader-xml": "npm:^0.1.25" + "@llm-tools/embedjs-openai": "npm:^0.1.25" "@reduxjs/toolkit": "npm:^2.2.5" "@types/adm-zip": "npm:^0" "@types/fs-extra": "npm:^11" @@ -2806,7 +2819,6 @@ __metadata: eslint-plugin-simple-import-sort: "npm:^12.1.1" eslint-plugin-unused-imports: "npm:^4.0.0" fs-extra: "npm:^11.2.0" - gpt-tokens: "npm:^1.3.10" html2canvas: "npm:^1.4.1" i18next: "npm:^23.11.5" lodash: "npm:^4.17.21" @@ -2832,10 +2844,12 @@ __metadata: rehype-raw: "npm:^7.0.0" remark-gfm: "npm:^4.0.0" remark-math: "npm:^6.0.0" + rollup-plugin-visualizer: "npm:^5.12.0" sass: "npm:^1.77.2" shiki: "npm:^1.22.2" styled-components: "npm:^6.1.11" tinycolor2: "npm:^1.6.0" + tokenx: "npm:^0.4.1" typescript: "npm:^5.6.2" uuid: "npm:^10.0.0" vite: "npm:^5.0.12" @@ -4208,38 +4222,6 @@ __metadata: languageName: node linkType: hard -"compute-cosine-similarity@npm:^1.1.0": - version: 1.1.0 - resolution: "compute-cosine-similarity@npm:1.1.0" - dependencies: - compute-dot: "npm:^1.1.0" - compute-l2norm: "npm:^1.1.0" - validate.io-array: "npm:^1.0.5" - validate.io-function: "npm:^1.0.2" - checksum: 10c0/82c5df828f0c3af2cf74e400916e904560fdbe61d740cb91f03b191f372408c439ab0afce8c053f6ca3fcf74d2f04bf0083e5b8b07fafceaaf50a6e35d191cb3 - languageName: node - linkType: hard - -"compute-dot@npm:^1.1.0": - version: 1.1.0 - resolution: "compute-dot@npm:1.1.0" - dependencies: - validate.io-array: "npm:^1.0.3" - validate.io-function: "npm:^1.0.2" - checksum: 10c0/1e81ab6a18d9fd0fae9265e501d00fe303d1305c7370db531d38ca35e09f9ceba66bec27c42992abd2f271f104f084db93a56e7f49424f4a4aea5c8265881d93 - languageName: node - linkType: hard - -"compute-l2norm@npm:^1.1.0": - version: 1.1.0 - resolution: "compute-l2norm@npm:1.1.0" - dependencies: - validate.io-array: "npm:^1.0.3" - validate.io-function: "npm:^1.0.2" - checksum: 10c0/af37fca05e662f221d73f41004972d95998c6927c666b95b9279466f34a06e374ae817194cbadd5c28d0e99a84526022b66f7d16e6e8e5cf9f909629ae381a93 - languageName: node - linkType: hard - "compute-scroll-into-view@npm:^3.0.2": version: 3.1.0 resolution: "compute-scroll-into-view@npm:3.1.0" @@ -4447,6 +4429,13 @@ __metadata: languageName: node linkType: hard +"data-uri-to-buffer@npm:^4.0.0": + version: 4.0.1 + resolution: "data-uri-to-buffer@npm:4.0.1" + checksum: 10c0/20a6b93107597530d71d4cb285acee17f66bcdfc03fd81040921a81252f19db27588d87fc8fc69e1950c55cfb0bf8ae40d0e5e21d907230813eb5d5a7f9eb45b + languageName: node + linkType: hard + "data-urls@npm:^5.0.0": version: 5.0.0 resolution: "data-urls@npm:5.0.0" @@ -4682,6 +4671,13 @@ __metadata: languageName: node linkType: hard +"define-lazy-prop@npm:^2.0.0": + version: 2.0.0 + resolution: "define-lazy-prop@npm:2.0.0" + checksum: 10c0/db6c63864a9d3b7dc9def55d52764968a5af296de87c1b2cc71d8be8142e445208071953649e0386a8cc37cfcf9a2067a47207f1eb9ff250c2a269658fdae422 + languageName: node + linkType: hard + "define-properties@npm:^1.1.3, define-properties@npm:^1.2.0, define-properties@npm:^1.2.1": version: 1.2.1 resolution: "define-properties@npm:1.2.1" @@ -4730,6 +4726,13 @@ __metadata: languageName: node linkType: hard +"detect-libc@npm:2.0.2": + version: 2.0.2 + resolution: "detect-libc@npm:2.0.2" + checksum: 10c0/a9f4ffcd2701525c589617d98afe5a5d0676c8ea82bcc4ed6f3747241b79f781d36437c59a5e855254c864d36a3e9f8276568b6b531c28d6e53b093a15703f11 + languageName: node + linkType: hard + "detect-libc@npm:^2.0.0": version: 2.0.3 resolution: "detect-libc@npm:2.0.3" @@ -5843,6 +5846,17 @@ __metadata: languageName: node linkType: hard +"fast-xml-parser@npm:^4.5.1": + version: 4.5.1 + resolution: "fast-xml-parser@npm:4.5.1" + dependencies: + strnum: "npm:^1.0.5" + bin: + fxparser: src/cli/cli.js + checksum: 10c0/70c6c675770d57d4b73716a1cdccff3780a5f818cffdab9c7560003e1724209001af32fbe7bb27a01107389b1998191c62e20104788ba17e218dfe063aa15b57 + languageName: node + linkType: hard + "fastq@npm:^1.6.0": version: 1.17.1 resolution: "fastq@npm:1.17.1" @@ -5861,6 +5875,16 @@ __metadata: languageName: node linkType: hard +"fetch-blob@npm:^3.1.2, fetch-blob@npm:^3.1.4": + version: 3.2.0 + resolution: "fetch-blob@npm:3.2.0" + dependencies: + node-domexception: "npm:^1.0.0" + web-streams-polyfill: "npm:^3.0.3" + checksum: 10c0/60054bf47bfa10fb0ba6cb7742acec2f37c1f56344f79a70bb8b1c48d77675927c720ff3191fa546410a0442c998d27ab05e9144c32d530d8a52fbe68f843b69 + languageName: node + linkType: hard + "fflate@npm:0.8.1": version: 0.8.1 resolution: "fflate@npm:0.8.1" @@ -6108,6 +6132,15 @@ __metadata: languageName: node linkType: hard +"formdata-polyfill@npm:^4.0.10": + version: 4.0.10 + resolution: "formdata-polyfill@npm:4.0.10" + dependencies: + fetch-blob: "npm:^3.1.2" + checksum: 10c0/5392ec484f9ce0d5e0d52fb5a78e7486637d516179b0eb84d81389d7eccf9ca2f663079da56f761355c0a65792810e3b345dc24db9a8bbbcf24ef3c8c88570c6 + languageName: node + linkType: hard + "fs-constants@npm:^1.0.0": version: 1.0.0 resolution: "fs-constants@npm:1.0.0" @@ -6513,17 +6546,6 @@ __metadata: languageName: node linkType: hard -"gpt-tokens@npm:^1.3.10": - version: 1.3.10 - resolution: "gpt-tokens@npm:1.3.10" - dependencies: - decimal.js: "npm:^10.4.3" - js-tiktoken: "npm:^1.0.14" - openai-chat-tokens: "npm:^0.2.8" - checksum: 10c0/9aa83bf1aecc3a11b8557769fa20d0f9daae61e3e79e1e308e2435069cee9c49f06563f68d9ce90b53c4981e84c92bf9bb43168f042e4606b51ccfce9210f95c - languageName: node - linkType: hard - "graceful-fs@npm:^4.1.10, graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.1.9, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.4, graceful-fs@npm:^4.2.6": version: 4.2.11 resolution: "graceful-fs@npm:4.2.11" @@ -7342,6 +7364,15 @@ __metadata: languageName: node linkType: hard +"is-docker@npm:^2.0.0, is-docker@npm:^2.1.1": + version: 2.2.1 + resolution: "is-docker@npm:2.2.1" + bin: + is-docker: cli.js + checksum: 10c0/e828365958d155f90c409cdbe958f64051d99e8aedc2c8c4cd7c89dcf35329daed42f7b99346f7828df013e27deb8f721cf9408ba878c76eb9e8290235fbcdcc + languageName: node + linkType: hard + "is-extglob@npm:^2.1.1": version: 2.1.1 resolution: "is-extglob@npm:2.1.1" @@ -7592,6 +7623,15 @@ __metadata: languageName: node linkType: hard +"is-wsl@npm:^2.2.0": + version: 2.2.0 + resolution: "is-wsl@npm:2.2.0" + dependencies: + is-docker: "npm:^2.0.0" + checksum: 10c0/a6fa2d370d21be487c0165c7a440d567274fbba1a817f2f0bfa41cc5e3af25041d84267baa22df66696956038a43973e72fca117918c91431920bdef490fa25e + languageName: node + linkType: hard + "isarray@npm:^2.0.5": version: 2.0.5 resolution: "isarray@npm:2.0.5" @@ -7701,6 +7741,13 @@ __metadata: languageName: node linkType: hard +"js-base64@npm:^3.7.5": + version: 3.7.7 + resolution: "js-base64@npm:3.7.7" + checksum: 10c0/3c905a7e78b601e4751b5e710edd0d6d045ce2d23eb84c9df03515371e1b291edc72808dc91e081cb9855aef6758292a2407006f4608ec3705373dd8baf2f80f + languageName: node + linkType: hard + "js-tiktoken@npm:^1.0.12": version: 1.0.15 resolution: "js-tiktoken@npm:1.0.15" @@ -7710,15 +7757,6 @@ __metadata: languageName: node linkType: hard -"js-tiktoken@npm:^1.0.14, js-tiktoken@npm:^1.0.7": - version: 1.0.14 - resolution: "js-tiktoken@npm:1.0.14" - dependencies: - base64-js: "npm:^1.5.1" - checksum: 10c0/63a88ebdca8f2d0c00581085251001d3c2784fa52388ce059097296e2cdc6b5335055bd3f8f8106969353dc4654fcc88257a16238f985425ae0d39da218230fa - languageName: node - linkType: hard - "js-tokens@npm:^3.0.0 || ^4.0.0, js-tokens@npm:^4.0.0": version: 4.0.0 resolution: "js-tokens@npm:4.0.0" @@ -8115,6 +8153,39 @@ __metadata: languageName: node linkType: hard +"libsql@npm:^0.4.4": + version: 0.4.7 + resolution: "libsql@npm:0.4.7" + dependencies: + "@libsql/darwin-arm64": "npm:0.4.7" + "@libsql/darwin-x64": "npm:0.4.7" + "@libsql/linux-arm64-gnu": "npm:0.4.7" + "@libsql/linux-arm64-musl": "npm:0.4.7" + "@libsql/linux-x64-gnu": "npm:0.4.7" + "@libsql/linux-x64-musl": "npm:0.4.7" + "@libsql/win32-x64-msvc": "npm:0.4.7" + "@neon-rs/load": "npm:^0.0.4" + detect-libc: "npm:2.0.2" + dependenciesMeta: + "@libsql/darwin-arm64": + optional: true + "@libsql/darwin-x64": + optional: true + "@libsql/linux-arm64-gnu": + optional: true + "@libsql/linux-arm64-musl": + optional: true + "@libsql/linux-x64-gnu": + optional: true + "@libsql/linux-x64-musl": + optional: true + "@libsql/win32-x64-msvc": + optional: true + checksum: 10c0/351952440e6bad3477e5f1bb1b9d6570d16e403b894f4a13c5c7e183a1307b2fb04a2fa902728cb8594a259e1726c51c61b822d545bbc88319b126ad15468a87 + conditions: (os=darwin | os=linux | os=win32) & (cpu=x64 | cpu=arm64 | cpu=wasm32) + languageName: node + linkType: hard + "lie@npm:~3.3.0": version: 3.3.0 resolution: "lie@npm:3.3.0" @@ -9154,6 +9225,15 @@ __metadata: languageName: node linkType: hard +"mime@npm:^4.0.6": + version: 4.0.6 + resolution: "mime@npm:4.0.6" + bin: + mime: bin/cli.js + checksum: 10c0/1797b1c6da4cdb817fc18a4b8d99d6034885946f3d3680c2e4eb18bf19d4a64b42559f1eae0d1607e216f584311f9f806b5bfa1426baebeae4807bec5e14188a + languageName: node + linkType: hard + "mimic-fn@npm:^2.1.0": version: 2.1.0 resolution: "mimic-fn@npm:2.1.0" @@ -9500,7 +9580,7 @@ __metadata: languageName: node linkType: hard -"node-domexception@npm:1.0.0": +"node-domexception@npm:1.0.0, node-domexception@npm:^1.0.0": version: 1.0.0 resolution: "node-domexception@npm:1.0.0" checksum: 10c0/5e5d63cda29856402df9472335af4bb13875e1927ad3be861dc5ebde38917aecbf9ae337923777af52a48c426b70148815e890a5d72760f1b4d758cc671b1a2b @@ -9528,6 +9608,17 @@ __metadata: languageName: node linkType: hard +"node-fetch@npm:^3.3.2": + version: 3.3.2 + resolution: "node-fetch@npm:3.3.2" + dependencies: + data-uri-to-buffer: "npm:^4.0.0" + fetch-blob: "npm:^3.1.4" + formdata-polyfill: "npm:^4.0.10" + checksum: 10c0/f3d5e56190562221398c9f5750198b34cf6113aa304e34ee97c94fd300ec578b25b2c2906edba922050fce983338fde0d5d34fcb0fc3336ade5bd0e429ad7538 + languageName: node + linkType: hard + "node-gyp@npm:8.x": version: 8.4.1 resolution: "node-gyp@npm:8.4.1" @@ -9755,15 +9846,6 @@ __metadata: languageName: node linkType: hard -"ollama@npm:^0.5.9": - version: 0.5.10 - resolution: "ollama@npm:0.5.10" - dependencies: - whatwg-fetch: "npm:^3.6.20" - checksum: 10c0/424b92ef640fb562c590bd8bf394c0cdab3eb93a1b70a58bcb98090957375e05130f41d94578124c0fc79b68448db351e3439e4dc3582ccdb7ae13343951c527 - languageName: node - linkType: hard - "omggif@npm:^1.0.10, omggif@npm:^1.0.9": version: 1.0.10 resolution: "omggif@npm:1.0.10" @@ -9798,12 +9880,14 @@ __metadata: languageName: node linkType: hard -"openai-chat-tokens@npm:^0.2.8": - version: 0.2.8 - resolution: "openai-chat-tokens@npm:0.2.8" +"open@npm:^8.4.0": + version: 8.4.2 + resolution: "open@npm:8.4.2" dependencies: - js-tiktoken: "npm:^1.0.7" - checksum: 10c0/b415fda706b408f29b4584998990f29ad7f80f2ac1e84179a0976742ba8a80859fedeae5745a9bfe73443d95960b77328610074952ad198a18bc0e5c0ceb5b7b + define-lazy-prop: "npm:^2.0.0" + is-docker: "npm:^2.1.1" + is-wsl: "npm:^2.2.0" + checksum: 10c0/bb6b3a58401dacdb0aad14360626faf3fb7fba4b77816b373495988b724fb48941cad80c1b65d62bb31a17609b2cd91c41a181602caea597ca80dfbcc27e84c9 languageName: node linkType: hard @@ -9851,6 +9935,28 @@ __metadata: languageName: node linkType: hard +"openai@npm:^4.77.0": + version: 4.77.0 + resolution: "openai@npm:4.77.0" + dependencies: + "@types/node": "npm:^18.11.18" + "@types/node-fetch": "npm:^2.6.4" + abort-controller: "npm:^3.0.0" + agentkeepalive: "npm:^4.2.1" + form-data-encoder: "npm:1.7.2" + formdata-node: "npm:^4.3.2" + node-fetch: "npm:^2.6.7" + peerDependencies: + zod: ^3.23.8 + peerDependenciesMeta: + zod: + optional: true + bin: + openai: bin/cli + checksum: 10c0/438e5acbcdc592ff192f294e936c10a8b71edf898b53afacb937da45f8d4e221e041bfcc84d6174c8dcb9ed4080b32760f8d94de1fcec7ab889046f1e1173f68 + languageName: node + linkType: hard + "openai@patch:openai@npm%3A4.76.2#~/.yarn/patches/openai-npm-4.76.2-8ff1374617.patch": version: 4.76.2 resolution: "openai@patch:openai@npm%3A4.76.2#~/.yarn/patches/openai-npm-4.76.2-8ff1374617.patch::version=4.76.2&hash=849914" @@ -10500,6 +10606,13 @@ __metadata: languageName: node linkType: hard +"promise-limit@npm:^2.7.0": + version: 2.7.0 + resolution: "promise-limit@npm:2.7.0" + checksum: 10c0/ce220a7e11c8d0541940a3d99cc424bd16a18451b295a263f6dbaa998585d2d1afa71fcb7bb29078a61e214d2f13d96e9b082e96e8e357fbe5c5936ef2459cba + languageName: node + linkType: hard + "promise-retry@npm:^2.0.1": version: 2.0.1 resolution: "promise-retry@npm:2.0.1" @@ -11449,13 +11562,6 @@ __metadata: languageName: node linkType: hard -"reflect-metadata@npm:^0.2.2": - version: 0.2.2 - resolution: "reflect-metadata@npm:0.2.2" - checksum: 10c0/1cd93a15ea291e420204955544637c264c216e7aac527470e393d54b4bb075f10a17e60d8168ec96600c7e0b9fcc0cb0bb6e91c3fbf5b0d8c9056f04e6ac1ec2 - languageName: node - linkType: hard - "reflect.getprototypeof@npm:^1.0.4": version: 1.0.6 resolution: "reflect.getprototypeof@npm:1.0.6" @@ -11828,6 +11934,25 @@ __metadata: languageName: node linkType: hard +"rollup-plugin-visualizer@npm:^5.12.0": + version: 5.12.0 + resolution: "rollup-plugin-visualizer@npm:5.12.0" + dependencies: + open: "npm:^8.4.0" + picomatch: "npm:^2.3.1" + source-map: "npm:^0.7.4" + yargs: "npm:^17.5.1" + peerDependencies: + rollup: 2.x || 3.x || 4.x + peerDependenciesMeta: + rollup: + optional: true + bin: + rollup-plugin-visualizer: dist/bin/cli.js + checksum: 10c0/0e44a641223377ebb472bb10f2b22efa773b5f6fbe8d54f197f07c68d7a432cbf00abad79a0aa1570f70c673c792f24700d926d663ed9a4d0ad8406ae5a0f4e4 + languageName: node + linkType: hard + "rollup@npm:^4.20.0": version: 4.21.3 resolution: "rollup@npm:4.21.3" @@ -12301,6 +12426,13 @@ __metadata: languageName: node linkType: hard +"source-map@npm:^0.7.4": + version: 0.7.4 + resolution: "source-map@npm:0.7.4" + checksum: 10c0/dc0cf3768fe23c345ea8760487f8c97ef6fca8a73c83cd7c9bf2fde8bc2c34adb9c0824d6feb14bc4f9e37fb522e18af621543f1289038a66ac7586da29aa7dc + languageName: node + linkType: hard + "space-separated-tokens@npm:^2.0.0": version: 2.0.2 resolution: "space-separated-tokens@npm:2.0.2" @@ -13010,6 +13142,13 @@ __metadata: languageName: node linkType: hard +"tokenx@npm:^0.4.1": + version: 0.4.1 + resolution: "tokenx@npm:0.4.1" + checksum: 10c0/377f4e3c31ff9dc57b5b6af0fb1ae821227dee5e1d87b92a3ab1a0ed25454f01185c709d73592002b0d3024de1c904c8f029c46ae1806677816e4659fb8c481e + languageName: node + linkType: hard + "tough-cookie@npm:^4.1.3": version: 4.1.4 resolution: "tough-cookie@npm:4.1.4" @@ -13593,20 +13732,6 @@ __metadata: languageName: node linkType: hard -"validate.io-array@npm:^1.0.3, validate.io-array@npm:^1.0.5": - version: 1.0.6 - resolution: "validate.io-array@npm:1.0.6" - checksum: 10c0/ece1e93d24fe1c92f5ec5983e186f7890021c9144c2ad0e45d76695267861e9ad0362474a038a240caf3ab30f7b7595738c7f6efe9f6f0f9ae94290d23c39ef6 - languageName: node - linkType: hard - -"validate.io-function@npm:^1.0.2": - version: 1.0.2 - resolution: "validate.io-function@npm:1.0.2" - checksum: 10c0/210b4bbf8c71c7863df122beae76387406eb960a6540b003568dcde2bbb4baac17a2c8f0eda014f0c5d2440396e87141e62028cc8758ddc61589e3425bd26c27 - languageName: node - linkType: hard - "verror@npm:1.10.0": version: 1.10.0 resolution: "verror@npm:1.10.0" @@ -13732,7 +13857,7 @@ __metadata: languageName: node linkType: hard -"web-streams-polyfill@npm:^3.2.1": +"web-streams-polyfill@npm:^3.0.3, web-streams-polyfill@npm:^3.2.1": version: 3.3.3 resolution: "web-streams-polyfill@npm:3.3.3" checksum: 10c0/64e855c47f6c8330b5436147db1c75cb7e7474d924166800e8e2aab5eb6c76aac4981a84261dd2982b3e754490900b99791c80ae1407a9fa0dcff74f82ea3a7f @@ -13783,13 +13908,6 @@ __metadata: languageName: node linkType: hard -"whatwg-fetch@npm:^3.6.20": - version: 3.6.20 - resolution: "whatwg-fetch@npm:3.6.20" - checksum: 10c0/fa972dd14091321d38f36a4d062298df58c2248393ef9e8b154493c347c62e2756e25be29c16277396046d6eaa4b11bd174f34e6403fff6aaca9fb30fa1ff46d - languageName: node - linkType: hard - "whatwg-mimetype@npm:^4.0.0": version: 4.0.0 resolution: "whatwg-mimetype@npm:4.0.0" @@ -13984,7 +14102,7 @@ __metadata: languageName: node linkType: hard -"ws@npm:^8.16.0": +"ws@npm:^8.13.0, ws@npm:^8.16.0": version: 8.18.0 resolution: "ws@npm:8.18.0" peerDependencies: @@ -14174,7 +14292,7 @@ __metadata: languageName: node linkType: hard -"yargs@npm:^17.6.2": +"yargs@npm:^17.5.1, yargs@npm:^17.6.2": version: 17.7.2 resolution: "yargs@npm:17.7.2" dependencies: