diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..d040cd2b --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,94 @@ +name: Release + +on: + workflow_dispatch: + inputs: + version: + description: 'Version (e.g. v1.2.3)' + required: true + type: string + push: + tags: + - v*.*.* + +permissions: + contents: write + +jobs: + release: + runs-on: ${{ matrix.os }} + + strategy: + matrix: + os: [macos-13, macos-latest] + arch: [x64, arm64] + exclude: + - os: windows-latest + arch: arm64 + - os: macos-latest + arch: x64 + - os: macos-13 + arch: arm64 + + steps: + - name: Check out Git repository + uses: actions/checkout@v3 + + - name: Install Node.js + uses: actions/setup-node@v3 + with: + node-version: 20 + arch: ${{ matrix.arch }} + + - name: Install corepack + run: corepack enable && corepack prepare yarn@4.3.1 --activate + + - name: Install Dependencies + run: yarn install + + - name: Build Linux + if: matrix.os == 'ubuntu-latest' + run: yarn build:linux + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + ARCH: ${{ matrix.arch }} + + - name: Build Mac + if: matrix.os == 'macos-13' + run: yarn build:mac + env: + CSC_LINK: ${{ secrets.CSC_LINK }} + CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }} + APPLE_ID: ${{ vars.APPLE_ID }} + APPLE_APP_SPECIFIC_PASSWORD: ${{ vars.APPLE_APP_SPECIFIC_PASSWORD }} + APPLE_TEAM_ID: ${{ vars.APPLE_TEAM_ID }} + GH_TOKEN: ${{ secrets.GH_TOKEN }} + ARCH: ${{ matrix.arch }} + + - name: Build Windows + if: matrix.os == 'windows-latest' + run: yarn build:win + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + ARCH: ${{ matrix.arch }} + + - name: Replace spaces in filenames + run: node scripts/replaceSpaces.js + + - name: Release + uses: softprops/action-gh-release@v2 + with: + draft: true + files: | + dist/*.exe + dist/*.zip + dist/*.dmg + dist/*.AppImage + dist/*.snap + dist/*.deb + dist/*.rpm + dist/*.tar.gz + dist/latest*.yml + dist/*.blockmap + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 352cdab2..59d138e9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,15 @@ jobs: strategy: matrix: - os: [macos-latest, windows-latest, ubuntu-latest] + os: [macos-13, macos-latest, windows-latest, ubuntu-latest] + arch: [x64, arm64] + exclude: + - os: windows-latest + arch: arm64 + - os: macos-latest + arch: x64 + - os: macos-13 + arch: arm64 steps: - name: Check out Git repository @@ -30,6 +38,7 @@ jobs: uses: actions/setup-node@v3 with: node-version: 20 + arch: ${{ matrix.arch }} - name: Install corepack run: corepack enable && corepack prepare yarn@4.3.1 --activate @@ -42,9 +51,10 @@ jobs: run: yarn build:linux env: GH_TOKEN: ${{ secrets.GH_TOKEN }} + ARCH: ${{ matrix.arch }} - name: Build Mac - if: matrix.os == 'macos-latest' + if: matrix.os == 'macos-13' run: yarn build:mac env: CSC_LINK: ${{ secrets.CSC_LINK }} @@ -53,12 +63,14 @@ jobs: APPLE_APP_SPECIFIC_PASSWORD: ${{ vars.APPLE_APP_SPECIFIC_PASSWORD }} APPLE_TEAM_ID: ${{ vars.APPLE_TEAM_ID }} GH_TOKEN: ${{ secrets.GH_TOKEN }} + ARCH: ${{ matrix.arch }} - name: Build Windows if: matrix.os == 'windows-latest' run: yarn build:win env: GH_TOKEN: ${{ secrets.GH_TOKEN }} + ARCH: ${{ matrix.arch }} - name: Replace spaces in filenames run: node scripts/replaceSpaces.js diff --git a/.yarn/patches/@electron-notarize-npm-2.3.2-535908a4bd.patch b/.yarn/patches/@electron-notarize-npm-2.3.2-535908a4bd.patch deleted file mode 100644 index 6df0a904..00000000 --- a/.yarn/patches/@electron-notarize-npm-2.3.2-535908a4bd.patch +++ /dev/null @@ -1,53 +0,0 @@ -diff --git a/lib/check-signature.js b/lib/check-signature.js -index 324568af71bcc4372c9f959131ecd24122848c86..677348e0a138ff608b2ac41f592d813b15ee4956 100644 ---- a/lib/check-signature.js -+++ b/lib/check-signature.js -@@ -41,16 +41,12 @@ const spawn_1 = require("./spawn"); - const debug_1 = __importDefault(require("debug")); - const d = (0, debug_1.default)('electron-notarize'); - const codesignDisplay = (opts) => __awaiter(void 0, void 0, void 0, function* () { -- const result = yield (0, spawn_1.spawn)('codesign', ['-dv', '-vvvv', '--deep', path.basename(opts.appPath)], { -- cwd: path.dirname(opts.appPath), -- }); -+ const result = yield (0, spawn_1.spawn)('codesign', ['-dv', '-vvvv', '--deep', opts.appPath]); - return result; - }); - const codesign = (opts) => __awaiter(void 0, void 0, void 0, function* () { - d('attempting to check codesign of app:', opts.appPath); -- const result = yield (0, spawn_1.spawn)('codesign', ['-vvv', '--deep', '--strict', path.basename(opts.appPath)], { -- cwd: path.dirname(opts.appPath), -- }); -+ const result = yield (0, spawn_1.spawn)('codesign', ['-vvv', '--deep', '--strict', opts.appPath]); - return result; - }); - function checkSignatures(opts) { -diff --git a/lib/notarytool.js b/lib/notarytool.js -index 1ab090efb2101fc8bee5553445e0349c54474421..a5ddfd922197449fc56078e4a7e9a2ee5d8d207d 100644 ---- a/lib/notarytool.js -+++ b/lib/notarytool.js -@@ -92,9 +92,7 @@ function notarizeAndWaitForNotaryTool(opts) { - else { - filePath = path.resolve(dir, `${path.parse(opts.appPath).name}.zip`); - d('zipping application to:', filePath); -- const zipResult = yield (0, spawn_1.spawn)('ditto', ['-c', '-k', '--sequesterRsrc', '--keepParent', path.basename(opts.appPath), filePath], { -- cwd: path.dirname(opts.appPath), -- }); -+ const zipResult = yield (0, spawn_1.spawn)('ditto', ['-c', '-k', '--sequesterRsrc', '--keepParent', opts.appPath, filePath]); - if (zipResult.code !== 0) { - throw new Error(`Failed to zip application, exited with code: ${zipResult.code}\n\n${zipResult.output}`); - } -diff --git a/lib/staple.js b/lib/staple.js -index 47dbd85b2fc279d999b57f47fb8171e1cc674436..f8829e6ac54fcd630a730d12d75acc1591b953b6 100644 ---- a/lib/staple.js -+++ b/lib/staple.js -@@ -43,9 +43,7 @@ const d = (0, debug_1.default)('electron-notarize:staple'); - function stapleApp(opts) { - return __awaiter(this, void 0, void 0, function* () { - d('attempting to staple app:', opts.appPath); -- const result = yield (0, spawn_1.spawn)('xcrun', ['stapler', 'staple', '-v', path.basename(opts.appPath)], { -- cwd: path.dirname(opts.appPath), -- }); -+ const result = yield (0, spawn_1.spawn)('xcrun', ['stapler', 'staple', '-v', opts.appPath]); - if (result.code !== 0) { - throw new Error(`Failed to staple your application with code: ${result.code}\n\n${result.output}`); - } diff --git a/electron-builder.yml b/electron-builder.yml index bdb3f958..26189869 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -13,6 +13,7 @@ files: - '!local' asarUnpack: - resources/** + - '**/*.{node,dll,metal,exp,lib}' win: executableName: Cherry Studio nsis: diff --git a/electron.vite.config.ts b/electron.vite.config.ts index 1b409788..f0646d89 100644 --- a/electron.vite.config.ts +++ b/electron.vite.config.ts @@ -16,6 +16,7 @@ export default defineConfig({ '@llm-tools/embedjs-loader-msoffice', '@llm-tools/embedjs-loader-xml', '@llm-tools/embedjs-loader-pdf', + '@llm-tools/embedjs-loader-sitemap', '@lancedb/lancedb' ] }) @@ -29,7 +30,7 @@ export default defineConfig({ }, build: { rollupOptions: { - external: ['@lancedb/lancedb', '@llm-tools/embedjs-loader-sitemap'] + external: ['@lancedb/lancedb'] } } }, diff --git a/package.json b/package.json index c87afcaa..7c23f243 100644 --- a/package.json +++ b/package.json @@ -28,9 +28,14 @@ "build": "npm run typecheck && electron-vite build", "postinstall": "electron-builder install-app-deps", "build:unpack": "dotenv npm run build && electron-builder --dir", - "build:win": "dotenv npm run build && electron-builder --win --publish never", - "build:mac": "dotenv electron-vite build && electron-builder --mac --publish never", - "build:linux": "dotenv electron-vite build && electron-builder --linux --publish never", + "build:win": "dotenv npm run build && electron-builder --win --$ARCH", + "build:win:x64": "dotenv npm run build && electron-builder --win --x64", + "build:mac": "dotenv electron-vite build && electron-builder --mac --$ARCH", + "build:mac:arm64": "dotenv electron-vite build && electron-builder --mac --arm64", + "build:mac:x64": "dotenv electron-vite build && electron-builder --mac --x64", + "build:linux": "dotenv electron-vite build && electron-builder --linux --$ARCH", + "build:linux:arm64": "dotenv electron-vite build && electron-builder --linux --arm64", + "build:linux:x64": "dotenv electron-vite build && electron-builder --linux --x64", "release": "node scripts/version.js", "publish": "yarn release patch push", "pulish:artifacts": "cd packages/artifacts && npm publish && cd -", @@ -40,12 +45,14 @@ "dependencies": { "@electron-toolkit/preload": "^3.0.0", "@electron-toolkit/utils": "^3.0.0", + "@electron/notarize": "^2.5.0", "@llm-tools/embedjs": "^0.1.24", "@llm-tools/embedjs-lancedb": "^0.1.24", "@llm-tools/embedjs-loader-csv": "^0.1.24", "@llm-tools/embedjs-loader-markdown": "^0.1.24", "@llm-tools/embedjs-loader-msoffice": "^0.1.24", "@llm-tools/embedjs-loader-pdf": "^0.1.24", + "@llm-tools/embedjs-loader-sitemap": "^0.1.24", "@llm-tools/embedjs-loader-web": "^0.1.24", "@llm-tools/embedjs-loader-xml": "^0.1.24", "@llm-tools/embedjs-ollama": "^0.1.24", @@ -138,7 +145,6 @@ "react-dom": "^17.0.0 || ^18.0.0" }, "resolutions": { - "@electron/notarize@npm:2.2.1": "patch:@electron/notarize@npm%3A2.3.2#~/.yarn/patches/@electron-notarize-npm-2.3.2-535908a4bd.patch", "pdf-parse@npm:1.1.1": "patch:pdf-parse@npm%3A1.1.1#~/.yarn/patches/pdf-parse-npm-1.1.1-04a6109b2a.patch" }, "packageManager": "yarn@4.5.0" diff --git a/src/main/services/KnowledgeService.ts b/src/main/services/KnowledgeService.ts index 3b87d370..e1b63b7b 100644 --- a/src/main/services/KnowledgeService.ts +++ b/src/main/services/KnowledgeService.ts @@ -7,9 +7,10 @@ import { LanceDb } from '@llm-tools/embedjs-lancedb' import { MarkdownLoader } from '@llm-tools/embedjs-loader-markdown' import { DocxLoader } from '@llm-tools/embedjs-loader-msoffice' import { PdfLoader } from '@llm-tools/embedjs-loader-pdf' +import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap' import { WebLoader } from '@llm-tools/embedjs-loader-web' import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai' -import { FileType, RagAppRequestParams } from '@types' +import { FileType, KnowledgeBaseParams, KnowledgeItem } from '@types' import { app } from 'electron' class KnowledgeService { @@ -25,7 +26,7 @@ class KnowledgeService { } } - private getRagApplication = async ({ id, model, apiKey, baseURL }: RagAppRequestParams): Promise => { + private getRagApplication = async ({ id, model, apiKey, baseURL }: KnowledgeBaseParams): Promise => { return new RAGApplicationBuilder() .setModel('NO_MODEL') .setEmbeddingModel( @@ -42,13 +43,13 @@ class KnowledgeService { public create = async ( _: Electron.IpcMainInvokeEvent, - { id, model, apiKey, baseURL }: RagAppRequestParams + { id, model, apiKey, baseURL }: KnowledgeBaseParams ): Promise => { this.getRagApplication({ id, model, apiKey, baseURL }) } - public reset = async (_: Electron.IpcMainInvokeEvent, { config }: { config: RagAppRequestParams }): Promise => { - const ragApplication = await this.getRagApplication(config) + public reset = async (_: Electron.IpcMainInvokeEvent, { base }: { base: KnowledgeBaseParams }): Promise => { + const ragApplication = await this.getRagApplication(base) await ragApplication.reset() } @@ -61,27 +62,41 @@ class KnowledgeService { public add = async ( _: Electron.IpcMainInvokeEvent, - { data, config }: { data: string | FileType; config: RagAppRequestParams } + { base, item }: { base: KnowledgeBaseParams; item: KnowledgeItem } ): Promise => { - const ragApplication = await this.getRagApplication(config) + const ragApplication = await this.getRagApplication(base) - if (typeof data === 'string') { - if (data.startsWith('http')) { - return await ragApplication.addLoader(new WebLoader({ urlOrContent: data })) + if (item.type === 'url') { + const content = item.content as string + if (content.startsWith('http')) { + return await ragApplication.addLoader(new WebLoader({ urlOrContent: content })) } - return await ragApplication.addLoader(new TextLoader({ text: data })) } - if (data.ext === '.pdf') { - return await ragApplication.addLoader(new PdfLoader({ filePathOrUrl: data.path }) as any) + if (item.type === 'sitemap') { + const content = item.content as string + return await ragApplication.addLoader(new SitemapLoader({ url: content })) } - if (data.ext === '.docx') { - return await ragApplication.addLoader(new DocxLoader({ filePathOrUrl: data.path }) as any) + if (item.type === 'note') { + const content = item.content as string + return await ragApplication.addLoader(new TextLoader({ text: content })) } - if (data.ext === '.md' || data.ext === '.mdx') { - return await ragApplication.addLoader(new MarkdownLoader({ filePathOrUrl: data.path }) as any) + if (item.type === 'file') { + const file = item.content as FileType + + if (file.ext === '.pdf') { + return await ragApplication.addLoader(new PdfLoader({ filePathOrUrl: file.path }) as any) + } + + if (file.ext === '.docx') { + return await ragApplication.addLoader(new DocxLoader({ filePathOrUrl: file.path }) as any) + } + + if (file.ext.startsWith('.md')) { + return await ragApplication.addLoader(new MarkdownLoader({ filePathOrUrl: file.path }) as any) + } } return { entriesAdded: 0, uniqueId: '', loaderType: '' } @@ -89,17 +104,17 @@ class KnowledgeService { public remove = async ( _: Electron.IpcMainInvokeEvent, - { uniqueId, config }: { uniqueId: string; config: RagAppRequestParams } + { uniqueId, base }: { uniqueId: string; base: KnowledgeBaseParams } ): Promise => { - const ragApplication = await this.getRagApplication(config) + const ragApplication = await this.getRagApplication(base) await ragApplication.deleteLoader(uniqueId) } public search = async ( _: Electron.IpcMainInvokeEvent, - { search, config }: { search: string; config: RagAppRequestParams } + { search, base }: { search: string; base: KnowledgeBaseParams } ): Promise => { - const ragApplication = await this.getRagApplication(config) + const ragApplication = await this.getRagApplication(base) return await ragApplication.search(search) } } diff --git a/src/preload/index.d.ts b/src/preload/index.d.ts index d0d526ec..4f34b94d 100644 --- a/src/preload/index.d.ts +++ b/src/preload/index.d.ts @@ -2,7 +2,7 @@ import { ElectronAPI } from '@electron-toolkit/preload' import { AddLoaderReturn, ExtractChunkData } from '@llm-tools/embedjs-interfaces' import { FileType } from '@renderer/types' import { WebDavConfig } from '@renderer/types' -import { AppInfo, LanguageVarious, RagAppRequestParams } from '@renderer/types' +import { AppInfo, KnowledgeBaseParams, KnowledgeItem, LanguageVarious } from '@renderer/types' import type { OpenDialogOptions } from 'electron' import type { UpdateInfo } from 'electron-updater' import { Readable } from 'stream' @@ -60,12 +60,12 @@ declare global { update: (shortcuts: Shortcut[]) => Promise } knowledgeBase: { - create: ({ id, model, apiKey, baseURL }: RagAppRequestParams) => Promise - reset: ({ config }: { config: RagAppRequestParams }) => Promise + create: ({ id, model, apiKey, baseURL }: KnowledgeBaseParams) => Promise + reset: ({ base }: { base: KnowledgeBaseParams }) => Promise delete: (id: string) => Promise - add: ({ data, config }: { data: string | FileType; config: RagAppRequestParams }) => Promise - remove: ({ uniqueId, config }: { uniqueId: string; config: RagAppRequestParams }) => Promise - search: ({ search, config }: { search: string; config: RagAppRequestParams }) => Promise + add: ({ base, item }: { base: KnowledgeBaseParams; item: KnowledgeItem }) => Promise + remove: ({ uniqueId, base }: { uniqueId: string; base: KnowledgeBaseParams }) => Promise + search: ({ search, base }: { search: string; base: KnowledgeBaseParams }) => Promise } } } diff --git a/src/preload/index.ts b/src/preload/index.ts index 657dd573..2190f749 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -1,5 +1,5 @@ import { electronAPI } from '@electron-toolkit/preload' -import { FileType, RagAppRequestParams, Shortcut, WebDavConfig } from '@types' +import { KnowledgeBaseParams, KnowledgeItem, Shortcut, WebDavConfig } from '@types' import { contextBridge, ipcRenderer, OpenDialogOptions } from 'electron' // Custom APIs for renderer @@ -52,16 +52,16 @@ const api = { update: (shortcuts: Shortcut[]) => ipcRenderer.invoke('shortcuts:update', shortcuts) }, knowledgeBase: { - create: ({ id, model, apiKey, baseURL }: RagAppRequestParams) => + create: ({ id, model, apiKey, baseURL }: KnowledgeBaseParams) => ipcRenderer.invoke('knowledge-base:create', { id, model, apiKey, baseURL }), - reset: ({ config }: { config: RagAppRequestParams }) => ipcRenderer.invoke('knowledge-base:reset', { config }), + reset: ({ base }: { base: KnowledgeBaseParams }) => ipcRenderer.invoke('knowledge-base:reset', { base }), delete: (id: string) => ipcRenderer.invoke('knowledge-base:delete', id), - add: ({ data, config }: { data: string | FileType; config: RagAppRequestParams }) => - ipcRenderer.invoke('knowledge-base:add', { data, config }), - remove: ({ uniqueId, config }: { uniqueId: string; config: RagAppRequestParams }) => - ipcRenderer.invoke('knowledge-base:remove', { uniqueId, config }), - search: ({ search, config }: { search: string; config: RagAppRequestParams }) => - ipcRenderer.invoke('knowledge-base:search', { search, config }) + add: ({ base, item }: { base: KnowledgeBaseParams; item: KnowledgeItem }) => + ipcRenderer.invoke('knowledge-base:add', { base, item }), + remove: ({ uniqueId, base }: { uniqueId: string; base: KnowledgeBaseParams }) => + ipcRenderer.invoke('knowledge-base:remove', { uniqueId, base }), + search: ({ search, base }: { search: string; base: KnowledgeBaseParams }) => + ipcRenderer.invoke('knowledge-base:search', { search, base }) } } diff --git a/src/renderer/src/hooks/useknowledge.ts b/src/renderer/src/hooks/useknowledge.ts index c4ab65f3..0af8b787 100644 --- a/src/renderer/src/hooks/useknowledge.ts +++ b/src/renderer/src/hooks/useknowledge.ts @@ -2,7 +2,7 @@ import { db } from '@renderer/databases/index' import KnowledgeQueue from '@renderer/queue/KnowledgeQueue' import FileManager from '@renderer/services/FileManager' -import { getRagAppRequestParams } from '@renderer/services/KnowledgeService' +import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService' import { RootState } from '@renderer/store' import { addBase, @@ -143,9 +143,8 @@ export const useKnowledge = (baseId: string) => { const removeItem = async (item: KnowledgeItem) => { dispatch(removeItemAction({ baseId, item })) if (base) { - const config = getRagAppRequestParams(base) if (item?.uniqueId) { - await window.api.knowledgeBase.remove({ uniqueId: item.uniqueId, config }) + await window.api.knowledgeBase.remove({ uniqueId: item.uniqueId, base: getKnowledgeBaseParams(base) }) } if (item.type === 'file' && typeof item.content === 'object') { await FileManager.deleteFile(item.content.id) diff --git a/src/renderer/src/i18n/locales/en-us.json b/src/renderer/src/i18n/locales/en-us.json index 483aae36..d80a19a1 100644 --- a/src/renderer/src/i18n/locales/en-us.json +++ b/src/renderer/src/i18n/locales/en-us.json @@ -560,8 +560,9 @@ "no_bases": "No knowledge bases available", "clear_selection": "Clear selection", "delete_confirm": "Are you sure you want to delete this knowledge base?", - "sitemaps": "Site Maps", - "add_sitemap": "Add Site Map" + "sitemaps": "Websites", + "add_sitemap": "Website Map", + "sitemap_placeholder": "Enter Website Map URL" } } } diff --git a/src/renderer/src/i18n/locales/ru-ru.json b/src/renderer/src/i18n/locales/ru-ru.json index d4d7d41a..f286df38 100644 --- a/src/renderer/src/i18n/locales/ru-ru.json +++ b/src/renderer/src/i18n/locales/ru-ru.json @@ -560,8 +560,9 @@ "no_bases": "База знаний не найдена", "clear_selection": "Очистить выбор", "delete_confirm": "Вы уверены, что хотите удалить эту базу знаний?", - "sitemaps": "Карта сайта", - "add_sitemap": "Добавить карту сайта" + "sitemaps": "Сайты", + "add_sitemap": "Карта сайта", + "sitemap_placeholder": "Введите URL карты сайта" } } } diff --git a/src/renderer/src/i18n/locales/zh-cn.json b/src/renderer/src/i18n/locales/zh-cn.json index 52bcd8da..2cf689f6 100644 --- a/src/renderer/src/i18n/locales/zh-cn.json +++ b/src/renderer/src/i18n/locales/zh-cn.json @@ -549,8 +549,9 @@ "no_bases": "暂无知识库", "clear_selection": "清除选择", "delete_confirm": "确定要删除此知识库吗?", - "sitemaps": "站点地图", - "add_sitemap": "添加站点地图" + "sitemaps": "网站", + "add_sitemap": "站点地图", + "sitemap_placeholder": "请输入站点地图 URL" } } } diff --git a/src/renderer/src/i18n/locales/zh-tw.json b/src/renderer/src/i18n/locales/zh-tw.json index a07e5e44..dc0f3121 100644 --- a/src/renderer/src/i18n/locales/zh-tw.json +++ b/src/renderer/src/i18n/locales/zh-tw.json @@ -548,8 +548,9 @@ "no_bases": "暫無知識庫", "clear_selection": "清除選擇", "delete_confirm": "確定要刪除此知識庫嗎?", - "sitemaps": "站點地圖", - "add_sitemap": "添加站點地圖" + "sitemaps": "網站", + "add_sitemap": "網站地圖", + "sitemap_placeholder": "請輸入網站地圖 URL" } } } diff --git a/src/renderer/src/pages/knowledge/KnowledgePage.tsx b/src/renderer/src/pages/knowledge/KnowledgePage.tsx index cd77a87b..13331a4b 100644 --- a/src/renderer/src/pages/knowledge/KnowledgePage.tsx +++ b/src/renderer/src/pages/knowledge/KnowledgePage.tsx @@ -27,7 +27,7 @@ const KnowledgePage: FC = () => { if (!selectedBase) { return setSelectedBase(bases[0]) } - if (selectedBase && !bases.includes(selectedBase)) { + if (selectedBase && !bases.find((base) => base.id === selectedBase.id)) { return setSelectedBase(bases[0]) } } diff --git a/src/renderer/src/pages/knowledge/components/AddKnowledgePopup.tsx b/src/renderer/src/pages/knowledge/components/AddKnowledgePopup.tsx index caa031be..51e681d2 100644 --- a/src/renderer/src/pages/knowledge/components/AddKnowledgePopup.tsx +++ b/src/renderer/src/pages/knowledge/components/AddKnowledgePopup.tsx @@ -2,7 +2,7 @@ import { TopView } from '@renderer/components/TopView' import { isEmbeddingModel } from '@renderer/config/models' import { useKnowledgeBases } from '@renderer/hooks/useknowledge' import { useProviders } from '@renderer/hooks/useProvider' -import { getRagAppRequestParams } from '@renderer/services/KnowledgeService' +import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService' import { getModelUniqId } from '@renderer/services/ModelService' import { Model } from '@renderer/types' import { Form, Input, Modal, Select } from 'antd' @@ -64,7 +64,7 @@ const PopupContainer: React.FC = ({ title, resolve }) => { updated_at: Date.now() } - await window.api.knowledgeBase.create(getRagAppRequestParams(newBase)) + await window.api.knowledgeBase.create(getKnowledgeBaseParams(newBase)) addKnowledgeBase(newBase as any) setOpen(false) diff --git a/src/renderer/src/pages/knowledge/components/KnowledgeSearchPopup.tsx b/src/renderer/src/pages/knowledge/components/KnowledgeSearchPopup.tsx index 8d5cfebf..b5f63140 100644 --- a/src/renderer/src/pages/knowledge/components/KnowledgeSearchPopup.tsx +++ b/src/renderer/src/pages/knowledge/components/KnowledgeSearchPopup.tsx @@ -1,6 +1,6 @@ import { ExtractChunkData } from '@llm-tools/embedjs-interfaces' import { TopView } from '@renderer/components/TopView' -import { getRagAppRequestParams } from '@renderer/services/KnowledgeService' +import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService' import { KnowledgeBase } from '@renderer/types' import { Input, List, Modal, Spin, Typography } from 'antd' import { useState } from 'react' @@ -37,7 +37,7 @@ const PopupContainer: React.FC = ({ base, resolve }) => { try { const searchResults = await window.api.knowledgeBase.search({ search: value, - config: getRagAppRequestParams(base) + base: getKnowledgeBaseParams(base) }) setResults(searchResults) } catch (error) { diff --git a/src/renderer/src/providers/BaseProvider.ts b/src/renderer/src/providers/BaseProvider.ts index 857f2179..c6fdc1b1 100644 --- a/src/renderer/src/providers/BaseProvider.ts +++ b/src/renderer/src/providers/BaseProvider.ts @@ -1,5 +1,5 @@ import { getOllamaKeepAliveTime } from '@renderer/hooks/useOllama' -import { getRagAppRequestParams } from '@renderer/services/KnowledgeService' +import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService' import store from '@renderer/store' import { Assistant, FileType, Message, Provider, Suggestion } from '@renderer/types' import { delay } from '@renderer/utils' @@ -96,7 +96,7 @@ export default abstract class BaseProvider { const searchResults = await window.api.knowledgeBase.search({ search: message.content, - config: getRagAppRequestParams(base) + base: getKnowledgeBaseParams(base) }) const references = take(searchResults, 5) diff --git a/src/renderer/src/queue/KnowledgeQueue.ts b/src/renderer/src/queue/KnowledgeQueue.ts index 24e979a8..51736ff7 100644 --- a/src/renderer/src/queue/KnowledgeQueue.ts +++ b/src/renderer/src/queue/KnowledgeQueue.ts @@ -1,6 +1,6 @@ import { AddLoaderReturn } from '@llm-tools/embedjs-interfaces' import db from '@renderer/databases' -import { getRagAppRequestParams } from '@renderer/services/KnowledgeService' +import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService' import store from '@renderer/store' import { clearCompletedProcessing, updateBaseItemUniqueId, updateItemProcessingStatus } from '@renderer/store/knowledge' import { KnowledgeItem } from '@renderer/types' @@ -142,7 +142,7 @@ class KnowledgeQueue { throw new Error(`[KnowledgeQueue] Knowledge base ${baseId} not found`) } - const requestParams = getRagAppRequestParams(base) + const baseParams = getKnowledgeBaseParams(base) const sourceItem = base.items.find((i) => i.id === item.id) if (!sourceItem) { @@ -155,26 +155,22 @@ class KnowledgeQueue { switch (item.type) { case 'file': console.log(`[KnowledgeQueue] Processing file: ${sourceItem.content}`) - result = await window.api.knowledgeBase.add({ data: sourceItem.content, config: requestParams }) - console.log(`[KnowledgeQueue] Result: ${JSON.stringify(result)}`) + result = await window.api.knowledgeBase.add({ base: baseParams, item: sourceItem }) break case 'url': console.log(`[KnowledgeQueue] Processing URL: ${sourceItem.content}`) - result = await window.api.knowledgeBase.add({ data: sourceItem.content, config: requestParams }) - console.log(`[KnowledgeQueue] Result: ${JSON.stringify(result)}`) + result = await window.api.knowledgeBase.add({ base: baseParams, item: sourceItem }) break case 'sitemap': console.log(`[KnowledgeQueue] Processing Sitemap: ${sourceItem.content}`) - result = await window.api.knowledgeBase.add({ data: sourceItem.content, config: requestParams }) - console.log(`[KnowledgeQueue] Result: ${JSON.stringify(result)}`) + result = await window.api.knowledgeBase.add({ base: baseParams, item: sourceItem }) break case 'note': console.log(`[KnowledgeQueue] Processing note: ${sourceItem.content}`) note = await db.knowledge_notes.get(item.id) if (!note) throw new Error(`Source note ${item.id} not found`) content = note.content as string - result = await window.api.knowledgeBase.add({ data: content, config: requestParams }) - console.log(`[KnowledgeQueue] Result: ${JSON.stringify(result)}`) + result = await window.api.knowledgeBase.add({ base: baseParams, item: { ...sourceItem, content } }) break } diff --git a/src/renderer/src/services/KnowledgeService.ts b/src/renderer/src/services/KnowledgeService.ts index 176ee97d..4fce5e91 100644 --- a/src/renderer/src/services/KnowledgeService.ts +++ b/src/renderer/src/services/KnowledgeService.ts @@ -1,12 +1,17 @@ import AiProvider from '@renderer/providers/AiProvider' -import { KnowledgeBase, RagAppRequestParams } from '@renderer/types' +import { KnowledgeBase, KnowledgeBaseParams } from '@renderer/types' +import { isEmpty } from 'lodash' import { getProviderByModel } from './AssistantService' -export const getRagAppRequestParams = (base: KnowledgeBase): RagAppRequestParams => { +export const getKnowledgeBaseParams = (base: KnowledgeBase): KnowledgeBaseParams => { const provider = getProviderByModel(base.model) const aiProvider = new AiProvider(provider) + if (provider.id === 'ollama' && isEmpty(provider.apiKey)) { + provider.apiKey = 'empty' + } + return { id: base.id, model: base.model.name, diff --git a/src/renderer/src/store/knowledge.ts b/src/renderer/src/store/knowledge.ts index 95dd7d6d..48fe5b3e 100644 --- a/src/renderer/src/store/knowledge.ts +++ b/src/renderer/src/store/knowledge.ts @@ -48,12 +48,20 @@ const knowledgeSlice = createSlice({ if (base) { if (action.payload.item.type === 'note') { base.items.push(action.payload.item) - } else if (action.payload.item.type === 'url') { + } + if (action.payload.item.type === 'url') { const urlExists = base.items.some((item) => item.content === action.payload.item.content) if (!urlExists) { base.items.push(action.payload.item) } - } else if (action.payload.item.type === 'file') { + } + if (action.payload.item.type === 'sitemap') { + const sitemapExists = base.items.some((item) => item.content === action.payload.item.content) + if (!sitemapExists) { + base.items.push(action.payload.item) + } + } + if (action.payload.item.type === 'file') { action.payload.item.created_at = new Date(action.payload.item.created_at).getTime() action.payload.item.updated_at = new Date(action.payload.item.updated_at).getTime() base.items.push(action.payload.item) diff --git a/src/renderer/src/types/index.ts b/src/renderer/src/types/index.ts index bb6f289e..a46409a6 100644 --- a/src/renderer/src/types/index.ts +++ b/src/renderer/src/types/index.ts @@ -197,6 +197,8 @@ export type KnowledgeItem = { retryCount?: number } +export type KnowledgeItemType = 'file' | 'url' | 'note' | 'sitemap' + export interface KnowledgeBase { id: string name: string @@ -207,7 +209,7 @@ export interface KnowledgeBase { updated_at: number } -export type RagAppRequestParams = { +export type KnowledgeBaseParams = { id: string model: string apiKey: string diff --git a/yarn.lock b/yarn.lock index 34502173..f0c21ac8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -504,25 +504,25 @@ __metadata: languageName: node linkType: hard -"@electron/notarize@npm:2.3.2": - version: 2.3.2 - resolution: "@electron/notarize@npm:2.3.2" +"@electron/notarize@npm:2.2.1": + version: 2.2.1 + resolution: "@electron/notarize@npm:2.2.1" dependencies: debug: "npm:^4.1.1" fs-extra: "npm:^9.0.1" promise-retry: "npm:^2.0.1" - checksum: 10c0/539ed5cd264c3885fd3ca9c0b243144e3e2856d767de3999da1e3f94f0d79db57cbb08862b640270dfad0292bc5345cd7177db096da2061e28e15a6b85946b32 + checksum: 10c0/d3fbbaaf26e809d4484f87826f02ba9108eba222a495ff533d9728a58a0cca6e267764baefc5616952318a6674eb6d3b7d07b1136ca0254da1c51012a0e6e6ae languageName: node linkType: hard -"@electron/notarize@patch:@electron/notarize@npm%3A2.3.2#~/.yarn/patches/@electron-notarize-npm-2.3.2-535908a4bd.patch": - version: 2.3.2 - resolution: "@electron/notarize@patch:@electron/notarize@npm%3A2.3.2#~/.yarn/patches/@electron-notarize-npm-2.3.2-535908a4bd.patch::version=2.3.2&hash=dd0991" +"@electron/notarize@npm:^2.5.0": + version: 2.5.0 + resolution: "@electron/notarize@npm:2.5.0" dependencies: debug: "npm:^4.1.1" fs-extra: "npm:^9.0.1" promise-retry: "npm:^2.0.1" - checksum: 10c0/37729f49effdf43fe6a4c5ea8f90c624c6c5f4eab3e040d91366a5b23aade9481148266d5015de64eb369ed75886664f22b9a7c34aeb9355978b78e2c901ea5e + checksum: 10c0/262c6a90db4b18c82abb2a8f5349d1bf19ac34a440fe6c01b8aee302b1c886a79906693e6c3fdba2a4efa23a6519abf2113a882b438f7b6687eb2daed3da2afa languageName: node linkType: hard @@ -1569,6 +1569,19 @@ __metadata: languageName: node linkType: hard +"@llm-tools/embedjs-loader-sitemap@npm:^0.1.24": + version: 0.1.24 + resolution: "@llm-tools/embedjs-loader-sitemap@npm:0.1.24" + dependencies: + "@llm-tools/embedjs-interfaces": "npm:0.1.24" + "@llm-tools/embedjs-loader-web": "npm:0.1.24" + debug: "npm:^4.4.0" + md5: "npm:^2.3.0" + sitemapper: "npm:^3.2.18" + checksum: 10c0/afda59d2f2e60cafd1398bf28b45ded8abc3849e354f85afd1b37c17cdc16d9483627352a85f90d6ecc52ff6aaa5a5c52bc6ebbfafdbc30b8005b26f6a95d5b9 + languageName: node + linkType: hard + "@llm-tools/embedjs-loader-web@npm:0.1.24, @llm-tools/embedjs-loader-web@npm:^0.1.24": version: 0.1.24 resolution: "@llm-tools/embedjs-loader-web@npm:0.1.24" @@ -2740,6 +2753,7 @@ __metadata: "@electron-toolkit/preload": "npm:^3.0.0" "@electron-toolkit/tsconfig": "npm:^1.0.1" "@electron-toolkit/utils": "npm:^3.0.0" + "@electron/notarize": "npm:^2.5.0" "@google/generative-ai": "npm:^0.21.0" "@hello-pangea/dnd": "npm:^16.6.0" "@kangfenmao/keyv-storage": "npm:^0.1.0" @@ -2749,6 +2763,7 @@ __metadata: "@llm-tools/embedjs-loader-markdown": "npm:^0.1.24" "@llm-tools/embedjs-loader-msoffice": "npm:^0.1.24" "@llm-tools/embedjs-loader-pdf": "npm:^0.1.24" + "@llm-tools/embedjs-loader-sitemap": "npm:^0.1.24" "@llm-tools/embedjs-loader-web": "npm:^0.1.24" "@llm-tools/embedjs-loader-xml": "npm:^0.1.24" "@llm-tools/embedjs-ollama": "npm:^0.1.24" @@ -6479,7 +6494,7 @@ __metadata: languageName: node linkType: hard -"got@npm:^11.8.5": +"got@npm:^11.8.0, got@npm:^11.8.5": version: 11.8.6 resolution: "got@npm:11.8.6" dependencies: @@ -7384,6 +7399,13 @@ __metadata: languageName: node linkType: hard +"is-gzip@npm:2.0.0": + version: 2.0.0 + resolution: "is-gzip@npm:2.0.0" + checksum: 10c0/80868d1c2e8a0cba249485bd8d99e98141366c2f5afa206ec6cdfc9b2393aa0c72b223ded82666e28e8d878f039fca7b19a1d0894c0eff30d330e27d1a2c9cde + languageName: node + linkType: hard + "is-hexadecimal@npm:^2.0.0": version: 2.0.1 resolution: "is-hexadecimal@npm:2.0.1" @@ -9918,7 +9940,7 @@ __metadata: languageName: node linkType: hard -"p-limit@npm:^3.0.2": +"p-limit@npm:^3.0.2, p-limit@npm:^3.1.0": version: 3.1.0 resolution: "p-limit@npm:3.1.0" dependencies: @@ -12186,6 +12208,18 @@ __metadata: languageName: node linkType: hard +"sitemapper@npm:^3.2.18": + version: 3.2.18 + resolution: "sitemapper@npm:3.2.18" + dependencies: + fast-xml-parser: "npm:^4.5.0" + got: "npm:^11.8.0" + is-gzip: "npm:2.0.0" + p-limit: "npm:^3.1.0" + checksum: 10c0/d23b0bb067a182313598494def92c8839f12ead34952a45d60afb7b5dd458e4b363892b1809cb8a99a8c358448cd0e18b2da9e9bed1289a7fbcda7a8f54971ec + languageName: node + linkType: hard + "slash@npm:^3.0.0": version: 3.0.0 resolution: "slash@npm:3.0.0"