whai/cherry-studio
1
1
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

build: add matrix

Browse Source
This commit is contained in:
kangfenmao 2024-12-19 15:07:17 +08:00
parent ca6027dd83
commit 9aa2a4727d
23 changed files with 263 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
.github/workflows/build.yml vendored Normal file
View File

@ -0,0 +1,94 @@
name: Release
on:
workflow_dispatch:
inputs:
version:
description: 'Version (e.g. v1.2.3)'
required: true
type: string
push:
tags:
- v*.*.*
permissions:
contents: write
jobs:
release:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [macos-13, macos-latest]
arch: [x64, arm64]
exclude:
- os: windows-latest
arch: arm64
- os: macos-latest
arch: x64
- os: macos-13
arch: arm64
steps:
- name: Check out Git repository
uses: actions/checkout@v3
- name: Install Node.js
uses: actions/setup-node@v3
with:
node-version: 20
arch: ${{ matrix.arch }}
- name: Install corepack
run: corepack enable && corepack prepare yarn@4.3.1 --activate
- name: Install Dependencies
run: yarn install
- name: Build Linux
if: matrix.os == 'ubuntu-latest'
run: yarn build:linux
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
ARCH: ${{ matrix.arch }}
- name: Build Mac
if: matrix.os == 'macos-13'
run: yarn build:mac
env:
CSC_LINK: ${{ secrets.CSC_LINK }}
CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }}
APPLE_ID: ${{ vars.APPLE_ID }}
APPLE_APP_SPECIFIC_PASSWORD: ${{ vars.APPLE_APP_SPECIFIC_PASSWORD }}
APPLE_TEAM_ID: ${{ vars.APPLE_TEAM_ID }}
GH_TOKEN: ${{ secrets.GH_TOKEN }}
ARCH: ${{ matrix.arch }}
- name: Build Windows
if: matrix.os == 'windows-latest'
run: yarn build:win
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
ARCH: ${{ matrix.arch }}
- name: Replace spaces in filenames
run: node scripts/replaceSpaces.js
- name: Release
uses: softprops/action-gh-release@v2
with:
draft: true
files: |
dist/*.exe
dist/*.zip
dist/*.dmg
dist/*.AppImage
dist/*.snap
dist/*.deb
dist/*.rpm
dist/*.tar.gz
dist/latest*.yml
dist/*.blockmap
env:
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}

16
.github/workflows/release.yml vendored
View File

@ -20,7 +20,15 @@ jobs:
strategy:
matrix:
os: [macos-latest, windows-latest, ubuntu-latest]
os: [macos-13, macos-latest, windows-latest, ubuntu-latest]
arch: [x64, arm64]
exclude:
- os: windows-latest
arch: arm64
- os: macos-latest
arch: x64
- os: macos-13
arch: arm64
steps:
- name: Check out Git repository
@ -30,6 +38,7 @@ jobs:
uses: actions/setup-node@v3
with:
node-version: 20
arch: ${{ matrix.arch }}
- name: Install corepack
run: corepack enable && corepack prepare yarn@4.3.1 --activate
@ -42,9 +51,10 @@ jobs:
run: yarn build:linux
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
ARCH: ${{ matrix.arch }}
- name: Build Mac
if: matrix.os == 'macos-latest'
if: matrix.os == 'macos-13'
run: yarn build:mac
env:
CSC_LINK: ${{ secrets.CSC_LINK }}
@ -53,12 +63,14 @@ jobs:
APPLE_APP_SPECIFIC_PASSWORD: ${{ vars.APPLE_APP_SPECIFIC_PASSWORD }}
APPLE_TEAM_ID: ${{ vars.APPLE_TEAM_ID }}
GH_TOKEN: ${{ secrets.GH_TOKEN }}
ARCH: ${{ matrix.arch }}
- name: Build Windows
if: matrix.os == 'windows-latest'
run: yarn build:win
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
ARCH: ${{ matrix.arch }}
- name: Replace spaces in filenames
run: node scripts/replaceSpaces.js

53
.yarn/patches/@electron-notarize-npm-2.3.2-535908a4bd.patch
View File

@ -1,53 +0,0 @@
diff --git a/lib/check-signature.js b/lib/check-signature.js
index 324568af71bcc4372c9f959131ecd24122848c86..677348e0a138ff608b2ac41f592d813b15ee4956 100644
--- a/lib/check-signature.js
+++ b/lib/check-signature.js
@@ -41,16 +41,12 @@ const spawn_1 = require("./spawn");
const debug_1 = __importDefault(require("debug"));
const d = (0, debug_1.default)('electron-notarize');
const codesignDisplay = (opts) => __awaiter(void 0, void 0, void 0, function* () {
- const result = yield (0, spawn_1.spawn)('codesign', ['-dv', '-vvvv', '--deep', path.basename(opts.appPath)], {
- cwd: path.dirname(opts.appPath),
- });
+ const result = yield (0, spawn_1.spawn)('codesign', ['-dv', '-vvvv', '--deep', opts.appPath]);
return result;
});
const codesign = (opts) => __awaiter(void 0, void 0, void 0, function* () {
d('attempting to check codesign of app:', opts.appPath);
- const result = yield (0, spawn_1.spawn)('codesign', ['-vvv', '--deep', '--strict', path.basename(opts.appPath)], {
- cwd: path.dirname(opts.appPath),
- });
+ const result = yield (0, spawn_1.spawn)('codesign', ['-vvv', '--deep', '--strict', opts.appPath]);
return result;
});
function checkSignatures(opts) {
diff --git a/lib/notarytool.js b/lib/notarytool.js
index 1ab090efb2101fc8bee5553445e0349c54474421..a5ddfd922197449fc56078e4a7e9a2ee5d8d207d 100644
--- a/lib/notarytool.js
+++ b/lib/notarytool.js
@@ -92,9 +92,7 @@ function notarizeAndWaitForNotaryTool(opts) {
else {
filePath = path.resolve(dir, `${path.parse(opts.appPath).name}.zip`);
d('zipping application to:', filePath);
- const zipResult = yield (0, spawn_1.spawn)('ditto', ['-c', '-k', '--sequesterRsrc', '--keepParent', path.basename(opts.appPath), filePath], {
- cwd: path.dirname(opts.appPath),
- });
+ const zipResult = yield (0, spawn_1.spawn)('ditto', ['-c', '-k', '--sequesterRsrc', '--keepParent', opts.appPath, filePath]);
if (zipResult.code !== 0) {
throw new Error(`Failed to zip application, exited with code: ${zipResult.code}\n\n${zipResult.output}`);
}
diff --git a/lib/staple.js b/lib/staple.js
index 47dbd85b2fc279d999b57f47fb8171e1cc674436..f8829e6ac54fcd630a730d12d75acc1591b953b6 100644
--- a/lib/staple.js
+++ b/lib/staple.js
@@ -43,9 +43,7 @@ const d = (0, debug_1.default)('electron-notarize:staple');
function stapleApp(opts) {
return __awaiter(this, void 0, void 0, function* () {
d('attempting to staple app:', opts.appPath);
- const result = yield (0, spawn_1.spawn)('xcrun', ['stapler', 'staple', '-v', path.basename(opts.appPath)], {
- cwd: path.dirname(opts.appPath),
- });
+ const result = yield (0, spawn_1.spawn)('xcrun', ['stapler', 'staple', '-v', opts.appPath]);
if (result.code !== 0) {
throw new Error(`Failed to staple your application with code: ${result.code}\n\n${result.output}`);
}

1
electron-builder.yml
View File

@ -13,6 +13,7 @@ files:
- '!local'
asarUnpack:
- resources/**
- '**/*.{node,dll,metal,exp,lib}'
win:
executableName: Cherry Studio
nsis:

3
electron.vite.config.ts
View File

@ -16,6 +16,7 @@ export default defineConfig({
'@llm-tools/embedjs-loader-msoffice',
'@llm-tools/embedjs-loader-xml',
'@llm-tools/embedjs-loader-pdf',
'@llm-tools/embedjs-loader-sitemap',
'@lancedb/lancedb'
]
})
@ -29,7 +30,7 @@ export default defineConfig({
},
build: {
rollupOptions: {
external: ['@lancedb/lancedb', '@llm-tools/embedjs-loader-sitemap']
external: ['@lancedb/lancedb']
}
}
},

14
package.json
View File

@ -28,9 +28,14 @@
"build": "npm run typecheck && electron-vite build",
"postinstall": "electron-builder install-app-deps",
"build:unpack": "dotenv npm run build && electron-builder --dir",
"build:win": "dotenv npm run build && electron-builder --win --publish never",
"build:mac": "dotenv electron-vite build && electron-builder --mac --publish never",
"build:linux": "dotenv electron-vite build && electron-builder --linux --publish never",
"build:win": "dotenv npm run build && electron-builder --win --$ARCH",
"build:win:x64": "dotenv npm run build && electron-builder --win --x64",
"build:mac": "dotenv electron-vite build && electron-builder --mac --$ARCH",
"build:mac:arm64": "dotenv electron-vite build && electron-builder --mac --arm64",
"build:mac:x64": "dotenv electron-vite build && electron-builder --mac --x64",
"build:linux": "dotenv electron-vite build && electron-builder --linux --$ARCH",
"build:linux:arm64": "dotenv electron-vite build && electron-builder --linux --arm64",
"build:linux:x64": "dotenv electron-vite build && electron-builder --linux --x64",
"release": "node scripts/version.js",
"publish": "yarn release patch push",
"pulish:artifacts": "cd packages/artifacts && npm publish && cd -",
@ -40,12 +45,14 @@
"dependencies": {
"@electron-toolkit/preload": "^3.0.0",
"@electron-toolkit/utils": "^3.0.0",
"@electron/notarize": "^2.5.0",
"@llm-tools/embedjs": "^0.1.24",
"@llm-tools/embedjs-lancedb": "^0.1.24",
"@llm-tools/embedjs-loader-csv": "^0.1.24",
"@llm-tools/embedjs-loader-markdown": "^0.1.24",
"@llm-tools/embedjs-loader-msoffice": "^0.1.24",
"@llm-tools/embedjs-loader-pdf": "^0.1.24",
"@llm-tools/embedjs-loader-sitemap": "^0.1.24",
"@llm-tools/embedjs-loader-web": "^0.1.24",
"@llm-tools/embedjs-loader-xml": "^0.1.24",
"@llm-tools/embedjs-ollama": "^0.1.24",
@ -138,7 +145,6 @@
"react-dom": "^17.0.0 || ^18.0.0"
},
"resolutions": {
"@electron/notarize@npm:2.2.1": "patch:@electron/notarize@npm%3A2.3.2#~/.yarn/patches/@electron-notarize-npm-2.3.2-535908a4bd.patch",
"pdf-parse@npm:1.1.1": "patch:pdf-parse@npm%3A1.1.1#~/.yarn/patches/pdf-parse-npm-1.1.1-04a6109b2a.patch"
},
"packageManager": "yarn@4.5.0"

57
src/main/services/KnowledgeService.ts
View File

@ -7,9 +7,10 @@ import { LanceDb } from '@llm-tools/embedjs-lancedb'
import { MarkdownLoader } from '@llm-tools/embedjs-loader-markdown'
import { DocxLoader } from '@llm-tools/embedjs-loader-msoffice'
import { PdfLoader } from '@llm-tools/embedjs-loader-pdf'
import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap'
import { WebLoader } from '@llm-tools/embedjs-loader-web'
import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'
import { FileType, RagAppRequestParams } from '@types'
import { FileType, KnowledgeBaseParams, KnowledgeItem } from '@types'
import { app } from 'electron'
class KnowledgeService {
@ -25,7 +26,7 @@ class KnowledgeService {
}
}
private getRagApplication = async ({ id, model, apiKey, baseURL }: RagAppRequestParams): Promise<RAGApplication> => {
private getRagApplication = async ({ id, model, apiKey, baseURL }: KnowledgeBaseParams): Promise<RAGApplication> => {
return new RAGApplicationBuilder()
.setModel('NO_MODEL')
.setEmbeddingModel(
@ -42,13 +43,13 @@ class KnowledgeService {
public create = async (
_: Electron.IpcMainInvokeEvent,
{ id, model, apiKey, baseURL }: RagAppRequestParams
{ id, model, apiKey, baseURL }: KnowledgeBaseParams
): Promise<void> => {
this.getRagApplication({ id, model, apiKey, baseURL })
}
public reset = async (_: Electron.IpcMainInvokeEvent, { config }: { config: RagAppRequestParams }): Promise<void> => {
const ragApplication = await this.getRagApplication(config)
public reset = async (_: Electron.IpcMainInvokeEvent, { base }: { base: KnowledgeBaseParams }): Promise<void> => {
const ragApplication = await this.getRagApplication(base)
await ragApplication.reset()
}
@ -61,27 +62,41 @@ class KnowledgeService {
public add = async (
_: Electron.IpcMainInvokeEvent,
{ data, config }: { data: string | FileType; config: RagAppRequestParams }
{ base, item }: { base: KnowledgeBaseParams; item: KnowledgeItem }
): Promise<AddLoaderReturn> => {
const ragApplication = await this.getRagApplication(config)
const ragApplication = await this.getRagApplication(base)
if (typeof data === 'string') {
if (data.startsWith('http')) {
return await ragApplication.addLoader(new WebLoader({ urlOrContent: data }))
if (item.type === 'url') {
const content = item.content as string
if (content.startsWith('http')) {
return await ragApplication.addLoader(new WebLoader({ urlOrContent: content }))
}
return await ragApplication.addLoader(new TextLoader({ text: data }))
}
if (data.ext === '.pdf') {
return await ragApplication.addLoader(new PdfLoader({ filePathOrUrl: data.path }) as any)
if (item.type === 'sitemap') {
const content = item.content as string
return await ragApplication.addLoader(new SitemapLoader({ url: content }))
}
if (data.ext === '.docx') {
return await ragApplication.addLoader(new DocxLoader({ filePathOrUrl: data.path }) as any)
if (item.type === 'note') {
const content = item.content as string
return await ragApplication.addLoader(new TextLoader({ text: content }))
}
if (data.ext === '.md' || data.ext === '.mdx') {
return await ragApplication.addLoader(new MarkdownLoader({ filePathOrUrl: data.path }) as any)
if (item.type === 'file') {
const file = item.content as FileType
if (file.ext === '.pdf') {
return await ragApplication.addLoader(new PdfLoader({ filePathOrUrl: file.path }) as any)
}
if (file.ext === '.docx') {
return await ragApplication.addLoader(new DocxLoader({ filePathOrUrl: file.path }) as any)
}
if (file.ext.startsWith('.md')) {
return await ragApplication.addLoader(new MarkdownLoader({ filePathOrUrl: file.path }) as any)
}
}
return { entriesAdded: 0, uniqueId: '', loaderType: '' }
@ -89,17 +104,17 @@ class KnowledgeService {
public remove = async (
_: Electron.IpcMainInvokeEvent,
{ uniqueId, config }: { uniqueId: string; config: RagAppRequestParams }
{ uniqueId, base }: { uniqueId: string; base: KnowledgeBaseParams }
): Promise<void> => {
const ragApplication = await this.getRagApplication(config)
const ragApplication = await this.getRagApplication(base)
await ragApplication.deleteLoader(uniqueId)
}
public search = async (
_: Electron.IpcMainInvokeEvent,
{ search, config }: { search: string; config: RagAppRequestParams }
{ search, base }: { search: string; base: KnowledgeBaseParams }
): Promise<ExtractChunkData[]> => {
const ragApplication = await this.getRagApplication(config)
const ragApplication = await this.getRagApplication(base)
return await ragApplication.search(search)
}
}

12
src/preload/index.d.ts vendored
View File

@ -2,7 +2,7 @@ import { ElectronAPI } from '@electron-toolkit/preload'
import { AddLoaderReturn, ExtractChunkData } from '@llm-tools/embedjs-interfaces'
import { FileType } from '@renderer/types'
import { WebDavConfig } from '@renderer/types'
import { AppInfo, LanguageVarious, RagAppRequestParams } from '@renderer/types'
import { AppInfo, KnowledgeBaseParams, KnowledgeItem, LanguageVarious } from '@renderer/types'
import type { OpenDialogOptions } from 'electron'
import type { UpdateInfo } from 'electron-updater'
import { Readable } from 'stream'
@ -60,12 +60,12 @@ declare global {
update: (shortcuts: Shortcut[]) => Promise<void>
}
knowledgeBase: {
create: ({ id, model, apiKey, baseURL }: RagAppRequestParams) => Promise<void>
reset: ({ config }: { config: RagAppRequestParams }) => Promise<void>
create: ({ id, model, apiKey, baseURL }: KnowledgeBaseParams) => Promise<void>
reset: ({ base }: { base: KnowledgeBaseParams }) => Promise<void>
delete: (id: string) => Promise<void>
add: ({ data, config }: { data: string | FileType; config: RagAppRequestParams }) => Promise<AddLoaderReturn>
remove: ({ uniqueId, config }: { uniqueId: string; config: RagAppRequestParams }) => Promise<void>
search: ({ search, config }: { search: string; config: RagAppRequestParams }) => Promise<ExtractChunkData[]>
add: ({ base, item }: { base: KnowledgeBaseParams; item: KnowledgeItem }) => Promise<AddLoaderReturn>
remove: ({ uniqueId, base }: { uniqueId: string; base: KnowledgeBaseParams }) => Promise<void>
search: ({ search, base }: { search: string; base: KnowledgeBaseParams }) => Promise<ExtractChunkData[]>
}
}
}

18
src/preload/index.ts
View File

@ -1,5 +1,5 @@
import { electronAPI } from '@electron-toolkit/preload'
import { FileType, RagAppRequestParams, Shortcut, WebDavConfig } from '@types'
import { KnowledgeBaseParams, KnowledgeItem, Shortcut, WebDavConfig } from '@types'
import { contextBridge, ipcRenderer, OpenDialogOptions } from 'electron'
// Custom APIs for renderer
@ -52,16 +52,16 @@ const api = {
update: (shortcuts: Shortcut[]) => ipcRenderer.invoke('shortcuts:update', shortcuts)
},
knowledgeBase: {
create: ({ id, model, apiKey, baseURL }: RagAppRequestParams) =>
create: ({ id, model, apiKey, baseURL }: KnowledgeBaseParams) =>
ipcRenderer.invoke('knowledge-base:create', { id, model, apiKey, baseURL }),
reset: ({ config }: { config: RagAppRequestParams }) => ipcRenderer.invoke('knowledge-base:reset', { config }),
reset: ({ base }: { base: KnowledgeBaseParams }) => ipcRenderer.invoke('knowledge-base:reset', { base }),
delete: (id: string) => ipcRenderer.invoke('knowledge-base:delete', id),
add: ({ data, config }: { data: string | FileType; config: RagAppRequestParams }) =>
ipcRenderer.invoke('knowledge-base:add', { data, config }),
remove: ({ uniqueId, config }: { uniqueId: string; config: RagAppRequestParams }) =>
ipcRenderer.invoke('knowledge-base:remove', { uniqueId, config }),
search: ({ search, config }: { search: string; config: RagAppRequestParams }) =>
ipcRenderer.invoke('knowledge-base:search', { search, config })
add: ({ base, item }: { base: KnowledgeBaseParams; item: KnowledgeItem }) =>
ipcRenderer.invoke('knowledge-base:add', { base, item }),
remove: ({ uniqueId, base }: { uniqueId: string; base: KnowledgeBaseParams }) =>
ipcRenderer.invoke('knowledge-base:remove', { uniqueId, base }),
search: ({ search, base }: { search: string; base: KnowledgeBaseParams }) =>
ipcRenderer.invoke('knowledge-base:search', { search, base })
}
}

5
src/renderer/src/hooks/useknowledge.ts
View File

@ -2,7 +2,7 @@
import { db } from '@renderer/databases/index'
import KnowledgeQueue from '@renderer/queue/KnowledgeQueue'
import FileManager from '@renderer/services/FileManager'
import { getRagAppRequestParams } from '@renderer/services/KnowledgeService'
import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService'
import { RootState } from '@renderer/store'
import {
addBase,
@ -143,9 +143,8 @@ export const useKnowledge = (baseId: string) => {
const removeItem = async (item: KnowledgeItem) => {
dispatch(removeItemAction({ baseId, item }))
if (base) {
const config = getRagAppRequestParams(base)
if (item?.uniqueId) {
await window.api.knowledgeBase.remove({ uniqueId: item.uniqueId, config })
await window.api.knowledgeBase.remove({ uniqueId: item.uniqueId, base: getKnowledgeBaseParams(base) })
}
if (item.type === 'file' && typeof item.content === 'object') {
await FileManager.deleteFile(item.content.id)

5
src/renderer/src/i18n/locales/en-us.json
View File

@ -560,8 +560,9 @@
"no_bases": "No knowledge bases available",
"clear_selection": "Clear selection",
"delete_confirm": "Are you sure you want to delete this knowledge base?",
"sitemaps": "Site Maps",
"add_sitemap": "Add Site Map"
"sitemaps": "Websites",
"add_sitemap": "Website Map",
"sitemap_placeholder": "Enter Website Map URL"
}
}
}

5
src/renderer/src/i18n/locales/ru-ru.json
View File

@ -560,8 +560,9 @@
"no_bases": "База знаний не найдена",
"clear_selection": "Очистить выбор",
"delete_confirm": "Вы уверены, что хотите удалить эту базу знаний?",
"sitemaps": "Карта сайта",
"add_sitemap": "Добавить карту сайта"
"sitemaps": "Сайты",
"add_sitemap": "Карта сайта",
"sitemap_placeholder": "Введите URL карты сайта"
}
}
}

5
src/renderer/src/i18n/locales/zh-cn.json
View File

@ -549,8 +549,9 @@
"no_bases": "暂无知识库",
"clear_selection": "清除选择",
"delete_confirm": "确定要删除此知识库吗?",
"sitemaps": "站点地图",
"add_sitemap": "添加站点地图"
"sitemaps": "网站",
"add_sitemap": "站点地图",
"sitemap_placeholder": "请输入站点地图 URL"
}
}
}

5
src/renderer/src/i18n/locales/zh-tw.json
View File

@ -548,8 +548,9 @@
"no_bases": "暫無知識庫",
"clear_selection": "清除選擇",
"delete_confirm": "確定要刪除此知識庫嗎?",
"sitemaps": "站點地圖",
"add_sitemap": "添加站點地圖"
"sitemaps": "網站",
"add_sitemap": "網站地圖",
"sitemap_placeholder": "請輸入網站地圖 URL"
}
}
}

2
src/renderer/src/pages/knowledge/KnowledgePage.tsx
View File

@ -27,7 +27,7 @@ const KnowledgePage: FC = () => {
if (!selectedBase) {
return setSelectedBase(bases[0])
}
if (selectedBase && !bases.includes(selectedBase)) {
if (selectedBase && !bases.find((base) => base.id === selectedBase.id)) {
return setSelectedBase(bases[0])
}
}

4
src/renderer/src/pages/knowledge/components/AddKnowledgePopup.tsx
View File

@ -2,7 +2,7 @@ import { TopView } from '@renderer/components/TopView'
import { isEmbeddingModel } from '@renderer/config/models'
import { useKnowledgeBases } from '@renderer/hooks/useknowledge'
import { useProviders } from '@renderer/hooks/useProvider'
import { getRagAppRequestParams } from '@renderer/services/KnowledgeService'
import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService'
import { getModelUniqId } from '@renderer/services/ModelService'
import { Model } from '@renderer/types'
import { Form, Input, Modal, Select } from 'antd'
@ -64,7 +64,7 @@ const PopupContainer: React.FC<Props> = ({ title, resolve }) => {
updated_at: Date.now()
}
await window.api.knowledgeBase.create(getRagAppRequestParams(newBase))
await window.api.knowledgeBase.create(getKnowledgeBaseParams(newBase))
addKnowledgeBase(newBase as any)
setOpen(false)

4
src/renderer/src/pages/knowledge/components/KnowledgeSearchPopup.tsx
View File

@ -1,6 +1,6 @@
import { ExtractChunkData } from '@llm-tools/embedjs-interfaces'
import { TopView } from '@renderer/components/TopView'
import { getRagAppRequestParams } from '@renderer/services/KnowledgeService'
import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService'
import { KnowledgeBase } from '@renderer/types'
import { Input, List, Modal, Spin, Typography } from 'antd'
import { useState } from 'react'
@ -37,7 +37,7 @@ const PopupContainer: React.FC<Props> = ({ base, resolve }) => {
try {
const searchResults = await window.api.knowledgeBase.search({
search: value,
config: getRagAppRequestParams(base)
base: getKnowledgeBaseParams(base)
})
setResults(searchResults)
} catch (error) {

4
src/renderer/src/providers/BaseProvider.ts
View File

@ -1,5 +1,5 @@
import { getOllamaKeepAliveTime } from '@renderer/hooks/useOllama'
import { getRagAppRequestParams } from '@renderer/services/KnowledgeService'
import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService'
import store from '@renderer/store'
import { Assistant, FileType, Message, Provider, Suggestion } from '@renderer/types'
import { delay } from '@renderer/utils'
@ -96,7 +96,7 @@ export default abstract class BaseProvider {
const searchResults = await window.api.knowledgeBase.search({
search: message.content,
config: getRagAppRequestParams(base)
base: getKnowledgeBaseParams(base)
})
const references = take(searchResults, 5)

16
src/renderer/src/queue/KnowledgeQueue.ts
View File

@ -1,6 +1,6 @@
import { AddLoaderReturn } from '@llm-tools/embedjs-interfaces'
import db from '@renderer/databases'
import { getRagAppRequestParams } from '@renderer/services/KnowledgeService'
import { getKnowledgeBaseParams } from '@renderer/services/KnowledgeService'
import store from '@renderer/store'
import { clearCompletedProcessing, updateBaseItemUniqueId, updateItemProcessingStatus } from '@renderer/store/knowledge'
import { KnowledgeItem } from '@renderer/types'
@ -142,7 +142,7 @@ class KnowledgeQueue {
throw new Error(`[KnowledgeQueue] Knowledge base ${baseId} not found`)
}
const requestParams = getRagAppRequestParams(base)
const baseParams = getKnowledgeBaseParams(base)
const sourceItem = base.items.find((i) => i.id === item.id)
if (!sourceItem) {
@ -155,26 +155,22 @@ class KnowledgeQueue {
switch (item.type) {
case 'file':
console.log(`[KnowledgeQueue] Processing file: ${sourceItem.content}`)
result = await window.api.knowledgeBase.add({ data: sourceItem.content, config: requestParams })
console.log(`[KnowledgeQueue] Result: ${JSON.stringify(result)}`)
result = await window.api.knowledgeBase.add({ base: baseParams, item: sourceItem })
break
case 'url':
console.log(`[KnowledgeQueue] Processing URL: ${sourceItem.content}`)
result = await window.api.knowledgeBase.add({ data: sourceItem.content, config: requestParams })
console.log(`[KnowledgeQueue] Result: ${JSON.stringify(result)}`)
result = await window.api.knowledgeBase.add({ base: baseParams, item: sourceItem })
break
case 'sitemap':
console.log(`[KnowledgeQueue] Processing Sitemap: ${sourceItem.content}`)
result = await window.api.knowledgeBase.add({ data: sourceItem.content, config: requestParams })
console.log(`[KnowledgeQueue] Result: ${JSON.stringify(result)}`)
result = await window.api.knowledgeBase.add({ base: baseParams, item: sourceItem })
break
case 'note':
console.log(`[KnowledgeQueue] Processing note: ${sourceItem.content}`)
note = await db.knowledge_notes.get(item.id)
if (!note) throw new Error(`Source note ${item.id} not found`)
content = note.content as string
result = await window.api.knowledgeBase.add({ data: content, config: requestParams })
console.log(`[KnowledgeQueue] Result: ${JSON.stringify(result)}`)
result = await window.api.knowledgeBase.add({ base: baseParams, item: { ...sourceItem, content } })
break
}

9
src/renderer/src/services/KnowledgeService.ts
View File

@ -1,12 +1,17 @@
import AiProvider from '@renderer/providers/AiProvider'
import { KnowledgeBase, RagAppRequestParams } from '@renderer/types'
import { KnowledgeBase, KnowledgeBaseParams } from '@renderer/types'
import { isEmpty } from 'lodash'
import { getProviderByModel } from './AssistantService'
export const getRagAppRequestParams = (base: KnowledgeBase): RagAppRequestParams => {
export const getKnowledgeBaseParams = (base: KnowledgeBase): KnowledgeBaseParams => {
const provider = getProviderByModel(base.model)
const aiProvider = new AiProvider(provider)
if (provider.id === 'ollama' && isEmpty(provider.apiKey)) {
provider.apiKey = 'empty'
}
return {
id: base.id,
model: base.model.name,

12
src/renderer/src/store/knowledge.ts
View File

@ -48,12 +48,20 @@ const knowledgeSlice = createSlice({
if (base) {
if (action.payload.item.type === 'note') {
base.items.push(action.payload.item)
} else if (action.payload.item.type === 'url') {
}
if (action.payload.item.type === 'url') {
const urlExists = base.items.some((item) => item.content === action.payload.item.content)
if (!urlExists) {
base.items.push(action.payload.item)
}
} else if (action.payload.item.type === 'file') {
}
if (action.payload.item.type === 'sitemap') {
const sitemapExists = base.items.some((item) => item.content === action.payload.item.content)
if (!sitemapExists) {
base.items.push(action.payload.item)
}
}
if (action.payload.item.type === 'file') {
action.payload.item.created_at = new Date(action.payload.item.created_at).getTime()
action.payload.item.updated_at = new Date(action.payload.item.updated_at).getTime()
base.items.push(action.payload.item)

4
src/renderer/src/types/index.ts
View File

@ -197,6 +197,8 @@ export type KnowledgeItem = {
retryCount?: number
}
export type KnowledgeItemType = 'file' | 'url' | 'note' | 'sitemap'
export interface KnowledgeBase {
id: string
name: string
@ -207,7 +209,7 @@ export interface KnowledgeBase {
updated_at: number
}
export type RagAppRequestParams = {
export type KnowledgeBaseParams = {
id: string
model: string
apiKey: string

54
yarn.lock
View File

@ -504,25 +504,25 @@ __metadata:
languageName: node
linkType: hard
"@electron/notarize@npm:2.3.2":
version: 2.3.2
resolution: "@electron/notarize@npm:2.3.2"
"@electron/notarize@npm:2.2.1":
version: 2.2.1
resolution: "@electron/notarize@npm:2.2.1"
dependencies:
debug: "npm:^4.1.1"
fs-extra: "npm:^9.0.1"
promise-retry: "npm:^2.0.1"
checksum: 10c0/539ed5cd264c3885fd3ca9c0b243144e3e2856d767de3999da1e3f94f0d79db57cbb08862b640270dfad0292bc5345cd7177db096da2061e28e15a6b85946b32
checksum: 10c0/d3fbbaaf26e809d4484f87826f02ba9108eba222a495ff533d9728a58a0cca6e267764baefc5616952318a6674eb6d3b7d07b1136ca0254da1c51012a0e6e6ae
languageName: node
linkType: hard
"@electron/notarize@patch:@electron/notarize@npm%3A2.3.2#~/.yarn/patches/@electron-notarize-npm-2.3.2-535908a4bd.patch":
version: 2.3.2
resolution: "@electron/notarize@patch:@electron/notarize@npm%3A2.3.2#~/.yarn/patches/@electron-notarize-npm-2.3.2-535908a4bd.patch::version=2.3.2&hash=dd0991"
"@electron/notarize@npm:^2.5.0":
version: 2.5.0
resolution: "@electron/notarize@npm:2.5.0"
dependencies:
debug: "npm:^4.1.1"
fs-extra: "npm:^9.0.1"
promise-retry: "npm:^2.0.1"
checksum: 10c0/37729f49effdf43fe6a4c5ea8f90c624c6c5f4eab3e040d91366a5b23aade9481148266d5015de64eb369ed75886664f22b9a7c34aeb9355978b78e2c901ea5e
checksum: 10c0/262c6a90db4b18c82abb2a8f5349d1bf19ac34a440fe6c01b8aee302b1c886a79906693e6c3fdba2a4efa23a6519abf2113a882b438f7b6687eb2daed3da2afa
languageName: node
linkType: hard
@ -1569,6 +1569,19 @@ __metadata:
languageName: node
linkType: hard
"@llm-tools/embedjs-loader-sitemap@npm:^0.1.24":
version: 0.1.24
resolution: "@llm-tools/embedjs-loader-sitemap@npm:0.1.24"
dependencies:
"@llm-tools/embedjs-interfaces": "npm:0.1.24"
"@llm-tools/embedjs-loader-web": "npm:0.1.24"
debug: "npm:^4.4.0"
md5: "npm:^2.3.0"
sitemapper: "npm:^3.2.18"
checksum: 10c0/afda59d2f2e60cafd1398bf28b45ded8abc3849e354f85afd1b37c17cdc16d9483627352a85f90d6ecc52ff6aaa5a5c52bc6ebbfafdbc30b8005b26f6a95d5b9
languageName: node
linkType: hard
"@llm-tools/embedjs-loader-web@npm:0.1.24, @llm-tools/embedjs-loader-web@npm:^0.1.24":
version: 0.1.24
resolution: "@llm-tools/embedjs-loader-web@npm:0.1.24"
@ -2740,6 +2753,7 @@ __metadata:
"@electron-toolkit/preload": "npm:^3.0.0"
"@electron-toolkit/tsconfig": "npm:^1.0.1"
"@electron-toolkit/utils": "npm:^3.0.0"
"@electron/notarize": "npm:^2.5.0"
"@google/generative-ai": "npm:^0.21.0"
"@hello-pangea/dnd": "npm:^16.6.0"
"@kangfenmao/keyv-storage": "npm:^0.1.0"
@ -2749,6 +2763,7 @@ __metadata:
"@llm-tools/embedjs-loader-markdown": "npm:^0.1.24"
"@llm-tools/embedjs-loader-msoffice": "npm:^0.1.24"
"@llm-tools/embedjs-loader-pdf": "npm:^0.1.24"
"@llm-tools/embedjs-loader-sitemap": "npm:^0.1.24"
"@llm-tools/embedjs-loader-web": "npm:^0.1.24"
"@llm-tools/embedjs-loader-xml": "npm:^0.1.24"
"@llm-tools/embedjs-ollama": "npm:^0.1.24"
@ -6479,7 +6494,7 @@ __metadata:
languageName: node
linkType: hard
"got@npm:^11.8.5":
"got@npm:^11.8.0, got@npm:^11.8.5":
version: 11.8.6
resolution: "got@npm:11.8.6"
dependencies:
@ -7384,6 +7399,13 @@ __metadata:
languageName: node
linkType: hard
"is-gzip@npm:2.0.0":
version: 2.0.0
resolution: "is-gzip@npm:2.0.0"
checksum: 10c0/80868d1c2e8a0cba249485bd8d99e98141366c2f5afa206ec6cdfc9b2393aa0c72b223ded82666e28e8d878f039fca7b19a1d0894c0eff30d330e27d1a2c9cde
languageName: node
linkType: hard
"is-hexadecimal@npm:^2.0.0":
version: 2.0.1
resolution: "is-hexadecimal@npm:2.0.1"
@ -9918,7 +9940,7 @@ __metadata:
languageName: node
linkType: hard
"p-limit@npm:^3.0.2":
"p-limit@npm:^3.0.2, p-limit@npm:^3.1.0":
version: 3.1.0
resolution: "p-limit@npm:3.1.0"
dependencies:
@ -12186,6 +12208,18 @@ __metadata:
languageName: node
linkType: hard
"sitemapper@npm:^3.2.18":
version: 3.2.18
resolution: "sitemapper@npm:3.2.18"
dependencies:
fast-xml-parser: "npm:^4.5.0"
got: "npm:^11.8.0"
is-gzip: "npm:2.0.0"
p-limit: "npm:^3.1.0"
checksum: 10c0/d23b0bb067a182313598494def92c8839f12ead34952a45d60afb7b5dd458e4b363892b1809cb8a99a8c358448cd0e18b2da9e9bed1289a7fbcda7a8f54971ec
languageName: node
linkType: hard
"slash@npm:^3.0.0":
version: 3.0.0
resolution: "slash@npm:3.0.0"