From aa4f94f8a4d3f23d13d48128c861425d4278f549 Mon Sep 17 00:00:00 2001 From: kangfenmao Date: Tue, 24 Dec 2024 16:34:29 +0800 Subject: [PATCH] build: download npm node native modules --- .github/workflows/release.yml | 27 +++++------- ...embedjs-libsql-npm-0.1.25-fad000d74c.patch | 31 ++----------- electron-builder.yml | 17 ++++++-- package.json | 13 +++--- scripts/after-pack.js | 42 ++++++++++++++++++ scripts/download-npm.js | 14 ++++++ .../{removeLocales.js => remove-locales.js} | 0 .../{replaceSpaces.js => replace-spaces.js} | 0 scripts/utils.js | 43 +++++++++++++++++++ yarn.lock | 25 ++++++++++- 10 files changed, 156 insertions(+), 56 deletions(-) create mode 100644 scripts/after-pack.js create mode 100644 scripts/download-npm.js rename scripts/{removeLocales.js => remove-locales.js} (100%) rename scripts/{replaceSpaces.js => replace-spaces.js} (100%) create mode 100644 scripts/utils.js diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad60467f..7d40050c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,15 +15,7 @@ jobs: strategy: matrix: - os: [macos-13, macos-latest, windows-latest, ubuntu-latest] - arch: [x64, arm64] - exclude: - - os: windows-latest - arch: arm64 - - os: macos-latest - arch: x64 - - os: macos-13 - arch: arm64 + os: [macos-latest, windows-latest, ubuntu-latest] steps: - name: Check out Git repository @@ -33,7 +25,6 @@ jobs: uses: actions/setup-node@v3 with: node-version: 20 - arch: ${{ matrix.arch }} - name: Install corepack run: corepack enable && corepack prepare yarn@4.3.1 --activate @@ -57,14 +48,18 @@ jobs: - name: Build Linux if: matrix.os == 'ubuntu-latest' - run: yarn build:linux + run: | + yarn download:npm linux + yarn build:linux + env: GH_TOKEN: ${{ secrets.GH_TOKEN }} - ARCH: ${{ matrix.arch }} - name: Build Mac - if: matrix.os == 'macos-13' || matrix.os == 'macos-latest' - run: yarn build:mac && mv dist/latest-mac.yml dist/latest-mac-${{ matrix.arch }}.yml + if: matrix.os == 'macos-latest' + run: | + yarn download:npm darwin + yarn build:mac env: CSC_LINK: ${{ secrets.CSC_LINK }} CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }} @@ -72,17 +67,15 @@ jobs: APPLE_APP_SPECIFIC_PASSWORD: ${{ vars.APPLE_APP_SPECIFIC_PASSWORD }} APPLE_TEAM_ID: ${{ vars.APPLE_TEAM_ID }} GH_TOKEN: ${{ secrets.GH_TOKEN }} - ARCH: ${{ matrix.arch }} - name: Build Windows if: matrix.os == 'windows-latest' run: yarn build:win env: GH_TOKEN: ${{ secrets.GH_TOKEN }} - ARCH: ${{ matrix.arch }} - name: Replace spaces in filenames - run: node scripts/replaceSpaces.js + run: node scripts/replace-spaces.js - name: Release uses: softprops/action-gh-release@v2 diff --git a/.yarn/patches/@llm-tools-embedjs-libsql-npm-0.1.25-fad000d74c.patch b/.yarn/patches/@llm-tools-embedjs-libsql-npm-0.1.25-fad000d74c.patch index ed51958c..0c794bfa 100644 --- a/.yarn/patches/@llm-tools-embedjs-libsql-npm-0.1.25-fad000d74c.patch +++ b/.yarn/patches/@llm-tools-embedjs-libsql-npm-0.1.25-fad000d74c.patch @@ -1,30 +1,5 @@ -diff --git a/src/libsql-db.cjs b/src/libsql-db.cjs -index 076e838a1a144fa80687aa27ed92f488c2b532cb..fd00187134cda688c20b2929934ca5bfd4899c42 100644 ---- a/src/libsql-db.cjs -+++ b/src/libsql-db.cjs -@@ -45,9 +45,9 @@ class LibSqlDb { - } - async similaritySearch(query, k) { - const statement = `SELECT id, pageContent, uniqueLoaderId, source, metadata, -- vector_distance_cos(vector, vector32('[${query.join(',')}]')) -+ vector_distance_cos(vector, vector32('[${query.join(',')}]')) as distance - FROM ${this.tableName} -- ORDER BY vector_distance_cos(vector, vector32('[${query.join(',')}]')) ASC -+ ORDER BY distance ASC - LIMIT ${k};`; - this.debug(`Executing statement - ${(0, embedjs_utils_1.truncateCenterString)(statement, 700)}`); - const results = await this.client.execute(statement); -@@ -56,7 +56,7 @@ class LibSqlDb { - return { - metadata, - pageContent: result.pageContent.toString(), -- score: 1, -+ score: result.distance, - }; - }); - } diff --git a/src/libsql-db.js b/src/libsql-db.js -index 58c42e4910bd0e53bc497ff9b9702b1f7a961266..3b124ff26f3b7cdca72eed5b872d7410c32eb70a 100644 +index 58c42e4910bd0e53bc497ff9b9702b1f7a961266..c02351dc95ba36fc4db97ddfb69206539fb4de1c 100644 --- a/src/libsql-db.js +++ b/src/libsql-db.js @@ -41,9 +41,9 @@ export class LibSqlDb { @@ -35,7 +10,7 @@ index 58c42e4910bd0e53bc497ff9b9702b1f7a961266..3b124ff26f3b7cdca72eed5b872d7410 + vector_distance_cos(vector, vector32('[${query.join(',')}]')) as distance FROM ${this.tableName} - ORDER BY vector_distance_cos(vector, vector32('[${query.join(',')}]')) ASC -+ ORDER BY distance ASC ++ ORDER BY distance DESC LIMIT ${k};`; this.debug(`Executing statement - ${truncateCenterString(statement, 700)}`); const results = await this.client.execute(statement); @@ -44,7 +19,7 @@ index 58c42e4910bd0e53bc497ff9b9702b1f7a961266..3b124ff26f3b7cdca72eed5b872d7410 metadata, pageContent: result.pageContent.toString(), - score: 1, -+ score: result.distance, ++ score: 1 - result.distance, }; }); } diff --git a/electron-builder.yml b/electron-builder.yml index 5970be70..e8449d57 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -27,7 +27,6 @@ files: - '!node_modules/pdf-parse/lib/pdf.js/{v1.9.426,v1.10.88,v2.0.550}' - '!node_modules/mammoth/{mammoth.browser.js,mammoth.browser.min.js}' - '!node_modules/html2canvas/dist/{html2canvas.min.js,html2canvas.esm.js}' - asarUnpack: - resources/** - '**/*.{node,dll,metal,exp,lib}' @@ -42,17 +41,29 @@ nsis: oneClick: false mac: entitlementsInherit: build/entitlements.mac.plist + notarize: false extendInfo: - NSCameraUsageDescription: Application requests access to the device's camera. - NSMicrophoneUsageDescription: Application requests access to the device's microphone. - NSDocumentsFolderUsageDescription: Application requests access to the user's Documents folder. - NSDownloadsFolderUsageDescription: Application requests access to the user's Downloads folder. - notarize: false + target: + - target: dmg + arch: + - arm64 + - x64 + - target: zip + arch: + - arm64 + - x64 dmg: artifactName: ${productName}-${version}-${arch}.${ext} linux: target: - target: AppImage + arch: + - arm64 + - x64 maintainer: electronjs.org category: Utility appImage: @@ -62,7 +73,7 @@ publish: url: https://cherrystudio.ocool.online electronDownload: mirror: https://npmmirror.com/mirrors/electron/ -afterPack: scripts/removeLocales.js +afterPack: scripts/after-pack.js afterSign: scripts/notarize.js releaseInfo: releaseNotes: | diff --git a/package.json b/package.json index 5096441d..82aa6b5b 100644 --- a/package.json +++ b/package.json @@ -25,24 +25,25 @@ "typecheck": "npm run typecheck:node && npm run typecheck:web", "start": "electron-vite preview", "dev": "electron-vite dev", - "analyze:renderer": "VISUALIZER_RENDERER=true yarn build", - "analyze:main": "VISUALIZER_MAIN=true yarn build", + "download:npm": "node scripts/download-npm.js", "build": "npm run typecheck && electron-vite build", "postinstall": "electron-builder install-app-deps", "build:unpack": "dotenv npm run build && electron-builder --dir", - "build:win": "dotenv npm run build && electron-builder --win --$ARCH", + "build:win": "dotenv npm run build && electron-builder --win", "build:win:x64": "dotenv npm run build && electron-builder --win --x64", - "build:mac": "dotenv electron-vite build && electron-builder --mac --$ARCH", + "build:mac": "dotenv electron-vite build && electron-builder --mac", "build:mac:arm64": "dotenv electron-vite build && electron-builder --mac --arm64", "build:mac:x64": "dotenv electron-vite build && electron-builder --mac --x64", - "build:linux": "dotenv electron-vite build && electron-builder --linux --$ARCH", + "build:linux": "dotenv electron-vite build && electron-builder --linux", "build:linux:arm64": "dotenv electron-vite build && electron-builder --linux --arm64", "build:linux:x64": "dotenv electron-vite build && electron-builder --linux --x64", "release": "node scripts/version.js", "publish": "yarn release patch push", "pulish:artifacts": "cd packages/artifacts && npm publish && cd -", "generate:agents": "yarn workspace @cherry-studio/database agents", - "generate:icons": "electron-icon-builder --input=./build/logo.png --output=build" + "generate:icons": "electron-icon-builder --input=./build/logo.png --output=build", + "analyze:renderer": "VISUALIZER_RENDERER=true yarn build", + "analyze:main": "VISUALIZER_MAIN=true yarn build" }, "dependencies": { "@electron-toolkit/preload": "^3.0.0", diff --git a/scripts/after-pack.js b/scripts/after-pack.js new file mode 100644 index 00000000..d14e24da --- /dev/null +++ b/scripts/after-pack.js @@ -0,0 +1,42 @@ +const { Arch } = require('electron-builder') +const { default: removeLocales } = require('./remove-locales') +const fs = require('fs') +const path = require('path') + +exports.default = async function (context) { + await removeLocales(context) + const platform = context.packager.platform.name + const arch = context.arch + + if (platform === 'mac') { + const nodeModulesPath = path.join( + context.appOutDir, + 'Cherry Studio.app', + 'Contents', + 'Resources', + 'app.asar.unpacked', + 'node_modules', + '@libsql' + ) + + keepLibsqlNodeModules(nodeModulesPath, arch === Arch.arm64 ? ['darwin-arm64'] : ['darwin-x64']) + } + + if (platform === 'linux') { + const nodeModulesPath = path.join(context.appOutDir, 'resources', 'app.asar.unpacked', 'node_modules', '@libsql') + keepLibsqlNodeModules( + nodeModulesPath, + arch === Arch.arm64 ? ['linux-arm64-gnu', 'linux-arm64-musl'] : ['linux-x64-gnu', 'linux-x64-musl'] + ) + } +} + +function keepLibsqlNodeModules(modulePath, arch) { + const dirs = fs.readdirSync(modulePath) + dirs + .filter((dir) => !arch.includes(dir)) + .forEach((dir) => { + fs.rmSync(path.join(modulePath, dir), { recursive: true, force: true }) + console.log(`Removed dir: ${dir}`, arch) + }) +} diff --git a/scripts/download-npm.js b/scripts/download-npm.js new file mode 100644 index 00000000..605f38fe --- /dev/null +++ b/scripts/download-npm.js @@ -0,0 +1,14 @@ +const { downloadNpmPackage } = require('./utils') + +async function downloadNpm(platform) { + if (!platform || platform === 'darwin') { + downloadNpmPackage('@libsql', '0.4.7', 'darwin', ['arm64', 'x64']) + } + + if (!platform || platform === 'linux') { + downloadNpmPackage('@libsql', '0.4.7', 'linux', ['arm64-gnu', 'x64-gnu']) + } +} + +const platformArg = process.argv[2] +downloadNpm(platformArg) diff --git a/scripts/removeLocales.js b/scripts/remove-locales.js similarity index 100% rename from scripts/removeLocales.js rename to scripts/remove-locales.js diff --git a/scripts/replaceSpaces.js b/scripts/replace-spaces.js similarity index 100% rename from scripts/replaceSpaces.js rename to scripts/replace-spaces.js diff --git a/scripts/utils.js b/scripts/utils.js new file mode 100644 index 00000000..4e3edb4b --- /dev/null +++ b/scripts/utils.js @@ -0,0 +1,43 @@ +const fs = require('fs') +const path = require('path') +const os = require('os') + +function downloadNpmPackage(package, version, platform, architectures = ['x64', 'arm64']) { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'npm-download-')) + + for (const arch of architectures) { + const targetDir = path.join('./node_modules/', package, `${platform}-${arch}`) + + // Skip if directory already exists + if (fs.existsSync(targetDir)) { + console.log(`${targetDir} already exists, skipping download...`) + continue + } + + const filename = path.join(tempDir, `${platform}-${arch}-${version}.tgz`) + const url = `https://registry.npmjs.org/${package}/${platform}-${arch}/-/${platform}-${arch}-${version}.tgz` + + try { + console.log(`Downloading ${filename}...`, url) + const { execSync } = require('child_process') + execSync(`curl --fail -o ${filename} ${url}`) + + console.log(`Extracting ${filename}...`) + execSync(`tar -xvf ${filename}`) + execSync(`rm -rf ${filename}`) + execSync(`mv package ${targetDir}`) + } catch (error) { + console.error(`Error processing ${filename}: ${error.message}`) + if (fs.existsSync(filename)) { + fs.unlinkSync(filename) + } + throw error + } + } + + fs.rmSync(tempDir, { recursive: true, force: true }) +} + +module.exports = { + downloadNpmPackage +} diff --git a/yarn.lock b/yarn.lock index 015b39cb..1db22a37 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1513,7 +1513,7 @@ __metadata: languageName: node linkType: hard -"@llm-tools/embedjs-libsql@npm:^0.1.25": +"@llm-tools/embedjs-libsql@npm:0.1.25": version: 0.1.25 resolution: "@llm-tools/embedjs-libsql@npm:0.1.25" dependencies: @@ -1525,6 +1525,18 @@ __metadata: languageName: node linkType: hard +"@llm-tools/embedjs-libsql@patch:@llm-tools/embedjs-libsql@npm%3A0.1.25#~/.yarn/patches/@llm-tools-embedjs-libsql-npm-0.1.25-fad000d74c.patch": + version: 0.1.25 + resolution: "@llm-tools/embedjs-libsql@patch:@llm-tools/embedjs-libsql@npm%3A0.1.25#~/.yarn/patches/@llm-tools-embedjs-libsql-npm-0.1.25-fad000d74c.patch::version=0.1.25&hash=7eafdc" + dependencies: + "@libsql/client": "npm:^0.14.0" + "@llm-tools/embedjs-interfaces": "npm:0.1.25" + "@llm-tools/embedjs-utils": "npm:0.1.25" + debug: "npm:^4.4.0" + checksum: 10c0/e7793caaf71fb977409d6ba97cb831d68fe5eff5c2ba8502d112fd0294ff004417c7685f86b1d45993f2722b6007976335413f9966b6b7e4676751ba41272041 + languageName: node + linkType: hard + "@llm-tools/embedjs-loader-csv@npm:^0.1.25": version: 0.1.25 resolution: "@llm-tools/embedjs-loader-csv@npm:0.1.25" @@ -1639,6 +1651,15 @@ __metadata: languageName: node linkType: hard +"@llm-tools/embedjs-utils@patch:@llm-tools/embedjs-utils@npm%3A0.1.25#~/.yarn/patches/@llm-tools-embedjs-utils-npm-0.1.25-fd8fe8a193.patch": + version: 0.1.25 + resolution: "@llm-tools/embedjs-utils@patch:@llm-tools/embedjs-utils@npm%3A0.1.25#~/.yarn/patches/@llm-tools-embedjs-utils-npm-0.1.25-fd8fe8a193.patch::version=0.1.25&hash=faf2ca" + dependencies: + "@llm-tools/embedjs-interfaces": "npm:0.1.25" + checksum: 10c0/c639f64a26023b21b2cd8b2bf5007541f4b1382ac3358d9404a531fcee66482ea94fe151b7d39b44a6cb903b975eb379101c4cca896d8a08a070330355e67c19 + languageName: node + linkType: hard + "@llm-tools/embedjs@npm:^0.1.25": version: 0.1.25 resolution: "@llm-tools/embedjs@npm:0.1.25" @@ -2740,7 +2761,7 @@ __metadata: "@hello-pangea/dnd": "npm:^16.6.0" "@kangfenmao/keyv-storage": "npm:^0.1.0" "@llm-tools/embedjs": "npm:^0.1.25" - "@llm-tools/embedjs-libsql": "npm:^0.1.25" + "@llm-tools/embedjs-libsql": "patch:@llm-tools/embedjs-libsql@npm%3A0.1.25#~/.yarn/patches/@llm-tools-embedjs-libsql-npm-0.1.25-fad000d74c.patch" "@llm-tools/embedjs-loader-csv": "npm:^0.1.25" "@llm-tools/embedjs-loader-markdown": "npm:^0.1.25" "@llm-tools/embedjs-loader-msoffice": "npm:^0.1.25"