build: download npm node native modules

.github/workflows/release.yml

@@ -15,15 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        os: [macos-13, macos-latest, windows-latest, ubuntu-latest]
-        arch: [x64, arm64]
-        exclude:
-          - os: windows-latest
-            arch: arm64
-          - os: macos-latest
-            arch: x64
-          - os: macos-13
-            arch: arm64
+        os: [macos-latest, windows-latest, ubuntu-latest]
 
     steps:
       - name: Check out Git repository
@@ -33,7 +25,6 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 20
-          arch: ${{ matrix.arch }}
 
       - name: Install corepack
         run: corepack enable && corepack prepare yarn@4.3.1 --activate
@@ -57,14 +48,18 @@ jobs:
 
       - name: Build Linux
         if: matrix.os == 'ubuntu-latest'
-        run: yarn build:linux
+        run: |
+          yarn download:npm linux
+          yarn build:linux
+
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
-          ARCH: ${{ matrix.arch }}
 
       - name: Build Mac
-        if: matrix.os == 'macos-13' || matrix.os == 'macos-latest'
-        run: yarn build:mac && mv dist/latest-mac.yml dist/latest-mac-${{ matrix.arch }}.yml
+        if: matrix.os == 'macos-latest'
+        run: |
+          yarn download:npm darwin
+          yarn build:mac
         env:
           CSC_LINK: ${{ secrets.CSC_LINK }}
           CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }}
@@ -72,17 +67,15 @@ jobs:
           APPLE_APP_SPECIFIC_PASSWORD: ${{ vars.APPLE_APP_SPECIFIC_PASSWORD }}
           APPLE_TEAM_ID: ${{ vars.APPLE_TEAM_ID }}
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
-          ARCH: ${{ matrix.arch }}
 
       - name: Build Windows
         if: matrix.os == 'windows-latest'
         run: yarn build:win
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
-          ARCH: ${{ matrix.arch }}
 
       - name: Replace spaces in filenames
-        run: node scripts/replaceSpaces.js
+        run: node scripts/replace-spaces.js
 
       - name: Release
         uses: softprops/action-gh-release@v2

.yarn/patches/@llm-tools-embedjs-libsql-npm-0.1.25-fad000d74c.patch

@@ -1,30 +1,5 @@
-diff --git a/src/libsql-db.cjs b/src/libsql-db.cjs
-index 076e838a1a144fa80687aa27ed92f488c2b532cb..fd00187134cda688c20b2929934ca5bfd4899c42 100644
---- a/src/libsql-db.cjs
-+++ b/src/libsql-db.cjs
-@@ -45,9 +45,9 @@ class LibSqlDb {
-     }
-     async similaritySearch(query, k) {
-         const statement = `SELECT id, pageContent, uniqueLoaderId, source, metadata,
--                vector_distance_cos(vector, vector32('[${query.join(',')}]'))
-+                vector_distance_cos(vector, vector32('[${query.join(',')}]')) as distance
-             FROM ${this.tableName}
--            ORDER BY vector_distance_cos(vector, vector32('[${query.join(',')}]')) ASC
-+            ORDER BY distance ASC
-             LIMIT ${k};`;
-         this.debug(`Executing statement - ${(0, embedjs_utils_1.truncateCenterString)(statement, 700)}`);
-         const results = await this.client.execute(statement);
-@@ -56,7 +56,7 @@ class LibSqlDb {
-             return {
-                 metadata,
-                 pageContent: result.pageContent.toString(),
--                score: 1,
-+                score: result.distance,
-             };
-         });
-     }
 diff --git a/src/libsql-db.js b/src/libsql-db.js
-index 58c42e4910bd0e53bc497ff9b9702b1f7a961266..3b124ff26f3b7cdca72eed5b872d7410c32eb70a 100644
+index 58c42e4910bd0e53bc497ff9b9702b1f7a961266..c02351dc95ba36fc4db97ddfb69206539fb4de1c 100644
 --- a/src/libsql-db.js
 +++ b/src/libsql-db.js
 @@ -41,9 +41,9 @@ export class LibSqlDb {
@@ -35,7 +10,7 @@ index 58c42e4910bd0e53bc497ff9b9702b1f7a961266..3b124ff26f3b7cdca72eed5b872d7410
 +                vector_distance_cos(vector, vector32('[${query.join(',')}]')) as distance
              FROM ${this.tableName}
 -            ORDER BY vector_distance_cos(vector, vector32('[${query.join(',')}]')) ASC
-+            ORDER BY distance ASC
++            ORDER BY distance DESC
              LIMIT ${k};`;
          this.debug(`Executing statement - ${truncateCenterString(statement, 700)}`);
          const results = await this.client.execute(statement);
@@ -44,7 +19,7 @@ index 58c42e4910bd0e53bc497ff9b9702b1f7a961266..3b124ff26f3b7cdca72eed5b872d7410
                  metadata,
                  pageContent: result.pageContent.toString(),
 -                score: 1,
-+                score: result.distance,
++                score: 1 - result.distance,
              };
          });
      }

electron-builder.yml

@@ -27,7 +27,6 @@ files:
   - '!node_modules/pdf-parse/lib/pdf.js/{v1.9.426,v1.10.88,v2.0.550}'
   - '!node_modules/mammoth/{mammoth.browser.js,mammoth.browser.min.js}'
   - '!node_modules/html2canvas/dist/{html2canvas.min.js,html2canvas.esm.js}'
-
 asarUnpack:
   - resources/**
   - '**/*.{node,dll,metal,exp,lib}'
@@ -42,17 +41,29 @@ nsis:
   oneClick: false
 mac:
   entitlementsInherit: build/entitlements.mac.plist
+  notarize: false
   extendInfo:
     - NSCameraUsageDescription: Application requests access to the device's camera.
     - NSMicrophoneUsageDescription: Application requests access to the device's microphone.
     - NSDocumentsFolderUsageDescription: Application requests access to the user's Documents folder.
     - NSDownloadsFolderUsageDescription: Application requests access to the user's Downloads folder.
-  notarize: false
+  target:
+    - target: dmg
+      arch:
+        - arm64
+        - x64
+    - target: zip
+      arch:
+        - arm64
+        - x64
 dmg:
   artifactName: ${productName}-${version}-${arch}.${ext}
 linux:
   target:
     - target: AppImage
+      arch:
+        - arm64
+        - x64
   maintainer: electronjs.org
   category: Utility
 appImage:
@@ -62,7 +73,7 @@ publish:
   url: https://cherrystudio.ocool.online
 electronDownload:
   mirror: https://npmmirror.com/mirrors/electron/
-afterPack: scripts/removeLocales.js
+afterPack: scripts/after-pack.js
 afterSign: scripts/notarize.js
 releaseInfo:
   releaseNotes: |

package.json

@@ -25,24 +25,25 @@
     "typecheck": "npm run typecheck:node && npm run typecheck:web",
     "start": "electron-vite preview",
     "dev": "electron-vite dev",
-    "analyze:renderer": "VISUALIZER_RENDERER=true yarn build",
-    "analyze:main": "VISUALIZER_MAIN=true yarn build",
+    "download:npm": "node scripts/download-npm.js",
     "build": "npm run typecheck && electron-vite build",
     "postinstall": "electron-builder install-app-deps",
     "build:unpack": "dotenv npm run build && electron-builder --dir",
-    "build:win": "dotenv npm run build && electron-builder --win --$ARCH",
+    "build:win": "dotenv npm run build && electron-builder --win",
     "build:win:x64": "dotenv npm run build && electron-builder --win --x64",
-    "build:mac": "dotenv electron-vite build && electron-builder --mac --$ARCH",
+    "build:mac": "dotenv electron-vite build && electron-builder --mac",
     "build:mac:arm64": "dotenv electron-vite build && electron-builder --mac --arm64",
     "build:mac:x64": "dotenv electron-vite build && electron-builder --mac --x64",
-    "build:linux": "dotenv electron-vite build && electron-builder --linux --$ARCH",
+    "build:linux": "dotenv electron-vite build && electron-builder --linux",
     "build:linux:arm64": "dotenv electron-vite build && electron-builder --linux --arm64",
     "build:linux:x64": "dotenv electron-vite build && electron-builder --linux --x64",
     "release": "node scripts/version.js",
     "publish": "yarn release patch push",
     "pulish:artifacts": "cd packages/artifacts && npm publish && cd -",
     "generate:agents": "yarn workspace @cherry-studio/database agents",
-    "generate:icons": "electron-icon-builder --input=./build/logo.png --output=build"
+    "generate:icons": "electron-icon-builder --input=./build/logo.png --output=build",
+    "analyze:renderer": "VISUALIZER_RENDERER=true yarn build",
+    "analyze:main": "VISUALIZER_MAIN=true yarn build"
   },
   "dependencies": {
     "@electron-toolkit/preload": "^3.0.0",

scripts/after-pack.js

@@ -0,0 +1,42 @@
+const { Arch } = require('electron-builder')
+const { default: removeLocales } = require('./remove-locales')
+const fs = require('fs')
+const path = require('path')
+
+exports.default = async function (context) {
+  await removeLocales(context)
+  const platform = context.packager.platform.name
+  const arch = context.arch
+
+  if (platform === 'mac') {
+    const nodeModulesPath = path.join(
+      context.appOutDir,
+      'Cherry Studio.app',
+      'Contents',
+      'Resources',
+      'app.asar.unpacked',
+      'node_modules',
+      '@libsql'
+    )
+
+    keepLibsqlNodeModules(nodeModulesPath, arch === Arch.arm64 ? ['darwin-arm64'] : ['darwin-x64'])
+  }
+
+  if (platform === 'linux') {
+    const nodeModulesPath = path.join(context.appOutDir, 'resources', 'app.asar.unpacked', 'node_modules', '@libsql')
+    keepLibsqlNodeModules(
+      nodeModulesPath,
+      arch === Arch.arm64 ? ['linux-arm64-gnu', 'linux-arm64-musl'] : ['linux-x64-gnu', 'linux-x64-musl']
+    )
+  }
+}
+
+function keepLibsqlNodeModules(modulePath, arch) {
+  const dirs = fs.readdirSync(modulePath)
+  dirs
+    .filter((dir) => !arch.includes(dir))
+    .forEach((dir) => {
+      fs.rmSync(path.join(modulePath, dir), { recursive: true, force: true })
+      console.log(`Removed dir: ${dir}`, arch)
+    })
+}

scripts/download-npm.js

@@ -0,0 +1,14 @@
+const { downloadNpmPackage } = require('./utils')
+
+async function downloadNpm(platform) {
+  if (!platform || platform === 'darwin') {
+    downloadNpmPackage('@libsql', '0.4.7', 'darwin', ['arm64', 'x64'])
+  }
+
+  if (!platform || platform === 'linux') {
+    downloadNpmPackage('@libsql', '0.4.7', 'linux', ['arm64-gnu', 'x64-gnu'])
+  }
+}
+
+const platformArg = process.argv[2]
+downloadNpm(platformArg)

scripts/utils.js

@@ -0,0 +1,43 @@
+const fs = require('fs')
+const path = require('path')
+const os = require('os')
+
+function downloadNpmPackage(package, version, platform, architectures = ['x64', 'arm64']) {
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'npm-download-'))
+
+  for (const arch of architectures) {
+    const targetDir = path.join('./node_modules/', package, `${platform}-${arch}`)
+
+    // Skip if directory already exists
+    if (fs.existsSync(targetDir)) {
+      console.log(`${targetDir} already exists, skipping download...`)
+      continue
+    }
+
+    const filename = path.join(tempDir, `${platform}-${arch}-${version}.tgz`)
+    const url = `https://registry.npmjs.org/${package}/${platform}-${arch}/-/${platform}-${arch}-${version}.tgz`
+
+    try {
+      console.log(`Downloading ${filename}...`, url)
+      const { execSync } = require('child_process')
+      execSync(`curl --fail -o ${filename} ${url}`)
+
+      console.log(`Extracting ${filename}...`)
+      execSync(`tar -xvf ${filename}`)
+      execSync(`rm -rf ${filename}`)
+      execSync(`mv package ${targetDir}`)
+    } catch (error) {
+      console.error(`Error processing ${filename}: ${error.message}`)
+      if (fs.existsSync(filename)) {
+        fs.unlinkSync(filename)
+      }
+      throw error
+    }
+  }
+
+  fs.rmSync(tempDir, { recursive: true, force: true })
+}
+
+module.exports = {
+  downloadNpmPackage
+}

yarn.lock

@@ -1513,7 +1513,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@llm-tools/embedjs-libsql@npm:^0.1.25":
+"@llm-tools/embedjs-libsql@npm:0.1.25":
   version: 0.1.25
   resolution: "@llm-tools/embedjs-libsql@npm:0.1.25"
   dependencies:
@@ -1525,6 +1525,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@llm-tools/embedjs-libsql@patch:@llm-tools/embedjs-libsql@npm%3A0.1.25#~/.yarn/patches/@llm-tools-embedjs-libsql-npm-0.1.25-fad000d74c.patch":
+  version: 0.1.25
+  resolution: "@llm-tools/embedjs-libsql@patch:@llm-tools/embedjs-libsql@npm%3A0.1.25#~/.yarn/patches/@llm-tools-embedjs-libsql-npm-0.1.25-fad000d74c.patch::version=0.1.25&hash=7eafdc"
+  dependencies:
+    "@libsql/client": "npm:^0.14.0"
+    "@llm-tools/embedjs-interfaces": "npm:0.1.25"
+    "@llm-tools/embedjs-utils": "npm:0.1.25"
+    debug: "npm:^4.4.0"
+  checksum: 10c0/e7793caaf71fb977409d6ba97cb831d68fe5eff5c2ba8502d112fd0294ff004417c7685f86b1d45993f2722b6007976335413f9966b6b7e4676751ba41272041
+  languageName: node
+  linkType: hard
+
 "@llm-tools/embedjs-loader-csv@npm:^0.1.25":
   version: 0.1.25
   resolution: "@llm-tools/embedjs-loader-csv@npm:0.1.25"
@@ -1639,6 +1651,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@llm-tools/embedjs-utils@patch:@llm-tools/embedjs-utils@npm%3A0.1.25#~/.yarn/patches/@llm-tools-embedjs-utils-npm-0.1.25-fd8fe8a193.patch":
+  version: 0.1.25
+  resolution: "@llm-tools/embedjs-utils@patch:@llm-tools/embedjs-utils@npm%3A0.1.25#~/.yarn/patches/@llm-tools-embedjs-utils-npm-0.1.25-fd8fe8a193.patch::version=0.1.25&hash=faf2ca"
+  dependencies:
+    "@llm-tools/embedjs-interfaces": "npm:0.1.25"
+  checksum: 10c0/c639f64a26023b21b2cd8b2bf5007541f4b1382ac3358d9404a531fcee66482ea94fe151b7d39b44a6cb903b975eb379101c4cca896d8a08a070330355e67c19
+  languageName: node
+  linkType: hard
+
 "@llm-tools/embedjs@npm:^0.1.25":
   version: 0.1.25
   resolution: "@llm-tools/embedjs@npm:0.1.25"
@@ -2740,7 +2761,7 @@ __metadata:
     "@hello-pangea/dnd": "npm:^16.6.0"
     "@kangfenmao/keyv-storage": "npm:^0.1.0"
     "@llm-tools/embedjs": "npm:^0.1.25"
-    "@llm-tools/embedjs-libsql": "npm:^0.1.25"
+    "@llm-tools/embedjs-libsql": "patch:@llm-tools/embedjs-libsql@npm%3A0.1.25#~/.yarn/patches/@llm-tools-embedjs-libsql-npm-0.1.25-fad000d74c.patch"
     "@llm-tools/embedjs-loader-csv": "npm:^0.1.25"
     "@llm-tools/embedjs-loader-markdown": "npm:^0.1.25"
     "@llm-tools/embedjs-loader-msoffice": "npm:^0.1.25"