Merge pull request #10 from vite-plugin/v2.1.0

V2.1.0

Author: caoxiemeihao

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.1.0 (2023-06-25)
+
+- 867f085 test: v2.1.0
+- 931ff60 feat: support auto detect native modules
+
 ## 2.0.2 (2023-06-23)
 
 - 2ac2913 chore: remove useless files

package.json

@@ -1,6 +1,6 @@
 {
   "name": "vite-plugin-native",
-  "version": "2.0.2",
+  "version": "2.1.0",
   "description": "Supports Node/Electron C/C++ native addons",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",

src/index.ts

@@ -9,7 +9,15 @@ import {
 import type { Configuration } from 'webpack'
 import type { NodeLoaderOptions, WebpackAssetRelocatorLoader } from './types'
 import { COLOURS } from 'vite-plugin-utils/function'
-import { createCjs, ensureDir, getInteropSnippet, getNatives } from './utils'
+import {
+  type NativeRecord,
+  type NativeRecordType,
+  createCjs,
+  ensureDir,
+  getInteropSnippet,
+  getDependenciesNatives,
+  resolveNativeRecord,
+} from './utils'
 
 export interface NativeOptions {
   /** @default 'node_natives' */
@@ -29,11 +37,15 @@ const TAG = '[vite-plugin-native]'
 const loader1 = '@vercel/webpack-asset-relocator-loader'
 const NativeExt = '.native.cjs'
 const InteropExt = '.interop.mjs'
+// https://github.com/vitejs/vite/blob/v5.3.1/packages/vite/src/node/plugins/index.ts#L55
+const bareImportRE = /^(?![a-zA-Z]:)[\w@](?!.*:\/\/)/
 // `nativesMap` is placed in the global scope and can be effective for multiple builds.
 const nativesMap = new Map<string, {
-  built: boolean
+  status: 'built' | 'resolved'
   nativeFilename: string
   interopFilename: string
+  type: NativeRecordType
+  nodeFiles: string[]
 }>
 
 export default function native(options: NativeOptions): Plugin {
@@ -49,64 +61,91 @@ export default function native(options: NativeOptions): Plugin {
       const outDir = config.build?.outDir ?? 'dist'
       output = normalizePath(path.join(resolvedRoot, outDir, assetsDir))
 
-      const natives = await getNatives(resolvedRoot)
-      options.natives ??= natives
+      const depsNativeRecord = await getDependenciesNatives(resolvedRoot)
+      const depsNatives = [...depsNativeRecord.keys()]
 
-      if (typeof options.natives === 'function') {
-        options.natives = options.natives(natives)
+      if (options.natives) {
+        const natives = Array.isArray(options.natives)
+          ? options.natives
+          : options.natives(depsNatives)
+        // TODO: bundle modules based on `natives`.
       }
 
-      const aliases: Alias[] = []
       const withDistAssetBase = (p: string) => (assetsDir && p) ? `${assetsDir}/${p}` : p
 
-      options.natives.length && ensureDir(output)
+      let detectedNativeRecord: NativeRecord = new Map
+      let detectedNatives: string[] = []
+
+      const alias: Alias = {
+        find: /(.*)/,
+        // Keep `customResolver` receive original source.
+        // @see https://github.com/rollup/plugins/blob/alias-v5.1.0/packages/alias/src/index.ts#L92
+        replacement: '$1',
+        async customResolver(source, importer) {
+          if (!importer) return
+          if (!bareImportRE.test(source)) return
+
+          if (!depsNativeRecord.has(source)) {
+            // Auto detection.
+            // e.g. serialport -> @serialport/bindings-cpp
+            const nativeRecord = await resolveNativeRecord(source, importer)
+            if (nativeRecord) {
+              detectedNativeRecord = new Map([...detectedNativeRecord, ...nativeRecord])
+              detectedNatives = [...depsNativeRecord.keys()]
+            }
+          }
+
+          if ([...depsNatives, ...detectedNatives].includes(source)) {
+            const nativeFilename = path.join(output, source + NativeExt)
+            const interopFilename = path.join(output, source + InteropExt)
 
-      for (const native of options.natives) {
-        const nativeFilename = path.join(output, native + NativeExt)
-        const interopFilename = path.join(output, native + InteropExt)
+            if (!nativesMap.get(source)) {
+              ensureDir(output)
 
-        aliases.push({
-          find: native,
-          replacement: interopFilename,
-          customResolver(source) {
-            const record = nativesMap.get(native)
-            if (!record?.built) {
               // Generate Vite and Webpack interop file.
-              const code = getInteropSnippet(native, `./${withDistAssetBase(native + NativeExt)}`)
-              fs.writeFileSync(interopFilename, code)
+              fs.writeFileSync(
+                interopFilename,
+                getInteropSnippet(source, `./${withDistAssetBase(source + NativeExt)}`),
+              )
 
               // We did not immediately call the `webpackBundle()` build here 
               // because `build.emptyOutDir = true` will cause the built file to be removed.
 
+              const isDetected = detectedNativeRecord.has(source)
+
               // Collect modules that are explicitly used.
-              nativesMap.set(native, { built: false, nativeFilename, interopFilename })
+              nativesMap.set(source, {
+                status: 'resolved',
+                nativeFilename,
+                interopFilename,
+                type: isDetected ? 'detected' : 'dependencies',
+                nodeFiles: isDetected
+                  ? detectedNativeRecord.get(source)?.nativeFiles!
+                  : depsNativeRecord.get(source)?.nativeFiles!,
+              })
             }
 
-            return { id: source }
-          },
-        })
+            return { id: interopFilename }
+          }
+        },
       }
 
-      const aliasKeys = aliases.map(({ find }) => find as string)
-
-      modifyAlias(config, aliases)
+      modifyAlias(config, [alias])
       // Run build are not necessary.
-      modifyOptimizeDeps(config, aliasKeys)
-    },
-    resolveId() {
-      // TODO: dynamic detect by bare moduleId. e.g. serialport
+      modifyOptimizeDeps(config, [...depsNatives, ...detectedNatives])
     },
     async buildEnd(error) {
       if (error) return
 
       // Must be explicitly specify use Webpack.
       if (options.webpack) {
         for (const [native, info] of nativesMap) {
-          if (info.built) continue
+          if (info.status === 'built') continue
 
           try {
             await webpackBundle(native, output, options.webpack)
-            info.built = true
+            // TODO: force copy *.node files to dist/node_modules path if Webpack can't bundle it correctly.
+            info.status = 'built'
           } catch (error: any) {
             console.error(`\n${TAG}`, error)
             process.exit(1)
@@ -148,7 +187,7 @@ async function webpackBundle(
   webpackOpts: NonNullable<NativeOptions['webpack']>
 ) {
   webpackOpts[loader1] ??= {}
-  const { validate, webpack } = cjs.require('webpack') as typeof import('webpack');
+  const { validate, webpack } = cjs.require('webpack') as typeof import('webpack')
   const assetBase = webpackOpts[loader1].outputAssetBase ??= 'native_modules'
 
   return new Promise<null>(async (resolve, reject) => {

src/utils.ts

@@ -6,6 +6,13 @@ import glob from 'fast-glob'
 import _libEsm from 'lib-esm'
 import { node_modules as findNodeModules } from 'vite-plugin-utils/function'
 
+export type NativeRecordType = 'dependencies' | 'detected'
+export type NativeRecord = Map<string, {
+  type: NativeRecordType
+  path: string
+  nativeFiles: string[]
+}>
+
 // @ts-ignore
 const libEsm: typeof import('lib-esm').default = _libEsm.default || _libEsm
 const cjs = createCjs(import.meta.url)
@@ -26,10 +33,16 @@ export function createCjs(url = import.meta.url) {
   }
 }
 
-export async function getNatives(root = process.cwd()) {
+export async function globNativeFiles(cwd: string) {
+  // @see https://github.com/electron/forge/blob/v7.4.0/packages/plugin/webpack/src/WebpackPlugin.ts#L192
+  const nativeFiles = await glob('**/*.node', { cwd })
+  return nativeFiles
+}
+
+export async function getDependenciesNatives(root = process.cwd()): Promise<NativeRecord> {
   const node_modules_paths = findNodeModules(root)
   // Native modules of package.json
-  const natives = []
+  const natives: NativeRecord = new Map
 
   for (const node_modules_path of node_modules_paths) {
     const pkgId = path.join(node_modules_path, '../package.json')
@@ -40,11 +53,14 @@ export async function getNatives(root = process.cwd()) {
 
       for (const dep of deps) {
         const depPath = path.join(node_modules_path, dep)
-        // @see https://github.com/electron/forge/blob/v7.4.0/packages/plugin/webpack/src/WebpackPlugin.ts#L192
-        const nativeFiles = await glob('**/*.node', { cwd: depPath })
+        const nativeFiles = await globNativeFiles(depPath)
 
         if (nativeFiles.length) {
-          natives.push(dep)
+          natives.set(dep, {
+            type: 'dependencies',
+            path: depPath,
+            nativeFiles,
+          })
         }
       }
     }
@@ -73,3 +89,22 @@ export function ensureDir(dir: string) {
   }
   return dir
 }
+
+export async function resolveNativeRecord(source: string, importer: string): Promise<NativeRecord | undefined> {
+  let modulePath: string | undefined
+  try {
+    const modulePackageJson = cjs.require.resolve(`${source}/package.json`, {
+      paths: [importer],
+    })
+    modulePath = path.dirname(modulePackageJson)
+  } catch { }
+
+  if (modulePath) {
+    const nodeFiles = await globNativeFiles(modulePath)
+    return new Map().set(source, {
+      type: 'deep-dependencies',
+      path: modulePath,
+      nodeFiles,
+    })
+  }
+}

test/fixtures/main.ts

@@ -1,11 +1,49 @@
+import path from 'node:path'
+
 // TODO: https://github.com/serialport/node-serialport/issues/2464
 // import * as serialport from 'serialport'
 import * as fsevents from 'fsevents'
 import sqlite3 from 'sqlite3'
-import better_sqlite3 from 'better-sqlite3'
+import BetterSqlite3 from 'better-sqlite3'
 
 export {
   fsevents,
   sqlite3,
-  better_sqlite3,
+}
+
+const root = path.join(__dirname, '..')
+
+export function initSqlite3() {
+  const sqlite3_db = path.join(root, 'db/sqlite3.db')
+
+  return new Promise<{
+    database: import('sqlite3').Database
+    error: Error | null
+  }>(resolve => {
+    const db = new (sqlite3.verbose().Database)(sqlite3_db, error => {
+      resolve({
+        database: db,
+        error,
+      })
+    })
+  })
+}
+
+export function initBetterSqlite3() {
+  const better_sqlite3_db = path.join(root, 'db/better-sqlite3.db')
+
+  return new Promise<{
+    database: import('better-sqlite3').Database
+    error: Error | null
+  }>(resolve => {
+    const db = new BetterSqlite3(better_sqlite3_db, { verbose: console.log })
+    // https://github.com/WiseLibs/better-sqlite3/blob/v11.1.0/docs/api.md#pragmastring-options---results
+    db.pragma('cache_size = 32000')
+    resolve({
+      database: db,
+      error: db.pragma('cache_size', { simple: true }) === 32000
+        ? null
+        : new Error('better-sqlite3 initialize failed'),
+    })
+  })
 }

test/index.test.ts

@@ -9,41 +9,6 @@ import {
 
 const root = path.join(__dirname, 'fixtures')
 
-function initSqlite3(sqlite3: typeof import('sqlite3')) {
-  const sqlite3_db = path.join(root, 'db/sqlite3.db')
-
-  return new Promise<{
-    database: import('sqlite3').Database
-    error: Error | null
-  }>(resolve => {
-    const db = new (sqlite3.verbose().Database)(sqlite3_db, error => {
-      resolve({
-        database: db,
-        error,
-      })
-    })
-  })
-}
-
-function initBetterSqlite3(BetterSqlite3: typeof import('better-sqlite3')) {
-  const better_sqlite3_db = path.join(root, 'db/better-sqlite3.db')
-
-  return new Promise<{
-    database: import('better-sqlite3').Database
-    error: Error | null
-  }>(resolve => {
-    const db = new BetterSqlite3(better_sqlite3_db, { verbose: console.log })
-    // https://github.com/WiseLibs/better-sqlite3/blob/v11.1.0/docs/api.md#pragmastring-options---results
-    db.pragma('cache_size = 32000')
-    resolve({
-      database: db,
-      error: db.pragma('cache_size', { simple: true }) === 32000
-        ? null
-        : new Error('better-sqlite3 initialize failed'),
-    })
-  })
-}
-
 beforeAll(async () => {
   for (const name of ['dist', 'db']) {
     fs.rmSync(path.join(root, name), { recursive: true, force: true })
@@ -57,16 +22,21 @@ test('vite-plugin-native', async () => {
   const main = require('./fixtures/dist/main')
   const fsevents = main.fsevents
   const sqlite3 = main.sqlite3
-  const sqlite3DB = await initSqlite3(main.sqlite3)
-  const better_sqlite3DB = await initBetterSqlite3(main.better_sqlite3)
+  const sqlite3DB = await main.initSqlite3()
+  const better_sqlite3DB = await main.initBetterSqlite3()
+
+  const fseventsKeys1 = Object.getOwnPropertyNames(fsevents).filter(name => name !== 'default')
+  // esm export members will be auto sort.
+  const fseventsKeys2 = Object.getOwnPropertyNames(require('fsevents')).sort((a, b) => a.localeCompare(b))
+  expect(fseventsKeys1).toEqual(fseventsKeys2)
 
-  expect(Object.getOwnPropertyNames(fsevents).filter(name => name !== 'default').reverse())
-    .toEqual(Object.getOwnPropertyNames(require('fsevents')))
+  const sqlite3Keys1 = Object.getOwnPropertyNames(sqlite3).filter(name => name !== 'default')
   // `require('sqlite3').path` will only be available after call `initSqlite3()`.
-  expect(Object.getOwnPropertyNames(sqlite3).filter(name => name !== 'default'))
-    .toEqual(Object.getOwnPropertyNames(require('sqlite3')).filter(name => name !== 'path'))
+  const sqlite3Keys2 = Object.getOwnPropertyNames(require('sqlite3')).filter(name => name !== 'path')
+  expect(sqlite3Keys1).toEqual(sqlite3Keys2)
   expect(sqlite3DB.database && typeof sqlite3DB.database).eq('object')
   expect(sqlite3DB.error).null
+
   expect(better_sqlite3DB.database && typeof better_sqlite3DB.database).eq('object')
   expect(better_sqlite3DB.error).null
 })