Merge pull request #11 from vite-plugin/v2.2.0

V2.2.0

Author: caoxiemeihao

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.2.0 (2023-06-27)
+
+- 64b881a feat: add `options.forceCopyIfUnbuilt`
+- 7c60957 feat: add `options.ignore`
+
 ## 2.1.0 (2023-06-25)
 
 - 867f085 test: v2.1.0

package.json

@@ -1,6 +1,6 @@
 {
   "name": "vite-plugin-native",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "description": "Supports Node/Electron C/C++ native addons",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -20,12 +20,14 @@
     "build": "vite build",
     "types": "tsc --emitDeclarationOnly",
     "prepublishOnly": "npm run build && npm run test",
+    "build:test": "vite build -c test/fixtures/vite.config.ts",
     "test": "vitest run"
   },
   "dependencies": {
     "@vercel/webpack-asset-relocator-loader": "1.7.3",
+    "dependencies-tree": "~0.2.0",
     "fast-glob": "^3.3.2",
-    "lib-esm": "^0.4.2",
+    "lib-esm": "~0.4.2",
     "node-loader": "^2.0.0",
     "webpack": "^5.70.0"
   },

src/index.ts

@@ -9,10 +9,11 @@ import {
 import type { Configuration } from 'webpack'
 import type { NodeLoaderOptions, WebpackAssetRelocatorLoader } from './types'
 import { COLOURS } from 'vite-plugin-utils/function'
+import { flatDependencies } from 'dependencies-tree'
 import {
-  type NativeRecord,
-  type NativeRecordType,
+  type ResolvedNativeRecord,
   createCjs,
+  copy,
   ensureDir,
   getInteropSnippet,
   getDependenciesNatives,
@@ -22,8 +23,15 @@ import {
 export interface NativeOptions {
   /** @default 'node_natives' */
   assetsDir?: string
-  /** By default native modules are automatically detected if this option is not explicitly configure by the user. */
+  /** 
+   * By default native modules are automatically detected if this option is not explicitly configure by the user.
+   * @deprecated use `ignore` option instead
+   */
   natives?: string[] | ((natives: string[]) => string[])
+  /** Ignore the specified native module. */
+  ignore?: (name: string) => boolean | undefined
+  /** Force copy *.node files to dist/node_modules path if Webpack can't bundle native modules correctly. */
+  forceCopyIfUnbuilt?: true
   /** Enable and configure webpack. */
   webpack?: {
     config?: (config: Configuration) => Configuration | undefined | Promise<Configuration | undefined>
@@ -35,18 +43,13 @@ export interface NativeOptions {
 const cjs = createCjs(import.meta.url)
 const TAG = '[vite-plugin-native]'
 const loader1 = '@vercel/webpack-asset-relocator-loader'
+const outputAssetBase = 'native_modules'
 const NativeExt = '.native.cjs'
 const InteropExt = '.interop.mjs'
 // https://github.com/vitejs/vite/blob/v5.3.1/packages/vite/src/node/plugins/index.ts#L55
 const bareImportRE = /^(?![a-zA-Z]:)[\w@](?!.*:\/\/)/
 // `nativesMap` is placed in the global scope and can be effective for multiple builds.
-const nativesMap = new Map<string, {
-  status: 'built' | 'resolved'
-  nativeFilename: string
-  interopFilename: string
-  type: NativeRecordType
-  nodeFiles: string[]
-}>
+const nativesMap = new Map<string, ResolvedNativeRecord>
 
 export default function native(options: NativeOptions): Plugin {
   const assetsDir = options.assetsDir ??= 'node_natives'
@@ -61,21 +64,16 @@ export default function native(options: NativeOptions): Plugin {
       const outDir = config.build?.outDir ?? 'dist'
       output = normalizePath(path.join(resolvedRoot, outDir, assetsDir))
 
-      const depsNativeRecord = await getDependenciesNatives(resolvedRoot)
-      const depsNatives = [...depsNativeRecord.keys()]
+      let nativeRecord = await getDependenciesNatives(resolvedRoot)
 
       if (options.natives) {
         const natives = Array.isArray(options.natives)
           ? options.natives
-          : options.natives(depsNatives)
+          : options.natives([...nativeRecord.keys()])
         // TODO: bundle modules based on `natives`.
       }
 
       const withDistAssetBase = (p: string) => (assetsDir && p) ? `${assetsDir}/${p}` : p
-
-      let detectedNativeRecord: NativeRecord = new Map
-      let detectedNatives: string[] = []
-
       const alias: Alias = {
         find: /(.*)/,
         // Keep `customResolver` receive original source.
@@ -85,67 +83,74 @@ export default function native(options: NativeOptions): Plugin {
           if (!importer) return
           if (!bareImportRE.test(source)) return
 
-          if (!depsNativeRecord.has(source)) {
-            // Auto detection.
+          if (!nativeRecord.has(source)) {
+            // Dynamic deep detection.
             // e.g. serialport -> @serialport/bindings-cpp
-            const nativeRecord = await resolveNativeRecord(source, importer)
-            if (nativeRecord) {
-              detectedNativeRecord = new Map([...detectedNativeRecord, ...nativeRecord])
-              detectedNatives = [...depsNativeRecord.keys()]
-            }
+            nativeRecord = new Map([...nativeRecord, ...(await resolveNativeRecord(source, importer) ?? [])])
           }
 
-          if ([...depsNatives, ...detectedNatives].includes(source)) {
-            const nativeFilename = path.join(output, source + NativeExt)
-            const interopFilename = path.join(output, source + InteropExt)
+          const nativeItem = nativeRecord.get(source)
+          if (!nativeItem) return
 
-            if (!nativesMap.get(source)) {
-              ensureDir(output)
+          if (options.ignore?.(source) === false) {
+            nativeItem.ignore = true
+            return
+          }
 
-              // Generate Vite and Webpack interop file.
-              fs.writeFileSync(
-                interopFilename,
-                getInteropSnippet(source, `./${withDistAssetBase(source + NativeExt)}`),
-              )
+          const nativeFilename = path.join(output, source + NativeExt)
+          const interopFilename = path.join(output, source + InteropExt)
 
-              // We did not immediately call the `webpackBundle()` build here 
-              // because `build.emptyOutDir = true` will cause the built file to be removed.
-
-              const isDetected = detectedNativeRecord.has(source)
-
-              // Collect modules that are explicitly used.
-              nativesMap.set(source, {
-                status: 'resolved',
-                nativeFilename,
-                interopFilename,
-                type: isDetected ? 'detected' : 'dependencies',
-                nodeFiles: isDetected
-                  ? detectedNativeRecord.get(source)?.nativeFiles!
-                  : depsNativeRecord.get(source)?.nativeFiles!,
-              })
-            }
+          if (!nativesMap.get(source)) {
+            ensureDir(path.dirname(interopFilename))
 
-            return { id: interopFilename }
+            // Generate Vite and Webpack interop file.
+            fs.writeFileSync(
+              interopFilename,
+              getInteropSnippet(source, `./${withDistAssetBase(source + NativeExt)}`),
+            )
+
+            // We did not immediately call the `webpackBundle()` build here 
+            // because `build.emptyOutDir = true` will cause the built file to be removed.
+
+            // Collect modules that are explicitly used.
+            nativesMap.set(source, {
+              status: 'resolved',
+              nativeFilename,
+              interopFilename,
+              native: nativeItem,
+            })
           }
+
+          return { id: interopFilename }
         },
       }
 
       modifyAlias(config, [alias])
       // Run build are not necessary.
-      modifyOptimizeDeps(config, [...depsNatives, ...detectedNatives])
+      modifyOptimizeDeps(config, [...nativeRecord.keys()])
     },
     async buildEnd(error) {
       if (error) return
 
       // Must be explicitly specify use Webpack.
       if (options.webpack) {
-        for (const [native, info] of nativesMap) {
-          if (info.status === 'built') continue
+        for (const item of nativesMap) {
+          const [name, native] = item
+          if (native.status === 'built') continue
+          if (native.native.ignore) continue
 
           try {
-            await webpackBundle(native, output, options.webpack)
-            // TODO: force copy *.node files to dist/node_modules path if Webpack can't bundle it correctly.
-            info.status = 'built'
+            await webpackBundle(name, output, options.webpack)
+
+            if (options.forceCopyIfUnbuilt) {
+              await forceCopyNativeFilesIfUnbuilt(
+                native,
+                output,
+                options.webpack[loader1]?.outputAssetBase ?? outputAssetBase,
+              )
+            }
+
+            native.status = 'built'
           } catch (error: any) {
             console.error(`\n${TAG}`, error)
             process.exit(1)
@@ -188,7 +193,7 @@ async function webpackBundle(
 ) {
   webpackOpts[loader1] ??= {}
   const { validate, webpack } = cjs.require('webpack') as typeof import('webpack')
-  const assetBase = webpackOpts[loader1].outputAssetBase ??= 'native_modules'
+  const assetBase = webpackOpts[loader1].outputAssetBase ??= outputAssetBase
 
   return new Promise<null>(async (resolve, reject) => {
     let options: Configuration = {
@@ -258,3 +263,40 @@ async function webpackBundle(
     })
   })
 }
+
+// Force copy *.node files to dist/node_modules path if Webpack can't bundle native modules correctly.
+async function forceCopyNativeFilesIfUnbuilt(
+  resolvedNative: ResolvedNativeRecord,
+  output: string,
+  assetBase: string,
+) {
+  const { nativeFilename } = resolvedNative
+  const { name: nativeName, path: nativeRoot, nativeFiles } = resolvedNative.native
+  const nativeOutput = path.posix.join(output, assetBase)
+  const nativeNodeModules = path.posix.join(output, 'node_modules')
+  const exists = nativeFiles
+    // e.g. ['build/Release/better_sqlite3.node', 'build/Release/test_extension.node']
+    .some((file) => fs.existsSync(path.join(nativeOutput, file)))
+
+  if (!exists) {
+    const nativeDest = path.join(nativeNodeModules, nativeName)
+    copy(nativeRoot, nativeDest)
+
+    const dependencies = await flatDependencies(nativeRoot)
+    for (const dep of dependencies) {
+      copy(dep.src, path.join(nativeNodeModules, dep.name))
+    }
+
+    let relativePath = path.posix.relative(path.dirname(nativeFilename), nativeDest)
+    if (!relativePath.startsWith('.')) {
+      relativePath = `./${relativePath}`
+    }
+    fs.writeFileSync(
+      path.join(nativeFilename),
+      `
+// This is a native module that cannot be built correctly.
+module.exports = require("${relativePath}");
+`.trim(),
+    )
+  }
+}

src/utils.ts

@@ -6,12 +6,20 @@ import glob from 'fast-glob'
 import _libEsm from 'lib-esm'
 import { node_modules as findNodeModules } from 'vite-plugin-utils/function'
 
-export type NativeRecordType = 'dependencies' | 'detected'
-export type NativeRecord = Map<string, {
-  type: NativeRecordType
+export interface ResolvedNativeRecord {
+  status: 'built' | 'resolved'
+  nativeFilename: string
+  interopFilename: string
+  native: NativeRecord
+}
+
+export interface NativeRecord {
+  name: string
+  type: 'dependencies' | 'detected'
   path: string
   nativeFiles: string[]
-}>
+  ignore?: true
+}
 
 // @ts-ignore
 const libEsm: typeof import('lib-esm').default = _libEsm.default || _libEsm
@@ -39,10 +47,10 @@ export async function globNativeFiles(cwd: string) {
   return nativeFiles
 }
 
-export async function getDependenciesNatives(root = process.cwd()): Promise<NativeRecord> {
+export async function getDependenciesNatives(root = process.cwd()): Promise<Map<string, NativeRecord>> {
   const node_modules_paths = findNodeModules(root)
   // Native modules of package.json
-  const natives: NativeRecord = new Map
+  const natives = new Map<string, NativeRecord>
 
   for (const node_modules_path of node_modules_paths) {
     const pkgId = path.join(node_modules_path, '../package.json')
@@ -52,11 +60,14 @@ export async function getDependenciesNatives(root = process.cwd()): Promise<Nati
       const deps = Object.keys(pkg.dependencies || {}).concat(Object.keys(pkg.devDependencies || {}))
 
       for (const dep of deps) {
+        if (natives.has(dep)) continue
+
         const depPath = path.join(node_modules_path, dep)
         const nativeFiles = await globNativeFiles(depPath)
 
         if (nativeFiles.length) {
           natives.set(dep, {
+            name: dep,
             type: 'dependencies',
             path: depPath,
             nativeFiles,
@@ -90,7 +101,7 @@ export function ensureDir(dir: string) {
   return dir
 }
 
-export async function resolveNativeRecord(source: string, importer: string): Promise<NativeRecord | undefined> {
+export async function resolveNativeRecord(source: string, importer: string): Promise<Map<string, NativeRecord> | undefined> {
   let modulePath: string | undefined
   try {
     const modulePackageJson = cjs.require.resolve(`${source}/package.json`, {
@@ -100,11 +111,32 @@ export async function resolveNativeRecord(source: string, importer: string): Pro
   } catch { }
 
   if (modulePath) {
-    const nodeFiles = await globNativeFiles(modulePath)
-    return new Map().set(source, {
-      type: 'deep-dependencies',
-      path: modulePath,
-      nodeFiles,
-    })
+    const nativeFiles = await globNativeFiles(modulePath)
+    if (nativeFiles.length) {
+      return new Map<string, NativeRecord>().set(source, {
+        name: source,
+        type: 'detected',
+        path: modulePath,
+        nativeFiles,
+      })
+    }
+  }
+}
+
+function copyDir(srcDir: string, destDir: string) {
+  fs.mkdirSync(destDir, { recursive: true })
+  for (const file of fs.readdirSync(srcDir)) {
+    const srcFile = path.resolve(srcDir, file)
+    const destFile = path.resolve(destDir, file)
+    copy(srcFile, destFile)
+  }
+}
+
+export function copy(src: string, dest: string) {
+  const stat = fs.statSync(src)
+  if (stat.isDirectory()) {
+    copyDir(src, dest)
+  } else {
+    fs.copyFileSync(src, dest)
   }
 }