diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..80bd6b9 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,95 @@ +# CLAUDE.md + +Guidance for working in this repository. + +## What this is + +`@jsonic/multisource` is a [Jsonic](https://jsonic.senecajs.org) plugin that +loads partial values from external sources (files, packages, in-memory maps) +into a single parse result. A directive character (`@` by default) marks a +reference; the plugin resolves it, processes the resolved text, and splices +the result into the parse output. + +The repository ships **two implementations**: + +- **TypeScript** — `src/`, tests in `test/`, built to `dist/` + `dist-test/`. +- **Go** — `go/`, a port published as the `github.com/jsonicjs/multisource/go` + module. + +## TypeScript is canonical + +**The TypeScript implementation is the source of truth.** The Go package is a +port that follows TS semantics. When the two diverge, **TS wins** — change Go +to match TS, not the other way around (unless you are deliberately fixing a +bug in TS first, then porting the fix). + +When adding or changing behaviour: + +1. Implement and test it in TypeScript first. +2. Port the change to Go, mirroring the public API names and behaviour as + closely as the language allows. +3. Keep both test suites and both `doc/` files in sync. + +## Build & test + +Use the `Makefile` targets (they wrap npm and go): + +```sh +make build # build-ts + build-go +make test # test-ts + test-go +make build-ts # npm run build (tsc --build src test) +make test-ts # npm test (node --test dist-test/*.test.js) +make build-go # cd go && go build ./... +make test-go # cd go && go test ./... +``` + +Notes: + +- TS tests run against the **built** output in `dist-test/`, so run + `make build-ts` (or `npm run build`) before `make test-ts`. +- `npm run reset` does a clean reinstall + build + test. +- Go has no external runtime deps beyond the other `jsonicjs/*/go` modules. + +## Resolver & processor parity + +Resolvers turn a reference into source text; processors turn source text into +a value. The public factory names are kept aligned across the two ports: + +| Resolver (TS) | Resolver (Go) | Resolves from | +| -------------------- | -------------------- | -------------------- | +| `makeMemResolver` | `MakeMemResolver` | in-memory `map` | +| `makeFileResolver` | `MakeFileResolver` | filesystem | +| `makePkgResolver` | `MakePkgResolver` | `node_modules` | + +| Processor (TS) | Processor (Go) | Handles | +| ------------------------- | ------------------- | ------------------ | +| default (string) | `DefaultProcessor` | unknown / raw | +| `makeJsonicProcessor` | `JsonicProcessor` | `.jsonic`, `.jsc` | +| (json via jsonic) | `JSONProcessor` | `.json` | +| `makeJavaScriptProcessor` | — (Node-only) | `.js` | + +## Known Go ↔ TS differences + +The ports are not line-for-line identical. Current intentional/known gaps: + +- **`MakePkgResolver` is a portable subset.** Go has no `require.resolve`, so + the Go resolver walks `node_modules` directories, honours `package.json` + `"main"` for bare references, and tries implicit extensions / index files. + It does not implement Node's full module-resolution algorithm (e.g. + conditional `"exports"`). +- **No virtual filesystem in Go.** The TS resolvers accept a `ctx.meta.fs` + (used by tests via `memfs`); the Go resolvers use the OS filesystem + directly. +- **No `.js` processor in Go.** Executing JavaScript modules is Node-specific. +- **Base-path resolution.** The Go plugin resolves a reference against + `opts.Path` once, before calling the resolver; it does not yet track each + parent file's directory for relative nested includes the way the TS + `resolvePathSpec` does via `ctx.meta.multisource.path`. + +If you close any of these gaps, update this list and both `doc/` files. + +## Releasing the Go module + +The Go module is versioned independently (see `const Version` in +`go/multisource.go`) and tagged `go/vX.Y.Z`. Use `make publish-go V=x.y.z`. +Do not bump or tag a release unless explicitly asked. diff --git a/README.md b/README.md index 579d5f8..8ac7df9 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,7 @@ single [Jsonic](https://jsonic.senecajs.org) parse result. ## Documentation -Documentation for both language implementations follows the -[Diátaxis](https://diataxis.fr) framework (Tutorials, How-to guides, -Explanation, Reference). +Documentation is available for both language implementations: - TypeScript: [`doc/multisource-ts.md`](doc/multisource-ts.md) - Go: [`doc/multisource-go.md`](doc/multisource-go.md) diff --git a/doc/multisource-go.md b/doc/multisource-go.md index 1635ea1..af403e4 100644 --- a/doc/multisource-go.md +++ b/doc/multisource-go.md @@ -71,6 +71,48 @@ out, _ := j.Parse(`{x: @a}`) ## How-to guides +### Load sources from the filesystem + +`MakeFileResolver` reads sources from disk, resolving the reference to an +absolute path and trying implicit extensions and index files when the path +has none: + +```go +j := multisource.MakeJsonic(multisource.MultiSourceOptions{ + Resolver: multisource.MakeFileResolver(), + Path: "config", // base directory for relative references +}) +out, _ := j.Parse(`{db: @"database.jsonic"}`) +``` + +Pass `FileResolverOptions` to supply a path transformer or preloaded +contents (checked before touching disk): + +```go +multisource.MakeFileResolver(multisource.FileResolverOptions{ + PathFinder: func(spec string) string { return "conf/" + spec }, + Preload: map[string]string{"/abs/path/a.jsonic": "{a:1}"}, +}) +``` + +### Resolve references from node_modules + +`MakePkgResolver` resolves references inside `node_modules` folders. With no +options it walks up from the current working directory; pass `Paths` to set +explicit search roots: + +```go +j := multisource.MakeJsonic(multisource.MultiSourceOptions{ + Resolver: multisource.MakePkgResolver(multisource.PkgResolverOptions{ + Paths: []string{"/path/to/project"}, + }), +}) +out, _ := j.Parse(`{cfg: @"some-pkg/config.jsonic"}`) +``` + +A bare package reference (`@"some-pkg"`) resolves via the package's +`package.json` `"main"`, falling back to index files. + ### Supply a custom resolver Implement the `Resolver` function type. It must populate `Resolution.Found` @@ -160,7 +202,7 @@ All added alternates share the `multisource` group tag, supplied via the ### `Version` ```go -const Version = "0.1.0" +const Version = "0.1.4" ``` Go module release version. @@ -196,11 +238,35 @@ Convenience wrapper around `MakeJsonic().Parse(src)`. | Name | Kind | | ---------------------- | --------------------- | -| `MakeMemResolver` | Resolver factory | +| `MakeMemResolver` | Resolver factory (in-memory map) | +| `MakeFileResolver` | Resolver factory (filesystem) | +| `MakePkgResolver` | Resolver factory (node_modules) | | `DefaultProcessor` | Raw passthrough | | `JSONProcessor` | `.json` via stdlib | | `JsonicProcessor` | `.jsonic`, `.jsc` | +These mirror the canonical TypeScript resolvers (`makeMemResolver`, +`makeFileResolver`, `makePkgResolver`). `MakePkgResolver` implements the +portable subset of the TypeScript package resolver: it walks `node_modules` +directories, honours a package's `package.json` `"main"` for bare references, +and tries implicit extensions and index files. It does not reproduce Node's +full `require.resolve` algorithm (for example, conditional `"exports"`). + +```go +// MakeFileResolver — load sources from disk. +type FileResolverOptions struct { + PathFinder func(spec string) string // Transform the raw reference path. + Preload map[string]string // Full path -> content, checked first. +} +func MakeFileResolver(opts ...FileResolverOptions) Resolver + +// MakePkgResolver — resolve references inside node_modules. +type PkgResolverOptions struct { + Paths []string // node_modules search roots; empty walks up from cwd. +} +func MakePkgResolver(opts ...PkgResolverOptions) Resolver +``` + ### Types ```go diff --git a/go/multisource.go b/go/multisource.go index a0fbcb6..57396b4 100644 --- a/go/multisource.go +++ b/go/multisource.go @@ -227,14 +227,30 @@ func buildPotentials(fullpath string, implicitExt []string) []string { return nil } potentials := []string{fullpath} - ext := path.Ext(fullpath) - if ext == "" { + + // Determine the final path segment in a separator-agnostic way: the + // in-memory resolver keys on forward slashes, while the file/pkg resolvers + // pass OS-native paths (e.g. Windows backslashes from filepath.Abs). + base := fullpath + if i := strings.LastIndexAny(fullpath, `/\`); i >= 0 { + base = fullpath[i+1:] + } + + if path.Ext(base) == "" { + // Implicit extensions. for _, ie := range implicitExt { potentials = append(potentials, fullpath+ie) } + // Folder index file. for _, ie := range implicitExt { potentials = append(potentials, fullpath+"/index"+ie) } + // Folder index file including the folder name, e.g. foo/index.foo.jsonic. + if base != "" && base != "." { + for _, ie := range implicitExt { + potentials = append(potentials, fullpath+"/index."+base+ie) + } + } } return potentials } diff --git a/go/multisource_test.go b/go/multisource_test.go index 18ad5c2..6cc8c95 100644 --- a/go/multisource_test.go +++ b/go/multisource_test.go @@ -230,10 +230,19 @@ func TestBuildPotentials(t *testing.T) { assert(t, "pot-2", p[2], "foo.jsc") assert(t, "pot-3", p[3], "foo.json") assert(t, "pot-idx-1", p[4], "foo/index.jsonic") + assert(t, "pot-idx-folder", p[7], "foo/index.foo.jsonic") p = buildPotentials("bar.json", exts) assert(t, "has-ext", len(p), 1) assert(t, "has-ext-0", p[0], "bar.json") + + // Windows-style (backslash) paths must extract the final segment, not the + // whole path, for the folder-name index variant. Regression guard: the + // `path` package only splits on '/', so filepath.Abs output on Windows + // (backslashes) would otherwise corrupt the folder name. + w := buildPotentials(`C:\proj\h`, exts) + assert(t, "win-no-ext", len(w), 10) + assert(t, "win-folder-index", w[7], `C:\proj\h/index.h.jsonic`) } func TestCustomProcessor(t *testing.T) { diff --git a/go/resolver.go b/go/resolver.go new file mode 100644 index 0000000..510e7cf --- /dev/null +++ b/go/resolver.go @@ -0,0 +1,194 @@ +/* Copyright (c) 2021-2025 Richard Rodger and other contributors, MIT License */ + +package multisource + +import ( + "encoding/json" + "os" + "path/filepath" +) + +// FileResolverOptions configures MakeFileResolver. +type FileResolverOptions struct { + // PathFinder transforms the raw reference path before resolution. + PathFinder func(spec string) string + // Preload maps full paths to content, consulted before reading from disk. + Preload map[string]string +} + +// MakeFileResolver creates a resolver that loads sources from the filesystem. +// +// It mirrors the TypeScript makeFileResolver: the reference is resolved to an +// absolute path; when the path has no extension, implicit extensions and index +// files are tried; and a preload map (full path -> content) is consulted before +// touching disk. +func MakeFileResolver(opts ...FileResolverOptions) Resolver { + var o FileResolverOptions + if len(opts) > 0 { + o = opts[0] + } + + return func(spec PathSpec, mopts *MultiSourceOptions) Resolution { + // A pathfinder transforms the raw reference before resolution. + if o.PathFinder != nil { + spec = ResolvePathSpec(o.PathFinder(spec.Path), spec.Base) + } + + res := Resolution{PathSpec: spec, Found: false} + if spec.Full == "" { + return res + } + + full := spec.Full + if abs, err := filepath.Abs(full); err == nil { + full = abs + } + res.Full = full + + potentials := buildPotentials(full, mopts.ImplicitExt) + res.Search = potentials + + for _, p := range potentials { + if src, ok := o.Preload[p]; ok { + res.Full = p + res.Kind = extKind(p) + res.Src = src + res.Found = true + return res + } + if src, ok := loadFile(p); ok { + res.Full = p + res.Kind = extKind(p) + res.Src = src + res.Found = true + return res + } + } + + return res + } +} + +// PkgResolverOptions configures MakePkgResolver. +type PkgResolverOptions struct { + // Paths lists directories whose node_modules folders are searched; each is + // also walked upwards. When empty, the resolver walks up from the current + // working directory. + Paths []string +} + +// MakePkgResolver creates a resolver that resolves references inside +// node_modules folders, mirroring the TypeScript makePkgResolver. +// +// Go has no equivalent of Node's require.resolve, so this implements the +// portable subset: it walks node_modules directories, honours a package's +// package.json "main" for bare references, and tries implicit extensions and +// index files. It does not implement Node's full module-resolution algorithm +// (for example, conditional "exports"). +func MakePkgResolver(opts ...PkgResolverOptions) Resolver { + var o PkgResolverOptions + if len(opts) > 0 { + o = opts[0] + } + + return func(spec PathSpec, mopts *MultiSourceOptions) Resolution { + res := Resolution{PathSpec: spec, Found: false} + ref := spec.Path + if ref == "" { + return res + } + + var roots []string + if len(o.Paths) > 0 { + roots = o.Paths + } else if cwd, err := os.Getwd(); err == nil { + roots = []string{cwd} + } + + seen := map[string]bool{} + var search []string + + for _, root := range roots { + for _, dir := range ancestorDirs(root) { + nm := filepath.Join(dir, "node_modules") + if seen[nm] { + continue + } + seen[nm] = true + + if full, src, ok := resolveInPkgDir(nm, ref, mopts.ImplicitExt, &search); ok { + res.Full = full + res.Kind = extKind(full) + res.Src = src + res.Found = true + res.Search = search + return res + } + } + } + + res.Search = search + return res + } +} + +// resolveInPkgDir resolves a package reference inside a node_modules directory, +// trying the reference directly (with implicit extensions and index files) and +// then the target package's package.json "main". +func resolveInPkgDir(nodeModules, ref string, exts []string, search *[]string) (full, src string, found bool) { + target := filepath.Join(nodeModules, filepath.FromSlash(ref)) + + for _, p := range buildPotentials(target, exts) { + *search = append(*search, p) + if s, ok := loadFile(p); ok { + return p, s, true + } + } + + // Bare package reference: honour package.json "main". + pkgJSON := filepath.Join(target, "package.json") + *search = append(*search, pkgJSON) + if data, err := os.ReadFile(pkgJSON); err == nil { + var meta struct { + Main string `json:"main"` + } + if json.Unmarshal(data, &meta) == nil && meta.Main != "" { + mainPath := filepath.Join(target, filepath.FromSlash(meta.Main)) + for _, p := range buildPotentials(mainPath, exts) { + *search = append(*search, p) + if s, ok := loadFile(p); ok { + return p, s, true + } + } + } + } + + return "", "", false +} + +// loadFile reads a file, reporting whether it was read successfully. A failed +// read is treated as the source not existing. +func loadFile(p string) (string, bool) { + b, err := os.ReadFile(p) + if err != nil { + return "", false + } + return string(b), true +} + +// ancestorDirs returns dir followed by each of its parent directories. +func ancestorDirs(dir string) []string { + if dir == "" { + return nil + } + var dirs []string + for { + dirs = append(dirs, dir) + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + } + return dirs +} diff --git a/go/resolver_test.go b/go/resolver_test.go new file mode 100644 index 0000000..84f2fa5 --- /dev/null +++ b/go/resolver_test.go @@ -0,0 +1,226 @@ +/* Copyright (c) 2025 Richard Rodger, MIT License */ + +package multisource + +import ( + "os" + "path/filepath" + "testing" +) + +// writeTestFile writes content to p, creating parent directories. +func writeTestFile(t *testing.T, p, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestFileResolver(t *testing.T) { + dir := t.TempDir() + writeTestFile(t, filepath.Join(dir, "a.jsonic"), `{a:1}`) + + j := MakeJsonic(MultiSourceOptions{ + Resolver: MakeFileResolver(), + Path: dir, + }) + + r, err := j.Parse(`{x: @a.jsonic}`) + if err != nil { + t.Fatal(err) + } + m, _ := r.(map[string]any) + assert(t, "file-ref", m["x"], map[string]any{"a": float64(1)}) +} + +func TestFileResolverImplicitExt(t *testing.T) { + dir := t.TempDir() + writeTestFile(t, filepath.Join(dir, "b.jsonic"), `{b:2}`) + + j := MakeJsonic(MultiSourceOptions{ + Resolver: MakeFileResolver(), + Path: dir, + }) + + r, err := j.Parse(`{x: @b}`) + if err != nil { + t.Fatal(err) + } + m, _ := r.(map[string]any) + assert(t, "file-implicit", m["x"], map[string]any{"b": float64(2)}) +} + +func TestFileResolverIndex(t *testing.T) { + dir := t.TempDir() + writeTestFile(t, filepath.Join(dir, "mod", "index.jsonic"), `{m:3}`) + + j := MakeJsonic(MultiSourceOptions{ + Resolver: MakeFileResolver(), + Path: dir, + }) + + r, err := j.Parse(`{x: @mod}`) + if err != nil { + t.Fatal(err) + } + m, _ := r.(map[string]any) + assert(t, "file-index", m["x"], map[string]any{"m": float64(3)}) +} + +func TestFileResolverFolderIndex(t *testing.T) { + dir := t.TempDir() + writeTestFile(t, filepath.Join(dir, "h", "index.h.jsonic"), `{h:7}`) + + j := MakeJsonic(MultiSourceOptions{ + Resolver: MakeFileResolver(), + Path: dir, + }) + + r, err := j.Parse(`{x: @h}`) + if err != nil { + t.Fatal(err) + } + m, _ := r.(map[string]any) + assert(t, "file-folder-index", m["x"], map[string]any{"h": float64(7)}) +} + +func TestFileResolverPreload(t *testing.T) { + dir := t.TempDir() + // No file on disk; provide content via preload keyed by absolute path. + abs, _ := filepath.Abs(filepath.Join(dir, "p.jsonic")) + + j := MakeJsonic(MultiSourceOptions{ + Resolver: MakeFileResolver(FileResolverOptions{ + Preload: map[string]string{abs: `{p:4}`}, + }), + Path: dir, + }) + + r, err := j.Parse(`{x: @p.jsonic}`) + if err != nil { + t.Fatal(err) + } + m, _ := r.(map[string]any) + assert(t, "file-preload", m["x"], map[string]any{"p": float64(4)}) +} + +func TestFileResolverPathFinder(t *testing.T) { + dir := t.TempDir() + writeTestFile(t, filepath.Join(dir, "sub", "a.jsonic"), `{a:1}`) + + j := MakeJsonic(MultiSourceOptions{ + Resolver: MakeFileResolver(FileResolverOptions{ + PathFinder: func(spec string) string { return "sub/" + spec }, + }), + Path: dir, + }) + + r, err := j.Parse(`{x: @a.jsonic}`) + if err != nil { + t.Fatal(err) + } + m, _ := r.(map[string]any) + assert(t, "file-pathfinder", m["x"], map[string]any{"a": float64(1)}) +} + +func TestFileResolverNotFound(t *testing.T) { + dir := t.TempDir() + j := MakeJsonic(MultiSourceOptions{ + Resolver: MakeFileResolver(), + Path: dir, + }) + + r, err := j.Parse(`{x: @missing.jsonic}`) + if err != nil { + t.Fatal(err) + } + m, _ := r.(map[string]any) + assert(t, "file-not-found", m["x"], nil) +} + +func TestPkgResolver(t *testing.T) { + dir := t.TempDir() + writeTestFile(t, filepath.Join(dir, "node_modules", "mypkg", "zed.jsonic"), `{zed:99}`) + + j := MakeJsonic(MultiSourceOptions{ + Resolver: MakePkgResolver(PkgResolverOptions{Paths: []string{dir}}), + }) + + r, err := j.Parse(`{c: @"mypkg/zed.jsonic"}`) + if err != nil { + t.Fatal(err) + } + m, _ := r.(map[string]any) + assert(t, "pkg-subpath", m["c"], map[string]any{"zed": float64(99)}) +} + +func TestPkgResolverMain(t *testing.T) { + dir := t.TempDir() + writeTestFile(t, filepath.Join(dir, "node_modules", "mypkg", "package.json"), `{"main":"main.jsonic"}`) + writeTestFile(t, filepath.Join(dir, "node_modules", "mypkg", "main.jsonic"), `{z:11}`) + + j := MakeJsonic(MultiSourceOptions{ + Resolver: MakePkgResolver(PkgResolverOptions{Paths: []string{dir}}), + }) + + r, err := j.Parse(`{z: @"mypkg"}`) + if err != nil { + t.Fatal(err) + } + m, _ := r.(map[string]any) + assert(t, "pkg-main", m["z"], map[string]any{"z": float64(11)}) +} + +func TestPkgResolverIndex(t *testing.T) { + dir := t.TempDir() + writeTestFile(t, filepath.Join(dir, "node_modules", "idxpkg", "index.jsonic"), `{i:5}`) + + j := MakeJsonic(MultiSourceOptions{ + Resolver: MakePkgResolver(PkgResolverOptions{Paths: []string{dir}}), + }) + + r, err := j.Parse(`{i: @"idxpkg"}`) + if err != nil { + t.Fatal(err) + } + m, _ := r.(map[string]any) + assert(t, "pkg-index", m["i"], map[string]any{"i": float64(5)}) +} + +func TestPkgResolverWalkUp(t *testing.T) { + // Package installed in an ancestor's node_modules; resolve from a nested dir. + root := t.TempDir() + writeTestFile(t, filepath.Join(root, "node_modules", "mypkg", "zed.jsonic"), `{zed:99}`) + nested := filepath.Join(root, "a", "b", "c") + if err := os.MkdirAll(nested, 0o755); err != nil { + t.Fatal(err) + } + + j := MakeJsonic(MultiSourceOptions{ + Resolver: MakePkgResolver(PkgResolverOptions{Paths: []string{nested}}), + }) + + r, err := j.Parse(`{c: @"mypkg/zed.jsonic"}`) + if err != nil { + t.Fatal(err) + } + m, _ := r.(map[string]any) + assert(t, "pkg-walkup", m["c"], map[string]any{"zed": float64(99)}) +} + +func TestPkgResolverNotFound(t *testing.T) { + dir := t.TempDir() + j := MakeJsonic(MultiSourceOptions{ + Resolver: MakePkgResolver(PkgResolverOptions{Paths: []string{dir}}), + }) + + r, err := j.Parse(`{x: @"nopkg/zed.jsonic"}`) + if err != nil { + t.Fatal(err) + } + m, _ := r.(map[string]any) + assert(t, "pkg-not-found", m["x"], nil) +} diff --git a/package.json b/package.json index f8b2050..cd99542 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "devDependencies": { "@types/node": "^25.6.0", "jsonic-multisource-pkg-test": "0.0.1", - "memfs": "4.57.2", + "memfs": "4.57.6", "typescript": "6.0.3" }, "peerDependencies": {