fs: let glob walk a mounted virtual file system

lib/internal/fs/glob.js

@@ -20,7 +20,7 @@ const {
   validateString,
   validateStringArray,
 } = require('internal/validators');
-const { Dirent, DirentFromStats } = require('internal/fs/utils');
+const { Dirent, DirentFromStats, vfsState } = require('internal/fs/utils');
 const {
   codes: {
     ERR_INVALID_ARG_TYPE,
@@ -126,7 +126,7 @@ class Glob {
   globSync() {
     const result = globBinding.globSync(
       this.#root, this.#patterns, this.#excludePatterns, this.#flags(),
-      this.#excludeAdapter(), this.#maxDepth);
+      this.#excludeAdapter(), this.#maxDepth, this.#reader());
     if (!this.#withFileTypes) {
       return result;
     }
@@ -141,7 +141,7 @@ class Glob {
   async globAll() {
     const handle = globBinding.globStart(
       this.#root, this.#patterns, this.#excludePatterns, this.#flags(),
-      this.#excludeAdapter(), this.#maxDepth);
+      this.#excludeAdapter(), this.#maxDepth, this.#reader());
     try {
       const { 0: paths, 1: types } = await handle.all();
       if (!this.#withFileTypes) {
@@ -161,7 +161,7 @@ class Glob {
   async * glob() {
     const handle = globBinding.globStart(
       this.#root, this.#patterns, this.#excludePatterns, this.#flags(),
-      this.#excludeAdapter(), this.#maxDepth);
+      this.#excludeAdapter(), this.#maxDepth, this.#reader());
     try {
       let done = false;
       while (!done) {
@@ -181,6 +181,13 @@ class Glob {
       (this.#withFileTypes ? kFlagWithFileTypes : 0);
   }
 
+  // The native walk reads the file system with libuv, which cannot see a
+  // mounted virtual file system: a walk that can reach one reads through
+  // the fs hooks instead.
+  #reader() {
+    return vfsState.handlers?.globReader(this.#root, this.#patterns);
+  }
+
   // Shapes the argument an `exclude` callback receives to
   // work both with and without Dirents
   #excludeAdapter() {

lib/internal/vfs/setup.js

@@ -2,6 +2,7 @@
 
 const {
   ArrayPrototypeForEach,
+  ArrayPrototypePush,
   MapPrototypeForEach,
   ObjectKeys,
   PromiseResolve,
@@ -13,7 +14,7 @@ const {
 
 const { Buffer } = require('buffer');
 const { isArrayBufferView } = require('internal/util/types');
-const { dirname, join, sep } = require('path');
+const { basename, dirname, isAbsolute, join, resolve, sep } = require('path');
 const { fileURLToPath, pathToFileURL, URL } = require('internal/url');
 const { kEmptyObject } = require('internal/util');
 const { validateObject } = require('internal/validators');
@@ -42,6 +43,22 @@ const permission = require('internal/process/permission');
 const { getOptionValue } = require('internal/options');
 const nativeModulesBinding = internalBinding('modules');
 const { UV_ENOENT } = internalBinding('uv');
+const {
+  kReadLstat,
+  kReadStat,
+  kReadRealpath,
+  kReadScandir,
+} = internalBinding('glob');
+const {
+  UV_DIRENT_UNKNOWN,
+  UV_DIRENT_FILE,
+  UV_DIRENT_DIR,
+  UV_DIRENT_LINK,
+  UV_DIRENT_FIFO,
+  UV_DIRENT_SOCKET,
+  UV_DIRENT_CHAR,
+  UV_DIRENT_BLOCK,
+} = internalBinding('constants').fs;
 let debug = require('internal/util/debuglog').debuglog('vfs', (fn) => {
   debug = fn;
 });
@@ -385,6 +402,69 @@ function rmRootSync(path, options) {
   throw createEROFS('rm', path);
 }
 
+/**
+ * The uv_dirent_type_t of a Dirent or Stats
+ * @param {object} entry
+ * @returns {number}
+ */
+function direntType(entry) {
+  if (entry.isSymbolicLink()) return UV_DIRENT_LINK;
+  if (entry.isDirectory()) return UV_DIRENT_DIR;
+  if (entry.isFile()) return UV_DIRENT_FILE;
+  if (entry.isFIFO()) return UV_DIRENT_FIFO;
+  if (entry.isSocket()) return UV_DIRENT_SOCKET;
+  if (entry.isCharacterDevice()) return UV_DIRENT_CHAR;
+  if (entry.isBlockDevice()) return UV_DIRENT_BLOCK;
+  return UV_DIRENT_UNKNOWN;
+}
+
+/**
+ * Serves the reads of a glob walk that can reach a mount, which the
+ * native walk cannot see: the fs functions dispatch to the mount, and to
+ * the real file system for every other path. A read that fails answers
+ * undefined, as the walk's own reads do. The parent of the reserved root
+ * (os.devNull) is a directory to the walk, which descends an absolute
+ * pattern from the root one segment at a time.
+ * @param {number} op
+ * @param {string} path
+ * @returns {number|boolean|string|[string[], number[]]|undefined}
+ */
+let lazyFs;
+function readForGlob(op, path) {
+  const fs = lazyFs ??= require('fs');
+  try {
+    if (normalizeMountedPath(path) === dirname(normalizedVfsRoot)) {
+      switch (op) {
+        case kReadLstat: return UV_DIRENT_DIR;
+        case kReadStat: return true;
+        case kReadRealpath: return path;
+        case kReadScandir:
+          return [[basename(normalizedVfsRoot)], [UV_DIRENT_DIR]];
+      }
+    }
+    switch (op) {
+      case kReadLstat:
+        return direntType(fs.lstatSync(path));
+      case kReadStat:
+        return fs.statSync(path).isDirectory();
+      case kReadRealpath:
+        return fs.realpathSync.native(path);
+      case kReadScandir: {
+        const dirents = fs.readdirSync(path, { withFileTypes: true });
+        const names = [];
+        const types = [];
+        for (let i = 0; i < dirents.length; i++) {
+          ArrayPrototypePush(names, dirents[i].name);
+          ArrayPrototypePush(types, direntType(dirents[i]));
+        }
+        return [names, types];
+      }
+    }
+  } catch {
+    return undefined;
+  }
+}
+
 function createVfsHandlers() {
   return {
     __proto__: null,
@@ -430,6 +510,17 @@ function createVfsHandlers() {
     statSync(path, options) {
       return vfsRead(path, 'stat', (vfs, n) => vfs.statSync(n, options));
     },
+    // The reads of a glob walk that can reach a mount: it starts under
+    // one, or a pattern names one absolutely. See internal/fs/glob.
+    globReader(cwd, patterns) {
+      if (findVFS(resolve(cwd)) !== null) return readForGlob;
+      for (let i = 0; i < patterns.length; i++) {
+        if (isAbsolute(patterns[i]) && findVFS(patterns[i]) !== null) {
+          return readForGlob;
+        }
+      }
+      return undefined;
+    },
     realpathSync(path, options) {
       const result = vfsRead(path, 'realpath', (vfs, n) => vfs.realpathSync(n));
       if (result !== undefined && options?.encoding === 'buffer') {

src/glob/glob_walker.cc

@@ -268,13 +268,6 @@ bool MatchFullPath(const CompiledPattern& pattern,
   return MatchPattern(pattern, PatternView(*scratch));
 }
 
-struct StatInfo {
-  bool exists = false;
-  bool is_dir = false;
-  bool is_link = false;
-  int type = UV_DIRENT_UNKNOWN;
-};
-
 // The dirent type a stat's mode describes; special files keep their real
 // type, as the previous implementation's lstat-built Dirents did.
 int TypeOfMode(uint64_t mode) {
@@ -311,11 +304,6 @@ int DirentTypeOf(const StatInfo& info) {
                        : UV_DIRENT_FILE;
 }
 
-struct DirEntry {
-  std::string name;
-  int type = UV_DIRENT_UNKNOWN;
-};
-
 // One blocking scandir with no Environment access, callable from any thread.
 std::vector<DirEntry> ScanDir(const char* path) {
   std::vector<DirEntry> entries;
@@ -652,6 +640,7 @@ class WalkerState {
   bool RunSlice(size_t max_results,
                 bool on_main_thread,
                 std::vector<WalkEntry>* out) {
+    CHECK_IMPLIES(options_.fs_reader != nullptr, on_main_thread);
     on_main_thread_ = on_main_thread;
     if (!started_) {
       started_ = true;
@@ -706,11 +695,13 @@ class WalkerState {
   // visited once and nothing can observe the traversal while it runs:
   // one pattern row, no '..' hops or absolute heads (revisits), no
   // symlink following (realpath chains), no exclude pattern or callback
-  // (pruning), and no permission model (per-path checks).
+  // (pruning), no reader (main-thread reads), and no permission model
+  // (per-path checks).
   bool ClassifyFree() const {
     if (rows_.size() != 1) return false;
     if (!excludes_.empty() || options_.exclude_filter != nullptr) return false;
     if (options_.follow_symlinks) return false;
+    if (options_.fs_reader != nullptr) return false;
     if (env_->permission()->enabled()) return false;
     const CompiledPattern::Row& row = *rows_[0];
     if (row.parts.empty()) return false;
@@ -922,18 +913,28 @@ class WalkerState {
     return fn(nullptr, &req->req, args..., nullptr);
   }
 
+  // A reader call that threw ends the walk, its exception pending.
+  void CheckReader() {
+    if (options_.fs_reader->failed()) aborted_ = true;
+  }
+
   StatInfo StatSync(const std::string& path) {
     auto it = stat_cache_.find(path);
     if (it != stat_cache_.end()) return it->second;
     StatInfo info;
     if (PermissionGranted(path)) {
-      fs::FSReqWrapSync req("lstat", path.c_str());
-      if (FsCall(&req, uv_fs_lstat, path.c_str()) == 0) {
-        const uint64_t mode = req.req.statbuf.st_mode;
-        info.exists = true;
-        info.is_dir = (mode & S_IFMT) == S_IFDIR;
-        info.is_link = (mode & S_IFMT) == S_IFLNK;
-        info.type = TypeOfMode(mode);
+      if (options_.fs_reader != nullptr) {
+        info = options_.fs_reader->Lstat(path);
+        CheckReader();
+      } else {
+        fs::FSReqWrapSync req("lstat", path.c_str());
+        if (FsCall(&req, uv_fs_lstat, path.c_str()) == 0) {
+          const uint64_t mode = req.req.statbuf.st_mode;
+          info.exists = true;
+          info.is_dir = (mode & S_IFMT) == S_IFDIR;
+          info.is_link = (mode & S_IFMT) == S_IFLNK;
+          info.type = TypeOfMode(mode);
+        }
       }
     }
     stat_cache_.emplace(path, info);
@@ -949,9 +950,14 @@ class WalkerState {
     if (it != follow_stat_cache_.end()) return it->second;
     bool is_dir = false;
     if (PermissionGranted(path)) {
-      fs::FSReqWrapSync req("stat", path.c_str());
-      if (FsCall(&req, uv_fs_stat, path.c_str()) == 0) {
-        is_dir = (req.req.statbuf.st_mode & S_IFMT) == S_IFDIR;
+      if (options_.fs_reader != nullptr) {
+        is_dir = options_.fs_reader->IsDirectory(path);
+        CheckReader();
+      } else {
+        fs::FSReqWrapSync req("stat", path.c_str());
+        if (FsCall(&req, uv_fs_stat, path.c_str()) == 0) {
+          is_dir = (req.req.statbuf.st_mode & S_IFMT) == S_IFDIR;
+        }
       }
     }
     follow_stat_cache_.emplace(path, is_dir);
@@ -963,10 +969,15 @@ class WalkerState {
     if (it != realpath_cache_.end()) return it->second;
     std::string real;
     if (PermissionGranted(path)) {
-      fs::FSReqWrapSync req("realpath", path.c_str());
-      if (FsCall(&req, uv_fs_realpath, path.c_str()) == 0 &&
-          req.req.ptr != nullptr) {
-        real = static_cast<const char*>(req.req.ptr);
+      if (options_.fs_reader != nullptr) {
+        real = options_.fs_reader->Realpath(path);
+        CheckReader();
+      } else {
+        fs::FSReqWrapSync req("realpath", path.c_str());
+        if (FsCall(&req, uv_fs_realpath, path.c_str()) == 0 &&
+            req.req.ptr != nullptr) {
+          real = static_cast<const char*>(req.req.ptr);
+        }
       }
     }
     return realpath_cache_.emplace(path, std::move(real)).first->second;
@@ -977,7 +988,10 @@ class WalkerState {
     if (it != readdir_cache_.end()) return it->second;
     std::vector<DirEntry> entries;
     if (PermissionGranted(path)) {
-      if (!prefetcher_.Take(path, &entries)) {
+      if (options_.fs_reader != nullptr) {
+        entries = options_.fs_reader->Scandir(path);
+        CheckReader();
+      } else if (!prefetcher_.Take(path, &entries)) {
         fs::FSReqWrapSync req("scandir", path.c_str());
         if (FsCall(&req, uv_fs_scandir, path.c_str(), 0) >= 0) {
           uv_dirent_t ent;
@@ -1003,6 +1017,7 @@ class WalkerState {
   }
 
   void MaybePrefetch(const QueueItem& item) {
+    if (options_.fs_reader != nullptr) return;
     const bool needs_listing =
         std::ranges::any_of(item.patterns, [](const PatternState& pattern) {
           return pattern.indexes.size() != 1 ||
@@ -1108,10 +1123,12 @@ class WalkerState {
     return depth;
   }
 
-  // Whether two spellings name one directory.
+  // Whether two spellings name one directory. A reader's stats carry no
+  // identity (a virtual file system need not have inodes), so its paths
+  // are told apart by realpath, as on Windows.
   bool SamePath(const std::string& a, const std::string& b) {
     if (a == b) return true;
-    if (kWindows) {
+    if (kWindows || options_.fs_reader != nullptr) {
       const std::string real = RealpathSync(a);
       return !real.empty() && real == RealpathSync(b);
     } else {

src/glob/glob_walker.h

@@ -34,6 +34,37 @@ class ExcludeFilter {
   virtual bool failed() const = 0;
 };
 
+// What lstat reports about a path.
+struct StatInfo {
+  bool exists = false;
+  bool is_dir = false;
+  bool is_link = false;
+  int type = UV_DIRENT_UNKNOWN;
+};
+
+struct DirEntry {
+  std::string name;
+  int type = UV_DIRENT_UNKNOWN;
+};
+
+// Reads the filesystem for a walk in place of libuv, for paths libuv
+// cannot see: a mounted virtual file system only answers through the
+// JavaScript fs hooks. A walk with a reader runs on the main thread, and
+// a read that fails answers as a missing path, as the walk's own reads do.
+class FsReader {
+ public:
+  virtual ~FsReader() = default;
+  virtual StatInfo Lstat(const std::string& path) = 0;
+  // Whether the path, with symbolic links followed, is a directory.
+  virtual bool IsDirectory(const std::string& path) = 0;
+  // Empty when the path cannot be resolved.
+  virtual std::string Realpath(const std::string& path) = 0;
+  virtual std::vector<DirEntry> Scandir(const std::string& path) = 0;
+  // Set once a call has thrown. The walk stops and the exception is left
+  // for the caller to propagate.
+  virtual bool failed() const = 0;
+};
+
 struct WalkOptions {
   // options.cwd, or ".": the string paths are resolved against.
   std::string cwd = ".";
@@ -45,6 +76,7 @@ struct WalkOptions {
   size_t max_depth = kNoDepthLimit;
 
   ExcludeFilter* exclude_filter = nullptr;
+  FsReader* fs_reader = nullptr;
 };
 
 struct WalkEntry {