mojavelinux · August 25, 2021 21:47 · Aug 25, 2021
diff --git a/graceful-vfs-src-with-gs-and-pipeline.js b/graceful-vfs-src-with-gs-and-pipeline.js
@@ -0,0 +1,83 @@
+const fs = require('fs')
+const gs = require('glob-stream')
+const File = require('vinyl')
+const { Transform } = require('stream')
+const map = (transform) => new Transform({ objectMode: true, transform })
+const ospath = require('path')
+//const isUTF8 = require('is-utf8')
+const { pipeline } = require('stream')
+
+function smartStat (path_, callback) {
+  fs.lstat(path_, (lstatErr, lstat) => {
+    if (lstatErr) {
+      callback(lstatErr) // NOTE extremely rare
+    } else if (lstat.isSymbolicLink()) {
+      fs.stat(path_, (statErr, stat) => {
+        //statErr ? callback(Object.assign(statErr, { link: true })) : callback(null, stat)
+        if (statErr) {
+          fs.readlink(path_, (_, link) => {
+            callback(Object.assign(statErr, { link }))
+          })
+        } else {
+          callback(null, stat)
+        }
+      })
+    } else {
+      callback(null, lstat)
+    }
+  })
+}
+
+;(async () => {
+  await new Promise((resolve, reject) => {
+    console.time('measure')
+    const accum = []
+    const cache = {}
+    const cwd = ospath.resolve('../docs')
+    pipeline(
+      gs('**/*', {
+        cwd,
+        //cwd: ospath.resolve('more-docs'),
+        follow: true,
+        nomount: true,
+        nosort: true,
+        uniqueBy: ((entries) => entries),
+        strict: false, // silently ignore files that cannot be read
+        cache, // we could reuse cache, but it's very low level and not likely to provide all the info we need
+        //nodir: true, // don't return directories, though it seems to come with a slight cost; but why?
+      }),
+      map(({ path: abspath }, _, next) => {
+        //to skip directories eagerly, map cache and enable this line
+        //if (Array.isArray(cache[abspath])) return next()
+        smartStat(abspath, (statErr, stat) => {
+          if (statErr) {
+            console.warn('skipping file: ' + abspath + (statErr.link ? ` (${statErr.code === 'ELOOP' ? 'symlink cycle' : 'broken symlink'}: ${statErr.link})` : ''))
+            return next()
+          }
+          if (stat.isDirectory()) return next()
+          fs.readFile(abspath, (readErr, contents) => {
+            if (readErr) {
+              console.warn('skipping file: ' + abspath)
+              return next()
+            }
+            //if (contents[0] === 0xEF && contents[1] === 0xBB && contents[2] === 0xBF && isUTF8(contents)) {
+            //  contents = contents.slice(3)
+            //}
+            accum.push(new File({ base: cwd, cwd, path: abspath, contents, stat }))
+            next()
+          })
+        })
+      }),
+      (err) => err ? reject(err) : resolve(accum)
+    )
+  })
+    .then((files) => {
+      console.timeEnd('measure')
+      console.log(files.length)
+      console.log(files[1].relative)
+      //files.forEach((file) => console.log(file.relative))
+    })
+    .catch((err) => {
+      console.warn('error: ' + err.message)
+    })
+})()
No results found