feat: use asar for twemoji assets

package.json

@@ -56,12 +56,14 @@
     "chalk": "4.0.0",
     "cheerio": "1.0.0-rc.3",
     "chokidar": "3.4.0",
+    "chromium-pickle-js": "0.2.0",
     "clean-css": "4.2.3",
     "command-exists": "1.2.9",
     "compression": "1.7.4",
     "connect-session-knex": "1.6.0",
     "cookie-parser": "1.4.5",
     "cors": "2.8.5",
+    "cuint": "0.2.2",
     "custom-error-instance": "2.1.2",
     "dependency-graph": "0.9.0",
     "diff": "4.0.2",

server/core/asar.js

@@ -0,0 +1,130 @@
+const pickle = require('chromium-pickle-js')
+const path = require('path')
+const UINT64 = require('cuint').UINT64
+const fs = require('fs')
+
+/* global WIKI */
+
+/**
+ * Based of express-serve-asar (https://github.com/toyobayashi/express-serve-asar)
+ * by Fenglin Li (https://github.com/toyobayashi)
+ */
+
+const packages = {
+  'twemoji': path.join(WIKI.ROOTPATH, `assets/svg/twemoji.asar`)
+}
+
+module.exports = {
+  fdCache: {},
+  async serve (pkgName, req, res, next) {
+    const file = this.readFilesystemSync(packages[pkgName])
+    const { filesystem, fd } = file
+    const info = filesystem.getFile(req.path.substring(1))
+    if (info) {
+      res.set({
+        'Content-Type': 'image/svg+xml',
+        'Content-Length': info.size
+      })
+
+      fs.createReadStream('', {
+        fd,
+        autoClose: false,
+        start: 8 + filesystem.headerSize + parseInt(info.offset, 10),
+        end: 8 + filesystem.headerSize + parseInt(info.offset, 10) + info.size - 1
+      }).on('error', (err) => {
+        WIKI.logger.warn(err)
+        res.sendStatus(404)
+      }).pipe(res.status(200))
+    } else {
+      res.sendStatus(404)
+    }
+  },
+  async unload () {
+    if (this.fdCache) {
+      WIKI.logger.info('Closing ASAR file descriptors...')
+      for (const fdItem in this.fdCache) {
+        fs.closeSync(this.fdCache[fdItem].fd)
+      }
+      this.fdCache = {}
+    }
+  },
+  readArchiveHeaderSync (fd) {
+    let size
+    let headerBuf
+
+    const sizeBuf = Buffer.alloc(8)
+    if (fs.readSync(fd, sizeBuf, 0, 8, null) !== 8) {
+      throw new Error('Unable to read header size')
+    }
+
+    const sizePickle = pickle.createFromBuffer(sizeBuf)
+    size = sizePickle.createIterator().readUInt32()
+    headerBuf = Buffer.alloc(size)
+    if (fs.readSync(fd, headerBuf, 0, size, null) !== size) {
+      throw new Error('Unable to read header')
+    }
+
+    const headerPickle = pickle.createFromBuffer(headerBuf)
+    const header = headerPickle.createIterator().readString()
+    return { header: JSON.parse(header), headerSize: size }
+  },
+  readFilesystemSync (archive) {
+    if (!this.fdCache[archive]) {
+      const fd = fs.openSync(archive, 'r')
+      const header = this.readArchiveHeaderSync(fd)
+      const filesystem = new Filesystem(archive)
+      filesystem.header = header.header
+      filesystem.headerSize = header.headerSize
+      this.fdCache[archive] = {
+        fd,
+        filesystem: filesystem
+      }
+    }
+
+    return this.fdCache[archive]
+  }
+}
+
+class Filesystem {
+  constructor (src) {
+    this.src = path.resolve(src)
+    this.header = { files: {} }
+    this.offset = UINT64(0)
+  }
+
+  searchNodeFromDirectory (p) {
+    let json = this.header
+    const dirs = p.split(path.sep)
+    for (const dir of dirs) {
+      if (dir !== '.') {
+        json = json.files[dir]
+      }
+    }
+    return json
+  }
+
+  getNode (p) {
+    const node = this.searchNodeFromDirectory(path.dirname(p))
+    const name = path.basename(p)
+    if (name) {
+      return node.files[name]
+    } else {
+      return node
+    }
+  }
+
+  getFile (p, followLinks) {
+    followLinks = typeof followLinks === 'undefined' ? true : followLinks
+    const info = this.getNode(p)
+
+    if (!info) {
+      return false
+    }
+
+    if (info.link && followLinks) {
+      return this.getFile(info.link)
+    } else {
+      return info
+    }
+  }
+}

server/core/kernel.js

@@ -41,6 +41,7 @@ module.exports = {
         outbound: new EventEmitter()
       }
       WIKI.extensions = require('./extensions')
+      WIKI.asar = require('./asar')
     } catch (err) {
       WIKI.logger.error(err)
       process.exit(1)
@@ -114,6 +115,9 @@ module.exports = {
     if (WIKI.scheduler) {
       WIKI.scheduler.stop()
     }
+    if (WIKI.asar) {
+      await WIKI.asar.unload()
+    }
     if (WIKI.servers) {
       await WIKI.servers.stopServers()
     }

server/master.js

@@ -53,6 +53,13 @@ module.exports = async () => {
   // ----------------------------------------
 
   app.use(favicon(path.join(WIKI.ROOTPATH, 'assets', 'favicon.ico')))
+  app.use('/_assets/svg/twemoji', async (req, res, next) => {
+    try {
+      WIKI.asar.serve('twemoji', req, res, next)
+    } catch (err) {
+      res.sendStatus(404)
+    }
+  })
   app.use('/_assets', express.static(path.join(WIKI.ROOTPATH, 'assets'), {
     index: false,
     maxAge: '7d'

yarn.lock

@@ -4886,6 +4886,11 @@ chrome-trace-event@^1.0.2:
   dependencies:
     tslib "^1.9.0"
 
+chromium-pickle-js@0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/chromium-pickle-js/-/chromium-pickle-js-0.2.0.tgz#04a106672c18b085ab774d983dfa3ea138f22205"
+  integrity sha1-BKEGZywYsIWrd02YPfo+oTjyIgU=
+
 ci-info@^1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-1.6.0.tgz#2ca20dbb9ceb32d4524a683303313f0304b1e497"
@@ -5834,6 +5839,11 @@ cssstyle@^2.2.0:
   dependencies:
     cssom "~0.3.6"
 
+cuint@0.2.2:
+  version "0.2.2"
+  resolved "https://registry.yarnpkg.com/cuint/-/cuint-0.2.2.tgz#408086d409550c2631155619e9fa7bcadc3b991b"
+  integrity sha1-QICG1AlVDCYxFVYZ6fp7ytw7mRs=
+
 custom-error-instance@2.1.2:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/custom-error-instance/-/custom-error-instance-2.1.2.tgz#dbf463ce4f12567421cc99efd2dd3fa9845a917b"