Merge pull request #231 from jfhbrook/revert-230-revert-228-brotli-encoding

jfhbrook · web-flow · commit 871a3c41ee25 · 2018-09-02T19:09:17.000-04:00
Fix and pull in brotli encoding (aka `Revert "Revert "Add support for brotli encoding""`)
diff --git a/.gitignore b/.gitignore
@@ -11,7 +11,6 @@ coverage
 *.dat
 *.out
 *.pid
-*.gz
 
 pids
 logs
diff --git a/README.md b/README.md
@@ -97,6 +97,7 @@ const opts = {
   cache: 'max-age=3600',
   cors: false,
   gzip: true,
+  brotli: false,
   defaultExt: 'html',
   handleError: true,
   serverHeader: true,
@@ -209,6 +210,16 @@ that the behavior is appropriate. If `./public/some-file.js.gz` is not valid
 gzip, this will fall back to `./public/some-file.js`. You can turn this off
 with `opts.gzip === false`.
 
+### `opts.brotli`
+### `--brotli`
+
+Serve `./public/some-file.js.br` in place of `./public/some-file.js` when the 
+[brotli encoded](https://github.com/google/brotli) version exists and ecstatic 
+determines that the behavior is appropriate. If the request does not contain 
+`br` in the HTTP `accept-encoding` header, ecstatic will instead attempt to 
+serve a gzipped version (if `opts.gzip` is `true`), or fall back to 
+`./public.some-file.js`. Defaults to **false**.
+
 ### `opts.serverHeader`
 ### `--no-server-header`
 
diff --git a/lib/ecstatic.js b/lib/ecstatic.js
@@ -32,7 +32,7 @@ function decodePathname(pathname) {
 
 
 // Check to see if we should try to compress a file with gzip.
-function shouldCompress(req) {
+function shouldCompressGzip(req) {
   const headers = req.headers;
 
   return headers && headers['accept-encoding'] &&
@@ -42,6 +42,16 @@ function shouldCompress(req) {
     ;
 }
 
+function shouldCompressBrotli(req) {
+  const headers = req.headers;
+
+  return headers && headers['accept-encoding'] &&
+    headers['accept-encoding']
+    .split(',')
+    .some(el => ['*', 'br'].indexOf(el.trim()) !== -1)
+  ;
+}
+
 function hasGzipId12(gzipped, cb) {
   const stream = fs.createReadStream(gzipped, { start: 0, end: 1 });
   let buffer = Buffer('');
@@ -166,7 +176,8 @@ module.exports = function createMiddleware(_dir, _options) {
     const parsed = url.parse(req.url);
     let pathname = null;
     let file = null;
-    let gzipped = null;
+    let gzippedFile = null;
+    let brotliFile = null;
 
     // Strip any null bytes from the url
     // This was at one point necessary because of an old bug in url.parse
@@ -198,7 +209,9 @@ module.exports = function createMiddleware(_dir, _options) {
         path.relative(path.join('/', baseDir), pathname)
       )
     );
-    gzipped = `${file}.gz`;
+    // determine compressed forms if they were to exist
+    gzippedFile = `${file}.gz`;
+    brotliFile = `${file}.br`;
 
     if (serverHeader !== false) {
       // Set common headers.
@@ -229,7 +242,7 @@ module.exports = function createMiddleware(_dir, _options) {
 
     function serve(stat) {
       // Do a MIME lookup, fall back to octet-stream and handle gzip
-      // special case.
+      // and brotli special case.
       const defaultType = opts.contentType || 'application/octet-stream';
       let contentType = mime.lookup(file, defaultType);
       let charSet;
@@ -238,19 +251,21 @@ module.exports = function createMiddleware(_dir, _options) {
       const etag = generateEtag(stat, weakEtags);
       let cacheControl = cache;
       let stream = null;
-
       if (contentType) {
         charSet = mime.charsets.lookup(contentType, 'utf-8');
         if (charSet) {
           contentType += `; charset=${charSet}`;
         }
       }
 
-      if (file === gzipped) { // is .gz picked up
+      if (file === gzippedFile) { // is .gz picked up
         res.setHeader('Content-Encoding', 'gzip');
-
         // strip gz ending and lookup mime type
         contentType = mime.lookup(path.basename(file, '.gz'), defaultType);
+      } else if (file === brotliFile) { // is .br picked up
+        res.setHeader('Content-Encoding', 'br');
+        // strip br ending and lookup mime type
+        contentType = mime.lookup(path.basename(file, '.br'), defaultType);
       }
 
       if (typeof cacheControl === 'function') {
@@ -401,13 +416,13 @@ module.exports = function createMiddleware(_dir, _options) {
       });
     }
 
-    // Look for a gzipped file if this is turned on
-    if (opts.gzip && shouldCompress(req)) {
-      fs.stat(gzipped, (err, stat) => {
+    // serve gzip file if exists and is valid
+    function tryServeWithGzip() {
+      fs.stat(gzippedFile, (err, stat) => {
         if (!err && stat.isFile()) {
-          hasGzipId12(gzipped, (gzipErr, isGzip) => {
+          hasGzipId12(gzippedFile, (gzipErr, isGzip) => {
             if (!gzipErr && isGzip) {
-              file = gzipped;
+              file = gzippedFile;
               serve(stat);
             } else {
               statFile();
@@ -417,6 +432,29 @@ module.exports = function createMiddleware(_dir, _options) {
           statFile();
         }
       });
+    }
+
+    // serve brotli file if exists, otherwise try gzip
+    function tryServeWithBrotli(shouldTryGzip) {
+      fs.stat(brotliFile, (err, stat) => {
+        if (!err && stat.isFile()) {
+          file = brotliFile;
+          serve(stat);
+        } else if (shouldTryGzip) {
+          tryServeWithGzip();
+        } else {
+          statFile();
+        }
+      });
+    }
+
+    const shouldTryBrotli = opts.brotli && shouldCompressBrotli(req);
+    const shouldTryGzip = opts.gzip && shouldCompressGzip(req);
+    // always try brotli first, next try gzip, finally serve without compression
+    if (shouldTryBrotli) {
+      tryServeWithBrotli(shouldTryGzip);
+    } else if (shouldTryGzip) {
+      tryServeWithGzip();
     } else {
       statFile();
     }
diff --git a/lib/ecstatic/defaults.json b/lib/ecstatic/defaults.json
@@ -8,6 +8,7 @@
   "cache": "max-age=3600",
   "cors": false,
   "gzip": true,
+  "brotli": false,
   "defaultExt": ".html",
   "handleError": true,
   "serverHeader": true,
diff --git a/lib/ecstatic/opts.js b/lib/ecstatic/opts.js
@@ -14,6 +14,7 @@ module.exports = (opts) => {
   let si = defaults.si;
   let cache = defaults.cache;
   let gzip = defaults.gzip;
+  let brotli = defaults.brotli;
   let defaultExt = defaults.defaultExt;
   let handleError = defaults.handleError;
   const headers = {};
@@ -105,6 +106,10 @@ module.exports = (opts) => {
       gzip = opts.gzip;
     }
 
+    if (typeof opts.brotli !== 'undefined' && opts.brotli !== null) {
+      brotli = opts.brotli;
+    }
+
     aliases.handleError.some((k) => {
       if (isDeclared(k)) {
         handleError = opts[k];
@@ -195,6 +200,7 @@ module.exports = (opts) => {
     defaultExt,
     baseDir: (opts && opts.baseDir) || '/',
     gzip,
+    brotli,
     handleError,
     headers,
     serverHeader,
diff --git a/test/compression.js b/test/compression.js
@@ -0,0 +1,187 @@
+'use strict';
+
+const test = require('tap').test;
+const ecstatic = require('../lib/ecstatic');
+const http = require('http');
+const request = require('request');
+
+const root = `${__dirname}/public`;
+
+test('serves brotli-encoded file when available', (t) => {
+  t.plan(3);
+
+  const server = http.createServer(ecstatic({
+    root,
+    brotli: true,
+    autoIndex: true
+  }));
+
+  server.listen(() => {
+    const port = server.address().port;
+    const options = {
+      uri: `http://localhost:${port}/brotli`,
+      headers: {
+        'accept-encoding': 'gzip, deflate, br'
+      }
+    };
+
+    request.get(options, (err, res) => {
+      t.ifError(err);
+      t.equal(res.statusCode, 200);
+      t.equal(res.headers['content-encoding'], 'br');
+    });
+  });
+  t.once('end', () => {
+    server.close();
+  });
+});
+
+test('serves gzip-encoded file when brotli not available', (t) => {
+  t.plan(3);
+
+  const server = http.createServer(ecstatic({
+    root,
+    brotli: true,
+    gzip: true,
+    autoIndex: true
+  }));
+
+  server.listen(() => {
+    const port = server.address().port;
+    const options = {
+      uri: `http://localhost:${port}/gzip`,
+      headers: {
+        'accept-encoding': 'gzip, deflate, br'
+      }
+    };
+
+    request.get(options, (err, res) => {
+      t.ifError(err);
+      t.equal(res.statusCode, 200);
+      t.equal(res.headers['content-encoding'], 'gzip');
+    });
+  });
+  t.once('end', () => {
+    server.close();
+  });
+});
+
+test('serves gzip-encoded file when brotli not accepted', (t) => {
+  t.plan(3);
+
+  const server = http.createServer(ecstatic({
+    root,
+    brotli: true,
+    gzip: true,
+    autoIndex: true
+  }));
+
+  server.listen(() => {
+    const port = server.address().port;
+    const options = {
+      uri: `http://localhost:${port}/brotli`,
+      headers: {
+        'accept-encoding': 'gzip, deflate'
+      }
+    };
+
+    request.get(options, (err, res) => {
+      t.ifError(err);
+      t.equal(res.statusCode, 200);
+      t.equal(res.headers['content-encoding'], 'gzip');
+    });
+  });
+  t.once('end', () => {
+    server.close();
+  });
+});
+
+test('serves gzip-encoded file when brotli not enabled', (t) => {
+  t.plan(3);
+
+  const server = http.createServer(ecstatic({
+    root,
+    brotli: false,
+    gzip: true,
+    autoIndex: true
+  }));
+
+  server.listen(() => {
+    const port = server.address().port;
+    const options = {
+      uri: `http://localhost:${port}/brotli`,
+      headers: {
+        'accept-encoding': 'gzip, deflate, br'
+      }
+    };
+
+    request.get(options, (err, res) => {
+      t.ifError(err);
+      t.equal(res.statusCode, 200);
+      t.equal(res.headers['content-encoding'], 'gzip');
+    });
+  });
+  t.once('end', () => {
+    server.close();
+  });
+});
+
+test('serves unencoded file when compression not accepted', (t) => {
+  t.plan(3);
+
+  const server = http.createServer(ecstatic({
+    root,
+    brotli: true,
+    gzip: true,
+    autoIndex: true
+  }));
+
+  server.listen(() => {
+    const port = server.address().port;
+    const options = {
+      uri: `http://localhost:${port}/brotli`,
+      headers: {
+        'accept-encoding': ''
+      }
+    };
+
+    request.get(options, (err, res) => {
+      t.ifError(err);
+      t.equal(res.statusCode, 200);
+      t.equal(res.headers['content-encoding'], undefined);
+    });
+  });
+  t.once('end', () => {
+    server.close();
+  });
+});
+
+test('serves unencoded file when compression not enabled', (t) => {
+  t.plan(3);
+
+  const server = http.createServer(ecstatic({
+    root,
+    brotli: false,
+    gzip: false,
+    autoIndex: true
+  }));
+
+  server.listen(() => {
+    const port = server.address().port;
+    const options = {
+      uri: `http://localhost:${port}/brotli`,
+      headers: {
+        'accept-encoding': 'gzip, deflate, br'
+      }
+    };
+
+    request.get(options, (err, res) => {
+      t.ifError(err);
+      t.equal(res.statusCode, 200);
+      t.equal(res.headers['content-encoding'], undefined);
+    });
+  });
+  t.once('end', () => {
+    server.close();
+  });
+});
diff --git a/test/public/brotli/fake_ecstatic b/test/public/brotli/fake_ecstatic
@@ -0,0 +1 @@
+ecstatic
diff --git a/test/public/brotli/fake_ecstatic.br b/test/public/brotli/fake_ecstatic.br
diff --git a/test/public/brotli/index.html b/test/public/brotli/index.html
@@ -0,0 +1 @@
+brotli, but I'm not compressed!!!
diff --git a/test/public/brotli/index.html.br b/test/public/brotli/index.html.br
@@ -0,0 +1,3 @@
+
+�brotli, compressed!!
+
diff --git a/test/public/brotli/index.html.gz b/test/public/brotli/index.html.gz
diff --git a/test/public/brotli/not_actually_brotli.br b/test/public/brotli/not_actually_brotli.br
@@ -0,0 +1 @@
+You've been duped! This is not compressed!
diff --git a/test/public/brotli/real_ecstatic b/test/public/brotli/real_ecstatic
@@ -0,0 +1 @@
+ecstatic
diff --git a/test/public/brotli/real_ecstatic.br b/test/public/brotli/real_ecstatic.br

-Original file line number
+Diff line change
 *.dat
 *.out
 *.pid
 -*.gz
 pids
 logs