Disallow scratch papers to be crawled by robots

64json · 64json · commit 83213cdb1819 · 2019-01-24T03:15:16.000-05:00
diff --git a/app/index.js b/app/index.js
@@ -1,3 +1,4 @@
+const path = require('path');
 const express = require('express');
 const app = express();
 
@@ -18,6 +19,9 @@ app.use((req, res, next) => {
     next();
   }
 });
+app.get('/robots.txt', (req, res) => {
+  res.sendFile(path.resolve(__dirname, '..', 'robots.txt'));
+});
 app.use(apiEndpoint, backend);
 app.use(frontend);
 
diff --git a/robots.txt b/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /scratch-paper/
