Merge pull request #10151 from internetarchive/fix-nginx-logs

mekarpeles · web-flow · commit 0045f029824f · 2025-01-15T14:28:35.000-05:00
Switch to using NJS for nginx IP anonymization
diff --git a/docker/Dockerfile.olbase b/docker/Dockerfile.olbase
@@ -38,22 +38,10 @@ RUN apt-get -qq update && apt-get install -y \
 COPY scripts/install_nodejs.sh ./
 RUN ./install_nodejs.sh && rm ./install_nodejs.sh
 
-# Install Archive.org nginx w/ IP anonymization
+# Install nginx
 USER root
-RUN apt-get update && apt-get install -y --no-install-recommends nginx curl letsencrypt \
-    # nginx-plus
-    apt-transport-https lsb-release ca-certificates wget \
-    # log rotation service for ol-nginx
-    logrotate \
-    # rsync service for pulling monthly sitemaps from ol-home0 to ol-www0
-    rsync
-COPY scripts/install_openresty.sh ./
-RUN ./install_openresty.sh && rm ./install_openresty.sh
-RUN rm /usr/sbin/nginx
-RUN curl -L https://archive.org/download/nginx/nginx -o /usr/sbin/nginx
-RUN chmod +x /usr/sbin/nginx
-# Remove the stock nginx config file
-RUN rm /etc/nginx/sites-enabled/default
+COPY scripts/install_nginx.sh ./
+RUN ./install_nginx.sh && rm ./install_nginx.sh
 
 RUN mkdir -p /var/log/openlibrary /var/lib/openlibrary && chown openlibrary:openlibrary /var/log/openlibrary /var/lib/openlibrary \
  && mkdir /openlibrary && chown openlibrary:openlibrary /openlibrary \
diff --git a/docker/covers_nginx.conf b/docker/covers_nginx.conf
@@ -14,6 +14,9 @@ server {
     ssl_protocols TLSv1.2 TLSv1.3;
     ssl_ciphers EECDH+CHACHA20:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;
     ssl_prefer_server_ciphers on;
+
+    # Needed for logging/IP anonymization
+    include /olsystem/etc/nginx/logging_periodics.conf;
 }
 
 # Docker's internal load balancing ends up with unbalanced connections eventually.
diff --git a/docker/nginx.conf b/docker/nginx.conf
@@ -1,3 +1,6 @@
+# Needed for IP anonymization
+load_module modules/ngx_http_js_module.so;
+
 user  www-data;
 
 # XXX-Anand: Oct 2013
@@ -25,7 +28,8 @@ http {
     server_names_hash_bucket_size   64;
     types_hash_bucket_size 64;
 
-    log_format iacombined '$remote_addr_ipscrub $host $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time';
+    # Logging / IP Anonymization; also need logging_periodics.conf inside a server block
+    include /olsystem/etc/nginx/logging.conf;
     access_log    /var/log/nginx/access.log iacombined;
 
     client_max_body_size 50m;
diff --git a/docker/web_nginx.conf b/docker/web_nginx.conf
@@ -32,6 +32,9 @@ server {
     ssl_protocols TLSv1.2 TLSv1.3;
     ssl_ciphers EECDH+CHACHA20:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;
     ssl_prefer_server_ciphers on;
+
+    # Needed for logging/IP anonymization
+    include /olsystem/etc/nginx/logging_periodics.conf;
 }
 
 server {
diff --git a/scripts/install_nginx.sh b/scripts/install_nginx.sh
@@ -0,0 +1,19 @@
+#! /bin/bash
+
+apt-get update
+
+# log rotation service for ol-nginx
+# rsync service for pulling monthly sitemaps from ol-home0 to ol-www0
+apt-get install -y --no-install-recommends curl \
+    logrotate \
+    rsync \
+    lsb-release
+
+# Add the NGINX signing key + Repo
+curl -fsSL https://nginx.org/keys/nginx_signing.key | tee /usr/share/keyrings/nginx-keyring.asc
+echo "deb [signed-by=/usr/share/keyrings/nginx-keyring.asc] http://nginx.org/packages/debian $(lsb_release -cs) nginx" \
+    > /etc/apt/sources.list.d/nginx.list
+
+# Install nginx and the NJS module
+apt-get update
+apt-get install -y --no-install-recommends nginx nginx-module-njs letsencrypt
diff --git a/scripts/install_openresty.sh b/scripts/install_openresty.sh

Original file line number	Diff line number	Diff line change
`@@ -14,6 +14,9 @@ server {`
`14`	`14`	`ssl_protocols TLSv1.2 TLSv1.3;`
`15`	`15`	`ssl_ciphers EECDH+CHACHA20:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;`
`16`	`16`	`ssl_prefer_server_ciphers on;`
	`17`	`+`
	`18`	`+ # Needed for logging/IP anonymization`
	`19`	`+ include /olsystem/etc/nginx/logging_periodics.conf;`
`17`	`20`	`}`
`18`	`21`
`19`	`22`	`# Docker's internal load balancing ends up with unbalanced connections eventually.`
Original file line number	Diff line number	Diff line change
`@@ -32,6 +32,9 @@ server {`
`32`	`32`	`ssl_protocols TLSv1.2 TLSv1.3;`
`33`	`33`	`ssl_ciphers EECDH+CHACHA20:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;`
`34`	`34`	`ssl_prefer_server_ciphers on;`
	`35`	`+`
	`36`	`+ # Needed for logging/IP anonymization`
	`37`	`+ include /olsystem/etc/nginx/logging_periodics.conf;`
`35`	`38`	`}`
`36`	`39`
`37`	`40`	`server {`