release: version 1.11.3.1 -> 1.11.3.2

Counterpart of:
https://github.com/pagespeed/mod_pagespeed/commit/f3639e84c0196a5f5151ff5e54ad57285db09b37

config

@@ -29,8 +29,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
     echo "   You need to separately download the pagespeed library:"
     echo ""
     echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.10.33.6.tar.gz"
-    echo "     $ tar -xzvf 1.10.33.6.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.11.33.2.tar.gz"
+    echo "     $ tar -xzvf 1.11.33.2.tar.gz # expands to psol/"
     echo ""
     echo "   Or see the installation instructions:"
     echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"

src/ngx_fetch.cc

@@ -315,7 +315,9 @@ bool NgxFetch::Init() {
   }
 
   if (!ParseUrl()) {
-    message_handler_->Message(kError, "NgxFetch: ParseUrl() failed");
+    message_handler_->Message(kError, 
+                              "NgxFetch: ParseUrl() failed for [%s]:%s",
+                              str_url_.c_str(), url_.err);
     return false;
   }
 
@@ -938,8 +940,8 @@ void NgxFetch::FixUserAgent() {
     user_agent += "NgxNativeFetcher";
   }
   GoogleString version = StrCat(
-      " ", kModPagespeedSubrequestUserAgent,
-      "/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING);
+      " (", kModPagespeedSubrequestUserAgent,
+      "/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING ")");
   if (!StringPiece(user_agent).ends_with(version)) {
     user_agent += version;
   }

src/ngx_pagespeed.cc

@@ -576,6 +576,10 @@ ngx_int_t copy_response_headers_to_ngx(
       continue;
     } else if (STR_EQ_LITERAL(name, "Transfer-Encoding")) {
       continue;
+    } else if (STR_EQ_LITERAL(name, "Vary") && value.len
+        && STR_EQ_LITERAL(value, "Accept-Encoding")) {
+      ps_request_ctx_t* ctx = ps_get_request_context(r);
+      ctx->psol_vary_accept_only = true;
     }
 
     ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(
@@ -678,6 +682,13 @@ char* ps_main_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 char* ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 
+// We want NGX_CONF_MULTI for some very old versions:
+//   https://github.com/pagespeed/ngx_pagespeed/commit/66f1b9aa
+// but it's gone in recent revisions, so provide a compat #define if needed
+#ifndef NGX_CONF_MULTI
+#define NGX_CONF_MULTI 0
+#endif
+
 // TODO(jud): Verify that all the offsets should be NGX_HTTP_SRV_CONF_OFFSET and
 // not NGX_HTTP_LOC_CONF_OFFSET or NGX_HTTP_MAIN_CONF_OFFSET.
 ngx_command_t ps_commands[] = {
@@ -1270,6 +1281,7 @@ ngx_int_t ps_decline_request(ngx_http_request_t* r) {
   ctx->driver->Cleanup();
   ctx->driver = NULL;
   ctx->location_field_set = false;
+  ctx->psol_vary_accept_only = false;
 
   // re init ctx
   ctx->html_rewrite = true;
@@ -1840,6 +1852,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     ctx->recorder = NULL;
     ctx->url_string = url_string;
     ctx->location_field_set = false;
+    ctx->psol_vary_accept_only = false;
 
     // Set up a cleanup handler on the request.
     ngx_http_cleanup_t* cleanup = ngx_http_cleanup_add(r, 0);
@@ -2157,6 +2170,13 @@ ngx_int_t ps_etag_header_filter(ngx_http_request_t* r) {
       break;
     }
   }
+
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+#if (NGX_HTTP_GZIP)
+  if (ctx && ctx->psol_vary_accept_only) {
+    r->gzip_vary = 0;
+  }
+#endif
   return ngx_http_ef_next_header_filter(r);
 }
 

src/ngx_pagespeed.h

@@ -108,6 +108,7 @@ typedef struct {
   // we should mirror that when we write it back. nginx may absolutify
   // Location: headers that start with '/' without regarding X-Forwarded-Proto.
   bool location_field_set;
+  bool psol_vary_accept_only;
 } ps_request_ctx_t;
 
 ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r);

test/nginx_system_test.sh

@@ -552,7 +552,7 @@ fetch_until "$URL" "fgrep -c .pagespeed." 1 --header=Host:www.google.com
 
 # If this accepts the Host header and fetches from google.com it will fail with
 # a 404.  Instead it should use a loopback fetch and succeed.
-URL="$HOSTNAME/mod_pagespeed_example/.pagespeed.ce.8CfGBvwDhH.css"
+URL="$HOSTNAME/mod_pagespeed_example/styles/big.css.pagespeed.ce.8CfGBvwDhH.css"
 check wget -O /dev/null --header=Host:www.google.com "$URL"
 
 start_test statistics load
@@ -1249,6 +1249,28 @@ OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1) || tr
 # We ignored the exit code, check if we got a 404 response.
 check_from "$OUT" fgrep -qi '404'
 
+if false; then
+# This test fails on 1.11, but passes on trunk.  While 2b4c097 fixed most users,
+# it's still possible to get two Vary: Accept-Encoding headers in this test, so
+# disable it for now.
+start_test Single Vary: Accept-Encoding header in IPRO flow
+URL=http://psol-vary.example.com/mod_pagespeed_example/styles/index_style.css
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
+# First hit will be recorded and passed on untouched
+MATCHES=$(echo "$OUT" | grep -c "Vary: Accept-Encoding") || true
+check [ $MATCHES -eq 1 ]
+
+# Fetch until we get a fully optimized response
+http_proxy=$SECONDARY_HOSTNAME \
+  fetch_until $URL "fgrep -c W/\"PSA" 1 --save-headers
+
+# Test the optimized response.
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
+MATCHES=$(echo "$OUT" | grep -c "Vary: Accept-Encoding") || true
+check [ $MATCHES -eq 1 ]
+
+fi
+
 start_test Shutting down.
 
 # Fire up some heavy load if ab is available to test a stressed shutdown
@@ -1324,6 +1346,8 @@ OUT=$(cat "$ERROR_LOG" \
     | grep -v "\\[warn\\].*remote\.cfg.*" \
     | grep -v "\\[warn\\].*end token not received.*" \
     | grep -v "\\[warn\\].*failed to hook next event.*" \
+    | grep -v "\\[warn\\].*Rewrite.*failed.*.pagespeed....0.foo.*" \
+    | grep -v "\\[warn\\].*A.blue.css.*but cannot access the original.*" \
     || true)
 
 check [ -z "$OUT" ]

test/pagespeed_test.conf.template

@@ -703,7 +703,44 @@ http {
     pagespeed EnableFilters rewrite_images,rewrite_css;
     pagespeed EnableFilters convert_to_webp_lossless;
     pagespeed EnableFilters in_place_optimize_for_browser;
+    pagespeed JpegRecompressionQuality 75;
+    pagespeed WebpRecompressionQuality 70;
     pagespeed InPlaceResourceOptimization on;
+    pagespeed AllowVaryOn "Accept";
+    pagespeed FileCachePath "@@IPRO_CACHE@@";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name ipro-for-browser-vary-on-auto.example.com;
+    root "@@SERVER_ROOT@@/mod_pagespeed_example";
+    pagespeed EnableFilters rewrite_images,rewrite_css;
+    pagespeed EnableFilters convert_to_webp_animated;
+    pagespeed EnableFilters convert_to_webp_lossless;
+    pagespeed EnableFilters in_place_optimize_for_browser;
+    pagespeed InPlaceResourceOptimization on;
+    # pagespeed AllowVaryOn "Accept";  # Default is "Auto".
+    pagespeed ImageRecompressionQuality 90;
+    pagespeed JpegRecompressionQuality 75;
+    pagespeed JpegRecompressionQualityForSmallScreens 55;
+    pagespeed JpegQualityForSaveData 35;
+    pagespeed WebpRecompressionQuality 70;
+    pagespeed WebpRecompressionQualityForSmallScreens 50;
+    pagespeed WebpQualityForSaveData 30;
+    pagespeed WebpAnimatedRecompressionQuality 60;
+    pagespeed FileCachePath "@@IPRO_CACHE@@";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name ipro-for-browser-vary-on-none.example.com;
+    root "@@SERVER_ROOT@@/mod_pagespeed_example";
+    pagespeed EnableFilters rewrite_images,in_place_optimize_for_browser;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed AllowVaryOn "None";
+    pagespeed ImageRecompressionQuality 75;
     pagespeed FileCachePath "@@IPRO_CACHE@@";
   }
 
@@ -1471,6 +1508,14 @@ http {
     pagespeed GlobalAdminDomains
      Allow everything-explicitly-allowed.example.com;
   }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name psol-vary.example.com;
+    pagespeed on;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
 
   server {
     listen @@PRIMARY_PORT@@;
@@ -1756,6 +1801,25 @@ http {
           https://www.gstatic.com/psa/static;
     }
 
+    location /mod_pagespeed_test/strip_subresource_hints/default/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed RewriteLevel CoreFilters;
+      pagespeed DisAllow *dontrewriteme*;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/default_passthrough/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed RewriteLevel PassThrough;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/preserve_on/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed PreserveSubresourceHints on;
+      pagespeed RewriteLevel CoreFilters;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/preserve_off/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed PreserveSubresourceHints off;
+      pagespeed RewriteLevel CoreFilters;
+    }
     # $host implicitly tests script variable support. I'd love to test it more
     # directly, but so far this is the best I've come up with and duplicating
     # the test doesn't seem to make sense.

test/run_tests.sh

@@ -23,9 +23,11 @@
 # Exits with status 2 if command line args are wrong.
 #
 # Usage:
-#   ./run_tests.sh primary_port secondary_port mod_pagespeed_dir pagespeed_test_host
+#   ./run_tests.sh primary_port secondary_port mod_pagespeed_dir ngx_binary
+#   pagespeed_test_host
 # Example:
-#   ./run_tests.sh 8050 8051 /path/to/mod_pagespeed www.modpagespeed.com
+#   ./run_tests.sh 8050 8051 /path/to/mod_pagespeed /path/to/nginx/binary
+#   selfsigned.modpagespeed.com
 #
 
 # Normally we test only with the native fetcher off.  Set