release: version 1.11.3.1 -> 1.11.3.2

Counterpart of:
https://github.com/pagespeed/mod_pagespeed/commit/f3639e84c0196a5f5151ff5e54ad57285db09b37

config

@@ -29,8 +29,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
     echo "   You need to separately download the pagespeed library:"
     echo ""
     echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.9.32.1.tar.gz"
-    echo "     $ tar -xzvf 1.9.32.1.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.11.33.2.tar.gz"
+    echo "     $ tar -xzvf 1.11.33.2.tar.gz # expands to psol/"
     echo ""
     echo "   Or see the installation instructions:"
     echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
@@ -71,6 +71,20 @@ else
   buildtype=Release
 fi
 
+# If the compiler is gcc, we want to use g++ to link, if at all possible,
+# so that -static-libstdc++ works.
+# Annoyingly, the feature test doesn't even use $LINK for linking, so that
+# needs an explicit -lstdc++
+pagespeed_libs=
+ps_maybe_gpp_base=`basename $CC| sed s/gcc/g++/`
+ps_maybe_gpp="`dirname $CC`/$ps_maybe_gpp_base"
+if [ -n "$NGX_GCC_VER" -a \( -x "$ps_maybe_gpp" \) ]; then
+  LINK=$ps_maybe_gpp
+  NGX_TEST_LD_OPT="$NGX_TEST_LD_OPT -lstdc++"
+else
+  pagespeed_libs="-lstdc++"
+fi
+
 # The compiler needs to know that __sync_add_and_fetch_4 is ok,
 # and this requires an instruction that didn't exist on i586 or i386.
 if [ "$uname_arch" = "i686" ]; then
@@ -79,6 +93,13 @@ fi
 
 CFLAGS="$CFLAGS $FLAG_MARCH"
 
+# For now, standardize on gcc-4.x ABI --- if we don't set this, people building
+# with new gcc defaulting to gcc-5 C++11 ABI will have build trouble linking
+# to our libpsol.a
+# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html
+CFLAGS="$CFLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
+CC_TEST_FLAGS="$CC_TEST_FLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
+
 case "$NGX_GCC_VER" in
   4.8*)
     # On GCC 4.8 and above, -Wall enables -Wunused-local-typedefs.  This breaks
@@ -97,6 +118,13 @@ case "$NGX_GCC_VER" in
   ;;
 esac
 
+# workaround for a bug in nginx-1.9.11, see:
+# http://hg.nginx.org/nginx/rev/ff1e625ae55b
+NGX_VERSION=`grep nginx_version src/core/nginx.h | sed -e 's/^.* \(.*\)$/\1/'`
+if [ "$NGX_VERSION" = "1009011" ]; then
+    CFLAGS="$CFLAGS -Wno-write-strings"
+fi
+
 if [ "$WNO_ERROR" = "YES" ]; then
     CFLAGS="$CFLAGS -Wno-error"
 fi
@@ -141,9 +169,10 @@ pagespeed_include="\
   $mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include"
 ngx_feature_path="$pagespeed_include"
 
-pagespeed_libs="-lstdc++ $psol_binary -lrt -pthread -lm"
+pagespeed_libs="$pagespeed_libs $psol_binary -lrt -pthread -lm"
 ngx_feature_libs="$pagespeed_libs"
 ngx_feature_test="
+
   GoogleString output_buffer;
   net_instaweb::StringWriter write_to_string(&output_buffer);
 
@@ -164,54 +193,7 @@ ngx_feature_test="
 # Test whether we have pagespeed and can compile and link against it.
 . "$ngx_addon_dir/cpp_feature"
 
-if [ $ngx_found = yes ]; then
-  ps_src="$ngx_addon_dir/src"
-  ngx_addon_name=ngx_pagespeed
-  NGX_ADDON_DEPS="$NGX_ADDON_DEPS \
-    $ps_src/log_message_handler.h \
-    $ps_src/ngx_base_fetch.h \
-    $ps_src/ngx_caching_headers.h \
-    $ps_src/ngx_event_connection.h \
-    $ps_src/ngx_fetch.h \
-    $ps_src/ngx_gzip_setter.h \
-    $ps_src/ngx_list_iterator.h \
-    $ps_src/ngx_message_handler.h \
-    $ps_src/ngx_pagespeed.h \
-    $ps_src/ngx_rewrite_driver_factory.h \
-    $ps_src/ngx_rewrite_options.h \
-    $ps_src/ngx_server_context.h \
-    $ps_src/ngx_url_async_fetcher.h \
-    $psol_binary"
-  NGX_ADDON_SRCS="$NGX_ADDON_SRCS \
-    $ps_src/log_message_handler.cc \
-    $ps_src/ngx_base_fetch.cc \
-    $ps_src/ngx_caching_headers.cc \
-    $ps_src/ngx_event_connection.cc \
-    $ps_src/ngx_fetch.cc \
-    $ps_src/ngx_gzip_setter.cc \
-    $ps_src/ngx_list_iterator.cc \
-    $ps_src/ngx_message_handler.cc \
-    $ps_src/ngx_pagespeed.cc \
-    $ps_src/ngx_rewrite_driver_factory.cc \
-    $ps_src/ngx_rewrite_options.cc \
-    $ps_src/ngx_server_context.cc \
-    $ps_src/ngx_url_async_fetcher.cc"
-
-  if [ "$position_aux" = "true" ] ; then
-     HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
-  else
-    # Make pagespeed run immediately before gzip.
-    HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
-      sed "s/$HTTP_GZIP_FILTER_MODULE/$HTTP_GZIP_FILTER_MODULE $ngx_addon_name/")
-  fi
-
-  # Make the etag header filter run immediately after gzip.
-  HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
-    sed "s/$HTTP_GZIP_FILTER_MODULE/ngx_pagespeed_etag_filter $HTTP_GZIP_FILTER_MODULE/")
-  CORE_LIBS="$CORE_LIBS $pagespeed_libs"
-  CORE_INCS="$CORE_INCS $pagespeed_include"
-  echo "List of modules (in reverse order of applicability): "$HTTP_FILTER_MODULES
-else
+if [ $ngx_found = no ]; then
   cat << END
 $0: error: module ngx_pagespeed requires the pagespeed optimization library.
 Look in obj/autoconf.err for more details.
@@ -219,4 +201,155 @@ END
   exit 1
 fi
 
+ps_src="$ngx_addon_dir/src"
+ngx_addon_name=ngx_pagespeed
+NGX_ADDON_DEPS="$NGX_ADDON_DEPS \
+$ps_src/log_message_handler.h \
+$ps_src/ngx_base_fetch.h \
+$ps_src/ngx_caching_headers.h \
+$ps_src/ngx_event_connection.h \
+$ps_src/ngx_fetch.h \
+$ps_src/ngx_gzip_setter.h \
+$ps_src/ngx_list_iterator.h \
+$ps_src/ngx_message_handler.h \
+$ps_src/ngx_pagespeed.h \
+$ps_src/ngx_rewrite_driver_factory.h \
+$ps_src/ngx_rewrite_options.h \
+$ps_src/ngx_server_context.h \
+$ps_src/ngx_url_async_fetcher.h \
+$psol_binary"
+NPS_SRCS=" \
+$ps_src/log_message_handler.cc \
+$ps_src/ngx_base_fetch.cc \
+$ps_src/ngx_caching_headers.cc \
+$ps_src/ngx_event_connection.cc \
+$ps_src/ngx_fetch.cc \
+$ps_src/ngx_gzip_setter.cc \
+$ps_src/ngx_list_iterator.cc \
+$ps_src/ngx_message_handler.cc \
+$ps_src/ngx_pagespeed.cc \
+$ps_src/ngx_rewrite_driver_factory.cc \
+$ps_src/ngx_rewrite_options.cc \
+$ps_src/ngx_server_context.cc \
+$ps_src/ngx_url_async_fetcher.cc"
+# Save our sources in a separate var since we may need it in config.make
+PS_NGX_SRCS="$NGX_ADDON_SRCS \
+$NPS_SRCS"
+
+# Make pagespeed run immediately before gzip and Brotli.
+if echo $HTTP_FILTER_MODULES | grep ngx_http_brotli_filter_module >/dev/null; then
+  next=ngx_http_brotli_filter_module
+elif [ $HTTP_GZIP = YES ]; then
+  next=ngx_http_gzip_filter_module
+else
+  next=ngx_http_range_header_filter_module
+fi
+
+if [ -n "$ngx_module_link" ]; then
+  # nginx-1.9.11+
+  ngx_module_type=HTTP_FILTER
+  ngx_module_name="ngx_pagespeed ngx_pagespeed_etag_filter"
+  ngx_module_incs="$ngx_feature_path"
+  ngx_module_deps=
+  ngx_module_srcs="$NPS_SRCS"
+  ngx_module_libs="$ngx_feature_libs"
+  ngx_module_order="ngx_http_range_header_filter_module\
+                    ngx_pagespeed_etag_filter\
+                    ngx_http_gzip_filter_module \
+                    ngx_http_brotli_filter_module \
+                    ngx_pagespeed \
+                    ngx_http_postpone_filter_module \
+                    ngx_http_ssi_filter_module \
+                    ngx_http_charset_filter_module \
+                    ngx_http_xslt_filter_module \
+                    ngx_http_image_filter_module \
+                    ngx_http_sub_filter_module \
+                    ngx_http_addition_filter_module \
+                    ngx_http_gunzip_filter_module \
+                    ngx_http_userid_filter_module \
+                    ngx_http_headers_filter_module"
+
+    . auto/module
+
+    if [ $ngx_module_link != DYNAMIC ]; then
+      # ngx_module_order doesn't work with static modules,
+      # so we must re-order filters here.
+      if [ "$position_aux" = "true" ] ; then
+        HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
+      else
+        HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES \
+                             | sed "s/ngx_pagespeed//" \
+                             | sed "s/$next/$next ngx_pagespeed/")
+      fi
+      # Make the etag header filter run immediately before range header filter.
+      HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES \
+                           | sed "s/ngx_pagespeed_etag_filter//" \
+                           | sed "s/ngx_http_range_header_filter_module/ngx_http_range_header_filter_module ngx_pagespeed_etag_filter/")
+    else
+      # config.make is not executed for dynamic modules
+      CFLAGS="$CFLAGS -Wno-c++11-extensions"
+      if [ "$position_aux" = "true" ] ; then
+        ngx_module_type=HTTP_AUX_FILTER
+        ngx_module_order=""
+      fi
+    fi
+else
+  CORE_LIBS="$CORE_LIBS $pagespeed_libs"
+  CORE_INCS="$CORE_INCS $pagespeed_include"
+  NGX_ADDON_SRCS="$PS_NGX_SRCS"
+  if [ "$position_aux" = "true" ] ; then
+    HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
+  else
+    HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES | sed "s/$next/$next $ngx_addon_name/")
+  fi
+
+  # Make the etag header filter run immediately before range header filter.
+  HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
+    sed "s/ngx_http_range_header_filter_module/ngx_http_range_header_filter_module ngx_pagespeed_etag_filter/")
+fi
+
+echo "List of modules (in reverse order of applicability): "$HTTP_FILTER_MODULES
+
+# Test whether the compiler is compatible
+ngx_feature="psol-compiler-compat"
+ngx_feature_name=""
+ngx_feature_run=no
+ngx_feature_incs=""
+ngx_feature_path=""
+ngx_feature_libs="-lstdc++"
+ngx_feature_test="
+
+#if defined(__clang__) && defined(__GLIBCXX__)
+   // See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning 
+   // for a list of various values of __GLIBCXX__. Note that they're not monotonic
+   // with respect to version numbers.
+   #if __GLIBCXX__ == 20120322  || __GLIBCXX__ == 20120614
+      #error \"clang is using libstdc++ 4.7.0 or 4.7.1, which can cause binary incompatibility.\"
+   #endif
+#endif
+
+#if !defined(__clang__) && defined(__GNUC__)
+   #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8)
+      #error \"GCC < 4.8 no longer supported. Please use gcc >= 4.8 or clang >= 3.3\"
+   #endif
+#endif
+
+#if defined(__clang__)
+   #if __clang_major__ < 3 || (__clang_major__ == 3 && __clang_minor__ < 3)
+      #error \"Please use gcc >= 4.8 or clang >= 3.3\"
+   #endif
+#endif
+"
+
+. "$ngx_addon_dir/cpp_feature"
+
+if [ $ngx_found = no ]; then
+  cat << END
+$0: error: module ngx_pagespeed requires gcc >= 4.8 or clang >= 3.3.
+See https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source for some recommendations.
+Look in objs/autoconf.err for more details.
+END
+  exit 1
+fi
+
 have=NGX_PAGESPEED . auto/have

config.make

@@ -0,0 +1,7 @@
+if [ -n "$NGX_CLANG_VER" ]; then
+  # Chromium headers assume clang is always in C++11 mode. Oblige it.
+  for ps_src_file in $PS_NGX_SRCS; do
+    ps_obj_file="$NGX_OBJS/addon/src/`basename $ps_src_file .cc`.o"
+    echo "$ps_obj_file : CFLAGS += --std=c++11" >> $NGX_MAKEFILE
+  done
+fi

src/log_message_handler.cc

@@ -38,7 +38,7 @@
 
 namespace {
 
-ngx_log_t* log = NULL;
+ngx_log_t* ngx_log = NULL;
 
 ngx_uint_t GetNgxLogLevel(int severity) {
   switch (severity) {
@@ -78,7 +78,7 @@ bool LogMessageHandler(int severity, const char* file, int line,
     message.resize(last_msg_character_index);
   }
 
-  ngx_log_error(this_log_level, log, 0, "[ngx_pagespeed %s] %s",
+  ngx_log_error(this_log_level, ngx_log, 0, "[ngx_pagespeed %s] %s",
                 net_instaweb::kModPagespeedVersion,
                 message.c_str());
 
@@ -99,12 +99,12 @@ namespace log_message_handler {
 
 
 void Install(ngx_log_t* log_in) {
-  log = log_in;
+  ngx_log = log_in;
   logging::SetLogMessageHandler(&LogMessageHandler);
 
   // All VLOG(2) and higher will be displayed as DEBUG logs if the nginx log
   // level is DEBUG.
-  if (log->log_level >= NGX_LOG_DEBUG) {
+  if (ngx_log->log_level >= NGX_LOG_DEBUG) {
     logging::SetMinLogLevel(-2);
   }
 }

src/ngx_base_fetch.cc

@@ -111,11 +111,15 @@ const char* BaseFetchTypeToCStr(NgxBaseFetchType type) {
       return "admin page";
     case kIproLookup:
       return "ipro lookup";
+    case kPageSpeedProxy:
+      return "pagespeed proxy";
   }
   CHECK(false);
   return "can't get here";
 }
 
+// TODO(oschaaf): replace the ngx_log_error with VLOGS or pass in a
+// MessageHandler and use that.
 void NgxBaseFetch::ReadCallback(const ps_event_data& data) {
   NgxBaseFetch* base_fetch = reinterpret_cast<NgxBaseFetch*>(data.sender);
   ngx_http_request_t* r = base_fetch->request();
@@ -136,19 +140,40 @@ void NgxBaseFetch::ReadCallback(const ps_event_data& data) {
   if (refcount == 0 || detached) {
     return;
   }
+
   ps_request_ctx_t* ctx = ps_get_request_context(r);
 
-  CHECK(data.sender == ctx->base_fetch);
-  CHECK(r->count > 0) << "r->count: " << r->count;
+  // If our request context was zeroed, skip this event.
+  // See https://github.com/pagespeed/ngx_pagespeed/issues/1081
+  if (ctx == NULL) {
+    // Should not happen normally, when it does this message will cause our
+    // system tests to fail.
+    ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
+        "pagespeed [%p] skipping event: request context gone", r);
+    return;
+  }
+
+  // Normally we expect the sender to equal the active NgxBaseFetch instance.
+  DCHECK(data.sender == ctx->base_fetch);
+
+  // If someone changed our request context or NgxBaseFetch, skip processing.
+  if (data.sender != ctx->base_fetch) {
+      ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
+          "pagespeed [%p] skipping event: event originating from disassociated"
+          " NgxBaseFetch instance.", r);
+      return;
+  }
 
   int rc;
+  bool run_posted = true;
   // If we are unlucky enough to have our connection finalized mid-ipro-lookup,
   // we must enter a different flow. Also see ps_in_place_check_header_filter().
   if ((ctx->base_fetch->base_fetch_type_ != kIproLookup)
       && r->connection->error) {
     ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
-      "pagespeed [%p] request already finalized", r);
+      "pagespeed [%p] request already finalized %d", r, r->count);
     rc = NGX_ERROR;
+    run_posted = false;
   } else {
     rc = ps_base_fetch::ps_base_fetch_handler(r);
   }
@@ -161,8 +186,11 @@ void NgxBaseFetch::ReadCallback(const ps_event_data& data) {
 
   ngx_connection_t* c = r->connection;
   ngx_http_finalize_request(r, rc);
-  // See http://forum.nginx.org/read.php?2,253006,253061
-  ngx_http_run_posted_requests(c);
+
+  if (run_posted) {
+    // See http://forum.nginx.org/read.php?2,253006,253061
+    ngx_http_run_posted_requests(c);
+  }
 }
 
 void NgxBaseFetch::Lock() {

src/ngx_base_fetch.h

@@ -68,7 +68,8 @@ enum NgxBaseFetchType {
   kIproLookup,
   kHtmlTransform,
   kPageSpeedResource,
-  kAdminPage
+  kAdminPage,
+  kPageSpeedProxy
 };
 
 class NgxBaseFetch : public AsyncFetch {

src/ngx_fetch.cc

@@ -315,7 +315,9 @@ bool NgxFetch::Init() {
   }
 
   if (!ParseUrl()) {
-    message_handler_->Message(kError, "NgxFetch: ParseUrl() failed");
+    message_handler_->Message(kError, 
+                              "NgxFetch: ParseUrl() failed for [%s]:%s",
+                              str_url_.c_str(), url_.err);
     return false;
   }
 
@@ -938,8 +940,8 @@ void NgxFetch::FixUserAgent() {
     user_agent += "NgxNativeFetcher";
   }
   GoogleString version = StrCat(
-      " ", kModPagespeedSubrequestUserAgent,
-      "/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING);
+      " (", kModPagespeedSubrequestUserAgent,
+      "/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING ")");
   if (!StringPiece(user_agent).ends_with(version)) {
     user_agent += version;
   }

src/ngx_pagespeed.cc

@@ -284,7 +284,8 @@ ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r) {
     // modules running after us to manipulate those responses.
     if (!status_ok && (ctx->base_fetch->base_fetch_type() != kHtmlTransform
                        && ctx->base_fetch->base_fetch_type() != kIproLookup)) {
-      return status_code;
+      ps_release_base_fetch(ctx);
+      return ngx_http_filter_finalize_request(r, NULL, status_code);
     }
 
     if (ctx->preserve_caching_headers != kDontPreserveHeaders) {
@@ -302,6 +303,12 @@ ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r) {
                 STR_CASE_EQ_LITERAL(header->key, "Last-Modified") ||
                 STR_CASE_EQ_LITERAL(header->key, "Expires"))))) {
           header->hash = 0;
+          if (STR_CASE_EQ_LITERAL(header->key, "Location")) {
+            // There's a possible issue with the location header, where setting
+            // the hash to 0 is not enough. See:
+            // https://github.com/nginx/nginx/blob/master/src/http/ngx_http_header_filter_module.c#L314
+            r->headers_out.location = NULL;
+          }
         }
       }
     } else {
@@ -452,11 +459,6 @@ void copy_response_headers_from_ngx(const ngx_http_request_t* r,
 
   headers->set_status_code(r->headers_out.status);
 
-  if (r->headers_out.location != NULL) {
-    headers->Add(HttpAttributes::kLocation,
-                 str_to_string_piece(r->headers_out.location->value));
-  }
-
   // Manually copy over the content type because it's not included in
   // request_->headers_out.headers.
   headers->Add(HttpAttributes::kContentType,
@@ -490,6 +492,10 @@ ngx_int_t copy_response_headers_to_ngx(
 
   ngx_int_t i;
   for (i = 0 ; i < pagespeed_headers.NumAttributes() ; i++) {
+    // For IPRO cache misses, these gs_ variables may point to freed memory
+    // when nginx writes the headers to the output as the NgxBaseFetch instance
+    // that owns this memory gets released during request processing. So we
+    // copy these strings later on.
     const GoogleString& name_gs = pagespeed_headers.Name(i);
     const GoogleString& value_gs = pagespeed_headers.Value(i);
 
@@ -510,6 +516,9 @@ ngx_int_t copy_response_headers_to_ngx(
     }  // else we don't preserve any headers.
 
     ngx_str_t name, value;
+    value.len = value_gs.size();
+    value.data = reinterpret_cast<u_char*>(
+        string_piece_to_pool_string(r->pool, value_gs.c_str()));
 
     // To prevent the gzip module from clearing weak etags, we output them
     // using a different name here. The etag header filter module runs behind
@@ -520,11 +529,15 @@ ngx_int_t copy_response_headers_to_ngx(
       name.data = reinterpret_cast<u_char*>(
           const_cast<char*>(kInternalEtagName));
     } else {
-      name.len = name_gs.length();
-      name.data = reinterpret_cast<u_char*>(const_cast<char*>(name_gs.data()));
+      name.len = name_gs.size();
+      name.data = reinterpret_cast<u_char*>(
+          string_piece_to_pool_string(r->pool, name_gs.c_str()));
+    }
+
+    // In case string_piece_to_pool_string failed:
+    if (name.data == NULL || value.data == NULL) {
+        return NGX_ERROR;
     }
-    value.len = value_gs.length();
-    value.data = reinterpret_cast<u_char*>(const_cast<char*>(value_gs.data()));
 
     // TODO(jefftk): If we're setting a cache control header we'd like to
     // prevent any downstream code from changing it.  Specifically, if we're
@@ -535,24 +548,17 @@ ngx_int_t copy_response_headers_to_ngx(
     // net/instaweb/apache/header_util:AddResponseHeadersToRequest
 
     // Make copies of name and value to put into headers_out.
-
-    u_char* value_s = ngx_pstrdup(r->pool, &value);
-    if (value_s == NULL) {
-      return NGX_ERROR;
-    }
-
     if (STR_EQ_LITERAL(name, "Cache-Control")) {
-      ps_set_cache_control(r, const_cast<char*>(value_gs.c_str()));
+      ps_set_cache_control(r, reinterpret_cast<char*>(value.data));
       continue;
     } else if (STR_EQ_LITERAL(name, "Content-Type")) {
       // Unlike all the other headers, content_type is just a string.
-      headers_out->content_type.data = value_s;
-      headers_out->content_type.len = value.len;
+      headers_out->content_type = value;
 
       // We should not include the charset when determining content_type_len, so
       // scan for the ';' that marks the start of the charset part.
       for (ngx_uint_t i = 0; i < value.len; i++) {
-        if (value_s[i] == ';') {
+        if (value.data[i] == ';') {
           break;
         }
         headers_out->content_type_len = i + 1;
@@ -570,11 +576,10 @@ ngx_int_t copy_response_headers_to_ngx(
       continue;
     } else if (STR_EQ_LITERAL(name, "Transfer-Encoding")) {
       continue;
-    }
-
-    u_char* name_s = ngx_pstrdup(r->pool, &name);
-    if (name_s == NULL) {
-      return NGX_ERROR;
+    } else if (STR_EQ_LITERAL(name, "Vary") && value.len
+        && STR_EQ_LITERAL(value, "Accept-Encoding")) {
+      ps_request_ctx_t* ctx = ps_get_request_context(r);
+      ctx->psol_vary_accept_only = true;
     }
 
     ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(
@@ -584,10 +589,10 @@ ngx_int_t copy_response_headers_to_ngx(
     }
 
     header->hash = 1;  // Include this header in the output.
+    header->key.data = name.data;
     header->key.len = name.len;
-    header->key.data = name_s;
+    header->value.data = value.data;
     header->value.len = value.len;
-    header->value.data = value_s;
 
     // Populate the shortcuts to commonly used headers.
     if (STR_EQ_LITERAL(name, "Date")) {
@@ -599,7 +604,10 @@ ngx_int_t copy_response_headers_to_ngx(
     } else if (STR_EQ_LITERAL(name, "Last-Modified")) {
       headers_out->last_modified = header;
     } else if (STR_EQ_LITERAL(name, "Location")) {
-      headers_out->location = header;
+      ps_request_ctx_t* ctx = ps_get_request_context(r);
+      if (ctx->location_field_set) {
+        headers_out->location = header;
+      }
     } else if (STR_EQ_LITERAL(name, "Server")) {
       headers_out->server = header;
     } else if (STR_EQ_LITERAL(name, "Content-Length")) {
@@ -607,6 +615,16 @@ ngx_int_t copy_response_headers_to_ngx(
       CHECK(pagespeed_headers.FindContentLength(&len));
       headers_out->content_length_n = len;
       headers_out->content_length = header;
+    } else if (STR_EQ_LITERAL(name, "Content-Encoding")) {
+      headers_out->content_encoding = header;
+    } else if (STR_EQ_LITERAL(name, "Refresh")) {
+      headers_out->refresh = header;
+    } else if (STR_EQ_LITERAL(name, "Content-Range")) {
+      headers_out->content_range = header;
+    } else if (STR_EQ_LITERAL(name, "Accept-Ranges")) {
+      headers_out->accept_ranges = header;
+    } else if (STR_EQ_LITERAL(name, "WWW-Authenticate")) {
+      headers_out->www_authenticate = header;
     }
   }
 
@@ -623,7 +641,7 @@ typedef struct {
 typedef struct {
   // If pagespeed is configured in some server block but not this one our
   // per-request code will be invoked but server context will be null.  In those
-  // cases we neet to short circuit, not changing anything.  Currently our
+  // cases we need to short circuit, not changing anything.  Currently our
   // header filter, body filter, and content handler all do this, but if anyone
   // adds another way for nginx to give us a request to process we need to check
   // there as well.
@@ -664,6 +682,13 @@ char* ps_main_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 char* ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 
+// We want NGX_CONF_MULTI for some very old versions:
+//   https://github.com/pagespeed/ngx_pagespeed/commit/66f1b9aa
+// but it's gone in recent revisions, so provide a compat #define if needed
+#ifndef NGX_CONF_MULTI
+#define NGX_CONF_MULTI 0
+#endif
+
 // TODO(jud): Verify that all the offsets should be NGX_HTTP_SRV_CONF_OFFSET and
 // not NGX_HTTP_LOC_CONF_OFFSET or NGX_HTTP_MAIN_CONF_OFFSET.
 ngx_command_t ps_commands[] = {
@@ -973,6 +998,7 @@ void* ps_create_main_conf(ngx_conf_t* cf) {
       "" /* hostname, not used */,
       -1 /* port, not used */);
   active_driver_factory = cfg_m->driver_factory;
+  active_driver_factory->LoggingInit(ngx_cycle->log, false);
   cfg_m->driver_factory->Init();
   ps_set_conf_cleanup_handler(cf, ps_cleanup_main_conf, cfg_m);
   return cfg_m;
@@ -1254,6 +1280,8 @@ ngx_int_t ps_decline_request(ngx_http_request_t* r) {
 
   ctx->driver->Cleanup();
   ctx->driver = NULL;
+  ctx->location_field_set = false;
+  ctx->psol_vary_accept_only = false;
 
   // re init ctx
   ctx->html_rewrite = true;
@@ -1650,23 +1678,29 @@ RequestRouting::Response ps_route_request(ngx_http_request_t* r) {
   const NgxRewriteOptions* global_options = cfg_s->server_context->config();
 
   StringPiece path = url.PathSansQuery();
-  if (StringCaseEqual(path, global_options->statistics_path())) {
+  if (StringCaseEqual(path, global_options->statistics_path()) &&
+      global_options->StatisticsAccessAllowed(url)) {
     return RequestRouting::kStatistics;
-  } else if (StringCaseEqual(path, global_options->global_statistics_path())) {
+  } else if (StringCaseEqual(path, global_options->global_statistics_path()) &&
+             global_options->GlobalStatisticsAccessAllowed(url)) {
     return RequestRouting::kGlobalStatistics;
-  } else if (StringCaseEqual(path, global_options->console_path())) {
+  } else if (StringCaseEqual(path, global_options->console_path()) &&
+             global_options->ConsoleAccessAllowed(url)) {
     return RequestRouting::kConsole;
-  } else if (StringCaseEqual(path, global_options->messages_path())) {
+  } else if (StringCaseEqual(path, global_options->messages_path()) &&
+             global_options->MessagesAccessAllowed(url)) {
     return RequestRouting::kMessages;
   } else if (
       // The admin handlers get everything under a path (/path/*) while all the
       // other handlers only get exact matches (/path).  So match all paths
       // starting with the handler path.
       !global_options->admin_path().empty() &&
-      StringCaseStartsWith(path, global_options->admin_path())) {
+      StringCaseStartsWith(path, global_options->admin_path()) &&
+      global_options->AdminAccessAllowed(url)) {
     return RequestRouting::kAdmin;
   } else if (!global_options->global_admin_path().empty() &&
-             StringCaseStartsWith(path, global_options->global_admin_path())) {
+             StringCaseStartsWith(path, global_options->global_admin_path()) &&
+             global_options->GlobalAdminAccessAllowed(url)) {
     return RequestRouting::kGlobalAdmin;
   } else if (global_options->enable_cache_purge() &&
              !global_options->purge_method().empty() &&
@@ -1817,6 +1851,8 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
 
     ctx->recorder = NULL;
     ctx->url_string = url_string;
+    ctx->location_field_set = false;
+    ctx->psol_vary_accept_only = false;
 
     // Set up a cleanup handler on the request.
     ngx_http_cleanup_t* cleanup = ngx_http_cleanup_add(r, 0);
@@ -1882,6 +1918,35 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     }
 
     return ps_async_wait_response(r);
+  } else if (!html_rewrite && response_category == RequestRouting::kResource) {
+    bool is_proxy = false;
+    GoogleString mapped_url;
+    GoogleString host_header;
+
+    if (options->domain_lawyer()->MapOriginUrl(
+            url, &mapped_url, &host_header, &is_proxy) && is_proxy) {
+      ps_create_base_fetch(ctx, request_context, request_headers.release(),
+                          kPageSpeedProxy);
+
+      RewriteDriver* driver;
+      if (custom_options.get() == NULL) {
+        driver = cfg_s->server_context->NewRewriteDriver(
+            ctx->base_fetch->request_context());
+      } else {
+        driver = cfg_s->server_context->NewCustomRewriteDriver(
+            custom_options.release(), ctx->base_fetch->request_context());
+      }
+
+      driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
+      driver->set_pagespeed_query_params(pagespeed_query_params);
+      driver->set_pagespeed_option_cookies(pagespeed_option_cookies);
+      cfg_s->proxy_fetch_factory->StartNewProxyFetch(
+          mapped_url, ctx->base_fetch, driver, NULL /*property_callback*/,
+          NULL /*original_content_fetch*/);
+
+      return ps_async_wait_response(r);
+    }
+
   }
 
   if (html_rewrite) {
@@ -1903,11 +1968,6 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
           custom_options.release(), ctx->base_fetch->request_context());
     }
 
-    StringPiece user_agent = ctx->base_fetch->request_headers()->Lookup1(
-        HttpAttributes::kUserAgent);
-    if (!user_agent.empty()) {
-      driver->SetUserAgent(user_agent);
-    }
     driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
     driver->set_pagespeed_query_params(pagespeed_query_params);
     driver->set_pagespeed_option_cookies(pagespeed_option_cookies);
@@ -1928,6 +1988,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
         url_string, ctx->base_fetch, driver,
         property_callback,
         NULL /* original_content_fetch */);
+    ctx->proxy_fetch->set_trusted_input(true);
     return NGX_OK;
   }
 
@@ -1948,13 +2009,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
           custom_options.release(), ctx->base_fetch->request_context());
     }
 
-    StringPiece user_agent = ctx->base_fetch->request_headers()->Lookup1(
-        HttpAttributes::kUserAgent);
-    if (!user_agent.empty()) {
-      driver->SetUserAgent(user_agent);
-    }
     driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
-
     ctx->driver = driver;
 
     cfg_s->server_context->message_handler()->Message(
@@ -2115,6 +2170,13 @@ ngx_int_t ps_etag_header_filter(ngx_http_request_t* r) {
       break;
     }
   }
+
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+#if (NGX_HTTP_GZIP)
+  if (ctx && ctx->psol_vary_accept_only) {
+    r->gzip_vary = 0;
+  }
+#endif
   return ngx_http_ef_next_header_filter(r);
 }
 
@@ -2198,6 +2260,8 @@ ngx_int_t ps_html_rewrite_header_filter(ngx_http_request_t* r) {
   }
 
   ps_strip_html_headers(r);
+  // See https://github.com/pagespeed/ngx_pagespeed/issues/819
+  ctx->location_field_set = r->headers_out.location != NULL;
 
   // TODO(jefftk): is this thread safe?
   copy_response_headers_from_ngx(r, ctx->base_fetch->response_headers());
@@ -2436,8 +2500,8 @@ ngx_int_t send_out_headers_and_body(
 
   rc = ngx_http_send_header(r);
 
-  if (rc != NGX_OK) {
-    return NGX_ERROR;
+  if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) {
+    return rc;
   }
 
   // Send the body.
@@ -3009,7 +3073,7 @@ ngx_int_t ps_init_module(ngx_cycle_t* cycle) {
       return NGX_ERROR;
     }
 
-    cfg_m->driver_factory->LoggingInit(cycle->log);
+    cfg_m->driver_factory->LoggingInit(cycle->log, true);
     cfg_m->driver_factory->RootInit();
   } else {
     delete cfg_m->driver_factory;
@@ -3023,7 +3087,9 @@ void ps_exit_child_process(ngx_cycle_t* cycle) {
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
   NgxBaseFetch::Terminate();
-  cfg_m->driver_factory->ShutDown();
+  if (cfg_m->driver_factory != NULL) {
+    cfg_m->driver_factory->ShutDown();
+  }
 }
 
 // Called when nginx forks worker processes.  No threads should be started
@@ -3041,7 +3107,7 @@ ngx_int_t ps_init_child_process(ngx_cycle_t* cycle) {
 
   // ChildInit() will initialise all ServerContexts, which we need to
   // create ProxyFetchFactories below
-  cfg_m->driver_factory->LoggingInit(cycle->log);
+  cfg_m->driver_factory->LoggingInit(cycle->log, true);
   cfg_m->driver_factory->ChildInit();
 
   ngx_http_core_main_conf_t* cmcf = static_cast<ngx_http_core_main_conf_t*>(

src/ngx_pagespeed.h

@@ -103,6 +103,12 @@ typedef struct {
   // We need to remember the URL here as well since we may modify what NGX
   // gets by stripping our special query params and honoring X-Forwarded-Proto.
   GoogleString url_string;
+
+  // We need to remember if the upstream had headers_out->location set, because
+  // we should mirror that when we write it back. nginx may absolutify
+  // Location: headers that start with '/' without regarding X-Forwarded-Proto.
+  bool location_field_set;
+  bool psol_vary_accept_only;
 } ps_request_ctx_t;
 
 ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r);

src/ngx_rewrite_driver_factory.cc

@@ -208,10 +208,11 @@ void NgxRewriteDriverFactory::StartThreads() {
   threads_started_ = true;
 }
 
-void NgxRewriteDriverFactory::LoggingInit(ngx_log_t* log) {
+void NgxRewriteDriverFactory::LoggingInit(
+    ngx_log_t* log, bool may_install_crash_handler) {
   log_ = log;
   net_instaweb::log_message_handler::Install(log);
-  if (install_crash_handler()) {
+  if (may_install_crash_handler && install_crash_handler()) {
     NgxMessageHandler::InstallCrashHandler(log);
   }
   ngx_message_handler_->set_log(log);

src/ngx_rewrite_driver_factory.h

@@ -115,7 +115,7 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
     return process_script_variables_;
   }
 
-  void LoggingInit(ngx_log_t* log);
+  void LoggingInit(ngx_log_t* log, bool may_install_crash_handler);
 
   virtual void ShutDownMessageHandlers();
 

src/ngx_rewrite_options.cc

@@ -284,7 +284,8 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
       !StringCaseEqual(directive, "DownstreamCachePurgeLocationPrefix") &&
       !StringCaseEqual(directive, "DownstreamCachePurgeMethod") &&
       !StringCaseEqual(directive,
-                       "DownstreamCacheRewrittenPercentageThreshold")) {
+                       "DownstreamCacheRewrittenPercentageThreshold") &&
+      !StringCaseEqual(directive, "ShardDomain")){
     compile_scripts = false;
   }
 

src/ngx_server_context.h

@@ -40,8 +40,9 @@ class NgxServerContext : public SystemServerContext {
       NgxRewriteDriverFactory* factory, StringPiece hostname, int port);
   virtual ~NgxServerContext();
 
-  // We expect to use ProxyFetch with HTML.
-  virtual bool ProxiesHtml() const { return true; }
+  // We don't allow ProxyFetch to fetch HTML via MapProxyDomain. We will call
+  // set_trusted_input() on any ProxyFetches we use to transform internal HTML.
+  virtual bool ProxiesHtml() const { return false; }
 
   // Call only when you need an NgxRewriteOptions.  If you don't need
   // nginx-specific behavior, call global_options() instead which doesn't

test/nginx_system_test.sh

@@ -40,6 +40,7 @@
 : ${RCPORT7:?"Set RCPORT7"}
 : ${MOD_PAGESPEED_DIR:?"Set MOD_PAGESPEED_DIR"}
 : ${NGINX_EXECUTABLE:?"Set NGINX_EXECUTABLE"}
+: ${PAGESPEED_TEST_HOST:?"Set PAGESPEED_TEST_HOST"}
 POSITION_AUX="${POSITION_AUX:-unset}"
 
 PRIMARY_HOSTNAME="localhost:$PRIMARY_PORT"
@@ -208,6 +209,7 @@ cat $PAGESPEED_CONF_TEMPLATE \
   | sed 's#@@RCPORT5@@#'"$RCPORT5"'#' \
   | sed 's#@@RCPORT6@@#'"$RCPORT6"'#' \
   | sed 's#@@RCPORT7@@#'"$RCPORT7"'#' \
+  | sed 's#@@PAGESPEED_TEST_HOST@@#'"$PAGESPEED_TEST_HOST"'#' \
   >> $PAGESPEED_CONF
 # make sure we substituted all the variables
 check_not_simple grep @@ $PAGESPEED_CONF
@@ -298,11 +300,16 @@ fi
 
 PSA_JS_LIBRARY_URL_PREFIX="pagespeed_custom_static"
 BEACON_HANDLER="ngx_pagespeed_beacon"
+STATISTICS_HANDLER="ngx_pagespeed_statistics"
+GLOBAL_STATISTICS_HANDLER="ngx_pagespeed_global_statistics"
+MESSAGES_HANDLER="ngx_pagespeed_message"
 STATISTICS_URL=http://$PRIMARY_HOSTNAME/ngx_pagespeed_statistics
 
 # An expected failure can be indicated like: "~In-place resource optimization~"
 PAGESPEED_EXPECTED_FAILURES="
 ~Cache purging with PageSpeed off in vhost, but on in directory.~
+~2-pass ipro with long ModPagespeedInPlaceRewriteDeadline~
+~3-pass ipro with short ModPagespeedInPlaceRewriteDeadline~
 "
 
 if [ "$POSITION_AUX" = "true" ] ; then
@@ -317,8 +324,8 @@ fi
 #
 # TODO(sligicki): When the prioritize critical css race condition is fixed, the
 # two prioritize_critical_css tests no longer need to be listed here.
-# TODO(oschaaf): Now that we wait after we send a SIGHUP for the new worker 
-# process to handle requests, check if we can remove more from the expected 
+# TODO(oschaaf): Now that we wait after we send a SIGHUP for the new worker
+# process to handle requests, check if we can remove more from the expected
 # failures here under valgrind.
 if $USE_VALGRIND; then
     PAGESPEED_EXPECTED_FAILURES+="
@@ -509,6 +516,17 @@ check $WGET_DUMP -O $FETCHED $HEADERS $URL
 # When enabled, we respect X-Forwarded-Proto and thus list base as https.
 check fgrep -q '<base href="https://' $FETCHED
 
+start_test Relative redirects starting with a forward slash survive.
+URL=http://xfp.example.com/redirect
+# wget seems a bit hairy here, when it comes to handling (relative) redirects.
+# I could not get this test going with wget, and that is why curl is used here.
+# TODO(oschaaf): debug wget some more and swap out curl here.
+OUT=$(curl -v --proxy $SECONDARY_HOSTNAME $URL 2>&1)
+check_from "$OUT" egrep -q '301 Moved Permanently'
+# The important part is that we don't end up with an absolute location here.
+check_from "$OUT" grep -q 'Location: /mod_pagespeed_example'
+check_not_from "$OUT" grep -q 'Location: http'
+
 # Test that loopback route fetcher works with vhosts not listening on
 # 127.0.0.1
 start_test IP choice for loopback fetches.
@@ -534,7 +552,7 @@ fetch_until "$URL" "fgrep -c .pagespeed." 1 --header=Host:www.google.com
 
 # If this accepts the Host header and fetches from google.com it will fail with
 # a 404.  Instead it should use a loopback fetch and succeed.
-URL="$HOSTNAME/mod_pagespeed_example/.pagespeed.ce.8CfGBvwDhH.css"
+URL="$HOSTNAME/mod_pagespeed_example/styles/big.css.pagespeed.ce.8CfGBvwDhH.css"
 check wget -O /dev/null --header=Host:www.google.com "$URL"
 
 start_test statistics load
@@ -561,7 +579,7 @@ check test $(scrape_stat image_rewrite_total_original_bytes) -ge 10000
 start_test "Reload config"
 
 # Fire up some heavy load if ab is available to test a stressed reload.
-# TODO(oschaaf): make sure we wait for the new worker to get ready to accept 
+# TODO(oschaaf): make sure we wait for the new worker to get ready to accept
 # requests.
 fire_ab_load
 
@@ -1148,6 +1166,19 @@ HEADERS="--header=Host:script-filters.example.com"
 OUT=$($WGET_DUMP -S $HEADERS $URL 2>&1)
 check_not_from "$OUT" fgrep -qi 'addInstrumentationInit'
 
+start_test Test that we can modify domain sharding via script variables.
+URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/rewrite_images.html"
+HEADERS="--header=Host:script-filters.example.com"
+OUT=$($WGET_DUMP -S $HEADERS $URL 2>&1)
+check_from "$OUT" fgrep "http://cdn1.example.com"
+check_from "$OUT" fgrep "http://cdn2.example.com"
+
+URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/rewrite_images.html"
+HEADERS="--header=Host:script-filters.example.com --header=X-Script:1"
+OUT=$($WGET_DUMP -S $HEADERS $URL 2>&1)
+check_not_from "$OUT" fgrep "http://cdn1.example.com"
+check_not_from "$OUT" fgrep "http://cdn2.example.com"
+
 if [ "$NATIVE_FETCHER" != "on" ]; then
   start_test Test that we can rewrite an HTTPS resource.
   fetch_until $TEST_ROOT/https_fetch/https_fetch.html \
@@ -1210,6 +1241,36 @@ check_from "$OUT" fgrep -qi '404'
 MATCHES=$(echo "$OUT" | grep -c "Cache-Control: override") || true
 check [ $MATCHES -eq 1 ]
 
+start_test Custom 404 does not crash.
+URL=http://custom404.example.com/mod_pagespeed_test/
+URL+=A.doesnotexist.css.pagespeed.cf.0.css
+# The 404 response makes wget exit with an error code, which we ignore.
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1) || true
+# We ignored the exit code, check if we got a 404 response.
+check_from "$OUT" fgrep -qi '404'
+
+if false; then
+# This test fails on 1.11, but passes on trunk.  While 2b4c097 fixed most users,
+# it's still possible to get two Vary: Accept-Encoding headers in this test, so
+# disable it for now.
+start_test Single Vary: Accept-Encoding header in IPRO flow
+URL=http://psol-vary.example.com/mod_pagespeed_example/styles/index_style.css
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
+# First hit will be recorded and passed on untouched
+MATCHES=$(echo "$OUT" | grep -c "Vary: Accept-Encoding") || true
+check [ $MATCHES -eq 1 ]
+
+# Fetch until we get a fully optimized response
+http_proxy=$SECONDARY_HOSTNAME \
+  fetch_until $URL "fgrep -c W/\"PSA" 1 --save-headers
+
+# Test the optimized response.
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
+MATCHES=$(echo "$OUT" | grep -c "Vary: Accept-Encoding") || true
+check [ $MATCHES -eq 1 ]
+
+fi
+
 start_test Shutting down.
 
 # Fire up some heavy load if ab is available to test a stressed shutdown
@@ -1279,9 +1340,14 @@ OUT=$(cat "$ERROR_LOG" \
     | grep -v "\\[error\\].*Could not create directories*" \
     | grep -v "\\[error\\].*opening temp file: No such file or directory.*" \
     | grep -v "\\[error\\].*remote\.cfg.*" \
+    | grep -v "\\[error\\].*Slow read operation on file.*" \
+    | grep -v "\\[error\\].*Slow ReadFile operation on file.*" \
+    | grep -v "\\[error\\].*Slow write operation on file.*" \
     | grep -v "\\[warn\\].*remote\.cfg.*" \
     | grep -v "\\[warn\\].*end token not received.*" \
     | grep -v "\\[warn\\].*failed to hook next event.*" \
+    | grep -v "\\[warn\\].*Rewrite.*failed.*.pagespeed....0.foo.*" \
+    | grep -v "\\[warn\\].*A.blue.css.*but cannot access the original.*" \
     || true)
 
 check [ -z "$OUT" ]

test/pagespeed_test.conf.template

@@ -24,6 +24,9 @@ http {
                    '"$http_user_agent"';
   access_log "@@ACCESS_LOG@@" cache;
 
+  # Don't put entries in the error log for 403s and 404s.
+  log_not_found off;
+
   proxy_cache_path "@@PROXY_CACHE@@"  levels=1:2   keys_zone=htmlcache:60m inactive=90m  max_size=50m;
   proxy_temp_path "@@TMP_PROXY_CACHE@@";
 
@@ -283,12 +286,31 @@ http {
 
     pagespeed RunExperiment on;
     pagespeed AnalyticsID "123-45-6734";
+    pagespeed UseAnalyticsJs false;
     pagespeed ExperimentVariable 2;
     pagespeed ExperimentSpec
       "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
     pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
     pagespeed ExperimentSpec "id=3;default;percent=0";
   }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name contentexperiment.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed InPlaceResourceOptimization off;
+
+    pagespeed RunExperiment on;
+    pagespeed AnalyticsID "123-45-6734";
+    pagespeed UseAnalyticsJs false;
+    pagespeed ExperimentVariable 2;
+    pagespeed ExperimentSpec
+      "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50;options=ContentExperimentID=ID:H1BpS2TCRFmJzZgjwyeBHQ,ContentExperimentVariantID=111";
+    pagespeed ExperimentSpec
+      "id=2;enable=recompress_images;percent=50;options=ContentExperimentID=ID:H1BpS2TCRFmJzZgjwyeBHQ,ContentExperimentVariantID=222";
+    pagespeed ExperimentSpec
+      "id=3;default;percent=0;options=ContentExperimentID=123,ContentExperimentVariantID=333";
+  }
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -599,8 +621,23 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name xfp.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-
     pagespeed RespectXForwardedProto on;
+
+    location /redirecting_origin {
+      pagespeed off;
+      # Hack: we clear the response headers using headers_more.
+      # If we don't, nginx will add an extra empty Location: headers here.
+      # It is kind of hard to get nginx to generate a relative location header
+      # that starts with "/".
+      more_clear_headers 'Location';
+      add_header Location /mod_pagespeed_example;
+      return 301;
+    }
+    location /redirect {
+      proxy_method GET;
+      proxy_pass http://127.0.0.1:@@SECONDARY_PORT@@/redirecting_origin;
+      proxy_set_header "Host" "xfp.example.com";
+    }
   }
 
   server {
@@ -666,7 +703,44 @@ http {
     pagespeed EnableFilters rewrite_images,rewrite_css;
     pagespeed EnableFilters convert_to_webp_lossless;
     pagespeed EnableFilters in_place_optimize_for_browser;
+    pagespeed JpegRecompressionQuality 75;
+    pagespeed WebpRecompressionQuality 70;
     pagespeed InPlaceResourceOptimization on;
+    pagespeed AllowVaryOn "Accept";
+    pagespeed FileCachePath "@@IPRO_CACHE@@";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name ipro-for-browser-vary-on-auto.example.com;
+    root "@@SERVER_ROOT@@/mod_pagespeed_example";
+    pagespeed EnableFilters rewrite_images,rewrite_css;
+    pagespeed EnableFilters convert_to_webp_animated;
+    pagespeed EnableFilters convert_to_webp_lossless;
+    pagespeed EnableFilters in_place_optimize_for_browser;
+    pagespeed InPlaceResourceOptimization on;
+    # pagespeed AllowVaryOn "Accept";  # Default is "Auto".
+    pagespeed ImageRecompressionQuality 90;
+    pagespeed JpegRecompressionQuality 75;
+    pagespeed JpegRecompressionQualityForSmallScreens 55;
+    pagespeed JpegQualityForSaveData 35;
+    pagespeed WebpRecompressionQuality 70;
+    pagespeed WebpRecompressionQualityForSmallScreens 50;
+    pagespeed WebpQualityForSaveData 30;
+    pagespeed WebpAnimatedRecompressionQuality 60;
+    pagespeed FileCachePath "@@IPRO_CACHE@@";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name ipro-for-browser-vary-on-none.example.com;
+    root "@@SERVER_ROOT@@/mod_pagespeed_example";
+    pagespeed EnableFilters rewrite_images,in_place_optimize_for_browser;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed AllowVaryOn "None";
+    pagespeed ImageRecompressionQuality 75;
     pagespeed FileCachePath "@@IPRO_CACHE@@";
   }
 
@@ -816,6 +890,13 @@ http {
     pagespeed ConsolePath /custom_pagespeed_console;
     pagespeed MessagesPath /custom_pagespeed_message;
     pagespeed AdminPath /custom_pagespeed_admin;
+
+    pagespeed StatisticsDomains Allow *;
+    pagespeed GlobalStatisticsDomains Allow *;
+    pagespeed MessagesDomains Allow *;
+    pagespeed ConsoleDomains Allow *;
+    pagespeed AdminDomains Allow *;
+    pagespeed GlobalAdminDomains Allow *;
   }
 
   server {
@@ -823,6 +904,13 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name inherit-paths.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed StatisticsDomains Allow *;
+    pagespeed GlobalStatisticsDomains Allow *;
+    pagespeed MessagesDomains Allow *;
+    pagespeed ConsoleDomains Allow *;
+    pagespeed AdminDomains Allow *;
+    pagespeed GlobalAdminDomains Allow *;
   }
 
   server {
@@ -916,6 +1004,14 @@ http {
     }
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name custom404.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    error_page 404 /404.html;
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -1128,6 +1224,31 @@ http {
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
 
+  # Test that pagespeed is disabled when sendfile headers are present.
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name uses-sendfile.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed EnableFilters inline_javascript,rewrite_javascript;
+    add_header 'X-Sendfile' 'blablabla';
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name doesnt-sendfile.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed EnableFilters inline_javascript,rewrite_javascript;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name uses-xaccelredirect.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed EnableFilters inline_javascript,rewrite_javascript;
+    add_header 'X-Accel-Redirect' 'blablabla';
+  }
+
   # Proxy + IPRO a gzip'd file for testing Issue 896.
   server {
     listen @@SECONDARY_PORT@@;
@@ -1227,11 +1348,15 @@ http {
     pagespeed FileCachePath "@@FILE_CACHE@@";
     root "@@SERVER_ROOT@@";
     set $filters "";
+    set $domain_shards "cdn1.example.com,cdn2.example.com";
     if ($http_X_Script) {
         set $filters "add_instrumentation";
+        set $domain_shards "";
     }
     pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_domains;
     pagespeed EnableFilters $filters;
+    pagespeed ShardDomain script-filters.example.com "$domain_shards";
   }
 
   server {
@@ -1311,6 +1436,87 @@ http {
     }
   }
 
+  pagespeed MessagesDomains Allow messages-allowed.example.com;
+  pagespeed MessagesDomains Allow cleared-inherited.example.com;
+  pagespeed MessagesDomains Allow cleared-inherited-reallowed.example.com;
+  pagespeed MessagesDomains Allow more-messages-allowed.example.com;
+  pagespeed MessagesDomains Allow anything-*-wildcard.example.com;
+  pagespeed MessagesDomains Allow localhost;
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name messages-allowed.example.com
+                messages-not-allowed.example.com
+                more-messages-allowed.example.com
+                anything-a-wildcard.example.com
+                anything-b-wildcard.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name messages-still-not-allowed.example.com
+                but-this-message-allowed.example.com
+                and-this-one.example.com;
+    pagespeed MessagesDomains Allow but-this-message-allowed.example.com;
+    pagespeed MessagesDomains Allow and-this-one.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name cleared-inherited.example.com
+                cleared-inherited-reallowed.example.com
+                messages-allowed-at-vhost.example.com
+                messages-not-allowed-at-vhost.example.com
+                anything-c-wildcard.example.com;
+    pagespeed MessagesDomains Disallow *;
+    pagespeed MessagesDomains Allow cleared-inherited-reallowed.example.com;
+    pagespeed MessagesDomains Allow messages-allowed-at-vhost.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name cleared-inherited-unlisted.example.com;
+    pagespeed MessagesDomains Allow *;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+  server {
+    server_name nothing-allowed.example.com;
+    pagespeed MessagesDomains Disallow *;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+  server {
+    server_name nothing-explicitly-allowed.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name everything-explicitly-allowed.example.com
+                everything-explicitly-allowed-but-aliased.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed StatisticsDomains Allow everything-explicitly-allowed.example.com;
+    pagespeed GlobalStatisticsDomains
+      Allow everything-explicitly-allowed.example.com;
+    pagespeed MessagesDomains Allow everything-explicitly-allowed.example.com;
+    pagespeed ConsoleDomains Allow everything-explicitly-allowed.example.com;
+    pagespeed AdminDomains Allow everything-explicitly-allowed.example.com;
+    pagespeed GlobalAdminDomains
+     Allow everything-explicitly-allowed.example.com;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name psol-vary.example.com;
+    pagespeed on;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+
   server {
     listen @@PRIMARY_PORT@@;
     listen [::]:@@PRIMARY_PORT@@;
@@ -1376,6 +1582,14 @@ http {
       http://www.modpagespeed.com/rewrite_javascript.js;
     pagespeed RetainComment " google_ad_section*";
 
+    # Test proxying of non-.pagespeed. resources.
+    pagespeed MapProxyDomain http://localhost:@@PRIMARY_PORT@@/modpagespeed_http
+                           http://@@PAGESPEED_TEST_HOST@@/do_not_modify;
+    pagespeed MapProxyDomain http://localhost:@@PRIMARY_PORT@@/content_type_present
+                           http://@@PAGESPEED_TEST_HOST@@:8091;
+    pagespeed MapProxyDomain http://localhost:@@PRIMARY_PORT@@/content_type_absent
+                           http://@@PAGESPEED_TEST_HOST@@:8092;
+
     add_header X-Extra-Header 1;
 
     # Establish a proxy mapping where the current server proxies an image
@@ -1494,6 +1708,30 @@ http {
       expires 5m;
     }
 
+    location /mod_pagespeed_test/ipro/wait/ {
+      # TODO(jmarantz): ModPagespeedInPlaceWaitForOptimized should be superfluous,
+      # or made equivalent to ModPagespeedInPlaceRewriteDeadlineMs -1, which waits
+      # forever.  Otherwise ModPagespeedInPlaceRewriteDeadlineMs should just have
+      # the specified deadline.
+      # # See https://github.com/pagespeed/mod_pagespeed/issues/1171 for more
+      # detailed discussion.
+      pagespeed InPlaceWaitForOptimized on;
+    }
+
+    location /mod_pagespeed_test/ipro/wait/long/ {
+      # Make the deadline long here for valgrind tests.  We could
+      # conditionalize this.
+      pagespeed InPlaceRewriteDeadlineMs 10000;
+    }
+
+    location /mod_pagespeed_test/ipro/wait/short/ {
+      pagespeed EnableFilters in_place_optimize_for_browser;
+
+      # Make the deadline short here as we expect to always miss it
+      # in tests.
+      pagespeed InPlaceRewriteDeadlineMs 1;
+    }
+
     location /mod_pagespeed_test/ipro/instant/wait/ {
       pagespeed InPlaceWaitForOptimized on;
 
@@ -1563,6 +1801,25 @@ http {
           https://www.gstatic.com/psa/static;
     }
 
+    location /mod_pagespeed_test/strip_subresource_hints/default/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed RewriteLevel CoreFilters;
+      pagespeed DisAllow *dontrewriteme*;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/default_passthrough/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed RewriteLevel PassThrough;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/preserve_on/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed PreserveSubresourceHints on;
+      pagespeed RewriteLevel CoreFilters;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/preserve_off/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed PreserveSubresourceHints off;
+      pagespeed RewriteLevel CoreFilters;
+    }
     # $host implicitly tests script variable support. I'd love to test it more
     # directly, but so far this is the best I've come up with and duplicating
     # the test doesn't seem to make sense.

test/run_tests.sh

@@ -23,9 +23,11 @@
 # Exits with status 2 if command line args are wrong.
 #
 # Usage:
-#   ./run_tests.sh primary_port secondary_port mod_pagespeed_dir
+#   ./run_tests.sh primary_port secondary_port mod_pagespeed_dir ngx_binary
+#   pagespeed_test_host
 # Example:
-#   ./run_tests.sh 8050 8051 /path/to/mod_pagespeed
+#   ./run_tests.sh 8050 8051 /path/to/mod_pagespeed /path/to/nginx/binary
+#   selfsigned.modpagespeed.com
 #
 
 # Normally we test only with the native fetcher off.  Set
@@ -43,7 +45,7 @@ RUN_TESTS=${RUN_TESTS:-true}
 # true.
 USE_VALGRIND=${USE_VALGRIND:-false}
 
-if [ "$#" -ne 4 ] ; then
+if [ "$#" -ne 5 ] ; then
   echo "Usage: $0 primary_port secondary_port mod_pagespeed_dir"
   echo "  nginx_executable"
   exit 2
@@ -53,6 +55,7 @@ PRIMARY_PORT="$1"
 SECONDARY_PORT="$2"
 MOD_PAGESPEED_DIR="$3"
 NGINX_EXECUTABLE="$4"
+PAGESPEED_TEST_HOST="$5"
 RCPORT1=9991
 RCPORT2=9992
 RCPORT3=9993
@@ -69,6 +72,7 @@ function run_test_checking_failure() {
     SECONDARY_PORT="$SECONDARY_PORT" \
     MOD_PAGESPEED_DIR="$MOD_PAGESPEED_DIR" \
     NGINX_EXECUTABLE="$NGINX_EXECUTABLE" \
+    PAGESPEED_TEST_HOST="$PAGESPEED_TEST_HOST" \
     RUN_TESTS="$RUN_TESTS" \
     RCPORT1="$RCPORT1" \
     RCPORT2="$RCPORT2" \