native fetcher ipv6 support - work in progress

This allows is to test again, but needs a lot of cleaning up.
More importantly, the google font inlining test fails, and I'm
not sure why that is, the fetch succeeds.

.gitignore

@@ -2,3 +2,4 @@ test/tmp
 psol/
 psol-*.tar.gz
 *.*.*.*.tar.gz
+nginx

.gitmodules

@@ -0,0 +1,18 @@
+[submodule "testing-dependencies/mod_pagespeed"]
+	path = testing-dependencies/mod_pagespeed
+	url = https://github.com/pagespeed/mod_pagespeed.git
+[submodule "testing-dependencies/ngx_cache_purge"]
+	path = testing-dependencies/ngx_cache_purge
+	url = https://github.com/FRiCKLE/ngx_cache_purge.git
+[submodule "testing-dependencies/nginx"]
+	path = testing-dependencies/nginx
+	url = https://github.com/nginx/nginx.git
+[submodule "testing-dependencies/set-misc-nginx-module"]
+	path = testing-dependencies/set-misc-nginx-module
+	url = https://github.com/openresty/set-misc-nginx-module
+[submodule "testing-dependencies/ngx_devel_kit"]
+	path = testing-dependencies/ngx_devel_kit
+	url = https://github.com/simpl/ngx_devel_kit
+[submodule "testing-dependencies/headers-more-nginx-module"]
+	path = testing-dependencies/headers-more-nginx-module
+	url = https://github.com/openresty/headers-more-nginx-module

.travis.yml

@@ -0,0 +1,33 @@
+language: c++
+env:
+  global:
+    - MAKEFLAGS=-j3
+# By default travis loads submodules serially, but we can load them in parallel
+# if we install an updated git and use --jobs.  Some timing numbers:
+#   serial: 257s
+#   jobs=4: 182s (29s to install new git, 153s to run the downloads)
+#   jobs=8: 179s (29s to install new git, 150s to run the downloads)
+# We can't use --depth=1, though, because github doesn't have
+# allowReachableSHA1InWant set.
+#
+# TODO(jefftk): once we're running on a server with git 2.8 or later we can have
+#               --jobs without installing a new git.
+git:
+  submodules: false
+before_install:
+  - sudo add-apt-repository --yes ppa:git-core/ppa
+  - sudo apt-get update
+  - sudo apt-get install git
+  - git submodule update --init --recursive --jobs=8
+install:
+  scripts/build_ngx_pagespeed.sh --devel --assume-yes
+script:
+  test/run_tests.sh $PWD/testing-dependencies/mod_pagespeed/ \
+                    $PWD/nginx/sbin/nginx
+dist: trusty
+sudo: required
+compiler:
+  - gcc
+notifications:
+  email:
+    - pagespeed-ci@googlegroups.com

PSOL_BINARY_URL

@@ -0,0 +1,4 @@
+In a release this file would contain the URL to download the pre-compiled PSOL
+binary, but on development branches (like this one) you have to build PSOL from
+source yourself.  See:
+  https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source

README.md

@@ -1,5 +1,7 @@
 ![ngx_pagespeed](https://lh6.googleusercontent.com/-qufedJIJq7Y/UXEvVYxyYvI/AAAAAAAADo8/JHDFQhs91_c/s401/04_ngx_pagespeed.png)
 
+[![Build Status](https://travis-ci.org/pagespeed/ngx_pagespeed.svg?branch=master)](https://travis-ci.org/pagespeed/ngx_pagespeed)
+
 ngx_pagespeed speeds up your site and reduces page load time by automatically
 applying web performance best practices to pages and associated assets (CSS,
 JavaScript, images) without requiring you to modify your existing content or

config

@@ -25,18 +25,28 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
 
   if [ ! -e "$mod_pagespeed_dir" ] ; then
     echo "ngx_pagespeed: pagespeed optimization library not found:"
-    echo ""
+
-    echo "   You need to separately download the pagespeed library:"
+    psol_binary_url="$($ngx_addon_dir/scripts/format_binary_url.sh \
-    echo ""
+                           $ngx_addon_dir/PSOL_BINARY_URL)"
-    echo "     $ cd /path/to/ngx_pagespeed"
+    if [[ "$psol_binary_url" != https://* ]]; then
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.9.32.1.tar.gz"
+      echo "
-    echo "     $ tar -xzvf 1.9.32.1.tar.gz # expands to psol/"
+  This is a development branch of ngx_pagespeed, which means there is no
-    echo ""
+  precompiled PSOL library available to link against.  Either build from a
-    echo "   Or see the installation instructions:"
+  release tag, like latest-beta, or build PSOL from source:
-    echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
+    https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source"
+      exit 1
+    fi
+
+    echo "
+  You need to separately download the pagespeed library:
+     $ cd $ngx_addon_dir
+     $ wget $psol_binary_url
+     $ tar -xzvf $(basename $psol_binary_url) # expands to psol/
+
+  Or see the installation instructions:
+     https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source"
     exit 1
   fi
-
 else
   build_from_source=true
 fi
@@ -67,6 +77,8 @@ fi
 
 if [ "$NGX_DEBUG" = "YES" ]; then
   buildtype=Debug
+  # If we're using a psol tarball that doesn't contain Debug/ (which is the case
+  # from 1.12 onward) then this will be overriden to buildtype=Release below.
 else
   buildtype=Release
 fi
@@ -80,10 +92,8 @@ ps_maybe_gpp_base=`basename $CC| sed s/gcc/g++/`
 ps_maybe_gpp="`dirname $CC`/$ps_maybe_gpp_base"
 if [ -n "$NGX_GCC_VER" -a \( -x "$ps_maybe_gpp" \) ]; then
   LINK=$ps_maybe_gpp
-  NGX_TEST_LD_OPT="$NGX_TEST_LD_OPT -lstdc++"
-else
-  pagespeed_libs="-lstdc++"
 fi
+pagespeed_libs="-lstdc++"
 
 # The compiler needs to know that __sync_add_and_fetch_4 is ok,
 # and this requires an instruction that didn't exist on i586 or i386.
@@ -98,7 +108,8 @@ CFLAGS="$CFLAGS $FLAG_MARCH"
 # to our libpsol.a
 # See https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html
 CFLAGS="$CFLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
-CC_TEST_FLAGS="$CC_TEST_FLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
+CC_OLD_TEST_FLAGS="$CC_TEST_FLAGS"
+CC_TEST_FLAGS="$CC_TEST_FLAGS -D_GLIBCXX_USE_CXX11_ABI=0 --std=c++11"
 
 case "$NGX_GCC_VER" in
   4.8*)
@@ -118,6 +129,13 @@ case "$NGX_GCC_VER" in
   ;;
 esac
 
+# workaround for a bug in nginx-1.9.11, see:
+# http://hg.nginx.org/nginx/rev/ff1e625ae55b
+NGX_VERSION=`grep nginx_version src/core/nginx.h | sed -e 's/^.* \(.*\)$/\1/'`
+if [ "$NGX_VERSION" = "1009011" ]; then
+    CFLAGS="$CFLAGS -Wno-write-strings"
+fi
+
 if [ "$WNO_ERROR" = "YES" ]; then
     CFLAGS="$CFLAGS -Wno-error"
 fi
@@ -128,6 +146,22 @@ if [ "$psol_binary" = "unset" ] ; then
     psol_binary="\
         $mod_pagespeed_dir/pagespeed/automatic/pagespeed_automatic.a"
   else
+    if ! [ -d "$ngx_addon_dir/psol/lib/$buildtype" ]; then
+      echo "
+You have set --with-debug for building nginx, but precompiled Debug binaries for
+PSOL, which ngx_pagespeed depends on, aren't available.  If you're trying to
+debug PSOL you need to build it from source.  If you just want to run nginx with
+debug-level logging you can use the Release binaries."
+      echo -n "
+Use the available Release binaries?"
+      read -p " [Y/n] " yn
+      if [[ "$yn" == N* || "$yn" == n* ]]; then
+        echo "Cancelled."
+        exit 1
+      fi
+
+      buildtype=Release
+    fi
     psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
     psol_binary="$psol_library_dir/pagespeed_automatic.a"
   fi
@@ -150,19 +184,21 @@ ngx_feature_incs="
 pagespeed_include="\
   $mod_pagespeed_dir \
   $mod_pagespeed_dir/third_party/chromium/src \
-  $mod_pagespeed_dir/third_party/google-sparsehash/src \
+  $mod_pagespeed_dir/third_party/google-sparsehash/src/src \
   $mod_pagespeed_dir/third_party/google-sparsehash/gen/arch/$os_name/$arch_name/include \
-  $mod_pagespeed_dir/third_party/protobuf/src \
+  $mod_pagespeed_dir/third_party/grpc/src/include \
+  $mod_pagespeed_dir/third_party/protobuf/src/src \
   $mod_pagespeed_dir/third_party/re2/src \
   $mod_pagespeed_dir/out/$buildtype/obj/gen \
   $mod_pagespeed_dir/out/$buildtype/obj/gen/protoc_out/instaweb \
   $mod_pagespeed_dir/third_party/apr/src/include \
   $mod_pagespeed_dir/third_party/aprutil/src/include \
   $mod_pagespeed_dir/third_party/apr/gen/arch/$os_name/$arch_name/include \
-  $mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include"
+  $mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include \
+  $mod_pagespeed_dir/url"
 ngx_feature_path="$pagespeed_include"
 
-pagespeed_libs="$pagespeed_libs $psol_binary -lrt -pthread -lm"
+pagespeed_libs="$psol_binary $pagespeed_libs -lrt -pthread -lm -luuid"
 ngx_feature_libs="$pagespeed_libs"
 ngx_feature_test="
 
@@ -186,72 +222,121 @@ ngx_feature_test="
 # Test whether we have pagespeed and can compile and link against it.
 . "$ngx_addon_dir/cpp_feature"
 
-if [ $ngx_found = yes ]; then
+if [ $ngx_found = no ]; then
-  ps_src="$ngx_addon_dir/src"
+  cat << END
-  ngx_addon_name=ngx_pagespeed
+$0: error: module ngx_pagespeed requires the pagespeed optimization library.
-  NGX_ADDON_DEPS="$NGX_ADDON_DEPS \
+Look in $PWD/$NGX_AUTOCONF_ERR for more details.
-    $ps_src/log_message_handler.h \
+END
-    $ps_src/ngx_base_fetch.h \
+  exit 1
-    $ps_src/ngx_caching_headers.h \
+fi
-    $ps_src/ngx_event_connection.h \
-    $ps_src/ngx_fetch.h \
-    $ps_src/ngx_gzip_setter.h \
-    $ps_src/ngx_list_iterator.h \
-    $ps_src/ngx_message_handler.h \
-    $ps_src/ngx_pagespeed.h \
-    $ps_src/ngx_rewrite_driver_factory.h \
-    $ps_src/ngx_rewrite_options.h \
-    $ps_src/ngx_server_context.h \
-    $ps_src/ngx_url_async_fetcher.h \
-    $psol_binary"
-  NGX_ADDON_SRCS="$NGX_ADDON_SRCS \
-    $ps_src/log_message_handler.cc \
-    $ps_src/ngx_base_fetch.cc \
-    $ps_src/ngx_caching_headers.cc \
-    $ps_src/ngx_event_connection.cc \
-    $ps_src/ngx_fetch.cc \
-    $ps_src/ngx_gzip_setter.cc \
-    $ps_src/ngx_list_iterator.cc \
-    $ps_src/ngx_message_handler.cc \
-    $ps_src/ngx_pagespeed.cc \
-    $ps_src/ngx_rewrite_driver_factory.cc \
-    $ps_src/ngx_rewrite_options.cc \
-    $ps_src/ngx_server_context.cc \
-    $ps_src/ngx_url_async_fetcher.cc"
-  # Save our sources in a separate var since we may need it in config.make
-  PS_NGX_SRCS="$NGX_ADDON_SRCS"
 
-  if [ "$position_aux" = "true" ] ; then
+ps_src="$ngx_addon_dir/src"
-     HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
+ngx_addon_name=ngx_pagespeed
-  else
+NGX_ADDON_DEPS="$NGX_ADDON_DEPS \
-    # Make pagespeed run immediately before gzip and Brotli.
+$ps_src/log_message_handler.h \
-    if echo $HTTP_FILTER_MODULES | grep ngx_http_brotli_filter_module >/dev/null; then
+$ps_src/ngx_base_fetch.h \
-      module=ngx_http_brotli_filter_module
+$ps_src/ngx_caching_headers.h \
-    elif [ $HTTP_GZIP = YES ]; then
+$ps_src/ngx_event_connection.h \
-      module=$HTTP_GZIP_FILTER_MODULE
+$ps_src/ngx_fetch.h \
+$ps_src/ngx_gzip_setter.h \
+$ps_src/ngx_list_iterator.h \
+$ps_src/ngx_message_handler.h \
+$ps_src/ngx_pagespeed.h \
+$ps_src/ngx_rewrite_driver_factory.h \
+$ps_src/ngx_rewrite_options.h \
+$ps_src/ngx_server_context.h \
+$ps_src/ngx_url_async_fetcher.h \
+$psol_binary"
+NPS_SRCS=" \
+$ps_src/log_message_handler.cc \
+$ps_src/ngx_base_fetch.cc \
+$ps_src/ngx_caching_headers.cc \
+$ps_src/ngx_event_connection.cc \
+$ps_src/ngx_fetch.cc \
+$ps_src/ngx_gzip_setter.cc \
+$ps_src/ngx_list_iterator.cc \
+$ps_src/ngx_message_handler.cc \
+$ps_src/ngx_pagespeed.cc \
+$ps_src/ngx_rewrite_driver_factory.cc \
+$ps_src/ngx_rewrite_options.cc \
+$ps_src/ngx_server_context.cc \
+$ps_src/ngx_url_async_fetcher.cc"
+# Save our sources in a separate var since we may need it in config.make
+PS_NGX_SRCS="$NGX_ADDON_SRCS \
+$NPS_SRCS"
+
+# Make pagespeed run immediately before gzip and Brotli.
+if echo $HTTP_FILTER_MODULES | grep ngx_http_brotli_filter_module >/dev/null; then
+  next=ngx_http_brotli_filter_module
+elif [ $HTTP_GZIP = YES ]; then
+  next=ngx_http_gzip_filter_module
+else
+  next=ngx_http_range_header_filter_module
+fi
+
+if [ -n "$ngx_module_link" ]; then
+  # nginx-1.9.11+
+  ngx_module_type=HTTP_FILTER
+  ngx_module_name="ngx_pagespeed ngx_pagespeed_etag_filter"
+  ngx_module_incs="$ngx_feature_path"
+  ngx_module_deps=
+  ngx_module_srcs="$NPS_SRCS"
+  ngx_module_libs="$ngx_feature_libs"
+  ngx_module_order="ngx_http_range_header_filter_module\
+                    ngx_pagespeed_etag_filter\
+                    ngx_http_gzip_filter_module \
+                    ngx_http_brotli_filter_module \
+                    ngx_pagespeed \
+                    ngx_http_postpone_filter_module \
+                    ngx_http_ssi_filter_module \
+                    ngx_http_charset_filter_module \
+                    ngx_http_xslt_filter_module \
+                    ngx_http_image_filter_module \
+                    ngx_http_sub_filter_module \
+                    ngx_http_addition_filter_module \
+                    ngx_http_gunzip_filter_module \
+                    ngx_http_userid_filter_module \
+                    ngx_http_headers_filter_module"
+
+    . auto/module
+
+    if [ $ngx_module_link != DYNAMIC ]; then
+      # ngx_module_order doesn't work with static modules,
+      # so we must re-order filters here.
+      if [ "$position_aux" = "true" ] ; then
+        HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
+      else
+        HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES \
+                             | sed "s/ngx_pagespeed//" \
+                             | sed "s/$next/$next ngx_pagespeed/")
+      fi
+      # Make the etag header filter run immediately before range header filter.
+      HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES \
+                           | sed "s/ngx_pagespeed_etag_filter//" \
+                           | sed "s/ngx_http_range_header_filter_module/ngx_http_range_header_filter_module ngx_pagespeed_etag_filter/")
     else
-      module=$HTTP_RANGE_HEADER_FILTER_MODULE
+      if [ "$position_aux" = "true" ] ; then
+        ngx_module_type=HTTP_AUX_FILTER
+        ngx_module_order=""
+      fi
     fi
-
+else
-    HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
+  CORE_LIBS="$CORE_LIBS $pagespeed_libs"
-      sed "s/$module/$module $ngx_addon_name/")
+  CORE_INCS="$CORE_INCS $pagespeed_include"
+  NGX_ADDON_SRCS="$PS_NGX_SRCS"
+  if [ "$position_aux" = "true" ] ; then
+    HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
+  else
+    HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES | sed "s/$next/$next $ngx_addon_name/")
   fi
 
   # Make the etag header filter run immediately before range header filter.
   HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
-    sed "s/$HTTP_RANGE_HEADER_FILTER_MODULE/$HTTP_RANGE_HEADER_FILTER_MODULE ngx_pagespeed_etag_filter/")
+    sed "s/ngx_http_range_header_filter_module/ngx_http_range_header_filter_module ngx_pagespeed_etag_filter/")
-
-  CORE_LIBS="$CORE_LIBS $pagespeed_libs"
-  CORE_INCS="$CORE_INCS $pagespeed_include"
-  echo "List of modules (in reverse order of applicability): "$HTTP_FILTER_MODULES
-else
-  cat << END
-$0: error: module ngx_pagespeed requires the pagespeed optimization library.
-Look in obj/autoconf.err for more details.
-END
-  exit 1
 fi
 
+echo "List of modules (in reverse order of applicability): "$HTTP_FILTER_MODULES
+
 # Test whether the compiler is compatible
 ngx_feature="psol-compiler-compat"
 ngx_feature_name=""
@@ -262,7 +347,7 @@ ngx_feature_libs="-lstdc++"
 ngx_feature_test="
 
 #if defined(__clang__) && defined(__GLIBCXX__)
-   // See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning 
+   // See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning
    // for a list of various values of __GLIBCXX__. Note that they're not monotonic
    // with respect to version numbers.
    #if __GLIBCXX__ == 20120322  || __GLIBCXX__ == 20120614
@@ -289,9 +374,11 @@ if [ $ngx_found = no ]; then
   cat << END
 $0: error: module ngx_pagespeed requires gcc >= 4.8 or clang >= 3.3.
 See https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source for some recommendations.
-Look in objs/autoconf.err for more details.
+Look in $PWD/$NGX_AUTOCONF_ERR for more details.
 END
   exit 1
 fi
 
 have=NGX_PAGESPEED . auto/have
+
+CC_TEST_FLAGS="$CC_OLD_TEST_FLAGS"

config.make

@@ -1,7 +1,7 @@
-if [ -n "$NGX_CLANG_VER" ]; then
+# Since nginx build system doesn't normally do C++, there is no CXXFLAGS for us
-  # Chromium headers assume clang is always in C++11 mode. Oblige it.
+# to touch, and compilers are understandably unhappy with --std=c++11 on C
-  for ps_src_file in $PS_NGX_SRCS; do
+# files. Hence, we hack the makefile to add it for just our sources.
-    ps_obj_file="$NGX_OBJS/addon/src/`basename $ps_src_file .cc`.o"
+for ps_src_file in $PS_NGX_SRCS; do
-    echo "$ps_obj_file : CFLAGS += --std=c++11" >> $NGX_MAKEFILE
+  ps_obj_file="$NGX_OBJS/addon/src/`basename $ps_src_file .cc`.o"
-  done
+  echo "$ps_obj_file : CFLAGS += --std=c++11" >> $NGX_MAKEFILE
-fi
+done

scripts/build_ngx_pagespeed.sh

@@ -0,0 +1,808 @@
+#!/bin/bash
+#
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function usage() {
+  echo "
+Usage: build_ngx_pagespeed.sh [options]
+
+  Installs ngx_pagespeed and its dependencies.  Can optionally build and install
+  nginx as well.  Can be run either as:
+
+     bash <(curl -f -L -sS https://ngxpagespeed.com/install) [options]
+
+  Or:
+
+     git clone git@github.com:pagespeed/ngx_pagespeed.git
+     cd ngx_pagespeed/
+     git checkout <branch>
+     scripts/build_ngx_pagespeed.sh [options]
+
+Options:
+  -v, --ngx-pagespeed-version <ngx_pagespeed version>
+      What version of ngx_pagespeed to build.  Valid options include:
+      * latest-beta
+      * latest-stable
+      * a version number, such as 1.11.33.4
+
+      If you don't specify a version, defaults to latest-stable unless --devel
+      is specified, in which case it defaults to master.
+
+      This option doesn't make sense if we're running within an existing
+      ngx_pagespeed checkout.
+
+  -n, --nginx-version <nginx version>
+      What version of nginx to build.  If not set, this script only prepares the
+      ngx_pagespeed module, and expects you to handle including it when you
+      build nginx.
+
+      If you pass in 'latest' then this script scrapes the nginx download page
+      and attempts to determine the latest version automatically.
+
+  -m, --dynamic-module
+      Build ngx_pagespeed as a dynamic module.
+
+  -b, --builddir <directory>
+      Where to build.  Defaults to \$HOME.
+
+  -p, --no-deps-check
+      By default, this script checks for the packages it depends on and tries to
+      install them.  If you have installed dependencies from source or are on a
+      non-deb non-rpm system, this won't work.  In that case, install the
+      dependencies yourself and pass --no-deps-check.
+
+  -s, --psol-from-source
+      Build PSOL from source instead of downloading a pre-built binary module.
+
+  -l, --devel
+      Sets up a development environment in ngx_pagespeed/nginx, building with
+      testing-only dependencies.  Includes --psol-from-source, conflicts with
+      --nginx-version.  Uses a 'git clone' checkout for ngx_pagespeed and nginx
+      instead of downloading a tarball.
+
+  -t, --build-type
+      When building PSOL from source, what to tell it for BUILD_TYPE.  Defaults
+      to 'Release' unless --devel is set in which case it defaults to 'Debug'.
+
+  -y, --assume-yes
+      Assume the answer to all prompts is 'yes, please continue'.  Intended for
+      automated usage, such as buildbots.
+
+  -a, --additional-nginx-configure-arguments
+      When running ./configure for nginx, you may want to specify additional
+      arguments, such as --with-http_ssl_module.  By default this script will
+      pause and prompt you for them, but this option lets you pass them in.  For
+      example, you might do:
+        -a '--with-http_ssl_module --with-cc-opt=\"-I /usr/local/include\"'
+
+  -d, --dryrun
+      Don't make any changes to the system, just print what changes you
+      would have made.
+
+  -h, --help
+      Print this message and exit."
+}
+
+RED=31
+GREEN=32
+YELLOW=33
+function begin_color() {
+  color="$1"
+  echo -e -n "\e[${color}m"
+}
+function end_color() {
+  echo -e -n "\e[0m"
+}
+
+function echo_color() {
+  color="$1"
+  shift
+  begin_color "$color"
+  echo "$@"
+  end_color
+}
+
+function error() {
+  local error_message="$@"
+  echo_color "$RED" -n "Error: " >&2
+  echo "$@" >&2
+}
+
+# Prints an error message and exits with an error code.
+function fail() {
+  error "$@"
+
+  # Normally I'd use $0 in "usage" here, but since most people will be running
+  # this via curl, that wouldn't actually give something useful.
+  echo >&2
+  echo "For usage information, run this script with --help" >&2
+  exit 1
+}
+
+
+function status() {
+  echo_color "$GREEN" "$@"
+}
+
+# Intended to be called as:
+#   bash <(curl dl.google.com/.../build_ngx_pagespeed.sh) <args>
+
+# If we set -e or -u then users of this script will see it silently exit on
+# failure.  Instead we need to check the exit status of each command manually.
+# The run function handles exit-status checking for system-changing commands.
+# Additionally, this allows us to easily have a dryrun mode where we don't
+# actually make any changes.
+INITIAL_ENV=$(printenv | sort)
+function run() {
+  if "$DRYRUN"; then
+    echo_color "$YELLOW" -n "would run"
+    echo " $@"
+    env_differences=$(comm -13 <(echo "$INITIAL_ENV") <(printenv | sort))
+    if [ -n "$env_differences" ]; then
+      echo "  with the following additional environment variables:"
+      echo "$env_differences" | sed 's/^/    /'
+    fi
+  else
+    if ! "$@"; then
+      error "Failure running '$@', exiting."
+      exit 1
+    fi
+  fi
+}
+
+function redhat_is_installed() {
+  local package_name="$1"
+  rpm -qa $package_name | grep -q .
+}
+
+function debian_is_installed() {
+  local package_name="$1"
+  dpkg -l $package_name | grep ^ii | grep -q .
+}
+
+function version_sort() {
+  # We'd rather use sort -V, but that's not available on Centos 5.  This works
+  # for versions in the form A.B.C.D or shorter, which is enough for our use.
+  sort -t '.' -k 1,1 -k 2,2 -k 3,3 -k 4,4 -g
+}
+
+# Compare two numeric versions in the form "A.B.C".  Works with version numbers
+# having up to four components, since that's enough to handle both nginx (3) and
+# ngx_pagespeed (4).
+function version_older_than() {
+  local test_version="$1"
+  local compare_to="$2"
+
+  local older_version=$(echo $@ | tr ' ' '\n' | version_sort | head -n 1)
+  test "$older_version" != "$compare_to"
+}
+
+function determine_latest_nginx_version() {
+  # Scrape nginx's download page to try to find the most recent nginx version.
+
+  nginx_download_url="https://nginx.org/en/download.html"
+  function report_error() {
+    fail "
+Couldn't automatically determine the latest nginx version: failed to $@
+$nginx_download_url"
+  }
+
+  nginx_download_page=$(curl -sS --fail "$nginx_download_url") || \
+    report_error "download"
+
+  download_refs=$(echo "$nginx_download_page" | \
+    grep -o '/download/nginx-[0-9.]*[.]tar[.]gz') || \
+    report_error "parse"
+
+  versions_available=$(echo "$download_refs" | \
+    sed -e 's~^/download/nginx-~~' -e 's~\.tar\.gz$~~') || \
+    report_error "extract versions from"
+
+  latest_version=$(echo "$versions_available" | version_sort | tail -n 1) || \
+    report_error "determine latest version from"
+
+  if version_older_than "$latest_version" "1.11.4"; then
+    fail "
+Expected the latest version of nginx to be at least 1.11.4 but found
+$latest_version on $nginx_download_url"
+  fi
+
+  echo "$latest_version"
+}
+
+# Usage:
+#  install_dependencies install_pkg_cmd is_pkg_installed_cmd dep1 dep2 ...
+#
+# install_pkg_cmd is a command to install a dependency
+# is_pkg_installed_cmd is a command that returns true if the dependency is
+#   already installed
+# each dependency is a package name
+function install_dependencies() {
+  local install_pkg_cmd="$1"
+  local is_pkg_installed_cmd="$2"
+  shift 2
+
+  local missing_dependencies=""
+
+  for package_name in "$@"; do
+    if ! $is_pkg_installed_cmd $package_name; then
+      missing_dependencies+="$package_name "
+    fi
+  done
+  if [ -n "$missing_dependencies" ]; then
+    status "Detected that we're missing the following depencencies:"
+    echo "  $missing_dependencies"
+    status "Installing them:"
+    run sudo $install_pkg_cmd $missing_dependencies
+  fi
+}
+
+function gcc_too_old() {
+  # We need gcc >= 4.8
+  local gcc_major_version=$(gcc -dumpversion | awk -F. '{print $1}')
+  if [ "$gcc_major_version" -lt 4 ]; then
+    return 0  # too old
+  elif [ "$gcc_major_version" -gt 4 ]; then
+    return 1  # plenty new
+  fi
+  # It's gcc 4.x, check if x >= 8:
+  local gcc_minor_version=$(gcc -dumpversion | awk -F. '{print $2}')
+  test "$gcc_minor_version" -lt 8
+}
+
+function continue_or_exit() {
+  if "$ASSUME_YES"; then
+    return
+  fi
+
+  local prompt="$1"
+  echo_color "$YELLOW" -n "$prompt"
+  read -p " [Y/n] " yn
+  if [[ "$yn" == N* || "$yn" == n* ]]; then
+    echo "Cancelled."
+    exit 0
+  fi
+}
+
+# If a string is very simple we don't need to quote it.  But we should quote
+# everything else to be safe.
+function needs_quoting() {
+  echo "$@" | grep -q '[^a-zA-Z0-9./_=-]'
+}
+
+function escape_for_quotes() {
+  echo "$@" | sed -e 's~\\~\\\\~g' -e "s~'~\\\\'~g"
+}
+
+function quote_arguments() {
+  local argument_str=""
+  for argument in "$@"; do
+    if [ -n "$argument_str" ]; then
+      argument_str+=" "
+    fi
+    if needs_quoting "$argument"; then
+      argument="'$(escape_for_quotes "$argument")'"
+    fi
+    argument_str+="$argument"
+  done
+  echo "$argument_str"
+}
+
+function build_ngx_pagespeed() {
+  getopt --test
+  if [ "$?" != 4 ]; then
+    # Even Centos 5 and Ubuntu 10 LTS have new-style getopt, so I don't expect
+    # this to be hit in practice on systems that are actually able to run
+    # PageSpeed.
+    fail "Your version of getopt is too old.  Exiting with no changes made."
+  fi
+
+  opts=$(getopt -o v:n:mb:pslt:ya:dh \
+    --longoptions ngx-pagespeed-version:,nginx-version:,dynamic-module \
+    --longoptions buildir:,no-deps-check,psol-from-source,devel,build-type: \
+    --longoptions assume-yes,additional-nginx-configure-arguments:,dryrun,help \
+    -n "$(basename "$0")" -- "$@")
+  if [ $? != 0 ]; then
+    usage
+    exit 1
+  fi
+  eval set -- "$opts"
+
+  NPS_VERSION="DEFAULT"
+  NGINX_VERSION=""
+  BUILDDIR="$HOME"
+  DO_DEPS_CHECK=true
+  PSOL_FROM_SOURCE=false
+  DEVEL=false
+  BUILD_TYPE=""
+  ASSUME_YES=false
+  DRYRUN=false
+  DYNAMIC_MODULE=false
+  while true; do
+    case "$1" in
+      -v | --ngx-pagespeed-version) shift
+        NPS_VERSION="$1"
+        shift
+        ;;
+      -n | --nginx-version) shift
+        NGINX_VERSION="$1"
+        shift
+        ;;
+      -m | --dynamic-module) shift
+        DYNAMIC_MODULE=true
+        ;;
+      -b | --builddir) shift
+        BUILDDIR="$1"
+        shift
+        ;;
+      -p | --no-deps-check) shift
+        DO_DEPS_CHECK=false
+        ;;
+      -s | --psol-from-source) shift
+        PSOL_FROM_SOURCE=true
+        ;;
+      -l | --devel) shift
+        DEVEL=true
+        ;;
+      -t | --build-type) shift
+        BUILD_TYPE="$1"
+        shift
+        ;;
+      -y | --assume-yes) shift
+        ASSUME_YES="true"
+        ;;
+      -a | --additional-nginx-configure-arguments) shift
+        ADDITIONAL_NGINX_CONFIGURE_ARGUMENTS="$1"
+        shift
+        ;;
+      -d | --dryrun) shift
+        DRYRUN="true"
+        ;;
+      -h | --help) shift
+        usage
+        exit 0
+        ;;
+      --) shift
+        break
+        ;;
+      *)
+        echo "Invalid argument: $1"
+        usage
+        exit 1
+        ;;
+    esac
+  done
+
+  USE_GIT_CHECKOUT="$DEVEL"
+  ALREADY_CHECKED_OUT=false
+  if [ -e PSOL_BINARY_URL ]; then
+    status "Detected that we're running in an existing ngx_pagespeed checkout."
+    USE_GIT_CHECKOUT=true
+    ALREADY_CHECKED_OUT=true
+  fi
+
+  if "$ALREADY_CHECKED_OUT"; then
+    if [ "$NPS_VERSION" != "DEFAULT" ]; then
+      fail \
+"The --ngx-pagespeed-version argument doesn't make sense when running within an existing checkout."
+    fi
+  elif [ "$NPS_VERSION" = "DEFAULT" ]; then
+    if "$DEVEL"; then
+      NPS_VERSION="master"
+    else
+      NPS_VERSION="latest-stable"
+    fi
+  fi
+
+  if [ ! -d "$BUILDDIR" ]; then
+    fail "Told to build in $BUILDDIR, but that directory doesn't exist."
+  fi
+
+  BUILD_NGINX=false
+  if [ -n "$NGINX_VERSION" ]; then
+    BUILD_NGINX=true
+  fi
+
+  if "$DEVEL"; then
+    PSOL_FROM_SOURCE=true
+    BUILD_NGINX=true
+    if [ -n "$NGINX_VERSION" ]; then
+      fail \
+"The --devel argument conflicts with --nginx.  In devel mode we use the version of nginx that's included as a submodule."
+    fi
+    if "$DYNAMIC_MODULE"; then
+      fail "Can't currently build a dynamic module in --devel mode."
+    fi
+  fi
+
+  if "$PSOL_FROM_SOURCE" && [ -z "$BUILD_TYPE" ]; then
+    if "$DEVEL"; then
+      BUILD_TYPE="Debug"
+    else
+      BUILD_TYPE="Release"
+    fi
+  elif [ -n "$BUILD_TYPE" ]; then
+    fail "Setting --build-type requires --psol-from-source or --devel."
+  fi
+
+  if [ "$NGINX_VERSION" = "latest" ]; then
+    # When this function fails it prints the debugging information needed first
+    # to stderr.
+    NGINX_VERSION=$(determine_latest_nginx_version) || exit 1
+  fi
+
+  if "$DYNAMIC_MODULE"; then
+    # Check that ngx_pagespeed and nginx are recent enough to support dynamic
+    # modules.  Unfortunately NPS_VERSION might be a tag, in which case we don't
+    # know.  If it's not a numeric version number, then assume it's recent
+    # enough and if it's not they'll get an ugly compilation error later.
+    # Luckily 1.10.33.5 was a while ago now.
+    #
+    # I'd like to use =~ here, but they changed syntax between v3 and v4 (quotes
+    # moved from mandatory to optional to prohibited).
+    if [[ "${NPS_VERSION#*[^0-9.]}" = "$NPS_VERSION" ]] &&
+         version_older_than "$NPS_VERSION" "1.10.33.5"; then
+      fail "
+You're trying to build ngx_pagespeed $NPS_VERSION as a dynamic module, but
+ngx_pagespeed didn't add support for dynamic modules until 1.10.33.5."
+    fi
+
+    if [ ! -z "NGINX_VERSION" ]; then
+      if version_older_than "$NGINX_VERSION" "1.9.13"; then
+        fail "
+You're trying to build nginx $NGINX_VERSION as a dynamic module but nginx didn't
+add support for dynamic modules in a way compatible with ngx_pagespeed until
+1.9.13."
+      fi
+    fi
+  fi
+
+  if "$DRYRUN"; then
+    TEMPDIR="/tmp/output-of-mktemp"
+  else
+    TEMPDIR=$(mktemp -d)
+    function cleanup_tempdir {
+      rm -rf "$TEMPDIR"
+    }
+    trap cleanup_tempdir EXIT
+  fi
+
+  extra_flags=()
+  # Now make sure our dependencies are installed.
+  if "$DO_DEPS_CHECK"; then
+    if [ -f /etc/debian_version ]; then
+      status "Detected debian-based distro."
+
+      install_dependencies "apt-get install" debian_is_installed \
+        "build-essential zlib1g-dev libpcre3 libpcre3-dev unzip"
+
+      if gcc_too_old; then
+        if [ ! -e /usr/lib/gcc-mozilla/bin/gcc ]; then
+          status "Detected that gcc is older than 4.8.  Installing gcc-mozilla"
+          status "which installs gcc-4.8 into /usr/lib/gcc-mozilla/ and doesn't"
+          status "affect your global gcc installation."
+          run sudo apt-get install gcc-mozilla
+        fi
+
+        extra_flags=("--with-cc=/usr/lib/gcc-mozilla/bin/gcc" \
+                     "--with-ld-opt=-static-libstdc++")
+      fi
+
+    elif [ -f /etc/redhat-release ]; then
+      status "Detected redhat-based distro."
+
+      install_dependencies "yum install" redhat_is_installed \
+        "gcc-c++ pcre-devel zlib-devel make unzip wget"
+      if gcc_too_old; then
+        if [ ! -e /opt/rh/devtoolset-2/root/usr/bin/gcc ]; then
+          redhat_major_version=$(
+            cat /etc/redhat-release | grep -o -E '[0-9]+' | head -n 1)
+          if [ "$redhat_major_version" == 5 ]; then
+            slc_version=5
+          elif [ "$redhat_major_version" == 6 ]; then
+            slc_version=6
+          else
+            fail "
+Unexpected major version $redhat_major_version in /etc/redhat-release:
+$(cat /etc/redhat-release) Expected 5 or 6."
+          fi
+
+          status "Detected that gcc is older than 4.8.  Scientific Linux"
+          status "provides a gcc package that installs gcc-4.8 into /opt/ and"
+          status "doesn't affect your global gcc installation."
+          slc_key="https://linux.web.cern.ch/linux/scientific6/docs/repository/"
+          slc_key+="cern/slc6X/i386/RPM-GPG-KEY-cern"
+          slc_key_out="$TEMPDIR/RPM-GPG-KEY-cern"
+          run sudo wget "$slc_key" -O "$slc_key_out"
+          run sudo rpm --import "$slc_key_out"
+
+          repo_fname="/etc/yum.repos.d/slc${slc_version}-devtoolset.repo"
+          if [ -e "$repo_fname" ]; then
+            fail "Expected $repo_fname not to exist; aborting."
+          fi
+
+          repo_url="https://linux.web.cern.ch/linux/scientific${slc_version}/"
+          repo_url+="/docs/repository/cern/devtoolset/"
+          repo_url+="slc${slc_version}-devtoolset.repo"
+          run sudo wget -O "$repo_fname" "$repo_url"
+          run sudo yum install devtoolset-2-gcc-c++ devtoolset-2-binutils
+        fi
+        extra_flags=("--with-cc=/opt/rh/devtoolset-2/root/usr/bin/gcc")
+      fi
+    else
+      fail "
+This doesn't appear to be a deb-based distro or an rpm-based one.  Not going to
+be able to install dependencies.  Please install dependencies manually and rerun
+with --no-deps-check."
+    fi
+    status "Operating system dependencies are all set."
+  else
+    status "Not checking whether operating system dependencies are installed."
+  fi
+
+  function delete_if_already_exists() {
+    if "$DRYRUN"; then return; fi
+
+    local directory="$1"
+    if [ -d "$directory" ]; then
+      if [ ${#directory} -lt 8 ]; then
+        fail "
+Not deleting $directory; name is suspiciously short.  Something is wrong."
+      fi
+
+      continue_or_exit "OK to delete $directory?"
+      run rm -rf "$directory"
+    fi
+  }
+
+  # In general, the zip github builds for tag foo unzips to ngx_pagespeed-foo,
+  # but it looks like they special case vVERSION tags to ngx_pagespeed-VERSION
+  if [[ "$NPS_VERSION" =~ ^[0-9]*[.][0-9]*[.][0-9]*[.][0-9]*$ ]]; then
+    # We've been given a numeric version number.  This has an associated tag
+    # in the form vVERSION-beta.
+    tag_name="v${NPS_VERSION}-beta"
+    nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}-beta"
+  else
+    # We've been given a tag name, like latest-beta.  Download that directly.
+    tag_name="$NPS_VERSION"
+    nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}"
+  fi
+
+  install_dir="this-only-makes-sense-in-devel-mode"
+  if "$USE_GIT_CHECKOUT"; then
+    # We're either doing a --devel build, or someone is running us from an
+    # existing git checkout.
+    nps_module_dir="$PWD"
+    install_dir="$nps_module_dir"
+    if "$ALREADY_CHECKED_OUT"; then
+      run cd "$nps_module_dir"
+    else
+      status "Checking out ngx_pagespeed..."
+      run git clone "git@github.com:pagespeed/ngx_pagespeed.git" \
+                    "$nps_module_dir"
+      run cd "$nps_module_dir"
+      run git checkout "$tag_name"
+    fi
+    submodules_dir="$nps_module_dir/testing-dependencies"
+    if "$DEVEL"; then
+      status "Downloading dependencies..."
+      run git submodule update --init --recursive
+      if [[ "$CONTINUOUS_INTEGRATION" != true ]]; then
+        status "Switching submodules over to git protocol."
+        # This lets us push to github by public key.
+        for config in $(find .git/ -name config) ; do
+          run sed -i s~https://github.com/~git@github.com:~ $config ;
+        done
+      fi
+    fi
+  else
+    nps_baseurl="https://github.com/pagespeed/ngx_pagespeed/archive"
+    nps_downloaded="$TEMPDIR/$nps_downloaded_fname.zip"
+    status "Downloading ngx_pagespeed..."
+    run wget "$nps_baseurl/$tag_name.zip" -O "$nps_downloaded"
+    nps_module_dir="$BUILDDIR/$nps_downloaded_fname"
+    delete_if_already_exists "$nps_module_dir"
+    status "Extracting ngx_pagespeed..."
+    run unzip -q "$nps_downloaded" -d "$BUILDDIR"
+    run cd "$nps_module_dir"
+  fi
+
+  MOD_PAGESPEED_DIR=""
+  PSOL_BINARY=""
+  if "$PSOL_FROM_SOURCE"; then
+    MOD_PAGESPEED_DIR="$PWD/testing-dependencies/mod_pagespeed"
+    git submodule update --init --recursive -- "$MOD_PAGESPEED_DIR"
+    run pushd "$MOD_PAGESPEED_DIR"
+
+    if "$DEVEL"; then
+      if [ ! -d "$HOME/apache2" ]; then
+        run install/build_development_apache.sh 2.2 prefork
+      fi
+      cd devel
+      run make apache_debug_psol
+      PSOL_BINARY="$MOD_PAGESPEED_DIR/out/$BUILD_TYPE/pagespeed_automatic.a"
+    else
+      run install/build_psol.sh --skip_tests --skip_packaging
+      PSOL_BINARY="$MOD_PAGESPEED_DIR/pagespeed/automatic/pagespeed_automatic.a"
+    fi
+    run popd
+  else
+    # Now we need to figure out what precompiled version of PSOL to build
+    # ngx_pagespeed against.
+    if "$DRYRUN"; then
+      psol_url="https://psol.example.com/cant-get-psol-version-in-dry-run.tar.gz"
+    elif [ -e PSOL_BINARY_URL ]; then
+      # Releases after 1.11.33.4 there is a PSOL_BINARY_URL file that tells us
+      # where to look.
+      psol_url="$(scripts/format_binary_url.sh PSOL_BINARY_URL)"
+      if [[ "$psol_url" != https://* ]]; then
+        fail "Got bad psol binary location information: $psol_url"
+      fi
+    else
+      # For past releases we have to grep it from the config file.  The url has
+      # always looked like this, and the config file has contained it since
+      # before we started tagging our ngx_pagespeed releases.
+      psol_url="$(grep -o \
+          "https://dl.google.com/dl/page-speed/psol/[0-9.]*.tar.gz" config)"
+      if [ -z "$psol_url" ]; then
+        fail "Couldn't find PSOL url in $PWD/config"
+      fi
+    fi
+
+    status "Downloading PSOL binary..."
+    run wget "$psol_url"
+
+    status "Extracting PSOL..."
+    run tar -xzf $(basename "$psol_url")  # extracts to psol/
+  fi
+
+  if "$DYNAMIC_MODULE"; then
+    add_module="--add-dynamic-module=$nps_module_dir"
+  else
+    add_module="--add-module=$nps_module_dir"
+  fi
+  configure_args=("$add_module" "${extra_flags[@]}")
+
+  if "$DEVEL"; then
+    configure_args=("${configure_args[@]}"
+                    "--prefix=$install_dir/nginx"
+                    "--add-module=$submodules_dir/ngx_cache_purge"
+                    "--add-module=$submodules_dir/ngx_devel_kit"
+                    "--add-module=$submodules_dir/set-misc-nginx-module"
+                    "--add-module=$submodules_dir/headers-more-nginx-module"
+                    "--with-ipv6"
+                    "--with-http_v2_module")
+    if [ "$BUILD_TYPE" = "Debug" ]; then
+      configure_args=("${configure_args[@]}" "--with-debug")
+    fi
+  fi
+
+  echo
+  if ! "$BUILD_NGINX"; then
+    # Just prepare the module for them to install.
+    status "ngx_pagespeed is ready to be built against nginx."
+    echo "When running ./configure:"
+    if "$PSOL_FROM_SOURCE"; then
+      echo "  Set the following environment variables:"
+      echo "    MOD_PAGESPEED_DIR=$MOD_PAGESPEED_DIR"
+      echo "    PSOL_BINARY=$PSOL_BINARY"
+    fi
+    echo "  Give ./configure the following arguments:"
+    echo "    $(quote_arguments "${configure_args[@]}")"
+    echo
+    if [ ${#extra_flags[@]} -eq 0 ]; then
+      echo "If this is for integration with an already-built nginx, make sure"
+      echo "to include any other arguments you originally passed to"
+      echo "./configure.  You can see these with 'nginx -V'."
+    else
+      echo "Note: because we need to set $(quote_arguments "${extra_flags[@]}")"
+      echo "on this platform, if you want to integrate ngx_pagespeed with an"
+      echo "already-built nginx you're going to need to rebuild your nginx with"
+      echo "those flags set."
+    fi
+  else
+    if "$DEVEL"; then
+      # Use the nginx we loaded as a submodule
+      nginx_dir="$submodules_dir/nginx"
+      configure_location="auto"
+    else
+      # Download and build the specified nginx version.
+      nginx_leaf="nginx-${NGINX_VERSION}.tar.gz"
+      nginx_fname="$TEMPDIR/$nginx_leaf"
+      status "Downloading nginx..."
+      run wget "http://nginx.org/download/$nginx_leaf" -O "$nginx_fname"
+      nginx_dir="$BUILDDIR/nginx-${NGINX_VERSION}/"
+      delete_if_already_exists "$nginx_dir"
+      status "Extracting nginx..."
+      run tar -xzf "$nginx_fname" --directory "$BUILDDIR"
+      configure_location="."
+    fi
+    run cd "$nginx_dir"
+
+    configure=("$configure_location/configure" "${configure_args[@]}")
+    additional_configure_args=""
+    if [ -z "${ADDITIONAL_NGINX_CONFIGURE_ARGUMENTS+x}" ]; then
+      if ! "$ASSUME_YES"; then
+        echo "About to build nginx.  Do you have any additional ./configure"
+        echo "arguments you would like to set?  For example, if you would like"
+        echo "to build nginx with https support give --with-http_ssl_module"
+        echo "If you don't have any, just press enter."
+        read -p "> " additional_configure_args
+      fi
+    else
+      additional_configure_args="$ADDITIONAL_NGINX_CONFIGURE_ARGUMENTS"
+    fi
+
+    if [ -n "$additional_configure_args" ]; then
+      # Split additional_configure_args respecting any internal quotation.
+      # Otherwise things like --with-cc-opt='-foo -bar' won't work.
+      eval additional_configure_args=("$additional_configure_args")
+      configure=("${configure[@]}" "${additional_configure_args[@]}")
+    fi
+
+    echo "About to configure nginx with:"
+    echo "   $(quote_arguments "${configure[@]}")"
+    continue_or_exit "Does this look right?"
+    MOD_PAGESPEED_DIR="$MOD_PAGESPEED_DIR" \
+      PSOL_BINARY="$PSOL_BINARY" \
+      run "${configure[@]}"
+
+    if ! "$DEVEL"; then
+      continue_or_exit "Build nginx?"
+    fi
+    run make
+
+    if "$DEVEL"; then
+      run make install
+
+      status "Nginx installed with ngx_pagespeed, and set up for development."
+      echo "To run tests:"
+      echo "  cd $nps_module_dir"
+      echo "  test/run_tests.sh"
+      echo
+      echo "To rebuild after changes:"
+      echo "  scripts/rebuild.sh"
+    else
+      continue_or_exit "Install nginx?"
+      run sudo make install
+
+      echo
+      if "$DYNAMIC_MODULE"; then
+        echo "Nginx installed with ngx_pagespeed support available as a"
+        echo "loadable module."
+        echo
+        echo "To load the ngx_pagespeed module, you'll need to add:"
+        echo "  load_module \"modules/ngx_pagespeed.so\";"
+        echo "at the top of your main nginx configuration file."
+      else
+        echo "Nginx installed with ngx_pagespeed support compiled-in."
+      fi
+      echo
+      echo "If this is a new installation you probably need an init script to"
+      echo "manage starting and stopping the nginx service.  See:"
+      echo "  http://wiki.nginx.org/InitScripts"
+      echo
+      echo "You'll also need to configure ngx_pagespeed if you haven't yet:"
+      echo "  https://developers.google.com/speed/pagespeed/module/configuration"
+    fi
+  fi
+  if "$DRYRUN"; then
+    echo_color "$YELLOW" "[this was a dry run; your system is unchanged]"
+  fi
+}
+
+# Start running things from a call at the end so if this script is executed
+# after a partial download it doesn't do anything.
+build_ngx_pagespeed "$@"

scripts/format_binary_url.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -u
+
+if [ $# -ne 1 ]; then
+  echo "Usage: $(basename $0) <url_file>" >&2
+  exit 1
+fi
+
+url_file=$1
+
+if [ ! -e "$url_file" ]; then
+  echo "Url file '$url_file' missing!" >&2
+fi
+
+# The size names must match install/build_psol.sh in mod_pagespeed
+if [ "$(uname -m)" = x86_64 ]; then
+  bit_size_name=x64
+else
+  bit_size_name=ia32
+fi
+
+sed -e 's/$BIT_SIZE_NAME\b/'$bit_size_name'/g' $url_file

scripts/pagespeed_libraries_generator.sh

@@ -1,5 +1,19 @@
 #!/bin/bash
 #
+# Copyright 2013 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 # Converts pagespeed_libraries.conf from Apache-format to Nginx-format,
 # supporting the canonicalize_javascript_libraries filter.
 # Inspired by https://github.com/pagespeed/ngx_pagespeed/issues/532

scripts/prepare_psol.sh

@@ -1,94 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2012 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Author: jefftk@google.com (Jeff Kaufman)
-#
-# Usage:
-#   scripts/prepare_psol.sh /path/to/mod_pagespeed/src
-#
-# Creates a directory psol/ and copies headers and a few source files from a
-# depot_tools (glient) checkout into psol/include.  Along with creating
-# binaries, this is a step in preparing psol.tar.gz for distribution.
-#
-
-set -u  # check for undefined variables
-set -e  # exit on failed commands
-
-if [ "$(basename "$PWD")" != "ngx_pagespeed" ] ; then
-  echo "$(basename $0) must be invoked from the ngx_pagespeed directory"
-  exit 1
-fi
-
-if [ $# -ne 1 ] ; then
-  echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
-  exit 1
-fi
-
-MOD_PAGESPEED_SRC="$1"
-
-if [ "$(basename "$(dirname "$MOD_PAGESPEED_SRC")")/$( \
-        basename "$MOD_PAGESPEED_SRC")" != "mod_pagespeed/src" ] ; then
-  echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
-  exit 1
-fi
-
-if [ -e psol ] ; then
-  echo "A psol/ directory already exists.  Move it somewhere else and rerun."
-  exit 1
-fi
-mkdir psol/
-# Copy over the .h files, plus a few selected .cc and .c files.
-rsync -arvz "$MOD_PAGESPEED_SRC/" "psol/include/" --prune-empty-dirs \
-  --exclude=".svn" \
-  --exclude=".git" \
-  --include='*.h' \
-  --include='*/' \
-  --include="apr_thread_compatible_pool.cc" \
-  --include="serf_url_async_fetcher.cc" \
-  --include="apr_mem_cache.cc" \
-  --include="key_value_codec.cc" \
-  --include="apr_memcache2.c" \
-  --include="loopback_route_fetcher.cc" \
-  --include="add_headers_fetcher.cc" \
-  --include="console_css_out.cc" \
-  --include="console_out.cc" \
-  --include="dense_hash_map" \
-  --include="dense_hash_set" \
-  --include="sparse_hash_map" \
-  --include="sparse_hash_set" \
-  --include="sparsetable" \
-  --include="mod_pagespeed_console_out.cc" \
-  --include="mod_pagespeed_console_css_out.cc" \
-  --include="mod_pagespeed_console_html_out.cc" \
-  --exclude='*'
-mkdir -p psol/lib/Debug/linux/ia32
-mkdir -p psol/lib/Debug/linux/x64
-mkdir -p psol/lib/Release/linux/ia32
-mkdir -p psol/lib/Release/linux/x64
-
-# Log that we did this.
-SVN_REVISION="$(svn info $MOD_PAGESPEED_SRC | grep Revision | awk '{print $2}')"
-SVN_TAG="$(svn info $MOD_PAGESPEED_SRC | grep URL |  awk -F/ '{print $(NF-1)}')"
-
-DATE="$(date +%F)"
-echo "${DATE}: Copied from mod_pagespeed ${SVN_TAG}@r${SVN_REVISION} ($USER)" \
-  >> psol/include_history.txt
-
-echo
-echo "Output is in psol/include.  Now put binaries in psol/lib following"
-echo "https://github.com/pagespeed/ngx_pagespeed/wiki/Building-Release-Binaries"
-echo "and then you can distribute PSOL."
-

scripts/rebuild.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Usage: scripts/rebuild.sh
+#
+# After building with "scripts/build_ngx_pagespeed.sh --devel", if you make
+# changes to ngx_pagespeed you'll need to rebuild it.  The underlying commands
+# aren't complicated, but it's faster to work if it's automated.
+
+set -e  # exit script if any command returns an error
+set -u  # exit the script if any variable is uninitialized
+
+this_dir="$(dirname $0)"
+cd "$this_dir/.."
+nps_dir="$PWD"
+
+cd "$nps_dir/testing-dependencies/mod_pagespeed/devel"
+make apache_debug_psol
+
+cd "$nps_dir/testing-dependencies/nginx/"
+make
+make install

src/ngx_base_fetch.cc

@@ -12,17 +12,20 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
+ * Author: jefftk@google.com (Jeff Kaufman)
  */
 
-// Author: jefftk@google.com (Jeff Kaufman)
+#include "ngx_pagespeed.h"  // Must come first, see comments in CollectHeaders.
+
 #include <unistd.h> //for usleep
 
 #include "ngx_base_fetch.h"
 #include "ngx_event_connection.h"
 #include "ngx_list_iterator.h"
 
-#include "ngx_pagespeed.h"
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
-
+#include "net/instaweb/rewriter/public/rewrite_options.h"
 #include "net/instaweb/rewriter/public/rewrite_stats.h"
 #include "pagespeed/kernel/base/google_message_handler.h"
 #include "pagespeed/kernel/base/message_handler.h"
@@ -38,14 +41,19 @@ const char kDone = 'D';
 NgxEventConnection* NgxBaseFetch::event_connection = NULL;
 int NgxBaseFetch::active_base_fetches = 0;
 
-NgxBaseFetch::NgxBaseFetch(ngx_http_request_t* r,
+NgxBaseFetch::NgxBaseFetch(StringPiece url,
+                           ngx_http_request_t* r,
                            NgxServerContext* server_context,
                            const RequestContextPtr& request_ctx,
                            PreserveCachingHeaders preserve_caching_headers,
-                           NgxBaseFetchType base_fetch_type)
+                           NgxBaseFetchType base_fetch_type,
+                           const RewriteOptions* options)
     : AsyncFetch(request_ctx),
+      url_(url.data(), url.size()),
       request_(r),
       server_context_(server_context),
+      options_(options),
+      need_flush_(false),
       done_called_(false),
       last_buf_sent_(false),
       references_(2),
@@ -118,6 +126,8 @@ const char* BaseFetchTypeToCStr(NgxBaseFetchType type) {
   return "can't get here";
 }
 
+// TODO(oschaaf): replace the ngx_log_error with VLOGS or pass in a
+// MessageHandler and use that.
 void NgxBaseFetch::ReadCallback(const ps_event_data& data) {
   NgxBaseFetch* base_fetch = reinterpret_cast<NgxBaseFetch*>(data.sender);
   ngx_http_request_t* r = base_fetch->request();
@@ -138,19 +148,40 @@ void NgxBaseFetch::ReadCallback(const ps_event_data& data) {
   if (refcount == 0 || detached) {
     return;
   }
+
   ps_request_ctx_t* ctx = ps_get_request_context(r);
 
-  CHECK(data.sender == ctx->base_fetch);
+  // If our request context was zeroed, skip this event.
-  CHECK(r->count > 0) << "r->count: " << r->count;
+  // See https://github.com/pagespeed/ngx_pagespeed/issues/1081
+  if (ctx == NULL) {
+    // Should not happen normally, when it does this message will cause our
+    // system tests to fail.
+    ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
+        "pagespeed [%p] skipping event: request context gone", r);
+    return;
+  }
+
+  // Normally we expect the sender to equal the active NgxBaseFetch instance.
+  DCHECK(data.sender == ctx->base_fetch);
+
+  // If someone changed our request context or NgxBaseFetch, skip processing.
+  if (data.sender != ctx->base_fetch) {
+      ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
+          "pagespeed [%p] skipping event: event originating from disassociated"
+          " NgxBaseFetch instance.", r);
+      return;
+  }
 
   int rc;
+  bool run_posted = true;
   // If we are unlucky enough to have our connection finalized mid-ipro-lookup,
   // we must enter a different flow. Also see ps_in_place_check_header_filter().
   if ((ctx->base_fetch->base_fetch_type_ != kIproLookup)
       && r->connection->error) {
     ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
-      "pagespeed [%p] request already finalized", r);
+      "pagespeed [%p] request already finalized %d", r, r->count);
     rc = NGX_ERROR;
+    run_posted = false;
   } else {
     rc = ps_base_fetch::ps_base_fetch_handler(r);
   }
@@ -163,8 +194,11 @@ void NgxBaseFetch::ReadCallback(const ps_event_data& data) {
 
   ngx_connection_t* c = r->connection;
   ngx_http_finalize_request(r, rc);
-  // See http://forum.nginx.org/read.php?2,253006,253061
+
-  ngx_http_run_posted_requests(c);
+  if (run_posted) {
+    // See http://forum.nginx.org/read.php?2,253006,253061
+    ngx_http_run_posted_requests(c);
+  }
 }
 
 void NgxBaseFetch::Lock() {
@@ -194,8 +228,11 @@ ngx_int_t NgxBaseFetch::CopyBufferToNginx(ngx_chain_t** link_ptr) {
     return NGX_AGAIN;
   }
 
-  int rc = string_piece_to_buffer_chain(
+  int rc = string_piece_to_buffer_chain(request_->pool, buffer_, link_ptr,
-      request_->pool, buffer_, link_ptr, done_called_ /* send_last_buf */);
+                                        done_called_ /* send_last_buf */,
+                                        need_flush_);
+  need_flush_ = false;
+
   if (rc != NGX_OK) {
     return rc;
   }
@@ -223,6 +260,13 @@ ngx_int_t NgxBaseFetch::CollectAccumulatedWrites(ngx_chain_t** link_ptr) {
 }
 
 ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
+  // nginx defines _FILE_OFFSET_BITS to 64, which changes the size of off_t.
+  // If a standard header is accidentally included before the nginx header,
+  // on a 32-bit system off_t will be 4 bytes and we don't assign all the
+  // bits of content_length_n. Sanity check that did not happen.
+  // This could use static_assert, but this file is not built with --std=c++11.
+  bool sanity_check_off_t[sizeof(off_t) == 8 ? 1 : -1] __attribute__ ((unused));
+
   const ResponseHeaders* pagespeed_headers = response_headers();
 
   if (content_length_known()) {
@@ -273,6 +317,9 @@ void NgxBaseFetch::HandleHeadersComplete() {
 }
 
 bool NgxBaseFetch::HandleFlush(MessageHandler* handler) {
+  Lock();
+  need_flush_ = true;
+  Unlock();
   RequestCollection(kFlush);  // A new part of the response body is available
   return true;
 }
@@ -305,4 +352,9 @@ void NgxBaseFetch::HandleDone(bool success) {
   DecrefAndDeleteIfUnreferenced();
 }
 
+bool NgxBaseFetch::IsCachedResultValid(const ResponseHeaders& headers) {
+  return OptionsAwareHTTPCacheCallback::IsCacheValid(
+      url_, *options_, request_context(), headers);
+}
+
 }  // namespace net_instaweb

src/ngx_base_fetch.h

@@ -43,6 +43,9 @@
 //    events it handles.
 //
 // When the last reference is dropped, this class will delete itself.
+//
+// TODO(jmarantz): consider sharing the cache-invalidation infrastructure
+// with ApacheFetch, using a common base class.
 
 #ifndef NGX_BASE_FETCH_H_
 #define NGX_BASE_FETCH_H_
@@ -59,6 +62,7 @@ extern "C" {
 #include "ngx_server_context.h"
 
 #include "net/instaweb/http/public/async_fetch.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
 #include "pagespeed/kernel/base/string.h"
 #include "pagespeed/kernel/http/headers.h"
 
@@ -74,10 +78,12 @@ enum NgxBaseFetchType {
 
 class NgxBaseFetch : public AsyncFetch {
  public:
-  NgxBaseFetch(ngx_http_request_t* r, NgxServerContext* server_context,
+  NgxBaseFetch(StringPiece url, ngx_http_request_t* r,
+               NgxServerContext* server_context,
                const RequestContextPtr& request_ctx,
                PreserveCachingHeaders preserve_caching_headers,
-               NgxBaseFetchType base_fetch_type);
+               NgxBaseFetchType base_fetch_type,
+               const RewriteOptions* options);
   virtual ~NgxBaseFetch();
 
   // Statically initializes event_connection, require for PSOL and nginx to
@@ -125,6 +131,8 @@ class NgxBaseFetch : public AsyncFetch {
   ngx_http_request_t* request() { return request_; }
   NgxBaseFetchType base_fetch_type() { return base_fetch_type_; }
 
+  bool IsCachedResultValid(const ResponseHeaders& headers) override;
+
  private:
   virtual bool HandleWrite(const StringPiece& sp, MessageHandler* handler);
   virtual bool HandleFlush(MessageHandler* handler);
@@ -152,13 +160,16 @@ class NgxBaseFetch : public AsyncFetch {
   int DecrefAndDeleteIfUnreferenced();
 
   static NgxEventConnection* event_connection;
-  
+
   // Live count of NgxBaseFetch instances that are currently in use.
   static int active_base_fetches;
 
+  GoogleString url_;
   ngx_http_request_t* request_;
   GoogleString buffer_;
   NgxServerContext* server_context_;
+  const RewriteOptions* options_;
+  bool need_flush_;
   bool done_called_;
   bool last_buf_sent_;
   // How many active references there are to this fetch. Starts at two,

src/ngx_fetch.cc

@@ -140,6 +140,8 @@ NgxConnection* NgxConnection::Connect(ngx_peer_connection_t* pc,
 
   int rc = ngx_event_connect_peer(pc);
   if (rc == NGX_ERROR || rc == NGX_DECLINED || rc == NGX_BUSY) {
+    ngx_log_error(NGX_LOG_DEBUG, pc->log, 0,
+                    "NgxFetch: ngx_event_connect_peer failed");
     return NULL;
   }
 
@@ -315,7 +317,9 @@ bool NgxFetch::Init() {
   }
 
   if (!ParseUrl()) {
-    message_handler_->Message(kError, "NgxFetch: ParseUrl() failed");
+    message_handler_->Message(kError, 
+                              "NgxFetch: ParseUrl() failed for [%s]:%s",
+                              str_url_.c_str(), url_.err);
     return false;
   }
 
@@ -361,14 +365,17 @@ bool NgxFetch::Init() {
 
   GoogleString s_ipaddress(reinterpret_cast<char*>(tmp_url->host.data),
                            tmp_url->host.len);
-  ngx_memzero(&sin_, sizeof(sin_));
-  sin_.sin_family = AF_INET;
-  sin_.sin_port = htons(tmp_url->port);
-  sin_.sin_addr.s_addr = inet_addr(s_ipaddress.c_str());
 
-  if (sin_.sin_addr.s_addr == INADDR_NONE) {
+  ngx_memzero(&sin_, sizeof(sin_));
-    // inet_addr returned INADDR_NONE, which means the hostname
+  if (s_ipaddress[0] == '[') {
-    // isn't a valid IP address.  Check DNS.
+      s_ipaddress = s_ipaddress.substr(1);
+  }
+  if (s_ipaddress[s_ipaddress.size() - 1] == ']') {
+      s_ipaddress = s_ipaddress.substr(0, s_ipaddress.size()-2);      
+  }
+  ngx_int_t res = ngx_parse_addr(pool_, &sin_, (u_char*)s_ipaddress.c_str(), s_ipaddress.size());
+  if (res != NGX_OK) {
+    // Hostname isn't a valid IP address.  Check DNS.
     ngx_resolver_ctx_t temp;
     temp.name.data = tmp_url->host.data;
     temp.name.len = tmp_url->host.len;
@@ -403,6 +410,16 @@ bool NgxFetch::Init() {
       return false;
     }
   } else {
+      /*
+    u_char text[NGX_SOCKADDR_STRLEN];
+    ngx_str_t addrtext;
+    addrtext.len = ngx_sock_ntop(fetch->sin_.sockaddr, fetch->sin_.socklen, 
+        text, NGX_SOCKADDR_STRLEN, htons(fetcher->proxy_.url.len ? htons(fetcher->proxy_.port) : fetch->url_.port));
+    ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                    "NgxFetch %p: Resolved host [%V] to [%V]", fetch,
+                    &resolver_ctx->name, &addrtext);
+    */
+    ((struct sockaddr_in *)sin_.sockaddr)->sin_port = htons(fetcher_->proxy_.url.len ? htons(fetcher_->proxy_.port) : url_.port);
     if (InitRequest() != NGX_OK) {
       message_handler()->Message(kError, "NgxFetch: InitRequest failed");
       return false;
@@ -533,14 +550,14 @@ void NgxFetch::ResolveDoneHandler(ngx_resolver_ctx_t* resolver_ctx) {
   }
 
   ngx_uint_t i;
-  // Find the first ipv4 address. We don't support ipv6 yet.
+  // Find the first ipv4 address. We don't support ipv6 yet in dns results.
   for (i = 0; i < resolver_ctx->naddrs; i++) {
     // Old versions of nginx and tengine have a different definition of addrs,
     // work around to make sure we are using the right type (ngx_addr_t*).
     ngx_addr_t* ngx_addrs = reinterpret_cast<ngx_addr_t*>(resolver_ctx->addrs);
     if (typeid(*ngx_addrs) == typeid(*resolver_ctx->addrs)) {
       if (reinterpret_cast<struct sockaddr_in*>(ngx_addrs[i].sockaddr)
-              ->sin_family == AF_INET) {
+             ->sin_family == AF_INET) {
         break;
       }
     } else {
@@ -564,37 +581,30 @@ void NgxFetch::ResolveDoneHandler(ngx_resolver_ctx_t* resolver_ctx) {
   ngx_memzero(&fetch->sin_, sizeof(fetch->sin_));
 
 #if (nginx_version < 1005008)
+  // TODO(oschaaf): need to test this again, might be broken.
   fetch->sin_.sin_addr.s_addr = resolver_ctx->addrs[i];
 #else
-  struct sockaddr_in* sin;
+  fetch->sin_ = resolver_ctx->addrs[i];
-
-  sin = reinterpret_cast<struct sockaddr_in*>(
-      resolver_ctx->addrs[i].sockaddr);
-
-  fetch->sin_.sin_family = sin->sin_family;
-  fetch->sin_.sin_addr.s_addr = sin->sin_addr.s_addr;
 #endif
 
-  fetch->sin_.sin_family = AF_INET;
-  fetch->sin_.sin_port = htons(fetch->url_.port);
-
   // Maybe we have Proxy
-  if (0 != fetcher->proxy_.url.len) {
+  ((struct sockaddr_in *)fetch->sin_.sockaddr)->sin_port = 
-    fetch->sin_.sin_port = htons(fetcher->proxy_.port);
+    htons(fetcher->proxy_.url.len ? htons(fetcher->proxy_.port) : fetch->url_.port);
-  }
-
-  char* ip_address = inet_ntoa(fetch->sin_.sin_addr);
 
+  u_char text[NGX_SOCKADDR_STRLEN];
+  ngx_str_t addrtext;
+  addrtext.len = ngx_sock_ntop(fetch->sin_.sockaddr, fetch->sin_.socklen, 
+    text, NGX_SOCKADDR_STRLEN, htons(fetcher->proxy_.url.len ? htons(fetcher->proxy_.port) : fetch->url_.port));
+  addrtext.data = text;
   ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
-                "NgxFetch %p: Resolved host [%V] to [%s]", fetch,
+                "NgxFetch %p: Resolved host [%V] to [%V]", fetch,
-                &resolver_ctx->name, ip_address);
+                &resolver_ctx->name, &addrtext);
-
-  fetch->release_resolver();
 
   if (fetch->InitRequest() != NGX_OK) {
     fetch->message_handler()->Message(kError, "NgxFetch: InitRequest failed");
     fetch->CallbackDone(false);
   }
+  fetch->release_resolver();
 }
 
 // Prepare the request data for this fetch, and hook the write event.
@@ -690,8 +700,8 @@ int NgxFetch::InitRequest() {
 int NgxFetch::Connect() {
   ngx_peer_connection_t pc;
   ngx_memzero(&pc, sizeof(pc));
-  pc.sockaddr = (struct sockaddr*)&sin_;
+  pc.sockaddr =  sin_.sockaddr;
-  pc.socklen = sizeof(struct sockaddr_in);
+  pc.socklen = sin_.socklen;
   pc.name = &url_.host;
 
   // get callback is dummy function, it just returns NGX_OK
@@ -938,8 +948,8 @@ void NgxFetch::FixUserAgent() {
     user_agent += "NgxNativeFetcher";
   }
   GoogleString version = StrCat(
-      " ", kModPagespeedSubrequestUserAgent,
+      " (", kModPagespeedSubrequestUserAgent,
-      "/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING);
+      "/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING ")");
   if (!StringPiece(user_agent).ends_with(version)) {
     user_agent += version;
   }

src/ngx_fetch.h

@@ -191,7 +191,8 @@ class NgxFetch : public PoolElement<NgxFetch> {
   int64 content_length_;
   bool content_length_known_;
 
-  struct sockaddr_in sin_;
+
+  ngx_addr_t sin_;
   ngx_log_t* log_;
   ngx_buf_t* out_;
   ngx_buf_t* in_;

src/ngx_message_handler.cc

@@ -97,7 +97,6 @@ void NgxMessageHandler::MessageSImpl(MessageType type,
   } else {
     GoogleMessageHandler::MessageSImpl(type, message);
   }
-  // Prepare a log message for the SharedCircularBuffer only.
   AddMessageToBuffer(type, message);
 }
 
@@ -111,6 +110,7 @@ void NgxMessageHandler::FileMessageSImpl(
   } else {
     GoogleMessageHandler::FileMessageSImpl(type, file, line, message);
   }
+  AddMessageToBuffer(type, file, line, message);
 }
 
 }  // namespace net_instaweb

src/ngx_pagespeed.cc

@@ -124,7 +124,7 @@ char* string_piece_to_pool_string(ngx_pool_t* pool, StringPiece sp) {
 // (potentially) longer string to nginx and want it to take ownership.
 ngx_int_t string_piece_to_buffer_chain(
     ngx_pool_t* pool, StringPiece sp, ngx_chain_t** link_ptr,
-    bool send_last_buf) {
+    bool send_last_buf, bool send_flush) {
   // Below, *link_ptr will be NULL if we're starting the chain, and the head
   // chain link.
   *link_ptr = NULL;
@@ -198,6 +198,9 @@ ngx_int_t string_piece_to_buffer_chain(
 
 
   CHECK(tail_link != NULL);
+  if (send_flush) {
+    tail_link->buf->flush = true;
+  }
   if (send_last_buf) {
     tail_link->buf->last_buf = true;
   }
@@ -236,6 +239,9 @@ ngx_http_output_body_filter_pt ngx_http_next_body_filter;
 ngx_int_t ps_base_fetch_filter(ngx_http_request_t* r, ngx_chain_t* in) {
   ps_request_ctx_t* ctx = ps_get_request_context(r);
 
+  if (r->header_only) {
+    return NGX_OK;
+  }
   if (ctx == NULL || ctx->base_fetch == NULL) {
     return ngx_http_next_body_filter(r, in);
   }
@@ -284,7 +290,9 @@ ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r) {
     // modules running after us to manipulate those responses.
     if (!status_ok && (ctx->base_fetch->base_fetch_type() != kHtmlTransform
                        && ctx->base_fetch->base_fetch_type() != kIproLookup)) {
-      return status_code;
+      ps_release_base_fetch(ctx);
+      ngx_http_filter_finalize_request(r, NULL, status_code);
+      return NGX_DONE;
     }
 
     if (ctx->preserve_caching_headers != kDontPreserveHeaders) {
@@ -302,6 +310,12 @@ ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r) {
                 STR_CASE_EQ_LITERAL(header->key, "Last-Modified") ||
                 STR_CASE_EQ_LITERAL(header->key, "Expires"))))) {
           header->hash = 0;
+          if (STR_CASE_EQ_LITERAL(header->key, "Location")) {
+            // There's a possible issue with the location header, where setting
+            // the hash to 0 is not enough. See:
+            // https://github.com/nginx/nginx/blob/master/src/http/ngx_http_header_filter_module.c#L314
+            r->headers_out.location = NULL;
+          }
         }
       }
     } else {
@@ -415,6 +429,32 @@ ngx_int_t ps_set_cache_control(ngx_http_request_t* r, char* cache_control) {
   return NGX_OK;
 }
 
+// Returns false if the header wasn't found.  Otherwise sets cache_control and
+// returns true;
+bool ps_get_cache_control(ngx_http_request_t* r, GoogleString* cache_control) {
+  // Use headers_out.cache_control instead of looking for Cache-Control in
+  // headers_out.headers, because if an upstream sent multiple Cache-Control
+  // headers they're already combined in headers_out.cache_control.
+  auto ccp = static_cast<ngx_table_elt_t**>(r->headers_out.cache_control.elts);
+  if (ccp == nullptr) {
+    return false;  // Header not present.
+  }
+  bool first_segment = true;
+  for (ngx_uint_t i = 0; i < r->headers_out.cache_control.nelts; i++) {
+    if (ccp[i]->hash == 0) {
+      continue;  // Elements with a hash of 0 are marked as excluded.
+    }
+    if (first_segment) {
+      first_segment = false;
+    } else {
+      cache_control->append(", ");
+    }
+    cache_control->append(reinterpret_cast<char*>(ccp[i]->value.data),
+                          ccp[i]->value.len);
+  }
+  return true;
+}
+
 template<class Headers>
 void copy_headers_from_table(const ngx_list_t &from, Headers* to) {
   // Standard nginx idiom for iterating over a list.  See ngx_list.h
@@ -452,11 +492,6 @@ void copy_response_headers_from_ngx(const ngx_http_request_t* r,
 
   headers->set_status_code(r->headers_out.status);
 
-  if (r->headers_out.location != NULL) {
-    headers->Add(HttpAttributes::kLocation,
-                 str_to_string_piece(r->headers_out.location->value));
-  }
-
   // Manually copy over the content type because it's not included in
   // request_->headers_out.headers.
   headers->Add(HttpAttributes::kContentType,
@@ -464,7 +499,8 @@ void copy_response_headers_from_ngx(const ngx_http_request_t* r,
 
   // When we don't have a date header, set one with the current time.
   if (headers->Lookup1(HttpAttributes::kDate) == NULL) {
-    headers->SetDate(ngx_current_msec);
+    PosixTimer timer;
+    headers->SetDate(timer.NowMs());
   }
 
   // TODO(oschaaf): ComputeCaching should be called in setupforhtml()?
@@ -490,6 +526,10 @@ ngx_int_t copy_response_headers_to_ngx(
 
   ngx_int_t i;
   for (i = 0 ; i < pagespeed_headers.NumAttributes() ; i++) {
+    // For IPRO cache misses, these gs_ variables may point to freed memory
+    // when nginx writes the headers to the output as the NgxBaseFetch instance
+    // that owns this memory gets released during request processing. So we
+    // copy these strings later on.
     const GoogleString& name_gs = pagespeed_headers.Name(i);
     const GoogleString& value_gs = pagespeed_headers.Value(i);
 
@@ -510,6 +550,9 @@ ngx_int_t copy_response_headers_to_ngx(
     }  // else we don't preserve any headers.
 
     ngx_str_t name, value;
+    value.len = value_gs.size();
+    value.data = reinterpret_cast<u_char*>(
+        string_piece_to_pool_string(r->pool, value_gs.c_str()));
 
     // To prevent the gzip module from clearing weak etags, we output them
     // using a different name here. The etag header filter module runs behind
@@ -520,11 +563,15 @@ ngx_int_t copy_response_headers_to_ngx(
       name.data = reinterpret_cast<u_char*>(
           const_cast<char*>(kInternalEtagName));
     } else {
-      name.len = name_gs.length();
+      name.len = name_gs.size();
-      name.data = reinterpret_cast<u_char*>(const_cast<char*>(name_gs.data()));
+      name.data = reinterpret_cast<u_char*>(
+          string_piece_to_pool_string(r->pool, name_gs.c_str()));
+    }
+
+    // In case string_piece_to_pool_string failed:
+    if (name.data == NULL || value.data == NULL) {
+        return NGX_ERROR;
     }
-    value.len = value_gs.length();
-    value.data = reinterpret_cast<u_char*>(const_cast<char*>(value_gs.data()));
 
     // TODO(jefftk): If we're setting a cache control header we'd like to
     // prevent any downstream code from changing it.  Specifically, if we're
@@ -535,24 +582,17 @@ ngx_int_t copy_response_headers_to_ngx(
     // net/instaweb/apache/header_util:AddResponseHeadersToRequest
 
     // Make copies of name and value to put into headers_out.
-
-    u_char* value_s = ngx_pstrdup(r->pool, &value);
-    if (value_s == NULL) {
-      return NGX_ERROR;
-    }
-
     if (STR_EQ_LITERAL(name, "Cache-Control")) {
-      ps_set_cache_control(r, const_cast<char*>(value_gs.c_str()));
+      ps_set_cache_control(r, reinterpret_cast<char*>(value.data));
       continue;
     } else if (STR_EQ_LITERAL(name, "Content-Type")) {
       // Unlike all the other headers, content_type is just a string.
-      headers_out->content_type.data = value_s;
+      headers_out->content_type = value;
-      headers_out->content_type.len = value.len;
 
       // We should not include the charset when determining content_type_len, so
       // scan for the ';' that marks the start of the charset part.
       for (ngx_uint_t i = 0; i < value.len; i++) {
-        if (value_s[i] == ';') {
+        if (value.data[i] == ';') {
           break;
         }
         headers_out->content_type_len = i + 1;
@@ -570,11 +610,10 @@ ngx_int_t copy_response_headers_to_ngx(
       continue;
     } else if (STR_EQ_LITERAL(name, "Transfer-Encoding")) {
       continue;
-    }
+    } else if (STR_EQ_LITERAL(name, "Vary") && value.len
-
+        && STR_EQ_LITERAL(value, "Accept-Encoding")) {
-    u_char* name_s = ngx_pstrdup(r->pool, &name);
+      ps_request_ctx_t* ctx = ps_get_request_context(r);
-    if (name_s == NULL) {
+      ctx->psol_vary_accept_only = true;
-      return NGX_ERROR;
     }
 
     ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(
@@ -584,10 +623,10 @@ ngx_int_t copy_response_headers_to_ngx(
     }
 
     header->hash = 1;  // Include this header in the output.
+    header->key.data = name.data;
     header->key.len = name.len;
-    header->key.data = name_s;
+    header->value.data = value.data;
     header->value.len = value.len;
-    header->value.data = value_s;
 
     // Populate the shortcuts to commonly used headers.
     if (STR_EQ_LITERAL(name, "Date")) {
@@ -636,7 +675,7 @@ typedef struct {
 typedef struct {
   // If pagespeed is configured in some server block but not this one our
   // per-request code will be invoked but server context will be null.  In those
-  // cases we neet to short circuit, not changing anything.  Currently our
+  // cases we need to short circuit, not changing anything.  Currently our
   // header filter, body filter, and content handler all do this, but if anyone
   // adds another way for nginx to give us a request to process we need to check
   // there as well.
@@ -677,6 +716,13 @@ char* ps_main_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 char* ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 
+// We want NGX_CONF_MULTI for some very old versions:
+//   https://github.com/pagespeed/ngx_pagespeed/commit/66f1b9aa
+// but it's gone in recent revisions, so provide a compat #define if needed
+#ifndef NGX_CONF_MULTI
+#define NGX_CONF_MULTI 0
+#endif
+
 // TODO(jud): Verify that all the offsets should be NGX_HTTP_SRV_CONF_OFFSET and
 // not NGX_HTTP_LOC_CONF_OFFSET or NGX_HTTP_MAIN_CONF_OFFSET.
 ngx_command_t ps_commands[] = {
@@ -706,6 +752,11 @@ ngx_command_t ps_commands[] = {
   ngx_null_command
 };
 
+bool ps_disabled(ps_srv_conf_t* cfg_s) {
+  return cfg_s->server_context == NULL ||
+         cfg_s->server_context->config()->unplugged();
+}
+
 void ps_ignore_sigpipe() {
   struct sigaction act;
   ngx_memzero(&act, sizeof(act));
@@ -838,18 +889,17 @@ char* ps_configure(ngx_conf_t* cf,
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cf->cycle, ngx_pagespeed));
   if (*options == NULL) {
-    *options = new NgxRewriteOptions(
+    *options = new NgxRewriteOptions(cfg_m->driver_factory->thread_system());
-        cfg_m->driver_factory->thread_system());
   }
 
-  bool process_script_variables = dynamic_cast<NgxRewriteDriverFactory*>(
+  ProcessScriptVariablesMode script_mode =
-      cfg_m->driver_factory)->process_script_variables();
+      dynamic_cast<NgxRewriteDriverFactory*>(cfg_m->driver_factory)
-
+          ->process_script_variables();
-  if (process_script_variables) {
+  if (script_mode != ProcessScriptVariablesMode::kOff) {
     // To be able to use '$', we map '$ps_dollar' to '$' via a script variable.
     ngx_str_t name = ngx_string("ps_dollar");
-    ngx_http_variable_t* var = ngx_http_add_variable(
+    ngx_http_variable_t* var =
-        cf, &name, NGX_HTTP_VAR_CHANGEABLE);
+        ngx_http_add_variable(cf, &name, NGX_HTTP_VAR_CHANGEABLE);
 
     if (var == NULL) {
       return const_cast<char*>(
@@ -860,7 +910,7 @@ char* ps_configure(ngx_conf_t* cf,
 
   const char* status = (*options)->ParseAndSetOptions(
       args, n_args, cf->pool, handler, cfg_m->driver_factory, option_scope, cf,
-      process_script_variables);
+      script_mode);
 
   // nginx expects us to return a string literal but doesn't mark it const.
   return const_cast<char*>(status);
@@ -986,6 +1036,7 @@ void* ps_create_main_conf(ngx_conf_t* cf) {
       "" /* hostname, not used */,
       -1 /* port, not used */);
   active_driver_factory = cfg_m->driver_factory;
+  active_driver_factory->LoggingInit(ngx_cycle->log, false);
   cfg_m->driver_factory->Init();
   ps_set_conf_cleanup_handler(cf, ps_cleanup_main_conf, cfg_m);
   return cfg_m;
@@ -1080,9 +1131,20 @@ char* ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child) {
 
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
-  cfg_m->driver_factory->set_main_conf(parent_cfg_s->options);
+  cfg_m->driver_factory->SetMainConf(parent_cfg_s->options);
   cfg_s->server_context = cfg_m->driver_factory->MakeNgxServerContext(
       "dummy_hostname", dummy_port);
+
+#if (NGX_HTTP_V2)
+  // Save the variable index of the "http2" variable, so we can use it
+  // at request time to lookup whether that's on. We do this conditionally
+  // since NGINX will complain to the user (at [emerg] level!) if it doesn't
+  // know about it.
+  ngx_str_t name = ngx_string("http2");
+  cfg_s->server_context->set_ngx_http2_variable_index(
+      ngx_http_get_variable_index(cf, &name));
+#endif
+
   // The server context sets some options when we call global_options(). So
   // let it do that, then merge in options we got from the config file.
   // Once we do that we're done with cfg_s->options.
@@ -1093,13 +1155,17 @@ char* ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child) {
   delete cfg_s->options;
   cfg_s->options = NULL;
 
-  if (cfg_s->server_context->global_options()->enabled()) {
+  if (!cfg_s->server_context->global_options()->unplugged()) {
     // Validate FileCachePath
     GoogleMessageHandler handler;
     const char* file_cache_path =
         cfg_s->server_context->config()->file_cache_path().c_str();
     if (file_cache_path[0] == '\0') {
-      return const_cast<char*>("FileCachePath must be set");
+      if (!cfg_s->server_context->global_options()->standby()) {
+        return const_cast<char*>("FileCachePath must be set, even for standby");
+      } else {
+        return const_cast<char*>("FileCachePath must be set");
+      }
     } else if (!cfg_m->driver_factory->file_system()->IsDir(
         file_cache_path, &handler).is_true()) {
       return const_cast<char*>(
@@ -1136,7 +1202,7 @@ char* ps_merge_loc_conf(ngx_conf_t* cf, void* parent, void* child) {
   ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
       ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
 
-  if (cfg_s->server_context == NULL) {
+  if (ps_disabled(cfg_s)) {
     // Pagespeed options cannot be defined only in location blocks.  There must
     // be at least a single "pagespeed off" in the main block or a server
     // block.
@@ -1268,6 +1334,7 @@ ngx_int_t ps_decline_request(ngx_http_request_t* r) {
   ctx->driver->Cleanup();
   ctx->driver = NULL;
   ctx->location_field_set = false;
+  ctx->psol_vary_accept_only = false;
 
   // re init ctx
   ctx->html_rewrite = true;
@@ -1277,6 +1344,9 @@ ngx_int_t ps_decline_request(ngx_http_request_t* r) {
 
   r->count++;
   r->phase_handler++;
+
+  //restore read_event_handler to what it was in ps_async_wait_response
+  r->read_event_handler = ngx_http_block_reading;
   r->write_event_handler = ngx_http_core_run_phases;
   ngx_http_core_run_phases(r);
   ngx_http_run_posted_requests(r->connection);
@@ -1288,6 +1358,13 @@ ngx_int_t ps_async_wait_response(ngx_http_request_t* r) {
   CHECK(ctx != NULL);
 
   r->count++;
+  // While we wait for PSOL to complete an async operation, there is a chance
+  // that the underlying connection gets closed,  or a http/2 RST_STREAM is
+  // received before the async operation completes. In that case we don't want
+  // to continue processing this flow. So we override the requests's read event
+  // handler with one that will make nginx abort request processing and execute
+  // our cleanup handlers instead of resuming request processing.
+  r->read_event_handler = ngx_http_test_reading;
   r->write_event_handler = ngx_http_request_empty_handler;
   ps_set_buffered(r, true);
   // We don't need to add a timer here, as it will be set by nginx.
@@ -1574,10 +1651,12 @@ void ps_release_base_fetch(ps_request_ctx_t* ctx) {
 }
 
 // TODO(chaizhenhua): merge into NgxBaseFetch ctor
-void ps_create_base_fetch(ps_request_ctx_t* ctx,
+void ps_create_base_fetch(StringPiece url,
-                               RequestContextPtr request_context,
+                          ps_request_ctx_t* ctx,
-                               RequestHeaders* request_headers,
+                          RequestContextPtr request_context,
-                               NgxBaseFetchType type) {
+                          RequestHeaders* request_headers,
+                          NgxBaseFetchType type,
+                          const RewriteOptions* options) {
   CHECK(ctx->base_fetch == NULL) << "Pre-existing base fetch!";
 
   ngx_http_request_t* r = ctx->r;
@@ -1587,9 +1666,9 @@ void ps_create_base_fetch(ps_request_ctx_t* ctx,
   // it, and call Done() on the associated parent (Proxy or Resource) fetch. If
   // we fail before creating the associated fetch then we need to call Done() on
   // the BaseFetch ourselves.
-  ctx->base_fetch = new NgxBaseFetch(r, cfg_s->server_context,
+  ctx->base_fetch = new NgxBaseFetch(url, r, cfg_s->server_context, request_context,
-                                     request_context,
+                                     ctx->preserve_caching_headers, type,
-                                     ctx->preserve_caching_headers, type);
+                                     options);
   ctx->base_fetch->SetRequestHeadersTakingOwnership(request_headers);
 }
 
@@ -1630,7 +1709,7 @@ void ps_release_request_context(void* data) {
 RequestRouting::Response ps_route_request(ngx_http_request_t* r) {
   ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
 
-  if (!cfg_s->server_context->global_options()->enabled()) {
+  if (ps_disabled(cfg_s)) {
     // Not enabled for this server block.
     return RequestRouting::kPagespeedDisabled;
   }
@@ -1772,11 +1851,6 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     options = cfg_s->server_context->global_options();
   }
 
-  if (!options->enabled()) {
-    // Disabled via query params or request headers.
-    return NGX_DECLINED;
-  }
-
   request_context->set_options(options->ComputeHttpOptions());
 
   // ps_determine_options modified url, removing any ModPagespeedFoo=Bar query
@@ -1803,6 +1877,14 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
       response_category == RequestRouting::kGlobalAdmin ||
       response_category == RequestRouting::kCachePurge;
 
+  // Normally if we're disabled we won't handle any requests, but if we're in
+  // standby mode we do want to handle requests for .pagespeed. resources.
+  if (options->unplugged() ||
+      (!options->enabled() && !pagespeed_resource)) {
+    // Disabled via query params or request headers.
+    return NGX_DECLINED;
+  }
+
   if (!html_rewrite) {
     // create request ctx
     CHECK(ctx == NULL);
@@ -1811,6 +1893,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     ctx->r = r;
     ctx->html_rewrite = false;
     ctx->in_place = false;
+    ctx->follow_flushes = options->follow_flushes();
     ctx->preserve_caching_headers = kDontPreserveHeaders;
 
     // See build_context_for_request() in mod_instaweb.cc
@@ -1838,6 +1921,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     ctx->recorder = NULL;
     ctx->url_string = url_string;
     ctx->location_field_set = false;
+    ctx->psol_vary_accept_only = false;
 
     // Set up a cleanup handler on the request.
     ngx_http_cleanup_t* cleanup = ngx_http_cleanup_add(r, 0);
@@ -1853,16 +1937,17 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
   if (pagespeed_resource) {
     // TODO(jefftk): Set using_spdy appropriately.  See
     // ProxyInterface::ProxyRequestCallback
-    ps_create_base_fetch(ctx, request_context, request_headers.release(),
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
-                         kPageSpeedResource);
+                         request_headers.release(), kPageSpeedResource,
+                         options);
     ResourceFetch::Start(
         url,
         custom_options.release() /* null if there aren't custom options */,
-        false /* using_spdy */, cfg_s->server_context, ctx->base_fetch);
+        cfg_s->server_context, ctx->base_fetch);
     return ps_async_wait_response(r);
   } else if (is_an_admin_handler) {
-    ps_create_base_fetch(ctx, request_context, request_headers.release(),
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
-                         kAdminPage);
+                         request_headers.release(), kAdminPage, options);
     QueryParams query_params;
     query_params.ParseFromUrl(url);
 
@@ -1910,8 +1995,8 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
 
     if (options->domain_lawyer()->MapOriginUrl(
             url, &mapped_url, &host_header, &is_proxy) && is_proxy) {
-      ps_create_base_fetch(ctx, request_context, request_headers.release(),
+      ps_create_base_fetch(url.Spec(), ctx, request_context,
-                          kPageSpeedProxy);
+                           request_headers.release(), kPageSpeedProxy, options);
 
       RewriteDriver* driver;
       if (custom_options.get() == NULL) {
@@ -1934,9 +2019,9 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
 
   }
 
-  if (html_rewrite) {
+  if (html_rewrite && options->IsAllowed(url.Spec())) {
-    ps_create_base_fetch(ctx, request_context, request_headers.release(),
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
-                         kHtmlTransform);
+                         request_headers.release(), kHtmlTransform, options);
     // Do not store driver in request_context, it's not safe.
     RewriteDriver* driver;
 
@@ -1964,8 +2049,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
             url,
             cfg_s->server_context,
             options,
-            ctx->base_fetch,
+            ctx->base_fetch);
-            false /* requires_blink_cohort (no longer unused) */);
 
     // Will call StartParse etc.  The rewrite driver will take care of deleting
     // itself if necessary.
@@ -1980,8 +2064,8 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
   if (options->in_place_rewriting_enabled() &&
       options->enabled() &&
       options->IsAllowed(url.Spec())) {
-    ps_create_base_fetch(ctx, request_context, request_headers.release(),
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
-                         kIproLookup);
+                         request_headers.release(), kIproLookup, options);
 
     // Do not store driver in request_context, it's not safe.
     RewriteDriver* driver;
@@ -2030,7 +2114,6 @@ void ps_send_to_pagespeed(ngx_http_request_t* r,
   int last_buf = 0;
   for (cur = in; cur != NULL; cur = cur->next) {
     last_buf = cur->buf->last_buf;
-
     // Buffers are not really the last buffer until they've been through
     // pagespeed.
     cur->buf->last_buf = 0;
@@ -2055,6 +2138,19 @@ void ps_send_to_pagespeed(ngx_http_request_t* r,
         }
       }
     }
+    if (cur->buf->flush && ctx->follow_flushes) {
+      // Calling ctx->proxy_fetch->Flush(cfg_s->handler) will be a no-op here,
+      // unless we have follow_flushes or flush_html enabled. Note that PSOL
+      // might aggregate multiple flushes into 1, and actually flush a little bit
+      // later due to html parser state and earlier scheduled operations.
+      // Also, unless we also set the flush flag on the nginx buffers we won't
+      // actually flush.
+      // Note that too many flushes could harm optimization over larger html
+      // fragments as PSOL gets less context to work with, e.g. it can't combine
+      // two css files if a flush happens in between.
+      ctx->proxy_fetch->Flush(cfg_s->handler);
+    }
+
     // We're done with buffers as we pass them through, so mark them as sent as
     // we go.
     cur->buf->pos = cur->buf->last;
@@ -2063,9 +2159,6 @@ void ps_send_to_pagespeed(ngx_http_request_t* r,
   if (last_buf) {
     ctx->proxy_fetch->Done(true /* success */);
     ctx->proxy_fetch = NULL;  // ProxyFetch deletes itself on Done().
-  } else {
-    // TODO(jefftk): Decide whether Flush() is warranted here.
-    ctx->proxy_fetch->Flush(cfg_s->handler);
   }
 }
 
@@ -2155,6 +2248,38 @@ ngx_int_t ps_etag_header_filter(ngx_http_request_t* r) {
       break;
     }
   }
+
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+#if (NGX_HTTP_GZIP)
+  if (ctx && ctx->psol_vary_accept_only) {
+    r->gzip_vary = 0;
+  }
+#endif
+
+  if (ctx && ctx->recorder) {
+    ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
+    int s_maxage_sec =
+        cfg_s->server_context->global_options()->EffectiveInPlaceSMaxAgeSec();
+    if (s_maxage_sec != -1) {
+      GoogleString existing_cache_control;
+      bool cache_control_present = ps_get_cache_control(
+          r, &existing_cache_control);
+      GoogleString updated_cache_control;
+      if (ResponseHeaders::ApplySMaxAge(s_maxage_sec,
+                                        existing_cache_control,
+                                        &updated_cache_control)) {
+        // We're modifing the cache control header; save a copy first.
+        // NULL indicates that the header was not present.
+        ctx->recorder->SaveCacheControl(
+            cache_control_present ? existing_cache_control.c_str() : NULL);
+
+        // Replace the cache-control with our new s-maxage-including one.
+        ps_set_cache_control(r, string_piece_to_pool_string(
+            r->pool, updated_cache_control));
+      }
+    }
+  }
+
   return ngx_http_ef_next_header_filter(r);
 }
 
@@ -2166,8 +2291,7 @@ ngx_http_output_body_filter_pt ngx_http_next_body_filter;
 // nginx so it can send them out to the browser.
 ngx_int_t ps_html_rewrite_header_filter(ngx_http_request_t* r) {
   ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
-  if (cfg_s->server_context == NULL) {
+  if (ps_disabled(cfg_s)) {
-    // Pagespeed is on for some server block but not this one.
     return ngx_http_next_header_filter(r);
   }
 
@@ -2251,8 +2375,7 @@ ngx_int_t ps_html_rewrite_header_filter(ngx_http_request_t* r) {
 
 ngx_int_t ps_html_rewrite_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
   ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
-  if (cfg_s->server_context == NULL) {
+  if (ps_disabled(cfg_s)) {
-    // Pagespeed is on for some server block but not this one.
     return ngx_http_next_body_filter(r, in);
   }
 
@@ -2382,6 +2505,7 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
         cache_url.c_str());
     const SystemRewriteOptions* options = SystemRewriteOptions::DynamicCast(
         ctx->driver->options());
+
     RequestContextPtr request_context(
         cfg_s->server_context->NewRequestContext(r));
     request_context->set_options(options->ComputeHttpOptions());
@@ -2401,7 +2525,7 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
         server_context->http_cache(),
         server_context->statistics(),
         message_handler);
-    // set in memory flag for in place_body_filter
+    // set in memory flag for in_place_body_filter
     r->filter_need_in_memory = 1;
 
     // We don't have the response headers at all yet because we haven't yet gone
@@ -2478,14 +2602,14 @@ ngx_int_t send_out_headers_and_body(
 
   rc = ngx_http_send_header(r);
 
-  if (rc != NGX_OK) {
+  if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) {
-    return NGX_ERROR;
+    return rc;
   }
 
   // Send the body.
   ngx_chain_t* out;
   rc = string_piece_to_buffer_chain(
-      r->pool, output, &out, true /* send_last_buf */);
+      r->pool, output, &out, true /* send_last_buf */, false);
   if (rc == NGX_ERROR) {
     return NGX_ERROR;
   }
@@ -2775,8 +2899,8 @@ ngx_int_t ps_beacon_handler(ngx_http_request_t* r) {
 // supply it to the user.
 ngx_int_t ps_content_handler(ngx_http_request_t* r) {
   ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
-  if (cfg_s->server_context == NULL) {
+  if (ps_disabled(cfg_s)) {
-    // Pagespeed is on for some server block but not this one.
+    // Pagespeed is not on for this server block.
     return NGX_DECLINED;
   }
 
@@ -2916,7 +3040,7 @@ ngx_int_t ps_preaccess_handler(ngx_http_request_t* r) {
 ngx_int_t ps_etag_filter_init(ngx_conf_t* cf) {
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
-  if (cfg_m->driver_factory != NULL) {
+  if (cfg_m != NULL && cfg_m->driver_factory != NULL) {
     ngx_http_ef_next_header_filter = ngx_http_top_header_filter;
     ngx_http_top_header_filter = ps_etag_header_filter;
   }
@@ -2998,6 +3122,11 @@ ngx_int_t ps_init_module(ngx_cycle_t* cycle) {
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
 
+  // See https://github.com/pagespeed/ngx_pagespeed/issues/1220
+  if (cfg_m == NULL) {
+    return NGX_OK;
+  }
+
   ngx_http_core_main_conf_t* cmcf = static_cast<ngx_http_core_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cycle, ngx_http_core_module));
   ngx_http_core_srv_conf_t** cscfp = static_cast<ngx_http_core_srv_conf_t**>(
@@ -3051,7 +3180,7 @@ ngx_int_t ps_init_module(ngx_cycle_t* cycle) {
       return NGX_ERROR;
     }
 
-    cfg_m->driver_factory->LoggingInit(cycle->log);
+    cfg_m->driver_factory->LoggingInit(cycle->log, true);
     cfg_m->driver_factory->RootInit();
   } else {
     delete cfg_m->driver_factory;
@@ -3065,7 +3194,9 @@ void ps_exit_child_process(ngx_cycle_t* cycle) {
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
   NgxBaseFetch::Terminate();
-  cfg_m->driver_factory->ShutDown();
+  if (cfg_m != NULL && cfg_m->driver_factory != NULL) {
+    cfg_m->driver_factory->ShutDown();
+  }
 }
 
 // Called when nginx forks worker processes.  No threads should be started
@@ -3073,7 +3204,7 @@ void ps_exit_child_process(ngx_cycle_t* cycle) {
 ngx_int_t ps_init_child_process(ngx_cycle_t* cycle) {
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
-  if (cfg_m->driver_factory == NULL) {
+  if (cfg_m == NULL || cfg_m->driver_factory == NULL) {
     return NGX_OK;
   }
 
@@ -3083,7 +3214,7 @@ ngx_int_t ps_init_child_process(ngx_cycle_t* cycle) {
 
   // ChildInit() will initialise all ServerContexts, which we need to
   // create ProxyFetchFactories below
-  cfg_m->driver_factory->LoggingInit(cycle->log);
+  cfg_m->driver_factory->LoggingInit(cycle->log, true);
   cfg_m->driver_factory->ChildInit();
 
   ngx_http_core_main_conf_t* cmcf = static_cast<ngx_http_core_main_conf_t*>(

src/ngx_pagespeed.h

@@ -52,7 +52,7 @@ class InPlaceResourceRecorder;
 // NGX_DECLINED immediately unless send_last_buf.
 ngx_int_t string_piece_to_buffer_chain(
     ngx_pool_t* pool, StringPiece sp,
-    ngx_chain_t** link_ptr, bool send_last_buf);
+    ngx_chain_t** link_ptr, bool send_last_buf, bool send_flush);
 
 StringPiece str_to_string_piece(ngx_str_t s);
 
@@ -108,6 +108,8 @@ typedef struct {
   // we should mirror that when we write it back. nginx may absolutify
   // Location: headers that start with '/' without regarding X-Forwarded-Proto.
   bool location_field_set;
+  bool psol_vary_accept_only;
+  bool follow_flushes;
 } ps_request_ctx_t;
 
 ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r);

src/ngx_rewrite_driver_factory.cc

@@ -69,7 +69,6 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
     SystemThreadSystem* system_thread_system, StringPiece hostname, int port)
     : SystemRewriteDriverFactory(process_context, system_thread_system,
         NULL /* default shared memory runtime */, hostname, port),
-      main_conf_(NULL),
       threads_started_(false),
       ngx_message_handler_(
           new NgxMessageHandler(timer(), thread_system()->NewMutex())),
@@ -83,7 +82,7 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
       ngx_shared_circular_buffer_(NULL),
       hostname_(hostname.as_string()),
       port_(port),
-      process_script_variables_(false),
+      process_script_variables_mode_(ProcessScriptVariablesMode::kOff),
       process_script_variables_set_(false),
       shut_down_(false) {
   InitializeDefaultOptions();
@@ -148,10 +147,19 @@ NamedLockManager* NgxRewriteDriverFactory::DefaultLockManager() {
 
 RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
   NgxRewriteOptions* options = new NgxRewriteOptions(thread_system());
+  // TODO(jefftk): figure out why using SetDefaultRewriteLevel like
+  // mod_pagespeed does in mod_instaweb.cc:create_dir_config() isn't enough here
+  // -- if you use that instead then ngx_pagespeed doesn't actually end up
+  // defaulting CoreFilters.
+  // See: https://github.com/pagespeed/ngx_pagespeed/issues/1190
   options->SetRewriteLevel(RewriteOptions::kCoreFilters);
   return options;
 }
 
+RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptionsForQuery() {
+  return new NgxRewriteOptions(thread_system());
+}
+
 bool NgxRewriteDriverFactory::CheckResolver() {
   if (use_native_fetcher_ && resolver_ == NULL) {
     return false;
@@ -208,10 +216,19 @@ void NgxRewriteDriverFactory::StartThreads() {
   threads_started_ = true;
 }
 
-void NgxRewriteDriverFactory::LoggingInit(ngx_log_t* log) {
+void NgxRewriteDriverFactory::SetMainConf(NgxRewriteOptions* main_options) {
+  // Propagate process-scope options from the copy we had during nginx option
+  // parsing to our own.
+  if (main_options != NULL) {
+    default_options()->MergeOnlyProcessScopeOptions(*main_options);
+  }
+}
+
+void NgxRewriteDriverFactory::LoggingInit(
+    ngx_log_t* log, bool may_install_crash_handler) {
   log_ = log;
   net_instaweb::log_message_handler::Install(log);
-  if (install_crash_handler()) {
+  if (may_install_crash_handler && install_crash_handler()) {
     NgxMessageHandler::InstallCrashHandler(log);
   }
   ngx_message_handler_->set_log(log);
@@ -249,4 +266,23 @@ void NgxRewriteDriverFactory::InitStats(Statistics* statistics) {
   InPlaceResourceRecorder::InitStats(statistics);
 }
 
+void NgxRewriteDriverFactory::PrepareForkedProcess(const char* name) {
+  ngx_pid = ngx_getpid();  // Needed for logging to have the right PIDs.
+  SystemRewriteDriverFactory::PrepareForkedProcess(name);
+}
+
+void NgxRewriteDriverFactory::NameProcess(const char* name) {
+  SystemRewriteDriverFactory::NameProcess(name);
+
+  // Superclass set status with prctl.  Nginx has a helper function for setting
+  // argv[0] as well, so let's use that.  We'll show up as:
+  //
+  //    nginx: pagespeed $name
+
+  char name_for_setproctitle[32];
+  snprintf(name_for_setproctitle, sizeof(name_for_setproctitle),
+           "pagespeed %s", name);
+  ngx_setproctitle(name_for_setproctitle);
+}
+
 }  // namespace net_instaweb

src/ngx_rewrite_driver_factory.h

@@ -48,6 +48,12 @@ class SlowWorker;
 class Statistics;
 class SystemThreadSystem;
 
+enum ProcessScriptVariablesMode {
+  kOff,
+  kLegacyRestricted,
+  kAll
+};
+
 class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  public:
   // We take ownership of the thread system.
@@ -63,8 +69,9 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   virtual Timer* DefaultTimer();
   virtual NamedLockManager* DefaultLockManager();
   // Create a new RewriteOptions.  In this implementation it will be an
-  // NgxRewriteOptions.
+  // NgxRewriteOptions, and it will have CoreFilters explicitly set.
   virtual RewriteOptions* NewRewriteOptions();
+  virtual RewriteOptions* NewRewriteOptionsForQuery();
   virtual ServerContext* NewDecodingServerContext();
   // Check resolver configured or not.
   bool CheckResolver();
@@ -90,7 +97,7 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
     InitStats(statistics);
   }
 
-  void set_main_conf(NgxRewriteOptions* main_conf) {  main_conf_ = main_conf; }
+  void SetMainConf(NgxRewriteOptions* main_conf);
 
   void set_resolver(ngx_resolver_t* resolver) {
     resolver_ = resolver;
@@ -111,32 +118,32 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   void set_native_fetcher_max_keepalive_requests(int x) {
     native_fetcher_max_keepalive_requests_ = x;
   }
-  bool process_script_variables() {
+  ProcessScriptVariablesMode process_script_variables() {
-    return process_script_variables_;
+    return process_script_variables_mode_;
   }
 
-  void LoggingInit(ngx_log_t* log);
+  void LoggingInit(ngx_log_t* log, bool may_install_crash_handler);
 
   virtual void ShutDownMessageHandlers();
 
   virtual void SetCircularBuffer(SharedCircularBuffer* buffer);
 
-  bool SetProcessScriptVariables(bool process_script_variables) {
+  bool SetProcessScriptVariables(ProcessScriptVariablesMode mode) {
     if (!process_script_variables_set_) {
-      process_script_variables_ = process_script_variables;
+      process_script_variables_mode_ = mode;
       process_script_variables_set_ = true;
       return true;
     }
     return false;
   }
 
+  virtual void PrepareForkedProcess(const char* name);
+
+  virtual void NameProcess(const char* name);
+
  private:
   Timer* timer_;
 
-  // main_conf will have only options set in the main block.  It may be NULL,
-  // and we do not take ownership.
-  NgxRewriteOptions* main_conf_;
-
   bool threads_started_;
   NgxMessageHandler* ngx_message_handler_;
   NgxMessageHandler* ngx_html_parse_message_handler_;
@@ -157,7 +164,7 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
 
   GoogleString hostname_;
   int port_;
-  bool process_script_variables_;
+  ProcessScriptVariablesMode process_script_variables_mode_;
   bool process_script_variables_set_;
   bool shut_down_;
 

src/ngx_rewrite_options.cc

@@ -112,7 +112,7 @@ void NgxRewriteOptions::AddProperties() {
       false);
   add_ngx_option(
       "", &NgxRewriteOptions::global_statistics_path_, "ngsp",
-      kGlobalStatisticsPath, kProcessScope,
+      kGlobalStatisticsPath, kProcessScopeStrict,
       "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics",
       false);
   add_ngx_option(
@@ -127,7 +127,8 @@ void NgxRewriteOptions::AddProperties() {
       kServerScope, "Set the admin path.  Ex: /pagespeed_admin", false);
   add_ngx_option(
       "", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
-      kProcessScope, "Set the global admin path.  Ex: /pagespeed_global_admin",
+      kProcessScopeStrict,
+      "Set the global admin path.  Ex: /pagespeed_global_admin",
       false);
 
   MergeSubclassProperties(ngx_properties_);
@@ -180,10 +181,10 @@ RewriteOptions::OptionScope NgxRewriteOptions::GetOptionScope(
        it != all_options().end(); ++it) {
     RewriteOptions::OptionBase* option = *it;
     if (option->option_name() == option_name) {
-      // We treat kProcessScope as kProcessScopeStrict, failing to start if an
+      // We treat kLegacyProcessScope as kProcessScopeStrict, failing to start
-      // option is out of place.
+      // if an option is out of place.
-      return option->scope() == kProcessScope ? kProcessScopeStrict
+      return option->scope() == kLegacyProcessScope ? kProcessScopeStrict
-                                              : option->scope();
+                                                    : option->scope();
     }
   }
   return kDirectoryScope;
@@ -191,15 +192,18 @@ RewriteOptions::OptionScope NgxRewriteOptions::GetOptionScope(
 
 RewriteOptions::OptionSettingResult NgxRewriteOptions::ParseAndSetOptions0(
     StringPiece directive, GoogleString* msg, MessageHandler* handler) {
-  if (IsDirective(directive, "on")) {
+  EnabledEnum enabled;
-    set_enabled(RewriteOptions::kEnabledOn);
+  if (!ParseFromString(directive, &enabled)) {
-  } else if (IsDirective(directive, "off")) {
-    set_enabled(RewriteOptions::kEnabledOff);
-  } else if (IsDirective(directive, "unplugged")) {
-    set_enabled(RewriteOptions::kEnabledUnplugged);
-  } else {
     return RewriteOptions::kOptionNameUnknown;
   }
+  if (enabled == RewriteOptions::kEnabledOff) {
+    // In ngx_pagespeed, for historical reasons, we treat "off" as "unplugged".
+    // Also, "off" is deprecated and people should be using "standby" or
+    // "unplugged" now depending on which sense they want.  See comment on
+    // RewriteOptions::EnabledEnum.
+    enabled = RewriteOptions::kEnabledUnplugged;
+  }
+  set_enabled(enabled);
   return RewriteOptions::kOptionOk;
 }
 
@@ -258,7 +262,8 @@ const char* ps_error_string_for_option(
 const char* NgxRewriteOptions::ParseAndSetOptions(
     StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
     NgxRewriteDriverFactory* driver_factory,
-    RewriteOptions::OptionScope scope, ngx_conf_t* cf, bool compile_scripts) {
+    RewriteOptions::OptionScope scope, ngx_conf_t* cf,
+    ProcessScriptVariablesMode script_mode) {
   CHECK_GE(n_args, 1);
 
   StringPiece directive = args[0];
@@ -274,20 +279,43 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
         pool, directive, "cannot be set at this scope.");
   }
 
+  bool compile_scripts = false;
+
+  if (script_mode != ProcessScriptVariablesMode::kOff) {
+    // In the old mode we only allowed a few, so restrict to those.
+    compile_scripts =
+        StringCaseStartsWith(directive, "LoadFromFile") ||
+        StringCaseEqual(directive, "EnableFilters") ||
+        StringCaseEqual(directive, "DisableFilters") ||
+        StringCaseEqual(directive, "DownstreamCachePurgeLocationPrefix") ||
+        StringCaseEqual(directive, "DownstreamCachePurgeMethod") ||
+        StringCaseEqual(directive,
+                        "DownstreamCacheRewrittenPercentageThreshold") ||
+        StringCaseEqual(directive, "ShardDomain");
+    // In the new behaviour we also allow scripting of query- and directory-
+    // scoped options.
+    compile_scripts |=
+        script_mode == ProcessScriptVariablesMode::kAll &&
+        (GetOptionScope(directive) <= RewriteOptions::kDirectoryScope ||
+         (StringCaseEqual(directive, "Allow") ||
+          StringCaseEqual(directive, "BlockingRewriteRefererUrls") ||
+          StringCaseEqual(directive, "Disallow") ||
+          StringCaseEqual(directive, "DistributableFilters") ||
+          StringCaseEqual(directive, "Domain") ||
+          StringCaseEqual(directive, "ExperimentVariable") ||
+          StringCaseEqual(directive, "ExperimentSpec") ||
+          StringCaseEqual(directive, "ForbidFilters") ||
+          StringCaseEqual(directive, "RetainComment") ||
+          StringCaseEqual(directive, "CustomFetchHeader") ||
+          StringCaseEqual(directive, "MapOriginDomain") ||
+          StringCaseEqual(directive, "MapProxyDomain") ||
+          StringCaseEqual(directive, "MapRewriteDomain") ||
+          StringCaseEqual(directive, "UrlValuedAttribute") ||
+          StringCaseEqual(directive, "Library")));
+  }
+
   ScriptLine* script_line;
   script_line = NULL;
-  // Note that LoadFromFile should not be scriptable on wildcard hosts,
-  // as browsers might be able to manipulate its natural use-case: $http_host.
-  if (!StringCaseStartsWith(directive, "LoadFromFile") &&
-      !StringCaseEqual(directive, "EnableFilters") &&
-      !StringCaseEqual(directive, "DisableFilters") &&
-      !StringCaseEqual(directive, "DownstreamCachePurgeLocationPrefix") &&
-      !StringCaseEqual(directive, "DownstreamCachePurgeMethod") &&
-      !StringCaseEqual(directive,
-                       "DownstreamCacheRewrittenPercentageThreshold") &&
-      !StringCaseEqual(directive, "ShardDomain")){
-    compile_scripts = false;
-  }
 
   if (n_args == 1 && StringCaseEqual(directive, "ClearInheritedScripts")) {
     clear_inherited_scripts_ = true;
@@ -363,24 +391,23 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
       }
     } else if (StringCaseEqual("ProcessScriptVariables", args[0])) {
       if (scope == RewriteOptions::kProcessScopeStrict) {
-        if (StringCaseEqual(arg, "on")) {
+        ProcessScriptVariablesMode mode;
-          if (driver_factory->SetProcessScriptVariables(true)) {
+        if (StringCaseEqual(arg, "all")) {
-            result = RewriteOptions::kOptionOk;
+          mode = ProcessScriptVariablesMode::kAll;
-          } else {
+        } else if (StringCaseEqual(arg, "on")) {
-            return const_cast<char*>(
+          mode = ProcessScriptVariablesMode::kLegacyRestricted;
-                "pagespeed ProcessScriptVariables: can only be set once");
-          }
         } else if (StringCaseEqual(arg, "off")) {
-          if (driver_factory->SetProcessScriptVariables(false)) {
+          mode = ProcessScriptVariablesMode::kOff;
-            result = RewriteOptions::kOptionOk;
-          } else {
-            return const_cast<char*>(
-                "pagespeed ProcessScriptVariables: can only be set once");
-          }
         } else {
           return const_cast<char*>(
               "pagespeed ProcessScriptVariables: invalid value");
         }
+        if (driver_factory->SetProcessScriptVariables(mode)) {
+          result = RewriteOptions::kOptionOk;
+        } else {
+          return const_cast<char*>(
+              "pagespeed ProcessScriptVariables: can only be set once");
+        }
       } else {
         return const_cast<char*>(
             "ProcessScriptVariables is only allowed at the top level");
@@ -391,7 +418,7 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
         result = driver_factory->ParseAndSetOption1(
             directive,
             arg,
-            scope >= RewriteOptions::kProcessScope,
+            scope >= RewriteOptions::kLegacyProcessScope,
             &msg,
             handler);
       }
@@ -404,7 +431,7 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
           directive,
           args[1],
           args[2],
-          scope >= RewriteOptions::kProcessScope,
+          scope >= RewriteOptions::kLegacyProcessScope,
           &msg,
           handler);
     }
@@ -476,7 +503,7 @@ bool NgxRewriteOptions::ExecuteScriptVariables(
 
       const char* status = ParseAndSetOptions(args, script_line->n_args(),
           r->pool, handler, driver_factory, script_line->scope(), NULL /*cf*/,
-          false /*compile scripts*/);
+          ProcessScriptVariablesMode::kOff);
 
       if (status != NULL) {
         script_error = true;

src/ngx_rewrite_options.h

@@ -29,6 +29,8 @@ extern "C" {
 
 #include <vector>
 
+#include "ngx_rewrite_driver_factory.h"
+
 #include "net/instaweb/rewriter/public/rewrite_options.h"
 #include "pagespeed/kernel/base/message_handler.h"
 #include "pagespeed/kernel/base/ref_counted_ptr.h"
@@ -131,7 +133,7 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   const char* ParseAndSetOptions(
       StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
       NgxRewriteDriverFactory* driver_factory, OptionScope scope,
-      ngx_conf_t* cf, bool compile_scripts);
+      ngx_conf_t* cf, ProcessScriptVariablesMode script_mode);
   bool ExecuteScriptVariables(
       ngx_http_request_t* r, MessageHandler* handler,
       NgxRewriteDriverFactory* driver_factory);

src/ngx_server_context.cc

@@ -35,7 +35,8 @@ namespace net_instaweb {
 
 NgxServerContext::NgxServerContext(
     NgxRewriteDriverFactory* factory, StringPiece hostname, int port)
-    : SystemServerContext(factory, hostname, port) {
+    : SystemServerContext(factory, hostname, port),
+      ngx_http2_variable_index_(NGX_ERROR) {
 }
 
 NgxServerContext::~NgxServerContext() { }
@@ -70,11 +71,23 @@ SystemRequestContext* NgxServerContext::NewRequestContext(
     local_ip.len = 0;
   }
 
-  return new SystemRequestContext(thread_system()->NewMutex(),
+  SystemRequestContext* ctx = new SystemRequestContext(
-                                  timer(),
+      thread_system()->NewMutex(), timer(),
-                                  ps_determine_host(r),
+      ps_determine_host(r), local_port, str_to_string_piece(local_ip));
-                                  local_port,
+
-                                  str_to_string_piece(local_ip));
+  // See if http2 is in use.
+  if (ngx_http2_variable_index_ >= 0) {
+    ngx_http_variable_value_t* val =
+        ngx_http_get_indexed_variable(r, ngx_http2_variable_index_);
+    if (val != NULL && val->valid) {
+      StringPiece str_val(reinterpret_cast<char*>(val->data), val->len);
+      if (str_val == "h2" || str_val == "h2c") {
+        ctx->set_using_http2(true);
+      }
+    }
+  }
+
+  return ctx;
 }
 
 GoogleString NgxServerContext::FormatOption(StringPiece option_name,

src/ngx_server_context.h

@@ -58,8 +58,18 @@ class NgxServerContext : public SystemServerContext {
 
   virtual GoogleString FormatOption(StringPiece option_name, StringPiece args);
 
+  void set_ngx_http2_variable_index(ngx_int_t idx) {
+    ngx_http2_variable_index_ = idx;
+  }
+
+  ngx_int_t ngx_http2_variable_index() const {
+    return ngx_http2_variable_index_;
+  }
+
  private:
   NgxRewriteDriverFactory* ngx_factory_;
+  // what index the "http2" var is, or NGX_ERROR.
+  ngx_int_t ngx_http2_variable_index_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxServerContext);
 };

test/nginx_system_test.sh

@@ -31,22 +31,38 @@
 : ${NATIVE_FETCHER:?"Set NATIVE_FETCHER to off or on"}
 : ${PRIMARY_PORT:?"Set PRIMARY_PORT"}
 : ${SECONDARY_PORT:?"Set SECONDARY_PORT"}
-: ${RCPORT1:?"Set RCPORT1"}
+: ${CONTROLLER_PORT:?"Set CONTROLLER_PORT"}
-: ${RCPORT2:?"Set RCPORT2"}
+: ${RCPORT:?"Set RCPORT"}
-: ${RCPORT2:?"Set RCPORT3"}
-: ${RCPORT4:?"Set RCPORT4"}
-: ${RCPORT5:?"Set RCPORT5"}
-: ${RCPORT6:?"Set RCPORT6"}
-: ${RCPORT7:?"Set RCPORT7"}
 : ${MOD_PAGESPEED_DIR:?"Set MOD_PAGESPEED_DIR"}
 : ${NGINX_EXECUTABLE:?"Set NGINX_EXECUTABLE"}
 : ${PAGESPEED_TEST_HOST:?"Set PAGESPEED_TEST_HOST"}
 POSITION_AUX="${POSITION_AUX:-unset}"
+RUN_CONTROLLER_TEST="${RUN_CONTROLLER_TEST:-off}"
 
 PRIMARY_HOSTNAME="localhost:$PRIMARY_PORT"
 SECONDARY_HOSTNAME="localhost:$SECONDARY_PORT"
 
-SERVER_ROOT="$MOD_PAGESPEED_DIR/src/install/"
+this_dir="$( cd $(dirname "$0") && pwd)"
+echo this_dir=$this_dir
+TEST_TMP="$this_dir/tmp"
+rm -rf "$TEST_TMP"
+mkdir -p "$TEST_TMP"
+echo TEST_TMP=$TEST_TMP
+
+if [ -d "$MOD_PAGESPEED_DIR/src" ]; then
+  MOD_PAGESPEED_DIR+="/src"
+fi
+
+APACHE_DOC_SRC="$MOD_PAGESPEED_DIR/install/"
+SERVER_ROOT="$TEST_TMP/root"
+echo SERVER_ROOT=$SERVER_ROOT
+rm -rf "$SERVER_ROOT"
+mkdir -p "$SERVER_ROOT"
+export APACHE_DOC_ROOT="$SERVER_ROOT"
+
+mkdir -p "$APACHE_DOC_ROOT"
+make -f "$APACHE_DOC_SRC/Makefile.tests" setup_doc_root \
+  INSTALL_DATA_DIR="$APACHE_DOC_SRC"
 
 # We need check and check_not before we source SYSTEM_TEST_FILE that provides
 # them.
@@ -136,17 +152,19 @@ function fire_ab_load() {
   sleep 2
  }
 
-this_dir="$( cd $(dirname "$0") && pwd)"
-
 # stop nginx/valgrind
 killall -s KILL nginx
-# TODO(oschaaf): Fix waiting for valgrind on 32 bits systems.
 killall -s KILL memcheck-amd64-
-while pgrep nginx > /dev/null; do sleep 1; done
+killall -s KILL memcheck-x86-
-while pgrep memcheck > /dev/null; do sleep 1; done
+SECONDS=0
+while pgrep -x nginx >/dev/null || pgrep memcheck >/dev/null;do
+  if [ $SECONDS -gt 20 ]; then
+    echo "Old processes won't die" >&2
+    exit 1
+  fi
+  sleep 1
+done
 
-TEST_TMP="$this_dir/tmp"
-rm -r "$TEST_TMP"
 check_simple mkdir -p "$TEST_TMP"
 PROXY_CACHE="$TEST_TMP/proxycache"
 TMP_PROXY_CACHE="$TEST_TMP/tmpproxycache"
@@ -176,6 +194,12 @@ else
   RESOLVER=""
 fi
 
+if [ "$RUN_CONTROLLER_TEST" = "on" ]; then
+  CONTROLLER="pagespeed ExperimentalCentralControllerPort $CONTROLLER_PORT;"
+else
+  CONTROLLER=""
+fi
+
 # set up the config file for the test
 PAGESPEED_CONF="$TEST_TMP/pagespeed_test.conf"
 PAGESPEED_CONF_TEMPLATE="$this_dir/pagespeed_test.conf.template"
@@ -200,15 +224,10 @@ cat $PAGESPEED_CONF_TEMPLATE \
   | sed 's#@@SERVER_ROOT@@#'"$SERVER_ROOT"'#' \
   | sed 's#@@PRIMARY_PORT@@#'"$PRIMARY_PORT"'#' \
   | sed 's#@@SECONDARY_PORT@@#'"$SECONDARY_PORT"'#' \
+  | sed 's#@@CONTROLLER@@#'"$CONTROLLER"'#' \
   | sed 's#@@NATIVE_FETCHER@@#'"$NATIVE_FETCHER"'#' \
   | sed 's#@@RESOLVER@@#'"$RESOLVER"'#' \
-  | sed 's#@@RCPORT1@@#'"$RCPORT1"'#' \
+  | sed 's#@@RCPORT@@#'"$RCPORT"'#' \
-  | sed 's#@@RCPORT2@@#'"$RCPORT2"'#' \
-  | sed 's#@@RCPORT3@@#'"$RCPORT3"'#' \
-  | sed 's#@@RCPORT4@@#'"$RCPORT4"'#' \
-  | sed 's#@@RCPORT5@@#'"$RCPORT5"'#' \
-  | sed 's#@@RCPORT6@@#'"$RCPORT6"'#' \
-  | sed 's#@@RCPORT7@@#'"$RCPORT7"'#' \
   | sed 's#@@PAGESPEED_TEST_HOST@@#'"$PAGESPEED_TEST_HOST"'#' \
   >> $PAGESPEED_CONF
 # make sure we substituted all the variables
@@ -220,7 +239,7 @@ if $USE_VALGRIND; then
             --show-possibly-lost=no --log-file=$TEST_TMP/valgrind.log \
             --suppressions="$this_dir/valgrind.sup" \
       $NGINX_EXECUTABLE -c $PAGESPEED_CONF) & VALGRIND_PID=$!
-  trap "echo 'terminating valgrind!' && kill -s sigterm $VALGRIND_PID" EXIT
+  trap "echo 'terminating valgrind!' && kill -s TERM $VALGRIND_PID" EXIT
   echo "Wait until nginx is ready to accept connections"
   while ! curl -I "http://$PRIMARY_HOSTNAME/mod_pagespeed_example/" 2>/dev/null; do
       sleep 0.1;
@@ -275,7 +294,7 @@ else
   if $USE_VALGRIND; then
     # Clear valgrind trap
     trap - EXIT
-    echo "To end valgrind, run 'kill -s quit $VALGRIND_PID'"
+    echo "To end valgrind, run 'kill -s TERM $VALGRIND_PID'"
   fi
   echo "Not running tests; commence manual testing"
   exit 4
@@ -289,8 +308,16 @@ CACHE_PURGE_METHODS="PURGE GET"
 
 SERVER_NAME=nginx
 
+# TODO(cheesy): add an nginx controller test variant.  Actually requires some
+# thought as we don't currently have a multi-phase test flow in ngx_pagespeeed,
+# as far as I can tell, and we really have to update the config and restart
+# since the controller is a global setting.
+RUN_CONTROLLER_TEST=${RUN_CONTROLLER_TEST:-off}
+
 # run generic system tests
-SYSTEM_TEST_FILE="$MOD_PAGESPEED_DIR/src/pagespeed/system/system_test.sh"
+PAGESPEED_DIR="$MOD_PAGESPEED_DIR/pagespeed"
+SYSTEM_TEST_FILE="$PAGESPEED_DIR/system/system_test.sh"
+REMOTE_CONFIG_TEST_FILE="$PAGESPEED_DIR/system/remote_config_test.sh"
 
 if [ ! -e "$SYSTEM_TEST_FILE" ] ; then
   echo "Not finding $SYSTEM_TEST_FILE -- is mod_pagespeed not in a parallel"
@@ -303,7 +330,8 @@ BEACON_HANDLER="ngx_pagespeed_beacon"
 STATISTICS_HANDLER="ngx_pagespeed_statistics"
 GLOBAL_STATISTICS_HANDLER="ngx_pagespeed_global_statistics"
 MESSAGES_HANDLER="ngx_pagespeed_message"
-STATISTICS_URL=http://$PRIMARY_HOSTNAME/ngx_pagespeed_statistics
+STATISTICS_URL=http://$PRIMARY_HOSTNAME/$STATISTICS_HANDLER
+GLOBAL_STATISTICS_URL=http://$PRIMARY_HOSTNAME/$GLOBAL_STATISTICS_HANDLER
 
 # An expected failure can be indicated like: "~In-place resource optimization~"
 PAGESPEED_EXPECTED_FAILURES="
@@ -332,7 +360,6 @@ if $USE_VALGRIND; then
 ~combine_css Maximum size of combined CSS.~
 ~prioritize_critical_css~
 ~prioritize_critical_css Able to read POST data from temp file.~
-~IPRO flow uses cache as expected.~
 ~IPRO flow doesn't copy uncacheable resources multiple times.~
 ~inline_unauthorized_resources allows unauthorized css selectors~
 "
@@ -343,6 +370,14 @@ fi
 set -- "$PRIMARY_HOSTNAME"
 source $SYSTEM_TEST_FILE
 
+# If we were only asked to run a single test, it ran in SYSTEM_TEST_FILE, so
+# we're done now and should exit.
+if [ -n ${TEST_TO_RUN+x} ]; then
+  check_failures_and_exit
+fi
+
+source $REMOTE_CONFIG_TEST_FILE
+
 # nginx-specific system tests
 
 start_test Test pagespeed directive inside if block inside location block.
@@ -552,7 +587,7 @@ fetch_until "$URL" "fgrep -c .pagespeed." 1 --header=Host:www.google.com
 
 # If this accepts the Host header and fetches from google.com it will fail with
 # a 404.  Instead it should use a loopback fetch and succeed.
-URL="$HOSTNAME/mod_pagespeed_example/.pagespeed.ce.8CfGBvwDhH.css"
+URL="$HOSTNAME/mod_pagespeed_example/styles/big.css.pagespeed.ce.8CfGBvwDhH.css"
 check wget -O /dev/null --header=Host:www.google.com "$URL"
 
 start_test statistics load
@@ -578,30 +613,93 @@ check test $(scrape_stat image_rewrite_total_original_bytes) -ge 10000
 # happens both before and after.
 start_test "Reload config"
 
+function find_exactly_once {
+  test $(grep -c "$1") -eq 1
+}
+
+if [ "$RUN_CONTROLLER_TEST" = "on" ]; then
+  function check_process_names() {
+    if ! $USE_VALGRIND; then
+      # There should be one babysitter and controller running.  Under valgrind
+      # process labels are confused, so skip the check then.
+
+      running=$(ps auxww | grep 'ngin[x]')
+      check_from "$running" find_exactly_once "nginx: pagespeed babysitter"
+      check_from "$running" find_exactly_once "nginx: pagespeed controller"
+    fi
+  }
+  check_process_names
+fi
+
 # Fire up some heavy load if ab is available to test a stressed reload.
 # TODO(oschaaf): make sure we wait for the new worker to get ready to accept
 # requests.
 fire_ab_load
 
-check wget $EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css \
+URL="$EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css"
-  -O /dev/null
+check wget "$URL" -O /dev/null
 check_simple "$NGINX_EXECUTABLE" -s reload -c "$PAGESPEED_CONF"
+
 # Wait for the new worker process with the new configuration to get ready, or
 # else the sudden reset of the shared mem statistics/cache might catch upcoming
 # tests unaware.
-while [ $(scrape_stat image_rewrite_total_original_bytes) -gt 0 ]
+function wait_for_new_worker() {
-do
+  while [ $(scrape_stat image_rewrite_total_original_bytes) -gt 0 ]; do
     echo "Waiting for new worker to get ready..."
     sleep .1
-done
+  done
-
+}
-check wget $EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css \
+wait_for_new_worker
-  -O /dev/null
+check wget "$URL" -O /dev/null
 if [ "$AB_PID" != "0" ]; then
     echo "Kill ab (pid: $AB_PID)"
     kill -s KILL $AB_PID &>/dev/null || true
 fi
 
+# There should still be just one babysitter and controller running.
+if [ "$RUN_CONTROLLER_TEST" = "on" ]; then
+  check_process_names
+
+  check grep "Writing a byte to a pipe to tell the old controller to exit." \
+    $ERROR_LOG
+  check grep "Root process is starting a new controller; shutting down." \
+    $ERROR_LOG
+fi
+
+start_test "Shared memory checkpointing"
+
+# We do two tests here:
+# 1. Metadata cache: check that an IPRO'd image is still fully optimized after a
+#    restart.
+# 2. Property cache: prioritize critical CSS persists beacon results across
+#    restarts.
+
+IPRO_URL="$EXAMPLE_ROOT/images/Cuppa.png"
+
+# Checkpoint beacon results to disk if we haven't already.
+check wget "$IPRO_URL" -O /dev/null
+test_prioritize_critical_css
+sleep 2
+check wget "$IPRO_URL" -O /dev/null
+test_prioritize_critical_css
+
+# Reload nginx again.
+check_simple "$NGINX_EXECUTABLE" -s reload -c "$PAGESPEED_CONF"
+wait_for_new_worker
+
+# The image should be fully optimized.
+OUT=$($CURL -sS -D- -o/dev/null "$IPRO_URL")
+check_from "$OUT" grep ^X-Original-Content-Length:
+check_from "$OUT" grep ^Content-Length:
+content_length=$(echo "$OUT" | grep ^Content-Length: | grep -o [0-9]*)
+original_content_length=$(echo "$OUT" | grep ^X-Original-Content-Length: \
+  | grep -o [0-9]*)
+check [ "$original_content_length" -gt "$content_length" ]
+
+# The beacon responses are stored in the metadata cache, so this can only pass
+# if we persisted the metadata across restarts.
+test_prioritize_critical_css_final
+
 # This is dependent upon having a beacon handler.
 test_filter add_instrumentation beacons load.
 
@@ -646,14 +744,14 @@ check touch "$SECONDARY_CACHE/cache.flush"
 check touch "$IPRO_CACHE/cache.flush"
 sleep 1
 
-CACHE_TESTING_DIR="$SERVER_ROOT/mod_pagespeed_test/cache_flush/"
+CACHE_TESTING_DIR="$SERVER_ROOT/cache_flush/"
 CACHE_TESTING_TMPDIR="$CACHE_TESTING_DIR/$$"
 mkdir "$CACHE_TESTING_TMPDIR"
 cp "$CACHE_TESTING_DIR/cache_flush_test.html" "$CACHE_TESTING_TMPDIR/"
 CSS_FILE="$CACHE_TESTING_TMPDIR/update.css"
 echo ".class myclass { color: $COLOR0; }" > "$CSS_FILE"
 
-URL_PATH="mod_pagespeed_test/cache_flush/$$/cache_flush_test.html"
+URL_PATH="cache_flush/$$/cache_flush_test.html"
 
 URL="$SECONDARY_HOSTNAME/$URL_PATH"
 CACHE_A="--header=Host:cache_a.example.com"
@@ -1097,7 +1195,7 @@ OUT=$($WGET_DUMP --header=Host:response-header-disable.example.com $URL)
 check_not_from "$OUT" egrep -qi 'addInstrumentationInit'
 
 # TODO(jmaessen, jefftk): Port proxying tests, which rely on pointing a
-# MapProxyDomain construct at a static server.  Perhaps localhost:8050 will
+# MapProxyDomain construct at a static server.  Perhaps $HOSTNAME will
 # serve, but the tests need to use different urls then.  For mod_pagespeed these
 # tests immediately precede the "scrape_secondary_stat" definition in
 # system_test.sh.
@@ -1241,13 +1339,48 @@ check_from "$OUT" fgrep -qi '404'
 MATCHES=$(echo "$OUT" | grep -c "Cache-Control: override") || true
 check [ $MATCHES -eq 1 ]
 
+start_test Custom 404 does not crash.
+URL=http://custom404.example.com/mod_pagespeed_test/
+URL+=A.doesnotexist.css.pagespeed.cf.0.css
+# The 404 response makes wget exit with an error code, which we ignore.
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1) || true
+# We ignored the exit code, check if we got a 404 response.
+check_from "$OUT" fgrep -qi '404'
+
+start_test Single Vary: Accept-Encoding header in IPRO flow
+URL=http://psol-vary.example.com/mod_pagespeed_example/styles/index_style.css
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
+# First hit will be recorded and passed on untouched
+MATCHES=$(echo "$OUT" | grep -c "Vary: Accept-Encoding") || true
+check [ $MATCHES -eq 1 ]
+
+# Fetch until we get a fully optimized response
+http_proxy=$SECONDARY_HOSTNAME \
+  fetch_until $URL "fgrep -c W/\"PSA" 1 --save-headers
+
+# Test the optimized response.
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
+MATCHES=$(echo "$OUT" | grep -c "Vary: Accept-Encoding") || true
+check [ $MATCHES -eq 1 ]
+
+start_test Follow flushes can be turned off.
+echo "Check that FollowFlushes off outputs a single chunk"
+check_flushing noflush 5.4 1
+
+start_test Special responses from php are handled OK.
+URL="http://special-response.example.com/A.foo.css.pagespeed.cf.0.css"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -S --content-on-error $URL 2>&1) || true
+check_from "$OUT" fgrep -qi '404'
+check_from "$OUT" fgrep -q "PHP with a call to flush"
+
 start_test Shutting down.
 
 # Fire up some heavy load if ab is available to test a stressed shutdown
 fire_ab_load
 
 if $USE_VALGRIND; then
-    kill -s quit $VALGRIND_PID
+    # SIGQUIT requests a graceful shutdown.
+    kill -s QUIT $VALGRIND_PID
     while pgrep memcheck > /dev/null; do sleep 1; done
     # Clear the previously set trap, we don't need it anymore.
     trap - EXIT
@@ -1256,12 +1389,12 @@ if $USE_VALGRIND; then
     check_not [ -s "$TEST_TMP/valgrind.log" ]
 else
     check_simple "$NGINX_EXECUTABLE" -s quit -c "$PAGESPEED_CONF"
-    while pgrep nginx > /dev/null; do sleep 1; done
+    while pgrep -x nginx > /dev/null; do sleep 1; done
 fi
 
 if [ "$AB_PID" != "0" ]; then
     echo "Kill ab (pid: $AB_PID)"
-    killall -s KILL $AB_PID &>/dev/null || true
+    kill -s KILL $AB_PID &>/dev/null || true
 fi
 
 start_test Logged output looks healthy.
@@ -1288,6 +1421,8 @@ OUT=$(cat "$ERROR_LOG" \
     | grep -v "\\[warn\\].*Resource based on.*ngx_pagespeed_statistics.*" \
     | grep -v "\\[warn\\].*Canceling 1 functions on sequence Shutdown.*" \
     | grep -v "\\[warn\\].*using uninitialized.*" \
+    | grep -v "\\[warn\\].*Controller process .* exited with wait status 9" \
+    | grep -v "\\[warn\\].*Controller process .* exited with wait status 15" \
     | grep -v "\\[error\\].*BadName*" \
     | grep -v "\\[error\\].*/mod_pagespeed/bad*" \
     | grep -v "\\[error\\].*doesnotexist.css.*" \
@@ -1296,7 +1431,7 @@ OUT=$(cat "$ERROR_LOG" \
     | grep -v "\\[error\\].*forbidden.example.com*" \
     | grep -v "\\[error\\].*custom-paths.example.com*" \
     | grep -v "\\[error\\].*bogus_format*" \
-    | grep -v "\\[error\\].*src/install/foo*" \
+    | grep -v "\\[error\\].*/install/foo*" \
     | grep -v "\\[error\\].*recv() failed*" \
     | grep -v "\\[error\\].*send() failed*" \
     | grep -v "\\[error\\].*Invalid url requested: js_defer.js.*" \
@@ -1316,6 +1451,12 @@ OUT=$(cat "$ERROR_LOG" \
     | grep -v "\\[warn\\].*remote\.cfg.*" \
     | grep -v "\\[warn\\].*end token not received.*" \
     | grep -v "\\[warn\\].*failed to hook next event.*" \
+    | grep -v "\\[warn\\].*Fetch timed out:.*" \
+    | grep -v "\\[warn\\].*Controller process .* exited with status code.*" \
+    | grep -v "\\[warn\\].*Rewrite.*failed.*.pagespeed....0.foo.*" \
+    | grep -v "\\[warn\\].*A.blue.css.*but cannot access the original.*" \
+    | grep -v "\\[warn\\].*Adding function to sequence.*" \
+    | grep -v "\\[warn\\].*special-response.*foo.css.*but cannot access the original.*" \
     || true)
 
 check [ -z "$OUT" ]

test/pagespeed_test.conf.template

@@ -30,7 +30,7 @@ http {
   proxy_cache_path "@@PROXY_CACHE@@"  levels=1:2   keys_zone=htmlcache:60m inactive=90m  max_size=50m;
   proxy_temp_path "@@TMP_PROXY_CACHE@@";
 
-  pagespeed ProcessScriptVariables on;
+  pagespeed ProcessScriptVariables all;
   pagespeed StatisticsPath /ngx_pagespeed_statistics;
   pagespeed GlobalStatisticsPath /ngx_pagespeed_global_statistics;
   pagespeed ConsolePath /pagespeed_console;
@@ -63,6 +63,12 @@ http {
   pagespeed CreateSharedMemoryMetadataCache "@@SHM_CACHE@@" 8192;
   pagespeed BlockingRewriteKey psatest;
 
+  # We need to checkpoint more frequently than usual so that our checkpointing
+  # test will definitly have seen one of these intervals and so not be flaky.
+  # This needs to have a 1s interval because the checkpointing test has a 2s
+  # sleep.
+  pagespeed ShmMetadataCacheCheckpointIntervalSec 1;
+
   # CriticalImagesBeaconEnabled is now on by default, but we disable in testing.
   # With this option enabled, the inline image system test will currently fail.
   # When critical image beaconing is enabled, only critical images are inlined.
@@ -79,6 +85,9 @@ http {
   pagespeed StatisticsLogging on;
   pagespeed LogDir "@@TEST_TMP@@/logdir";
 
+  # Expanded to CentralControllerPort directive when RUN_CONTROLLER_TEST=on
+  @@CONTROLLER@@
+
   server {
     # Sets up a logical home-page server on
     # max-cacheable-content-length.example.com.  This server is only used to
@@ -95,6 +104,70 @@ http {
     pagespeed MaxCacheableContentLength 85;
   }
 
+  # Test how load from file handles maximum sizes.
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name lff-large-files.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed MaxCacheableContentLength 4096;
+    pagespeed LoadFromFile
+      "http://lff-large-files.example.com/"
+      "@@SERVER_ROOT@@/";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name lff-large-files-no-fallback.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed MaxCacheableContentLength 4096;
+    # http://lff-large-files-no-fallback.example.com/foo.css won't load, because
+    # we've configured a LoadFromFile pattern that doesn't match the path the
+    # webserver would take.
+    pagespeed LoadFromFile
+      "http://lff-large-files-no-fallback.example.com/"
+      "@@SERVER_ROOT@@/mod_pagespeed_example/styles/";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name pagespeed-off.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed off;
+    pagespeed EnableFilters collapse_whitespace;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name pagespeed-on.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed on;
+    pagespeed EnableFilters collapse_whitespace;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name pagespeed-standby.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed standby;
+    pagespeed EnableFilters collapse_whitespace;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name pagespeed-unplugged.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed unplugged;
+    pagespeed EnableFilters collapse_whitespace;
+  }
+
   pagespeed UseNativeFetcher "@@NATIVE_FETCHER@@";
   @@RESOLVER@@
 
@@ -137,7 +210,7 @@ http {
     # inline_images (ii), defer_javascript (dj), webp (jw) and lossless_webp
     # (ws).
     if ($http_user_agent ~*
-        "Chrome/[2][3-9]+\.|Chrome/[[3-9][0-9]+\.|Chrome/[0-9]{3,}\.") {
+        "Chrome/[2][3-9]+\.|Chrome/[3-9][0-9]+\.|Chrome/[0-9]{3,}\.") {
       set $ps_capability_list "ll,ii,dj,jw,ws:";
     }
     # Cache-fragment 3: This fragment contains (a) Desktop User-Agents that
@@ -195,7 +268,7 @@ http {
 
     # Block 6: Location block with proxy_cache directives.
     location /mod_pagespeed_test/cachable_rewritten_html/ {
-      # 1: Upstream PageSpeed server is running at localhost:8050.
+      # 1: Upstream PageSpeed server is running at localhost:@@PRIMARY_PORT@@.
       proxy_pass http://localhost:@@PRIMARY_PORT@@;
       # 2: Use htmlcache as the zone for caching.
       proxy_cache htmlcache;
@@ -335,6 +408,29 @@ http {
       "id=1;percent=100;matches_device_type=mobile;enable=recompress_images";
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name experiment.ajax.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel CoreFilters;
+    pagespeed DisableFilters add_instrumentation;
+    pagespeed JsInlineMaxBytes 1;
+    pagespeed RunExperiment on;
+    pagespeed UseAnalyticsJs false;
+    pagespeed ExperimentSpec "id=1;percent=100;level=CoreFilters;enable=collapse_whitespace;options=JsInlineMaxBytes=1";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name ajax.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel CoreFilters;
+    pagespeed EnableFilters add_instrumentation;
+    pagespeed JsInlineMaxBytes 1;
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -564,6 +660,10 @@ http {
     pagespeed UrlValuedAttribute img alt-src Image;
     pagespeed UrlValuedAttribute video alt-a Image;
     pagespeed UrlValuedAttribute video alt-b Image;
+    pagespeed UrlValuedAttribute link data-stylesheet Stylesheet;
+    pagespeed UrlValuedAttribute span data-stylesheet-a Stylesheet;
+    pagespeed UrlValuedAttribute span data-stylesheet-b Stylesheet;
+    pagespeed UrlValuedAttribute span data-stylesheet-c Stylesheet;
 
     # Also test that we can redefine spec-defined attributes.
     pagespeed UrlValuedAttribute blockquote cite Image;
@@ -703,7 +803,44 @@ http {
     pagespeed EnableFilters rewrite_images,rewrite_css;
     pagespeed EnableFilters convert_to_webp_lossless;
     pagespeed EnableFilters in_place_optimize_for_browser;
+    pagespeed JpegRecompressionQuality 75;
+    pagespeed WebpRecompressionQuality 70;
     pagespeed InPlaceResourceOptimization on;
+    pagespeed AllowVaryOn "Accept";
+    pagespeed FileCachePath "@@IPRO_CACHE@@";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name ipro-for-browser-vary-on-auto.example.com;
+    root "@@SERVER_ROOT@@/mod_pagespeed_example";
+    pagespeed EnableFilters rewrite_images,rewrite_css;
+    pagespeed EnableFilters convert_to_webp_animated;
+    pagespeed EnableFilters convert_to_webp_lossless;
+    pagespeed EnableFilters in_place_optimize_for_browser;
+    pagespeed InPlaceResourceOptimization on;
+    # pagespeed AllowVaryOn "Accept";  # Default is "Auto".
+    pagespeed ImageRecompressionQuality 90;
+    pagespeed JpegRecompressionQuality 75;
+    pagespeed JpegRecompressionQualityForSmallScreens 55;
+    pagespeed JpegQualityForSaveData 35;
+    pagespeed WebpRecompressionQuality 70;
+    pagespeed WebpRecompressionQualityForSmallScreens 50;
+    pagespeed WebpQualityForSaveData 30;
+    pagespeed WebpAnimatedRecompressionQuality 60;
+    pagespeed FileCachePath "@@IPRO_CACHE@@";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name ipro-for-browser-vary-on-none.example.com;
+    root "@@SERVER_ROOT@@/mod_pagespeed_example";
+    pagespeed EnableFilters rewrite_images,in_place_optimize_for_browser;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed AllowVaryOn "None";
+    pagespeed ImageRecompressionQuality 75;
     pagespeed FileCachePath "@@IPRO_CACHE@@";
   }
 
@@ -771,7 +908,7 @@ http {
     server_name www.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@;
+    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
@@ -785,7 +922,7 @@ http {
     server_name origin.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@;
+    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
@@ -799,7 +936,7 @@ http {
     server_name cdn.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@;
+    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
@@ -967,6 +1104,14 @@ http {
     }
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name custom404.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    error_page 404 /404.html;
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -1135,7 +1280,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT1@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/standard";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1143,7 +1288,8 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-partially-invalid.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT2@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl
+              "http://127.0.0.1:@@RCPORT@@/partly-invalid";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1151,7 +1297,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-invalid.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT3@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/invalid";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1159,7 +1305,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-failed-fetch.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT5@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/fail-future";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1167,7 +1313,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-slow-fetch.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT6@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/timeout";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1175,7 +1321,41 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-experiment.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT7@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/experiment";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-slightly-slow-fetch.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl
+              "http://127.0.0.1:@@RCPORT@@/slightly-slow";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-slightly-slow-expired-fetch.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/slow-expired";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-forbidden.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/forbidden";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-initially-forbidden.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl
+              "http://127.0.0.1:@@RCPORT@@/forbidden-once";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
 
@@ -1204,6 +1384,17 @@ http {
     add_header 'X-Accel-Redirect' 'blablabla';
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name map-static-domain.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed MapRewriteDomain http://static-cdn.example.com
+                               http://map-static-domain.example.com;
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters defer_javascript,rewrite_domains;
+  }
+
   # Proxy + IPRO a gzip'd file for testing Issue 896.
   server {
     listen @@SECONDARY_PORT@@;
@@ -1221,6 +1412,44 @@ http {
     }
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name compressedcache.example.com;
+    pagespeed on;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed EnableFilters rewrite_css;
+    pagespeed HttpCacheCompressionLevel 9;
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name uncompressedcache.example.com;
+    pagespeed on;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed EnableFilters rewrite_css;
+    pagespeed HttpCacheCompressionLevel 0;
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -1283,7 +1512,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name proxy-post-origin.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
   }
 
   server {
@@ -1301,7 +1530,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name script-filters.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
     set $filters "";
     set $domain_shards "cdn1.example.com,cdn2.example.com";
     if ($http_X_Script) {
@@ -1323,7 +1552,7 @@ http {
     pagespeed EnableCachePurge on;
 
     pagespeed PurgeMethod PURGE;
-    root "@@SERVER_ROOT@@/mod_pagespeed_test/purge";
+    root "@@SERVER_ROOT@@/purge";
     pagespeed FileCachePath "@@FILE_CACHE@@_purge";
     pagespeed DisableFilters add_instrumentation;
     pagespeed RewriteLevel PassThrough;
@@ -1339,7 +1568,7 @@ http {
     pagespeed EnableCachePurge on;
 
     pagespeed PurgeMethod PURGE;
-    root "@@SERVER_ROOT@@/mod_pagespeed_test/purge";
+    root "@@SERVER_ROOT@@/purge";
     pagespeed FileCachePath "@@FILE_CACHE@@_dir_on";
     pagespeed DisableFilters add_instrumentation;
     pagespeed RewriteLevel PassThrough;
@@ -1357,7 +1586,7 @@ http {
     pagespeed InPlaceResourceOptimization off;
     pagespeed off;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
     location /mod_pagespeed_test/nostore {
       add_header "Cache-Control" "max-age=12345";
       add_header "Cache-Control" "public, no-store";
@@ -1372,7 +1601,7 @@ http {
     pagespeed InPlaceResourceOptimization off;
     pagespeed off;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
     location / {
       add_header "PageSpeedFilters" "add_instrumentation";
     }
@@ -1385,7 +1614,7 @@ http {
     pagespeed InPlaceResourceOptimization off;
     pagespeed off;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
     location / {
       add_header "PageSpeed" "off";
     }
@@ -1463,7 +1692,118 @@ http {
     pagespeed GlobalAdminDomains
      Allow everything-explicitly-allowed.example.com;
   }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name psol-vary.example.com;
+    pagespeed on;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name flush.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed RewriteDeadlinePerFlushMs 1;
 
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name noflush.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed RewriteDeadlinePerFlushMs 1;
+    pagespeed FollowFlushes off;
+
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name special-response.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    error_page 404 /mod_pagespeed_test/php_withflush.php;
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+    }
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name debug-filters.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters debug;
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name broken-fetch.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@/broken-fetch";
+
+    # Set up a fetch proxy that will 404 every request.
+    pagespeed FetchProxy "brokenfetch.example.com:1111";
+
+    # Prevent loopback fetch by explicitly authorizing the domain.  Loopback
+    # fetches would work, and for this test, we're trying to have fetches fail,
+    # so they are instead picked up by the ipro recorder.
+    pagespeed Domain http://broken-fetch.example.com;
+    pagespeed InPlaceResourceOptimization on;
+
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_javascript;
+    pagespeed DisableFilters add_instrumentation;
+    pagespeed CriticalImagesBeaconEnabled false;
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name image-rewrite-with-flush.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed UrlValuedAttribute li data-thumb image;
+    pagespeed EnableFilters core;
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
   server {
     listen @@PRIMARY_PORT@@;
     listen [::]:@@PRIMARY_PORT@@;
@@ -1477,6 +1817,14 @@ http {
     }
 
     location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" {
+      # TODO(oschaaf): The following breaks nginx starting up, as the bad header
+      # value gets rejected during configuration parsing. However, we can't easily
+      # build a test for that right now.
+      # pagespeed AddResourceHeader "X-Fo\n\no" "Bar";
+      # pagespeed AddResourceHeader "X-Foo" "Ba\nr";
+
+      # For testing that AddResourceHeader gets passed through
+      pagespeed AddResourceHeader "X-Foo" "Bar";
       add_header "" "";
     }
 
@@ -1528,6 +1876,8 @@ http {
     pagespeed Library 43 1o978_K0_LNE5_ystNklf
       http://www.modpagespeed.com/rewrite_javascript.js;
     pagespeed RetainComment " google_ad_section*";
+    pagespeed PermitIdsForCssCombining "allowed-to-combine-1-*";
+    pagespeed PermitIdsForCssCombining "allowed-to-combine-2-*";
 
     # Test proxying of non-.pagespeed. resources.
     pagespeed MapProxyDomain http://localhost:@@PRIMARY_PORT@@/modpagespeed_http
@@ -1655,6 +2005,53 @@ http {
       expires 5m;
     }
 
+    location /mod_pagespeed_test/example.json {
+      expires 10m;
+    }
+
+    location /mod_pagespeed_test/ipro/cc200/ {
+      add_header "Cache-Control" "max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200p/ {
+      add_header "Cache-Control" "private, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200nc/ {
+      add_header "Cache-Control" "no-cache, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200ns/ {
+      add_header "Cache-Control" "no-store, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200nt/ {
+      add_header "Cache-Control" "no-transform, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma5/ {
+      add_header "Cache-Control" "s-maxage=5, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50/ {
+      add_header "Cache-Control" "s-maxage=50, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50nsp/ {
+      add_header "Cache-Control" "s-maxage=50,max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc9/ {
+      add_header "Cache-Control" "max-age=9";
+    }
+    location /mod_pagespeed_test/ipro/cc200cc9/ {
+      add_header "Cache-Control" "max-age=200, max-age=9";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50sma5/ {
+      add_header "Cache-Control" "max-age=200, s-maxage=50, s-maxage=5";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50cc9/ {
+      add_header "Cache-Control" "max-age=200, s-maxage=50, max-age=9";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50cc9nsp/ {
+      add_header "Cache-Control" "max-age=200,s-maxage=50,max-age=9";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50sma51/ {
+      add_header "Cache-Control" "max-age=200, s-maxage=50, s-maxage=51";
+    }
+
     location /mod_pagespeed_test/ipro/wait/ {
       # TODO(jmarantz): ModPagespeedInPlaceWaitForOptimized should be superfluous,
       # or made equivalent to ModPagespeedInPlaceRewriteDeadlineMs -1, which waits
@@ -1695,14 +2092,16 @@ http {
     }
 
     location /mod_pagespeed_test/ipro/instant/deadline/ {
-      pagespeed InPlaceRewriteDeadlineMs -1;
+      set $ps_deadline -1;
+      pagespeed ImageRecompressionQuality $ps_deadline;
+      pagespeed InPlaceRewriteDeadlineMs $ps_deadline;
     }
 
     # Test to make sure that user-authenticated resources do not get cached and
     # optimized.
     location /mod_pagespeed_test/auth/ {
        auth_basic "Restricted";
-       auth_basic_user_file "@@SERVER_ROOT@@mod_pagespeed_test/auth/passwd.conf";
+       auth_basic_user_file "@@SERVER_ROOT@@/mod_pagespeed_test/auth/passwd.conf";
     }
 
     location /mod_pagespeed_test/ipro/cookie/ {
@@ -1743,11 +2142,31 @@ http {
     pagespeed FetchHttps enable;
     location /mod_pagespeed_test/https_fetch/ {
       pagespeed DisableFilters inline_images;
+      set $ps_gstatic "https://www.gstatic.com/psa/static";
       pagespeed MapProxyDomain
           http://localhost:@@PRIMARY_PORT@@/https_gstatic_dot_com
-          https://www.gstatic.com/psa/static;
+          $ps_gstatic;
     }
 
+    location /mod_pagespeed_test/strip_subresource_hints/default/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed RewriteLevel CoreFilters;
+      pagespeed DisAllow *dontrewriteme*;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/default_passthrough/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed RewriteLevel PassThrough;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/preserve_on/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed PreserveSubresourceHints on;
+      pagespeed RewriteLevel CoreFilters;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/preserve_off/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed PreserveSubresourceHints off;
+      pagespeed RewriteLevel CoreFilters;
+    }
     # $host implicitly tests script variable support. I'd love to test it more
     # directly, but so far this is the best I've come up with and duplicating
     # the test doesn't seem to make sense.
@@ -1755,6 +2174,11 @@ http {
         "http://$host:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/instant/"
         "@@SERVER_ROOT@@/mod_pagespeed_test/ipro/instant/";
 
+    location /mod_pagespeed_test/public/ {
+      add_header "Cache-Control" "public, max-age=600";
+      pagespeed PreserveUrlRelativity off;
+    }
+
     pagespeed EnableFilters remove_comments;
 
     # Test LoadFromFile mapping by mapping one dir to another.
@@ -1768,7 +2192,6 @@ http {
       "@@SERVER_ROOT@@/mod_pagespeed_test/load_from_file/file_dir/httponly/";
     pagespeed LoadFromFileRuleMatch Disallow \.ssp.css$ps_dollar;
     pagespeed LoadFromFileRuleMatch Allow exception\.ssp\.css$ps_dollar;
-
     #charset koi8-r;
 
     #access_log  logs/host.access.log  main;
@@ -1815,6 +2238,7 @@ http {
     image/svg+xml                         svg svgz;
     image/webp                            webp;
 
+    application/json                      json;
     application/java-archive              jar war ear;
     application/mac-binhex40              hqx;
     application/msword                    doc;

test/run_tests.sh

@@ -23,10 +23,19 @@
 # Exits with status 2 if command line args are wrong.
 #
 # Usage:
-#   ./run_tests.sh primary_port secondary_port mod_pagespeed_dir pagespeed_test_host
+#   ./run_tests.sh
-# Example:
+# Or:
-#   ./run_tests.sh 8050 8051 /path/to/mod_pagespeed www.modpagespeed.com
+#   ./run_tests.sh /path/to/mod_pagespeed /path/to/nginx/binary
 #
+# If you built ngx_pagespeed with "scripts/build_ngx_pagespeed.sh --devel" then
+# you don't need to pass any arguments to run_tests.sh.  Otherwise, you'll need
+# to tell it where to find a mod_pagespeed checkout (for example html files etc)
+# and the nginx binary to test.
+#
+# By default the test script uses several ports.  If you have a port conflict
+# and need to override one you can do that by setting the relevant environment
+# variable.  For example:
+#   PRIMARY_PORT=1234 ./run_tests.sh /.../mod_pagespeed /.../nginx/binary
 
 # Normally we test only with the native fetcher off.  Set
 # TEST_NATIVE_FETCHER=true to also test the native fetcher, set
@@ -43,28 +52,28 @@ RUN_TESTS=${RUN_TESTS:-true}
 # true.
 USE_VALGRIND=${USE_VALGRIND:-false}
 
-if [ "$#" -ne 5 ] ; then
+if [ "$#" -eq 0 ]; then
-  echo "Usage: $0 primary_port secondary_port mod_pagespeed_dir"
+  MOD_PAGESPEED_DIR="testing-dependencies/mod_pagespeed/"
-  echo "  nginx_executable"
+  NGINX_EXECUTABLE="nginx/sbin/nginx"
+elif [ "$#" -eq 2 ]; then
+  MOD_PAGESPEED_DIR="$1"
+  NGINX_EXECUTABLE="$2"
+else
+  echo "Usage: $0 [mod_pagespeed_dir nginx_executable]"
   exit 2
 fi
 
-PRIMARY_PORT="$1"
+: ${PRIMARY_PORT:=8050}
-SECONDARY_PORT="$2"
+: ${SECONDARY_PORT:=8051}
-MOD_PAGESPEED_DIR="$3"
+: ${CONTROLLER_PORT:=8053}
-NGINX_EXECUTABLE="$4"
+: ${RCPORT:=9991}
-PAGESPEED_TEST_HOST="$5"
+: ${PAGESPEED_TEST_HOST:=selfsigned.modpagespeed.com}
-RCPORT1=9991
+: ${PHP_PORT:=9000}
-RCPORT2=9992
-RCPORT3=9993
-RCPORT4=9994
-RCPORT5=9995
-RCPORT6=9996
-RCPORT7=9997
 
 this_dir="$( cd $(dirname "$0") && pwd)"
 
 function run_test_checking_failure() {
+  "$MOD_PAGESPEED_DIR/install/start_php.sh" "$PHP_PORT"
   USE_VALGRIND="$USE_VALGRIND" \
     PRIMARY_PORT="$PRIMARY_PORT" \
     SECONDARY_PORT="$SECONDARY_PORT" \
@@ -72,13 +81,8 @@ function run_test_checking_failure() {
     NGINX_EXECUTABLE="$NGINX_EXECUTABLE" \
     PAGESPEED_TEST_HOST="$PAGESPEED_TEST_HOST" \
     RUN_TESTS="$RUN_TESTS" \
-    RCPORT1="$RCPORT1" \
+    CONTROLLER_PORT="$CONTROLLER_PORT" \
-    RCPORT2="$RCPORT2" \
+    RCPORT="$RCPORT" \
-    RCPORT3="$RCPORT3" \
-    RCPORT4="$RCPORT4" \
-    RCPORT5="$RCPORT5" \
-    RCPORT6="$RCPORT6" \
-    RCPORT7="$RCPORT7" \
     bash "$this_dir/nginx_system_test.sh"
   STATUS=$?
   echo "With $@ setup."

test/valgrind.sup

@@ -163,3 +163,31 @@
    fun:_ZN3re23DFA10AddToQueueEPNS0_5WorkqEij
    ...
 }
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Value8
+   fun:_ZNK3re210SparseSetTIvE8containsEi
+   ...
+}
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Cond
+   fun:_ZNK3re210SparseSetTIvE8containsEi
+   ...
+}
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Value8
+   fun:_ZNK3re211SparseArrayIiE9has_indexEi
+   ...
+}
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Cond
+   fun:_ZNK3re211SparseArrayIiE9has_indexEi
+   ...
+}