Merge remote-tracking branch 'origin/trunk-tracking' into jefftk-better-error-message

We want to parse $additional_configure_args into an array the way the
shell would, which we're using eval for.  But because we were missing
quotes whitespace was being stripped.  Basically:

    $ function lines() { for x in "$@"; do echo "$x"; done; }

    $ lines a "b   c" d
    a
    b   c
    d

    $ a='a "b   c" d'
    $ eval e=($a)
    $ eval f=("$a")

    $ lines "${e[@]}"
    a
    b c
    d

    $ lines "${f[@]}"
    a
    b   c
    d

We were doing this like e when we should have been doing it like f.

.gitignore

@@ -1,4 +1,4 @@
 test/tmp
 psol/
 psol-*.tar.gz
-
+*.*.*.*.tar.gz

.travis.yml

@@ -0,0 +1,66 @@
+language: c++
+# Remove broken repo, per: https://github.com/travis-ci/travis-ci/issues/6588
+# Undo this once the repo is fixed.
+before_install:
+  - "sudo add-apt-repository --remove 'http://us-central1.gce.archive.ubuntu.com/ubuntu/ main restricted'"
+  - "sudo add-apt-repository --remove 'http://us-central1.gce.archive.ubuntu.com/ubuntu/ universe'"
+  - "sudo add-apt-repository --remove 'http://us-central1.gce.archive.ubuntu.com/ubuntu/ multiverse'"
+  - "sudo add-apt-repository http://archive.ubuntu.com/ubuntu/"
+  - "sudo add-apt-repository 'http://archive.ubuntu.com/ubuntu/ universe'"
+  - "sudo add-apt-repository 'http://archive.ubuntu.com/ubuntu/ multiverse'"
+  - "sudo apt-get -qq update"
+install:
+  - sudo sh -c 'echo "deb http://opensource.wandisco.com/ubuntu precise svn18" >> /etc/apt/sources.list.d/subversion18.list'
+  - sudo wget -q http://opensource.wandisco.com/wandisco-debian.gpg -O- | sudo apt-key add -
+  - sudo apt-get update 2>&1 > /dev/null
+  - mv $TRAVIS_BUILD_DIR ~/ngxpagespeed
+  - sudo apt-get install build-essential zlib1g-dev libpcre3 libpcre3-dev unzip g++ python subversion gperf make devscripts fakeroot git curl netcat-traditional gcc-mozilla clang-3.4 2>&1 > /dev/null
+  - sudo mv /bin/nc.traditional /usr/bin/nc
+  - export PATH=/usr/lib/gcc-mozilla/bin:$PATH
+  - sudo ln -sf /usr/lib/gcc-mozilla/lib/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libstdc++.so.6
+  - sudo sh -c 'echo "image/webp webp" >> /etc/mime.types'
+  - mkdir -p ~/bin
+  - cd ~/bin
+  - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
+  - mkdir ~/mod_pagespeed
+  - cd ~/mod_pagespeed
+  - git clone https://github.com/pagespeed/mod_pagespeed.git src
+  - cd src
+  - ~/bin/depot_tools/gclient config https://github.com/pagespeed/mod_pagespeed.git --unmanaged --name=$PWD
+  - ~/bin/depot_tools/gclient sync --force --jobs=1
+  - cd ~/mod_pagespeed/src/pagespeed/automatic
+  - make BUILDTYPE=Release -C ../../pagespeed/automatic AR.host="$PWD/../../build/wrappers/ar.sh" AR.target="$PWD/../../build/wrappers/ar.sh" all
+  - cd ~
+  - git clone https://github.com/FRiCKLE/ngx_cache_purge.git
+  - NGX_CACHE_PURGE=$PWD/ngx_cache_purge
+  - wget https://openresty.org/download/ngx_openresty-1.9.7.2.tar.gz
+  - tar xzf ngx_openresty-*.tar.gz
+  - cd ngx_openresty-*/
+  - ./configure --with-luajit
+  - make
+  - NGX_DEVEL_KIT=$(echo $HOME/ngx_openresty-*/build/ngx_devel_kit-*/)
+  - SET_MISC_MODULE=$(echo $HOME/ngx_openresty-*/build/set-misc-nginx-module-*/)
+  - HEADERS_MORE_MODULE=$(echo $HOME/ngx_openresty-*/build/headers-more-nginx-module-*/)
+  - cd ~
+  - wget https://github.com/nginx/nginx/archive/branches/default.zip
+  - unzip default.zip
+  - cd nginx-branches-default
+  - MOD_PAGESPEED_DIR="$HOME/mod_pagespeed/src" ./auto/configure --add-module=$HOME/ngxpagespeed --add-module="$NGX_CACHE_PURGE" --add-module="$NGX_DEVEL_KIT" --add-module="$SET_MISC_MODULE" --add-module="$HEADERS_MORE_MODULE" --with-ipv6
+  - make
+  - sudo make install
+script:
+  - echo "build successful"
+  - echo "cd ~/ngxpagespeed"
+  - echo "sudo ./test/run_tests.sh 8050 8051 $HOME/mod_pagespeed/src $HOME/nginx-branches-default/objs selfsigned.modpagespeed.com"
+  - echo "sudo ./test/run_tests.sh 8050 8051 $HOME/mod_pagespeed $HOME/nginx-branches-default/objs/nginx selfsigned.modpagespeed.com"
+sudo: required
+compiler:
+  - gcc
+notifications:
+  email:
+    - cheesy@google.com
+    - jefftk@google.com
+    - morlovich@google.com
+    - jmarantz@google.com
+    - huibao@google.com
+    - jcrowell@google.com

PSOL_BINARY_URL

@@ -0,0 +1,4 @@
+In a release this file would contain the URL to download the pre-compiled PSOL
+binary, but on development branches (like this one) you have to build PSOL from
+source yourself.  See:
+  https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source

README.md

@@ -1,5 +1,7 @@
 ![ngx_pagespeed](https://lh6.googleusercontent.com/-qufedJIJq7Y/UXEvVYxyYvI/AAAAAAAADo8/JHDFQhs91_c/s401/04_ngx_pagespeed.png)
 
+[![Build Status](https://travis-ci.org/pagespeed/ngx_pagespeed.svg?branch=trunk-tracking)](https://travis-ci.org/pagespeed/ngx_pagespeed)
+
 ngx_pagespeed speeds up your site and reduces page load time by automatically
 applying web performance best practices to pages and associated assets (CSS,
 JavaScript, images) without requiring you to modify your existing content or

config

@@ -17,53 +17,39 @@
 #   PSOL_BINARY: absolute path to pagespeed_automatic.a
 
 mod_pagespeed_dir="${MOD_PAGESPEED_DIR:-unset}"
+position_aux="${POSITION_AUX:-unset}"
+
 if [ "$mod_pagespeed_dir" = "unset" ] ; then
   mod_pagespeed_dir="$ngx_addon_dir/psol/include"
   build_from_source=false
 
   if [ ! -e "$mod_pagespeed_dir" ] ; then
     echo "ngx_pagespeed: pagespeed optimization library not found:"
-    echo ""
+
-    echo "   You need to separately download the pagespeed library:"
+    psol_binary_url="$(cat $ngx_addon_dir/PSOL_BINARY_URL)"
-    echo ""
+    if [[ "$psol_binary_url" != https://* ]]; then
-    echo "     $ cd /path/to/ngx_pagespeed"
+      echo "
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.8.31.2.tar.gz"
+  This is a development branch of ngx_pagespeed, which means there is no
-    echo "     $ tar -xzvf 1.8.31.2.tar.gz # expands to psol/"
+  precompiled PSOL library available to link against.  Either build from a
-    echo ""
+  release tag, like latest-beta, or build PSOL from source:
-    echo "   Or see the installation instructions:"
+    https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source"
-    echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
+      exit 1
+    fi
+
+    echo "
+  You need to separately download the pagespeed library:
+     $ cd $ngx_addon_dir
+     $ wget $psol_binary_url
+     $ tar -xzvf $(basename $psol_binary_url) # expands to psol/
+
+  Or see the installation instructions:
+     https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source"
     exit 1
   fi
-
 else
   build_from_source=true
 fi
 
-psol_binary="${PSOL_BINARY:-unset}"
-if [ "$psol_binary" = "unset" ] ; then
-  if $build_from_source ; then
-    psol_binary="\
-        $mod_pagespeed_dir/net/instaweb/automatic/pagespeed_automatic.a"
-  else
-    psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
-    psol_binary="$psol_library_dir/pagespeed_automatic.a"
-  fi
-fi
-
-echo "mod_pagespeed_dir=$mod_pagespeed_dir"
-echo "build_from_source=$build_from_source"
-
-ngx_feature="psol"
-ngx_feature_name=""
-ngx_feature_run=no
-ngx_feature_incs="
-#include \"net/instaweb/htmlparse/public/html_parse.h\"
-#include \"net/instaweb/htmlparse/public/html_writer_filter.h\"
-#include \"net/instaweb/util/public/string.h\"
-#include \"net/instaweb/util/public/string_writer.h\"
-#include \"net/instaweb/util/public/null_message_handler.h\"
-"
-
 os_name='unknown_os'
 arch_name='unknown_arch'
 uname_os=`uname`
@@ -94,6 +80,18 @@ else
   buildtype=Release
 fi
 
+# If the compiler is gcc, we want to use g++ to link, if at all possible,
+# so that -static-libstdc++ works.
+# Annoyingly, the feature test doesn't even use $LINK for linking, so that
+# needs an explicit -lstdc++
+pagespeed_libs=
+ps_maybe_gpp_base=`basename $CC| sed s/gcc/g++/`
+ps_maybe_gpp="`dirname $CC`/$ps_maybe_gpp_base"
+if [ -n "$NGX_GCC_VER" -a \( -x "$ps_maybe_gpp" \) ]; then
+  LINK=$ps_maybe_gpp
+fi
+pagespeed_libs="-lstdc++"
+
 # The compiler needs to know that __sync_add_and_fetch_4 is ok,
 # and this requires an instruction that didn't exist on i586 or i386.
 if [ "$uname_arch" = "i686" ]; then
@@ -102,6 +100,14 @@ fi
 
 CFLAGS="$CFLAGS $FLAG_MARCH"
 
+# For now, standardize on gcc-4.x ABI --- if we don't set this, people building
+# with new gcc defaulting to gcc-5 C++11 ABI will have build trouble linking
+# to our libpsol.a
+# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html
+CFLAGS="$CFLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
+CC_OLD_TEST_FLAGS="$CC_TEST_FLAGS"
+CC_TEST_FLAGS="$CC_TEST_FLAGS -D_GLIBCXX_USE_CXX11_ABI=0 --std=c++11"
+
 case "$NGX_GCC_VER" in
   4.8*)
     # On GCC 4.8 and above, -Wall enables -Wunused-local-typedefs.  This breaks
@@ -120,28 +126,63 @@ case "$NGX_GCC_VER" in
   ;;
 esac
 
+# workaround for a bug in nginx-1.9.11, see:
+# http://hg.nginx.org/nginx/rev/ff1e625ae55b
+NGX_VERSION=`grep nginx_version src/core/nginx.h | sed -e 's/^.* \(.*\)$/\1/'`
+if [ "$NGX_VERSION" = "1009011" ]; then
+    CFLAGS="$CFLAGS -Wno-write-strings"
+fi
+
 if [ "$WNO_ERROR" = "YES" ]; then
     CFLAGS="$CFLAGS -Wno-error"
 fi
 
+psol_binary="${PSOL_BINARY:-unset}"
+if [ "$psol_binary" = "unset" ] ; then
+  if $build_from_source ; then
+    psol_binary="\
+        $mod_pagespeed_dir/pagespeed/automatic/pagespeed_automatic.a"
+  else
+    psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
+    psol_binary="$psol_library_dir/pagespeed_automatic.a"
+  fi
+fi
+
+echo "mod_pagespeed_dir=$mod_pagespeed_dir"
+echo "build_from_source=$build_from_source"
+
+ngx_feature="psol"
+ngx_feature_name=""
+ngx_feature_run=no
+ngx_feature_incs="
+#include \"pagespeed/kernel/base/string.h\"
+#include \"pagespeed/kernel/base/string_writer.h\"
+#include \"pagespeed/kernel/base/null_message_handler.h\"
+#include \"pagespeed/kernel/html/html_parse.h\"
+#include \"pagespeed/kernel/html/html_writer_filter.h\"
+"
+
 pagespeed_include="\
   $mod_pagespeed_dir \
   $mod_pagespeed_dir/third_party/chromium/src \
   $mod_pagespeed_dir/third_party/google-sparsehash/src \
   $mod_pagespeed_dir/third_party/google-sparsehash/gen/arch/$os_name/$arch_name/include \
-  $mod_pagespeed_dir/third_party/protobuf/src \
+  $mod_pagespeed_dir/third_party/grpc/src/include \
+  $mod_pagespeed_dir/third_party/protobuf/src/src \
   $mod_pagespeed_dir/third_party/re2/src \
   $mod_pagespeed_dir/out/$buildtype/obj/gen \
   $mod_pagespeed_dir/out/$buildtype/obj/gen/protoc_out/instaweb \
   $mod_pagespeed_dir/third_party/apr/src/include \
   $mod_pagespeed_dir/third_party/aprutil/src/include \
   $mod_pagespeed_dir/third_party/apr/gen/arch/$os_name/$arch_name/include \
-  $mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include"
+  $mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include \
+  $mod_pagespeed_dir/url"
 ngx_feature_path="$pagespeed_include"
 
-pagespeed_libs="-lstdc++ $psol_binary -lrt -pthread -lm"
+pagespeed_libs="$psol_binary $pagespeed_libs -lrt -pthread -lm"
 ngx_feature_libs="$pagespeed_libs"
 ngx_feature_test="
+
   GoogleString output_buffer;
   net_instaweb::StringWriter write_to_string(&output_buffer);
 
@@ -162,52 +203,163 @@ ngx_feature_test="
 # Test whether we have pagespeed and can compile and link against it.
 . "$ngx_addon_dir/cpp_feature"
 
-if [ $ngx_found = yes ]; then
+if [ $ngx_found = no ]; then
-  ps_src="$ngx_addon_dir/src"
-  ngx_addon_name=ngx_pagespeed
-  NGX_ADDON_DEPS="$NGX_ADDON_DEPS \
-    $ps_src/log_message_handler.h \
-    $ps_src/ngx_base_fetch.h \
-    $ps_src/ngx_caching_headers.h \
-    $ps_src/ngx_fetch.h \
-    $ps_src/ngx_gzip_setter.h \
-    $ps_src/ngx_list_iterator.h \
-    $ps_src/ngx_message_handler.h \
-    $ps_src/ngx_pagespeed.h \
-    $ps_src/ngx_rewrite_driver_factory.h \
-    $ps_src/ngx_rewrite_options.h \
-    $ps_src/ngx_server_context.h \
-    $ps_src/ngx_url_async_fetcher.h \
-    $psol_binary"
-  NGX_ADDON_SRCS="$NGX_ADDON_SRCS \
-    $ps_src/log_message_handler.cc \
-    $ps_src/ngx_base_fetch.cc \
-    $ps_src/ngx_caching_headers.cc \
-    $ps_src/ngx_fetch.cc \
-    $ps_src/ngx_gzip_setter.cc \
-    $ps_src/ngx_list_iterator.cc \
-    $ps_src/ngx_message_handler.cc \
-    $ps_src/ngx_pagespeed.cc \
-    $ps_src/ngx_rewrite_driver_factory.cc \
-    $ps_src/ngx_rewrite_options.cc \
-    $ps_src/ngx_server_context.cc \
-    $ps_src/ngx_url_async_fetcher.cc"
-
-  # Make pagespeed run immediately before gzip.
-  HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
-    sed "s/$HTTP_GZIP_FILTER_MODULE/$HTTP_GZIP_FILTER_MODULE $ngx_addon_name/")
-  # Make the etag header filter run immediately after gzip.
-  HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
-    sed "s/$HTTP_GZIP_FILTER_MODULE/ngx_pagespeed_etag_filter $HTTP_GZIP_FILTER_MODULE/")
-  CORE_LIBS="$CORE_LIBS $pagespeed_libs"
-  CORE_INCS="$CORE_INCS $pagespeed_include"
-  echo "List of modules (in reverse order of applicability): "$HTTP_FILTER_MODULES
-else
   cat << END
 $0: error: module ngx_pagespeed requires the pagespeed optimization library.
-Look in obj/autoconf.err for more details.
+Look in $PWD/$NGX_AUTOCONF_ERR for more details.
+END
+  exit 1
+fi
+
+ps_src="$ngx_addon_dir/src"
+ngx_addon_name=ngx_pagespeed
+NGX_ADDON_DEPS="$NGX_ADDON_DEPS \
+$ps_src/log_message_handler.h \
+$ps_src/ngx_base_fetch.h \
+$ps_src/ngx_caching_headers.h \
+$ps_src/ngx_event_connection.h \
+$ps_src/ngx_fetch.h \
+$ps_src/ngx_gzip_setter.h \
+$ps_src/ngx_list_iterator.h \
+$ps_src/ngx_message_handler.h \
+$ps_src/ngx_pagespeed.h \
+$ps_src/ngx_rewrite_driver_factory.h \
+$ps_src/ngx_rewrite_options.h \
+$ps_src/ngx_server_context.h \
+$ps_src/ngx_url_async_fetcher.h \
+$psol_binary"
+NPS_SRCS=" \
+$ps_src/log_message_handler.cc \
+$ps_src/ngx_base_fetch.cc \
+$ps_src/ngx_caching_headers.cc \
+$ps_src/ngx_event_connection.cc \
+$ps_src/ngx_fetch.cc \
+$ps_src/ngx_gzip_setter.cc \
+$ps_src/ngx_list_iterator.cc \
+$ps_src/ngx_message_handler.cc \
+$ps_src/ngx_pagespeed.cc \
+$ps_src/ngx_rewrite_driver_factory.cc \
+$ps_src/ngx_rewrite_options.cc \
+$ps_src/ngx_server_context.cc \
+$ps_src/ngx_url_async_fetcher.cc"
+# Save our sources in a separate var since we may need it in config.make
+PS_NGX_SRCS="$NGX_ADDON_SRCS \
+$NPS_SRCS"
+
+# Make pagespeed run immediately before gzip and Brotli.
+if echo $HTTP_FILTER_MODULES | grep ngx_http_brotli_filter_module >/dev/null; then
+  next=ngx_http_brotli_filter_module
+elif [ $HTTP_GZIP = YES ]; then
+  next=ngx_http_gzip_filter_module
+else
+  next=ngx_http_range_header_filter_module
+fi
+
+if [ -n "$ngx_module_link" ]; then
+  # nginx-1.9.11+
+  ngx_module_type=HTTP_FILTER
+  ngx_module_name="ngx_pagespeed ngx_pagespeed_etag_filter"
+  ngx_module_incs="$ngx_feature_path"
+  ngx_module_deps=
+  ngx_module_srcs="$NPS_SRCS"
+  ngx_module_libs="$ngx_feature_libs"
+  ngx_module_order="ngx_http_range_header_filter_module\
+                    ngx_pagespeed_etag_filter\
+                    ngx_http_gzip_filter_module \
+                    ngx_http_brotli_filter_module \
+                    ngx_pagespeed \
+                    ngx_http_postpone_filter_module \
+                    ngx_http_ssi_filter_module \
+                    ngx_http_charset_filter_module \
+                    ngx_http_xslt_filter_module \
+                    ngx_http_image_filter_module \
+                    ngx_http_sub_filter_module \
+                    ngx_http_addition_filter_module \
+                    ngx_http_gunzip_filter_module \
+                    ngx_http_userid_filter_module \
+                    ngx_http_headers_filter_module"
+
+    . auto/module
+
+    if [ $ngx_module_link != DYNAMIC ]; then
+      # ngx_module_order doesn't work with static modules,
+      # so we must re-order filters here.
+      if [ "$position_aux" = "true" ] ; then
+        HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
+      else
+        HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES \
+                             | sed "s/ngx_pagespeed//" \
+                             | sed "s/$next/$next ngx_pagespeed/")
+      fi
+      # Make the etag header filter run immediately before range header filter.
+      HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES \
+                           | sed "s/ngx_pagespeed_etag_filter//" \
+                           | sed "s/ngx_http_range_header_filter_module/ngx_http_range_header_filter_module ngx_pagespeed_etag_filter/")
+    else
+      if [ "$position_aux" = "true" ] ; then
+        ngx_module_type=HTTP_AUX_FILTER
+        ngx_module_order=""
+      fi
+    fi
+else
+  CORE_LIBS="$CORE_LIBS $pagespeed_libs"
+  CORE_INCS="$CORE_INCS $pagespeed_include"
+  NGX_ADDON_SRCS="$PS_NGX_SRCS"
+  if [ "$position_aux" = "true" ] ; then
+    HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
+  else
+    HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES | sed "s/$next/$next $ngx_addon_name/")
+  fi
+
+  # Make the etag header filter run immediately before range header filter.
+  HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
+    sed "s/ngx_http_range_header_filter_module/ngx_http_range_header_filter_module ngx_pagespeed_etag_filter/")
+fi
+
+echo "List of modules (in reverse order of applicability): "$HTTP_FILTER_MODULES
+
+# Test whether the compiler is compatible
+ngx_feature="psol-compiler-compat"
+ngx_feature_name=""
+ngx_feature_run=no
+ngx_feature_incs=""
+ngx_feature_path=""
+ngx_feature_libs="-lstdc++"
+ngx_feature_test="
+
+#if defined(__clang__) && defined(__GLIBCXX__)
+   // See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning
+   // for a list of various values of __GLIBCXX__. Note that they're not monotonic
+   // with respect to version numbers.
+   #if __GLIBCXX__ == 20120322  || __GLIBCXX__ == 20120614
+      #error \"clang is using libstdc++ 4.7.0 or 4.7.1, which can cause binary incompatibility.\"
+   #endif
+#endif
+
+#if !defined(__clang__) && defined(__GNUC__)
+   #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8)
+      #error \"GCC < 4.8 no longer supported. Please use gcc >= 4.8 or clang >= 3.3\"
+   #endif
+#endif
+
+#if defined(__clang__)
+   #if __clang_major__ < 3 || (__clang_major__ == 3 && __clang_minor__ < 3)
+      #error \"Please use gcc >= 4.8 or clang >= 3.3\"
+   #endif
+#endif
+"
+
+. "$ngx_addon_dir/cpp_feature"
+
+if [ $ngx_found = no ]; then
+  cat << END
+$0: error: module ngx_pagespeed requires gcc >= 4.8 or clang >= 3.3.
+See https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source for some recommendations.
+Look in $PWD/$NGX_AUTOCONF_ERR for more details.
 END
   exit 1
 fi
 
 have=NGX_PAGESPEED . auto/have
+
+CC_TEST_FLAGS="$CC_OLD_TEST_FLAGS"

config.make

@@ -0,0 +1,7 @@
+# Since nginx build system doesn't normally do C++, there is no CXXFLAGS for us
+# to touch, and compilers are understandably unhappy with --std=c++11 on C
+# files. Hence, we hack the makefile to add it for just our sources.
+for ps_src_file in $PS_NGX_SRCS; do
+  ps_obj_file="$NGX_OBJS/addon/src/`basename $ps_src_file .cc`.o"
+  echo "$ps_obj_file : CFLAGS += --std=c++11" >> $NGX_MAKEFILE
+done

scripts/build_ngx_pagespeed.sh

@@ -0,0 +1,532 @@
+#!/bin/bash
+
+function usage() {
+  echo "
+Usage: build_ngx_pagespeed.sh [options]
+
+  Installs ngx_pagespeed and its dependencies.  Can optionally build and install
+  nginx as well.
+
+Options:
+  -v, --ngx-pagespeed-version <ngx_pagespeed version>
+      What version of ngx_pagespeed to build.  Valid options include:
+      * latest-beta
+      * latest-stable
+      * a version number, such as 1.11.33.4
+
+      If you don't specify a version, defaults to latest-stable.
+
+  -n, --nginx-version <nginx version>
+      What version of nginx to build.  If not set, this script only prepares the
+      ngx_pagespeed module, and expects you to handle including it when you
+      build nginx.
+
+      If you pass in 'latest' then this script scrapes the nginx download page
+      and attempts to determine the latest version automatically.
+
+  -m, --dynamic-module
+      Build ngx_pagespeed as a dynamic module.
+
+  -b, --builddir <directory>
+      Where to build.  Defaults to \$HOME.
+
+  -p, --no-deps-check
+      By default, this script checks for the packages it depends on and tries to
+      install them.  If you have installed dependencies from source or are on a
+      non-deb non-rpm system, this won't work.  In that case, install the
+      dependencies yourself and pass --no-deps-check.
+
+  -d, --dryrun
+      Don't make any changes to the system, just print what changes you
+      would have made.
+
+  -h, --help
+      Print this message and exit."
+}
+
+# Prints an error message and exits with an error code.
+function fail() {
+  local error_message="$@"
+  echo "$@" >&2
+
+  # Normally I'd use $0 in "usage" here, but since most people will be running
+  # this via curl, that wouldn't actually give something useful.
+  echo >&2
+  echo "For usage information, run this script with --help" >&2
+  exit 1
+}
+
+# Intended to be called as:
+#   bash <(curl dl.google.com/.../build_ngx_pagespeed.sh) <args>
+
+# If we set -e or -u then users of this script will see it silently exit on
+# failure.  Instead we need to check the exit status of each command manually.
+# The run function handles exit-status checking for system-changing commands.
+# Additionally, this allows us to easily have a dryrun mode where we don't
+# actually make any changes.
+function run() {
+  if "$DRYRUN"; then
+    echo "would run $@"
+  else
+    if ! "$@"; then
+      echo "Failure running $@, exiting."
+      exit 1
+    fi
+  fi
+}
+
+function redhat_is_installed() {
+  local package_name="$1"
+  rpm -qa $package_name | grep -q .
+}
+
+function debian_is_installed() {
+  local package_name="$1"
+  dpkg -l $package_name | grep ^ii | grep -q .
+}
+
+function version_sort() {
+  # We'd rather use sort -V, but that's not available on Centos 5.  This works
+  # for versions in the form A.B.C.D or shorter, which is enough for our use.
+  sort -t '.' -k 1,1 -k 2,2 -k 3,3 -k 4,4 -g
+}
+
+# Compare two numeric versions in the form "A.B.C".  Works with version numbers
+# having up to four components, since that's enough to handle both nginx (3) and
+# ngx_pagespeed (4).
+function version_older_than() {
+  local test_version="$1"
+  local compare_to="$2"
+
+  local older_version=$(echo $@ | tr ' ' '\n' | version_sort | head -n 1)
+  test "$older_version" != "$compare_to"
+}
+
+function determine_latest_nginx_version() {
+  # Scrape nginx's download page to try to find the most recent nginx version.
+
+  nginx_download_url="https://nginx.org/en/download.html"
+  function report_error() {
+    fail "
+Couldn't automatically determine the latest nginx version: failed to $@
+$nginx_download_url"
+  }
+
+  nginx_download_page=$(curl -sS --fail "$nginx_download_url") || \
+    report_error "download"
+
+  download_refs=$(echo "$nginx_download_page" | \
+    grep -o '/download/nginx-[0-9.]*[.]tar[.]gz') || \
+    report_error "parse"
+
+  versions_available=$(echo "$download_refs" | \
+    sed -e 's~^/download/nginx-~~' -e 's~\.tar\.gz$~~') || \
+    report_error "extract versions from"
+
+  latest_version=$(echo "$versions_available" | version_sort | tail -n 1) || \
+    report_error "determine latest version from"
+
+  if version_older_than "$latest_version" "1.11.4"; then
+    fail "
+Expected the latest version of nginx to be at least 1.11.4 but found
+$latest_version on $nginx_download_url"
+  fi
+
+  echo "$latest_version"
+}
+
+# Usage:
+#  install_dependencies install_pkg_cmd is_pkg_installed_cmd dep1 dep2 ...
+#
+# install_pkg_cmd is a command to install a dependency
+# is_pkg_installed_cmd is a command that returns true if the dependency is
+#   already installed
+# each dependency is a package name
+function install_dependencies() {
+  local install_pkg_cmd="$1"
+  local is_pkg_installed_cmd="$2"
+  shift 2
+
+  local missing_dependencies=""
+
+  for package_name in "$@"; do
+    if ! $is_pkg_installed_cmd $package_name; then
+      missing_dependencies+="$package_name "
+    fi
+  done
+  if [ -n "$missing_dependencies" ]; then
+    echo "Detected that we're missing the following depencencies:"
+    echo "  $missing_dependencies"
+    echo "Installing them:"
+    run sudo $install_pkg_cmd $missing_dependencies
+  fi
+}
+
+function gcc_too_old() {
+  # We need gcc >= 4.8
+  local gcc_major_version=$(gcc -dumpversion | awk -F. '{print $1}')
+  if [ "$gcc_major_version" -lt 4 ]; then
+    return 0  # too old
+  elif [ "$gcc_major_version" -gt 4 ]; then
+    return 1  # plenty new
+  fi
+  # It's gcc 4.x, check if x >= 8:
+  local gcc_minor_version=$(gcc -dumpversion | awk -F. '{print $2}')
+  test "$gcc_minor_version" -lt 8
+}
+
+function continue_or_exit() {
+  local prompt="$1"
+  read -p "$prompt [Y/n] " yn
+  if [[ "$yn" == N* || "$yn" == n* ]]; then
+    echo "Cancelled."
+    exit 0
+  fi
+}
+
+# If a string is very simple we don't need to quote it.  But we should quote
+# everything else to be safe.
+function needs_quoting() {
+  echo "$@" | grep -q '[^a-zA-Z0-9./_=-]'
+}
+
+function escape_for_quotes() {
+  echo "$@" | sed -e 's~\\~\\\\~g' -e "s~'~\\\\'~g"
+}
+
+function quote_arguments() {
+  local argument_str=""
+  for argument in "$@"; do
+    if [ -n "$argument_str" ]; then
+      argument_str+=" "
+    fi
+    if needs_quoting "$argument"; then
+      argument="'$(escape_for_quotes "$argument")'"
+    fi
+    argument_str+="$argument"
+  done
+  echo "$argument_str"
+}
+
+function build_ngx_pagespeed() {
+  getopt --test
+  if [ "$?" != 4 ]; then
+    # Even Centos 5 and Ubuntu 10 LTS have new-style getopt, so I don't expect
+    # this to be hit in practice on systems that are actually able to run
+    # PageSpeed.
+    fail "Your version of getopt is too old.  Exiting with no changes made."
+  fi
+
+  opts=$(getopt -o v:n:mb:pdh \
+    --longoptions ngx-pagespeed-version:,nginx-version:,dynamic-module \
+    --longoptions buildir:,no-deps-check,dryrun,help \
+    -n "$(basename "$0")" -- "$@")
+  if [ $? != 0 ]; then
+    usage
+    exit 1
+  fi
+  eval set -- "$opts"
+
+  NPS_VERSION="latest-stable"
+  NGINX_VERSION=""
+  BUILDDIR="$HOME"
+  DO_DEPS_CHECK=true
+  DRYRUN=false
+  DYNAMIC_MODULE=false
+  while true; do
+    case "$1" in
+      -v | --ngx-pagespeed-version) shift
+        NPS_VERSION="$1"
+        shift
+        ;;
+      -n | --nginx-version) shift
+        NGINX_VERSION="$1"
+        shift
+        ;;
+      -m | --dynamic-module) shift
+        DYNAMIC_MODULE=true
+        ;;
+      -b | --builddir) shift
+        BUILDDIR="$1"
+        shift
+        ;;
+      -p | --no-deps-check) shift
+        DO_DEPS_CHECK=false
+        ;;
+      -d | --dryrun) shift
+        DRYRUN="true"
+        ;;
+      -h | --help) shift
+        usage
+        exit 0
+        ;;
+      --) shift
+        break
+        ;;
+      *)
+        echo "Invalid argument: $1"
+        usage
+        exit 1
+        ;;
+    esac
+  done
+
+  if [ ! -d "$BUILDDIR" ]; then
+    fail "Told to build in $BUILDDIR, but that directory doesn't exist."
+  fi
+
+  if [ "$NGINX_VERSION" = "latest" ]; then
+    # When this function fails it prints the debugging information needed first
+    # to stderr.
+    NGINX_VERSION=$(determine_latest_nginx_version) || exit 1
+  fi
+
+  if "$DYNAMIC_MODULE"; then
+    # Check that ngx_pagespeed and nginx are recent enough to support dynamic
+    # modules.
+    if version_older_than "$NPS_VERSION" "1.10.33.5"; then
+      fail "
+You're trying to build ngx_pagespeed $NPS_VERSION as a dynamic module, but
+ngx_pagespeed didn't add support for dynamic modules until 1.10.33.5."
+    fi
+
+    if [ ! -z "NGINX_VERSION" ]; then
+      if version_older_than "$NGINX_VERSION" "1.9.13"; then
+        fail "
+You're trying to build nginx $NGINX_VERSION as a dynamic module but nginx didn't
+add support for dynamic modules in a way compatible with ngx_pagespeed until
+1.9.13."
+      fi
+    fi
+  fi
+
+  if "$DRYRUN"; then
+    TEMPDIR="/tmp/output-of-mktemp"
+  else
+    TEMPDIR=$(mktemp -d)
+    function cleanup_tempdir {
+      rm -rf "$TEMPDIR"
+    }
+    trap cleanup_tempdir EXIT
+  fi
+
+  extra_flags=()
+  # Now make sure our dependencies are installed.
+  if "$DO_DEPS_CHECK"; then
+    if [ -f /etc/debian_version ]; then
+      echo "Detected debian-based distro."
+
+      install_dependencies "apt-get install" debian_is_installed \
+        "build-essential zlib1g-dev libpcre3 libpcre3-dev unzip"
+
+      if gcc_too_old; then
+        if [ ! -e /usr/lib/gcc-mozilla/bin/gcc ]; then
+          echo "Detected that gcc is older than 4.8.  Installing gcc-mozilla"
+          echo "which installs gcc-4.8 into /usr/lib/gcc-mozilla/ and doesn't"
+          echo "affect your global gcc installation."
+          run sudo apt-get install gcc-mozilla
+        fi
+
+        extra_flags=("--with-cc=/usr/lib/gcc-mozilla/bin/gcc" \
+                     "--with-ld-opt=-static-libstdc++")
+      fi
+
+    elif [ -f /etc/redhat-release ]; then
+      echo "Detected redhat-based distro."
+
+      install_dependencies "yum install" redhat_is_installed \
+        "gcc-c++ pcre-devel zlib-devel make unzip wget"
+      if gcc_too_old; then
+        if [ ! -e /opt/rh/devtoolset-2/root/usr/bin/gcc ]; then
+          redhat_major_version=$(
+            cat /etc/redhat-release | grep -o -E '[0-9]+' | head -n 1)
+          if [ "$redhat_major_version" == 5 ]; then
+            slc_version=5
+          elif [ "$redhat_major_version" == 6 ]; then
+            slc_version=6
+          else
+            fail "
+Unexpected major version $redhat_major_version in /etc/redhat-release:
+$(cat /etc/redhat-release) Expected 5 or 6."
+          fi
+
+          echo "Detected that gcc is older than 4.8.  Scientific Linux provides"
+          echo "a gcc package that installs gcc-4.8 into /opt/ and doesn't"
+          echo "affect your global gcc installation."
+          slc_key="https://linux.web.cern.ch/linux/scientific6/docs/repository/"
+          slc_key+="cern/slc6X/i386/RPM-GPG-KEY-cern"
+          slc_key_out="$TEMPDIR/RPM-GPG-KEY-cern"
+          run sudo wget "$slc_key" -O "$slc_key_out"
+          run sudo rpm --import "$slc_key_out"
+
+          repo_fname="/etc/yum.repos.d/slc${slc_version}-devtoolset.repo"
+          if [ -e "$repo_fname" ]; then
+            fail "Expected $repo_fname not to exist; aborting."
+          fi
+
+          repo_url="https://linux.web.cern.ch/linux/scientific${slc_version}/"
+          repo_url+="/docs/repository/cern/devtoolset/"
+          repo_url+="slc${slc_version}-devtoolset.repo"
+          run sudo wget -O "$repo_fname" "$repo_url"
+          run sudo yum install devtoolset-2-gcc-c++ devtoolset-2-binutils
+        fi
+        extra_flags=("--with-cc=/opt/rh/devtoolset-2/root/usr/bin/gcc")
+      fi
+    else
+      fail "
+This doesn't appear to be a deb-based distro or an rpm-based one.  Not going to
+be able to install dependencies.  Please install dependencies manually and rerun
+with --depsinstalled."
+    fi
+    echo "Dependencies are all set."
+  else
+    echo "Not checking whether dependencies are installed."
+  fi
+
+  function delete_if_already_exists() {
+    if "$DRYRUN"; then return; fi
+
+    local directory="$1"
+    if [ -d "$directory" ]; then
+      if [ ${#directory} -lt 8 ]; then
+        fail "
+Not deleting $directory; name is suspiciously short.  Something is wrong."
+        exit 1
+      fi
+
+      continue_or_exit "OK to delete $directory?"
+      run rm -rf "$directory"
+    fi
+  }
+
+  nps_baseurl="https://github.com/pagespeed/ngx_pagespeed/archive"
+  # In general, the zip github builds for tag foo unzips to ngx_pagespeed-foo,
+  # but it looks like they special case vVERSION tags to ngx_pagespeed-VERSION.
+  if [[ "$NPS_VERSION" =~ ^[0-9]*[.][0-9]*[.][0-9]*[.][0-9]*$ ]]; then
+    # We've been given a numeric version number.  This has an associated tag in
+    # the form vVERSION-beta.
+    nps_url_fname="v${NPS_VERSION}-beta"
+    nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}-beta"
+  else
+    # We've been given a tag name, like latest-beta.  Download that directly.
+    nps_url_fname="$NPS_VERSION"
+    nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}"
+  fi
+
+  nps_downloaded="$TEMPDIR/$nps_downloaded_fname.zip"
+  run wget "$nps_baseurl/$nps_url_fname.zip" -O "$nps_downloaded"
+  nps_module_dir="$BUILDDIR/$nps_downloaded_fname"
+  delete_if_already_exists "$nps_module_dir"
+  echo "Extracting ngx_pagespeed..."
+  run unzip -q "$nps_downloaded" -d "$BUILDDIR"
+  run cd "$nps_module_dir"
+
+  # Now we need to figure out what precompiled version of PSOL to build
+  # ngx_pagespeed against.
+  if "$DRYRUN"; then
+    psol_url="https://psol.example.com/cant-get-psol-version-in-dry-run.tar.gz"
+  elif [ -e PSOL_BINARY_URL ]; then
+    # Releases after 1.11.33.4 there is a PSOL_BINARY_URL file that tells us
+    # where to look.
+    psol_url="$(cat PSOL_BINARY_URL)"
+    if [[ "$psol_url" != https://* ]]; then
+      fail "Got bad psol binary location information: $psol_url"
+    fi
+  else
+    # For past releases we have to grep it from the config file.  The url has
+    # always looked like this, and the config file has contained it since before
+    # we started tagging our ngx_pagespeed releases.
+    psol_url="$(
+      grep -o "https://dl.google.com/dl/page-speed/psol/[0-9.]*.tar.gz" config)"
+    if [ -z "$psol_url" ]; then
+      fail "Couldn't find PSOL url in $PWD/config"
+    fi
+  fi
+
+  run wget "$psol_url"
+  echo "Extracting PSOL..."
+  run tar -xzf $(basename "$psol_url")  # extracts to psol/
+
+  if "$DYNAMIC_MODULE"; then
+    add_module="--add-dynamic-module=$nps_module_dir"
+  else
+    add_module="--add-module=$nps_module_dir"
+  fi
+  configure_args=("$add_module" "${extra_flags[@]}")
+
+  echo
+  if [ -z "$NGINX_VERSION" ]; then
+    # They didn't specify an nginx version, so we're just preparing the
+    # module for them to install.
+    echo "ngx_pagespeed is ready to be built against nginx."
+    echo "When running ./configure pass in:"
+    echo "  $(quote_arguments "${configure_args[@]}")"
+    if [ ${#extra_flags[@]} -eq 0 ]; then
+      echo "If this is for integration with an already-built nginx, make sure"
+      echo "to include any other arguments you originally passed to ./configure"
+      echo "You can see these with 'nginx -V'."
+    else
+      echo "Note: because we need to set $(quote_arguments "${extra_flags[@]}")"
+      echo "on this platform, if you want to integrate ngx_pagespeed with an"
+      echo "already-built nginx you're going to need to rebuild your nginx with"
+      echo "those flags set."
+    fi
+  else
+    # Download and build nginx.
+    nginx_leaf="nginx-${NGINX_VERSION}.tar.gz"
+    nginx_fname="$TEMPDIR/$nginx_leaf"
+    run wget "http://nginx.org/download/$nginx_leaf" -O "$nginx_fname"
+    nginx_dir="$BUILDDIR/nginx-${NGINX_VERSION}/"
+    delete_if_already_exists "$nginx_dir"
+    echo "Extracting nginx..."
+    run tar -xzf "$nginx_fname" --directory "$BUILDDIR"
+    "$DRYRUN" || cd "$nginx_dir"
+
+    configure=("./configure" "${configure_args[@]}")
+    echo "About to build nginx.  Do you have any additional ./configure"
+    echo "arguments you would like to set?  For example, if you would like"
+    echo "to build nginx with https support give --with-http_ssl_module"
+    echo "If you don't have any, just press enter."
+    read -p "> " additional_configure_args
+    if [ -n "$additional_configure_args" ]; then
+      # Split additional_configure_args respecting any internal quotation.
+      # Otherwise things like --with-cc-opt='-foo -bar' won't work.
+      eval additional_configure_args=("$additional_configure_args")
+      configure=("${configure[@]}" "${additional_configure_args[@]}")
+    fi
+    echo "About to configure nginx with:"
+    echo "   $(quote_arguments "${configure[@]}")"
+    continue_or_exit "Does this look right?"
+    run "${configure[@]}"
+
+    continue_or_exit "Build nginx?"
+    run make
+
+    continue_or_exit "Install nginx?"
+    run sudo make install
+
+    echo
+    if "$DYNAMIC_MODULE"; then
+      echo "Nginx installed with ngx_pagespeed support available as a"
+      echo "loadable module."
+      echo
+      echo "To load the ngx_pagespeed module, you'll need to add:"
+      echo "  load_module \"modules/ngx_pagespeed.so\";"
+      echo "at the top of your main nginx configuration file."
+    else
+      echo "Nginx installed with ngx_pagespeed support compiled-in."
+    fi
+    echo
+    echo "If this is a new installation you probably need an init script to"
+    echo "manage starting and stopping the nginx service.  See:"
+    echo "  http://wiki.nginx.org/InitScripts"
+    echo
+    echo "You'll also need to configure ngx_pagespeed if you haven't yet:"
+    echo "  https://developers.google.com/speed/pagespeed/module/configuration"
+  fi
+  "$DRYRUN" && echo "[this was a dry run; your system is unchanged]"
+}
+
+# Start running things from a call at the end so if this script is executed
+# after a partial download it doesn't do anything.
+build_ngx_pagespeed "$@"

scripts/pagespeed_libraries_generator.sh

@@ -13,9 +13,9 @@
 # Author: vid@zippykid.com (Vid Luther)
 #         jefftk@google.com (Jeff Kaufman)
 
-URL="https://modpagespeed.googlecode.com/svn/trunk/src/"
+URL="https://github.com/pagespeed/mod_pagespeed/raw/master/"
 URL+="net/instaweb/genfiles/conf/pagespeed_libraries.conf"
-curl -s "$URL" \
+curl -L -s -S "$URL" \
     | grep ModPagespeedLibrary \
     | while read library size hash url ; do
   echo "  pagespeed Library $size $hash $url;"

src/log_message_handler.cc

@@ -29,7 +29,7 @@
 #include "base/debug/stack_trace.h"
 #include "base/logging.h"
 #include "net/instaweb/public/version.h"
-#include "net/instaweb/util/public/string_util.h"
+#include "pagespeed/kernel/base/string_util.h"
 
 // Make sure we don't attempt to use LOG macros here, since doing so
 // would cause us to go into an infinite log loop.
@@ -38,7 +38,7 @@
 
 namespace {
 
-ngx_log_t* log = NULL;
+ngx_log_t* ngx_log = NULL;
 
 ngx_uint_t GetNgxLogLevel(int severity) {
   switch (severity) {
@@ -78,7 +78,7 @@ bool LogMessageHandler(int severity, const char* file, int line,
     message.resize(last_msg_character_index);
   }
 
-  ngx_log_error(this_log_level, log, 0, "[ngx_pagespeed %s] %s",
+  ngx_log_error(this_log_level, ngx_log, 0, "[ngx_pagespeed %s] %s",
                 net_instaweb::kModPagespeedVersion,
                 message.c_str());
 
@@ -99,12 +99,12 @@ namespace log_message_handler {
 
 
 void Install(ngx_log_t* log_in) {
-  log = log_in;
+  ngx_log = log_in;
   logging::SetLogMessageHandler(&LogMessageHandler);
 
   // All VLOG(2) and higher will be displayed as DEBUG logs if the nginx log
   // level is DEBUG.
-  if (log->log_level >= NGX_LOG_DEBUG) {
+  if (ngx_log->log_level >= NGX_LOG_DEBUG) {
     logging::SetMinLogLevel(-2);
   }
 }

src/ngx_base_fetch.cc

@@ -12,40 +12,193 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
+ * Author: jefftk@google.com (Jeff Kaufman)
  */
 
-// Author: jefftk@google.com (Jeff Kaufman)
+#include "ngx_pagespeed.h"  // Must come first, see comments in CollectHeaders.
+
+#include <unistd.h> //for usleep
 
 #include "ngx_base_fetch.h"
+#include "ngx_event_connection.h"
 #include "ngx_list_iterator.h"
 
-#include "ngx_pagespeed.h"
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
-
+#include "net/instaweb/rewriter/public/rewrite_options.h"
-#include "net/instaweb/http/public/response_headers.h"
 #include "net/instaweb/rewriter/public/rewrite_stats.h"
-#include "net/instaweb/util/public/google_message_handler.h"
+#include "pagespeed/kernel/base/google_message_handler.h"
-#include "net/instaweb/util/public/message_handler.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/posix_timer.h"
+#include "pagespeed/kernel/http/response_headers.h"
 
 namespace net_instaweb {
 
-NgxBaseFetch::NgxBaseFetch(ngx_http_request_t* r, int pipe_fd,
+const char kHeadersComplete = 'H';
+const char kFlush = 'F';
+const char kDone = 'D';
+
+NgxEventConnection* NgxBaseFetch::event_connection = NULL;
+int NgxBaseFetch::active_base_fetches = 0;
+
+NgxBaseFetch::NgxBaseFetch(StringPiece url,
+                           ngx_http_request_t* r,
                            NgxServerContext* server_context,
                            const RequestContextPtr& request_ctx,
-                           PreserveCachingHeaders preserve_caching_headers)
+                           PreserveCachingHeaders preserve_caching_headers,
+                           NgxBaseFetchType base_fetch_type,
+                           const RewriteOptions* options)
     : AsyncFetch(request_ctx),
+      url_(url.data(), url.size()),
       request_(r),
       server_context_(server_context),
+      options_(options),
+      need_flush_(false),
       done_called_(false),
       last_buf_sent_(false),
-      pipe_fd_(pipe_fd),
       references_(2),
-      handle_error_(true),
+      base_fetch_type_(base_fetch_type),
-      preserve_caching_headers_(preserve_caching_headers) {
+      preserve_caching_headers_(preserve_caching_headers),
+      detached_(false),
+      suppress_(false) {
   if (pthread_mutex_init(&mutex_, NULL)) CHECK(0);
+  __sync_add_and_fetch(&NgxBaseFetch::active_base_fetches, 1);
 }
 
 NgxBaseFetch::~NgxBaseFetch() {
   pthread_mutex_destroy(&mutex_);
+  __sync_add_and_fetch(&NgxBaseFetch::active_base_fetches, -1);
+}
+
+bool NgxBaseFetch::Initialize(ngx_cycle_t* cycle) {
+  CHECK(event_connection == NULL) << "event connection already set";
+  event_connection = new NgxEventConnection(ReadCallback);
+  return event_connection->Init(cycle);
+}
+
+void NgxBaseFetch::Terminate() {
+  if (event_connection != NULL) {
+    GoogleMessageHandler handler;
+    PosixTimer timer;
+    int64 timeout_us = Timer::kSecondUs * 30;
+    int64 end_us = timer.NowUs() + timeout_us;
+    static unsigned int sleep_microseconds = 100;
+
+    handler.Message(
+        kInfo,"NgxBaseFetch::Terminate rounding up %d active base fetches.",
+        NgxBaseFetch::active_base_fetches);
+
+    // Try to continue processing and get the active base fetch count to 0
+    // untill the timeout expires.
+    // TODO(oschaaf): This needs more work.
+    while (NgxBaseFetch::active_base_fetches > 0 && end_us > timer.NowUs()) {
+      event_connection->Drain();
+      usleep(sleep_microseconds);
+    }
+
+    if (NgxBaseFetch::active_base_fetches != 0) {
+      handler.Message(
+          kWarning,"NgxBaseFetch::Terminate timed out with %d active base fetches.",
+          NgxBaseFetch::active_base_fetches);
+    }
+
+    // Close down the named pipe.
+    event_connection->Shutdown();
+    delete event_connection;
+    event_connection = NULL;
+  }
+}
+
+const char* BaseFetchTypeToCStr(NgxBaseFetchType type) {
+  switch(type) {
+    case kPageSpeedResource:
+      return "ps resource";
+    case kHtmlTransform:
+      return "html transform";
+    case kAdminPage:
+      return "admin page";
+    case kIproLookup:
+      return "ipro lookup";
+    case kPageSpeedProxy:
+      return "pagespeed proxy";
+  }
+  CHECK(false);
+  return "can't get here";
+}
+
+// TODO(oschaaf): replace the ngx_log_error with VLOGS or pass in a
+// MessageHandler and use that.
+void NgxBaseFetch::ReadCallback(const ps_event_data& data) {
+  NgxBaseFetch* base_fetch = reinterpret_cast<NgxBaseFetch*>(data.sender);
+  ngx_http_request_t* r = base_fetch->request();
+  bool detached = base_fetch->detached();
+#if (NGX_DEBUG)  // `type` is unused if NGX_DEBUG isn't set, needed for -Werror.
+  const char* type = BaseFetchTypeToCStr(base_fetch->base_fetch_type_);
+#endif
+  int refcount = base_fetch->DecrementRefCount();
+
+#if (NGX_DEBUG)
+  ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
+     "pagespeed [%p] event: %c. bf:%p (%s) - refcnt:%d - det: %c", r,
+     data.type, base_fetch, type, refcount, detached ? 'Y': 'N');
+#endif
+
+  // If we ended up destructing the base fetch, or the request context is
+  // detached, skip this event.
+  if (refcount == 0 || detached) {
+    return;
+  }
+
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+
+  // If our request context was zeroed, skip this event.
+  // See https://github.com/pagespeed/ngx_pagespeed/issues/1081
+  if (ctx == NULL) {
+    // Should not happen normally, when it does this message will cause our
+    // system tests to fail.
+    ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
+        "pagespeed [%p] skipping event: request context gone", r);
+    return;
+  }
+
+  // Normally we expect the sender to equal the active NgxBaseFetch instance.
+  DCHECK(data.sender == ctx->base_fetch);
+
+  // If someone changed our request context or NgxBaseFetch, skip processing.
+  if (data.sender != ctx->base_fetch) {
+      ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
+          "pagespeed [%p] skipping event: event originating from disassociated"
+          " NgxBaseFetch instance.", r);
+      return;
+  }
+
+  int rc;
+  bool run_posted = true;
+  // If we are unlucky enough to have our connection finalized mid-ipro-lookup,
+  // we must enter a different flow. Also see ps_in_place_check_header_filter().
+  if ((ctx->base_fetch->base_fetch_type_ != kIproLookup)
+      && r->connection->error) {
+    ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
+      "pagespeed [%p] request already finalized %d", r, r->count);
+    rc = NGX_ERROR;
+    run_posted = false;
+  } else {
+    rc = ps_base_fetch::ps_base_fetch_handler(r);
+  }
+
+#if (NGX_DEBUG)
+  ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
+                "pagespeed [%p] ps_base_fetch_handler() returned %d for %c",
+                r, rc, data.type);
+#endif
+
+  ngx_connection_t* c = r->connection;
+  ngx_http_finalize_request(r, rc);
+
+  if (run_posted) {
+    // See http://forum.nginx.org/read.php?2,253006,253061
+    ngx_http_run_posted_requests(c);
+  }
 }
 
 void NgxBaseFetch::Lock() {
@@ -75,8 +228,11 @@ ngx_int_t NgxBaseFetch::CopyBufferToNginx(ngx_chain_t** link_ptr) {
     return NGX_AGAIN;
   }
 
-  int rc = string_piece_to_buffer_chain(
+  int rc = string_piece_to_buffer_chain(request_->pool, buffer_, link_ptr,
-      request_->pool, buffer_, link_ptr, done_called_ /* send_last_buf */);
+                                        done_called_ /* send_last_buf */,
+                                        need_flush_);
+  need_flush_ = false;
+
   if (rc != NGX_OK) {
     return rc;
   }
@@ -104,6 +260,13 @@ ngx_int_t NgxBaseFetch::CollectAccumulatedWrites(ngx_chain_t** link_ptr) {
 }
 
 ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
+  // nginx defines _FILE_OFFSET_BITS to 64, which changes the size of off_t.
+  // If a standard header is accidentally included before the nginx header,
+  // on a 32-bit system off_t will be 4 bytes and we don't assign all the
+  // bits of content_length_n. Sanity check that did not happen.
+  // This could use static_assert, but this file is not built with --std=c++11.
+  bool sanity_check_off_t[sizeof(off_t) == 8 ? 1 : -1] __attribute__ ((unused));
+
   const ResponseHeaders* pagespeed_headers = response_headers();
 
   if (content_length_known()) {
@@ -115,21 +278,19 @@ ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
                                       preserve_caching_headers_);
 }
 
-void NgxBaseFetch::RequestCollection() {
+void NgxBaseFetch::RequestCollection(char type) {
-  int rc;
+  if (suppress_) {
-  char c = 'A';  // What byte we write is arbitrary.
+    return;
-  while (true) {
+  }
-    rc = write(pipe_fd_, &c, 1);
-    if (rc == 1) {
-      break;
-    } else if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
-      // TODO(jefftk): is this rare enough that spinning isn't a problem?  Could
-      // we get into a case where the pipe fills up and we spin forever?
 
-    } else {
+  // We must optimistically increment the refcount, and decrement it
-      perror("NgxBaseFetch::RequestCollection");
+  // when we conclude we failed. If we only increment on a successfull write,
-      break;
+  // there's a small chance that between writing and adding to the refcount
-    }
+  // both pagespeed and nginx will release their refcount -- destructing
+  // this NgxBaseFetch instance.
+  IncrementRefCount();
+  if (!event_connection->WriteEvent(type, this)) {
+    DecrementRefCount();
   }
 }
 
@@ -137,43 +298,63 @@ void NgxBaseFetch::HandleHeadersComplete() {
   int status_code = response_headers()->status_code();
   bool status_ok = (status_code != 0) && (status_code < 400);
 
-  if (status_ok || handle_error_) {
+  if ((base_fetch_type_ != kIproLookup) || status_ok) {
     // If this is a 404 response we need to count it in the stats.
     if (response_headers()->status_code() == HttpStatus::kNotFound) {
       server_context_->rewrite_stats()->resource_404_count()->Add(1);
     }
   }
 
-  RequestCollection();  // Headers available.
+  RequestCollection(kHeadersComplete);  // Headers available.
+
+  // For the IPRO lookup, supress notification of the nginx side here.
+  // If we send both the headerscomplete event and the one from done, nasty
+  // stuff will happen if we loose the race with with the nginx side destructing
+  // this base fetch instance.
+  if (base_fetch_type_ == kIproLookup && !status_ok) {
+    suppress_ = true;
+  }
 }
 
 bool NgxBaseFetch::HandleFlush(MessageHandler* handler) {
-  RequestCollection();  // A new part of the response body is available.
+  Lock();
+  need_flush_ = true;
+  Unlock();
+  RequestCollection(kFlush);  // A new part of the response body is available
   return true;
 }
 
-void NgxBaseFetch::Release() {
+int NgxBaseFetch::DecrementRefCount() {
-  DecrefAndDeleteIfUnreferenced();
+  return DecrefAndDeleteIfUnreferenced();
 }
 
-void NgxBaseFetch::DecrefAndDeleteIfUnreferenced() {
+int NgxBaseFetch::IncrementRefCount() {
+  return __sync_add_and_fetch(&references_, 1);
+}
+
+int NgxBaseFetch::DecrefAndDeleteIfUnreferenced() {
   // Creates a full memory barrier.
-  if (__sync_add_and_fetch(&references_, -1) == 0) {
+  int r = __sync_add_and_fetch(&references_, -1);
+  if (r == 0) {
     delete this;
   }
+  return r;
 }
 
 void NgxBaseFetch::HandleDone(bool success) {
   // TODO(jefftk): it's possible that instead of locking here we can just modify
   // CopyBufferToNginx to only read done_called_ once.
+  CHECK(!done_called_) << "Done already called!";
   Lock();
   done_called_ = true;
   Unlock();
-
+  RequestCollection(kDone);
-  close(pipe_fd_);  // Indicates to nginx that we're done with the rewrite.
-  pipe_fd_ = -1;
-
   DecrefAndDeleteIfUnreferenced();
 }
 
+bool NgxBaseFetch::IsCachedResultValid(const ResponseHeaders& headers) {
+  return OptionsAwareHTTPCacheCallback::IsCacheValid(
+      url_, *options_, request_context(), headers);
+}
+
 }  // namespace net_instaweb

src/ngx_base_fetch.h

@@ -17,22 +17,35 @@
 // Author: jefftk@google.com (Jeff Kaufman)
 //
 // Collects output from pagespeed and buffers it until nginx asks for it.
-// Notifies nginx via pipe to call CollectAccumulatedWrites() on flush.
+// Notifies nginx via NgxEventConnection to call ReadCallback() when
+// the headers are computed, when a flush should be performed, and when done.
 //
 //  - nginx creates a base fetch and passes it to a new proxy fetch.
 //  - The proxy fetch manages rewriting and thread complexity, and through
 //    several chained steps passes rewritten html to HandleWrite().
 //  - Written data is buffered.
-//  - When Flush() is called the base fetch writes a byte to a pipe nginx is
+//  - When HandleHeadersComplete(), HandleFlush(), or HandleDone() is called by
-//    watching so nginx knows to call CollectAccumulatedWrites() to pick up the
+//    PSOL, events are written to NgxEventConnection which will end up being
-//    rewritten html.
+//    handled by ReadCallback() on nginx's thread.
-//  - When Done() is called the base fetch closes the pipe, which tells nginx to
+//    When applicable, request processing will be continued via a call to
-//    make a final call to CollectAccumulatedWrites().
+//    ps_base_fetch_handler().
+//  - ps_base_fetch_handler() will pull the header and body bytes from PSOL
+//    via CollectAccumulatedWrites() and write those to the module's output.
 //
-// This class is referred two in two places: the proxy fetch and nginx's
+// This class is referred to in three places: the proxy fetch, nginx's request,
-// request.  It must stay alive until both are finished.  The proxy fetch will
+// and pending events written to the associated NgxEventConnection. It must stay
-// call Done() to indicate this; nginx will call Release().  Once both Done()
+// alive until the proxy fetch and nginx request are finished, and no more
-// and Release() have been called this class will delete itself.
+// events are pending.
+//  - The proxy fetch will call Done() to indicate this.
+//  - nginx will call Detach() when the associated request is handled
+//    completely (e.g. the request context is about to be destroyed).
+//  - ReadCallback() will call DecrementRefCount() on instances associated to
+//    events it handles.
+//
+// When the last reference is dropped, this class will delete itself.
+//
+// TODO(jmarantz): consider sharing the cache-invalidation infrastructure
+// with ApacheFetch, using a common base class.
 
 #ifndef NGX_BASE_FETCH_H_
 #define NGX_BASE_FETCH_H_
@@ -45,22 +58,46 @@ extern "C" {
 
 #include "ngx_pagespeed.h"
 
+#include "ngx_event_connection.h"
 #include "ngx_server_context.h"
 
 #include "net/instaweb/http/public/async_fetch.h"
-#include "net/instaweb/http/public/headers.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
-#include "net/instaweb/util/public/string.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/http/headers.h"
 
 namespace net_instaweb {
 
+enum NgxBaseFetchType {
+  kIproLookup,
+  kHtmlTransform,
+  kPageSpeedResource,
+  kAdminPage,
+  kPageSpeedProxy
+};
+
 class NgxBaseFetch : public AsyncFetch {
  public:
-  NgxBaseFetch(ngx_http_request_t* r, int pipe_fd,
+  NgxBaseFetch(StringPiece url, ngx_http_request_t* r,
                NgxServerContext* server_context,
                const RequestContextPtr& request_ctx,
-               PreserveCachingHeaders preserve_caching_headers);
+               PreserveCachingHeaders preserve_caching_headers,
+               NgxBaseFetchType base_fetch_type,
+               const RewriteOptions* options);
   virtual ~NgxBaseFetch();
 
+  // Statically initializes event_connection, require for PSOL and nginx to
+  // communicate.
+  static bool Initialize(ngx_cycle_t* cycle);
+
+  // Attempts to finish up request processing queued up in the named pipe and
+  // PSOL for a fixed amount of time. If time is up, a fast and rough shutdown
+  // is attempted.
+  // Statically terminates and NULLS event_connection.
+  static void Terminate();
+
+  static void ReadCallback(const ps_event_data& data);
+
   // Puts a chain in link_ptr if we have any output data buffered.  Returns
   // NGX_OK on success, NGX_ERROR on errors.  If there's no data to send, sends
   // data only if Done() has been called.  Indicates the end of output by
@@ -77,9 +114,24 @@ class NgxBaseFetch : public AsyncFetch {
   // time for resource fetches.  Not called at all for proxy fetches.
   ngx_int_t CollectHeaders(ngx_http_headers_out_t* headers_out);
 
-  // Called by nginx when it's done with us.
+  // Called by nginx to decrement the refcount.
-  void Release();
+  int DecrementRefCount();
-  void set_handle_error(bool x) { handle_error_ = x; }
+
+  // Called by pagespeed to increment the refcount.
+  int IncrementRefCount();
+
+  // Detach() is called when the nginx side releases this base fetch. It
+  // sets detached_ to true and decrements the refcount. We need to know
+  // this to be able to handle events which nginx request context has been
+  // released while the event was in-flight.
+  void Detach() { detached_ = true; DecrementRefCount(); }
+
+  bool detached() { return detached_; }
+
+  ngx_http_request_t* request() { return request_; }
+  NgxBaseFetchType base_fetch_type() { return base_fetch_type_; }
+
+  bool IsCachedResultValid(const ResponseHeaders& headers) override;
 
  private:
   virtual bool HandleWrite(const StringPiece& sp, MessageHandler* handler);
@@ -89,7 +141,7 @@ class NgxBaseFetch : public AsyncFetch {
 
   // Indicate to nginx that we would like it to call
   // CollectAccumulatedWrites().
-  void RequestCollection();
+  void RequestCollection(char type);
 
   // Lock must be acquired first.
   // Returns:
@@ -105,20 +157,32 @@ class NgxBaseFetch : public AsyncFetch {
 
   // Called by Done() and Release().  Decrements our reference count, and if
   // it's zero we delete ourself.
-  void DecrefAndDeleteIfUnreferenced();
+  int DecrefAndDeleteIfUnreferenced();
 
+  static NgxEventConnection* event_connection;
+
+  // Live count of NgxBaseFetch instances that are currently in use.
+  static int active_base_fetches;
+
+  GoogleString url_;
   ngx_http_request_t* request_;
   GoogleString buffer_;
   NgxServerContext* server_context_;
+  const RewriteOptions* options_;
+  bool need_flush_;
   bool done_called_;
   bool last_buf_sent_;
-  int pipe_fd_;
   // How many active references there are to this fetch. Starts at two,
-  // decremented once when Done() is called and once when Release() is called.
+  // decremented once when Done() is called and once when Detach() is called.
+  // Incremented for each event written by pagespeed for this NgxBaseFetch, and
+  // decremented on the nginx side for each event read for it.
   int references_;
   pthread_mutex_t mutex_;
-  bool handle_error_;
+  NgxBaseFetchType base_fetch_type_;
   PreserveCachingHeaders preserve_caching_headers_;
+  // Set to true just before the nginx side releases its reference
+  bool detached_;
+  bool suppress_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxBaseFetch);
 };

src/ngx_event_connection.cc

@@ -0,0 +1,172 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+
+extern "C" {
+
+#include <ngx_channel.h>
+
+}
+
+#include "ngx_event_connection.h"
+
+#include "pagespeed/kernel/base/google_message_handler.h"
+#include "pagespeed/kernel/base/message_handler.h"
+
+namespace net_instaweb {
+
+  NgxEventConnection::NgxEventConnection(callbackPtr callback)
+    : event_handler_(callback) {
+}
+
+bool NgxEventConnection::Init(ngx_cycle_t* cycle) {
+  int file_descriptors[2];
+
+  if (pipe(file_descriptors) != 0) {
+    ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, "pagespeed: pipe() failed");
+    return false;
+  }
+  if (ngx_nonblocking(file_descriptors[0]) == -1) {
+    ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
+                  ngx_nonblocking_n "pagespeed:  pipe[0] failed");
+  } else if (ngx_nonblocking(file_descriptors[1]) == -1) {
+    ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
+                  ngx_nonblocking_n "pagespeed:  pipe[1] failed");
+  } else if (!CreateNgxConnection(cycle, file_descriptors[0])) {
+    ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
+                  "pagespeed: failed to create connection.");
+  } else {
+    pipe_read_fd_ = file_descriptors[0];
+    pipe_write_fd_ = file_descriptors[1];
+    return true;
+  }
+  close(file_descriptors[0]);
+  close(file_descriptors[1]);
+  return false;
+}
+
+bool NgxEventConnection::CreateNgxConnection(ngx_cycle_t* cycle,
+                                             ngx_fd_t pipe_fd) {
+  // pipe_fd (the read side of the pipe will end up as c->fd on the
+  // underlying ngx_connection_t that gets created here)
+  ngx_int_t rc = ngx_add_channel_event(cycle, pipe_fd, NGX_READ_EVENT,
+      &NgxEventConnection::ReadEventHandler);
+  return rc  == NGX_OK;
+}
+
+void NgxEventConnection::ReadEventHandler(ngx_event_t* ev) {
+  ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
+  ngx_int_t result = ngx_handle_read_event(ev, 0);
+  if (result != NGX_OK) {
+    CHECK(false) << "pagespeed: ngx_handle_read_event error: " << result;
+  }
+
+  if (ev->timedout) {
+    ev->timedout = 0;
+    return;
+  }
+
+  if (!NgxEventConnection::ReadAndNotify(c->fd)) {
+    // This was copied from ngx_channel_handler(): for epoll, we need to call
+    // ngx_del_conn(). Sadly, no documentation as to why.
+    if (ngx_event_flags & NGX_USE_EPOLL_EVENT) {
+      ngx_del_conn(c, 0);
+    }
+    ngx_close_connection(c);
+    ngx_del_event(ev, NGX_READ_EVENT, 0);
+  }
+}
+
+// Deserialize ps_event_data's from the pipe as they become available.
+// Subsequently do some bookkeeping, cleanup, and error checking to keep
+// the mess out of ps_base_fetch_handler.
+bool NgxEventConnection::ReadAndNotify(ngx_fd_t fd) {
+  while (true) {
+    // We read only one ps_event_data at a time for now:
+    // We can end up recursing all the way and end up calling ourselves here.
+    // If that happens in the middle of looping over multiple ps_event_data's we
+    // have obtained with read(), the results from the next read() will make us
+    // process events out of order. Which can give headaches.
+    // Alternatively, we could maintain a queue to make sure we process in
+    // sequence
+    ps_event_data data;
+    ngx_int_t size = read(fd, static_cast<void*>(&data), sizeof(data));
+
+    if (size == -1) {
+      if (errno == EINTR) {
+        continue;
+      // TODO(oschaaf): should we worry about spinning here?
+      } else if (ngx_errno == EAGAIN || ngx_errno == EWOULDBLOCK) {
+        return true;
+      }
+    }
+
+    if (size <= 0) {
+      return false;
+    }
+
+    data.connection->event_handler_(data);
+    return true;
+  }
+}
+
+bool NgxEventConnection::WriteEvent(void* sender) {
+  return WriteEvent('X' /* Anything char is fine */, sender);
+}
+
+bool NgxEventConnection::WriteEvent(char type, void* sender) {
+  ssize_t size = 0;
+  ps_event_data data;
+
+  ngx_memzero(&data, sizeof(data));
+  data.type = type;
+  data.sender = sender;
+  data.connection = this;
+
+  while (true) {
+    size = write(pipe_write_fd_,
+                 static_cast<void*>(&data), sizeof(data));
+    if (size == sizeof(data)) {
+      return true;
+    } else if (size == -1) {
+      // TODO(oschaaf): should we worry about spinning here?
+      if (ngx_errno == EINTR || ngx_errno == EAGAIN
+          || ngx_errno == EWOULDBLOCK) {
+        continue;
+      } else {
+        return false;
+      }
+    } else {
+      CHECK(false) << "pagespeed: unexpected return value from write(): "
+                   << size;
+    }
+  }
+  CHECK(false) << "Should not get here";
+  return false;
+}
+
+// Reads and processes what is available in the pipe.
+void NgxEventConnection::Drain() {
+  NgxEventConnection::ReadAndNotify(pipe_read_fd_);
+}
+
+void NgxEventConnection::Shutdown() {
+  close(pipe_write_fd_);
+  close(pipe_read_fd_);
+}
+
+}  // namespace net_instaweb

src/ngx_event_connection.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+//
+// NgxEventConnection implements a means to send events from other threads to
+// nginx's event loop, and is implemented by a named pipe under the hood.
+// A single instance is used by NgxBaseFetch, and one instance is created per
+// NgxUrlAsyncFetcher when native fetching is on.
+
+#ifndef NGX_EVENT_CONNECTION_H_
+#define NGX_EVENT_CONNECTION_H_
+
+extern "C" {
+#include <ngx_http.h>
+}
+
+#include <pthread.h>
+
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/http/headers.h"
+
+namespace net_instaweb {
+
+class NgxEventConnection;
+
+// Represents a single event that can be written to or read from the pipe.
+// Technically, sender is the only data we need to send. type and connection are
+// included to provide a means to trace the events along with some more
+// info.
+typedef struct {
+  char type;
+  void* sender;
+  NgxEventConnection* connection;
+} ps_event_data;
+
+// Handler signature for receiving events
+typedef void (*callbackPtr)(const ps_event_data&);
+
+// Abstracts a connection to nginx through which events can be written.
+class NgxEventConnection {
+ public:
+  explicit NgxEventConnection(callbackPtr handler);
+
+  // Creates the file descriptors and ngx_connection_t required for event
+  // messaging between pagespeed and nginx.
+  bool Init(ngx_cycle_t* cycle);
+  // Shuts down the underlying file descriptors and connection created in Init()
+  void Shutdown();
+  // Constructs a ps_event_data and writes it to the underlying named pipe.
+  bool WriteEvent(char type, void* sender);
+  // Convenience overload for clients that have a single event type.
+  bool WriteEvent(void* sender);
+  // Reads and processes what is available in the named pipe's buffer.
+  void Drain();
+ private:
+  static bool CreateNgxConnection(ngx_cycle_t* cycle, ngx_fd_t pipe_fd);
+  static void ReadEventHandler(ngx_event_t* e);
+  static bool ReadAndNotify(ngx_fd_t fd);
+
+  callbackPtr event_handler_;
+  // We own these file descriptors
+  ngx_fd_t pipe_write_fd_;
+  ngx_fd_t pipe_read_fd_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxEventConnection);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_EVENT_CONNECTION_H_

src/ngx_fetch.h

@@ -39,24 +39,72 @@ extern "C" {
 
 #include "ngx_url_async_fetcher.h"
 #include <vector>
-#include "net/instaweb/util/public/basictypes.h"
-#include "net/instaweb/util/public/pool.h"
-#include "net/instaweb/util/public/string.h"
 #include "net/instaweb/http/public/url_async_fetcher.h"
-#include "net/instaweb/http/public/response_headers.h"
+#include "pagespeed/kernel/base/basictypes.h"
-#include "net/instaweb/http/public/response_headers_parser.h"
+#include "pagespeed/kernel/base/pool.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/http/response_headers.h"
+#include "pagespeed/kernel/http/response_headers_parser.h"
+#include "pagespeed/kernel/thread/pthread_mutex.h"
+
 
 namespace net_instaweb {
 
 typedef bool (*response_handler_pt)(ngx_connection_t* c);
 
 class NgxUrlAsyncFetcher;
+class NgxConnection;
+
+class NgxConnection : public PoolElement<NgxConnection> {
+ public:
+  NgxConnection(MessageHandler* handler, int max_keepalive_requests);
+  ~NgxConnection();
+  void SetSock(u_char *sockaddr, socklen_t socklen) {
+    socklen_ = socklen;
+    ngx_memcpy(&sockaddr_, sockaddr, socklen);
+  }
+  // Close ensures that NgxConnection deletes itself at the appropriate time,
+  // which can be after receiving a non-keepalive response, or when the remote
+  // server closes the connection when the NgxConnection is pooled and idle.
+  void Close();
+
+  // Once keepalive is disabled, it can't be toggled back on.
+  void set_keepalive(bool k) { keepalive_ = keepalive_ && k; }
+  bool keepalive() { return keepalive_; }
+
+  typedef Pool<NgxConnection> NgxConnectionPool;
+
+  static NgxConnection* Connect(ngx_peer_connection_t* pc,
+                                MessageHandler* handler,
+                                int max_keepalive_requests);
+  static void IdleWriteHandler(ngx_event_t* ev);
+  static void IdleReadHandler(ngx_event_t* ev);
+  // Terminate will cleanup any idle connections upon shutdown.
+  static void Terminate();
+
+  static NgxConnectionPool connection_pool;
+  static PthreadMutex connection_pool_mutex;
+
+  // c_ is owned by NgxConnection and freed in ::Close()
+  ngx_connection_t* c_;
+  static const int64 keepalive_timeout_ms;
+  static const GoogleString ka_header;
+
+ private:
+  int max_keepalive_requests_;
+  bool keepalive_;
+  socklen_t socklen_;
+  u_char sockaddr_[NGX_SOCKADDRLEN];
+  MessageHandler* handler_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxConnection);
+};
+
 class NgxFetch : public PoolElement<NgxFetch> {
  public:
   NgxFetch(const GoogleString& url,
            AsyncFetch* async_fetch,
            MessageHandler* message_handler,
-           ngx_msec_t timeout_ms,
            ngx_log_t* log);
   ~NgxFetch();
 
@@ -112,19 +160,19 @@ class NgxFetch : public PoolElement<NgxFetch> {
     response_handler = handler;
   }
   // Only the Static functions could be used in callbacks.
-  static void NgxFetchResolveDone(ngx_resolver_ctx_t* ctx);
+  static void ResolveDoneHandler(ngx_resolver_ctx_t* ctx);
   // Write the request.
-  static void NgxFetchWrite(ngx_event_t* wev);
+  static void ConnectionWriteHandler(ngx_event_t* wev);
   // Wait for the response.
-  static void NgxFetchRead(ngx_event_t* rev);
+  static void ConnectionReadHandler(ngx_event_t* rev);
   // Read and parse the first status line.
-  static bool NgxFetchHandleStatusLine(ngx_connection_t* c);
+  static bool HandleStatusLine(ngx_connection_t* c);
   // Read and parse the HTTP headers.
-  static bool NgxFetchHandleHeader(ngx_connection_t* c);
+  static bool HandleHeader(ngx_connection_t* c);
   // Read the response body.
-  static bool NgxFetchHandleBody(ngx_connection_t* c);
+  static bool HandleBody(ngx_connection_t* c);
   // Cancel the fetch when it's timeout.
-  static void NgxFetchTimeout(ngx_event_t* tev);
+  static void TimeoutHandler(ngx_event_t* tev);
 
   // Add the pagespeed User-Agent.
   void FixUserAgent();
@@ -139,7 +187,6 @@ class NgxFetch : public PoolElement<NgxFetch> {
   int64 bytes_received_;
   int64 fetch_start_ms_;
   int64 fetch_end_ms_;
-  int64 timeout_ms_;
   bool done_;
   int64 content_length_;
   bool content_length_known_;
@@ -152,7 +199,7 @@ class NgxFetch : public PoolElement<NgxFetch> {
   ngx_http_request_t* r_;
   ngx_http_status_t* status_;
   ngx_event_t* timeout_event_;
-  ngx_connection_t* connection_;
+  NgxConnection* connection_;
   ngx_resolver_ctx_t* resolver_ctx_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxFetch);

src/ngx_gzip_setter.h

@@ -60,7 +60,7 @@ extern "C" {
 
 #include "ngx_pagespeed.h"
 
-#include "net/instaweb/util/public/basictypes.h"
+#include "pagespeed/kernel/base/basictypes.h"
 
 namespace net_instaweb {
 

src/ngx_message_handler.cc

@@ -18,13 +18,13 @@
 
 #include <signal.h>
 
-#include "net/instaweb/util/public/abstract_mutex.h"
-#include "net/instaweb/util/public/debug.h"
-#include "net/instaweb/util/public/shared_circular_buffer.h"
-#include "net/instaweb/util/public/string_util.h"
 #include "net/instaweb/public/version.h"
+#include "pagespeed/kernel/base/abstract_mutex.h"
+#include "pagespeed/kernel/base/debug.h"
 #include "pagespeed/kernel/base/posix_timer.h"
+#include "pagespeed/kernel/base/string_util.h"
 #include "pagespeed/kernel/base/time_util.h"
+#include "pagespeed/kernel/sharedmem/shared_circular_buffer.h"
 
 namespace {
 
@@ -88,32 +88,29 @@ ngx_uint_t NgxMessageHandler::GetNgxLogLevel(MessageType type) {
   return NGX_LOG_ALERT;
 }
 
-void NgxMessageHandler::MessageVImpl(MessageType type, const char* msg,
+void NgxMessageHandler::MessageSImpl(MessageType type,
-                                     va_list args) {
+                                     const GoogleString& message) {
-  ngx_uint_t log_level = GetNgxLogLevel(type);
-  GoogleString formatted_message = Format(msg, args);
   if (log_ != NULL) {
+    ngx_uint_t log_level = GetNgxLogLevel(type);
     ngx_log_error(log_level, log_, 0/*ngx_err_t*/, "[%s %s] %s",
-                  kModuleName, kModPagespeedVersion, formatted_message.c_str());
+                  kModuleName, kModPagespeedVersion, message.c_str());
   } else {
-    GoogleMessageHandler::MessageVImpl(type, msg, args);
+    GoogleMessageHandler::MessageSImpl(type, message);
   }
-  // Prepare a log message for the SharedCircularBuffer only.
+  AddMessageToBuffer(type, message);
-  AddMessageToBuffer(type, formatted_message);
 }
 
-void NgxMessageHandler::FileMessageVImpl(MessageType type, const char* file,
+void NgxMessageHandler::FileMessageSImpl(
-                                         int line, const char* msg,
+    MessageType type, const char* file, int line, const GoogleString& message) {
-                                         va_list args) {
-  ngx_uint_t log_level = GetNgxLogLevel(type);
-  GoogleString formatted_message = Format(msg, args);
   if (log_ != NULL) {
+    ngx_uint_t log_level = GetNgxLogLevel(type);
     ngx_log_error(log_level, log_, 0/*ngx_err_t*/, "[%s %s] %s:%d:%s",
                   kModuleName, kModPagespeedVersion, file, line,
-                  formatted_message.c_str());
+                  message.c_str());
   } else {
-    GoogleMessageHandler::FileMessageVImpl(type, file, line, msg, args);
+    GoogleMessageHandler::FileMessageSImpl(type, file, line, message);
   }
+  AddMessageToBuffer(type, file, line, message);
 }
 
 }  // namespace net_instaweb

src/ngx_message_handler.h

@@ -18,17 +18,21 @@
 #define NGX_MESSAGE_HANDLER_H_
 
 extern "C" {
+  #include <ngx_auto_config.h>
+#if (NGX_THREADS)
+  #include <ngx_thread.h>
+#endif
   #include <ngx_core.h>
   #include <ngx_log.h>
 }
 
 #include <cstdarg>
 
-#include "net/instaweb/system/public/system_message_handler.h"
+#include "pagespeed/kernel/base/basictypes.h"
-#include "net/instaweb/util/public/basictypes.h"
+#include "pagespeed/kernel/base/message_handler.h"
-#include "net/instaweb/util/public/message_handler.h"
+#include "pagespeed/kernel/base/string.h"
-#include "net/instaweb/util/public/string.h"
+#include "pagespeed/kernel/base/string_util.h"
-#include "net/instaweb/util/public/string_util.h"
+#include "pagespeed/system/system_message_handler.h"
 
 namespace net_instaweb {
 
@@ -49,10 +53,10 @@ class NgxMessageHandler : public SystemMessageHandler {
   ngx_log_t* log() { return log_; }
 
  protected:
-  virtual void MessageVImpl(MessageType type, const char* msg, va_list args);
+  virtual void MessageSImpl(MessageType type, const GoogleString& message);
 
-  virtual void FileMessageVImpl(MessageType type, const char* filename,
+  virtual void FileMessageSImpl(MessageType type, const char* file,
-                                int line, const char* msg, va_list args);
+                                int line, const GoogleString& message);
 
  private:
   ngx_uint_t GetNgxLogLevel(MessageType type);

src/ngx_pagespeed.h

@@ -34,8 +34,8 @@ extern "C" {
 }
 
 #include "base/logging.h"
-#include "net/instaweb/http/public/response_headers.h"
+#include "pagespeed/kernel/base/string_util.h"
-#include "net/instaweb/util/public/string_util.h"
+#include "pagespeed/kernel/http/response_headers.h"
 
 namespace net_instaweb {
 
@@ -52,7 +52,7 @@ class InPlaceResourceRecorder;
 // NGX_DECLINED immediately unless send_last_buf.
 ngx_int_t string_piece_to_buffer_chain(
     ngx_pool_t* pool, StringPiece sp,
-    ngx_chain_t** link_ptr, bool send_last_buf);
+    ngx_chain_t** link_ptr, bool send_last_buf, bool send_flush);
 
 StringPiece str_to_string_piece(ngx_str_t s);
 
@@ -84,15 +84,11 @@ enum PreserveCachingHeaders {
 typedef struct {
   NgxBaseFetch* base_fetch;
 
-  ngx_connection_t* pagespeed_connection;
   ngx_http_request_t* r;
 
   bool html_rewrite;
   bool in_place;
 
-  bool write_pending;
-  bool fetch_done;
-
   PreserveCachingHeaders preserve_caching_headers;
 
   // for html rewrite
@@ -107,8 +103,16 @@ typedef struct {
   // We need to remember the URL here as well since we may modify what NGX
   // gets by stripping our special query params and honoring X-Forwarded-Proto.
   GoogleString url_string;
+
+  // We need to remember if the upstream had headers_out->location set, because
+  // we should mirror that when we write it back. nginx may absolutify
+  // Location: headers that start with '/' without regarding X-Forwarded-Proto.
+  bool location_field_set;
+  bool psol_vary_accept_only;
+  bool follow_flushes;
 } ps_request_ctx_t;
 
+ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r);
 
 void copy_request_headers_from_ngx(const ngx_http_request_t* r,
                                    RequestHeaders* headers);
@@ -123,6 +127,12 @@ ngx_int_t copy_response_headers_to_ngx(
 
 StringPiece ps_determine_host(ngx_http_request_t* r);
 
+namespace ps_base_fetch {
+
+ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r);
+
+}  // namespace ps_base_fetch
+
 }  // namespace net_instaweb
 
 #endif  // NGX_PAGESPEED_H_

src/ngx_rewrite_driver_factory.cc

@@ -26,30 +26,30 @@
 #include "ngx_server_context.h"
 #include "ngx_url_async_fetcher.h"
 
-#include "net/instaweb/http/public/content_type.h"
 #include "net/instaweb/http/public/rate_controller.h"
 #include "net/instaweb/http/public/rate_controlling_url_async_fetcher.h"
 #include "net/instaweb/http/public/wget_url_fetcher.h"
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
 #include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
 #include "net/instaweb/rewriter/public/server_context.h"
-#include "net/instaweb/system/public/in_place_resource_recorder.h"
-#include "net/instaweb/system/public/serf_url_async_fetcher.h"
-#include "net/instaweb/system/public/system_caches.h"
-#include "net/instaweb/system/public/system_rewrite_options.h"
-#include "net/instaweb/util/public/google_message_handler.h"
-#include "net/instaweb/util/public/null_shared_mem.h"
-#include "net/instaweb/util/public/posix_timer.h"
 #include "net/instaweb/util/public/property_cache.h"
-#include "net/instaweb/util/public/scheduler_thread.h"
+#include "pagespeed/kernel/base/google_message_handler.h"
-#include "net/instaweb/util/public/shared_circular_buffer.h"
+#include "pagespeed/kernel/base/null_shared_mem.h"
-#include "net/instaweb/util/public/shared_mem_statistics.h"
+#include "pagespeed/kernel/base/posix_timer.h"
-#include "net/instaweb/util/public/slow_worker.h"
+#include "pagespeed/kernel/base/stdio_file_system.h"
-#include "net/instaweb/util/public/stdio_file_system.h"
+#include "pagespeed/kernel/base/string.h"
-#include "net/instaweb/util/public/string.h"
+#include "pagespeed/kernel/base/string_util.h"
-#include "net/instaweb/util/public/string_util.h"
+#include "pagespeed/kernel/base/thread_system.h"
-#include "net/instaweb/util/public/thread_system.h"
+#include "pagespeed/kernel/http/content_type.h"
+#include "pagespeed/kernel/sharedmem/shared_circular_buffer.h"
+#include "pagespeed/kernel/sharedmem/shared_mem_statistics.h"
 #include "pagespeed/kernel/thread/pthread_shared_mem.h"
+#include "pagespeed/kernel/thread/scheduler_thread.h"
+#include "pagespeed/kernel/thread/slow_worker.h"
+#include "pagespeed/system/in_place_resource_recorder.h"
+#include "pagespeed/system/serf_url_async_fetcher.h"
+#include "pagespeed/system/system_caches.h"
+#include "pagespeed/system/system_rewrite_options.h"
 
 namespace net_instaweb {
 
@@ -69,7 +69,6 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
     SystemThreadSystem* system_thread_system, StringPiece hostname, int port)
     : SystemRewriteDriverFactory(process_context, system_thread_system,
         NULL /* default shared memory runtime */, hostname, port),
-      main_conf_(NULL),
       threads_started_(false),
       ngx_message_handler_(
           new NgxMessageHandler(timer(), thread_system()->NewMutex())),
@@ -78,11 +77,14 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
       log_(NULL),
       resolver_timeout_(NGX_CONF_UNSET_MSEC),
       use_native_fetcher_(false),
+      // 100 Aligns to nginx's server-side default.
+      native_fetcher_max_keepalive_requests_(100),
       ngx_shared_circular_buffer_(NULL),
       hostname_(hostname.as_string()),
       port_(port),
-      process_script_variables_(false),
+      process_script_variables_mode_(ProcessScriptVariablesMode::kOff),
-      process_script_variables_set_(false) {
+      process_script_variables_set_(false),
+      shut_down_(false) {
   InitializeDefaultOptions();
   default_options()->set_beacon_url("/ngx_pagespeed_beacon");
   SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
@@ -112,6 +114,7 @@ UrlAsyncFetcher* NgxRewriteDriverFactory::AllocateFetcher(
         resolver_timeout_,
         config->blocking_fetch_timeout_ms(),
         resolver_,
+        native_fetcher_max_keepalive_requests_,
         thread_system(),
         message_handler());
     ngx_url_async_fetchers_.push_back(fetcher);
@@ -144,18 +147,17 @@ NamedLockManager* NgxRewriteDriverFactory::DefaultLockManager() {
 
 RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
   NgxRewriteOptions* options = new NgxRewriteOptions(thread_system());
+  // TODO(jefftk): figure out why using SetDefaultRewriteLevel like
+  // mod_pagespeed does in mod_instaweb.cc:create_dir_config() isn't enough here
+  // -- if you use that instead then ngx_pagespeed doesn't actually end up
+  // defaulting CoreFilters.
+  // See: https://github.com/pagespeed/ngx_pagespeed/issues/1190
   options->SetRewriteLevel(RewriteOptions::kCoreFilters);
   return options;
 }
 
-bool NgxRewriteDriverFactory::InitNgxUrlAsyncFetchers() {
+RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptionsForQuery() {
-  log_ = ngx_cycle->log;
+  return new NgxRewriteOptions(thread_system());
-  for (size_t i = 0; i < ngx_url_async_fetchers_.size(); ++i) {
-    if (!ngx_url_async_fetchers_[i]->Init()) {
-      return false;
-    }
-  }
-  return true;
 }
 
 bool NgxRewriteDriverFactory::CheckResolver() {
@@ -183,6 +185,13 @@ ServerContext* NgxRewriteDriverFactory::NewServerContext() {
   return NULL;
 }
 
+void NgxRewriteDriverFactory::ShutDown() {
+  if (!shut_down_) {
+    shut_down_ = true;
+    SystemRewriteDriverFactory::ShutDown();
+  }
+}
+
 void NgxRewriteDriverFactory::ShutDownMessageHandlers() {
   ngx_message_handler_->set_buffer(NULL);
   ngx_html_parse_message_handler_->set_buffer(NULL);
@@ -207,9 +216,19 @@ void NgxRewriteDriverFactory::StartThreads() {
   threads_started_ = true;
 }
 
-void NgxRewriteDriverFactory::LoggingInit(ngx_log_t* log) {
+void NgxRewriteDriverFactory::SetMainConf(NgxRewriteOptions* main_options) {
+  // Propagate process-scope options from the copy we had during nginx option
+  // parsing to our own.
+  if (main_options != NULL) {
+    default_options()->MergeOnlyProcessScopeOptions(*main_options);
+  }
+}
+
+void NgxRewriteDriverFactory::LoggingInit(
+    ngx_log_t* log, bool may_install_crash_handler) {
+  log_ = log;
   net_instaweb::log_message_handler::Install(log);
-  if (install_crash_handler()) {
+  if (may_install_crash_handler && install_crash_handler()) {
     NgxMessageHandler::InstallCrashHandler(log);
   }
   ngx_message_handler_->set_log(log);
@@ -247,4 +266,23 @@ void NgxRewriteDriverFactory::InitStats(Statistics* statistics) {
   InPlaceResourceRecorder::InitStats(statistics);
 }
 
+void NgxRewriteDriverFactory::PrepareForkedProcess(const char* name) {
+  ngx_pid = ngx_getpid();  // Needed for logging to have the right PIDs.
+  SystemRewriteDriverFactory::PrepareForkedProcess(name);
+}
+
+void NgxRewriteDriverFactory::NameProcess(const char* name) {
+  SystemRewriteDriverFactory::NameProcess(name);
+
+  // Superclass set status with prctl.  Nginx has a helper function for setting
+  // argv[0] as well, so let's use that.  We'll show up as:
+  //
+  //    nginx: pagespeed $name
+
+  char name_for_setproctitle[32];
+  snprintf(name_for_setproctitle, sizeof(name_for_setproctitle),
+           "pagespeed %s", name);
+  ngx_setproctitle(name_for_setproctitle);
+}
+
 }  // namespace net_instaweb

src/ngx_rewrite_driver_factory.h

@@ -20,6 +20,10 @@
 #define NGX_REWRITE_DRIVER_FACTORY_H_
 
 extern "C" {
+  #include <ngx_auto_config.h>
+#if (NGX_THREADS)
+  #include <ngx_thread.h>
+#endif
   #include <ngx_core.h>
   #include <ngx_http.h>
   #include <ngx_config.h>
@@ -28,13 +32,9 @@ extern "C" {
 
 #include <set>
 
-#include "net/instaweb/system/public/system_rewrite_driver_factory.h"
+#include "pagespeed/kernel/base/md5_hasher.h"
-#include "net/instaweb/util/public/md5_hasher.h"
+#include "pagespeed/kernel/base/scoped_ptr.h"
-#include "net/instaweb/util/public/scoped_ptr.h"
+#include "pagespeed/system/system_rewrite_driver_factory.h"
-
-// TODO(oschaaf): We should reparent ApacheRewriteDriverFactory and
-// NgxRewriteDriverFactory to a new class OriginRewriteDriverFactory and factor
-// out as much as possible.
 
 namespace net_instaweb {
 
@@ -48,6 +48,12 @@ class SlowWorker;
 class Statistics;
 class SystemThreadSystem;
 
+enum ProcessScriptVariablesMode {
+  kOff,
+  kLegacyRestricted,
+  kAll
+};
+
 class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  public:
   // We take ownership of the thread system.
@@ -63,10 +69,10 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   virtual Timer* DefaultTimer();
   virtual NamedLockManager* DefaultLockManager();
   // Create a new RewriteOptions.  In this implementation it will be an
-  // NgxRewriteOptions.
+  // NgxRewriteOptions, and it will have CoreFilters explicitly set.
   virtual RewriteOptions* NewRewriteOptions();
+  virtual RewriteOptions* NewRewriteOptionsForQuery();
   virtual ServerContext* NewDecodingServerContext();
-  bool InitNgxUrlAsyncFetchers();
   // Check resolver configured or not.
   bool CheckResolver();
 
@@ -76,6 +82,7 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   static void InitStats(Statistics* statistics);
   NgxServerContext* MakeNgxServerContext(StringPiece hostname, int port);
   virtual ServerContext* NewServerContext();
+  virtual void ShutDown();
 
   // Starts pagespeed threads if they've not been started already.  Must be
   // called after the caller has finished any forking it intends to do.
@@ -90,7 +97,7 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
     InitStats(statistics);
   }
 
-  void set_main_conf(NgxRewriteOptions* main_conf) {  main_conf_ = main_conf; }
+  void SetMainConf(NgxRewriteOptions* main_conf);
 
   void set_resolver(ngx_resolver_t* resolver) {
     resolver_ = resolver;
@@ -105,32 +112,38 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   void set_use_native_fetcher(bool x) {
     use_native_fetcher_ = x;
   }
-  bool process_script_variables() {
+  int native_fetcher_max_keepalive_requests() {
-    return process_script_variables_;
+    return native_fetcher_max_keepalive_requests_;
+  }
+  void set_native_fetcher_max_keepalive_requests(int x) {
+    native_fetcher_max_keepalive_requests_ = x;
+  }
+  ProcessScriptVariablesMode process_script_variables() {
+    return process_script_variables_mode_;
   }
 
-  void LoggingInit(ngx_log_t* log);
+  void LoggingInit(ngx_log_t* log, bool may_install_crash_handler);
 
   virtual void ShutDownMessageHandlers();
 
   virtual void SetCircularBuffer(SharedCircularBuffer* buffer);
 
-  bool SetProcessScriptVariables(bool process_script_variables) {
+  bool SetProcessScriptVariables(ProcessScriptVariablesMode mode) {
     if (!process_script_variables_set_) {
-      process_script_variables_ = process_script_variables;
+      process_script_variables_mode_ = mode;
       process_script_variables_set_ = true;
       return true;
     }
     return false;
   }
 
+  virtual void PrepareForkedProcess(const char* name);
+
+  virtual void NameProcess(const char* name);
+
  private:
   Timer* timer_;
 
-  // main_conf will have only options set in the main block.  It may be NULL,
-  // and we do not take ownership.
-  NgxRewriteOptions* main_conf_;
-
   bool threads_started_;
   NgxMessageHandler* ngx_message_handler_;
   NgxMessageHandler* ngx_html_parse_message_handler_;
@@ -140,6 +153,8 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   ngx_msec_t resolver_timeout_;
   ngx_resolver_t* resolver_;
   bool use_native_fetcher_;
+  int native_fetcher_max_keepalive_requests_;
+
   typedef std::set<NgxMessageHandler*> NgxMessageHandlerSet;
   NgxMessageHandlerSet server_context_message_handlers_;
 
@@ -149,8 +164,9 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
 
   GoogleString hostname_;
   int port_;
-  bool process_script_variables_;
+  ProcessScriptVariablesMode process_script_variables_mode_;
   bool process_script_variables_set_;
+  bool shut_down_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
 };

src/ngx_rewrite_options.cc

@@ -30,9 +30,9 @@ extern "C" {
 #include "net/instaweb/public/version.h"
 #include "net/instaweb/rewriter/public/file_load_policy.h"
 #include "net/instaweb/rewriter/public/rewrite_options.h"
-#include "net/instaweb/system/public/system_caches.h"
+#include "pagespeed/kernel/base/message_handler.h"
-#include "net/instaweb/util/public/message_handler.h"
+#include "pagespeed/kernel/base/timer.h"
-#include "net/instaweb/util/public/timer.h"
+#include "pagespeed/system/system_caches.h"
 
 namespace net_instaweb {
 
@@ -72,12 +72,14 @@ const char* const server_only_options[] = {
   "LoadFromFileMatch",
   "LoadFromFileRule",
   "LoadFromFileRuleMatch",
-  "UseNativeFetcher"
+  "UseNativeFetcher",
+  "NativeFetcherMaxKeepaliveRequests"
 };
 
 // Options that can only be used in the main (http) option scope.
 const char* const main_only_options[] = {
-  "UseNativeFetcher"
+  "UseNativeFetcher",
+  "NativeFetcherMaxKeepaliveRequests"
 };
 
 }  // namespace
@@ -110,7 +112,7 @@ void NgxRewriteOptions::AddProperties() {
       false);
   add_ngx_option(
       "", &NgxRewriteOptions::global_statistics_path_, "ngsp",
-      kGlobalStatisticsPath, kProcessScope,
+      kGlobalStatisticsPath, kProcessScopeStrict,
       "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics",
       false);
   add_ngx_option(
@@ -125,7 +127,8 @@ void NgxRewriteOptions::AddProperties() {
       kServerScope, "Set the admin path.  Ex: /pagespeed_admin", false);
   add_ngx_option(
       "", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
-      kProcessScope, "Set the global admin path.  Ex: /pagespeed_global_admin",
+      kProcessScopeStrict,
+      "Set the global admin path.  Ex: /pagespeed_global_admin",
       false);
 
   MergeSubclassProperties(ngx_properties_);
@@ -178,10 +181,10 @@ RewriteOptions::OptionScope NgxRewriteOptions::GetOptionScope(
        it != all_options().end(); ++it) {
     RewriteOptions::OptionBase* option = *it;
     if (option->option_name() == option_name) {
-      // We treat kProcessScope as kProcessScopeStrict, failing to start if an
+      // We treat kLegacyProcessScope as kProcessScopeStrict, failing to start
-      // option is out of place.
+      // if an option is out of place.
-      return option->scope() == kProcessScope ? kProcessScopeStrict
+      return option->scope() == kLegacyProcessScope ? kProcessScopeStrict
-                                              : option->scope();
+                                                    : option->scope();
     }
   }
   return kDirectoryScope;
@@ -256,7 +259,8 @@ const char* ps_error_string_for_option(
 const char* NgxRewriteOptions::ParseAndSetOptions(
     StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
     NgxRewriteDriverFactory* driver_factory,
-    RewriteOptions::OptionScope scope, ngx_conf_t* cf, bool compile_scripts) {
+    RewriteOptions::OptionScope scope, ngx_conf_t* cf,
+    ProcessScriptVariablesMode script_mode) {
   CHECK_GE(n_args, 1);
 
   StringPiece directive = args[0];
@@ -272,14 +276,43 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
         pool, directive, "cannot be set at this scope.");
   }
 
+  bool compile_scripts = false;
+
+  if (script_mode != ProcessScriptVariablesMode::kOff) {
+    // In the old mode we only allowed a few, so restrict to those.
+    compile_scripts =
+        StringCaseStartsWith(directive, "LoadFromFile") ||
+        StringCaseEqual(directive, "EnableFilters") ||
+        StringCaseEqual(directive, "DisableFilters") ||
+        StringCaseEqual(directive, "DownstreamCachePurgeLocationPrefix") ||
+        StringCaseEqual(directive, "DownstreamCachePurgeMethod") ||
+        StringCaseEqual(directive,
+                        "DownstreamCacheRewrittenPercentageThreshold") ||
+        StringCaseEqual(directive, "ShardDomain");
+    // In the new behaviour we also allow scripting of query- and directory-
+    // scoped options.
+    compile_scripts |=
+        script_mode == ProcessScriptVariablesMode::kAll &&
+        (GetOptionScope(directive) <= RewriteOptions::kDirectoryScope ||
+         (StringCaseEqual(directive, "Allow") ||
+          StringCaseEqual(directive, "BlockingRewriteRefererUrls") ||
+          StringCaseEqual(directive, "Disallow") ||
+          StringCaseEqual(directive, "DistributableFilters") ||
+          StringCaseEqual(directive, "Domain") ||
+          StringCaseEqual(directive, "ExperimentVariable") ||
+          StringCaseEqual(directive, "ExperimentSpec") ||
+          StringCaseEqual(directive, "ForbidFilters") ||
+          StringCaseEqual(directive, "RetainComment") ||
+          StringCaseEqual(directive, "CustomFetchHeader") ||
+          StringCaseEqual(directive, "MapOriginDomain") ||
+          StringCaseEqual(directive, "MapProxyDomain") ||
+          StringCaseEqual(directive, "MapRewriteDomain") ||
+          StringCaseEqual(directive, "UrlValuedAttribute") ||
+          StringCaseEqual(directive, "Library")));
+  }
+
   ScriptLine* script_line;
   script_line = NULL;
-  // Only allow script variable support for LoadFromFile for now.
-  // Note that LoadFromFile should not be scriptable on wildcard hosts,
-  // as browsers might be able to manipulate its natural use-case: $http_host.
-  if (!StringCaseStartsWith(directive, "LoadFromFile")) {
-    compile_scripts = false;
-  }
 
   if (n_args == 1 && StringCaseEqual(directive, "ClearInheritedScripts")) {
     clear_inherited_scripts_ = true;
@@ -343,26 +376,35 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
       result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
           arg, driver_factory,
           &NgxRewriteDriverFactory::set_use_native_fetcher);
+    } else if (IsDirective(directive, "NativeFetcherMaxKeepaliveRequests")) {
+      int max_keepalive_requests;
+      if (StringToInt(arg, &max_keepalive_requests) &&
+          max_keepalive_requests > 0) {
+        driver_factory->set_native_fetcher_max_keepalive_requests(
+            max_keepalive_requests);
+        result = RewriteOptions::kOptionOk;
+      } else {
+        result = RewriteOptions::kOptionValueInvalid;
+      }
     } else if (StringCaseEqual("ProcessScriptVariables", args[0])) {
       if (scope == RewriteOptions::kProcessScopeStrict) {
-        if (StringCaseEqual(arg, "on")) {
+        ProcessScriptVariablesMode mode;
-          if (driver_factory->SetProcessScriptVariables(true)) {
+        if (StringCaseEqual(arg, "all")) {
-            result = RewriteOptions::kOptionOk;
+          mode = ProcessScriptVariablesMode::kAll;
-          } else {
+        } else if (StringCaseEqual(arg, "on")) {
-            return const_cast<char*>(
+          mode = ProcessScriptVariablesMode::kLegacyRestricted;
-                "pagespeed ProcessScriptVariables: can only be set once");
-          }
         } else if (StringCaseEqual(arg, "off")) {
-          if (driver_factory->SetProcessScriptVariables(false)) {
+          mode = ProcessScriptVariablesMode::kOff;
-            result = RewriteOptions::kOptionOk;
-          } else {
-            return const_cast<char*>(
-                "pagespeed ProcessScriptVariables: can only be set once");
-          }
         } else {
           return const_cast<char*>(
               "pagespeed ProcessScriptVariables: invalid value");
         }
+        if (driver_factory->SetProcessScriptVariables(mode)) {
+          result = RewriteOptions::kOptionOk;
+        } else {
+          return const_cast<char*>(
+              "pagespeed ProcessScriptVariables: can only be set once");
+        }
       } else {
         return const_cast<char*>(
             "ProcessScriptVariables is only allowed at the top level");
@@ -373,7 +415,7 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
         result = driver_factory->ParseAndSetOption1(
             directive,
             arg,
-            scope >= RewriteOptions::kProcessScope,
+            scope >= RewriteOptions::kLegacyProcessScope,
             &msg,
             handler);
       }
@@ -386,7 +428,7 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
           directive,
           args[1],
           args[2],
-          scope >= RewriteOptions::kProcessScope,
+          scope >= RewriteOptions::kLegacyProcessScope,
           &msg,
           handler);
     }
@@ -458,7 +500,7 @@ bool NgxRewriteOptions::ExecuteScriptVariables(
 
       const char* status = ParseAndSetOptions(args, script_line->n_args(),
           r->pool, handler, driver_factory, script_line->scope(), NULL /*cf*/,
-          false /*compile scripts*/);
+          ProcessScriptVariablesMode::kOff);
 
       if (status != NULL) {
         script_error = true;

src/ngx_rewrite_options.h

@@ -29,11 +29,13 @@ extern "C" {
 
 #include <vector>
 
-#include "net/instaweb/util/public/message_handler.h"
+#include "ngx_rewrite_driver_factory.h"
-#include "net/instaweb/util/public/ref_counted_ptr.h"
+
-#include "net/instaweb/util/public/stl_util.h"          // for STLDeleteElements
 #include "net/instaweb/rewriter/public/rewrite_options.h"
-#include "net/instaweb/system/public/system_rewrite_options.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/ref_counted_ptr.h"
+#include "pagespeed/kernel/base/stl_util.h"          // for STLDeleteElements
+#include "pagespeed/system/system_rewrite_options.h"
 
 #define NGX_PAGESPEED_MAX_ARGS 10
 
@@ -131,7 +133,7 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   const char* ParseAndSetOptions(
       StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
       NgxRewriteDriverFactory* driver_factory, OptionScope scope,
-      ngx_conf_t* cf, bool compile_scripts);
+      ngx_conf_t* cf, ProcessScriptVariablesMode script_mode);
   bool ExecuteScriptVariables(
       ngx_http_request_t* r, MessageHandler* handler,
       NgxRewriteDriverFactory* driver_factory);

src/ngx_server_context.cc

@@ -27,15 +27,16 @@ extern "C" {
 #include "ngx_rewrite_driver_factory.h"
 #include "ngx_rewrite_options.h"
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
-#include "net/instaweb/system/public/add_headers_fetcher.h"
+#include "pagespeed/system/add_headers_fetcher.h"
-#include "net/instaweb/system/public/loopback_route_fetcher.h"
+#include "pagespeed/system/loopback_route_fetcher.h"
-#include "net/instaweb/system/public/system_request_context.h"
+#include "pagespeed/system/system_request_context.h"
 
 namespace net_instaweb {
 
 NgxServerContext::NgxServerContext(
     NgxRewriteDriverFactory* factory, StringPiece hostname, int port)
-    : SystemServerContext(factory, hostname, port) {
+    : SystemServerContext(factory, hostname, port),
+      ngx_http2_variable_index_(NGX_ERROR) {
 }
 
 NgxServerContext::~NgxServerContext() { }
@@ -48,7 +49,7 @@ SystemRequestContext* NgxServerContext::NewRequestContext(
     ngx_http_request_t* r) {
   // Based on ngx_http_variable_server_port.
   bool port_set = false;
-  int local_port;
+  int local_port = 0;
 #if (NGX_HAVE_INET6)
   if (r->connection->local_sockaddr->sa_family == AF_INET6) {
     local_port = ntohs(reinterpret_cast<struct sockaddr_in6*>(
@@ -70,11 +71,23 @@ SystemRequestContext* NgxServerContext::NewRequestContext(
     local_ip.len = 0;
   }
 
-  return new SystemRequestContext(thread_system()->NewMutex(),
+  SystemRequestContext* ctx = new SystemRequestContext(
-                                  timer(),
+      thread_system()->NewMutex(), timer(),
-                                  ps_determine_host(r),
+      ps_determine_host(r), local_port, str_to_string_piece(local_ip));
-                                  local_port,
+
-                                  str_to_string_piece(local_ip));
+  // See if http2 is in use.
+  if (ngx_http2_variable_index_ >= 0) {
+    ngx_http_variable_value_t* val =
+        ngx_http_get_indexed_variable(r, ngx_http2_variable_index_);
+    if (val != NULL && val->valid) {
+      StringPiece str_val(reinterpret_cast<char*>(val->data), val->len);
+      if (str_val == "h2" || str_val == "h2c") {
+        ctx->set_using_http2(true);
+      }
+    }
+  }
+
+  return ctx;
 }
 
 GoogleString NgxServerContext::FormatOption(StringPiece option_name,

src/ngx_server_context.h

@@ -22,7 +22,7 @@
 #define NGX_SERVER_CONTEXT_H_
 
 #include "ngx_message_handler.h"
-#include "net/instaweb/system/public/system_server_context.h"
+#include "pagespeed/system/system_server_context.h"
 
 extern "C" {
 #include <ngx_http.h>
@@ -40,8 +40,9 @@ class NgxServerContext : public SystemServerContext {
       NgxRewriteDriverFactory* factory, StringPiece hostname, int port);
   virtual ~NgxServerContext();
 
-  // We expect to use ProxyFetch with HTML.
+  // We don't allow ProxyFetch to fetch HTML via MapProxyDomain. We will call
-  virtual bool ProxiesHtml() const { return true; }
+  // set_trusted_input() on any ProxyFetches we use to transform internal HTML.
+  virtual bool ProxiesHtml() const { return false; }
 
   // Call only when you need an NgxRewriteOptions.  If you don't need
   // nginx-specific behavior, call global_options() instead which doesn't
@@ -57,8 +58,18 @@ class NgxServerContext : public SystemServerContext {
 
   virtual GoogleString FormatOption(StringPiece option_name, StringPiece args);
 
+  void set_ngx_http2_variable_index(ngx_int_t idx) {
+    ngx_http2_variable_index_ = idx;
+  }
+
+  ngx_int_t ngx_http2_variable_index() const {
+    return ngx_http2_variable_index_;
+  }
+
  private:
   NgxRewriteDriverFactory* ngx_factory_;
+  // what index the "http2" var is, or NGX_ERROR.
+  ngx_int_t ngx_http2_variable_index_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxServerContext);
 };

src/ngx_url_async_fetcher.cc

@@ -31,22 +31,22 @@ extern "C" {
 #include <map>
 #include <set>
 
-#include "net/instaweb/util/public/basictypes.h"
 #include "net/instaweb/http/public/async_fetch.h"
 #include "net/instaweb/http/public/inflating_fetch.h"
-#include "net/instaweb/http/public/request_headers.h"
-#include "net/instaweb/http/public/response_headers.h"
-#include "net/instaweb/http/public/response_headers_parser.h"
 #include "net/instaweb/public/version.h"
-#include "net/instaweb/util/public/condvar.h"
+#include "pagespeed/kernel/base/basictypes.h"
-#include "net/instaweb/util/public/message_handler.h"
+#include "pagespeed/kernel/base/condvar.h"
-#include "net/instaweb/util/public/pool.h"
+#include "pagespeed/kernel/base/message_handler.h"
-#include "net/instaweb/util/public/pool_element.h"
+#include "pagespeed/kernel/base/pool.h"
-#include "net/instaweb/util/public/statistics.h"
+#include "pagespeed/kernel/base/pool_element.h"
-#include "net/instaweb/util/public/string_util.h"
+#include "pagespeed/kernel/base/statistics.h"
-#include "net/instaweb/util/public/thread_system.h"
+#include "pagespeed/kernel/base/string_util.h"
-#include "net/instaweb/util/public/timer.h"
+#include "pagespeed/kernel/base/thread_system.h"
-#include "net/instaweb/util/public/writer.h"
+#include "pagespeed/kernel/base/timer.h"
+#include "pagespeed/kernel/base/writer.h"
+#include "pagespeed/kernel/http/request_headers.h"
+#include "pagespeed/kernel/http/response_headers.h"
+#include "pagespeed/kernel/http/response_headers_parser.h"
 
 namespace net_instaweb {
 
@@ -55,6 +55,7 @@ namespace net_instaweb {
                                          ngx_msec_t resolver_timeout,
                                          ngx_msec_t fetch_timeout,
                                          ngx_resolver_t* resolver,
+                                         int max_keepalive_requests,
                                          ThreadSystem* thread_system,
                                          MessageHandler* handler)
     : fetchers_count_(0),
@@ -63,7 +64,9 @@ namespace net_instaweb {
       byte_count_(0),
       thread_system_(thread_system),
       message_handler_(handler),
-      mutex_(NULL) {
+      mutex_(NULL),
+      max_keepalive_requests_(max_keepalive_requests),
+      event_connection_(NULL) {
     resolver_timeout_ = resolver_timeout;
     fetch_timeout_ = fetch_timeout;
     ngx_memzero(&proxy_, sizeof(proxy_));
@@ -74,12 +77,17 @@ namespace net_instaweb {
     mutex_ = thread_system_->NewMutex();
     log_ = log;
     pool_ = NULL;
-    command_connection_ = NULL;
-    pipe_fd_ = -1;
     resolver_ = resolver;
+    // If init fails, set shutdown_ so no fetches will be attempted.
+    if (!Init(const_cast<ngx_cycle_t*>(ngx_cycle))) {
+      shutdown_ = true;
+      message_handler_->Message(
+          kError, "NgxUrlAsyncFetcher failed to init, fetching disabled.");
+    }
   }
 
   NgxUrlAsyncFetcher::~NgxUrlAsyncFetcher() {
+    DCHECK(shutdown_)  << "Shut down before destructing NgxUrlAsyncFetcher.";
     message_handler_->Message(
         kInfo,
         "Destruct NgxUrlAsyncFetcher with [%d] active fetchers",
@@ -87,19 +95,12 @@ namespace net_instaweb {
 
     CancelActiveFetches();
     active_fetches_.DeleteAll();
+    NgxConnection::Terminate();
 
     if (pool_ != NULL) {
       ngx_destroy_pool(pool_);
       pool_ = NULL;
     }
-    if (command_connection_ != NULL) {
-      ngx_close_connection(command_connection_);
-      command_connection_ = NULL;
-    }
-    if (pipe_fd_ != -1) {
-      close(pipe_fd_);
-      pipe_fd_ = -1;
-    }
     if (mutex_ != NULL) {
       delete mutex_;
       mutex_ = NULL;
@@ -150,9 +151,13 @@ namespace net_instaweb {
 
   // Create the pool for fetcher, create the pipe, add the read event for main
   // thread. It should be called in the worker process.
-  bool NgxUrlAsyncFetcher::Init() {
+  bool NgxUrlAsyncFetcher::Init(ngx_cycle_t* cycle) {
-    log_ = ngx_cycle->log;
+    log_ = cycle->log;
-
+    CHECK(event_connection_ == NULL) << "event connection already set";
+    event_connection_ = new NgxEventConnection(ReadCallback);
+    if (!event_connection_->Init(cycle)) {
+      return false;
+    }
     if (pool_ == NULL) {
       pool_ = ngx_create_pool(4096, log_);
       if (pool_ == NULL) {
@@ -162,41 +167,6 @@ namespace net_instaweb {
       }
     }
 
-    int pipe_fds[2];
-    int rc = pipe(pipe_fds);
-    if (rc != 0) {
-      ngx_log_error(NGX_LOG_ERR, log_, 0, "pipe() failed");
-      return false;
-    }
-    if (ngx_nonblocking(pipe_fds[0]) == -1) {
-      ngx_log_error(NGX_LOG_ERR, log_, 0, "nonblocking pipe[0] failed");
-      return false;
-    }
-    if (ngx_nonblocking(pipe_fds[1]) == -1) {
-      ngx_log_error(NGX_LOG_ERR, log_, 0, "nonblocking pipe[1] failed");
-      return false;
-    }
-
-    pipe_fd_ = pipe_fds[1];
-    command_connection_ = ngx_get_connection(pipe_fds[0], log_);
-    if (command_connection_ == NULL) {
-      close(pipe_fds[1]);
-      close(pipe_fds[0]);
-      pipe_fd_ = -1;
-      return false;
-    }
-
-    command_connection_->recv = ngx_recv;
-    command_connection_->send = ngx_send;
-    command_connection_->recv_chain = ngx_recv_chain;
-    command_connection_->send_chain = ngx_send_chain;
-    command_connection_->log = log_;
-    command_connection_->read->log = log_;
-    command_connection_->write->log = log_;
-    command_connection_->data = this;
-    command_connection_->read->handler = CommandHandler;
-    ngx_add_event(command_connection_->read, NGX_READ_EVENT, 0);
-
     if (proxy_.url.len == 0) {
       return true;
     }
@@ -212,8 +182,29 @@ namespace net_instaweb {
   }
 
   void NgxUrlAsyncFetcher::ShutDown() {
-      shutdown_ = true;
+    shutdown_ = true;
-      SendCmd('S');
+    if (!pending_fetches_.empty()) {
+      for (Pool<NgxFetch>::iterator p = pending_fetches_.begin(),
+           e = pending_fetches_.end(); p != e; p++) {
+        NgxFetch* fetch = *p;
+        fetch->CallbackDone(false);
+      }
+      pending_fetches_.DeleteAll();
+    }
+
+    if (!active_fetches_.empty()) {
+      for (Pool<NgxFetch>::iterator p = active_fetches_.begin(),
+           e = active_fetches_.end(); p != e; p++) {
+        NgxFetch* fetch = *p;
+        fetch->CallbackDone(false);
+      }
+      active_fetches_.Clear();
+    }
+    if (event_connection_ != NULL) {
+      event_connection_->Shutdown();
+      delete event_connection_;
+      event_connection_ = NULL;
+    }
   }
 
   // It's called in the rewrite thread. All the fetches are started at
@@ -221,92 +212,46 @@ namespace net_instaweb {
   void NgxUrlAsyncFetcher::Fetch(const GoogleString& url,
                                  MessageHandler* message_handler,
                                  AsyncFetch* async_fetch) {
+    // Don't accept new fetches when shut down. This flow is also entered when
+    // we did not initialize properly in ::Init().
+    if (shutdown_) {
+      async_fetch->Done(false);
+      return;
+    }
     async_fetch = EnableInflation(async_fetch);
     NgxFetch* fetch = new NgxFetch(url, async_fetch,
-          message_handler, fetch_timeout_, log_);
+          message_handler, log_);
     ScopedMutex lock(mutex_);
     pending_fetches_.Add(fetch);
-    SendCmd('F');
-  }
 
-  // send command to nginx main thread
+    // TODO(oschaaf): thread safety on written vs shutdown.
-  // 'F' : start a fetch
+    // It is possible that shutdown() is called after writing an event? In that
-  // 'S' : shutdown the fetcher
+    // case, this could (rarely) fail when it shouldn't.
-  bool NgxUrlAsyncFetcher::SendCmd(const char command) {
+    bool written = event_connection_->WriteEvent(this);
-    int rc;
+    CHECK(written || shutdown_) << "NgxUrlAsyncFetcher: event write failure";
-    while (true) {
-      rc = write(pipe_fd_, &command, 1);
-      if (rc == 1) {
-        return true;
-      } else if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
-        // TODO(junmin): It's rare. But it need be fixed.
-      } else {
-        return false;
-      }
-    }
-    return true;
   }
 
   // This is the read event which is called in the main thread.
   // It will do the real work. Add the work event and start the fetch.
-  void NgxUrlAsyncFetcher::CommandHandler(ngx_event_t* cmdev) {
+  void NgxUrlAsyncFetcher::ReadCallback(const ps_event_data& data) {
-    char command;
+    std::vector<NgxFetch*> to_start;
-    int rc;
+    NgxUrlAsyncFetcher* fetcher = reinterpret_cast<NgxUrlAsyncFetcher*>(
-    ngx_connection_t* c = static_cast<ngx_connection_t*>(cmdev->data);
+      data.sender);
-    NgxUrlAsyncFetcher* fetcher = static_cast<NgxUrlAsyncFetcher*>(c->data);
-    do {
-      rc = read(c->fd, &command, 1);
-    } while (rc == -1 && errno == EINTR);
 
-    CHECK(rc == -1 || rc == 0 || rc == 1);
+    fetcher->mutex_->Lock();
+    fetcher->completed_fetches_.DeleteAll();
 
-    if (rc == -1 || rc == 0) {
+    for (Pool<NgxFetch>::iterator p = fetcher->pending_fetches_.begin(),
-      // EAGAIN
+             e = fetcher->pending_fetches_.end(); p != e; p++) {
-      return;
+      NgxFetch* fetch = *p;
+      to_start.push_back(fetch);
     }
 
-    std::vector<NgxFetch*> to_start;
+    fetcher->pending_fetches_.Clear();
+    fetcher->mutex_->Unlock();
 
-    switch (command) {
+    for (size_t i = 0; i < to_start.size(); i++) {
-      // All the new fetches are appended in the pending_fetches.
+      fetcher->StartFetch(to_start[i]);
-      // Start all these fetches.
-      case 'F':
-        fetcher->mutex_->Lock();
-        fetcher->completed_fetches_.DeleteAll();
-        for (Pool<NgxFetch>::iterator p = fetcher->pending_fetches_.begin(),
-                 e = fetcher->pending_fetches_.end(); p != e; p++) {
-          NgxFetch* fetch = *p;
-          to_start.push_back(fetch);
-        }
-
-        fetcher->pending_fetches_.Clear();
-        fetcher->mutex_->Unlock();
-
-        for (size_t i = 0; i < to_start.size(); i++) {
-          fetcher->StartFetch(to_start[i]);
-        }
-        CHECK(ngx_handle_read_event(cmdev, 0) == NGX_OK);
-        break;
-
-      // Shutdown all the fetches.
-      case 'S':
-        if (!fetcher->pending_fetches_.empty()) {
-          fetcher->pending_fetches_.DeleteAll();
-        }
-
-        if (!fetcher->active_fetches_.empty()) {
-          for (Pool<NgxFetch>::iterator p = fetcher->active_fetches_.begin(),
-               e = fetcher->active_fetches_.end(); p != e; p++) {
-            NgxFetch* fetch = *p;
-            fetch->CallbackDone(false);
-          }
-          fetcher->active_fetches_.Clear();
-        }
-        CHECK(ngx_del_event(cmdev, NGX_READ_EVENT, 0) == NGX_OK);
-        break;
-
-      default:
-        break;
     }
 
     return;

src/ngx_url_async_fetcher.h

@@ -33,11 +33,14 @@ extern "C" {
 }
 
 #include <vector>
+
+#include "ngx_event_connection.h"
+
 #include "net/instaweb/http/public/url_async_fetcher.h"
-#include "net/instaweb/util/public/basictypes.h"
+#include "pagespeed/kernel/base/basictypes.h"
-#include "net/instaweb/util/public/pool.h"
+#include "pagespeed/kernel/base/pool.h"
-#include "net/instaweb/util/public/string.h"
+#include "pagespeed/kernel/base/string.h"
-#include "net/instaweb/util/public/thread_system.h"
+#include "pagespeed/kernel/base/thread_system.h"
 
 
 namespace net_instaweb {
@@ -53,27 +56,27 @@ class NgxUrlAsyncFetcher : public UrlAsyncFetcher {
   NgxUrlAsyncFetcher(
       const char* proxy, ngx_log_t* log, ngx_msec_t resolver_timeout,
       ngx_msec_t fetch_timeout, ngx_resolver_t* resolver,
-      ThreadSystem* thread_system, MessageHandler* handler);
+      int max_keepalive_requests, ThreadSystem* thread_system,
+      MessageHandler* handler);
 
   ~NgxUrlAsyncFetcher();
 
   // It should be called in the module init_process callback function. Do some
   // intializations which can't be done in the master process
-  bool Init();
+  bool Init(ngx_cycle_t* cycle);
 
   // shutdown all the fetches.
   virtual void ShutDown();
 
+  // the read handler in the main thread
+  static void ReadCallback(const ps_event_data& data);
+
   virtual bool SupportsHttps() const { return false; }
 
   virtual void Fetch(const GoogleString& url,
                      MessageHandler* message_handler,
                      AsyncFetch* callback);
 
-  // send the command from the current thread to main thread
-  bool SendCmd(const char command);
-  // the read handler in the main thread
-  static void CommandHandler(ngx_event_t* cmdev);
   bool StartFetch(NgxFetch* fetch);
 
   // Remove the completed fetch from the active fetch set, and put it into a
@@ -136,12 +139,13 @@ class NgxUrlAsyncFetcher : public UrlAsyncFetcher {
 
   ngx_pool_t* pool_;
   ngx_log_t* log_;
-  ngx_connection_t* command_connection_;  // the command pipe
-  int pipe_fd_;  // the write pipe end
   ngx_resolver_t* resolver_;
+  int max_keepalive_requests_;
   ngx_msec_t resolver_timeout_;
   ngx_msec_t fetch_timeout_;
 
+  NgxEventConnection* event_connection_;
+
   DISALLOW_COPY_AND_ASSIGN(NgxUrlAsyncFetcher);
 };
 

test/pagespeed_test.conf.template

@@ -24,10 +24,13 @@ http {
                    '"$http_user_agent"';
   access_log "@@ACCESS_LOG@@" cache;
 
+  # Don't put entries in the error log for 403s and 404s.
+  log_not_found off;
+
   proxy_cache_path "@@PROXY_CACHE@@"  levels=1:2   keys_zone=htmlcache:60m inactive=90m  max_size=50m;
   proxy_temp_path "@@TMP_PROXY_CACHE@@";
 
-  pagespeed ProcessScriptVariables on;
+  pagespeed ProcessScriptVariables all;
   pagespeed StatisticsPath /ngx_pagespeed_statistics;
   pagespeed GlobalStatisticsPath /ngx_pagespeed_global_statistics;
   pagespeed ConsolePath /pagespeed_console;
@@ -38,6 +41,10 @@ http {
   pagespeed StaticAssetPrefix /pagespeed_custom_static/;
 
   pagespeed MessageBufferSize 200000;
+  # Increase the default fetcher timeout to resolve sporadic flakeyness when
+  # the native fetcher uses 8.8.8.8 to resolve.
+  pagespeed FetcherTimeoutMs 10000;
+  pagespeed NativeFetcherMaxKeepaliveRequests 50;
 
   root "@@SERVER_ROOT@@";
 
@@ -56,6 +63,12 @@ http {
   pagespeed CreateSharedMemoryMetadataCache "@@SHM_CACHE@@" 8192;
   pagespeed BlockingRewriteKey psatest;
 
+  # We need to checkpoint more frequently than usual so that our checkpointing
+  # test will definitly have seen one of these intervals and so not be flaky.
+  # This needs to have a 1s interval because the checkpointing test has a 2s
+  # sleep.
+  pagespeed ShmMetadataCacheCheckpointIntervalSec 1;
+
   # CriticalImagesBeaconEnabled is now on by default, but we disable in testing.
   # With this option enabled, the inline image system test will currently fail.
   # When critical image beaconing is enabled, only critical images are inlined.
@@ -72,6 +85,9 @@ http {
   pagespeed StatisticsLogging on;
   pagespeed LogDir "@@TEST_TMP@@/logdir";
 
+  # Expanded to CentralControllerPort directive when RUN_CONTROLLER_TEST=on
+  @@CONTROLLER@@
+
   server {
     # Sets up a logical home-page server on
     # max-cacheable-content-length.example.com.  This server is only used to
@@ -88,6 +104,34 @@ http {
     pagespeed MaxCacheableContentLength 85;
   }
 
+  # Test how load from file handles maximum sizes.
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name lff-large-files.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed MaxCacheableContentLength 4096;
+    pagespeed LoadFromFile
+      "http://lff-large-files.example.com/"
+      "@@SERVER_ROOT@@/";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name lff-large-files-no-fallback.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed MaxCacheableContentLength 4096;
+    # http://lff-large-files-no-fallback.example.com/foo.css won't load, because
+    # we've configured a LoadFromFile pattern that doesn't match the path the
+    # webserver would take.
+    pagespeed LoadFromFile
+      "http://lff-large-files-no-fallback.example.com/"
+      "@@SERVER_ROOT@@/mod_pagespeed_example/styles/";
+  }
+
   pagespeed UseNativeFetcher "@@NATIVE_FETCHER@@";
   @@RESOLVER@@
 
@@ -130,7 +174,7 @@ http {
     # inline_images (ii), defer_javascript (dj), webp (jw) and lossless_webp
     # (ws).
     if ($http_user_agent ~*
-        "Chrome/[2][3-9]+\.|Chrome/[[3-9][0-9]+\.|Chrome/[0-9]{3,}\.") {
+        "Chrome/[2][3-9]+\.|Chrome/[3-9][0-9]+\.|Chrome/[0-9]{3,}\.") {
       set $ps_capability_list "ll,ii,dj,jw,ws:";
     }
     # Cache-fragment 3: This fragment contains (a) Desktop User-Agents that
@@ -262,14 +306,14 @@ http {
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
-    server_name mpd.example.com;
+    server_name secondary.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
 
-    pagespeed MapProxyDomain mpd.example.com/gstatic_images
+    pagespeed MapProxyDomain secondary.example.com/gstatic_images
                              http://www.gstatic.com/psa/static;
   }
 
-  # These two vhosts are for testing the experiment framework (Furious).
+  # These three vhosts are for testing the experiment framework (Furious).
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -279,12 +323,31 @@ http {
 
     pagespeed RunExperiment on;
     pagespeed AnalyticsID "123-45-6734";
+    pagespeed UseAnalyticsJs false;
     pagespeed ExperimentVariable 2;
     pagespeed ExperimentSpec
       "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
     pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
     pagespeed ExperimentSpec "id=3;default;percent=0";
   }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name contentexperiment.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed InPlaceResourceOptimization off;
+
+    pagespeed RunExperiment on;
+    pagespeed AnalyticsID "123-45-6734";
+    pagespeed UseAnalyticsJs false;
+    pagespeed ExperimentVariable 2;
+    pagespeed ExperimentSpec
+      "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50;options=ContentExperimentID=ID:H1BpS2TCRFmJzZgjwyeBHQ,ContentExperimentVariantID=111";
+    pagespeed ExperimentSpec
+      "id=2;enable=recompress_images;percent=50;options=ContentExperimentID=ID:H1BpS2TCRFmJzZgjwyeBHQ,ContentExperimentVariantID=222";
+    pagespeed ExperimentSpec
+      "id=3;default;percent=0;options=ContentExperimentID=123,ContentExperimentVariantID=333";
+  }
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -298,6 +361,16 @@ http {
     pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
     pagespeed ExperimentSpec "id=3;default;percent=0";
   }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name experiment.devicematch.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed RunExperiment on;
+    pagespeed ExperimentSpec
+      "id=1;percent=100;matches_device_type=mobile;enable=recompress_images";
+  }
 
   server {
     listen @@SECONDARY_PORT@@;
@@ -368,6 +441,34 @@ http {
     pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
   }
 
+  # Set up a reverse proxy (rproxy.) and origin (origin.) as vhosts for
+  # showing that we can configure PageSpeed via response headers.
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name rproxy.rmcomments.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed DisableFilters add_instrumentation,remove_comments;
+
+    # Note that we don't enable remove_comments here; that setting comes from
+    # the response headers from origin.rmcomments.example.com
+    location / {
+      proxy_pass http://127.0.0.1:@@SECONDARY_PORT@@/;
+      proxy_set_header "Host" "origin.rmcomments.example.com";
+    }
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name origin.rmcomments.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed off;
+    location / {
+      add_header PageSpeedFilters remove_comments;
+    }
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -383,9 +484,6 @@ http {
   # Build a configuration hierarchy where at the root we have turned on
   # OptimizeForBandwidth, and in various subdirectories we override settings
   # to make them more aggressive.
-  #
-  # In Apache we can do this all with Directory blocks, but to get the same
-  # inheretence in Nginx we need to have location blocks inside a server block.
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -503,6 +601,10 @@ http {
     pagespeed UrlValuedAttribute img alt-src Image;
     pagespeed UrlValuedAttribute video alt-a Image;
     pagespeed UrlValuedAttribute video alt-b Image;
+    pagespeed UrlValuedAttribute link data-stylesheet Stylesheet;
+    pagespeed UrlValuedAttribute span data-stylesheet-a Stylesheet;
+    pagespeed UrlValuedAttribute span data-stylesheet-b Stylesheet;
+    pagespeed UrlValuedAttribute span data-stylesheet-c Stylesheet;
 
     # Also test that we can redefine spec-defined attributes.
     pagespeed UrlValuedAttribute blockquote cite Image;
@@ -544,15 +646,6 @@ http {
     pagespeed EnableFilters rewrite_domains;
   }
 
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name retaincomment.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-
-    pagespeed RetainComment " google_ad_section*";
-  }
-
   server {
     # Test host for shared memory cache.
     listen @@SECONDARY_PORT@@;
@@ -569,8 +662,23 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name xfp.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-
     pagespeed RespectXForwardedProto on;
+
+    location /redirecting_origin {
+      pagespeed off;
+      # Hack: we clear the response headers using headers_more.
+      # If we don't, nginx will add an extra empty Location: headers here.
+      # It is kind of hard to get nginx to generate a relative location header
+      # that starts with "/".
+      more_clear_headers 'Location';
+      add_header Location /mod_pagespeed_example;
+      return 301;
+    }
+    location /redirect {
+      proxy_method GET;
+      proxy_pass http://127.0.0.1:@@SECONDARY_PORT@@/redirecting_origin;
+      proxy_set_header "Host" "xfp.example.com";
+    }
   }
 
   server {
@@ -636,7 +744,44 @@ http {
     pagespeed EnableFilters rewrite_images,rewrite_css;
     pagespeed EnableFilters convert_to_webp_lossless;
     pagespeed EnableFilters in_place_optimize_for_browser;
+    pagespeed JpegRecompressionQuality 75;
+    pagespeed WebpRecompressionQuality 70;
     pagespeed InPlaceResourceOptimization on;
+    pagespeed AllowVaryOn "Accept";
+    pagespeed FileCachePath "@@IPRO_CACHE@@";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name ipro-for-browser-vary-on-auto.example.com;
+    root "@@SERVER_ROOT@@/mod_pagespeed_example";
+    pagespeed EnableFilters rewrite_images,rewrite_css;
+    pagespeed EnableFilters convert_to_webp_animated;
+    pagespeed EnableFilters convert_to_webp_lossless;
+    pagespeed EnableFilters in_place_optimize_for_browser;
+    pagespeed InPlaceResourceOptimization on;
+    # pagespeed AllowVaryOn "Accept";  # Default is "Auto".
+    pagespeed ImageRecompressionQuality 90;
+    pagespeed JpegRecompressionQuality 75;
+    pagespeed JpegRecompressionQualityForSmallScreens 55;
+    pagespeed JpegQualityForSaveData 35;
+    pagespeed WebpRecompressionQuality 70;
+    pagespeed WebpRecompressionQualityForSmallScreens 50;
+    pagespeed WebpQualityForSaveData 30;
+    pagespeed WebpAnimatedRecompressionQuality 60;
+    pagespeed FileCachePath "@@IPRO_CACHE@@";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name ipro-for-browser-vary-on-none.example.com;
+    root "@@SERVER_ROOT@@/mod_pagespeed_example";
+    pagespeed EnableFilters rewrite_images,in_place_optimize_for_browser;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed AllowVaryOn "None";
+    pagespeed ImageRecompressionQuality 75;
     pagespeed FileCachePath "@@IPRO_CACHE@@";
   }
 
@@ -704,7 +849,7 @@ http {
     server_name www.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@;
+    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
@@ -718,7 +863,7 @@ http {
     server_name origin.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@;
+    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
@@ -732,7 +877,7 @@ http {
     server_name cdn.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@;
+    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
@@ -786,6 +931,13 @@ http {
     pagespeed ConsolePath /custom_pagespeed_console;
     pagespeed MessagesPath /custom_pagespeed_message;
     pagespeed AdminPath /custom_pagespeed_admin;
+
+    pagespeed StatisticsDomains Allow *;
+    pagespeed GlobalStatisticsDomains Allow *;
+    pagespeed MessagesDomains Allow *;
+    pagespeed ConsoleDomains Allow *;
+    pagespeed AdminDomains Allow *;
+    pagespeed GlobalAdminDomains Allow *;
   }
 
   server {
@@ -793,18 +945,13 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name inherit-paths.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-  }
 
-  server {
+    pagespeed StatisticsDomains Allow *;
-    listen @@SECONDARY_PORT@@;
+    pagespeed GlobalStatisticsDomains Allow *;
-    listen [::]:@@SECONDARY_PORT@@;
+    pagespeed MessagesDomains Allow *;
-    server_name notransform.example.com;
+    pagespeed ConsoleDomains Allow *;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed AdminDomains Allow *;
-
+    pagespeed GlobalAdminDomains Allow *;
-    pagespeed RewriteLevel PassThrough;
-    pagespeed EnableFilters rewrite_images;
-
-    add_header Cache-Control no-transform;
   }
 
   server {
@@ -865,7 +1012,10 @@ http {
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
     pagespeed RewriteLevel PassThrough;
     pagespeed on;
-    add_header PageSpeedFilters add_instrumentation;
+    location / {
+      proxy_pass http://localhost:@@SECONDARY_PORT@@;
+      proxy_set_header Host "enable-filter-header-origin.example.com";
+    }
   }
 
   server {
@@ -875,7 +1025,32 @@ http {
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
     pagespeed EnableFilters add_instrumentation;
     pagespeed on;
-    add_header PageSpeed off;
+    location / {
+      proxy_pass http://localhost:@@SECONDARY_PORT@@;
+      proxy_set_header Host "pagespeed-off-header-origin.example.com";
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name headers.example.com;
+    pagespeed InPlaceRewriteDeadlineMs -1;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed LoadFromFile "http://headers.example.com/"
+              "@@SERVER_ROOT@@/";
+    location /mod_pagespeed_test/ {
+      more_set_headers "Server: override";
+      more_set_headers -s '404' 'Cache-Control: override';
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name custom404.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    error_page 404 /404.html;
   }
 
   server {
@@ -958,7 +1133,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name date.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    add_header "Date" "Date: Fri, 16 Oct 2009 23:05:07 GMT";
+    add_header "Date" "Fri, 16 Oct 2009 23:05:07 GMT";
   }
 
   server {
@@ -1040,6 +1215,92 @@ http {
     pagespeed DisableFilters remove_comments,add_instrumentation;
   }
 
+  # Start remote config servers.
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT1@@/remote.cfg";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-partially-invalid.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT2@@/remote.cfg";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-invalid.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT3@@/remote.cfg";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-failed-fetch.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT5@@/remote.cfg";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-slow-fetch.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT6@@/remote.cfg";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-experiment.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT7@@/remote.cfg";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+
+  # Test that pagespeed is disabled when sendfile headers are present.
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name uses-sendfile.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed EnableFilters inline_javascript,rewrite_javascript;
+    add_header 'X-Sendfile' 'blablabla';
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name doesnt-sendfile.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed EnableFilters inline_javascript,rewrite_javascript;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name uses-xaccelredirect.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed EnableFilters inline_javascript,rewrite_javascript;
+    add_header 'X-Accel-Redirect' 'blablabla';
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name map-static-domain.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed MapRewriteDomain http://static-cdn.example.com
+                               http://map-static-domain.example.com;
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters defer_javascript,rewrite_domains;
+  }
+
   # Proxy + IPRO a gzip'd file for testing Issue 896.
   server {
     listen @@SECONDARY_PORT@@;
@@ -1057,6 +1318,44 @@ http {
     }
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name compressedcache.example.com;
+    pagespeed on;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed EnableFilters rewrite_css;
+    pagespeed HttpCacheCompressionLevel 9;
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name uncompressedcache.example.com;
+    pagespeed on;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed EnableFilters rewrite_css;
+    pagespeed HttpCacheCompressionLevel 0;
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -1119,7 +1418,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name proxy-post-origin.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
   }
 
   server {
@@ -1132,6 +1431,24 @@ http {
     pagespeed FileCachePath "@@FILE_CACHE@@";
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name script-filters.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    root "@@SERVER_ROOT@@/";
+    set $filters "";
+    set $domain_shards "cdn1.example.com,cdn2.example.com";
+    if ($http_X_Script) {
+        set $filters "add_instrumentation";
+        set $domain_shards "";
+    }
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_domains;
+    pagespeed EnableFilters $filters;
+    pagespeed ShardDomain script-filters.example.com "$domain_shards";
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -1141,13 +1458,218 @@ http {
     pagespeed EnableCachePurge on;
 
     pagespeed PurgeMethod PURGE;
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/purge";
     pagespeed FileCachePath "@@FILE_CACHE@@_purge";
     pagespeed DisableFilters add_instrumentation;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css;
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name psoff-dir-on.example.com;
+
+    # Test purging individual URLs without flushing the entire metadata cache.
+    pagespeed EnableCachePurge on;
+
+    pagespeed PurgeMethod PURGE;
+    root "@@SERVER_ROOT@@/purge";
+    pagespeed FileCachePath "@@FILE_CACHE@@_dir_on";
+    pagespeed DisableFilters add_instrumentation;
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_css;
+    pagespeed off;
+    location / {
+      pagespeed on;
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name  cc-header-origin.example.com;
+    pagespeed InPlaceResourceOptimization off;
+    pagespeed off;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    root "@@SERVER_ROOT@@/";
+    location /mod_pagespeed_test/nostore {
+      add_header "Cache-Control" "max-age=12345";
+      add_header "Cache-Control" "public, no-store";
+      add_header "Cache-Control" "max-age=14";
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name  enable-filter-header-origin.example.com;
+    pagespeed InPlaceResourceOptimization off;
+    pagespeed off;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    root "@@SERVER_ROOT@@/";
+    location / {
+      add_header "PageSpeedFilters" "add_instrumentation";
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name  pagespeed-off-header-origin.example.com;
+    pagespeed InPlaceResourceOptimization off;
+    pagespeed off;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    root "@@SERVER_ROOT@@/";
+    location / {
+      add_header "PageSpeed" "off";
+    }
+  }
+
+  pagespeed MessagesDomains Allow messages-allowed.example.com;
+  pagespeed MessagesDomains Allow cleared-inherited.example.com;
+  pagespeed MessagesDomains Allow cleared-inherited-reallowed.example.com;
+  pagespeed MessagesDomains Allow more-messages-allowed.example.com;
+  pagespeed MessagesDomains Allow anything-*-wildcard.example.com;
+  pagespeed MessagesDomains Allow localhost;
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name messages-allowed.example.com
+                messages-not-allowed.example.com
+                more-messages-allowed.example.com
+                anything-a-wildcard.example.com
+                anything-b-wildcard.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name messages-still-not-allowed.example.com
+                but-this-message-allowed.example.com
+                and-this-one.example.com;
+    pagespeed MessagesDomains Allow but-this-message-allowed.example.com;
+    pagespeed MessagesDomains Allow and-this-one.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name cleared-inherited.example.com
+                cleared-inherited-reallowed.example.com
+                messages-allowed-at-vhost.example.com
+                messages-not-allowed-at-vhost.example.com
+                anything-c-wildcard.example.com;
+    pagespeed MessagesDomains Disallow *;
+    pagespeed MessagesDomains Allow cleared-inherited-reallowed.example.com;
+    pagespeed MessagesDomains Allow messages-allowed-at-vhost.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name cleared-inherited-unlisted.example.com;
+    pagespeed MessagesDomains Allow *;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+  server {
+    server_name nothing-allowed.example.com;
+    pagespeed MessagesDomains Disallow *;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+  server {
+    server_name nothing-explicitly-allowed.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name everything-explicitly-allowed.example.com
+                everything-explicitly-allowed-but-aliased.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed StatisticsDomains Allow everything-explicitly-allowed.example.com;
+    pagespeed GlobalStatisticsDomains
+      Allow everything-explicitly-allowed.example.com;
+    pagespeed MessagesDomains Allow everything-explicitly-allowed.example.com;
+    pagespeed ConsoleDomains Allow everything-explicitly-allowed.example.com;
+    pagespeed AdminDomains Allow everything-explicitly-allowed.example.com;
+    pagespeed GlobalAdminDomains
+     Allow everything-explicitly-allowed.example.com;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name psol-vary.example.com;
+    pagespeed on;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name flush.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed RewriteDeadlinePerFlushMs 1;
+
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name noflush.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed RewriteDeadlinePerFlushMs 1;
+    pagespeed FollowFlushes off;
+
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name special-response.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    error_page 404 /mod_pagespeed_test/php_withflush.php;
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+    }
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name debug-filters.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters debug;
+  }
   server {
     listen @@PRIMARY_PORT@@;
     listen [::]:@@PRIMARY_PORT@@;
@@ -1161,6 +1683,14 @@ http {
     }
 
     location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" {
+      # TODO(oschaaf): The following breaks nginx starting up, as the bad header
+      # value gets rejected during configuration parsing. However, we can't easily
+      # build a test for that right now.
+      # pagespeed AddResourceHeader "X-Fo\n\no" "Bar";
+      # pagespeed AddResourceHeader "X-Foo" "Ba\nr";
+
+      # For testing that AddResourceHeader gets passed through
+      pagespeed AddResourceHeader "X-Foo" "Bar";
       add_header "" "";
     }
 
@@ -1192,6 +1722,10 @@ http {
       add_header 'Cache-Control' 'no-transform';
     }
 
+    location /mod_pagespeed_test/no_transform {
+      add_header 'Cache-Control' 'no-transform';
+    }
+
     # uncomment the following two lines if you're testing memcached
     #pagespeed MemcachedServers "localhost:11211";
     #pagespeed MemcachedThreads 1;
@@ -1206,7 +1740,18 @@ http {
     #pagespeed ExperimentSpec "id=2;percent=50";
 
     pagespeed Library 43 1o978_K0_LNE5_ystNklf
-    http://www.modpagespeed.com/rewrite_javascript.js;
+      http://www.modpagespeed.com/rewrite_javascript.js;
+    pagespeed RetainComment " google_ad_section*";
+    pagespeed PermitIdsForCssCombining "allowed-to-combine-1-*";
+    pagespeed PermitIdsForCssCombining "allowed-to-combine-2-*";
+
+    # Test proxying of non-.pagespeed. resources.
+    pagespeed MapProxyDomain http://localhost:@@PRIMARY_PORT@@/modpagespeed_http
+                           http://@@PAGESPEED_TEST_HOST@@/do_not_modify;
+    pagespeed MapProxyDomain http://localhost:@@PRIMARY_PORT@@/content_type_present
+                           http://@@PAGESPEED_TEST_HOST@@:8091;
+    pagespeed MapProxyDomain http://localhost:@@PRIMARY_PORT@@/content_type_absent
+                           http://@@PAGESPEED_TEST_HOST@@:8092;
 
     add_header X-Extra-Header 1;
 
@@ -1232,9 +1777,8 @@ http {
     pagespeed AvoidRenamingIntrospectiveJavascript off;
 
     location /mod_pagespeed_test/nostore {
-      add_header "Cache-Control" "max-age=12345";
+      proxy_pass http://localhost:@@SECONDARY_PORT@@;
-      add_header "Cache-Control" "public, no-store";
+      proxy_set_header Host "cc-header-origin.example.com";
-      add_header "Cache-Control" "max-age=14";
     }
 
     location /mod_pagespeed_test/forbid_all_disabled/disabled {
@@ -1327,6 +1871,77 @@ http {
       expires 5m;
     }
 
+    location /mod_pagespeed_test/example.json {
+      expires 10m;
+    }
+
+    location /mod_pagespeed_test/ipro/cc200/ {
+      add_header "Cache-Control" "max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200p/ {
+      add_header "Cache-Control" "private, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200nc/ {
+      add_header "Cache-Control" "no-cache, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200ns/ {
+      add_header "Cache-Control" "no-store, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200nt/ {
+      add_header "Cache-Control" "no-transform, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma5/ {
+      add_header "Cache-Control" "s-maxage=5, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50/ {
+      add_header "Cache-Control" "s-maxage=50, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50nsp/ {
+      add_header "Cache-Control" "s-maxage=50,max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc9/ {
+      add_header "Cache-Control" "max-age=9";
+    }
+    location /mod_pagespeed_test/ipro/cc200cc9/ {
+      add_header "Cache-Control" "max-age=200, max-age=9";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50sma5/ {
+      add_header "Cache-Control" "max-age=200, s-maxage=50, s-maxage=5";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50cc9/ {
+      add_header "Cache-Control" "max-age=200, s-maxage=50, max-age=9";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50cc9nsp/ {
+      add_header "Cache-Control" "max-age=200,s-maxage=50,max-age=9";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50sma51/ {
+      add_header "Cache-Control" "max-age=200, s-maxage=50, s-maxage=51";
+    }
+
+    location /mod_pagespeed_test/ipro/wait/ {
+      # TODO(jmarantz): ModPagespeedInPlaceWaitForOptimized should be superfluous,
+      # or made equivalent to ModPagespeedInPlaceRewriteDeadlineMs -1, which waits
+      # forever.  Otherwise ModPagespeedInPlaceRewriteDeadlineMs should just have
+      # the specified deadline.
+      # # See https://github.com/pagespeed/mod_pagespeed/issues/1171 for more
+      # detailed discussion.
+      pagespeed InPlaceWaitForOptimized on;
+    }
+
+    location /mod_pagespeed_test/ipro/wait/long/ {
+      # Make the deadline long here for valgrind tests.  We could
+      # conditionalize this.
+      pagespeed InPlaceRewriteDeadlineMs 10000;
+    }
+
+    location /mod_pagespeed_test/ipro/wait/short/ {
+      pagespeed EnableFilters in_place_optimize_for_browser;
+
+      # Make the deadline short here as we expect to always miss it
+      # in tests.
+      pagespeed InPlaceRewriteDeadlineMs 1;
+    }
+
     location /mod_pagespeed_test/ipro/instant/wait/ {
       pagespeed InPlaceWaitForOptimized on;
 
@@ -1343,14 +1958,16 @@ http {
     }
 
     location /mod_pagespeed_test/ipro/instant/deadline/ {
-      pagespeed InPlaceRewriteDeadlineMs -1;
+      set $ps_deadline -1;
+      pagespeed ImageRecompressionQuality $ps_deadline;
+      pagespeed InPlaceRewriteDeadlineMs $ps_deadline;
     }
 
     # Test to make sure that user-authenticated resources do not get cached and
     # optimized.
     location /mod_pagespeed_test/auth/ {
        auth_basic "Restricted";
-       auth_basic_user_file "@@SERVER_ROOT@@mod_pagespeed_test/auth/passwd.conf";
+       auth_basic_user_file "@@SERVER_ROOT@@/mod_pagespeed_test/auth/passwd.conf";
     }
 
     location /mod_pagespeed_test/ipro/cookie/ {
@@ -1391,11 +2008,31 @@ http {
     pagespeed FetchHttps enable;
     location /mod_pagespeed_test/https_fetch/ {
       pagespeed DisableFilters inline_images;
+      set $ps_gstatic "https://www.gstatic.com/psa/static";
       pagespeed MapProxyDomain
           http://localhost:@@PRIMARY_PORT@@/https_gstatic_dot_com
-          https://www.gstatic.com/psa/static;
+          $ps_gstatic;
     }
 
+    location /mod_pagespeed_test/strip_subresource_hints/default/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed RewriteLevel CoreFilters;
+      pagespeed DisAllow *dontrewriteme*;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/default_passthrough/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed RewriteLevel PassThrough;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/preserve_on/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed PreserveSubresourceHints on;
+      pagespeed RewriteLevel CoreFilters;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/preserve_off/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed PreserveSubresourceHints off;
+      pagespeed RewriteLevel CoreFilters;
+    }
     # $host implicitly tests script variable support. I'd love to test it more
     # directly, but so far this is the best I've come up with and duplicating
     # the test doesn't seem to make sense.
@@ -1403,6 +2040,11 @@ http {
         "http://$host:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/instant/"
         "@@SERVER_ROOT@@/mod_pagespeed_test/ipro/instant/";
 
+    location /mod_pagespeed_test/public/ {
+      add_header "Cache-Control" "public, max-age=600";
+      pagespeed PreserveUrlRelativity off;
+    }
+
     pagespeed EnableFilters remove_comments;
 
     # Test LoadFromFile mapping by mapping one dir to another.
@@ -1416,7 +2058,6 @@ http {
       "@@SERVER_ROOT@@/mod_pagespeed_test/load_from_file/file_dir/httponly/";
     pagespeed LoadFromFileRuleMatch Disallow \.ssp.css$ps_dollar;
     pagespeed LoadFromFileRuleMatch Allow exception\.ssp\.css$ps_dollar;
-
     #charset koi8-r;
 
     #access_log  logs/host.access.log  main;
@@ -1463,6 +2104,7 @@ http {
     image/svg+xml                         svg svgz;
     image/webp                            webp;
 
+    application/json                      json;
     application/java-archive              jar war ear;
     application/mac-binhex40              hqx;
     application/msword                    doc;

test/run_tests.sh

@@ -23,9 +23,11 @@
 # Exits with status 2 if command line args are wrong.
 #
 # Usage:
-#   ./run_tests.sh primary_port secondary_port mod_pagespeed_dir
+#   ./run_tests.sh primary_port secondary_port mod_pagespeed_dir ngx_binary
+#   pagespeed_test_host
 # Example:
-#   ./run_tests.sh 8050 8051 /path/to/mod_pagespeed
+#   ./run_tests.sh 8050 8051 /path/to/mod_pagespeed /path/to/nginx/binary
+#   selfsigned.modpagespeed.com
 #
 
 # Normally we test only with the native fetcher off.  Set
@@ -43,7 +45,7 @@ RUN_TESTS=${RUN_TESTS:-true}
 # true.
 USE_VALGRIND=${USE_VALGRIND:-false}
 
-if [ "$#" -ne 4 ] ; then
+if [ "$#" -ne 5 ] ; then
   echo "Usage: $0 primary_port secondary_port mod_pagespeed_dir"
   echo "  nginx_executable"
   exit 2
@@ -53,6 +55,15 @@ PRIMARY_PORT="$1"
 SECONDARY_PORT="$2"
 MOD_PAGESPEED_DIR="$3"
 NGINX_EXECUTABLE="$4"
+PAGESPEED_TEST_HOST="$5"
+CONTROLLER_PORT=8053
+RCPORT1=9991
+RCPORT2=9992
+RCPORT3=9993
+RCPORT4=9994
+RCPORT5=9995
+RCPORT6=9996
+RCPORT7=9997
 
 this_dir="$( cd $(dirname "$0") && pwd)"
 
@@ -62,7 +73,16 @@ function run_test_checking_failure() {
     SECONDARY_PORT="$SECONDARY_PORT" \
     MOD_PAGESPEED_DIR="$MOD_PAGESPEED_DIR" \
     NGINX_EXECUTABLE="$NGINX_EXECUTABLE" \
+    PAGESPEED_TEST_HOST="$PAGESPEED_TEST_HOST" \
     RUN_TESTS="$RUN_TESTS" \
+    CONTROLLER_PORT="$CONTROLLER_PORT" \
+    RCPORT1="$RCPORT1" \
+    RCPORT2="$RCPORT2" \
+    RCPORT3="$RCPORT3" \
+    RCPORT4="$RCPORT4" \
+    RCPORT5="$RCPORT5" \
+    RCPORT6="$RCPORT6" \
+    RCPORT7="$RCPORT7" \
     bash "$this_dir/nginx_system_test.sh"
   STATUS=$?
   echo "With $@ setup."

test/valgrind.sup

@@ -163,3 +163,31 @@
    fun:_ZN3re23DFA10AddToQueueEPNS0_5WorkqEij
    ...
 }
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Value8
+   fun:_ZNK3re210SparseSetTIvE8containsEi
+   ...
+}
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Cond
+   fun:_ZNK3re210SparseSetTIvE8containsEi
+   ...
+}
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Value8
+   fun:_ZNK3re211SparseArrayIiE9has_indexEi
+   ...
+}
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Cond
+   fun:_ZNK3re211SparseArrayIiE9has_indexEi
+   ...
+}