native fetcher ipv6 support - work in progress

This allows is to test again, but needs a lot of cleaning up.
More importantly, the google font inlining test fails, and I'm
not sure why that is, the fetch succeeds.

.gitignore

@@ -2,3 +2,4 @@ test/tmp
 psol/
 psol-*.tar.gz
 *.*.*.*.tar.gz
+nginx

.gitmodules

@@ -0,0 +1,18 @@
+[submodule "testing-dependencies/mod_pagespeed"]
+	path = testing-dependencies/mod_pagespeed
+	url = https://github.com/pagespeed/mod_pagespeed.git
+[submodule "testing-dependencies/ngx_cache_purge"]
+	path = testing-dependencies/ngx_cache_purge
+	url = https://github.com/FRiCKLE/ngx_cache_purge.git
+[submodule "testing-dependencies/nginx"]
+	path = testing-dependencies/nginx
+	url = https://github.com/nginx/nginx.git
+[submodule "testing-dependencies/set-misc-nginx-module"]
+	path = testing-dependencies/set-misc-nginx-module
+	url = https://github.com/openresty/set-misc-nginx-module
+[submodule "testing-dependencies/ngx_devel_kit"]
+	path = testing-dependencies/ngx_devel_kit
+	url = https://github.com/simpl/ngx_devel_kit
+[submodule "testing-dependencies/headers-more-nginx-module"]
+	path = testing-dependencies/headers-more-nginx-module
+	url = https://github.com/openresty/headers-more-nginx-module

.travis.yml

@@ -0,0 +1,33 @@
+language: c++
+env:
+  global:
+    - MAKEFLAGS=-j3
+# By default travis loads submodules serially, but we can load them in parallel
+# if we install an updated git and use --jobs.  Some timing numbers:
+#   serial: 257s
+#   jobs=4: 182s (29s to install new git, 153s to run the downloads)
+#   jobs=8: 179s (29s to install new git, 150s to run the downloads)
+# We can't use --depth=1, though, because github doesn't have
+# allowReachableSHA1InWant set.
+#
+# TODO(jefftk): once we're running on a server with git 2.8 or later we can have
+#               --jobs without installing a new git.
+git:
+  submodules: false
+before_install:
+  - sudo add-apt-repository --yes ppa:git-core/ppa
+  - sudo apt-get update
+  - sudo apt-get install git
+  - git submodule update --init --recursive --jobs=8
+install:
+  scripts/build_ngx_pagespeed.sh --devel --assume-yes
+script:
+  test/run_tests.sh $PWD/testing-dependencies/mod_pagespeed/ \
+                    $PWD/nginx/sbin/nginx
+dist: trusty
+sudo: required
+compiler:
+  - gcc
+notifications:
+  email:
+    - pagespeed-ci@googlegroups.com

PSOL_BINARY_URL

@@ -0,0 +1,4 @@
+In a release this file would contain the URL to download the pre-compiled PSOL
+binary, but on development branches (like this one) you have to build PSOL from
+source yourself.  See:
+  https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source

README.md

@@ -1,5 +1,7 @@
 ![ngx_pagespeed](https://lh6.googleusercontent.com/-qufedJIJq7Y/UXEvVYxyYvI/AAAAAAAADo8/JHDFQhs91_c/s401/04_ngx_pagespeed.png)
 
+[![Build Status](https://travis-ci.org/pagespeed/ngx_pagespeed.svg?branch=master)](https://travis-ci.org/pagespeed/ngx_pagespeed)
+
 ngx_pagespeed speeds up your site and reduces page load time by automatically
 applying web performance best practices to pages and associated assets (CSS,
 JavaScript, images) without requiring you to modify your existing content or

config

@@ -25,18 +25,28 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
 
   if [ ! -e "$mod_pagespeed_dir" ] ; then
     echo "ngx_pagespeed: pagespeed optimization library not found:"
-    echo ""
-    echo "   You need to separately download the pagespeed library:"
-    echo ""
-    echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.11.33.0.tar.gz"
-    echo "     $ tar -xzvf 1.11.33.0.tar.gz # expands to psol/"
-    echo ""
-    echo "   Or see the installation instructions:"
-    echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
+
+    psol_binary_url="$($ngx_addon_dir/scripts/format_binary_url.sh \
+                           $ngx_addon_dir/PSOL_BINARY_URL)"
+    if [[ "$psol_binary_url" != https://* ]]; then
+      echo "
+  This is a development branch of ngx_pagespeed, which means there is no
+  precompiled PSOL library available to link against.  Either build from a
+  release tag, like latest-beta, or build PSOL from source:
+    https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source"
+      exit 1
+    fi
+
+    echo "
+  You need to separately download the pagespeed library:
+     $ cd $ngx_addon_dir
+     $ wget $psol_binary_url
+     $ tar -xzvf $(basename $psol_binary_url) # expands to psol/
+
+  Or see the installation instructions:
+     https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source"
     exit 1
   fi
-
 else
   build_from_source=true
 fi
@@ -67,6 +77,8 @@ fi
 
 if [ "$NGX_DEBUG" = "YES" ]; then
   buildtype=Debug
+  # If we're using a psol tarball that doesn't contain Debug/ (which is the case
+  # from 1.12 onward) then this will be overriden to buildtype=Release below.
 else
   buildtype=Release
 fi
@@ -80,10 +92,8 @@ ps_maybe_gpp_base=`basename $CC| sed s/gcc/g++/`
 ps_maybe_gpp="`dirname $CC`/$ps_maybe_gpp_base"
 if [ -n "$NGX_GCC_VER" -a \( -x "$ps_maybe_gpp" \) ]; then
   LINK=$ps_maybe_gpp
-  NGX_TEST_LD_OPT="$NGX_TEST_LD_OPT -lstdc++"
-else
-  pagespeed_libs="-lstdc++"
 fi
+pagespeed_libs="-lstdc++"
 
 # The compiler needs to know that __sync_add_and_fetch_4 is ok,
 # and this requires an instruction that didn't exist on i586 or i386.
@@ -98,7 +108,8 @@ CFLAGS="$CFLAGS $FLAG_MARCH"
 # to our libpsol.a
 # See https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html
 CFLAGS="$CFLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
-CC_TEST_FLAGS="$CC_TEST_FLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
+CC_OLD_TEST_FLAGS="$CC_TEST_FLAGS"
+CC_TEST_FLAGS="$CC_TEST_FLAGS -D_GLIBCXX_USE_CXX11_ABI=0 --std=c++11"
 
 case "$NGX_GCC_VER" in
   4.8*)
@@ -135,6 +146,22 @@ if [ "$psol_binary" = "unset" ] ; then
     psol_binary="\
         $mod_pagespeed_dir/pagespeed/automatic/pagespeed_automatic.a"
   else
+    if ! [ -d "$ngx_addon_dir/psol/lib/$buildtype" ]; then
+      echo "
+You have set --with-debug for building nginx, but precompiled Debug binaries for
+PSOL, which ngx_pagespeed depends on, aren't available.  If you're trying to
+debug PSOL you need to build it from source.  If you just want to run nginx with
+debug-level logging you can use the Release binaries."
+      echo -n "
+Use the available Release binaries?"
+      read -p " [Y/n] " yn
+      if [[ "$yn" == N* || "$yn" == n* ]]; then
+        echo "Cancelled."
+        exit 1
+      fi
+
+      buildtype=Release
+    fi
     psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
     psol_binary="$psol_library_dir/pagespeed_automatic.a"
   fi
@@ -157,19 +184,21 @@ ngx_feature_incs="
 pagespeed_include="\
   $mod_pagespeed_dir \
   $mod_pagespeed_dir/third_party/chromium/src \
-  $mod_pagespeed_dir/third_party/google-sparsehash/src \
+  $mod_pagespeed_dir/third_party/google-sparsehash/src/src \
   $mod_pagespeed_dir/third_party/google-sparsehash/gen/arch/$os_name/$arch_name/include \
-  $mod_pagespeed_dir/third_party/protobuf/src \
+  $mod_pagespeed_dir/third_party/grpc/src/include \
+  $mod_pagespeed_dir/third_party/protobuf/src/src \
   $mod_pagespeed_dir/third_party/re2/src \
   $mod_pagespeed_dir/out/$buildtype/obj/gen \
   $mod_pagespeed_dir/out/$buildtype/obj/gen/protoc_out/instaweb \
   $mod_pagespeed_dir/third_party/apr/src/include \
   $mod_pagespeed_dir/third_party/aprutil/src/include \
   $mod_pagespeed_dir/third_party/apr/gen/arch/$os_name/$arch_name/include \
-  $mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include"
+  $mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include \
+  $mod_pagespeed_dir/url"
 ngx_feature_path="$pagespeed_include"
 
-pagespeed_libs="$pagespeed_libs $psol_binary -lrt -pthread -lm"
+pagespeed_libs="$psol_binary $pagespeed_libs -lrt -pthread -lm -luuid"
 ngx_feature_libs="$pagespeed_libs"
 ngx_feature_test="
 
@@ -196,7 +225,7 @@ ngx_feature_test="
 if [ $ngx_found = no ]; then
   cat << END
 $0: error: module ngx_pagespeed requires the pagespeed optimization library.
-Look in obj/autoconf.err for more details.
+Look in $PWD/$NGX_AUTOCONF_ERR for more details.
 END
   exit 1
 fi
@@ -286,8 +315,6 @@ if [ -n "$ngx_module_link" ]; then
                            | sed "s/ngx_pagespeed_etag_filter//" \
                            | sed "s/ngx_http_range_header_filter_module/ngx_http_range_header_filter_module ngx_pagespeed_etag_filter/")
     else
-      # config.make is not executed for dynamic modules
-      CFLAGS="$CFLAGS -Wno-c++11-extensions"
       if [ "$position_aux" = "true" ] ; then
         ngx_module_type=HTTP_AUX_FILTER
         ngx_module_order=""
@@ -320,7 +347,7 @@ ngx_feature_libs="-lstdc++"
 ngx_feature_test="
 
 #if defined(__clang__) && defined(__GLIBCXX__)
-   // See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning 
+   // See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning
    // for a list of various values of __GLIBCXX__. Note that they're not monotonic
    // with respect to version numbers.
    #if __GLIBCXX__ == 20120322  || __GLIBCXX__ == 20120614
@@ -347,9 +374,11 @@ if [ $ngx_found = no ]; then
   cat << END
 $0: error: module ngx_pagespeed requires gcc >= 4.8 or clang >= 3.3.
 See https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source for some recommendations.
-Look in objs/autoconf.err for more details.
+Look in $PWD/$NGX_AUTOCONF_ERR for more details.
 END
   exit 1
 fi
 
 have=NGX_PAGESPEED . auto/have
+
+CC_TEST_FLAGS="$CC_OLD_TEST_FLAGS"

config.make

@@ -1,7 +1,7 @@
-if [ -n "$NGX_CLANG_VER" ]; then
-  # Chromium headers assume clang is always in C++11 mode. Oblige it.
-  for ps_src_file in $PS_NGX_SRCS; do
-    ps_obj_file="$NGX_OBJS/addon/src/`basename $ps_src_file .cc`.o"
-    echo "$ps_obj_file : CFLAGS += --std=c++11" >> $NGX_MAKEFILE
-  done
-fi
+# Since nginx build system doesn't normally do C++, there is no CXXFLAGS for us
+# to touch, and compilers are understandably unhappy with --std=c++11 on C
+# files. Hence, we hack the makefile to add it for just our sources.
+for ps_src_file in $PS_NGX_SRCS; do
+  ps_obj_file="$NGX_OBJS/addon/src/`basename $ps_src_file .cc`.o"
+  echo "$ps_obj_file : CFLAGS += --std=c++11" >> $NGX_MAKEFILE
+done

scripts/build_ngx_pagespeed.sh

@@ -0,0 +1,808 @@
+#!/bin/bash
+#
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function usage() {
+  echo "
+Usage: build_ngx_pagespeed.sh [options]
+
+  Installs ngx_pagespeed and its dependencies.  Can optionally build and install
+  nginx as well.  Can be run either as:
+
+     bash <(curl -f -L -sS https://ngxpagespeed.com/install) [options]
+
+  Or:
+
+     git clone git@github.com:pagespeed/ngx_pagespeed.git
+     cd ngx_pagespeed/
+     git checkout <branch>
+     scripts/build_ngx_pagespeed.sh [options]
+
+Options:
+  -v, --ngx-pagespeed-version <ngx_pagespeed version>
+      What version of ngx_pagespeed to build.  Valid options include:
+      * latest-beta
+      * latest-stable
+      * a version number, such as 1.11.33.4
+
+      If you don't specify a version, defaults to latest-stable unless --devel
+      is specified, in which case it defaults to master.
+
+      This option doesn't make sense if we're running within an existing
+      ngx_pagespeed checkout.
+
+  -n, --nginx-version <nginx version>
+      What version of nginx to build.  If not set, this script only prepares the
+      ngx_pagespeed module, and expects you to handle including it when you
+      build nginx.
+
+      If you pass in 'latest' then this script scrapes the nginx download page
+      and attempts to determine the latest version automatically.
+
+  -m, --dynamic-module
+      Build ngx_pagespeed as a dynamic module.
+
+  -b, --builddir <directory>
+      Where to build.  Defaults to \$HOME.
+
+  -p, --no-deps-check
+      By default, this script checks for the packages it depends on and tries to
+      install them.  If you have installed dependencies from source or are on a
+      non-deb non-rpm system, this won't work.  In that case, install the
+      dependencies yourself and pass --no-deps-check.
+
+  -s, --psol-from-source
+      Build PSOL from source instead of downloading a pre-built binary module.
+
+  -l, --devel
+      Sets up a development environment in ngx_pagespeed/nginx, building with
+      testing-only dependencies.  Includes --psol-from-source, conflicts with
+      --nginx-version.  Uses a 'git clone' checkout for ngx_pagespeed and nginx
+      instead of downloading a tarball.
+
+  -t, --build-type
+      When building PSOL from source, what to tell it for BUILD_TYPE.  Defaults
+      to 'Release' unless --devel is set in which case it defaults to 'Debug'.
+
+  -y, --assume-yes
+      Assume the answer to all prompts is 'yes, please continue'.  Intended for
+      automated usage, such as buildbots.
+
+  -a, --additional-nginx-configure-arguments
+      When running ./configure for nginx, you may want to specify additional
+      arguments, such as --with-http_ssl_module.  By default this script will
+      pause and prompt you for them, but this option lets you pass them in.  For
+      example, you might do:
+        -a '--with-http_ssl_module --with-cc-opt=\"-I /usr/local/include\"'
+
+  -d, --dryrun
+      Don't make any changes to the system, just print what changes you
+      would have made.
+
+  -h, --help
+      Print this message and exit."
+}
+
+RED=31
+GREEN=32
+YELLOW=33
+function begin_color() {
+  color="$1"
+  echo -e -n "\e[${color}m"
+}
+function end_color() {
+  echo -e -n "\e[0m"
+}
+
+function echo_color() {
+  color="$1"
+  shift
+  begin_color "$color"
+  echo "$@"
+  end_color
+}
+
+function error() {
+  local error_message="$@"
+  echo_color "$RED" -n "Error: " >&2
+  echo "$@" >&2
+}
+
+# Prints an error message and exits with an error code.
+function fail() {
+  error "$@"
+
+  # Normally I'd use $0 in "usage" here, but since most people will be running
+  # this via curl, that wouldn't actually give something useful.
+  echo >&2
+  echo "For usage information, run this script with --help" >&2
+  exit 1
+}
+
+
+function status() {
+  echo_color "$GREEN" "$@"
+}
+
+# Intended to be called as:
+#   bash <(curl dl.google.com/.../build_ngx_pagespeed.sh) <args>
+
+# If we set -e or -u then users of this script will see it silently exit on
+# failure.  Instead we need to check the exit status of each command manually.
+# The run function handles exit-status checking for system-changing commands.
+# Additionally, this allows us to easily have a dryrun mode where we don't
+# actually make any changes.
+INITIAL_ENV=$(printenv | sort)
+function run() {
+  if "$DRYRUN"; then
+    echo_color "$YELLOW" -n "would run"
+    echo " $@"
+    env_differences=$(comm -13 <(echo "$INITIAL_ENV") <(printenv | sort))
+    if [ -n "$env_differences" ]; then
+      echo "  with the following additional environment variables:"
+      echo "$env_differences" | sed 's/^/    /'
+    fi
+  else
+    if ! "$@"; then
+      error "Failure running '$@', exiting."
+      exit 1
+    fi
+  fi
+}
+
+function redhat_is_installed() {
+  local package_name="$1"
+  rpm -qa $package_name | grep -q .
+}
+
+function debian_is_installed() {
+  local package_name="$1"
+  dpkg -l $package_name | grep ^ii | grep -q .
+}
+
+function version_sort() {
+  # We'd rather use sort -V, but that's not available on Centos 5.  This works
+  # for versions in the form A.B.C.D or shorter, which is enough for our use.
+  sort -t '.' -k 1,1 -k 2,2 -k 3,3 -k 4,4 -g
+}
+
+# Compare two numeric versions in the form "A.B.C".  Works with version numbers
+# having up to four components, since that's enough to handle both nginx (3) and
+# ngx_pagespeed (4).
+function version_older_than() {
+  local test_version="$1"
+  local compare_to="$2"
+
+  local older_version=$(echo $@ | tr ' ' '\n' | version_sort | head -n 1)
+  test "$older_version" != "$compare_to"
+}
+
+function determine_latest_nginx_version() {
+  # Scrape nginx's download page to try to find the most recent nginx version.
+
+  nginx_download_url="https://nginx.org/en/download.html"
+  function report_error() {
+    fail "
+Couldn't automatically determine the latest nginx version: failed to $@
+$nginx_download_url"
+  }
+
+  nginx_download_page=$(curl -sS --fail "$nginx_download_url") || \
+    report_error "download"
+
+  download_refs=$(echo "$nginx_download_page" | \
+    grep -o '/download/nginx-[0-9.]*[.]tar[.]gz') || \
+    report_error "parse"
+
+  versions_available=$(echo "$download_refs" | \
+    sed -e 's~^/download/nginx-~~' -e 's~\.tar\.gz$~~') || \
+    report_error "extract versions from"
+
+  latest_version=$(echo "$versions_available" | version_sort | tail -n 1) || \
+    report_error "determine latest version from"
+
+  if version_older_than "$latest_version" "1.11.4"; then
+    fail "
+Expected the latest version of nginx to be at least 1.11.4 but found
+$latest_version on $nginx_download_url"
+  fi
+
+  echo "$latest_version"
+}
+
+# Usage:
+#  install_dependencies install_pkg_cmd is_pkg_installed_cmd dep1 dep2 ...
+#
+# install_pkg_cmd is a command to install a dependency
+# is_pkg_installed_cmd is a command that returns true if the dependency is
+#   already installed
+# each dependency is a package name
+function install_dependencies() {
+  local install_pkg_cmd="$1"
+  local is_pkg_installed_cmd="$2"
+  shift 2
+
+  local missing_dependencies=""
+
+  for package_name in "$@"; do
+    if ! $is_pkg_installed_cmd $package_name; then
+      missing_dependencies+="$package_name "
+    fi
+  done
+  if [ -n "$missing_dependencies" ]; then
+    status "Detected that we're missing the following depencencies:"
+    echo "  $missing_dependencies"
+    status "Installing them:"
+    run sudo $install_pkg_cmd $missing_dependencies
+  fi
+}
+
+function gcc_too_old() {
+  # We need gcc >= 4.8
+  local gcc_major_version=$(gcc -dumpversion | awk -F. '{print $1}')
+  if [ "$gcc_major_version" -lt 4 ]; then
+    return 0  # too old
+  elif [ "$gcc_major_version" -gt 4 ]; then
+    return 1  # plenty new
+  fi
+  # It's gcc 4.x, check if x >= 8:
+  local gcc_minor_version=$(gcc -dumpversion | awk -F. '{print $2}')
+  test "$gcc_minor_version" -lt 8
+}
+
+function continue_or_exit() {
+  if "$ASSUME_YES"; then
+    return
+  fi
+
+  local prompt="$1"
+  echo_color "$YELLOW" -n "$prompt"
+  read -p " [Y/n] " yn
+  if [[ "$yn" == N* || "$yn" == n* ]]; then
+    echo "Cancelled."
+    exit 0
+  fi
+}
+
+# If a string is very simple we don't need to quote it.  But we should quote
+# everything else to be safe.
+function needs_quoting() {
+  echo "$@" | grep -q '[^a-zA-Z0-9./_=-]'
+}
+
+function escape_for_quotes() {
+  echo "$@" | sed -e 's~\\~\\\\~g' -e "s~'~\\\\'~g"
+}
+
+function quote_arguments() {
+  local argument_str=""
+  for argument in "$@"; do
+    if [ -n "$argument_str" ]; then
+      argument_str+=" "
+    fi
+    if needs_quoting "$argument"; then
+      argument="'$(escape_for_quotes "$argument")'"
+    fi
+    argument_str+="$argument"
+  done
+  echo "$argument_str"
+}
+
+function build_ngx_pagespeed() {
+  getopt --test
+  if [ "$?" != 4 ]; then
+    # Even Centos 5 and Ubuntu 10 LTS have new-style getopt, so I don't expect
+    # this to be hit in practice on systems that are actually able to run
+    # PageSpeed.
+    fail "Your version of getopt is too old.  Exiting with no changes made."
+  fi
+
+  opts=$(getopt -o v:n:mb:pslt:ya:dh \
+    --longoptions ngx-pagespeed-version:,nginx-version:,dynamic-module \
+    --longoptions buildir:,no-deps-check,psol-from-source,devel,build-type: \
+    --longoptions assume-yes,additional-nginx-configure-arguments:,dryrun,help \
+    -n "$(basename "$0")" -- "$@")
+  if [ $? != 0 ]; then
+    usage
+    exit 1
+  fi
+  eval set -- "$opts"
+
+  NPS_VERSION="DEFAULT"
+  NGINX_VERSION=""
+  BUILDDIR="$HOME"
+  DO_DEPS_CHECK=true
+  PSOL_FROM_SOURCE=false
+  DEVEL=false
+  BUILD_TYPE=""
+  ASSUME_YES=false
+  DRYRUN=false
+  DYNAMIC_MODULE=false
+  while true; do
+    case "$1" in
+      -v | --ngx-pagespeed-version) shift
+        NPS_VERSION="$1"
+        shift
+        ;;
+      -n | --nginx-version) shift
+        NGINX_VERSION="$1"
+        shift
+        ;;
+      -m | --dynamic-module) shift
+        DYNAMIC_MODULE=true
+        ;;
+      -b | --builddir) shift
+        BUILDDIR="$1"
+        shift
+        ;;
+      -p | --no-deps-check) shift
+        DO_DEPS_CHECK=false
+        ;;
+      -s | --psol-from-source) shift
+        PSOL_FROM_SOURCE=true
+        ;;
+      -l | --devel) shift
+        DEVEL=true
+        ;;
+      -t | --build-type) shift
+        BUILD_TYPE="$1"
+        shift
+        ;;
+      -y | --assume-yes) shift
+        ASSUME_YES="true"
+        ;;
+      -a | --additional-nginx-configure-arguments) shift
+        ADDITIONAL_NGINX_CONFIGURE_ARGUMENTS="$1"
+        shift
+        ;;
+      -d | --dryrun) shift
+        DRYRUN="true"
+        ;;
+      -h | --help) shift
+        usage
+        exit 0
+        ;;
+      --) shift
+        break
+        ;;
+      *)
+        echo "Invalid argument: $1"
+        usage
+        exit 1
+        ;;
+    esac
+  done
+
+  USE_GIT_CHECKOUT="$DEVEL"
+  ALREADY_CHECKED_OUT=false
+  if [ -e PSOL_BINARY_URL ]; then
+    status "Detected that we're running in an existing ngx_pagespeed checkout."
+    USE_GIT_CHECKOUT=true
+    ALREADY_CHECKED_OUT=true
+  fi
+
+  if "$ALREADY_CHECKED_OUT"; then
+    if [ "$NPS_VERSION" != "DEFAULT" ]; then
+      fail \
+"The --ngx-pagespeed-version argument doesn't make sense when running within an existing checkout."
+    fi
+  elif [ "$NPS_VERSION" = "DEFAULT" ]; then
+    if "$DEVEL"; then
+      NPS_VERSION="master"
+    else
+      NPS_VERSION="latest-stable"
+    fi
+  fi
+
+  if [ ! -d "$BUILDDIR" ]; then
+    fail "Told to build in $BUILDDIR, but that directory doesn't exist."
+  fi
+
+  BUILD_NGINX=false
+  if [ -n "$NGINX_VERSION" ]; then
+    BUILD_NGINX=true
+  fi
+
+  if "$DEVEL"; then
+    PSOL_FROM_SOURCE=true
+    BUILD_NGINX=true
+    if [ -n "$NGINX_VERSION" ]; then
+      fail \
+"The --devel argument conflicts with --nginx.  In devel mode we use the version of nginx that's included as a submodule."
+    fi
+    if "$DYNAMIC_MODULE"; then
+      fail "Can't currently build a dynamic module in --devel mode."
+    fi
+  fi
+
+  if "$PSOL_FROM_SOURCE" && [ -z "$BUILD_TYPE" ]; then
+    if "$DEVEL"; then
+      BUILD_TYPE="Debug"
+    else
+      BUILD_TYPE="Release"
+    fi
+  elif [ -n "$BUILD_TYPE" ]; then
+    fail "Setting --build-type requires --psol-from-source or --devel."
+  fi
+
+  if [ "$NGINX_VERSION" = "latest" ]; then
+    # When this function fails it prints the debugging information needed first
+    # to stderr.
+    NGINX_VERSION=$(determine_latest_nginx_version) || exit 1
+  fi
+
+  if "$DYNAMIC_MODULE"; then
+    # Check that ngx_pagespeed and nginx are recent enough to support dynamic
+    # modules.  Unfortunately NPS_VERSION might be a tag, in which case we don't
+    # know.  If it's not a numeric version number, then assume it's recent
+    # enough and if it's not they'll get an ugly compilation error later.
+    # Luckily 1.10.33.5 was a while ago now.
+    #
+    # I'd like to use =~ here, but they changed syntax between v3 and v4 (quotes
+    # moved from mandatory to optional to prohibited).
+    if [[ "${NPS_VERSION#*[^0-9.]}" = "$NPS_VERSION" ]] &&
+         version_older_than "$NPS_VERSION" "1.10.33.5"; then
+      fail "
+You're trying to build ngx_pagespeed $NPS_VERSION as a dynamic module, but
+ngx_pagespeed didn't add support for dynamic modules until 1.10.33.5."
+    fi
+
+    if [ ! -z "NGINX_VERSION" ]; then
+      if version_older_than "$NGINX_VERSION" "1.9.13"; then
+        fail "
+You're trying to build nginx $NGINX_VERSION as a dynamic module but nginx didn't
+add support for dynamic modules in a way compatible with ngx_pagespeed until
+1.9.13."
+      fi
+    fi
+  fi
+
+  if "$DRYRUN"; then
+    TEMPDIR="/tmp/output-of-mktemp"
+  else
+    TEMPDIR=$(mktemp -d)
+    function cleanup_tempdir {
+      rm -rf "$TEMPDIR"
+    }
+    trap cleanup_tempdir EXIT
+  fi
+
+  extra_flags=()
+  # Now make sure our dependencies are installed.
+  if "$DO_DEPS_CHECK"; then
+    if [ -f /etc/debian_version ]; then
+      status "Detected debian-based distro."
+
+      install_dependencies "apt-get install" debian_is_installed \
+        "build-essential zlib1g-dev libpcre3 libpcre3-dev unzip"
+
+      if gcc_too_old; then
+        if [ ! -e /usr/lib/gcc-mozilla/bin/gcc ]; then
+          status "Detected that gcc is older than 4.8.  Installing gcc-mozilla"
+          status "which installs gcc-4.8 into /usr/lib/gcc-mozilla/ and doesn't"
+          status "affect your global gcc installation."
+          run sudo apt-get install gcc-mozilla
+        fi
+
+        extra_flags=("--with-cc=/usr/lib/gcc-mozilla/bin/gcc" \
+                     "--with-ld-opt=-static-libstdc++")
+      fi
+
+    elif [ -f /etc/redhat-release ]; then
+      status "Detected redhat-based distro."
+
+      install_dependencies "yum install" redhat_is_installed \
+        "gcc-c++ pcre-devel zlib-devel make unzip wget"
+      if gcc_too_old; then
+        if [ ! -e /opt/rh/devtoolset-2/root/usr/bin/gcc ]; then
+          redhat_major_version=$(
+            cat /etc/redhat-release | grep -o -E '[0-9]+' | head -n 1)
+          if [ "$redhat_major_version" == 5 ]; then
+            slc_version=5
+          elif [ "$redhat_major_version" == 6 ]; then
+            slc_version=6
+          else
+            fail "
+Unexpected major version $redhat_major_version in /etc/redhat-release:
+$(cat /etc/redhat-release) Expected 5 or 6."
+          fi
+
+          status "Detected that gcc is older than 4.8.  Scientific Linux"
+          status "provides a gcc package that installs gcc-4.8 into /opt/ and"
+          status "doesn't affect your global gcc installation."
+          slc_key="https://linux.web.cern.ch/linux/scientific6/docs/repository/"
+          slc_key+="cern/slc6X/i386/RPM-GPG-KEY-cern"
+          slc_key_out="$TEMPDIR/RPM-GPG-KEY-cern"
+          run sudo wget "$slc_key" -O "$slc_key_out"
+          run sudo rpm --import "$slc_key_out"
+
+          repo_fname="/etc/yum.repos.d/slc${slc_version}-devtoolset.repo"
+          if [ -e "$repo_fname" ]; then
+            fail "Expected $repo_fname not to exist; aborting."
+          fi
+
+          repo_url="https://linux.web.cern.ch/linux/scientific${slc_version}/"
+          repo_url+="/docs/repository/cern/devtoolset/"
+          repo_url+="slc${slc_version}-devtoolset.repo"
+          run sudo wget -O "$repo_fname" "$repo_url"
+          run sudo yum install devtoolset-2-gcc-c++ devtoolset-2-binutils
+        fi
+        extra_flags=("--with-cc=/opt/rh/devtoolset-2/root/usr/bin/gcc")
+      fi
+    else
+      fail "
+This doesn't appear to be a deb-based distro or an rpm-based one.  Not going to
+be able to install dependencies.  Please install dependencies manually and rerun
+with --no-deps-check."
+    fi
+    status "Operating system dependencies are all set."
+  else
+    status "Not checking whether operating system dependencies are installed."
+  fi
+
+  function delete_if_already_exists() {
+    if "$DRYRUN"; then return; fi
+
+    local directory="$1"
+    if [ -d "$directory" ]; then
+      if [ ${#directory} -lt 8 ]; then
+        fail "
+Not deleting $directory; name is suspiciously short.  Something is wrong."
+      fi
+
+      continue_or_exit "OK to delete $directory?"
+      run rm -rf "$directory"
+    fi
+  }
+
+  # In general, the zip github builds for tag foo unzips to ngx_pagespeed-foo,
+  # but it looks like they special case vVERSION tags to ngx_pagespeed-VERSION
+  if [[ "$NPS_VERSION" =~ ^[0-9]*[.][0-9]*[.][0-9]*[.][0-9]*$ ]]; then
+    # We've been given a numeric version number.  This has an associated tag
+    # in the form vVERSION-beta.
+    tag_name="v${NPS_VERSION}-beta"
+    nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}-beta"
+  else
+    # We've been given a tag name, like latest-beta.  Download that directly.
+    tag_name="$NPS_VERSION"
+    nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}"
+  fi
+
+  install_dir="this-only-makes-sense-in-devel-mode"
+  if "$USE_GIT_CHECKOUT"; then
+    # We're either doing a --devel build, or someone is running us from an
+    # existing git checkout.
+    nps_module_dir="$PWD"
+    install_dir="$nps_module_dir"
+    if "$ALREADY_CHECKED_OUT"; then
+      run cd "$nps_module_dir"
+    else
+      status "Checking out ngx_pagespeed..."
+      run git clone "git@github.com:pagespeed/ngx_pagespeed.git" \
+                    "$nps_module_dir"
+      run cd "$nps_module_dir"
+      run git checkout "$tag_name"
+    fi
+    submodules_dir="$nps_module_dir/testing-dependencies"
+    if "$DEVEL"; then
+      status "Downloading dependencies..."
+      run git submodule update --init --recursive
+      if [[ "$CONTINUOUS_INTEGRATION" != true ]]; then
+        status "Switching submodules over to git protocol."
+        # This lets us push to github by public key.
+        for config in $(find .git/ -name config) ; do
+          run sed -i s~https://github.com/~git@github.com:~ $config ;
+        done
+      fi
+    fi
+  else
+    nps_baseurl="https://github.com/pagespeed/ngx_pagespeed/archive"
+    nps_downloaded="$TEMPDIR/$nps_downloaded_fname.zip"
+    status "Downloading ngx_pagespeed..."
+    run wget "$nps_baseurl/$tag_name.zip" -O "$nps_downloaded"
+    nps_module_dir="$BUILDDIR/$nps_downloaded_fname"
+    delete_if_already_exists "$nps_module_dir"
+    status "Extracting ngx_pagespeed..."
+    run unzip -q "$nps_downloaded" -d "$BUILDDIR"
+    run cd "$nps_module_dir"
+  fi
+
+  MOD_PAGESPEED_DIR=""
+  PSOL_BINARY=""
+  if "$PSOL_FROM_SOURCE"; then
+    MOD_PAGESPEED_DIR="$PWD/testing-dependencies/mod_pagespeed"
+    git submodule update --init --recursive -- "$MOD_PAGESPEED_DIR"
+    run pushd "$MOD_PAGESPEED_DIR"
+
+    if "$DEVEL"; then
+      if [ ! -d "$HOME/apache2" ]; then
+        run install/build_development_apache.sh 2.2 prefork
+      fi
+      cd devel
+      run make apache_debug_psol
+      PSOL_BINARY="$MOD_PAGESPEED_DIR/out/$BUILD_TYPE/pagespeed_automatic.a"
+    else
+      run install/build_psol.sh --skip_tests --skip_packaging
+      PSOL_BINARY="$MOD_PAGESPEED_DIR/pagespeed/automatic/pagespeed_automatic.a"
+    fi
+    run popd
+  else
+    # Now we need to figure out what precompiled version of PSOL to build
+    # ngx_pagespeed against.
+    if "$DRYRUN"; then
+      psol_url="https://psol.example.com/cant-get-psol-version-in-dry-run.tar.gz"
+    elif [ -e PSOL_BINARY_URL ]; then
+      # Releases after 1.11.33.4 there is a PSOL_BINARY_URL file that tells us
+      # where to look.
+      psol_url="$(scripts/format_binary_url.sh PSOL_BINARY_URL)"
+      if [[ "$psol_url" != https://* ]]; then
+        fail "Got bad psol binary location information: $psol_url"
+      fi
+    else
+      # For past releases we have to grep it from the config file.  The url has
+      # always looked like this, and the config file has contained it since
+      # before we started tagging our ngx_pagespeed releases.
+      psol_url="$(grep -o \
+          "https://dl.google.com/dl/page-speed/psol/[0-9.]*.tar.gz" config)"
+      if [ -z "$psol_url" ]; then
+        fail "Couldn't find PSOL url in $PWD/config"
+      fi
+    fi
+
+    status "Downloading PSOL binary..."
+    run wget "$psol_url"
+
+    status "Extracting PSOL..."
+    run tar -xzf $(basename "$psol_url")  # extracts to psol/
+  fi
+
+  if "$DYNAMIC_MODULE"; then
+    add_module="--add-dynamic-module=$nps_module_dir"
+  else
+    add_module="--add-module=$nps_module_dir"
+  fi
+  configure_args=("$add_module" "${extra_flags[@]}")
+
+  if "$DEVEL"; then
+    configure_args=("${configure_args[@]}"
+                    "--prefix=$install_dir/nginx"
+                    "--add-module=$submodules_dir/ngx_cache_purge"
+                    "--add-module=$submodules_dir/ngx_devel_kit"
+                    "--add-module=$submodules_dir/set-misc-nginx-module"
+                    "--add-module=$submodules_dir/headers-more-nginx-module"
+                    "--with-ipv6"
+                    "--with-http_v2_module")
+    if [ "$BUILD_TYPE" = "Debug" ]; then
+      configure_args=("${configure_args[@]}" "--with-debug")
+    fi
+  fi
+
+  echo
+  if ! "$BUILD_NGINX"; then
+    # Just prepare the module for them to install.
+    status "ngx_pagespeed is ready to be built against nginx."
+    echo "When running ./configure:"
+    if "$PSOL_FROM_SOURCE"; then
+      echo "  Set the following environment variables:"
+      echo "    MOD_PAGESPEED_DIR=$MOD_PAGESPEED_DIR"
+      echo "    PSOL_BINARY=$PSOL_BINARY"
+    fi
+    echo "  Give ./configure the following arguments:"
+    echo "    $(quote_arguments "${configure_args[@]}")"
+    echo
+    if [ ${#extra_flags[@]} -eq 0 ]; then
+      echo "If this is for integration with an already-built nginx, make sure"
+      echo "to include any other arguments you originally passed to"
+      echo "./configure.  You can see these with 'nginx -V'."
+    else
+      echo "Note: because we need to set $(quote_arguments "${extra_flags[@]}")"
+      echo "on this platform, if you want to integrate ngx_pagespeed with an"
+      echo "already-built nginx you're going to need to rebuild your nginx with"
+      echo "those flags set."
+    fi
+  else
+    if "$DEVEL"; then
+      # Use the nginx we loaded as a submodule
+      nginx_dir="$submodules_dir/nginx"
+      configure_location="auto"
+    else
+      # Download and build the specified nginx version.
+      nginx_leaf="nginx-${NGINX_VERSION}.tar.gz"
+      nginx_fname="$TEMPDIR/$nginx_leaf"
+      status "Downloading nginx..."
+      run wget "http://nginx.org/download/$nginx_leaf" -O "$nginx_fname"
+      nginx_dir="$BUILDDIR/nginx-${NGINX_VERSION}/"
+      delete_if_already_exists "$nginx_dir"
+      status "Extracting nginx..."
+      run tar -xzf "$nginx_fname" --directory "$BUILDDIR"
+      configure_location="."
+    fi
+    run cd "$nginx_dir"
+
+    configure=("$configure_location/configure" "${configure_args[@]}")
+    additional_configure_args=""
+    if [ -z "${ADDITIONAL_NGINX_CONFIGURE_ARGUMENTS+x}" ]; then
+      if ! "$ASSUME_YES"; then
+        echo "About to build nginx.  Do you have any additional ./configure"
+        echo "arguments you would like to set?  For example, if you would like"
+        echo "to build nginx with https support give --with-http_ssl_module"
+        echo "If you don't have any, just press enter."
+        read -p "> " additional_configure_args
+      fi
+    else
+      additional_configure_args="$ADDITIONAL_NGINX_CONFIGURE_ARGUMENTS"
+    fi
+
+    if [ -n "$additional_configure_args" ]; then
+      # Split additional_configure_args respecting any internal quotation.
+      # Otherwise things like --with-cc-opt='-foo -bar' won't work.
+      eval additional_configure_args=("$additional_configure_args")
+      configure=("${configure[@]}" "${additional_configure_args[@]}")
+    fi
+
+    echo "About to configure nginx with:"
+    echo "   $(quote_arguments "${configure[@]}")"
+    continue_or_exit "Does this look right?"
+    MOD_PAGESPEED_DIR="$MOD_PAGESPEED_DIR" \
+      PSOL_BINARY="$PSOL_BINARY" \
+      run "${configure[@]}"
+
+    if ! "$DEVEL"; then
+      continue_or_exit "Build nginx?"
+    fi
+    run make
+
+    if "$DEVEL"; then
+      run make install
+
+      status "Nginx installed with ngx_pagespeed, and set up for development."
+      echo "To run tests:"
+      echo "  cd $nps_module_dir"
+      echo "  test/run_tests.sh"
+      echo
+      echo "To rebuild after changes:"
+      echo "  scripts/rebuild.sh"
+    else
+      continue_or_exit "Install nginx?"
+      run sudo make install
+
+      echo
+      if "$DYNAMIC_MODULE"; then
+        echo "Nginx installed with ngx_pagespeed support available as a"
+        echo "loadable module."
+        echo
+        echo "To load the ngx_pagespeed module, you'll need to add:"
+        echo "  load_module \"modules/ngx_pagespeed.so\";"
+        echo "at the top of your main nginx configuration file."
+      else
+        echo "Nginx installed with ngx_pagespeed support compiled-in."
+      fi
+      echo
+      echo "If this is a new installation you probably need an init script to"
+      echo "manage starting and stopping the nginx service.  See:"
+      echo "  http://wiki.nginx.org/InitScripts"
+      echo
+      echo "You'll also need to configure ngx_pagespeed if you haven't yet:"
+      echo "  https://developers.google.com/speed/pagespeed/module/configuration"
+    fi
+  fi
+  if "$DRYRUN"; then
+    echo_color "$YELLOW" "[this was a dry run; your system is unchanged]"
+  fi
+}
+
+# Start running things from a call at the end so if this script is executed
+# after a partial download it doesn't do anything.
+build_ngx_pagespeed "$@"

scripts/format_binary_url.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -u
+
+if [ $# -ne 1 ]; then
+  echo "Usage: $(basename $0) <url_file>" >&2
+  exit 1
+fi
+
+url_file=$1
+
+if [ ! -e "$url_file" ]; then
+  echo "Url file '$url_file' missing!" >&2
+fi
+
+# The size names must match install/build_psol.sh in mod_pagespeed
+if [ "$(uname -m)" = x86_64 ]; then
+  bit_size_name=x64
+else
+  bit_size_name=ia32
+fi
+
+sed -e 's/$BIT_SIZE_NAME\b/'$bit_size_name'/g' $url_file

scripts/pagespeed_libraries_generator.sh

@@ -1,5 +1,19 @@
 #!/bin/bash
 #
+# Copyright 2013 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 # Converts pagespeed_libraries.conf from Apache-format to Nginx-format,
 # supporting the canonicalize_javascript_libraries filter.
 # Inspired by https://github.com/pagespeed/ngx_pagespeed/issues/532

scripts/prepare_psol.sh

@@ -1,94 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2012 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Author: jefftk@google.com (Jeff Kaufman)
-#
-# Usage:
-#   scripts/prepare_psol.sh /path/to/mod_pagespeed/src
-#
-# Creates a directory psol/ and copies headers and a few source files from a
-# depot_tools (glient) checkout into psol/include.  Along with creating
-# binaries, this is a step in preparing psol.tar.gz for distribution.
-#
-
-set -u  # check for undefined variables
-set -e  # exit on failed commands
-
-if [ "$(basename "$PWD")" != "ngx_pagespeed" ] ; then
-  echo "$(basename $0) must be invoked from the ngx_pagespeed directory"
-  exit 1
-fi
-
-if [ $# -ne 1 ] ; then
-  echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
-  exit 1
-fi
-
-MOD_PAGESPEED_SRC="$1"
-
-if [ "$(basename "$(dirname "$MOD_PAGESPEED_SRC")")/$( \
-        basename "$MOD_PAGESPEED_SRC")" != "mod_pagespeed/src" ] ; then
-  echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
-  exit 1
-fi
-
-if [ -e psol ] ; then
-  echo "A psol/ directory already exists.  Move it somewhere else and rerun."
-  exit 1
-fi
-mkdir psol/
-# Copy over the .h files, plus a few selected .cc and .c files.
-rsync -arvz "$MOD_PAGESPEED_SRC/" "psol/include/" --prune-empty-dirs \
-  --exclude=".svn" \
-  --exclude=".git" \
-  --include='*.h' \
-  --include='*/' \
-  --include="apr_thread_compatible_pool.cc" \
-  --include="serf_url_async_fetcher.cc" \
-  --include="apr_mem_cache.cc" \
-  --include="key_value_codec.cc" \
-  --include="apr_memcache2.c" \
-  --include="loopback_route_fetcher.cc" \
-  --include="add_headers_fetcher.cc" \
-  --include="console_css_out.cc" \
-  --include="console_out.cc" \
-  --include="dense_hash_map" \
-  --include="dense_hash_set" \
-  --include="sparse_hash_map" \
-  --include="sparse_hash_set" \
-  --include="sparsetable" \
-  --include="mod_pagespeed_console_out.cc" \
-  --include="mod_pagespeed_console_css_out.cc" \
-  --include="mod_pagespeed_console_html_out.cc" \
-  --exclude='*'
-mkdir -p psol/lib/Debug/linux/ia32
-mkdir -p psol/lib/Debug/linux/x64
-mkdir -p psol/lib/Release/linux/ia32
-mkdir -p psol/lib/Release/linux/x64
-
-# Log that we did this.
-SVN_REVISION="$(svn info $MOD_PAGESPEED_SRC | grep Revision | awk '{print $2}')"
-SVN_TAG="$(svn info $MOD_PAGESPEED_SRC | grep URL |  awk -F/ '{print $(NF-1)}')"
-
-DATE="$(date +%F)"
-echo "${DATE}: Copied from mod_pagespeed ${SVN_TAG}@r${SVN_REVISION} ($USER)" \
-  >> psol/include_history.txt
-
-echo
-echo "Output is in psol/include.  Now put binaries in psol/lib following"
-echo "https://github.com/pagespeed/ngx_pagespeed/wiki/Building-Release-Binaries"
-echo "and then you can distribute PSOL."
-

scripts/rebuild.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Usage: scripts/rebuild.sh
+#
+# After building with "scripts/build_ngx_pagespeed.sh --devel", if you make
+# changes to ngx_pagespeed you'll need to rebuild it.  The underlying commands
+# aren't complicated, but it's faster to work if it's automated.
+
+set -e  # exit script if any command returns an error
+set -u  # exit the script if any variable is uninitialized
+
+this_dir="$(dirname $0)"
+cd "$this_dir/.."
+nps_dir="$PWD"
+
+cd "$nps_dir/testing-dependencies/mod_pagespeed/devel"
+make apache_debug_psol
+
+cd "$nps_dir/testing-dependencies/nginx/"
+make
+make install

src/ngx_base_fetch.cc

@@ -12,17 +12,20 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
+ * Author: jefftk@google.com (Jeff Kaufman)
  */
 
-// Author: jefftk@google.com (Jeff Kaufman)
+#include "ngx_pagespeed.h"  // Must come first, see comments in CollectHeaders.
+
 #include <unistd.h> //for usleep
 
 #include "ngx_base_fetch.h"
 #include "ngx_event_connection.h"
 #include "ngx_list_iterator.h"
 
-#include "ngx_pagespeed.h"
-
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
 #include "net/instaweb/rewriter/public/rewrite_stats.h"
 #include "pagespeed/kernel/base/google_message_handler.h"
 #include "pagespeed/kernel/base/message_handler.h"
@@ -38,14 +41,19 @@ const char kDone = 'D';
 NgxEventConnection* NgxBaseFetch::event_connection = NULL;
 int NgxBaseFetch::active_base_fetches = 0;
 
-NgxBaseFetch::NgxBaseFetch(ngx_http_request_t* r,
+NgxBaseFetch::NgxBaseFetch(StringPiece url,
+                           ngx_http_request_t* r,
                            NgxServerContext* server_context,
                            const RequestContextPtr& request_ctx,
                            PreserveCachingHeaders preserve_caching_headers,
-                           NgxBaseFetchType base_fetch_type)
+                           NgxBaseFetchType base_fetch_type,
+                           const RewriteOptions* options)
     : AsyncFetch(request_ctx),
+      url_(url.data(), url.size()),
       request_(r),
       server_context_(server_context),
+      options_(options),
+      need_flush_(false),
       done_called_(false),
       last_buf_sent_(false),
       references_(2),
@@ -220,8 +228,11 @@ ngx_int_t NgxBaseFetch::CopyBufferToNginx(ngx_chain_t** link_ptr) {
     return NGX_AGAIN;
   }
 
-  int rc = string_piece_to_buffer_chain(
-      request_->pool, buffer_, link_ptr, done_called_ /* send_last_buf */);
+  int rc = string_piece_to_buffer_chain(request_->pool, buffer_, link_ptr,
+                                        done_called_ /* send_last_buf */,
+                                        need_flush_);
+  need_flush_ = false;
+
   if (rc != NGX_OK) {
     return rc;
   }
@@ -249,6 +260,13 @@ ngx_int_t NgxBaseFetch::CollectAccumulatedWrites(ngx_chain_t** link_ptr) {
 }
 
 ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
+  // nginx defines _FILE_OFFSET_BITS to 64, which changes the size of off_t.
+  // If a standard header is accidentally included before the nginx header,
+  // on a 32-bit system off_t will be 4 bytes and we don't assign all the
+  // bits of content_length_n. Sanity check that did not happen.
+  // This could use static_assert, but this file is not built with --std=c++11.
+  bool sanity_check_off_t[sizeof(off_t) == 8 ? 1 : -1] __attribute__ ((unused));
+
   const ResponseHeaders* pagespeed_headers = response_headers();
 
   if (content_length_known()) {
@@ -299,6 +317,9 @@ void NgxBaseFetch::HandleHeadersComplete() {
 }
 
 bool NgxBaseFetch::HandleFlush(MessageHandler* handler) {
+  Lock();
+  need_flush_ = true;
+  Unlock();
   RequestCollection(kFlush);  // A new part of the response body is available
   return true;
 }
@@ -331,4 +352,9 @@ void NgxBaseFetch::HandleDone(bool success) {
   DecrefAndDeleteIfUnreferenced();
 }
 
+bool NgxBaseFetch::IsCachedResultValid(const ResponseHeaders& headers) {
+  return OptionsAwareHTTPCacheCallback::IsCacheValid(
+      url_, *options_, request_context(), headers);
+}
+
 }  // namespace net_instaweb

src/ngx_base_fetch.h

@@ -43,6 +43,9 @@
 //    events it handles.
 //
 // When the last reference is dropped, this class will delete itself.
+//
+// TODO(jmarantz): consider sharing the cache-invalidation infrastructure
+// with ApacheFetch, using a common base class.
 
 #ifndef NGX_BASE_FETCH_H_
 #define NGX_BASE_FETCH_H_
@@ -59,6 +62,7 @@ extern "C" {
 #include "ngx_server_context.h"
 
 #include "net/instaweb/http/public/async_fetch.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
 #include "pagespeed/kernel/base/string.h"
 #include "pagespeed/kernel/http/headers.h"
 
@@ -74,10 +78,12 @@ enum NgxBaseFetchType {
 
 class NgxBaseFetch : public AsyncFetch {
  public:
-  NgxBaseFetch(ngx_http_request_t* r, NgxServerContext* server_context,
+  NgxBaseFetch(StringPiece url, ngx_http_request_t* r,
+               NgxServerContext* server_context,
                const RequestContextPtr& request_ctx,
                PreserveCachingHeaders preserve_caching_headers,
-               NgxBaseFetchType base_fetch_type);
+               NgxBaseFetchType base_fetch_type,
+               const RewriteOptions* options);
   virtual ~NgxBaseFetch();
 
   // Statically initializes event_connection, require for PSOL and nginx to
@@ -125,6 +131,8 @@ class NgxBaseFetch : public AsyncFetch {
   ngx_http_request_t* request() { return request_; }
   NgxBaseFetchType base_fetch_type() { return base_fetch_type_; }
 
+  bool IsCachedResultValid(const ResponseHeaders& headers) override;
+
  private:
   virtual bool HandleWrite(const StringPiece& sp, MessageHandler* handler);
   virtual bool HandleFlush(MessageHandler* handler);
@@ -152,13 +160,16 @@ class NgxBaseFetch : public AsyncFetch {
   int DecrefAndDeleteIfUnreferenced();
 
   static NgxEventConnection* event_connection;
-  
+
   // Live count of NgxBaseFetch instances that are currently in use.
   static int active_base_fetches;
 
+  GoogleString url_;
   ngx_http_request_t* request_;
   GoogleString buffer_;
   NgxServerContext* server_context_;
+  const RewriteOptions* options_;
+  bool need_flush_;
   bool done_called_;
   bool last_buf_sent_;
   // How many active references there are to this fetch. Starts at two,

src/ngx_fetch.cc

@@ -140,6 +140,8 @@ NgxConnection* NgxConnection::Connect(ngx_peer_connection_t* pc,
 
   int rc = ngx_event_connect_peer(pc);
   if (rc == NGX_ERROR || rc == NGX_DECLINED || rc == NGX_BUSY) {
+    ngx_log_error(NGX_LOG_DEBUG, pc->log, 0,
+                    "NgxFetch: ngx_event_connect_peer failed");
     return NULL;
   }
 
@@ -363,14 +365,17 @@ bool NgxFetch::Init() {
 
   GoogleString s_ipaddress(reinterpret_cast<char*>(tmp_url->host.data),
                            tmp_url->host.len);
-  ngx_memzero(&sin_, sizeof(sin_));
-  sin_.sin_family = AF_INET;
-  sin_.sin_port = htons(tmp_url->port);
-  sin_.sin_addr.s_addr = inet_addr(s_ipaddress.c_str());
 
-  if (sin_.sin_addr.s_addr == INADDR_NONE) {
-    // inet_addr returned INADDR_NONE, which means the hostname
-    // isn't a valid IP address.  Check DNS.
+  ngx_memzero(&sin_, sizeof(sin_));
+  if (s_ipaddress[0] == '[') {
+      s_ipaddress = s_ipaddress.substr(1);
+  }
+  if (s_ipaddress[s_ipaddress.size() - 1] == ']') {
+      s_ipaddress = s_ipaddress.substr(0, s_ipaddress.size()-2);      
+  }
+  ngx_int_t res = ngx_parse_addr(pool_, &sin_, (u_char*)s_ipaddress.c_str(), s_ipaddress.size());
+  if (res != NGX_OK) {
+    // Hostname isn't a valid IP address.  Check DNS.
     ngx_resolver_ctx_t temp;
     temp.name.data = tmp_url->host.data;
     temp.name.len = tmp_url->host.len;
@@ -405,6 +410,16 @@ bool NgxFetch::Init() {
       return false;
     }
   } else {
+      /*
+    u_char text[NGX_SOCKADDR_STRLEN];
+    ngx_str_t addrtext;
+    addrtext.len = ngx_sock_ntop(fetch->sin_.sockaddr, fetch->sin_.socklen, 
+        text, NGX_SOCKADDR_STRLEN, htons(fetcher->proxy_.url.len ? htons(fetcher->proxy_.port) : fetch->url_.port));
+    ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                    "NgxFetch %p: Resolved host [%V] to [%V]", fetch,
+                    &resolver_ctx->name, &addrtext);
+    */
+    ((struct sockaddr_in *)sin_.sockaddr)->sin_port = htons(fetcher_->proxy_.url.len ? htons(fetcher_->proxy_.port) : url_.port);
     if (InitRequest() != NGX_OK) {
       message_handler()->Message(kError, "NgxFetch: InitRequest failed");
       return false;
@@ -535,14 +550,14 @@ void NgxFetch::ResolveDoneHandler(ngx_resolver_ctx_t* resolver_ctx) {
   }
 
   ngx_uint_t i;
-  // Find the first ipv4 address. We don't support ipv6 yet.
+  // Find the first ipv4 address. We don't support ipv6 yet in dns results.
   for (i = 0; i < resolver_ctx->naddrs; i++) {
     // Old versions of nginx and tengine have a different definition of addrs,
     // work around to make sure we are using the right type (ngx_addr_t*).
     ngx_addr_t* ngx_addrs = reinterpret_cast<ngx_addr_t*>(resolver_ctx->addrs);
     if (typeid(*ngx_addrs) == typeid(*resolver_ctx->addrs)) {
       if (reinterpret_cast<struct sockaddr_in*>(ngx_addrs[i].sockaddr)
-              ->sin_family == AF_INET) {
+             ->sin_family == AF_INET) {
         break;
       }
     } else {
@@ -566,37 +581,30 @@ void NgxFetch::ResolveDoneHandler(ngx_resolver_ctx_t* resolver_ctx) {
   ngx_memzero(&fetch->sin_, sizeof(fetch->sin_));
 
 #if (nginx_version < 1005008)
+  // TODO(oschaaf): need to test this again, might be broken.
   fetch->sin_.sin_addr.s_addr = resolver_ctx->addrs[i];
 #else
-  struct sockaddr_in* sin;
-
-  sin = reinterpret_cast<struct sockaddr_in*>(
-      resolver_ctx->addrs[i].sockaddr);
-
-  fetch->sin_.sin_family = sin->sin_family;
-  fetch->sin_.sin_addr.s_addr = sin->sin_addr.s_addr;
+  fetch->sin_ = resolver_ctx->addrs[i];
 #endif
 
-  fetch->sin_.sin_family = AF_INET;
-  fetch->sin_.sin_port = htons(fetch->url_.port);
-
   // Maybe we have Proxy
-  if (0 != fetcher->proxy_.url.len) {
-    fetch->sin_.sin_port = htons(fetcher->proxy_.port);
-  }
-
-  char* ip_address = inet_ntoa(fetch->sin_.sin_addr);
+  ((struct sockaddr_in *)fetch->sin_.sockaddr)->sin_port = 
+    htons(fetcher->proxy_.url.len ? htons(fetcher->proxy_.port) : fetch->url_.port);
 
+  u_char text[NGX_SOCKADDR_STRLEN];
+  ngx_str_t addrtext;
+  addrtext.len = ngx_sock_ntop(fetch->sin_.sockaddr, fetch->sin_.socklen, 
+    text, NGX_SOCKADDR_STRLEN, htons(fetcher->proxy_.url.len ? htons(fetcher->proxy_.port) : fetch->url_.port));
+  addrtext.data = text;
   ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
-                "NgxFetch %p: Resolved host [%V] to [%s]", fetch,
-                &resolver_ctx->name, ip_address);
-
-  fetch->release_resolver();
+                "NgxFetch %p: Resolved host [%V] to [%V]", fetch,
+                &resolver_ctx->name, &addrtext);
 
   if (fetch->InitRequest() != NGX_OK) {
     fetch->message_handler()->Message(kError, "NgxFetch: InitRequest failed");
     fetch->CallbackDone(false);
   }
+  fetch->release_resolver();
 }
 
 // Prepare the request data for this fetch, and hook the write event.
@@ -692,8 +700,8 @@ int NgxFetch::InitRequest() {
 int NgxFetch::Connect() {
   ngx_peer_connection_t pc;
   ngx_memzero(&pc, sizeof(pc));
-  pc.sockaddr = (struct sockaddr*)&sin_;
-  pc.socklen = sizeof(struct sockaddr_in);
+  pc.sockaddr =  sin_.sockaddr;
+  pc.socklen = sin_.socklen;
   pc.name = &url_.host;
 
   // get callback is dummy function, it just returns NGX_OK
@@ -940,8 +948,8 @@ void NgxFetch::FixUserAgent() {
     user_agent += "NgxNativeFetcher";
   }
   GoogleString version = StrCat(
-      " ", kModPagespeedSubrequestUserAgent,
-      "/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING);
+      " (", kModPagespeedSubrequestUserAgent,
+      "/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING ")");
   if (!StringPiece(user_agent).ends_with(version)) {
     user_agent += version;
   }

src/ngx_fetch.h

@@ -191,7 +191,8 @@ class NgxFetch : public PoolElement<NgxFetch> {
   int64 content_length_;
   bool content_length_known_;
 
-  struct sockaddr_in sin_;
+
+  ngx_addr_t sin_;
   ngx_log_t* log_;
   ngx_buf_t* out_;
   ngx_buf_t* in_;

src/ngx_message_handler.cc

@@ -97,7 +97,6 @@ void NgxMessageHandler::MessageSImpl(MessageType type,
   } else {
     GoogleMessageHandler::MessageSImpl(type, message);
   }
-  // Prepare a log message for the SharedCircularBuffer only.
   AddMessageToBuffer(type, message);
 }
 
@@ -111,6 +110,7 @@ void NgxMessageHandler::FileMessageSImpl(
   } else {
     GoogleMessageHandler::FileMessageSImpl(type, file, line, message);
   }
+  AddMessageToBuffer(type, file, line, message);
 }
 
 }  // namespace net_instaweb

src/ngx_pagespeed.cc

@@ -124,7 +124,7 @@ char* string_piece_to_pool_string(ngx_pool_t* pool, StringPiece sp) {
 // (potentially) longer string to nginx and want it to take ownership.
 ngx_int_t string_piece_to_buffer_chain(
     ngx_pool_t* pool, StringPiece sp, ngx_chain_t** link_ptr,
-    bool send_last_buf) {
+    bool send_last_buf, bool send_flush) {
   // Below, *link_ptr will be NULL if we're starting the chain, and the head
   // chain link.
   *link_ptr = NULL;
@@ -198,6 +198,9 @@ ngx_int_t string_piece_to_buffer_chain(
 
 
   CHECK(tail_link != NULL);
+  if (send_flush) {
+    tail_link->buf->flush = true;
+  }
   if (send_last_buf) {
     tail_link->buf->last_buf = true;
   }
@@ -236,6 +239,9 @@ ngx_http_output_body_filter_pt ngx_http_next_body_filter;
 ngx_int_t ps_base_fetch_filter(ngx_http_request_t* r, ngx_chain_t* in) {
   ps_request_ctx_t* ctx = ps_get_request_context(r);
 
+  if (r->header_only) {
+    return NGX_OK;
+  }
   if (ctx == NULL || ctx->base_fetch == NULL) {
     return ngx_http_next_body_filter(r, in);
   }
@@ -285,7 +291,8 @@ ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r) {
     if (!status_ok && (ctx->base_fetch->base_fetch_type() != kHtmlTransform
                        && ctx->base_fetch->base_fetch_type() != kIproLookup)) {
       ps_release_base_fetch(ctx);
-      return ngx_http_filter_finalize_request(r, NULL, status_code);
+      ngx_http_filter_finalize_request(r, NULL, status_code);
+      return NGX_DONE;
     }
 
     if (ctx->preserve_caching_headers != kDontPreserveHeaders) {
@@ -422,6 +429,32 @@ ngx_int_t ps_set_cache_control(ngx_http_request_t* r, char* cache_control) {
   return NGX_OK;
 }
 
+// Returns false if the header wasn't found.  Otherwise sets cache_control and
+// returns true;
+bool ps_get_cache_control(ngx_http_request_t* r, GoogleString* cache_control) {
+  // Use headers_out.cache_control instead of looking for Cache-Control in
+  // headers_out.headers, because if an upstream sent multiple Cache-Control
+  // headers they're already combined in headers_out.cache_control.
+  auto ccp = static_cast<ngx_table_elt_t**>(r->headers_out.cache_control.elts);
+  if (ccp == nullptr) {
+    return false;  // Header not present.
+  }
+  bool first_segment = true;
+  for (ngx_uint_t i = 0; i < r->headers_out.cache_control.nelts; i++) {
+    if (ccp[i]->hash == 0) {
+      continue;  // Elements with a hash of 0 are marked as excluded.
+    }
+    if (first_segment) {
+      first_segment = false;
+    } else {
+      cache_control->append(", ");
+    }
+    cache_control->append(reinterpret_cast<char*>(ccp[i]->value.data),
+                          ccp[i]->value.len);
+  }
+  return true;
+}
+
 template<class Headers>
 void copy_headers_from_table(const ngx_list_t &from, Headers* to) {
   // Standard nginx idiom for iterating over a list.  See ngx_list.h
@@ -466,7 +499,8 @@ void copy_response_headers_from_ngx(const ngx_http_request_t* r,
 
   // When we don't have a date header, set one with the current time.
   if (headers->Lookup1(HttpAttributes::kDate) == NULL) {
-    headers->SetDate(ngx_current_msec);
+    PosixTimer timer;
+    headers->SetDate(timer.NowMs());
   }
 
   // TODO(oschaaf): ComputeCaching should be called in setupforhtml()?
@@ -682,6 +716,13 @@ char* ps_main_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 char* ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 
+// We want NGX_CONF_MULTI for some very old versions:
+//   https://github.com/pagespeed/ngx_pagespeed/commit/66f1b9aa
+// but it's gone in recent revisions, so provide a compat #define if needed
+#ifndef NGX_CONF_MULTI
+#define NGX_CONF_MULTI 0
+#endif
+
 // TODO(jud): Verify that all the offsets should be NGX_HTTP_SRV_CONF_OFFSET and
 // not NGX_HTTP_LOC_CONF_OFFSET or NGX_HTTP_MAIN_CONF_OFFSET.
 ngx_command_t ps_commands[] = {
@@ -711,6 +752,11 @@ ngx_command_t ps_commands[] = {
   ngx_null_command
 };
 
+bool ps_disabled(ps_srv_conf_t* cfg_s) {
+  return cfg_s->server_context == NULL ||
+         cfg_s->server_context->config()->unplugged();
+}
+
 void ps_ignore_sigpipe() {
   struct sigaction act;
   ngx_memzero(&act, sizeof(act));
@@ -843,18 +889,17 @@ char* ps_configure(ngx_conf_t* cf,
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cf->cycle, ngx_pagespeed));
   if (*options == NULL) {
-    *options = new NgxRewriteOptions(
-        cfg_m->driver_factory->thread_system());
+    *options = new NgxRewriteOptions(cfg_m->driver_factory->thread_system());
   }
 
-  bool process_script_variables = dynamic_cast<NgxRewriteDriverFactory*>(
-      cfg_m->driver_factory)->process_script_variables();
-
-  if (process_script_variables) {
+  ProcessScriptVariablesMode script_mode =
+      dynamic_cast<NgxRewriteDriverFactory*>(cfg_m->driver_factory)
+          ->process_script_variables();
+  if (script_mode != ProcessScriptVariablesMode::kOff) {
     // To be able to use '$', we map '$ps_dollar' to '$' via a script variable.
     ngx_str_t name = ngx_string("ps_dollar");
-    ngx_http_variable_t* var = ngx_http_add_variable(
-        cf, &name, NGX_HTTP_VAR_CHANGEABLE);
+    ngx_http_variable_t* var =
+        ngx_http_add_variable(cf, &name, NGX_HTTP_VAR_CHANGEABLE);
 
     if (var == NULL) {
       return const_cast<char*>(
@@ -865,7 +910,7 @@ char* ps_configure(ngx_conf_t* cf,
 
   const char* status = (*options)->ParseAndSetOptions(
       args, n_args, cf->pool, handler, cfg_m->driver_factory, option_scope, cf,
-      process_script_variables);
+      script_mode);
 
   // nginx expects us to return a string literal but doesn't mark it const.
   return const_cast<char*>(status);
@@ -1086,9 +1131,20 @@ char* ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child) {
 
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
-  cfg_m->driver_factory->set_main_conf(parent_cfg_s->options);
+  cfg_m->driver_factory->SetMainConf(parent_cfg_s->options);
   cfg_s->server_context = cfg_m->driver_factory->MakeNgxServerContext(
       "dummy_hostname", dummy_port);
+
+#if (NGX_HTTP_V2)
+  // Save the variable index of the "http2" variable, so we can use it
+  // at request time to lookup whether that's on. We do this conditionally
+  // since NGINX will complain to the user (at [emerg] level!) if it doesn't
+  // know about it.
+  ngx_str_t name = ngx_string("http2");
+  cfg_s->server_context->set_ngx_http2_variable_index(
+      ngx_http_get_variable_index(cf, &name));
+#endif
+
   // The server context sets some options when we call global_options(). So
   // let it do that, then merge in options we got from the config file.
   // Once we do that we're done with cfg_s->options.
@@ -1099,13 +1155,17 @@ char* ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child) {
   delete cfg_s->options;
   cfg_s->options = NULL;
 
-  if (cfg_s->server_context->global_options()->enabled()) {
+  if (!cfg_s->server_context->global_options()->unplugged()) {
     // Validate FileCachePath
     GoogleMessageHandler handler;
     const char* file_cache_path =
         cfg_s->server_context->config()->file_cache_path().c_str();
     if (file_cache_path[0] == '\0') {
-      return const_cast<char*>("FileCachePath must be set");
+      if (!cfg_s->server_context->global_options()->standby()) {
+        return const_cast<char*>("FileCachePath must be set, even for standby");
+      } else {
+        return const_cast<char*>("FileCachePath must be set");
+      }
     } else if (!cfg_m->driver_factory->file_system()->IsDir(
         file_cache_path, &handler).is_true()) {
       return const_cast<char*>(
@@ -1142,7 +1202,7 @@ char* ps_merge_loc_conf(ngx_conf_t* cf, void* parent, void* child) {
   ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
       ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
 
-  if (cfg_s->server_context == NULL) {
+  if (ps_disabled(cfg_s)) {
     // Pagespeed options cannot be defined only in location blocks.  There must
     // be at least a single "pagespeed off" in the main block or a server
     // block.
@@ -1284,6 +1344,9 @@ ngx_int_t ps_decline_request(ngx_http_request_t* r) {
 
   r->count++;
   r->phase_handler++;
+
+  //restore read_event_handler to what it was in ps_async_wait_response
+  r->read_event_handler = ngx_http_block_reading;
   r->write_event_handler = ngx_http_core_run_phases;
   ngx_http_core_run_phases(r);
   ngx_http_run_posted_requests(r->connection);
@@ -1295,6 +1358,13 @@ ngx_int_t ps_async_wait_response(ngx_http_request_t* r) {
   CHECK(ctx != NULL);
 
   r->count++;
+  // While we wait for PSOL to complete an async operation, there is a chance
+  // that the underlying connection gets closed,  or a http/2 RST_STREAM is
+  // received before the async operation completes. In that case we don't want
+  // to continue processing this flow. So we override the requests's read event
+  // handler with one that will make nginx abort request processing and execute
+  // our cleanup handlers instead of resuming request processing.
+  r->read_event_handler = ngx_http_test_reading;
   r->write_event_handler = ngx_http_request_empty_handler;
   ps_set_buffered(r, true);
   // We don't need to add a timer here, as it will be set by nginx.
@@ -1581,10 +1651,12 @@ void ps_release_base_fetch(ps_request_ctx_t* ctx) {
 }
 
 // TODO(chaizhenhua): merge into NgxBaseFetch ctor
-void ps_create_base_fetch(ps_request_ctx_t* ctx,
-                               RequestContextPtr request_context,
-                               RequestHeaders* request_headers,
-                               NgxBaseFetchType type) {
+void ps_create_base_fetch(StringPiece url,
+                          ps_request_ctx_t* ctx,
+                          RequestContextPtr request_context,
+                          RequestHeaders* request_headers,
+                          NgxBaseFetchType type,
+                          const RewriteOptions* options) {
   CHECK(ctx->base_fetch == NULL) << "Pre-existing base fetch!";
 
   ngx_http_request_t* r = ctx->r;
@@ -1594,9 +1666,9 @@ void ps_create_base_fetch(ps_request_ctx_t* ctx,
   // it, and call Done() on the associated parent (Proxy or Resource) fetch. If
   // we fail before creating the associated fetch then we need to call Done() on
   // the BaseFetch ourselves.
-  ctx->base_fetch = new NgxBaseFetch(r, cfg_s->server_context,
-                                     request_context,
-                                     ctx->preserve_caching_headers, type);
+  ctx->base_fetch = new NgxBaseFetch(url, r, cfg_s->server_context, request_context,
+                                     ctx->preserve_caching_headers, type,
+                                     options);
   ctx->base_fetch->SetRequestHeadersTakingOwnership(request_headers);
 }
 
@@ -1637,7 +1709,7 @@ void ps_release_request_context(void* data) {
 RequestRouting::Response ps_route_request(ngx_http_request_t* r) {
   ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
 
-  if (!cfg_s->server_context->global_options()->enabled()) {
+  if (ps_disabled(cfg_s)) {
     // Not enabled for this server block.
     return RequestRouting::kPagespeedDisabled;
   }
@@ -1779,11 +1851,6 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     options = cfg_s->server_context->global_options();
   }
 
-  if (!options->enabled()) {
-    // Disabled via query params or request headers.
-    return NGX_DECLINED;
-  }
-
   request_context->set_options(options->ComputeHttpOptions());
 
   // ps_determine_options modified url, removing any ModPagespeedFoo=Bar query
@@ -1810,6 +1877,14 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
       response_category == RequestRouting::kGlobalAdmin ||
       response_category == RequestRouting::kCachePurge;
 
+  // Normally if we're disabled we won't handle any requests, but if we're in
+  // standby mode we do want to handle requests for .pagespeed. resources.
+  if (options->unplugged() ||
+      (!options->enabled() && !pagespeed_resource)) {
+    // Disabled via query params or request headers.
+    return NGX_DECLINED;
+  }
+
   if (!html_rewrite) {
     // create request ctx
     CHECK(ctx == NULL);
@@ -1818,6 +1893,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     ctx->r = r;
     ctx->html_rewrite = false;
     ctx->in_place = false;
+    ctx->follow_flushes = options->follow_flushes();
     ctx->preserve_caching_headers = kDontPreserveHeaders;
 
     // See build_context_for_request() in mod_instaweb.cc
@@ -1861,16 +1937,17 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
   if (pagespeed_resource) {
     // TODO(jefftk): Set using_spdy appropriately.  See
     // ProxyInterface::ProxyRequestCallback
-    ps_create_base_fetch(ctx, request_context, request_headers.release(),
-                         kPageSpeedResource);
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
+                         request_headers.release(), kPageSpeedResource,
+                         options);
     ResourceFetch::Start(
         url,
         custom_options.release() /* null if there aren't custom options */,
-        false /* using_spdy */, cfg_s->server_context, ctx->base_fetch);
+        cfg_s->server_context, ctx->base_fetch);
     return ps_async_wait_response(r);
   } else if (is_an_admin_handler) {
-    ps_create_base_fetch(ctx, request_context, request_headers.release(),
-                         kAdminPage);
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
+                         request_headers.release(), kAdminPage, options);
     QueryParams query_params;
     query_params.ParseFromUrl(url);
 
@@ -1918,8 +1995,8 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
 
     if (options->domain_lawyer()->MapOriginUrl(
             url, &mapped_url, &host_header, &is_proxy) && is_proxy) {
-      ps_create_base_fetch(ctx, request_context, request_headers.release(),
-                          kPageSpeedProxy);
+      ps_create_base_fetch(url.Spec(), ctx, request_context,
+                           request_headers.release(), kPageSpeedProxy, options);
 
       RewriteDriver* driver;
       if (custom_options.get() == NULL) {
@@ -1942,9 +2019,9 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
 
   }
 
-  if (html_rewrite) {
-    ps_create_base_fetch(ctx, request_context, request_headers.release(),
-                         kHtmlTransform);
+  if (html_rewrite && options->IsAllowed(url.Spec())) {
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
+                         request_headers.release(), kHtmlTransform, options);
     // Do not store driver in request_context, it's not safe.
     RewriteDriver* driver;
 
@@ -1972,8 +2049,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
             url,
             cfg_s->server_context,
             options,
-            ctx->base_fetch,
-            false /* requires_blink_cohort (no longer unused) */);
+            ctx->base_fetch);
 
     // Will call StartParse etc.  The rewrite driver will take care of deleting
     // itself if necessary.
@@ -1988,8 +2064,8 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
   if (options->in_place_rewriting_enabled() &&
       options->enabled() &&
       options->IsAllowed(url.Spec())) {
-    ps_create_base_fetch(ctx, request_context, request_headers.release(),
-                         kIproLookup);
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
+                         request_headers.release(), kIproLookup, options);
 
     // Do not store driver in request_context, it's not safe.
     RewriteDriver* driver;
@@ -2038,7 +2114,6 @@ void ps_send_to_pagespeed(ngx_http_request_t* r,
   int last_buf = 0;
   for (cur = in; cur != NULL; cur = cur->next) {
     last_buf = cur->buf->last_buf;
-
     // Buffers are not really the last buffer until they've been through
     // pagespeed.
     cur->buf->last_buf = 0;
@@ -2063,6 +2138,19 @@ void ps_send_to_pagespeed(ngx_http_request_t* r,
         }
       }
     }
+    if (cur->buf->flush && ctx->follow_flushes) {
+      // Calling ctx->proxy_fetch->Flush(cfg_s->handler) will be a no-op here,
+      // unless we have follow_flushes or flush_html enabled. Note that PSOL
+      // might aggregate multiple flushes into 1, and actually flush a little bit
+      // later due to html parser state and earlier scheduled operations.
+      // Also, unless we also set the flush flag on the nginx buffers we won't
+      // actually flush.
+      // Note that too many flushes could harm optimization over larger html
+      // fragments as PSOL gets less context to work with, e.g. it can't combine
+      // two css files if a flush happens in between.
+      ctx->proxy_fetch->Flush(cfg_s->handler);
+    }
+
     // We're done with buffers as we pass them through, so mark them as sent as
     // we go.
     cur->buf->pos = cur->buf->last;
@@ -2071,9 +2159,6 @@ void ps_send_to_pagespeed(ngx_http_request_t* r,
   if (last_buf) {
     ctx->proxy_fetch->Done(true /* success */);
     ctx->proxy_fetch = NULL;  // ProxyFetch deletes itself on Done().
-  } else {
-    // TODO(jefftk): Decide whether Flush() is warranted here.
-    ctx->proxy_fetch->Flush(cfg_s->handler);
   }
 }
 
@@ -2170,6 +2255,31 @@ ngx_int_t ps_etag_header_filter(ngx_http_request_t* r) {
     r->gzip_vary = 0;
   }
 #endif
+
+  if (ctx && ctx->recorder) {
+    ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
+    int s_maxage_sec =
+        cfg_s->server_context->global_options()->EffectiveInPlaceSMaxAgeSec();
+    if (s_maxage_sec != -1) {
+      GoogleString existing_cache_control;
+      bool cache_control_present = ps_get_cache_control(
+          r, &existing_cache_control);
+      GoogleString updated_cache_control;
+      if (ResponseHeaders::ApplySMaxAge(s_maxage_sec,
+                                        existing_cache_control,
+                                        &updated_cache_control)) {
+        // We're modifing the cache control header; save a copy first.
+        // NULL indicates that the header was not present.
+        ctx->recorder->SaveCacheControl(
+            cache_control_present ? existing_cache_control.c_str() : NULL);
+
+        // Replace the cache-control with our new s-maxage-including one.
+        ps_set_cache_control(r, string_piece_to_pool_string(
+            r->pool, updated_cache_control));
+      }
+    }
+  }
+
   return ngx_http_ef_next_header_filter(r);
 }
 
@@ -2181,8 +2291,7 @@ ngx_http_output_body_filter_pt ngx_http_next_body_filter;
 // nginx so it can send them out to the browser.
 ngx_int_t ps_html_rewrite_header_filter(ngx_http_request_t* r) {
   ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
-  if (cfg_s->server_context == NULL) {
-    // Pagespeed is on for some server block but not this one.
+  if (ps_disabled(cfg_s)) {
     return ngx_http_next_header_filter(r);
   }
 
@@ -2266,8 +2375,7 @@ ngx_int_t ps_html_rewrite_header_filter(ngx_http_request_t* r) {
 
 ngx_int_t ps_html_rewrite_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
   ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
-  if (cfg_s->server_context == NULL) {
-    // Pagespeed is on for some server block but not this one.
+  if (ps_disabled(cfg_s)) {
     return ngx_http_next_body_filter(r, in);
   }
 
@@ -2397,6 +2505,7 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
         cache_url.c_str());
     const SystemRewriteOptions* options = SystemRewriteOptions::DynamicCast(
         ctx->driver->options());
+
     RequestContextPtr request_context(
         cfg_s->server_context->NewRequestContext(r));
     request_context->set_options(options->ComputeHttpOptions());
@@ -2416,7 +2525,7 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
         server_context->http_cache(),
         server_context->statistics(),
         message_handler);
-    // set in memory flag for in place_body_filter
+    // set in memory flag for in_place_body_filter
     r->filter_need_in_memory = 1;
 
     // We don't have the response headers at all yet because we haven't yet gone
@@ -2500,7 +2609,7 @@ ngx_int_t send_out_headers_and_body(
   // Send the body.
   ngx_chain_t* out;
   rc = string_piece_to_buffer_chain(
-      r->pool, output, &out, true /* send_last_buf */);
+      r->pool, output, &out, true /* send_last_buf */, false);
   if (rc == NGX_ERROR) {
     return NGX_ERROR;
   }
@@ -2790,8 +2899,8 @@ ngx_int_t ps_beacon_handler(ngx_http_request_t* r) {
 // supply it to the user.
 ngx_int_t ps_content_handler(ngx_http_request_t* r) {
   ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
-  if (cfg_s->server_context == NULL) {
-    // Pagespeed is on for some server block but not this one.
+  if (ps_disabled(cfg_s)) {
+    // Pagespeed is not on for this server block.
     return NGX_DECLINED;
   }
 
@@ -2931,7 +3040,7 @@ ngx_int_t ps_preaccess_handler(ngx_http_request_t* r) {
 ngx_int_t ps_etag_filter_init(ngx_conf_t* cf) {
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
-  if (cfg_m->driver_factory != NULL) {
+  if (cfg_m != NULL && cfg_m->driver_factory != NULL) {
     ngx_http_ef_next_header_filter = ngx_http_top_header_filter;
     ngx_http_top_header_filter = ps_etag_header_filter;
   }
@@ -3013,6 +3122,11 @@ ngx_int_t ps_init_module(ngx_cycle_t* cycle) {
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
 
+  // See https://github.com/pagespeed/ngx_pagespeed/issues/1220
+  if (cfg_m == NULL) {
+    return NGX_OK;
+  }
+
   ngx_http_core_main_conf_t* cmcf = static_cast<ngx_http_core_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cycle, ngx_http_core_module));
   ngx_http_core_srv_conf_t** cscfp = static_cast<ngx_http_core_srv_conf_t**>(
@@ -3080,7 +3194,7 @@ void ps_exit_child_process(ngx_cycle_t* cycle) {
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
   NgxBaseFetch::Terminate();
-  if (cfg_m->driver_factory != NULL) {
+  if (cfg_m != NULL && cfg_m->driver_factory != NULL) {
     cfg_m->driver_factory->ShutDown();
   }
 }
@@ -3090,7 +3204,7 @@ void ps_exit_child_process(ngx_cycle_t* cycle) {
 ngx_int_t ps_init_child_process(ngx_cycle_t* cycle) {
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
-  if (cfg_m->driver_factory == NULL) {
+  if (cfg_m == NULL || cfg_m->driver_factory == NULL) {
     return NGX_OK;
   }
 

src/ngx_pagespeed.h

@@ -52,7 +52,7 @@ class InPlaceResourceRecorder;
 // NGX_DECLINED immediately unless send_last_buf.
 ngx_int_t string_piece_to_buffer_chain(
     ngx_pool_t* pool, StringPiece sp,
-    ngx_chain_t** link_ptr, bool send_last_buf);
+    ngx_chain_t** link_ptr, bool send_last_buf, bool send_flush);
 
 StringPiece str_to_string_piece(ngx_str_t s);
 
@@ -109,6 +109,7 @@ typedef struct {
   // Location: headers that start with '/' without regarding X-Forwarded-Proto.
   bool location_field_set;
   bool psol_vary_accept_only;
+  bool follow_flushes;
 } ps_request_ctx_t;
 
 ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r);

src/ngx_rewrite_driver_factory.cc

@@ -69,7 +69,6 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
     SystemThreadSystem* system_thread_system, StringPiece hostname, int port)
     : SystemRewriteDriverFactory(process_context, system_thread_system,
         NULL /* default shared memory runtime */, hostname, port),
-      main_conf_(NULL),
       threads_started_(false),
       ngx_message_handler_(
           new NgxMessageHandler(timer(), thread_system()->NewMutex())),
@@ -83,7 +82,7 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
       ngx_shared_circular_buffer_(NULL),
       hostname_(hostname.as_string()),
       port_(port),
-      process_script_variables_(false),
+      process_script_variables_mode_(ProcessScriptVariablesMode::kOff),
       process_script_variables_set_(false),
       shut_down_(false) {
   InitializeDefaultOptions();
@@ -148,10 +147,19 @@ NamedLockManager* NgxRewriteDriverFactory::DefaultLockManager() {
 
 RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
   NgxRewriteOptions* options = new NgxRewriteOptions(thread_system());
+  // TODO(jefftk): figure out why using SetDefaultRewriteLevel like
+  // mod_pagespeed does in mod_instaweb.cc:create_dir_config() isn't enough here
+  // -- if you use that instead then ngx_pagespeed doesn't actually end up
+  // defaulting CoreFilters.
+  // See: https://github.com/pagespeed/ngx_pagespeed/issues/1190
   options->SetRewriteLevel(RewriteOptions::kCoreFilters);
   return options;
 }
 
+RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptionsForQuery() {
+  return new NgxRewriteOptions(thread_system());
+}
+
 bool NgxRewriteDriverFactory::CheckResolver() {
   if (use_native_fetcher_ && resolver_ == NULL) {
     return false;
@@ -208,6 +216,14 @@ void NgxRewriteDriverFactory::StartThreads() {
   threads_started_ = true;
 }
 
+void NgxRewriteDriverFactory::SetMainConf(NgxRewriteOptions* main_options) {
+  // Propagate process-scope options from the copy we had during nginx option
+  // parsing to our own.
+  if (main_options != NULL) {
+    default_options()->MergeOnlyProcessScopeOptions(*main_options);
+  }
+}
+
 void NgxRewriteDriverFactory::LoggingInit(
     ngx_log_t* log, bool may_install_crash_handler) {
   log_ = log;
@@ -250,4 +266,23 @@ void NgxRewriteDriverFactory::InitStats(Statistics* statistics) {
   InPlaceResourceRecorder::InitStats(statistics);
 }
 
+void NgxRewriteDriverFactory::PrepareForkedProcess(const char* name) {
+  ngx_pid = ngx_getpid();  // Needed for logging to have the right PIDs.
+  SystemRewriteDriverFactory::PrepareForkedProcess(name);
+}
+
+void NgxRewriteDriverFactory::NameProcess(const char* name) {
+  SystemRewriteDriverFactory::NameProcess(name);
+
+  // Superclass set status with prctl.  Nginx has a helper function for setting
+  // argv[0] as well, so let's use that.  We'll show up as:
+  //
+  //    nginx: pagespeed $name
+
+  char name_for_setproctitle[32];
+  snprintf(name_for_setproctitle, sizeof(name_for_setproctitle),
+           "pagespeed %s", name);
+  ngx_setproctitle(name_for_setproctitle);
+}
+
 }  // namespace net_instaweb

src/ngx_rewrite_driver_factory.h

@@ -48,6 +48,12 @@ class SlowWorker;
 class Statistics;
 class SystemThreadSystem;
 
+enum ProcessScriptVariablesMode {
+  kOff,
+  kLegacyRestricted,
+  kAll
+};
+
 class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  public:
   // We take ownership of the thread system.
@@ -63,8 +69,9 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   virtual Timer* DefaultTimer();
   virtual NamedLockManager* DefaultLockManager();
   // Create a new RewriteOptions.  In this implementation it will be an
-  // NgxRewriteOptions.
+  // NgxRewriteOptions, and it will have CoreFilters explicitly set.
   virtual RewriteOptions* NewRewriteOptions();
+  virtual RewriteOptions* NewRewriteOptionsForQuery();
   virtual ServerContext* NewDecodingServerContext();
   // Check resolver configured or not.
   bool CheckResolver();
@@ -90,7 +97,7 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
     InitStats(statistics);
   }
 
-  void set_main_conf(NgxRewriteOptions* main_conf) {  main_conf_ = main_conf; }
+  void SetMainConf(NgxRewriteOptions* main_conf);
 
   void set_resolver(ngx_resolver_t* resolver) {
     resolver_ = resolver;
@@ -111,8 +118,8 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   void set_native_fetcher_max_keepalive_requests(int x) {
     native_fetcher_max_keepalive_requests_ = x;
   }
-  bool process_script_variables() {
-    return process_script_variables_;
+  ProcessScriptVariablesMode process_script_variables() {
+    return process_script_variables_mode_;
   }
 
   void LoggingInit(ngx_log_t* log, bool may_install_crash_handler);
@@ -121,22 +128,22 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
 
   virtual void SetCircularBuffer(SharedCircularBuffer* buffer);
 
-  bool SetProcessScriptVariables(bool process_script_variables) {
+  bool SetProcessScriptVariables(ProcessScriptVariablesMode mode) {
     if (!process_script_variables_set_) {
-      process_script_variables_ = process_script_variables;
+      process_script_variables_mode_ = mode;
       process_script_variables_set_ = true;
       return true;
     }
     return false;
   }
 
+  virtual void PrepareForkedProcess(const char* name);
+
+  virtual void NameProcess(const char* name);
+
  private:
   Timer* timer_;
 
-  // main_conf will have only options set in the main block.  It may be NULL,
-  // and we do not take ownership.
-  NgxRewriteOptions* main_conf_;
-
   bool threads_started_;
   NgxMessageHandler* ngx_message_handler_;
   NgxMessageHandler* ngx_html_parse_message_handler_;
@@ -157,7 +164,7 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
 
   GoogleString hostname_;
   int port_;
-  bool process_script_variables_;
+  ProcessScriptVariablesMode process_script_variables_mode_;
   bool process_script_variables_set_;
   bool shut_down_;
 

src/ngx_rewrite_options.cc

@@ -112,7 +112,7 @@ void NgxRewriteOptions::AddProperties() {
       false);
   add_ngx_option(
       "", &NgxRewriteOptions::global_statistics_path_, "ngsp",
-      kGlobalStatisticsPath, kProcessScope,
+      kGlobalStatisticsPath, kProcessScopeStrict,
       "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics",
       false);
   add_ngx_option(
@@ -127,7 +127,8 @@ void NgxRewriteOptions::AddProperties() {
       kServerScope, "Set the admin path.  Ex: /pagespeed_admin", false);
   add_ngx_option(
       "", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
-      kProcessScope, "Set the global admin path.  Ex: /pagespeed_global_admin",
+      kProcessScopeStrict,
+      "Set the global admin path.  Ex: /pagespeed_global_admin",
       false);
 
   MergeSubclassProperties(ngx_properties_);
@@ -180,10 +181,10 @@ RewriteOptions::OptionScope NgxRewriteOptions::GetOptionScope(
        it != all_options().end(); ++it) {
     RewriteOptions::OptionBase* option = *it;
     if (option->option_name() == option_name) {
-      // We treat kProcessScope as kProcessScopeStrict, failing to start if an
-      // option is out of place.
-      return option->scope() == kProcessScope ? kProcessScopeStrict
-                                              : option->scope();
+      // We treat kLegacyProcessScope as kProcessScopeStrict, failing to start
+      // if an option is out of place.
+      return option->scope() == kLegacyProcessScope ? kProcessScopeStrict
+                                                    : option->scope();
     }
   }
   return kDirectoryScope;
@@ -191,15 +192,18 @@ RewriteOptions::OptionScope NgxRewriteOptions::GetOptionScope(
 
 RewriteOptions::OptionSettingResult NgxRewriteOptions::ParseAndSetOptions0(
     StringPiece directive, GoogleString* msg, MessageHandler* handler) {
-  if (IsDirective(directive, "on")) {
-    set_enabled(RewriteOptions::kEnabledOn);
-  } else if (IsDirective(directive, "off")) {
-    set_enabled(RewriteOptions::kEnabledOff);
-  } else if (IsDirective(directive, "unplugged")) {
-    set_enabled(RewriteOptions::kEnabledUnplugged);
-  } else {
+  EnabledEnum enabled;
+  if (!ParseFromString(directive, &enabled)) {
     return RewriteOptions::kOptionNameUnknown;
   }
+  if (enabled == RewriteOptions::kEnabledOff) {
+    // In ngx_pagespeed, for historical reasons, we treat "off" as "unplugged".
+    // Also, "off" is deprecated and people should be using "standby" or
+    // "unplugged" now depending on which sense they want.  See comment on
+    // RewriteOptions::EnabledEnum.
+    enabled = RewriteOptions::kEnabledUnplugged;
+  }
+  set_enabled(enabled);
   return RewriteOptions::kOptionOk;
 }
 
@@ -258,7 +262,8 @@ const char* ps_error_string_for_option(
 const char* NgxRewriteOptions::ParseAndSetOptions(
     StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
     NgxRewriteDriverFactory* driver_factory,
-    RewriteOptions::OptionScope scope, ngx_conf_t* cf, bool compile_scripts) {
+    RewriteOptions::OptionScope scope, ngx_conf_t* cf,
+    ProcessScriptVariablesMode script_mode) {
   CHECK_GE(n_args, 1);
 
   StringPiece directive = args[0];
@@ -274,20 +279,43 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
         pool, directive, "cannot be set at this scope.");
   }
 
+  bool compile_scripts = false;
+
+  if (script_mode != ProcessScriptVariablesMode::kOff) {
+    // In the old mode we only allowed a few, so restrict to those.
+    compile_scripts =
+        StringCaseStartsWith(directive, "LoadFromFile") ||
+        StringCaseEqual(directive, "EnableFilters") ||
+        StringCaseEqual(directive, "DisableFilters") ||
+        StringCaseEqual(directive, "DownstreamCachePurgeLocationPrefix") ||
+        StringCaseEqual(directive, "DownstreamCachePurgeMethod") ||
+        StringCaseEqual(directive,
+                        "DownstreamCacheRewrittenPercentageThreshold") ||
+        StringCaseEqual(directive, "ShardDomain");
+    // In the new behaviour we also allow scripting of query- and directory-
+    // scoped options.
+    compile_scripts |=
+        script_mode == ProcessScriptVariablesMode::kAll &&
+        (GetOptionScope(directive) <= RewriteOptions::kDirectoryScope ||
+         (StringCaseEqual(directive, "Allow") ||
+          StringCaseEqual(directive, "BlockingRewriteRefererUrls") ||
+          StringCaseEqual(directive, "Disallow") ||
+          StringCaseEqual(directive, "DistributableFilters") ||
+          StringCaseEqual(directive, "Domain") ||
+          StringCaseEqual(directive, "ExperimentVariable") ||
+          StringCaseEqual(directive, "ExperimentSpec") ||
+          StringCaseEqual(directive, "ForbidFilters") ||
+          StringCaseEqual(directive, "RetainComment") ||
+          StringCaseEqual(directive, "CustomFetchHeader") ||
+          StringCaseEqual(directive, "MapOriginDomain") ||
+          StringCaseEqual(directive, "MapProxyDomain") ||
+          StringCaseEqual(directive, "MapRewriteDomain") ||
+          StringCaseEqual(directive, "UrlValuedAttribute") ||
+          StringCaseEqual(directive, "Library")));
+  }
+
   ScriptLine* script_line;
   script_line = NULL;
-  // Note that LoadFromFile should not be scriptable on wildcard hosts,
-  // as browsers might be able to manipulate its natural use-case: $http_host.
-  if (!StringCaseStartsWith(directive, "LoadFromFile") &&
-      !StringCaseEqual(directive, "EnableFilters") &&
-      !StringCaseEqual(directive, "DisableFilters") &&
-      !StringCaseEqual(directive, "DownstreamCachePurgeLocationPrefix") &&
-      !StringCaseEqual(directive, "DownstreamCachePurgeMethod") &&
-      !StringCaseEqual(directive,
-                       "DownstreamCacheRewrittenPercentageThreshold") &&
-      !StringCaseEqual(directive, "ShardDomain")){
-    compile_scripts = false;
-  }
 
   if (n_args == 1 && StringCaseEqual(directive, "ClearInheritedScripts")) {
     clear_inherited_scripts_ = true;
@@ -363,24 +391,23 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
       }
     } else if (StringCaseEqual("ProcessScriptVariables", args[0])) {
       if (scope == RewriteOptions::kProcessScopeStrict) {
-        if (StringCaseEqual(arg, "on")) {
-          if (driver_factory->SetProcessScriptVariables(true)) {
-            result = RewriteOptions::kOptionOk;
-          } else {
-            return const_cast<char*>(
-                "pagespeed ProcessScriptVariables: can only be set once");
-          }
+        ProcessScriptVariablesMode mode;
+        if (StringCaseEqual(arg, "all")) {
+          mode = ProcessScriptVariablesMode::kAll;
+        } else if (StringCaseEqual(arg, "on")) {
+          mode = ProcessScriptVariablesMode::kLegacyRestricted;
         } else if (StringCaseEqual(arg, "off")) {
-          if (driver_factory->SetProcessScriptVariables(false)) {
-            result = RewriteOptions::kOptionOk;
-          } else {
-            return const_cast<char*>(
-                "pagespeed ProcessScriptVariables: can only be set once");
-          }
+          mode = ProcessScriptVariablesMode::kOff;
         } else {
           return const_cast<char*>(
               "pagespeed ProcessScriptVariables: invalid value");
         }
+        if (driver_factory->SetProcessScriptVariables(mode)) {
+          result = RewriteOptions::kOptionOk;
+        } else {
+          return const_cast<char*>(
+              "pagespeed ProcessScriptVariables: can only be set once");
+        }
       } else {
         return const_cast<char*>(
             "ProcessScriptVariables is only allowed at the top level");
@@ -391,7 +418,7 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
         result = driver_factory->ParseAndSetOption1(
             directive,
             arg,
-            scope >= RewriteOptions::kProcessScope,
+            scope >= RewriteOptions::kLegacyProcessScope,
             &msg,
             handler);
       }
@@ -404,7 +431,7 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
           directive,
           args[1],
           args[2],
-          scope >= RewriteOptions::kProcessScope,
+          scope >= RewriteOptions::kLegacyProcessScope,
           &msg,
           handler);
     }
@@ -476,7 +503,7 @@ bool NgxRewriteOptions::ExecuteScriptVariables(
 
       const char* status = ParseAndSetOptions(args, script_line->n_args(),
           r->pool, handler, driver_factory, script_line->scope(), NULL /*cf*/,
-          false /*compile scripts*/);
+          ProcessScriptVariablesMode::kOff);
 
       if (status != NULL) {
         script_error = true;

src/ngx_rewrite_options.h

@@ -29,6 +29,8 @@ extern "C" {
 
 #include <vector>
 
+#include "ngx_rewrite_driver_factory.h"
+
 #include "net/instaweb/rewriter/public/rewrite_options.h"
 #include "pagespeed/kernel/base/message_handler.h"
 #include "pagespeed/kernel/base/ref_counted_ptr.h"
@@ -131,7 +133,7 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   const char* ParseAndSetOptions(
       StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
       NgxRewriteDriverFactory* driver_factory, OptionScope scope,
-      ngx_conf_t* cf, bool compile_scripts);
+      ngx_conf_t* cf, ProcessScriptVariablesMode script_mode);
   bool ExecuteScriptVariables(
       ngx_http_request_t* r, MessageHandler* handler,
       NgxRewriteDriverFactory* driver_factory);

src/ngx_server_context.cc

@@ -35,7 +35,8 @@ namespace net_instaweb {
 
 NgxServerContext::NgxServerContext(
     NgxRewriteDriverFactory* factory, StringPiece hostname, int port)
-    : SystemServerContext(factory, hostname, port) {
+    : SystemServerContext(factory, hostname, port),
+      ngx_http2_variable_index_(NGX_ERROR) {
 }
 
 NgxServerContext::~NgxServerContext() { }
@@ -70,11 +71,23 @@ SystemRequestContext* NgxServerContext::NewRequestContext(
     local_ip.len = 0;
   }
 
-  return new SystemRequestContext(thread_system()->NewMutex(),
-                                  timer(),
-                                  ps_determine_host(r),
-                                  local_port,
-                                  str_to_string_piece(local_ip));
+  SystemRequestContext* ctx = new SystemRequestContext(
+      thread_system()->NewMutex(), timer(),
+      ps_determine_host(r), local_port, str_to_string_piece(local_ip));
+
+  // See if http2 is in use.
+  if (ngx_http2_variable_index_ >= 0) {
+    ngx_http_variable_value_t* val =
+        ngx_http_get_indexed_variable(r, ngx_http2_variable_index_);
+    if (val != NULL && val->valid) {
+      StringPiece str_val(reinterpret_cast<char*>(val->data), val->len);
+      if (str_val == "h2" || str_val == "h2c") {
+        ctx->set_using_http2(true);
+      }
+    }
+  }
+
+  return ctx;
 }
 
 GoogleString NgxServerContext::FormatOption(StringPiece option_name,

src/ngx_server_context.h

@@ -58,8 +58,18 @@ class NgxServerContext : public SystemServerContext {
 
   virtual GoogleString FormatOption(StringPiece option_name, StringPiece args);
 
+  void set_ngx_http2_variable_index(ngx_int_t idx) {
+    ngx_http2_variable_index_ = idx;
+  }
+
+  ngx_int_t ngx_http2_variable_index() const {
+    return ngx_http2_variable_index_;
+  }
+
  private:
   NgxRewriteDriverFactory* ngx_factory_;
+  // what index the "http2" var is, or NGX_ERROR.
+  ngx_int_t ngx_http2_variable_index_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxServerContext);
 };

test/nginx_system_test.sh

@@ -31,22 +31,38 @@
 : ${NATIVE_FETCHER:?"Set NATIVE_FETCHER to off or on"}
 : ${PRIMARY_PORT:?"Set PRIMARY_PORT"}
 : ${SECONDARY_PORT:?"Set SECONDARY_PORT"}
-: ${RCPORT1:?"Set RCPORT1"}
-: ${RCPORT2:?"Set RCPORT2"}
-: ${RCPORT2:?"Set RCPORT3"}
-: ${RCPORT4:?"Set RCPORT4"}
-: ${RCPORT5:?"Set RCPORT5"}
-: ${RCPORT6:?"Set RCPORT6"}
-: ${RCPORT7:?"Set RCPORT7"}
+: ${CONTROLLER_PORT:?"Set CONTROLLER_PORT"}
+: ${RCPORT:?"Set RCPORT"}
 : ${MOD_PAGESPEED_DIR:?"Set MOD_PAGESPEED_DIR"}
 : ${NGINX_EXECUTABLE:?"Set NGINX_EXECUTABLE"}
 : ${PAGESPEED_TEST_HOST:?"Set PAGESPEED_TEST_HOST"}
 POSITION_AUX="${POSITION_AUX:-unset}"
+RUN_CONTROLLER_TEST="${RUN_CONTROLLER_TEST:-off}"
 
 PRIMARY_HOSTNAME="localhost:$PRIMARY_PORT"
 SECONDARY_HOSTNAME="localhost:$SECONDARY_PORT"
 
-SERVER_ROOT="$MOD_PAGESPEED_DIR/src/install/"
+this_dir="$( cd $(dirname "$0") && pwd)"
+echo this_dir=$this_dir
+TEST_TMP="$this_dir/tmp"
+rm -rf "$TEST_TMP"
+mkdir -p "$TEST_TMP"
+echo TEST_TMP=$TEST_TMP
+
+if [ -d "$MOD_PAGESPEED_DIR/src" ]; then
+  MOD_PAGESPEED_DIR+="/src"
+fi
+
+APACHE_DOC_SRC="$MOD_PAGESPEED_DIR/install/"
+SERVER_ROOT="$TEST_TMP/root"
+echo SERVER_ROOT=$SERVER_ROOT
+rm -rf "$SERVER_ROOT"
+mkdir -p "$SERVER_ROOT"
+export APACHE_DOC_ROOT="$SERVER_ROOT"
+
+mkdir -p "$APACHE_DOC_ROOT"
+make -f "$APACHE_DOC_SRC/Makefile.tests" setup_doc_root \
+  INSTALL_DATA_DIR="$APACHE_DOC_SRC"
 
 # We need check and check_not before we source SYSTEM_TEST_FILE that provides
 # them.
@@ -136,17 +152,19 @@ function fire_ab_load() {
   sleep 2
  }
 
-this_dir="$( cd $(dirname "$0") && pwd)"
-
 # stop nginx/valgrind
 killall -s KILL nginx
-# TODO(oschaaf): Fix waiting for valgrind on 32 bits systems.
 killall -s KILL memcheck-amd64-
-while pgrep nginx > /dev/null; do sleep 1; done
-while pgrep memcheck > /dev/null; do sleep 1; done
+killall -s KILL memcheck-x86-
+SECONDS=0
+while pgrep -x nginx >/dev/null || pgrep memcheck >/dev/null;do
+  if [ $SECONDS -gt 20 ]; then
+    echo "Old processes won't die" >&2
+    exit 1
+  fi
+  sleep 1
+done
 
-TEST_TMP="$this_dir/tmp"
-rm -r "$TEST_TMP"
 check_simple mkdir -p "$TEST_TMP"
 PROXY_CACHE="$TEST_TMP/proxycache"
 TMP_PROXY_CACHE="$TEST_TMP/tmpproxycache"
@@ -176,6 +194,12 @@ else
   RESOLVER=""
 fi
 
+if [ "$RUN_CONTROLLER_TEST" = "on" ]; then
+  CONTROLLER="pagespeed ExperimentalCentralControllerPort $CONTROLLER_PORT;"
+else
+  CONTROLLER=""
+fi
+
 # set up the config file for the test
 PAGESPEED_CONF="$TEST_TMP/pagespeed_test.conf"
 PAGESPEED_CONF_TEMPLATE="$this_dir/pagespeed_test.conf.template"
@@ -200,15 +224,10 @@ cat $PAGESPEED_CONF_TEMPLATE \
   | sed 's#@@SERVER_ROOT@@#'"$SERVER_ROOT"'#' \
   | sed 's#@@PRIMARY_PORT@@#'"$PRIMARY_PORT"'#' \
   | sed 's#@@SECONDARY_PORT@@#'"$SECONDARY_PORT"'#' \
+  | sed 's#@@CONTROLLER@@#'"$CONTROLLER"'#' \
   | sed 's#@@NATIVE_FETCHER@@#'"$NATIVE_FETCHER"'#' \
   | sed 's#@@RESOLVER@@#'"$RESOLVER"'#' \
-  | sed 's#@@RCPORT1@@#'"$RCPORT1"'#' \
-  | sed 's#@@RCPORT2@@#'"$RCPORT2"'#' \
-  | sed 's#@@RCPORT3@@#'"$RCPORT3"'#' \
-  | sed 's#@@RCPORT4@@#'"$RCPORT4"'#' \
-  | sed 's#@@RCPORT5@@#'"$RCPORT5"'#' \
-  | sed 's#@@RCPORT6@@#'"$RCPORT6"'#' \
-  | sed 's#@@RCPORT7@@#'"$RCPORT7"'#' \
+  | sed 's#@@RCPORT@@#'"$RCPORT"'#' \
   | sed 's#@@PAGESPEED_TEST_HOST@@#'"$PAGESPEED_TEST_HOST"'#' \
   >> $PAGESPEED_CONF
 # make sure we substituted all the variables
@@ -220,7 +239,7 @@ if $USE_VALGRIND; then
             --show-possibly-lost=no --log-file=$TEST_TMP/valgrind.log \
             --suppressions="$this_dir/valgrind.sup" \
       $NGINX_EXECUTABLE -c $PAGESPEED_CONF) & VALGRIND_PID=$!
-  trap "echo 'terminating valgrind!' && kill -s sigterm $VALGRIND_PID" EXIT
+  trap "echo 'terminating valgrind!' && kill -s TERM $VALGRIND_PID" EXIT
   echo "Wait until nginx is ready to accept connections"
   while ! curl -I "http://$PRIMARY_HOSTNAME/mod_pagespeed_example/" 2>/dev/null; do
       sleep 0.1;
@@ -275,7 +294,7 @@ else
   if $USE_VALGRIND; then
     # Clear valgrind trap
     trap - EXIT
-    echo "To end valgrind, run 'kill -s quit $VALGRIND_PID'"
+    echo "To end valgrind, run 'kill -s TERM $VALGRIND_PID'"
   fi
   echo "Not running tests; commence manual testing"
   exit 4
@@ -289,8 +308,16 @@ CACHE_PURGE_METHODS="PURGE GET"
 
 SERVER_NAME=nginx
 
+# TODO(cheesy): add an nginx controller test variant.  Actually requires some
+# thought as we don't currently have a multi-phase test flow in ngx_pagespeeed,
+# as far as I can tell, and we really have to update the config and restart
+# since the controller is a global setting.
+RUN_CONTROLLER_TEST=${RUN_CONTROLLER_TEST:-off}
+
 # run generic system tests
-SYSTEM_TEST_FILE="$MOD_PAGESPEED_DIR/src/pagespeed/system/system_test.sh"
+PAGESPEED_DIR="$MOD_PAGESPEED_DIR/pagespeed"
+SYSTEM_TEST_FILE="$PAGESPEED_DIR/system/system_test.sh"
+REMOTE_CONFIG_TEST_FILE="$PAGESPEED_DIR/system/remote_config_test.sh"
 
 if [ ! -e "$SYSTEM_TEST_FILE" ] ; then
   echo "Not finding $SYSTEM_TEST_FILE -- is mod_pagespeed not in a parallel"
@@ -303,7 +330,8 @@ BEACON_HANDLER="ngx_pagespeed_beacon"
 STATISTICS_HANDLER="ngx_pagespeed_statistics"
 GLOBAL_STATISTICS_HANDLER="ngx_pagespeed_global_statistics"
 MESSAGES_HANDLER="ngx_pagespeed_message"
-STATISTICS_URL=http://$PRIMARY_HOSTNAME/ngx_pagespeed_statistics
+STATISTICS_URL=http://$PRIMARY_HOSTNAME/$STATISTICS_HANDLER
+GLOBAL_STATISTICS_URL=http://$PRIMARY_HOSTNAME/$GLOBAL_STATISTICS_HANDLER
 
 # An expected failure can be indicated like: "~In-place resource optimization~"
 PAGESPEED_EXPECTED_FAILURES="
@@ -332,7 +360,6 @@ if $USE_VALGRIND; then
 ~combine_css Maximum size of combined CSS.~
 ~prioritize_critical_css~
 ~prioritize_critical_css Able to read POST data from temp file.~
-~IPRO flow uses cache as expected.~
 ~IPRO flow doesn't copy uncacheable resources multiple times.~
 ~inline_unauthorized_resources allows unauthorized css selectors~
 "
@@ -343,6 +370,14 @@ fi
 set -- "$PRIMARY_HOSTNAME"
 source $SYSTEM_TEST_FILE
 
+# If we were only asked to run a single test, it ran in SYSTEM_TEST_FILE, so
+# we're done now and should exit.
+if [ -n ${TEST_TO_RUN+x} ]; then
+  check_failures_and_exit
+fi
+
+source $REMOTE_CONFIG_TEST_FILE
+
 # nginx-specific system tests
 
 start_test Test pagespeed directive inside if block inside location block.
@@ -578,30 +613,93 @@ check test $(scrape_stat image_rewrite_total_original_bytes) -ge 10000
 # happens both before and after.
 start_test "Reload config"
 
+function find_exactly_once {
+  test $(grep -c "$1") -eq 1
+}
+
+if [ "$RUN_CONTROLLER_TEST" = "on" ]; then
+  function check_process_names() {
+    if ! $USE_VALGRIND; then
+      # There should be one babysitter and controller running.  Under valgrind
+      # process labels are confused, so skip the check then.
+
+      running=$(ps auxww | grep 'ngin[x]')
+      check_from "$running" find_exactly_once "nginx: pagespeed babysitter"
+      check_from "$running" find_exactly_once "nginx: pagespeed controller"
+    fi
+  }
+  check_process_names
+fi
+
 # Fire up some heavy load if ab is available to test a stressed reload.
 # TODO(oschaaf): make sure we wait for the new worker to get ready to accept
 # requests.
 fire_ab_load
 
-check wget $EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css \
-  -O /dev/null
+URL="$EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css"
+check wget "$URL" -O /dev/null
 check_simple "$NGINX_EXECUTABLE" -s reload -c "$PAGESPEED_CONF"
+
 # Wait for the new worker process with the new configuration to get ready, or
 # else the sudden reset of the shared mem statistics/cache might catch upcoming
 # tests unaware.
-while [ $(scrape_stat image_rewrite_total_original_bytes) -gt 0 ]
-do
+function wait_for_new_worker() {
+  while [ $(scrape_stat image_rewrite_total_original_bytes) -gt 0 ]; do
     echo "Waiting for new worker to get ready..."
     sleep .1
-done
-
-check wget $EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css \
-  -O /dev/null
+  done
+}
+wait_for_new_worker
+check wget "$URL" -O /dev/null
 if [ "$AB_PID" != "0" ]; then
     echo "Kill ab (pid: $AB_PID)"
     kill -s KILL $AB_PID &>/dev/null || true
 fi
 
+# There should still be just one babysitter and controller running.
+if [ "$RUN_CONTROLLER_TEST" = "on" ]; then
+  check_process_names
+
+  check grep "Writing a byte to a pipe to tell the old controller to exit." \
+    $ERROR_LOG
+  check grep "Root process is starting a new controller; shutting down." \
+    $ERROR_LOG
+fi
+
+start_test "Shared memory checkpointing"
+
+# We do two tests here:
+# 1. Metadata cache: check that an IPRO'd image is still fully optimized after a
+#    restart.
+# 2. Property cache: prioritize critical CSS persists beacon results across
+#    restarts.
+
+IPRO_URL="$EXAMPLE_ROOT/images/Cuppa.png"
+
+# Checkpoint beacon results to disk if we haven't already.
+check wget "$IPRO_URL" -O /dev/null
+test_prioritize_critical_css
+sleep 2
+check wget "$IPRO_URL" -O /dev/null
+test_prioritize_critical_css
+
+# Reload nginx again.
+check_simple "$NGINX_EXECUTABLE" -s reload -c "$PAGESPEED_CONF"
+wait_for_new_worker
+
+# The image should be fully optimized.
+OUT=$($CURL -sS -D- -o/dev/null "$IPRO_URL")
+check_from "$OUT" grep ^X-Original-Content-Length:
+check_from "$OUT" grep ^Content-Length:
+content_length=$(echo "$OUT" | grep ^Content-Length: | grep -o [0-9]*)
+original_content_length=$(echo "$OUT" | grep ^X-Original-Content-Length: \
+  | grep -o [0-9]*)
+check [ "$original_content_length" -gt "$content_length" ]
+
+# The beacon responses are stored in the metadata cache, so this can only pass
+# if we persisted the metadata across restarts.
+test_prioritize_critical_css_final
+
 # This is dependent upon having a beacon handler.
 test_filter add_instrumentation beacons load.
 
@@ -646,14 +744,14 @@ check touch "$SECONDARY_CACHE/cache.flush"
 check touch "$IPRO_CACHE/cache.flush"
 sleep 1
 
-CACHE_TESTING_DIR="$SERVER_ROOT/mod_pagespeed_test/cache_flush/"
+CACHE_TESTING_DIR="$SERVER_ROOT/cache_flush/"
 CACHE_TESTING_TMPDIR="$CACHE_TESTING_DIR/$$"
 mkdir "$CACHE_TESTING_TMPDIR"
 cp "$CACHE_TESTING_DIR/cache_flush_test.html" "$CACHE_TESTING_TMPDIR/"
 CSS_FILE="$CACHE_TESTING_TMPDIR/update.css"
 echo ".class myclass { color: $COLOR0; }" > "$CSS_FILE"
 
-URL_PATH="mod_pagespeed_test/cache_flush/$$/cache_flush_test.html"
+URL_PATH="cache_flush/$$/cache_flush_test.html"
 
 URL="$SECONDARY_HOSTNAME/$URL_PATH"
 CACHE_A="--header=Host:cache_a.example.com"
@@ -1097,7 +1195,7 @@ OUT=$($WGET_DUMP --header=Host:response-header-disable.example.com $URL)
 check_not_from "$OUT" egrep -qi 'addInstrumentationInit'
 
 # TODO(jmaessen, jefftk): Port proxying tests, which rely on pointing a
-# MapProxyDomain construct at a static server.  Perhaps localhost:8050 will
+# MapProxyDomain construct at a static server.  Perhaps $HOSTNAME will
 # serve, but the tests need to use different urls then.  For mod_pagespeed these
 # tests immediately precede the "scrape_secondary_stat" definition in
 # system_test.sh.
@@ -1249,10 +1347,6 @@ OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1) || tr
 # We ignored the exit code, check if we got a 404 response.
 check_from "$OUT" fgrep -qi '404'
 
-if false; then
-# This test fails on 1.11, but passes on trunk.  While 2b4c097 fixed most users,
-# it's still possible to get two Vary: Accept-Encoding headers in this test, so
-# disable it for now.
 start_test Single Vary: Accept-Encoding header in IPRO flow
 URL=http://psol-vary.example.com/mod_pagespeed_example/styles/index_style.css
 OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
@@ -1269,7 +1363,15 @@ OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
 MATCHES=$(echo "$OUT" | grep -c "Vary: Accept-Encoding") || true
 check [ $MATCHES -eq 1 ]
 
-fi
+start_test Follow flushes can be turned off.
+echo "Check that FollowFlushes off outputs a single chunk"
+check_flushing noflush 5.4 1
+
+start_test Special responses from php are handled OK.
+URL="http://special-response.example.com/A.foo.css.pagespeed.cf.0.css"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -S --content-on-error $URL 2>&1) || true
+check_from "$OUT" fgrep -qi '404'
+check_from "$OUT" fgrep -q "PHP with a call to flush"
 
 start_test Shutting down.
 
@@ -1277,7 +1379,8 @@ start_test Shutting down.
 fire_ab_load
 
 if $USE_VALGRIND; then
-    kill -s quit $VALGRIND_PID
+    # SIGQUIT requests a graceful shutdown.
+    kill -s QUIT $VALGRIND_PID
     while pgrep memcheck > /dev/null; do sleep 1; done
     # Clear the previously set trap, we don't need it anymore.
     trap - EXIT
@@ -1286,12 +1389,12 @@ if $USE_VALGRIND; then
     check_not [ -s "$TEST_TMP/valgrind.log" ]
 else
     check_simple "$NGINX_EXECUTABLE" -s quit -c "$PAGESPEED_CONF"
-    while pgrep nginx > /dev/null; do sleep 1; done
+    while pgrep -x nginx > /dev/null; do sleep 1; done
 fi
 
 if [ "$AB_PID" != "0" ]; then
     echo "Kill ab (pid: $AB_PID)"
-    killall -s KILL $AB_PID &>/dev/null || true
+    kill -s KILL $AB_PID &>/dev/null || true
 fi
 
 start_test Logged output looks healthy.
@@ -1318,6 +1421,8 @@ OUT=$(cat "$ERROR_LOG" \
     | grep -v "\\[warn\\].*Resource based on.*ngx_pagespeed_statistics.*" \
     | grep -v "\\[warn\\].*Canceling 1 functions on sequence Shutdown.*" \
     | grep -v "\\[warn\\].*using uninitialized.*" \
+    | grep -v "\\[warn\\].*Controller process .* exited with wait status 9" \
+    | grep -v "\\[warn\\].*Controller process .* exited with wait status 15" \
     | grep -v "\\[error\\].*BadName*" \
     | grep -v "\\[error\\].*/mod_pagespeed/bad*" \
     | grep -v "\\[error\\].*doesnotexist.css.*" \
@@ -1326,7 +1431,7 @@ OUT=$(cat "$ERROR_LOG" \
     | grep -v "\\[error\\].*forbidden.example.com*" \
     | grep -v "\\[error\\].*custom-paths.example.com*" \
     | grep -v "\\[error\\].*bogus_format*" \
-    | grep -v "\\[error\\].*src/install/foo*" \
+    | grep -v "\\[error\\].*/install/foo*" \
     | grep -v "\\[error\\].*recv() failed*" \
     | grep -v "\\[error\\].*send() failed*" \
     | grep -v "\\[error\\].*Invalid url requested: js_defer.js.*" \
@@ -1346,8 +1451,12 @@ OUT=$(cat "$ERROR_LOG" \
     | grep -v "\\[warn\\].*remote\.cfg.*" \
     | grep -v "\\[warn\\].*end token not received.*" \
     | grep -v "\\[warn\\].*failed to hook next event.*" \
+    | grep -v "\\[warn\\].*Fetch timed out:.*" \
+    | grep -v "\\[warn\\].*Controller process .* exited with status code.*" \
     | grep -v "\\[warn\\].*Rewrite.*failed.*.pagespeed....0.foo.*" \
     | grep -v "\\[warn\\].*A.blue.css.*but cannot access the original.*" \
+    | grep -v "\\[warn\\].*Adding function to sequence.*" \
+    | grep -v "\\[warn\\].*special-response.*foo.css.*but cannot access the original.*" \
     || true)
 
 check [ -z "$OUT" ]

test/pagespeed_test.conf.template

@@ -30,7 +30,7 @@ http {
   proxy_cache_path "@@PROXY_CACHE@@"  levels=1:2   keys_zone=htmlcache:60m inactive=90m  max_size=50m;
   proxy_temp_path "@@TMP_PROXY_CACHE@@";
 
-  pagespeed ProcessScriptVariables on;
+  pagespeed ProcessScriptVariables all;
   pagespeed StatisticsPath /ngx_pagespeed_statistics;
   pagespeed GlobalStatisticsPath /ngx_pagespeed_global_statistics;
   pagespeed ConsolePath /pagespeed_console;
@@ -63,6 +63,12 @@ http {
   pagespeed CreateSharedMemoryMetadataCache "@@SHM_CACHE@@" 8192;
   pagespeed BlockingRewriteKey psatest;
 
+  # We need to checkpoint more frequently than usual so that our checkpointing
+  # test will definitly have seen one of these intervals and so not be flaky.
+  # This needs to have a 1s interval because the checkpointing test has a 2s
+  # sleep.
+  pagespeed ShmMetadataCacheCheckpointIntervalSec 1;
+
   # CriticalImagesBeaconEnabled is now on by default, but we disable in testing.
   # With this option enabled, the inline image system test will currently fail.
   # When critical image beaconing is enabled, only critical images are inlined.
@@ -79,6 +85,9 @@ http {
   pagespeed StatisticsLogging on;
   pagespeed LogDir "@@TEST_TMP@@/logdir";
 
+  # Expanded to CentralControllerPort directive when RUN_CONTROLLER_TEST=on
+  @@CONTROLLER@@
+
   server {
     # Sets up a logical home-page server on
     # max-cacheable-content-length.example.com.  This server is only used to
@@ -95,6 +104,70 @@ http {
     pagespeed MaxCacheableContentLength 85;
   }
 
+  # Test how load from file handles maximum sizes.
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name lff-large-files.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed MaxCacheableContentLength 4096;
+    pagespeed LoadFromFile
+      "http://lff-large-files.example.com/"
+      "@@SERVER_ROOT@@/";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name lff-large-files-no-fallback.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed MaxCacheableContentLength 4096;
+    # http://lff-large-files-no-fallback.example.com/foo.css won't load, because
+    # we've configured a LoadFromFile pattern that doesn't match the path the
+    # webserver would take.
+    pagespeed LoadFromFile
+      "http://lff-large-files-no-fallback.example.com/"
+      "@@SERVER_ROOT@@/mod_pagespeed_example/styles/";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name pagespeed-off.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed off;
+    pagespeed EnableFilters collapse_whitespace;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name pagespeed-on.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed on;
+    pagespeed EnableFilters collapse_whitespace;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name pagespeed-standby.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed standby;
+    pagespeed EnableFilters collapse_whitespace;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name pagespeed-unplugged.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed unplugged;
+    pagespeed EnableFilters collapse_whitespace;
+  }
+
   pagespeed UseNativeFetcher "@@NATIVE_FETCHER@@";
   @@RESOLVER@@
 
@@ -137,7 +210,7 @@ http {
     # inline_images (ii), defer_javascript (dj), webp (jw) and lossless_webp
     # (ws).
     if ($http_user_agent ~*
-        "Chrome/[2][3-9]+\.|Chrome/[[3-9][0-9]+\.|Chrome/[0-9]{3,}\.") {
+        "Chrome/[2][3-9]+\.|Chrome/[3-9][0-9]+\.|Chrome/[0-9]{3,}\.") {
       set $ps_capability_list "ll,ii,dj,jw,ws:";
     }
     # Cache-fragment 3: This fragment contains (a) Desktop User-Agents that
@@ -195,7 +268,7 @@ http {
 
     # Block 6: Location block with proxy_cache directives.
     location /mod_pagespeed_test/cachable_rewritten_html/ {
-      # 1: Upstream PageSpeed server is running at localhost:8050.
+      # 1: Upstream PageSpeed server is running at localhost:@@PRIMARY_PORT@@.
       proxy_pass http://localhost:@@PRIMARY_PORT@@;
       # 2: Use htmlcache as the zone for caching.
       proxy_cache htmlcache;
@@ -335,6 +408,29 @@ http {
       "id=1;percent=100;matches_device_type=mobile;enable=recompress_images";
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name experiment.ajax.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel CoreFilters;
+    pagespeed DisableFilters add_instrumentation;
+    pagespeed JsInlineMaxBytes 1;
+    pagespeed RunExperiment on;
+    pagespeed UseAnalyticsJs false;
+    pagespeed ExperimentSpec "id=1;percent=100;level=CoreFilters;enable=collapse_whitespace;options=JsInlineMaxBytes=1";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name ajax.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel CoreFilters;
+    pagespeed EnableFilters add_instrumentation;
+    pagespeed JsInlineMaxBytes 1;
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -564,6 +660,10 @@ http {
     pagespeed UrlValuedAttribute img alt-src Image;
     pagespeed UrlValuedAttribute video alt-a Image;
     pagespeed UrlValuedAttribute video alt-b Image;
+    pagespeed UrlValuedAttribute link data-stylesheet Stylesheet;
+    pagespeed UrlValuedAttribute span data-stylesheet-a Stylesheet;
+    pagespeed UrlValuedAttribute span data-stylesheet-b Stylesheet;
+    pagespeed UrlValuedAttribute span data-stylesheet-c Stylesheet;
 
     # Also test that we can redefine spec-defined attributes.
     pagespeed UrlValuedAttribute blockquote cite Image;
@@ -808,7 +908,7 @@ http {
     server_name www.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@;
+    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
@@ -822,7 +922,7 @@ http {
     server_name origin.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@;
+    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
@@ -836,7 +936,7 @@ http {
     server_name cdn.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@;
+    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
@@ -1180,7 +1280,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT1@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/standard";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1188,7 +1288,8 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-partially-invalid.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT2@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl
+              "http://127.0.0.1:@@RCPORT@@/partly-invalid";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1196,7 +1297,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-invalid.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT3@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/invalid";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1204,7 +1305,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-failed-fetch.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT5@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/fail-future";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1212,7 +1313,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-slow-fetch.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT6@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/timeout";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1220,7 +1321,41 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-experiment.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT7@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/experiment";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-slightly-slow-fetch.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl
+              "http://127.0.0.1:@@RCPORT@@/slightly-slow";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-slightly-slow-expired-fetch.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/slow-expired";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-forbidden.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/forbidden";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-initially-forbidden.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl
+              "http://127.0.0.1:@@RCPORT@@/forbidden-once";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
 
@@ -1249,6 +1384,17 @@ http {
     add_header 'X-Accel-Redirect' 'blablabla';
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name map-static-domain.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed MapRewriteDomain http://static-cdn.example.com
+                               http://map-static-domain.example.com;
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters defer_javascript,rewrite_domains;
+  }
+
   # Proxy + IPRO a gzip'd file for testing Issue 896.
   server {
     listen @@SECONDARY_PORT@@;
@@ -1266,6 +1412,44 @@ http {
     }
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name compressedcache.example.com;
+    pagespeed on;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed EnableFilters rewrite_css;
+    pagespeed HttpCacheCompressionLevel 9;
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name uncompressedcache.example.com;
+    pagespeed on;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed EnableFilters rewrite_css;
+    pagespeed HttpCacheCompressionLevel 0;
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -1328,7 +1512,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name proxy-post-origin.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
   }
 
   server {
@@ -1346,7 +1530,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name script-filters.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
     set $filters "";
     set $domain_shards "cdn1.example.com,cdn2.example.com";
     if ($http_X_Script) {
@@ -1368,7 +1552,7 @@ http {
     pagespeed EnableCachePurge on;
 
     pagespeed PurgeMethod PURGE;
-    root "@@SERVER_ROOT@@/mod_pagespeed_test/purge";
+    root "@@SERVER_ROOT@@/purge";
     pagespeed FileCachePath "@@FILE_CACHE@@_purge";
     pagespeed DisableFilters add_instrumentation;
     pagespeed RewriteLevel PassThrough;
@@ -1384,7 +1568,7 @@ http {
     pagespeed EnableCachePurge on;
 
     pagespeed PurgeMethod PURGE;
-    root "@@SERVER_ROOT@@/mod_pagespeed_test/purge";
+    root "@@SERVER_ROOT@@/purge";
     pagespeed FileCachePath "@@FILE_CACHE@@_dir_on";
     pagespeed DisableFilters add_instrumentation;
     pagespeed RewriteLevel PassThrough;
@@ -1402,7 +1586,7 @@ http {
     pagespeed InPlaceResourceOptimization off;
     pagespeed off;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
     location /mod_pagespeed_test/nostore {
       add_header "Cache-Control" "max-age=12345";
       add_header "Cache-Control" "public, no-store";
@@ -1417,7 +1601,7 @@ http {
     pagespeed InPlaceResourceOptimization off;
     pagespeed off;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
     location / {
       add_header "PageSpeedFilters" "add_instrumentation";
     }
@@ -1430,7 +1614,7 @@ http {
     pagespeed InPlaceResourceOptimization off;
     pagespeed off;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
     location / {
       add_header "PageSpeed" "off";
     }
@@ -1516,7 +1700,110 @@ http {
     pagespeed InPlaceResourceOptimization on;
     pagespeed FileCachePath "@@FILE_CACHE@@";
   }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name flush.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed RewriteDeadlinePerFlushMs 1;
 
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name noflush.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed RewriteDeadlinePerFlushMs 1;
+    pagespeed FollowFlushes off;
+
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name special-response.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    error_page 404 /mod_pagespeed_test/php_withflush.php;
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+    }
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name debug-filters.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters debug;
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name broken-fetch.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@/broken-fetch";
+
+    # Set up a fetch proxy that will 404 every request.
+    pagespeed FetchProxy "brokenfetch.example.com:1111";
+
+    # Prevent loopback fetch by explicitly authorizing the domain.  Loopback
+    # fetches would work, and for this test, we're trying to have fetches fail,
+    # so they are instead picked up by the ipro recorder.
+    pagespeed Domain http://broken-fetch.example.com;
+    pagespeed InPlaceResourceOptimization on;
+
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_javascript;
+    pagespeed DisableFilters add_instrumentation;
+    pagespeed CriticalImagesBeaconEnabled false;
+  }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name image-rewrite-with-flush.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed UrlValuedAttribute li data-thumb image;
+    pagespeed EnableFilters core;
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
   server {
     listen @@PRIMARY_PORT@@;
     listen [::]:@@PRIMARY_PORT@@;
@@ -1530,6 +1817,14 @@ http {
     }
 
     location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" {
+      # TODO(oschaaf): The following breaks nginx starting up, as the bad header
+      # value gets rejected during configuration parsing. However, we can't easily
+      # build a test for that right now.
+      # pagespeed AddResourceHeader "X-Fo\n\no" "Bar";
+      # pagespeed AddResourceHeader "X-Foo" "Ba\nr";
+
+      # For testing that AddResourceHeader gets passed through
+      pagespeed AddResourceHeader "X-Foo" "Bar";
       add_header "" "";
     }
 
@@ -1581,6 +1876,8 @@ http {
     pagespeed Library 43 1o978_K0_LNE5_ystNklf
       http://www.modpagespeed.com/rewrite_javascript.js;
     pagespeed RetainComment " google_ad_section*";
+    pagespeed PermitIdsForCssCombining "allowed-to-combine-1-*";
+    pagespeed PermitIdsForCssCombining "allowed-to-combine-2-*";
 
     # Test proxying of non-.pagespeed. resources.
     pagespeed MapProxyDomain http://localhost:@@PRIMARY_PORT@@/modpagespeed_http
@@ -1708,6 +2005,53 @@ http {
       expires 5m;
     }
 
+    location /mod_pagespeed_test/example.json {
+      expires 10m;
+    }
+
+    location /mod_pagespeed_test/ipro/cc200/ {
+      add_header "Cache-Control" "max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200p/ {
+      add_header "Cache-Control" "private, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200nc/ {
+      add_header "Cache-Control" "no-cache, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200ns/ {
+      add_header "Cache-Control" "no-store, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200nt/ {
+      add_header "Cache-Control" "no-transform, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma5/ {
+      add_header "Cache-Control" "s-maxage=5, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50/ {
+      add_header "Cache-Control" "s-maxage=50, max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50nsp/ {
+      add_header "Cache-Control" "s-maxage=50,max-age=200";
+    }
+    location /mod_pagespeed_test/ipro/cc9/ {
+      add_header "Cache-Control" "max-age=9";
+    }
+    location /mod_pagespeed_test/ipro/cc200cc9/ {
+      add_header "Cache-Control" "max-age=200, max-age=9";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50sma5/ {
+      add_header "Cache-Control" "max-age=200, s-maxage=50, s-maxage=5";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50cc9/ {
+      add_header "Cache-Control" "max-age=200, s-maxage=50, max-age=9";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50cc9nsp/ {
+      add_header "Cache-Control" "max-age=200,s-maxage=50,max-age=9";
+    }
+    location /mod_pagespeed_test/ipro/cc200sma50sma51/ {
+      add_header "Cache-Control" "max-age=200, s-maxage=50, s-maxage=51";
+    }
+
     location /mod_pagespeed_test/ipro/wait/ {
       # TODO(jmarantz): ModPagespeedInPlaceWaitForOptimized should be superfluous,
       # or made equivalent to ModPagespeedInPlaceRewriteDeadlineMs -1, which waits
@@ -1748,14 +2092,16 @@ http {
     }
 
     location /mod_pagespeed_test/ipro/instant/deadline/ {
-      pagespeed InPlaceRewriteDeadlineMs -1;
+      set $ps_deadline -1;
+      pagespeed ImageRecompressionQuality $ps_deadline;
+      pagespeed InPlaceRewriteDeadlineMs $ps_deadline;
     }
 
     # Test to make sure that user-authenticated resources do not get cached and
     # optimized.
     location /mod_pagespeed_test/auth/ {
        auth_basic "Restricted";
-       auth_basic_user_file "@@SERVER_ROOT@@mod_pagespeed_test/auth/passwd.conf";
+       auth_basic_user_file "@@SERVER_ROOT@@/mod_pagespeed_test/auth/passwd.conf";
     }
 
     location /mod_pagespeed_test/ipro/cookie/ {
@@ -1796,9 +2142,10 @@ http {
     pagespeed FetchHttps enable;
     location /mod_pagespeed_test/https_fetch/ {
       pagespeed DisableFilters inline_images;
+      set $ps_gstatic "https://www.gstatic.com/psa/static";
       pagespeed MapProxyDomain
           http://localhost:@@PRIMARY_PORT@@/https_gstatic_dot_com
-          https://www.gstatic.com/psa/static;
+          $ps_gstatic;
     }
 
     location /mod_pagespeed_test/strip_subresource_hints/default/ {
@@ -1827,6 +2174,11 @@ http {
         "http://$host:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/instant/"
         "@@SERVER_ROOT@@/mod_pagespeed_test/ipro/instant/";
 
+    location /mod_pagespeed_test/public/ {
+      add_header "Cache-Control" "public, max-age=600";
+      pagespeed PreserveUrlRelativity off;
+    }
+
     pagespeed EnableFilters remove_comments;
 
     # Test LoadFromFile mapping by mapping one dir to another.
@@ -1840,7 +2192,6 @@ http {
       "@@SERVER_ROOT@@/mod_pagespeed_test/load_from_file/file_dir/httponly/";
     pagespeed LoadFromFileRuleMatch Disallow \.ssp.css$ps_dollar;
     pagespeed LoadFromFileRuleMatch Allow exception\.ssp\.css$ps_dollar;
-
     #charset koi8-r;
 
     #access_log  logs/host.access.log  main;
@@ -1887,6 +2238,7 @@ http {
     image/svg+xml                         svg svgz;
     image/webp                            webp;
 
+    application/json                      json;
     application/java-archive              jar war ear;
     application/mac-binhex40              hqx;
     application/msword                    doc;

test/run_tests.sh

@@ -23,10 +23,19 @@
 # Exits with status 2 if command line args are wrong.
 #
 # Usage:
-#   ./run_tests.sh primary_port secondary_port mod_pagespeed_dir pagespeed_test_host
-# Example:
-#   ./run_tests.sh 8050 8051 /path/to/mod_pagespeed www.modpagespeed.com
+#   ./run_tests.sh
+# Or:
+#   ./run_tests.sh /path/to/mod_pagespeed /path/to/nginx/binary
 #
+# If you built ngx_pagespeed with "scripts/build_ngx_pagespeed.sh --devel" then
+# you don't need to pass any arguments to run_tests.sh.  Otherwise, you'll need
+# to tell it where to find a mod_pagespeed checkout (for example html files etc)
+# and the nginx binary to test.
+#
+# By default the test script uses several ports.  If you have a port conflict
+# and need to override one you can do that by setting the relevant environment
+# variable.  For example:
+#   PRIMARY_PORT=1234 ./run_tests.sh /.../mod_pagespeed /.../nginx/binary
 
 # Normally we test only with the native fetcher off.  Set
 # TEST_NATIVE_FETCHER=true to also test the native fetcher, set
@@ -43,28 +52,28 @@ RUN_TESTS=${RUN_TESTS:-true}
 # true.
 USE_VALGRIND=${USE_VALGRIND:-false}
 
-if [ "$#" -ne 5 ] ; then
-  echo "Usage: $0 primary_port secondary_port mod_pagespeed_dir"
-  echo "  nginx_executable"
+if [ "$#" -eq 0 ]; then
+  MOD_PAGESPEED_DIR="testing-dependencies/mod_pagespeed/"
+  NGINX_EXECUTABLE="nginx/sbin/nginx"
+elif [ "$#" -eq 2 ]; then
+  MOD_PAGESPEED_DIR="$1"
+  NGINX_EXECUTABLE="$2"
+else
+  echo "Usage: $0 [mod_pagespeed_dir nginx_executable]"
   exit 2
 fi
 
-PRIMARY_PORT="$1"
-SECONDARY_PORT="$2"
-MOD_PAGESPEED_DIR="$3"
-NGINX_EXECUTABLE="$4"
-PAGESPEED_TEST_HOST="$5"
-RCPORT1=9991
-RCPORT2=9992
-RCPORT3=9993
-RCPORT4=9994
-RCPORT5=9995
-RCPORT6=9996
-RCPORT7=9997
+: ${PRIMARY_PORT:=8050}
+: ${SECONDARY_PORT:=8051}
+: ${CONTROLLER_PORT:=8053}
+: ${RCPORT:=9991}
+: ${PAGESPEED_TEST_HOST:=selfsigned.modpagespeed.com}
+: ${PHP_PORT:=9000}
 
 this_dir="$( cd $(dirname "$0") && pwd)"
 
 function run_test_checking_failure() {
+  "$MOD_PAGESPEED_DIR/install/start_php.sh" "$PHP_PORT"
   USE_VALGRIND="$USE_VALGRIND" \
     PRIMARY_PORT="$PRIMARY_PORT" \
     SECONDARY_PORT="$SECONDARY_PORT" \
@@ -72,13 +81,8 @@ function run_test_checking_failure() {
     NGINX_EXECUTABLE="$NGINX_EXECUTABLE" \
     PAGESPEED_TEST_HOST="$PAGESPEED_TEST_HOST" \
     RUN_TESTS="$RUN_TESTS" \
-    RCPORT1="$RCPORT1" \
-    RCPORT2="$RCPORT2" \
-    RCPORT3="$RCPORT3" \
-    RCPORT4="$RCPORT4" \
-    RCPORT5="$RCPORT5" \
-    RCPORT6="$RCPORT6" \
-    RCPORT7="$RCPORT7" \
+    CONTROLLER_PORT="$CONTROLLER_PORT" \
+    RCPORT="$RCPORT" \
     bash "$this_dir/nginx_system_test.sh"
   STATUS=$?
   echo "With $@ setup."

test/valgrind.sup

@@ -163,3 +163,31 @@
    fun:_ZN3re23DFA10AddToQueueEPNS0_5WorkqEij
    ...
 }
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Value8
+   fun:_ZNK3re210SparseSetTIvE8containsEi
+   ...
+}
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Cond
+   fun:_ZNK3re210SparseSetTIvE8containsEi
+   ...
+}
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Value8
+   fun:_ZNK3re211SparseArrayIiE9has_indexEi
+   ...
+}
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Cond
+   fun:_ZNK3re211SparseArrayIiE9has_indexEi
+   ...
+}