Release version 1.12.34.3

Address a mistake I made in https://github.com/pagespeed/ngx_pagespeed/pull/1458

While testing the draft for 1.12.34.3 I noticed that older nginx
versions would fail to compile. Switch kNginx13_1_4 to a #define so we
can use it to properly compare.

Note: this will have to be backported to master

.travis.yml

@@ -1,35 +1,20 @@
 language: c++
-# Remove broken repo, per: https://github.com/travis-ci/travis-ci/issues/6588
-# Undo this once the repo is fixed.
-before_install:
-  - "sudo add-apt-repository --remove 'http://us-central1.gce.archive.ubuntu.com/ubuntu/ main restricted'"
-  - "sudo add-apt-repository --remove 'http://us-central1.gce.archive.ubuntu.com/ubuntu/ universe'"
-  - "sudo add-apt-repository --remove 'http://us-central1.gce.archive.ubuntu.com/ubuntu/ multiverse'"
-  - "sudo add-apt-repository http://archive.ubuntu.com/ubuntu/"
-  - "sudo add-apt-repository 'http://archive.ubuntu.com/ubuntu/ universe'"
-  - "sudo add-apt-repository 'http://archive.ubuntu.com/ubuntu/ multiverse'"
-  - "sudo apt-get -qq update"
 install:
-  - sudo sh -c 'echo "deb http://opensource.wandisco.com/ubuntu precise svn18" >> /etc/apt/sources.list.d/subversion18.list'
-  - sudo wget -q http://opensource.wandisco.com/wandisco-debian.gpg -O- | sudo apt-key add -
-  - sudo apt-get update 2>&1 > /dev/null
   - mv $TRAVIS_BUILD_DIR ~/ngxpagespeed
-  - sudo apt-get install build-essential zlib1g-dev libpcre3 libpcre3-dev unzip g++ python subversion gperf make devscripts fakeroot git curl netcat-traditional gcc-mozilla clang-3.4 2>&1 > /dev/null
-  - sudo mv /bin/nc.traditional /usr/bin/nc
+  - sudo apt-get install build-essential zlib1g-dev libpcre3 libpcre3-dev unzip g++ python gperf make devscripts fakeroot git curl netcat-traditional gcc-mozilla clang-3.4 2>&1 > /dev/null
   - export PATH=/usr/lib/gcc-mozilla/bin:$PATH
   - sudo ln -sf /usr/lib/gcc-mozilla/lib/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libstdc++.so.6
-  - sudo sh -c 'echo "image/webp webp" >> /etc/mime.types'
-  - mkdir -p ~/bin
-  - cd ~/bin
-  - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
+#  - sudo sh -c 'echo "image/webp webp" >> /etc/mime.types'
+#  - mkdir -p ~/bin
+#  - cd ~/bin
+#  - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
   - mkdir ~/mod_pagespeed
   - cd ~/mod_pagespeed
-  - git clone https://github.com/pagespeed/mod_pagespeed.git src
+  - git clone --recursive https://github.com/pagespeed/mod_pagespeed.git src
   - cd src
-  - ~/bin/depot_tools/gclient config https://github.com/pagespeed/mod_pagespeed.git --unmanaged --name=$PWD
-  - ~/bin/depot_tools/gclient sync --force --jobs=1
+  - python build/gyp_chromium --depth=.
   - cd ~/mod_pagespeed/src/pagespeed/automatic
-  - make BUILDTYPE=Release -C ../../pagespeed/automatic AR.host="$PWD/../../build/wrappers/ar.sh" AR.target="$PWD/../../build/wrappers/ar.sh" all
+  - make BUILDTYPE=Release -C ../../pagespeed/automatic all
   - cd ~
   - git clone https://github.com/FRiCKLE/ngx_cache_purge.git
   - NGX_CACHE_PURGE=$PWD/ngx_cache_purge
@@ -51,16 +36,10 @@ install:
 script:
   - echo "build successful"
   - echo "cd ~/ngxpagespeed"
-  - echo "sudo ./test/run_tests.sh 8050 8051 $HOME/mod_pagespeed/src $HOME/nginx-branches-default/objs selfsigned.modpagespeed.com"
-  - echo "sudo ./test/run_tests.sh 8050 8051 $HOME/mod_pagespeed $HOME/nginx-branches-default/objs/nginx selfsigned.modpagespeed.com"
+  - echo "sudo ./test/run_tests.sh $HOME/mod_pagespeed $HOME/nginx-branches-default/objs/nginx"
 sudo: required
 compiler:
   - gcc
 notifications:
   email:
-    - cheesy@google.com
-    - jefftk@google.com
-    - morlovich@google.com
-    - jmarantz@google.com
-    - huibao@google.com
-    - jcrowell@google.com
+    - pagespeed-ci@googlegroups.com

PSOL_BINARY_URL

@@ -1,4 +1 @@
-In a release this file would contain the URL to download the pre-compiled PSOL
-binary, but on development branches (like this one) you have to build PSOL from
-source yourself.  See:
-  https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source
+https://dl.google.com/dl/page-speed/psol/1.12.34.2-$BIT_SIZE_NAME.tar.gz

config

@@ -26,7 +26,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
   if [ ! -e "$mod_pagespeed_dir" ] ; then
     echo "ngx_pagespeed: pagespeed optimization library not found:"
 
-    psol_binary_url="$(cat $ngx_addon_dir/PSOL_BINARY_URL)"
+    psol_binary_url="$($ngx_addon_dir/scripts/format_binary_url.sh \
+                           $ngx_addon_dir/PSOL_BINARY_URL)"
     if [[ "$psol_binary_url" != https://* ]]; then
       echo "
   This is a development branch of ngx_pagespeed, which means there is no
@@ -76,6 +77,8 @@ fi
 
 if [ "$NGX_DEBUG" = "YES" ]; then
   buildtype=Debug
+  # If we're using a psol tarball that doesn't contain Debug/ (which is the case
+  # from 1.12 onward) then this will be overriden to buildtype=Release below.
 else
   buildtype=Release
 fi
@@ -143,6 +146,22 @@ if [ "$psol_binary" = "unset" ] ; then
     psol_binary="\
         $mod_pagespeed_dir/pagespeed/automatic/pagespeed_automatic.a"
   else
+    if ! [ -d "$ngx_addon_dir/psol/lib/$buildtype" ]; then
+      echo "
+You have set --with-debug for building nginx, but precompiled Debug binaries for
+PSOL, which ngx_pagespeed depends on, aren't available.  If you're trying to
+debug PSOL you need to build it from source.  If you just want to run nginx with
+debug-level logging you can use the Release binaries."
+      echo -n "
+Use the available Release binaries?"
+      read -p " [Y/n] " yn
+      if [[ "$yn" == N* || "$yn" == n* ]]; then
+        echo "Cancelled."
+        exit 1
+      fi
+
+      buildtype=Release
+    fi
     psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
     psol_binary="$psol_library_dir/pagespeed_automatic.a"
   fi
@@ -165,7 +184,7 @@ ngx_feature_incs="
 pagespeed_include="\
   $mod_pagespeed_dir \
   $mod_pagespeed_dir/third_party/chromium/src \
-  $mod_pagespeed_dir/third_party/google-sparsehash/src \
+  $mod_pagespeed_dir/third_party/google-sparsehash/src/src \
   $mod_pagespeed_dir/third_party/google-sparsehash/gen/arch/$os_name/$arch_name/include \
   $mod_pagespeed_dir/third_party/grpc/src/include \
   $mod_pagespeed_dir/third_party/protobuf/src/src \

scripts/build_ngx_pagespeed.sh

@@ -376,7 +376,7 @@ $(cat /etc/redhat-release) Expected 5 or 6."
       fail "
 This doesn't appear to be a deb-based distro or an rpm-based one.  Not going to
 be able to install dependencies.  Please install dependencies manually and rerun
-with --depsinstalled."
+with --no-deps-check."
     fi
     echo "Dependencies are all set."
   else
@@ -428,7 +428,7 @@ Not deleting $directory; name is suspiciously short.  Something is wrong."
   elif [ -e PSOL_BINARY_URL ]; then
     # Releases after 1.11.33.4 there is a PSOL_BINARY_URL file that tells us
     # where to look.
-    psol_url="$(cat PSOL_BINARY_URL)"
+    psol_url="$(scripts/format_binary_url.sh PSOL_BINARY_URL)"
     if [[ "$psol_url" != https://* ]]; then
       fail "Got bad psol binary location information: $psol_url"
     fi
@@ -524,7 +524,9 @@ Not deleting $directory; name is suspiciously short.  Something is wrong."
     echo "You'll also need to configure ngx_pagespeed if you haven't yet:"
     echo "  https://developers.google.com/speed/pagespeed/module/configuration"
   fi
-  "$DRYRUN" && echo "[this was a dry run; your system is unchanged]"
+  if "$DRYRUN"; then
+    echo "[this was a dry run; your system is unchanged]"
+  fi
 }
 
 # Start running things from a call at the end so if this script is executed

scripts/format_binary_url.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+set -u
+
+if [ $# -ne 1 ]; then
+  echo "Usage: $(basename $0) <url_file>" >&2
+  exit 1
+fi
+
+url_file=$1
+
+if [ ! -e "$url_file" ]; then
+  echo "Url file '$url_file' missing!" >&2
+fi
+
+# The size names must match install/build_psol.sh in mod_pagespeed
+if [ "$(uname -m)" = x86_64 ]; then
+  bit_size_name=x64
+else
+  bit_size_name=ia32
+fi
+
+sed -e 's/$BIT_SIZE_NAME\b/'$bit_size_name'/g' $url_file

scripts/prepare_psol.sh

@@ -1,94 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2012 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Author: jefftk@google.com (Jeff Kaufman)
-#
-# Usage:
-#   scripts/prepare_psol.sh /path/to/mod_pagespeed/src
-#
-# Creates a directory psol/ and copies headers and a few source files from a
-# depot_tools (glient) checkout into psol/include.  Along with creating
-# binaries, this is a step in preparing psol.tar.gz for distribution.
-#
-
-set -u  # check for undefined variables
-set -e  # exit on failed commands
-
-if [ "$(basename "$PWD")" != "ngx_pagespeed" ] ; then
-  echo "$(basename $0) must be invoked from the ngx_pagespeed directory"
-  exit 1
-fi
-
-if [ $# -ne 1 ] ; then
-  echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
-  exit 1
-fi
-
-MOD_PAGESPEED_SRC="$1"
-
-if [ "$(basename "$(dirname "$MOD_PAGESPEED_SRC")")/$( \
-        basename "$MOD_PAGESPEED_SRC")" != "mod_pagespeed/src" ] ; then
-  echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
-  exit 1
-fi
-
-if [ -e psol ] ; then
-  echo "A psol/ directory already exists.  Move it somewhere else and rerun."
-  exit 1
-fi
-mkdir psol/
-# Copy over the .h files, plus a few selected .cc and .c files.
-rsync -arvz "$MOD_PAGESPEED_SRC/" "psol/include/" --prune-empty-dirs \
-  --exclude=".svn" \
-  --exclude=".git" \
-  --include='*.h' \
-  --include='*/' \
-  --include="apr_thread_compatible_pool.cc" \
-  --include="serf_url_async_fetcher.cc" \
-  --include="apr_mem_cache.cc" \
-  --include="key_value_codec.cc" \
-  --include="apr_memcache2.c" \
-  --include="loopback_route_fetcher.cc" \
-  --include="add_headers_fetcher.cc" \
-  --include="console_css_out.cc" \
-  --include="console_out.cc" \
-  --include="dense_hash_map" \
-  --include="dense_hash_set" \
-  --include="sparse_hash_map" \
-  --include="sparse_hash_set" \
-  --include="sparsetable" \
-  --include="mod_pagespeed_console_out.cc" \
-  --include="mod_pagespeed_console_css_out.cc" \
-  --include="mod_pagespeed_console_html_out.cc" \
-  --exclude='*'
-mkdir -p psol/lib/Debug/linux/ia32
-mkdir -p psol/lib/Debug/linux/x64
-mkdir -p psol/lib/Release/linux/ia32
-mkdir -p psol/lib/Release/linux/x64
-
-# Log that we did this.
-SVN_REVISION="$(svn info $MOD_PAGESPEED_SRC | grep Revision | awk '{print $2}')"
-SVN_TAG="$(svn info $MOD_PAGESPEED_SRC | grep URL |  awk -F/ '{print $(NF-1)}')"
-
-DATE="$(date +%F)"
-echo "${DATE}: Copied from mod_pagespeed ${SVN_TAG}@r${SVN_REVISION} ($USER)" \
-  >> psol/include_history.txt
-
-echo
-echo "Output is in psol/include.  Now put binaries in psol/lib following"
-echo "https://github.com/pagespeed/ngx_pagespeed/wiki/Building-Release-Binaries"
-echo "and then you can distribute PSOL."
-

src/ngx_pagespeed.cc

@@ -84,6 +84,7 @@ extern ngx_module_t ngx_pagespeed;
 
 // Needed for SystemRewriteDriverFactory to use shared memory.
 #define PAGESPEED_SUPPORT_POSIX_SHARED_MEM
+#define NGINX_1_13_4 1013004
 
 net_instaweb::NgxRewriteDriverFactory* active_driver_factory = NULL;
 
@@ -3007,13 +3008,17 @@ ngx_int_t ps_preaccess_handler(ngx_http_request_t* r) {
   ph = cmcf->phase_engine.handlers;
 
   i = r->phase_handler;
+
   // move handlers before try_files && content phase
+  // As of nginx 1.13.4 we will be right before the try_files module
+  #if (nginx_version < NGINX_1_13_4)
   while (ph[i + 1].checker != ngx_http_core_try_files_phase &&
          ph[i + 1].checker != ngx_http_core_content_phase) {
     ph[i] = ph[i + 1];
     ph[i].next--;
     i++;
   }
+  #endif
 
   // insert ps phase handler
   ph[i].checker = ps_phase_handler;
@@ -3069,8 +3074,17 @@ ngx_int_t ps_init(ngx_conf_t* cf) {
     ngx_http_core_main_conf_t* cmcf = static_cast<ngx_http_core_main_conf_t*>(
         ngx_http_conf_get_module_main_conf(cf, ngx_http_core_module));
 
+    int phase = NGX_HTTP_PREACCESS_PHASE;
+
+    // As of nginx 1.13.4, try_files has changed.
+    // https://github.com/nginx/nginx/commit/129b06dc5dfab7b4513a4f274b3778cd9b8a6a22
+#if (nginx_version >= NGINX_1_13_4)
+    phase = NGX_HTTP_PRECONTENT_PHASE;
+#endif
+
     ngx_http_handler_pt* h = static_cast<ngx_http_handler_pt*>(
-        ngx_array_push(&cmcf->phases[NGX_HTTP_PREACCESS_PHASE].handlers));
+        ngx_array_push(&cmcf->phases[phase].handlers));
+
     if (h == NULL) {
       return NGX_ERROR;
     }

src/ngx_rewrite_driver_factory.cc

@@ -87,6 +87,10 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
       shut_down_(false) {
   InitializeDefaultOptions();
   default_options()->set_beacon_url("/ngx_pagespeed_beacon");
+  // Put in the version of the ngx-pagespeed release. We don't release mod_pagespeed
+  // so we cannot rely on PSOL to hand us to right version. Note that if we perform
+  // another release, this must be either updated or removed.
+  default_options()->set_x_header_value("1.12.34.3-0");
   SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
       default_options());
   system_options->set_file_cache_clean_inode_limit(500000);

test/nginx_system_test.sh

@@ -32,13 +32,7 @@
 : ${PRIMARY_PORT:?"Set PRIMARY_PORT"}
 : ${SECONDARY_PORT:?"Set SECONDARY_PORT"}
 : ${CONTROLLER_PORT:?"Set CONTROLLER_PORT"}
-: ${RCPORT1:?"Set RCPORT1"}
-: ${RCPORT2:?"Set RCPORT2"}
-: ${RCPORT2:?"Set RCPORT3"}
-: ${RCPORT4:?"Set RCPORT4"}
-: ${RCPORT5:?"Set RCPORT5"}
-: ${RCPORT6:?"Set RCPORT6"}
-: ${RCPORT7:?"Set RCPORT7"}
+: ${RCPORT:?"Set RCPORT"}
 : ${MOD_PAGESPEED_DIR:?"Set MOD_PAGESPEED_DIR"}
 : ${NGINX_EXECUTABLE:?"Set NGINX_EXECUTABLE"}
 : ${PAGESPEED_TEST_HOST:?"Set PAGESPEED_TEST_HOST"}
@@ -197,7 +191,7 @@ else
 fi
 
 if [ "$RUN_CONTROLLER_TEST" = "on" ]; then
-  CONTROLLER="pagespeed CentralControllerPort $CONTROLLER_PORT;"
+  CONTROLLER="pagespeed ExperimentalCentralControllerPort $CONTROLLER_PORT;"
 else
   CONTROLLER=""
 fi
@@ -229,13 +223,7 @@ cat $PAGESPEED_CONF_TEMPLATE \
   | sed 's#@@CONTROLLER@@#'"$CONTROLLER"'#' \
   | sed 's#@@NATIVE_FETCHER@@#'"$NATIVE_FETCHER"'#' \
   | sed 's#@@RESOLVER@@#'"$RESOLVER"'#' \
-  | sed 's#@@RCPORT1@@#'"$RCPORT1"'#' \
-  | sed 's#@@RCPORT2@@#'"$RCPORT2"'#' \
-  | sed 's#@@RCPORT3@@#'"$RCPORT3"'#' \
-  | sed 's#@@RCPORT4@@#'"$RCPORT4"'#' \
-  | sed 's#@@RCPORT5@@#'"$RCPORT5"'#' \
-  | sed 's#@@RCPORT6@@#'"$RCPORT6"'#' \
-  | sed 's#@@RCPORT7@@#'"$RCPORT7"'#' \
+  | sed 's#@@RCPORT@@#'"$RCPORT"'#' \
   | sed 's#@@PAGESPEED_TEST_HOST@@#'"$PAGESPEED_TEST_HOST"'#' \
   >> $PAGESPEED_CONF
 # make sure we substituted all the variables
@@ -1196,7 +1184,7 @@ OUT=$($WGET_DUMP --header=Host:response-header-disable.example.com $URL)
 check_not_from "$OUT" egrep -qi 'addInstrumentationInit'
 
 # TODO(jmaessen, jefftk): Port proxying tests, which rely on pointing a
-# MapProxyDomain construct at a static server.  Perhaps localhost:8050 will
+# MapProxyDomain construct at a static server.  Perhaps $HOSTNAME will
 # serve, but the tests need to use different urls then.  For mod_pagespeed these
 # tests immediately precede the "scrape_secondary_stat" definition in
 # system_test.sh.

test/pagespeed_test.conf.template

@@ -232,7 +232,7 @@ http {
 
     # Block 6: Location block with proxy_cache directives.
     location /mod_pagespeed_test/cachable_rewritten_html/ {
-      # 1: Upstream PageSpeed server is running at localhost:8050.
+      # 1: Upstream PageSpeed server is running at localhost:@@PRIMARY_PORT@@.
       proxy_pass http://localhost:@@PRIMARY_PORT@@;
       # 2: Use htmlcache as the zone for caching.
       proxy_cache htmlcache;
@@ -372,6 +372,29 @@ http {
       "id=1;percent=100;matches_device_type=mobile;enable=recompress_images";
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name experiment.ajax.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel CoreFilters;
+    pagespeed DisableFilters add_instrumentation;
+    pagespeed JsInlineMaxBytes 1;
+    pagespeed RunExperiment on;
+    pagespeed UseAnalyticsJs false;
+    pagespeed ExperimentSpec "id=1;percent=100;level=CoreFilters;enable=collapse_whitespace;options=JsInlineMaxBytes=1";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name ajax.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel CoreFilters;
+    pagespeed EnableFilters add_instrumentation;
+    pagespeed JsInlineMaxBytes 1;
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -1221,7 +1244,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT1@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/standard";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1229,7 +1252,8 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-partially-invalid.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT2@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl
+              "http://127.0.0.1:@@RCPORT@@/partly-invalid";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1237,7 +1261,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-invalid.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT3@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/invalid";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1245,7 +1269,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-failed-fetch.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT5@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/fail-future";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1253,7 +1277,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-slow-fetch.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT6@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/timeout";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
   server {
@@ -1261,7 +1285,41 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name remote-config-experiment.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT7@@/remote.cfg";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/experiment";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-slightly-slow-fetch.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl
+              "http://127.0.0.1:@@RCPORT@@/slightly-slow";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-slightly-slow-expired-fetch.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/slow-expired";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-forbidden.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/forbidden";
+    pagespeed RemoteConfigurationTimeoutMs 1500;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name remote-config-initially-forbidden.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RemoteConfigurationUrl
+              "http://127.0.0.1:@@RCPORT@@/forbidden-once";
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
 
@@ -1670,6 +1728,27 @@ http {
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters debug;
   }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name broken-fetch.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@/broken-fetch";
+
+    # Set up a fetch proxy that will 404 every request.
+    pagespeed FetchProxy "brokenfetch.example.com:1111";
+
+    # Prevent loopback fetch by explicitly authorizing the domain.  Loopback
+    # fetches would work, and for this test, we're trying to have fetches fail,
+    # so they are instead picked up by the ipro recorder.
+    pagespeed Domain http://broken-fetch.example.com;
+    pagespeed InPlaceResourceOptimization on;
+
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_javascript;
+    pagespeed DisableFilters add_instrumentation;
+    pagespeed CriticalImagesBeaconEnabled false;
+  }
   server {
     listen @@PRIMARY_PORT@@;
     listen [::]:@@PRIMARY_PORT@@;

test/run_tests.sh

@@ -23,12 +23,12 @@
 # Exits with status 2 if command line args are wrong.
 #
 # Usage:
-#   ./run_tests.sh primary_port secondary_port mod_pagespeed_dir ngx_binary
-#   pagespeed_test_host
-# Example:
-#   ./run_tests.sh 8050 8051 /path/to/mod_pagespeed /path/to/nginx/binary
-#   selfsigned.modpagespeed.com
+#   ./run_tests.sh /path/to/mod_pagespeed /path/to/nginx/binary
 #
+# By default the test script uses several ports.  If you have a port conflict
+# and need to override one you can do that by setting the relevant environment
+# variable.  For example:
+#   PRIMARY_PORT=1234 ./run_tests.sh /.../mod_pagespeed /.../nginx/binary
 
 # Normally we test only with the native fetcher off.  Set
 # TEST_NATIVE_FETCHER=true to also test the native fetcher, set
@@ -45,25 +45,19 @@ RUN_TESTS=${RUN_TESTS:-true}
 # true.
 USE_VALGRIND=${USE_VALGRIND:-false}
 
-if [ "$#" -ne 5 ] ; then
-  echo "Usage: $0 primary_port secondary_port mod_pagespeed_dir"
-  echo "  nginx_executable"
+if [ "$#" -ne 2 ] ; then
+  echo "Usage: $0 mod_pagespeed_dir nginx_executable"
   exit 2
 fi
 
-PRIMARY_PORT="$1"
-SECONDARY_PORT="$2"
-MOD_PAGESPEED_DIR="$3"
-NGINX_EXECUTABLE="$4"
-PAGESPEED_TEST_HOST="$5"
-CONTROLLER_PORT=8053
-RCPORT1=9991
-RCPORT2=9992
-RCPORT3=9993
-RCPORT4=9994
-RCPORT5=9995
-RCPORT6=9996
-RCPORT7=9997
+MOD_PAGESPEED_DIR="$1"
+NGINX_EXECUTABLE="$2"
+
+: ${PRIMARY_PORT:=8050}
+: ${SECONDARY_PORT:=8051}
+: ${CONTROLLER_PORT:=8053}
+: ${RCPORT:=9991}
+: ${PAGESPEED_TEST_HOST:=selfsigned.modpagespeed.com}
 
 this_dir="$( cd $(dirname "$0") && pwd)"
 
@@ -76,13 +70,7 @@ function run_test_checking_failure() {
     PAGESPEED_TEST_HOST="$PAGESPEED_TEST_HOST" \
     RUN_TESTS="$RUN_TESTS" \
     CONTROLLER_PORT="$CONTROLLER_PORT" \
-    RCPORT1="$RCPORT1" \
-    RCPORT2="$RCPORT2" \
-    RCPORT3="$RCPORT3" \
-    RCPORT4="$RCPORT4" \
-    RCPORT5="$RCPORT5" \
-    RCPORT6="$RCPORT6" \
-    RCPORT7="$RCPORT7" \
+    RCPORT="$RCPORT" \
     bash "$this_dir/nginx_system_test.sh"
   STATUS=$?
   echo "With $@ setup."