update mps

I initially planned to get this working on Ubuntu 12, but decided it wasn't
worth it and switched to Ubuntu 14 (Trusty).  Now it passes tests on Travis, so
I've added other people's emails back to the set of people to send emails to.

Also update mps while we're at it.

.gitignore

@@ -2,3 +2,4 @@ test/tmp
 psol/
 psol-*.tar.gz
 *.*.*.*.tar.gz
+nginx

.gitmodules

@@ -0,0 +1,18 @@
+[submodule "testing-dependencies/mod_pagespeed"]
+	path = testing-dependencies/mod_pagespeed
+	url = https://github.com/pagespeed/mod_pagespeed.git
+[submodule "testing-dependencies/ngx_cache_purge"]
+	path = testing-dependencies/ngx_cache_purge
+	url = https://github.com/FRiCKLE/ngx_cache_purge.git
+[submodule "testing-dependencies/nginx"]
+	path = testing-dependencies/nginx
+	url = https://github.com/nginx/nginx.git
+[submodule "testing-dependencies/set-misc-nginx-module"]
+	path = testing-dependencies/set-misc-nginx-module
+	url = https://github.com/openresty/set-misc-nginx-module
+[submodule "testing-dependencies/ngx_devel_kit"]
+	path = testing-dependencies/ngx_devel_kit
+	url = https://github.com/simpl/ngx_devel_kit
+[submodule "testing-dependencies/headers-more-nginx-module"]
+	path = testing-dependencies/headers-more-nginx-module
+	url = https://github.com/openresty/headers-more-nginx-module

.travis.yml

@@ -1,42 +1,30 @@
 language: c++
+env:
+  global:
+    - MAKEFLAGS=-j3
+# By default travis loads submodules serially, but we can load them in parallel
+# if we install an updated git and use --jobs.  Some timing numbers:
+#   serial: 257s
+#   jobs=4: 182s (29s to install new git, 153s to run the downloads)
+#   jobs=8: 179s (29s to install new git, 150s to run the downloads)
+# We can't use --depth=1, though, because github doesn't have
+# allowReachableSHA1InWant set.
+#
+# TODO(jefftk): once we're running on a server with git 2.8 or later we can have
+#               --jobs without installing a new git.
+git:
+  submodules: false
+before_install:
+  - sudo add-apt-repository --yes ppa:git-core/ppa
+  - sudo apt-get update
+  - sudo apt-get install git
+  - git submodule update --init --recursive --jobs=8
 install:
-  - mv $TRAVIS_BUILD_DIR ~/ngxpagespeed
-  - sudo apt-get install build-essential zlib1g-dev libpcre3 libpcre3-dev unzip g++ python gperf make devscripts fakeroot git curl netcat-traditional gcc-mozilla clang-3.4 2>&1 > /dev/null
-  - export PATH=/usr/lib/gcc-mozilla/bin:$PATH
-  - sudo ln -sf /usr/lib/gcc-mozilla/lib/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libstdc++.so.6
-#  - sudo sh -c 'echo "image/webp webp" >> /etc/mime.types'
-#  - mkdir -p ~/bin
-#  - cd ~/bin
-#  - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
-  - mkdir ~/mod_pagespeed
-  - cd ~/mod_pagespeed
-  - git clone --recursive https://github.com/pagespeed/mod_pagespeed.git src
-  - cd src
-  - python build/gyp_chromium --depth=.
-  - cd ~/mod_pagespeed/src/pagespeed/automatic
-  - make BUILDTYPE=Release -C ../../pagespeed/automatic all
-  - cd ~
-  - git clone https://github.com/FRiCKLE/ngx_cache_purge.git
-  - NGX_CACHE_PURGE=$PWD/ngx_cache_purge
-  - wget https://openresty.org/download/ngx_openresty-1.9.7.2.tar.gz
-  - tar xzf ngx_openresty-*.tar.gz
-  - cd ngx_openresty-*/
-  - ./configure --with-luajit
-  - make
-  - NGX_DEVEL_KIT=$(echo $HOME/ngx_openresty-*/build/ngx_devel_kit-*/)
-  - SET_MISC_MODULE=$(echo $HOME/ngx_openresty-*/build/set-misc-nginx-module-*/)
-  - HEADERS_MORE_MODULE=$(echo $HOME/ngx_openresty-*/build/headers-more-nginx-module-*/)
-  - cd ~
-  - wget https://github.com/nginx/nginx/archive/branches/default.zip
-  - unzip default.zip
-  - cd nginx-branches-default
-  - MOD_PAGESPEED_DIR="$HOME/mod_pagespeed/src" ./auto/configure --add-module=$HOME/ngxpagespeed --add-module="$NGX_CACHE_PURGE" --add-module="$NGX_DEVEL_KIT" --add-module="$SET_MISC_MODULE" --add-module="$HEADERS_MORE_MODULE" --with-ipv6
-  - make
-  - sudo make install
+  scripts/build_ngx_pagespeed.sh --devel --assume-yes
 script:
-  - echo "build successful"
-  - echo "cd ~/ngxpagespeed"
-  - echo "sudo ./test/run_tests.sh $HOME/mod_pagespeed $HOME/nginx-branches-default/objs/nginx"
+  test/run_tests.sh $PWD/testing-dependencies/mod_pagespeed/ \
+                    $PWD/nginx/sbin/nginx
+dist: trusty
 sudo: required
 compiler:
   - gcc
@@ -47,4 +35,3 @@ notifications:
     - morlovich@google.com
     - jmarantz@google.com
     - huibao@google.com
-    - jcrowell@google.com

PSOL_BINARY_URL

@@ -1,4 +1,4 @@
 In a release this file would contain the URL to download the pre-compiled PSOL
 binary, but on development branches (like this one) you have to build PSOL from
 source yourself.  See:
-  https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source
+  https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source

README.md

@@ -1,6 +1,6 @@
 ![ngx_pagespeed](https://lh6.googleusercontent.com/-qufedJIJq7Y/UXEvVYxyYvI/AAAAAAAADo8/JHDFQhs91_c/s401/04_ngx_pagespeed.png)
 
-[![Build Status](https://travis-ci.org/pagespeed/ngx_pagespeed.svg?branch=trunk-tracking)](https://travis-ci.org/pagespeed/ngx_pagespeed)
+[![Build Status](https://travis-ci.org/pagespeed/ngx_pagespeed.svg?branch=master)](https://travis-ci.org/pagespeed/ngx_pagespeed)
 
 ngx_pagespeed speeds up your site and reduces page load time by automatically
 applying web performance best practices to pages and associated assets (CSS,

config

@@ -26,7 +26,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
   if [ ! -e "$mod_pagespeed_dir" ] ; then
     echo "ngx_pagespeed: pagespeed optimization library not found:"
 
-    psol_binary_url="$(cat $ngx_addon_dir/PSOL_BINARY_URL)"
+    psol_binary_url="$($ngx_addon_dir/scripts/format_binary_url.sh \
+                           $ngx_addon_dir/PSOL_BINARY_URL)"
     if [[ "$psol_binary_url" != https://* ]]; then
       echo "
   This is a development branch of ngx_pagespeed, which means there is no
@@ -76,6 +77,8 @@ fi
 
 if [ "$NGX_DEBUG" = "YES" ]; then
   buildtype=Debug
+  # If we're using a psol tarball that doesn't contain Debug/ (which is the case
+  # from 1.12 onward) then this will be overriden to buildtype=Release below.
 else
   buildtype=Release
 fi
@@ -143,6 +146,22 @@ if [ "$psol_binary" = "unset" ] ; then
     psol_binary="\
         $mod_pagespeed_dir/pagespeed/automatic/pagespeed_automatic.a"
   else
+    if ! [ -d "$ngx_addon_dir/psol/lib/$buildtype" ]; then
+      echo "
+You have set --with-debug for building nginx, but precompiled Debug binaries for
+PSOL, which ngx_pagespeed depends on, aren't available.  If you're trying to
+debug PSOL you need to build it from source.  If you just want to run nginx with
+debug-level logging you can use the Release binaries."
+      echo -n "
+Use the available Release binaries?"
+      read -p " [Y/n] " yn
+      if [[ "$yn" == N* || "$yn" == n* ]]; then
+        echo "Cancelled."
+        exit 1
+      fi
+
+      buildtype=Release
+    fi
     psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
     psol_binary="$psol_library_dir/pagespeed_automatic.a"
   fi
@@ -179,7 +198,7 @@ pagespeed_include="\
   $mod_pagespeed_dir/url"
 ngx_feature_path="$pagespeed_include"
 
-pagespeed_libs="$psol_binary $pagespeed_libs -lrt -pthread -lm"
+pagespeed_libs="$psol_binary $pagespeed_libs -lrt -pthread -lm -luuid"
 ngx_feature_libs="$pagespeed_libs"
 ngx_feature_test="
 

scripts/build_ngx_pagespeed.sh

@@ -1,11 +1,34 @@
 #!/bin/bash
+#
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 function usage() {
   echo "
 Usage: build_ngx_pagespeed.sh [options]
 
   Installs ngx_pagespeed and its dependencies.  Can optionally build and install
-  nginx as well.
+  nginx as well.  Can be run either as:
+
+     bash <(curl -f -L -sS https://ngxpagespeed.com/install) [options]
+
+  Or:
+
+     git clone git@github.com:pagespeed/ngx_pagespeed.git
+     cd ngx_pagespeed/
+     git checkout <branch>
+     scripts/build_ngx_pagespeed.sh [options]
 
 Options:
   -v, --ngx-pagespeed-version <ngx_pagespeed version>
@@ -14,7 +37,11 @@ Options:
       * latest-stable
       * a version number, such as 1.11.33.4
 
-      If you don't specify a version, defaults to latest-stable.
+      If you don't specify a version, defaults to latest-stable unless --devel
+      is specified, in which case it defaults to master.
+
+      This option doesn't make sense if we're running within an existing
+      ngx_pagespeed checkout.
 
   -n, --nginx-version <nginx version>
       What version of nginx to build.  If not set, this script only prepares the
@@ -36,6 +63,23 @@ Options:
       non-deb non-rpm system, this won't work.  In that case, install the
       dependencies yourself and pass --no-deps-check.
 
+  -s, --psol-from-source
+      Build PSOL from source instead of downloading a pre-built binary module.
+
+  -l, --devel
+      Sets up a development environment in ngx_pagespeed/nginx, building with
+      testing-only dependencies.  Includes --psol-from-source, conflicts with
+      --nginx-version.  Uses a 'git clone' checkout for ngx_pagespeed and nginx
+      instead of downloading a tarball.
+
+  -t, --build-type
+      When building PSOL from source, what to tell it for BUILD_TYPE.  Defaults
+      to 'Release' unless --devel is set in which case it defaults to 'Debug'.
+
+  -y, --assume-yes
+      Assume the answer to all prompts is 'yes, please continue'.  Intended for
+      automated usage, such as buildbots.
+
   -d, --dryrun
       Don't make any changes to the system, just print what changes you
       would have made.
@@ -44,10 +88,34 @@ Options:
       Print this message and exit."
 }
 
+RED=31
+GREEN=32
+YELLOW=33
+function begin_color() {
+  color="$1"
+  echo -e -n "\e[${color}m"
+}
+function end_color() {
+  echo -e -n "\e[0m"
+}
+
+function echo_color() {
+  color="$1"
+  shift
+  begin_color "$color"
+  echo "$@"
+  end_color
+}
+
+function error() {
+  local error_message="$@"
+  echo_color "$RED" -n "Error: " >&2
+  echo "$@" >&2
+}
+
 # Prints an error message and exits with an error code.
 function fail() {
-  local error_message="$@"
-  echo "$@" >&2
+  error "$@"
 
   # Normally I'd use $0 in "usage" here, but since most people will be running
   # this via curl, that wouldn't actually give something useful.
@@ -56,6 +124,11 @@ function fail() {
   exit 1
 }
 
+
+function status() {
+  echo_color "$GREEN" "$@"
+}
+
 # Intended to be called as:
 #   bash <(curl dl.google.com/.../build_ngx_pagespeed.sh) <args>
 
@@ -64,12 +137,19 @@ function fail() {
 # The run function handles exit-status checking for system-changing commands.
 # Additionally, this allows us to easily have a dryrun mode where we don't
 # actually make any changes.
+INITIAL_ENV=$(printenv | sort)
 function run() {
   if "$DRYRUN"; then
-    echo "would run $@"
+    echo_color "$YELLOW" -n "would run"
+    echo " $@"
+    env_differences=$(comm -13 <(echo "$INITIAL_ENV") <(printenv | sort))
+    if [ -n "$env_differences" ]; then
+      echo "  with the following additional environment variables:"
+      echo "$env_differences" | sed 's/^/    /'
+    fi
   else
     if ! "$@"; then
-      echo "Failure running $@, exiting."
+      error "Failure running '$@', exiting."
       exit 1
     fi
   fi
@@ -155,9 +235,9 @@ function install_dependencies() {
     fi
   done
   if [ -n "$missing_dependencies" ]; then
-    echo "Detected that we're missing the following depencencies:"
+    status "Detected that we're missing the following depencencies:"
     echo "  $missing_dependencies"
-    echo "Installing them:"
+    status "Installing them:"
     run sudo $install_pkg_cmd $missing_dependencies
   fi
 }
@@ -176,8 +256,13 @@ function gcc_too_old() {
 }
 
 function continue_or_exit() {
+  if "$ASSUME_YES"; then
+    return
+  fi
+
   local prompt="$1"
-  read -p "$prompt [Y/n] " yn
+  echo_color "$YELLOW" -n "$prompt"
+  read -p " [Y/n] " yn
   if [[ "$yn" == N* || "$yn" == n* ]]; then
     echo "Cancelled."
     exit 0
@@ -217,9 +302,10 @@ function build_ngx_pagespeed() {
     fail "Your version of getopt is too old.  Exiting with no changes made."
   fi
 
-  opts=$(getopt -o v:n:mb:pdh \
+  opts=$(getopt -o v:n:mb:pslt:ydh \
     --longoptions ngx-pagespeed-version:,nginx-version:,dynamic-module \
-    --longoptions buildir:,no-deps-check,dryrun,help \
+    --longoptions buildir:,no-deps-check,psol-from-source,devel,build-type: \
+    --longoptions assume-yes,dryrun,help \
     -n "$(basename "$0")" -- "$@")
   if [ $? != 0 ]; then
     usage
@@ -227,10 +313,14 @@ function build_ngx_pagespeed() {
   fi
   eval set -- "$opts"
 
-  NPS_VERSION="latest-stable"
+  NPS_VERSION="DEFAULT"
   NGINX_VERSION=""
   BUILDDIR="$HOME"
   DO_DEPS_CHECK=true
+  PSOL_FROM_SOURCE=false
+  DEVEL=false
+  BUILD_TYPE=""
+  ASSUME_YES=false
   DRYRUN=false
   DYNAMIC_MODULE=false
   while true; do
@@ -253,6 +343,19 @@ function build_ngx_pagespeed() {
       -p | --no-deps-check) shift
         DO_DEPS_CHECK=false
         ;;
+      -s | --psol-from-source) shift
+        PSOL_FROM_SOURCE=true
+        ;;
+      -l | --devel) shift
+        DEVEL=true
+        ;;
+      -t | --build-type) shift
+        BUILD_TYPE="$1"
+        shift
+        ;;
+      -y | --assume-yes) shift
+        ASSUME_YES="true"
+        ;;
       -d | --dryrun) shift
         DRYRUN="true"
         ;;
@@ -271,10 +374,58 @@ function build_ngx_pagespeed() {
     esac
   done
 
+  USE_GIT_CHECKOUT="$DEVEL"
+  ALREADY_CHECKED_OUT=false
+  if [ -e PSOL_BINARY_URL ]; then
+    status "Detected that we're running in an existing ngx_pagespeed checkout."
+    USE_GIT_CHECKOUT=true
+    ALREADY_CHECKED_OUT=true
+  fi
+
+  if "$ALREADY_CHECKED_OUT"; then
+    if [ "$NPS_VERSION" != "DEFAULT" ]; then
+      fail \
+"The --ngx-pagespeed-version argument doesn't make sense when running within an existing checkout."
+    fi
+  elif [ "$NPS_VERSION" = "DEFAULT" ]; then
+    if "$DEVEL"; then
+      NPS_VERSION="master"
+    else
+      NPS_VERSION="latest-stable"
+    fi
+  fi
+
   if [ ! -d "$BUILDDIR" ]; then
     fail "Told to build in $BUILDDIR, but that directory doesn't exist."
   fi
 
+  BUILD_NGINX=false
+  if [ -n "$NGINX_VERSION" ]; then
+    BUILD_NGINX=true
+  fi
+
+  if "$DEVEL"; then
+    PSOL_FROM_SOURCE=true
+    BUILD_NGINX=true
+    if [ -n "$NGINX_VERSION" ]; then
+      fail \
+"The --devel argument conflicts with --nginx.  In devel mode we use the version of nginx that's included as a submodule."
+    fi
+    if "$DYNAMIC_MODULE"; then
+      fail "Can't currently build a dynamic module in --devel mode."
+    fi
+  fi
+
+  if "$PSOL_FROM_SOURCE" && [ -z "$BUILD_TYPE" ]; then
+    if "$DEVEL"; then
+      BUILD_TYPE="Debug"
+    else
+      BUILD_TYPE="Release"
+    fi
+  elif [ -n "$BUILD_TYPE" ]; then
+    fail "Setting --build-type requires --psol-from-source or --devel."
+  fi
+
   if [ "$NGINX_VERSION" = "latest" ]; then
     # When this function fails it prints the debugging information needed first
     # to stderr.
@@ -283,8 +434,15 @@ function build_ngx_pagespeed() {
 
   if "$DYNAMIC_MODULE"; then
     # Check that ngx_pagespeed and nginx are recent enough to support dynamic
-    # modules.
-    if version_older_than "$NPS_VERSION" "1.10.33.5"; then
+    # modules.  Unfortunately NPS_VERSION might be a tag, in which case we don't
+    # know.  If it's not a numeric version number, then assume it's recent
+    # enough and if it's not they'll get an ugly compilation error later.
+    # Luckily 1.10.33.5 was a while ago now.
+    #
+    # I'd like to use =~ here, but they changed syntax between v3 and v4 (quotes
+    # moved from mandatory to optional to prohibited).
+    if [[ "${NPS_VERSION#*[^0-9.]}" = "$NPS_VERSION" ]] &&
+         version_older_than "$NPS_VERSION" "1.10.33.5"; then
       fail "
 You're trying to build ngx_pagespeed $NPS_VERSION as a dynamic module, but
 ngx_pagespeed didn't add support for dynamic modules until 1.10.33.5."
@@ -314,16 +472,16 @@ add support for dynamic modules in a way compatible with ngx_pagespeed until
   # Now make sure our dependencies are installed.
   if "$DO_DEPS_CHECK"; then
     if [ -f /etc/debian_version ]; then
-      echo "Detected debian-based distro."
+      status "Detected debian-based distro."
 
       install_dependencies "apt-get install" debian_is_installed \
         "build-essential zlib1g-dev libpcre3 libpcre3-dev unzip"
 
       if gcc_too_old; then
         if [ ! -e /usr/lib/gcc-mozilla/bin/gcc ]; then
-          echo "Detected that gcc is older than 4.8.  Installing gcc-mozilla"
-          echo "which installs gcc-4.8 into /usr/lib/gcc-mozilla/ and doesn't"
-          echo "affect your global gcc installation."
+          status "Detected that gcc is older than 4.8.  Installing gcc-mozilla"
+          status "which installs gcc-4.8 into /usr/lib/gcc-mozilla/ and doesn't"
+          status "affect your global gcc installation."
           run sudo apt-get install gcc-mozilla
         fi
 
@@ -332,7 +490,7 @@ add support for dynamic modules in a way compatible with ngx_pagespeed until
       fi
 
     elif [ -f /etc/redhat-release ]; then
-      echo "Detected redhat-based distro."
+      status "Detected redhat-based distro."
 
       install_dependencies "yum install" redhat_is_installed \
         "gcc-c++ pcre-devel zlib-devel make unzip wget"
@@ -350,9 +508,9 @@ Unexpected major version $redhat_major_version in /etc/redhat-release:
 $(cat /etc/redhat-release) Expected 5 or 6."
           fi
 
-          echo "Detected that gcc is older than 4.8.  Scientific Linux provides"
-          echo "a gcc package that installs gcc-4.8 into /opt/ and doesn't"
-          echo "affect your global gcc installation."
+          status "Detected that gcc is older than 4.8.  Scientific Linux"
+          status "provides a gcc package that installs gcc-4.8 into /opt/ and"
+          status "doesn't affect your global gcc installation."
           slc_key="https://linux.web.cern.ch/linux/scientific6/docs/repository/"
           slc_key+="cern/slc6X/i386/RPM-GPG-KEY-cern"
           slc_key_out="$TEMPDIR/RPM-GPG-KEY-cern"
@@ -378,9 +536,9 @@ This doesn't appear to be a deb-based distro or an rpm-based one.  Not going to
 be able to install dependencies.  Please install dependencies manually and rerun
 with --no-deps-check."
     fi
-    echo "Dependencies are all set."
+    status "Operating system dependencies are all set."
   else
-    echo "Not checking whether dependencies are installed."
+    status "Not checking whether operating system dependencies are installed."
   fi
 
   function delete_if_already_exists() {
@@ -391,7 +549,6 @@ with --no-deps-check."
       if [ ${#directory} -lt 8 ]; then
         fail "
 Not deleting $directory; name is suspiciously short.  Something is wrong."
-        exit 1
       fi
 
       continue_or_exit "OK to delete $directory?"
@@ -399,53 +556,106 @@ Not deleting $directory; name is suspiciously short.  Something is wrong."
     fi
   }
 
-  nps_baseurl="https://github.com/pagespeed/ngx_pagespeed/archive"
   # In general, the zip github builds for tag foo unzips to ngx_pagespeed-foo,
-  # but it looks like they special case vVERSION tags to ngx_pagespeed-VERSION.
+  # but it looks like they special case vVERSION tags to ngx_pagespeed-VERSION
   if [[ "$NPS_VERSION" =~ ^[0-9]*[.][0-9]*[.][0-9]*[.][0-9]*$ ]]; then
-    # We've been given a numeric version number.  This has an associated tag in
-    # the form vVERSION-beta.
-    nps_url_fname="v${NPS_VERSION}-beta"
+    # We've been given a numeric version number.  This has an associated tag
+    # in the form vVERSION-beta.
+    tag_name="v${NPS_VERSION}-beta"
     nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}-beta"
   else
     # We've been given a tag name, like latest-beta.  Download that directly.
-    nps_url_fname="$NPS_VERSION"
+    tag_name="$NPS_VERSION"
     nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}"
   fi
 
-  nps_downloaded="$TEMPDIR/$nps_downloaded_fname.zip"
-  run wget "$nps_baseurl/$nps_url_fname.zip" -O "$nps_downloaded"
-  nps_module_dir="$BUILDDIR/$nps_downloaded_fname"
-  delete_if_already_exists "$nps_module_dir"
-  echo "Extracting ngx_pagespeed..."
-  run unzip -q "$nps_downloaded" -d "$BUILDDIR"
-  run cd "$nps_module_dir"
-
-  # Now we need to figure out what precompiled version of PSOL to build
-  # ngx_pagespeed against.
-  if "$DRYRUN"; then
-    psol_url="https://psol.example.com/cant-get-psol-version-in-dry-run.tar.gz"
-  elif [ -e PSOL_BINARY_URL ]; then
-    # Releases after 1.11.33.4 there is a PSOL_BINARY_URL file that tells us
-    # where to look.
-    psol_url="$(cat PSOL_BINARY_URL)"
-    if [[ "$psol_url" != https://* ]]; then
-      fail "Got bad psol binary location information: $psol_url"
+  install_dir="this-only-makes-sense-in-devel-mode"
+  if "$USE_GIT_CHECKOUT"; then
+    # We're either doing a --devel build, or someone is running us from an
+    # existing git checkout.
+    nps_module_dir="$PWD"
+    install_dir="$nps_module_dir"
+    if "$ALREADY_CHECKED_OUT"; then
+      run cd "$nps_module_dir"
+    else
+      status "Checking out ngx_pagespeed..."
+      run git clone "git@github.com:pagespeed/ngx_pagespeed.git" \
+                    "$nps_module_dir"
+      run cd "$nps_module_dir"
+      run git checkout "$tag_name"
+    fi
+    submodules_dir="$nps_module_dir/testing-dependencies"
+    if "$DEVEL"; then
+      status "Downloading dependencies..."
+      run git submodule update --init --recursive
+      if [[ "$CONTINUOUS_INTEGRATION" != true ]]; then
+        status "Switching submodules over to git protocol."
+        # This lets us push to github by public key.
+        for config in $(find .git/ -name config) ; do
+          run sed -i s~https://github.com/~git@github.com:~ $config ;
+        done
+      fi
     fi
   else
-    # For past releases we have to grep it from the config file.  The url has
-    # always looked like this, and the config file has contained it since before
-    # we started tagging our ngx_pagespeed releases.
-    psol_url="$(
-      grep -o "https://dl.google.com/dl/page-speed/psol/[0-9.]*.tar.gz" config)"
-    if [ -z "$psol_url" ]; then
-      fail "Couldn't find PSOL url in $PWD/config"
-    fi
+    nps_baseurl="https://github.com/pagespeed/ngx_pagespeed/archive"
+    nps_downloaded="$TEMPDIR/$nps_downloaded_fname.zip"
+    status "Downloading ngx_pagespeed..."
+    run wget "$nps_baseurl/$tag_name.zip" -O "$nps_downloaded"
+    nps_module_dir="$BUILDDIR/$nps_downloaded_fname"
+    delete_if_already_exists "$nps_module_dir"
+    status "Extracting ngx_pagespeed..."
+    run unzip -q "$nps_downloaded" -d "$BUILDDIR"
+    run cd "$nps_module_dir"
   fi
 
-  run wget "$psol_url"
-  echo "Extracting PSOL..."
-  run tar -xzf $(basename "$psol_url")  # extracts to psol/
+  MOD_PAGESPEED_DIR=""
+  PSOL_BINARY=""
+  if "$PSOL_FROM_SOURCE"; then
+    MOD_PAGESPEED_DIR="$PWD/testing-dependencies/mod_pagespeed"
+    git submodule update --init --recursive -- "$MOD_PAGESPEED_DIR"
+    run pushd "$MOD_PAGESPEED_DIR"
+
+    if "$DEVEL"; then
+      if [ ! -d "$HOME/apache2" ]; then
+        run install/build_development_apache.sh 2.2 prefork
+      fi
+      cd devel
+      run make apache_debug_psol
+      PSOL_BINARY="$MOD_PAGESPEED_DIR/out/$BUILD_TYPE/pagespeed_automatic.a"
+    else
+      run install/build_psol.sh --skip_tests --skip_packaging
+      PSOL_BINARY="$MOD_PAGESPEED_DIR/pagespeed/automatic/pagespeed_automatic.a"
+    fi
+    run popd
+  else
+    # Now we need to figure out what precompiled version of PSOL to build
+    # ngx_pagespeed against.
+    if "$DRYRUN"; then
+      psol_url="https://psol.example.com/cant-get-psol-version-in-dry-run.tar.gz"
+    elif [ -e PSOL_BINARY_URL ]; then
+      # Releases after 1.11.33.4 there is a PSOL_BINARY_URL file that tells us
+      # where to look.
+      psol_url="$(scripts/format_binary_url.sh PSOL_BINARY_URL)"
+      if [[ "$psol_url" != https://* ]]; then
+        fail "Got bad psol binary location information: $psol_url"
+      fi
+    else
+      # For past releases we have to grep it from the config file.  The url has
+      # always looked like this, and the config file has contained it since
+      # before we started tagging our ngx_pagespeed releases.
+      psol_url="$(grep -o \
+          "https://dl.google.com/dl/page-speed/psol/[0-9.]*.tar.gz" config)"
+      if [ -z "$psol_url" ]; then
+        fail "Couldn't find PSOL url in $PWD/config"
+      fi
+    fi
+
+    status "Downloading PSOL binary..."
+    run wget "$psol_url"
+
+    status "Extracting PSOL..."
+    run tar -xzf $(basename "$psol_url")  # extracts to psol/
+  fi
 
   if "$DYNAMIC_MODULE"; then
     add_module="--add-dynamic-module=$nps_module_dir"
@@ -454,17 +664,37 @@ Not deleting $directory; name is suspiciously short.  Something is wrong."
   fi
   configure_args=("$add_module" "${extra_flags[@]}")
 
+  if "$DEVEL"; then
+    configure_args=("${configure_args[@]}"
+                    "--prefix=$install_dir/nginx"
+                    "--add-module=$submodules_dir/ngx_cache_purge"
+                    "--add-module=$submodules_dir/ngx_devel_kit"
+                    "--add-module=$submodules_dir/set-misc-nginx-module"
+                    "--add-module=$submodules_dir/headers-more-nginx-module"
+                    "--with-ipv6"
+                    "--with-http_v2_module")
+    if [ "$BUILD_TYPE" = "Debug" ]; then
+      configure_args=("${configure_args[@]}" "--with-debug")
+    fi
+  fi
+
   echo
-  if [ -z "$NGINX_VERSION" ]; then
-    # They didn't specify an nginx version, so we're just preparing the
-    # module for them to install.
-    echo "ngx_pagespeed is ready to be built against nginx."
-    echo "When running ./configure pass in:"
-    echo "  $(quote_arguments "${configure_args[@]}")"
+  if ! "$BUILD_NGINX"; then
+    # Just prepare the module for them to install.
+    status "ngx_pagespeed is ready to be built against nginx."
+    echo "When running ./configure:"
+    if "$PSOL_FROM_SOURCE"; then
+      echo "  Set the following environment variables:"
+      echo "    MOD_PAGESPEED_DIR=$MOD_PAGESPEED_DIR"
+      echo "    PSOL_BINARY=$PSOL_BINARY"
+    fi
+    echo "  Give ./configure the following arguments:"
+    echo "    $(quote_arguments "${configure_args[@]}")"
+    echo
     if [ ${#extra_flags[@]} -eq 0 ]; then
       echo "If this is for integration with an already-built nginx, make sure"
-      echo "to include any other arguments you originally passed to ./configure"
-      echo "You can see these with 'nginx -V'."
+      echo "to include any other arguments you originally passed to"
+      echo "./configure.  You can see these with 'nginx -V'."
     else
       echo "Note: because we need to set $(quote_arguments "${extra_flags[@]}")"
       echo "on this platform, if you want to integrate ngx_pagespeed with an"
@@ -472,60 +702,98 @@ Not deleting $directory; name is suspiciously short.  Something is wrong."
       echo "those flags set."
     fi
   else
-    # Download and build nginx.
-    nginx_leaf="nginx-${NGINX_VERSION}.tar.gz"
-    nginx_fname="$TEMPDIR/$nginx_leaf"
-    run wget "http://nginx.org/download/$nginx_leaf" -O "$nginx_fname"
-    nginx_dir="$BUILDDIR/nginx-${NGINX_VERSION}/"
-    delete_if_already_exists "$nginx_dir"
-    echo "Extracting nginx..."
-    run tar -xzf "$nginx_fname" --directory "$BUILDDIR"
-    "$DRYRUN" || cd "$nginx_dir"
+    if "$DEVEL"; then
+      # Use the nginx we loaded as a submodule
+      nginx_dir="$submodules_dir/nginx"
+      configure_location="auto"
+    else
+      # Download and build the specified nginx version.
+      nginx_leaf="nginx-${NGINX_VERSION}.tar.gz"
+      nginx_fname="$TEMPDIR/$nginx_leaf"
+      status "Downloading nginx..."
+      run wget "http://nginx.org/download/$nginx_leaf" -O "$nginx_fname"
+      nginx_dir="$BUILDDIR/nginx-${NGINX_VERSION}/"
+      delete_if_already_exists "$nginx_dir"
+      status "Extracting nginx..."
+      run tar -xzf "$nginx_fname" --directory "$BUILDDIR"
+      configure_location="."
+    fi
+    run cd "$nginx_dir"
 
-    configure=("./configure" "${configure_args[@]}")
-    echo "About to build nginx.  Do you have any additional ./configure"
-    echo "arguments you would like to set?  For example, if you would like"
-    echo "to build nginx with https support give --with-http_ssl_module"
-    echo "If you don't have any, just press enter."
-    read -p "> " additional_configure_args
-    if [ -n "$additional_configure_args" ]; then
-      # Split additional_configure_args respecting any internal quotation.
-      # Otherwise things like --with-cc-opt='-foo -bar' won't work.
-      eval additional_configure_args=("$additional_configure_args")
-      configure=("${configure[@]}" "${additional_configure_args[@]}")
+    configure=("$configure_location/configure" "${configure_args[@]}")
+    if ! "$ASSUME_YES"; then
+      echo "About to build nginx.  Do you have any additional ./configure"
+      echo "arguments you would like to set?  For example, if you would like"
+      echo "to build nginx with https support give --with-http_ssl_module"
+      echo "If you don't have any, just press enter."
+      read -p "> " additional_configure_args
+      if [ -n "$additional_configure_args" ]; then
+        # Split additional_configure_args respecting any internal quotation.
+        # Otherwise things like --with-cc-opt='-foo -bar' won't work.
+        eval additional_configure_args=("$additional_configure_args")
+        configure=("${configure[@]}" "${additional_configure_args[@]}")
+      fi
     fi
     echo "About to configure nginx with:"
     echo "   $(quote_arguments "${configure[@]}")"
     continue_or_exit "Does this look right?"
-    run "${configure[@]}"
+    MOD_PAGESPEED_DIR="$MOD_PAGESPEED_DIR" \
+      PSOL_BINARY="$PSOL_BINARY" \
+      run "${configure[@]}"
 
-    continue_or_exit "Build nginx?"
+    if ! "$DEVEL"; then
+      continue_or_exit "Build nginx?"
+    fi
     run make
 
-    continue_or_exit "Install nginx?"
-    run sudo make install
+    if "$DEVEL"; then
+      run make install
 
-    echo
-    if "$DYNAMIC_MODULE"; then
-      echo "Nginx installed with ngx_pagespeed support available as a"
-      echo "loadable module."
+      status "Nginx installed with ngx_pagespeed, and set up for testing."
+      # TODO(jefftk): pull these out into separate scripts.
+      echo "To run tests, pick a pair of ports like 8050 and 8051 and run:"
+      echo "  cd $nps_module_dir"
+      echo "  RUN_TESTS=true \\"
+      echo "  USE_VALGRIND=false \\"
+      echo "  TEST_NATIVE_FETCHER=false \\"
+      echo "  TEST_SERF_FETCHER=true \\"
+      echo "  test/run_tests.sh \\"
+      echo "    $MOD_PAGESPEED_DIR \\"
+      echo "    $install_dir/nginx/sbin/nginx"
       echo
-      echo "To load the ngx_pagespeed module, you'll need to add:"
-      echo "  load_module \"modules/ngx_pagespeed.so\";"
-      echo "at the top of your main nginx configuration file."
+      echo "To rebuild after changes:"
+      echo "  First, if you change things in PSOL or update it:"
+      echo "    cd $MOD_PAGESPEED_DIR/devel"
+      echo "    make apache_debug_psol"
+      echo "  Then, whether or not you updated PSOL, rebuild nginx:"
+      echo "    cd $install_dir/nginx"
+      echo "    make && make install"
     else
-      echo "Nginx installed with ngx_pagespeed support compiled-in."
+      continue_or_exit "Install nginx?"
+      run sudo make install
+
+      echo
+      if "$DYNAMIC_MODULE"; then
+        echo "Nginx installed with ngx_pagespeed support available as a"
+        echo "loadable module."
+        echo
+        echo "To load the ngx_pagespeed module, you'll need to add:"
+        echo "  load_module \"modules/ngx_pagespeed.so\";"
+        echo "at the top of your main nginx configuration file."
+      else
+        echo "Nginx installed with ngx_pagespeed support compiled-in."
+      fi
+      echo
+      echo "If this is a new installation you probably need an init script to"
+      echo "manage starting and stopping the nginx service.  See:"
+      echo "  http://wiki.nginx.org/InitScripts"
+      echo
+      echo "You'll also need to configure ngx_pagespeed if you haven't yet:"
+      echo "  https://developers.google.com/speed/pagespeed/module/configuration"
     fi
-    echo
-    echo "If this is a new installation you probably need an init script to"
-    echo "manage starting and stopping the nginx service.  See:"
-    echo "  http://wiki.nginx.org/InitScripts"
-    echo
-    echo "You'll also need to configure ngx_pagespeed if you haven't yet:"
-    echo "  https://developers.google.com/speed/pagespeed/module/configuration"
   fi
   if "$DRYRUN"; then
-    echo "[this was a dry run; your system is unchanged]"
+    echo_color "$YELLOW" "[this was a dry run; your system is unchanged]"
   fi
 }
 

scripts/format_binary_url.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -u
+
+if [ $# -ne 1 ]; then
+  echo "Usage: $(basename $0) <url_file>" >&2
+  exit 1
+fi
+
+url_file=$1
+
+if [ ! -e "$url_file" ]; then
+  echo "Url file '$url_file' missing!" >&2
+fi
+
+# The size names must match install/build_psol.sh in mod_pagespeed
+if [ "$(uname -m)" = x86_64 ]; then
+  bit_size_name=x64
+else
+  bit_size_name=ia32
+fi
+
+sed -e 's/$BIT_SIZE_NAME\b/'$bit_size_name'/g' $url_file

scripts/pagespeed_libraries_generator.sh

@@ -1,5 +1,19 @@
 #!/bin/bash
 #
+# Copyright 2013 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 # Converts pagespeed_libraries.conf from Apache-format to Nginx-format,
 # supporting the canonicalize_javascript_libraries filter.
 # Inspired by https://github.com/pagespeed/ngx_pagespeed/issues/532

scripts/prepare_psol.sh

@@ -1,94 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2012 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Author: jefftk@google.com (Jeff Kaufman)
-#
-# Usage:
-#   scripts/prepare_psol.sh /path/to/mod_pagespeed/src
-#
-# Creates a directory psol/ and copies headers and a few source files from a
-# depot_tools (glient) checkout into psol/include.  Along with creating
-# binaries, this is a step in preparing psol.tar.gz for distribution.
-#
-
-set -u  # check for undefined variables
-set -e  # exit on failed commands
-
-if [ "$(basename "$PWD")" != "ngx_pagespeed" ] ; then
-  echo "$(basename $0) must be invoked from the ngx_pagespeed directory"
-  exit 1
-fi
-
-if [ $# -ne 1 ] ; then
-  echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
-  exit 1
-fi
-
-MOD_PAGESPEED_SRC="$1"
-
-if [ "$(basename "$(dirname "$MOD_PAGESPEED_SRC")")/$( \
-        basename "$MOD_PAGESPEED_SRC")" != "mod_pagespeed/src" ] ; then
-  echo "Usage: $(basename $0) /path/to/mod_pagespeed/src"
-  exit 1
-fi
-
-if [ -e psol ] ; then
-  echo "A psol/ directory already exists.  Move it somewhere else and rerun."
-  exit 1
-fi
-mkdir psol/
-# Copy over the .h files, plus a few selected .cc and .c files.
-rsync -arvz "$MOD_PAGESPEED_SRC/" "psol/include/" --prune-empty-dirs \
-  --exclude=".svn" \
-  --exclude=".git" \
-  --include='*.h' \
-  --include='*/' \
-  --include="apr_thread_compatible_pool.cc" \
-  --include="serf_url_async_fetcher.cc" \
-  --include="apr_mem_cache.cc" \
-  --include="key_value_codec.cc" \
-  --include="apr_memcache2.c" \
-  --include="loopback_route_fetcher.cc" \
-  --include="add_headers_fetcher.cc" \
-  --include="console_css_out.cc" \
-  --include="console_out.cc" \
-  --include="dense_hash_map" \
-  --include="dense_hash_set" \
-  --include="sparse_hash_map" \
-  --include="sparse_hash_set" \
-  --include="sparsetable" \
-  --include="mod_pagespeed_console_out.cc" \
-  --include="mod_pagespeed_console_css_out.cc" \
-  --include="mod_pagespeed_console_html_out.cc" \
-  --exclude='*'
-mkdir -p psol/lib/Debug/linux/ia32
-mkdir -p psol/lib/Debug/linux/x64
-mkdir -p psol/lib/Release/linux/ia32
-mkdir -p psol/lib/Release/linux/x64
-
-# Log that we did this.
-SVN_REVISION="$(svn info $MOD_PAGESPEED_SRC | grep Revision | awk '{print $2}')"
-SVN_TAG="$(svn info $MOD_PAGESPEED_SRC | grep URL |  awk -F/ '{print $(NF-1)}')"
-
-DATE="$(date +%F)"
-echo "${DATE}: Copied from mod_pagespeed ${SVN_TAG}@r${SVN_REVISION} ($USER)" \
-  >> psol/include_history.txt
-
-echo
-echo "Output is in psol/include.  Now put binaries in psol/lib following"
-echo "https://github.com/pagespeed/ngx_pagespeed/wiki/Building-Release-Binaries"
-echo "and then you can distribute PSOL."
-

test/nginx_system_test.sh

@@ -49,7 +49,11 @@ rm -rf "$TEST_TMP"
 mkdir -p "$TEST_TMP"
 echo TEST_TMP=$TEST_TMP
 
-APACHE_DOC_SRC="$MOD_PAGESPEED_DIR/src/install/"
+if [ -d "$MOD_PAGESPEED_DIR/src" ]; then
+  MOD_PAGESPEED_DIR+="/src"
+fi
+
+APACHE_DOC_SRC="$MOD_PAGESPEED_DIR/install/"
 SERVER_ROOT="$TEST_TMP/root"
 echo SERVER_ROOT=$SERVER_ROOT
 rm -rf "$SERVER_ROOT"
@@ -311,7 +315,7 @@ SERVER_NAME=nginx
 RUN_CONTROLLER_TEST=${RUN_CONTROLLER_TEST:-off}
 
 # run generic system tests
-PAGESPEED_DIR="$MOD_PAGESPEED_DIR/src/pagespeed"
+PAGESPEED_DIR="$MOD_PAGESPEED_DIR/pagespeed"
 SYSTEM_TEST_FILE="$PAGESPEED_DIR/system/system_test.sh"
 REMOTE_CONFIG_TEST_FILE="$PAGESPEED_DIR/system/remote_config_test.sh"
 
@@ -1420,7 +1424,7 @@ OUT=$(cat "$ERROR_LOG" \
     | grep -v "\\[error\\].*forbidden.example.com*" \
     | grep -v "\\[error\\].*custom-paths.example.com*" \
     | grep -v "\\[error\\].*bogus_format*" \
-    | grep -v "\\[error\\].*src/install/foo*" \
+    | grep -v "\\[error\\].*/install/foo*" \
     | grep -v "\\[error\\].*recv() failed*" \
     | grep -v "\\[error\\].*send() failed*" \
     | grep -v "\\[error\\].*Invalid url requested: js_defer.js.*" \

test/run_tests.sh

@@ -58,10 +58,12 @@ NGINX_EXECUTABLE="$2"
 : ${CONTROLLER_PORT:=8053}
 : ${RCPORT:=9991}
 : ${PAGESPEED_TEST_HOST:=selfsigned.modpagespeed.com}
+: ${PHP_PORT:=9000}
 
 this_dir="$( cd $(dirname "$0") && pwd)"
 
 function run_test_checking_failure() {
+  "$MOD_PAGESPEED_DIR/install/start_php.sh" "$PHP_PORT"
   USE_VALGRIND="$USE_VALGRIND" \
     PRIMARY_PORT="$PRIMARY_PORT" \
     SECONDARY_PORT="$SECONDARY_PORT" \