Merge branch 'jmarantz-config-with-response-headers-test' of github.com:pagespeed/ngx_pagespeed into jmarantz-config-with-response-headers-test

README.md

@@ -19,15 +19,94 @@ optimizations, see our <a href="http://ngxpagespeed.com">demonstration site</a>.
 
 ## How to build
 
-Follow the steps on <a
-href="https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source">build
-ngx_pagespeed from source</a>.
+Because nginx does not support dynamic loading of modules, you need to compile
+nginx from source to add ngx_pagespeed. Alternatively, if you're using Tengine you can [install ngx_pagespeed without
+recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_pagespeed-with-Tengine).
+
+1. Install dependencies:
+
+   ```bash
+   # These are for RedHat, CentOS, and Fedora.
+   $ sudo yum install gcc-c++ pcre-dev pcre-devel zlib-devel make
+
+   # These are for Debian. Ubuntu will be similar.
+   $ sudo apt-get install build-essential zlib1g-dev libpcre3 libpcre3-dev
+   ```
+
+2. Download ngx_pagespeed:
+
+   ```bash
+   $ cd ~
+   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.7.30.4-beta.zip
+   $ unzip v1.7.30.4-beta.zip # or unzip v1.7.30.4-beta
+   $ cd ngx_pagespeed-1.7.30.4-beta/
+   $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.4.tar.gz
+   $ tar -xzvf 1.7.30.4.tar.gz # expands to psol/
+   ```
+
+3. Download and build nginx:
+
+   ```bash
+   $ cd ~
+   $ # check http://nginx.org/en/download.html for the latest version
+   $ wget http://nginx.org/download/nginx-1.6.0.tar.gz
+   $ tar -xvzf nginx-1.6.0.tar.gz
+   $ cd nginx-1.6.0/
+   $ ./configure --add-module=$HOME/ngx_pagespeed-1.7.30.4-beta
+   $ make
+   $ sudo make install
+   ```
+
+If this doesn't work see the [build
+troubleshooting](https://github.com/pagespeed/ngx_pagespeed/wiki/Build-Troubleshooting) page.
+
+This will use a binary PageSpeed Optimization Library, but it's also possible to
+[build PSOL from
+source](https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source).
+
+Note: ngx_pagespeed currently doesn't support Windows or MacOS because the
+underlying PSOL library doesn't.
 
 ## How to use
 
-Follow the steps on <a
-href="https://developers.google.com/speed/pagespeed/module/configuration">PageSpeed
-configuration</a>.
+In your `nginx.conf`, add to the main or server block:
+
+```nginx
+pagespeed on;
+pagespeed FileCachePath /var/ngx_pagespeed_cache;  # Use tmpfs for best results.
+```
+
+In every server block where pagespeed is enabled add:
+
+```apache
+#  Ensure requests for pagespeed optimized resources go to the pagespeed
+#  handler and no extraneous headers get set.
+location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" { add_header "" ""; }
+location ~ "^/ngx_pagespeed_static/" { }
+location ~ "^/ngx_pagespeed_beacon$" { }
+location /ngx_pagespeed_statistics { allow 127.0.0.1; deny all; }
+location /ngx_pagespeed_global_statistics { allow 127.0.0.1; deny all; }
+location /ngx_pagespeed_message { allow 127.0.0.1; deny all; }
+location /pagespeed_console { allow 127.0.0.1; deny all; }
+```
+
+To confirm that the module is loaded, fetch a page and check that you see the
+`X-Page-Speed` header:
+
+```bash
+$ curl -I 'http://localhost:8050/some_page/' | grep X-Page-Speed
+X-Page-Speed: 1.7.30.4-...
+```
+
+Looking at the source of a few pages you should see various changes, such as
+urls being replaced with new ones like `yellow.css.pagespeed.ce.lzJ8VcVi1l.css`.
+
+For complete documentation, see [Using
+PageSpeed](https://developers.google.com/speed/pagespeed/module/using).
+
+There are extensive system tests which cover most of ngx_pagespeed's
+functionality.  Consider [testing your
+installation](https://github.com/pagespeed/ngx_pagespeed/wiki/Testing).
 
 For feedback, questions, and to follow
 the progress of the project:
@@ -36,3 +115,20 @@ the progress of the project:
   list](https://groups.google.com/forum/#!forum/ngx-pagespeed-discuss)
 - [ngx-pagespeed-announce mailing
   list](https://groups.google.com/forum/#!forum/ngx-pagespeed-announce)
+
+Note: The
+[canonicalize_javascript_libraries](https://developers.google.com/speed/pagespeed/module/filter-canonicalize-js)
+depends on `pagespeed_libraries.conf` which is distributed in Apache's format.
+To convert it to the Nginx format, run:
+
+```bash
+$ scripts/pagespeed_libraries_generator.sh > ~/pagespeed_libraries.conf
+$ sudo mv ~/pagespeed_libraries.conf /etc/nginx/
+```
+
+And then include it in your Nginx configuration by reference:
+
+```nginx
+include pagespeed_libraries.conf;
+pagespeed EnableFilters canonicalize_javascript_libraries;
+```

config

@@ -27,8 +27,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
     echo "   You need to separately download the pagespeed library:"
     echo ""
     echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.9.32.2.tar.gz"
-    echo "     $ tar -xzvf 1.9.32.2.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.4.tar.gz"
+    echo "     $ tar -xzvf 1.7.30.4.tar.gz # expands to psol/"
     echo ""
     echo "   Or see the installation instructions:"
     echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
@@ -39,6 +39,20 @@ else
   build_from_source=true
 fi
 
+echo "mod_pagespeed_dir=$mod_pagespeed_dir"
+echo "build_from_source=$build_from_source"
+
+ngx_feature="psol"
+ngx_feature_name=""
+ngx_feature_run=no
+ngx_feature_incs="
+#include \"net/instaweb/htmlparse/public/html_parse.h\"
+#include \"net/instaweb/htmlparse/public/html_writer_filter.h\"
+#include \"net/instaweb/util/public/string.h\"
+#include \"net/instaweb/util/public/string_writer.h\"
+#include \"net/instaweb/util/public/null_message_handler.h\"
+"
+
 os_name='unknown_os'
 arch_name='unknown_arch'
 uname_os=`uname`
@@ -99,31 +113,6 @@ if [ "$WNO_ERROR" = "YES" ]; then
     CFLAGS="$CFLAGS -Wno-error"
 fi
 
-psol_binary="${PSOL_BINARY:-unset}"
-if [ "$psol_binary" = "unset" ] ; then
-  if $build_from_source ; then
-    psol_binary="\
-        $mod_pagespeed_dir/net/instaweb/automatic/pagespeed_automatic.a"
-  else
-    psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
-    psol_binary="$psol_library_dir/pagespeed_automatic.a"
-  fi
-fi
-
-echo "mod_pagespeed_dir=$mod_pagespeed_dir"
-echo "build_from_source=$build_from_source"
-
-ngx_feature="psol"
-ngx_feature_name=""
-ngx_feature_run=no
-ngx_feature_incs="
-#include \"pagespeed/kernel/base/string.h\"
-#include \"pagespeed/kernel/base/string_writer.h\"
-#include \"pagespeed/kernel/base/null_message_handler.h\"
-#include \"pagespeed/kernel/html/html_parse.h\"
-#include \"pagespeed/kernel/html/html_writer_filter.h\"
-"
-
 pagespeed_include="\
   $mod_pagespeed_dir \
   $mod_pagespeed_dir/third_party/chromium/src \
@@ -139,7 +128,15 @@ pagespeed_include="\
   $mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include"
 ngx_feature_path="$pagespeed_include"
 
-pagespeed_libs="-lstdc++ $psol_binary -lrt -pthread -lm"
+if $build_from_source ; then
+  psol_library_binaries="\
+    $mod_pagespeed_dir/net/instaweb/automatic/pagespeed_automatic.a"
+else
+  psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
+  psol_library_binaries="$psol_library_dir/pagespeed_automatic.a"
+fi
+
+pagespeed_libs="-lstdc++ $psol_library_binaries -lrt -pthread -lm"
 ngx_feature_libs="$pagespeed_libs"
 ngx_feature_test="
   GoogleString output_buffer;
@@ -170,7 +167,6 @@ if [ $ngx_found = yes ]; then
     $ps_src/ngx_base_fetch.h \
     $ps_src/ngx_caching_headers.h \
     $ps_src/ngx_fetch.h \
-    $ps_src/ngx_gzip_setter.h \
     $ps_src/ngx_list_iterator.h \
     $ps_src/ngx_message_handler.h \
     $ps_src/ngx_pagespeed.h \
@@ -178,13 +174,12 @@ if [ $ngx_found = yes ]; then
     $ps_src/ngx_rewrite_options.h \
     $ps_src/ngx_server_context.h \
     $ps_src/ngx_url_async_fetcher.h \
-    $psol_binary"
+    $psol_library_binaries"
   NGX_ADDON_SRCS="$NGX_ADDON_SRCS \
     $ps_src/log_message_handler.cc \
     $ps_src/ngx_base_fetch.cc \
     $ps_src/ngx_caching_headers.cc \
     $ps_src/ngx_fetch.cc \
-    $ps_src/ngx_gzip_setter.cc \
     $ps_src/ngx_list_iterator.cc \
     $ps_src/ngx_message_handler.cc \
     $ps_src/ngx_pagespeed.cc \
@@ -205,7 +200,7 @@ if [ $ngx_found = yes ]; then
 else
   cat << END
 $0: error: module ngx_pagespeed requires the pagespeed optimization library.
-Look in objs/autoconf.err for more details.
+Look in obj/autoconf.err for more details.
 END
   exit 1
 fi

src/ngx_base_fetch.cc

@@ -39,7 +39,7 @@ NgxBaseFetch::NgxBaseFetch(ngx_http_request_t* r, int pipe_fd,
       last_buf_sent_(false),
       pipe_fd_(pipe_fd),
       references_(2),
-      ipro_lookup_(false),
+      handle_error_(true),
       preserve_caching_headers_(preserve_caching_headers) {
   if (pthread_mutex_init(&mutex_, NULL)) CHECK(0);
 }
@@ -137,20 +137,14 @@ void NgxBaseFetch::HandleHeadersComplete() {
   int status_code = response_headers()->status_code();
   bool status_ok = (status_code != 0) && (status_code < 400);
 
-  if (!ipro_lookup_ || status_ok) {
+  if (status_ok || handle_error_) {
     // If this is a 404 response we need to count it in the stats.
     if (response_headers()->status_code() == HttpStatus::kNotFound) {
       server_context_->rewrite_stats()->resource_404_count()->Add(1);
     }
   }
 
-  // For the IPRO lookup, supress notification of the nginx side here.
-  // If we send both this event and the one from done, nasty stuff will happen
-  // if we loose the race with with the nginx side destructing this base fetch
-  // instance (and thereby clearing the byte and its pending extraneous event.
-  if (!ipro_lookup_) {
-    RequestCollection();  // Headers available.
-  }
+  RequestCollection();  // Headers available.
 }
 
 bool NgxBaseFetch::HandleFlush(MessageHandler* handler) {

src/ngx_base_fetch.h

@@ -79,7 +79,7 @@ class NgxBaseFetch : public AsyncFetch {
 
   // Called by nginx when it's done with us.
   void Release();
-  void set_ipro_lookup(bool x) { ipro_lookup_ = x; }
+  void set_handle_error(bool x) { handle_error_ = x; }
 
  private:
   virtual bool HandleWrite(const StringPiece& sp, MessageHandler* handler);
@@ -117,7 +117,7 @@ class NgxBaseFetch : public AsyncFetch {
   // decremented once when Done() is called and once when Release() is called.
   int references_;
   pthread_mutex_t mutex_;
-  bool ipro_lookup_;
+  bool handle_error_;
   PreserveCachingHeaders preserve_caching_headers_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxBaseFetch);

src/ngx_fetch.h

@@ -16,15 +16,6 @@
 
 // Author: x.dinic@gmail.com (Junmin Xiong)
 //
-// PageSpeed needs some way to talk to the internet and request resources.  For
-// example, if it's optimizing www.example.com/index.html and it sees html with
-// <img src="//images.example.com/cat.jpg"> and images.example.com is authorized
-// for rewriting in the config, then it needs to fetch cat.jpg from
-// images.example.com and optimize it.  In apache (always) and nginx (by
-// default) we use a fetcher called "serf".  This works fine, but it does run
-// its own event loop.  To be more efficient, this is a "native" fetcher that
-// uses nginx's event loop.
-//
 // The fetch is started by the main thread. It will fetch the remote resource
 // from the specific url asynchronously.
 
@@ -51,13 +42,12 @@ namespace net_instaweb {
 typedef bool (*response_handler_pt)(ngx_connection_t* c);
 
 class NgxUrlAsyncFetcher;
-class NgxConnection;
-
 class NgxFetch : public PoolElement<NgxFetch> {
  public:
   NgxFetch(const GoogleString& url,
            AsyncFetch* async_fetch,
            MessageHandler* message_handler,
+           ngx_msec_t timeout_ms,
            ngx_log_t* log);
   ~NgxFetch();
 
@@ -113,19 +103,19 @@ class NgxFetch : public PoolElement<NgxFetch> {
     response_handler = handler;
   }
   // Only the Static functions could be used in callbacks.
-  static void ResolveDoneHandler(ngx_resolver_ctx_t* ctx);
+  static void NgxFetchResolveDone(ngx_resolver_ctx_t* ctx);
   // Write the request.
-  static void ConnectionWriteHandler(ngx_event_t* wev);
+  static void NgxFetchWrite(ngx_event_t* wev);
   // Wait for the response.
-  static void ConnectionReadHandler(ngx_event_t* rev);
+  static void NgxFetchRead(ngx_event_t* rev);
   // Read and parse the first status line.
-  static bool HandleStatusLine(ngx_connection_t* c);
+  static bool NgxFetchHandleStatusLine(ngx_connection_t* c);
   // Read and parse the HTTP headers.
-  static bool HandleHeader(ngx_connection_t* c);
+  static bool NgxFetchHandleHeader(ngx_connection_t* c);
   // Read the response body.
-  static bool HandleBody(ngx_connection_t* c);
+  static bool NgxFetchHandleBody(ngx_connection_t* c);
   // Cancel the fetch when it's timeout.
-  static void TimeoutHandler(ngx_event_t* tev);
+  static void NgxFetchTimeout(ngx_event_t* tev);
 
   // Add the pagespeed User-Agent.
   void FixUserAgent();
@@ -140,6 +130,7 @@ class NgxFetch : public PoolElement<NgxFetch> {
   int64 bytes_received_;
   int64 fetch_start_ms_;
   int64 fetch_end_ms_;
+  int64 timeout_ms_;
   bool done_;
   int64 content_length_;
   bool content_length_known_;
@@ -152,7 +143,7 @@ class NgxFetch : public PoolElement<NgxFetch> {
   ngx_http_request_t* r_;
   ngx_http_status_t* status_;
   ngx_event_t* timeout_event_;
-  NgxConnection* connection_;
+  ngx_connection_t* connection_;
   ngx_resolver_ctx_t* resolver_ctx_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxFetch);

src/ngx_gzip_setter.cc

@@ -1,403 +0,0 @@
-/*
- * Copyright 2014 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: kspoelstra@we-amp.com (Kees Spoelstra)
-
-#include "ngx_gzip_setter.h"
-
-#include <ngx_conf_file.h>
-
-namespace net_instaweb {
-
-NgxGZipSetter g_gzip_setter;
-
-extern "C" {
-  // These functions replace the setters for:
-  //   gzip
-  //   gzip_types
-  //   gzip_http_version
-  //   gzip_vary
-  //
-  // If these functions are called it means there is an explicit gzip
-  // configuration. The gzip configuration set by pagespeed is then rolled
-  // back and pagespeed will stop enabling gzip automatically.
-  char* ngx_gzip_redirect_conf_set_flag_slot(
-      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
-    if (g_gzip_setter.enabled()) {
-      g_gzip_setter.RollBackAndDisable(cf);
-    }
-    char* ret = ngx_conf_set_flag_slot(cf, cmd, conf);
-    return ret;
-  }
-
-  char* ngx_gzip_redirect_http_types_slot(
-      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
-    if (g_gzip_setter.enabled()) {
-      g_gzip_setter.RollBackAndDisable(cf);
-    }
-    char* ret = ngx_http_types_slot(cf, cmd, conf);
-    return ret;
-  }
-
-  char* ngx_gzip_redirect_conf_set_enum_slot(
-      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
-    if (g_gzip_setter.enabled()) {
-      g_gzip_setter.RollBackAndDisable(cf);
-    }
-    char* ret = ngx_conf_set_enum_slot(cf, cmd, conf);
-    return ret;
-  }
-  char* ngx_gzip_redirect_conf_set_bitmask_slot(
-      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
-    if (g_gzip_setter.enabled()) {
-      g_gzip_setter.RollBackAndDisable(cf);
-    }
-    char* ret = ngx_conf_set_bitmask_slot(cf, cmd, conf);
-    return ret;
-  }
-}
-
-NgxGZipSetter::NgxGZipSetter() : enabled_(0) { }
-NgxGZipSetter::~NgxGZipSetter() { }
-
-// Helper functions to determine signature.
-bool HasLocalConfig(ngx_command_t* command) {
-  return (!(command->type & (NGX_DIRECT_CONF|NGX_MAIN_CONF)) &&
-          command->conf == NGX_HTTP_LOC_CONF_OFFSET);
-}
-bool IsNgxFlagCommand(ngx_command_t* command) {
-  return (command->set == ngx_conf_set_flag_slot &&
-          HasLocalConfig(command));
-}
-bool IsNgxHttpTypesCommand(ngx_command_t* command) {
-  return (command->set == ngx_http_types_slot &&
-          HasLocalConfig(command));
-}
-bool IsNgxEnumCommand(ngx_command_t* command) {
-  return (command->set == ngx_conf_set_enum_slot &&
-          HasLocalConfig(command));
-}
-bool IsNgxBitmaskCommand(ngx_command_t* command) {
-  return (command->set == ngx_conf_set_bitmask_slot &&
-          HasLocalConfig(command));
-}
-
-// Initialize the NgxGzipSetter.
-// Find the gzip, gzip_vary, gzip_http_version and gzip_types commands in the
-// gzip module. Enable if the signature of the zip command matches with what we
-// trust. Also sets up redirects for the configurations. These redirect handle
-// a rollback if expicit configuration is found.
-// If commands are not found the method will inform the user by logging.
-void NgxGZipSetter::Init(ngx_conf_t* cf) {
-#if (NGX_HTTP_GZIP)
-  bool gzip_signature_mismatch = false;
-  bool other_signature_mismatch = false;
-  for (int m = 0; ngx_modules[m] != NULL; m++) {
-    if (ngx_modules[m]->commands != NULL) {
-      for (int c = 0; ngx_modules[m]->commands[c].name.len; c++) {
-        ngx_command_t* current_command =& ngx_modules[m]->commands[c];
-
-        // We look for the gzip command, and the exact signature we trust
-        // this means configured as an config location offset
-        // and a ngx_flag_t setter.
-        // Also see:
-        //   ngx_conf_handler in ngx_conf_file.c
-        //   ngx_http_gzip_filter_commands in ngx_http_gzip_filter.c
-        if (gzip_command_.command_ == NULL &&
-            STR_EQ_LITERAL(current_command->name, "gzip")) {
-          if (IsNgxFlagCommand(current_command)) {
-            current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
-            gzip_command_.command_ = current_command;
-            gzip_command_.module_ = ngx_modules[m];
-            enabled_ = 1;
-          } else {
-            ngx_conf_log_error(
-                NGX_LOG_WARN, cf, 0,
-                "pagespeed: cannot set gzip, signature mismatch");
-            gzip_signature_mismatch = true;
-          }
-        }
-
-        if (!gzip_http_version_command_.command_ &&
-            STR_EQ_LITERAL(current_command->name, "gzip_http_version")) {
-          if (IsNgxEnumCommand(current_command)) {
-            current_command->set = ngx_gzip_redirect_conf_set_enum_slot;
-            gzip_http_version_command_.command_ = current_command;
-            gzip_http_version_command_.module_ = ngx_modules[m];
-          } else {
-            ngx_conf_log_error(
-                NGX_LOG_WARN, cf, 0,
-                "pagespeed: cannot set gzip_http_version, signature mismatch");
-            other_signature_mismatch = true;
-          }
-        }
-
-        if (!gzip_proxied_command_.command_ &&
-            STR_EQ_LITERAL(current_command->name, "gzip_proxied")) {
-          if (IsNgxBitmaskCommand(current_command)) {
-            current_command->set = ngx_gzip_redirect_conf_set_bitmask_slot;
-            gzip_proxied_command_.command_ = current_command;
-            gzip_proxied_command_.module_ = ngx_modules[m];
-          } else {
-            ngx_conf_log_error(
-                NGX_LOG_WARN, cf, 0,
-                "pagespeed: cannot set gzip_proxied, signature mismatch");
-            other_signature_mismatch = true;
-          }
-        }
-
-        if (!gzip_http_types_command_.command_ &&
-            STR_EQ_LITERAL(current_command->name, "gzip_types")) {
-          if (IsNgxHttpTypesCommand(current_command)) {
-            current_command->set = ngx_gzip_redirect_http_types_slot;
-            gzip_http_types_command_.command_ = current_command;
-            gzip_http_types_command_.module_ = ngx_modules[m];
-          } else {
-            ngx_conf_log_error(
-                NGX_LOG_WARN, cf, 0,
-                "pagespeed: cannot set gzip_types, signature mismatch");
-            other_signature_mismatch = true;
-          }
-        }
-
-        if (!gzip_vary_command_.command_ &&
-            STR_EQ_LITERAL(current_command->name, "gzip_vary")) {
-          if (IsNgxFlagCommand(current_command)) {
-            current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
-            gzip_vary_command_.command_ = current_command;
-            gzip_vary_command_.module_ = ngx_modules[m];
-          } else {
-            ngx_conf_log_error(
-                NGX_LOG_WARN, cf, 0,
-                "pagespeed: cannot set gzip_vary, signature mismatch");
-            other_signature_mismatch = true;
-          }
-        }
-      }
-    }
-  }
-  if (gzip_signature_mismatch) {
-    return;  // Already logged error.
-  } else if (!enabled_) {
-    // Looked through all the available commands and didn't find the "gzip" one.
-    ngx_conf_log_error(
-        NGX_LOG_WARN, cf, 0, "pagespeed: cannot set gzip, command not found");
-    return;
-  } else if (other_signature_mismatch) {
-    return;  // Already logged error.
-  } else if (!gzip_vary_command_.command_) {
-    ngx_conf_log_error(
-        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_vary");
-    return;
-  } else if (!gzip_http_types_command_.command_) {
-    ngx_conf_log_error(
-        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_types");
-    return;
-  } else if (!gzip_http_version_command_.command_) {
-    ngx_conf_log_error(
-        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_http_version");
-    return;
-  } else if (!gzip_proxied_command_.command_) {
-    ngx_conf_log_error(
-        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_proxied");
-    return;
-  } else {
-    return;  // Success.
-  }
-#else
-  ngx_conf_log_error(
-      NGX_LOG_WARN, cf, 0, "pagespeed: gzip not compiled into nginx");
-  return;
-#endif
-}
-
-void* ngx_command_ctx::GetConfPtr(ngx_conf_t* cf) {
-  return GetModuleConfPtr(cf) + command_->offset;
-}
-
-char* ngx_command_ctx::GetModuleConfPtr(ngx_conf_t* cf) {
-  return reinterpret_cast<char*>(
-      ngx_http_conf_get_module_loc_conf(cf, (*(module_))));
-}
-
-void NgxGZipSetter::SetNgxConfFlag(ngx_conf_t* cf,
-                                   ngx_command_ctx* command_ctx,
-                                   ngx_flag_t value) {
-  ngx_flag_t* flag = reinterpret_cast<ngx_flag_t*>(command_ctx->GetConfPtr(cf));
-  *flag = value;
-  // Save the flag position for possible rollback.
-  ngx_flags_set_.push_back(flag);
-}
-
-void NgxGZipSetter::SetNgxConfEnum(ngx_conf_t* cf,
-                                   ngx_command_ctx* command_ctx,
-                                   ngx_uint_t value) {
-  ngx_uint_t* enum_to_set =
-      reinterpret_cast<ngx_uint_t*>(command_ctx->GetConfPtr(cf));
-  *enum_to_set = value;
-  ngx_uint_set_.push_back(enum_to_set);
-}
-
-void NgxGZipSetter::SetNgxConfBitmask(ngx_conf_t* cf,
-                                      ngx_command_ctx* command_ctx,
-                                      ngx_uint_t value) {
-  ngx_uint_t* enum_to_set =
-      reinterpret_cast<ngx_uint_t*>(command_ctx->GetConfPtr(cf));
-  *enum_to_set = value;
-  ngx_uint_set_.push_back(enum_to_set);
-}
-
-// These are the content types we want to compress.
-ngx_str_t gzip_http_types[] = {
-  ngx_string("application/ecmascript"),
-  ngx_string("application/javascript"),
-  ngx_string("application/json"),
-  ngx_string("application/pdf"),
-  ngx_string("application/postscript"),
-  ngx_string("application/x-javascript"),
-  ngx_string("image/svg+xml"),
-  ngx_string("text/css"),
-  ngx_string("text/csv"),
-  // ngx_string("text/html"),  // This is the default implied value.
-  ngx_string("text/javascript"),
-  ngx_string("text/plain"),
-  ngx_string("text/xml"),
-  ngx_null_string  // Indicates end of array.
-};
-
-gzs_enable_result NgxGZipSetter::SetGZipForLocation(ngx_conf_t* cf,
-                                                    bool value) {
-  if (!enabled_) {
-    return kEnableGZipNotEnabled;
-  }
-  if (gzip_command_.command_) {
-    SetNgxConfFlag(cf, &gzip_command_, value);
-  }
-  return kEnableGZipOk;
-}
-
-void NgxGZipSetter::EnableGZipForLocation(ngx_conf_t* cf) {
-  if (!enabled_) {
-    return;
-  }
-
-  // When we get called twice for the same location{}, we ignore the second call
-  // to prevent adding duplicate gzip http types and so on.
-  ngx_flag_t* flag =
-      reinterpret_cast<ngx_flag_t*>(gzip_command_.GetConfPtr(cf));
-  if (*flag == 1) {
-    return;
-  }
-  SetGZipForLocation(cf, true);
-  if (gzip_vary_command_.command_) {
-    SetNgxConfFlag(cf, &gzip_vary_command_, 1);
-  }
-  if (gzip_http_version_command_.command_) {
-    SetNgxConfEnum(cf, &gzip_http_version_command_, NGX_HTTP_VERSION_10);
-  }
-  if (gzip_proxied_command_.command_) {
-    SetNgxConfBitmask(
-        cf, &gzip_http_version_command_, NGX_HTTP_GZIP_PROXIED_ANY);
-  }
-
-  // This is actually the most prone to future API changes, because gzip_types
-  // is not a simple type like ngx_flag_t. The signature check should be enough
-  // to prevent problems.
-  AddGZipHTTPTypes(cf);
-  return;
-}
-
-void NgxGZipSetter::AddGZipHTTPTypes(ngx_conf_t* cf) {
-  if (gzip_http_types_command_.command_) {
-    // Following should not happen, but if it does return gracefully.
-    if (cf->args->nalloc < 2) {
-      ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
-                         "pagespeed: unexpected small cf->args in gzip_types");
-      return;
-    }
-
-    ngx_command_t* command = gzip_http_types_command_.command_;
-    char* gzip_conf = reinterpret_cast<char* >(
-        gzip_http_types_command_.GetModuleConfPtr(cf));
-
-    // Backup the old settings.
-    ngx_str_t old_elt0 = reinterpret_cast<ngx_str_t*>(cf->args->elts)[0];
-    ngx_str_t old_elt1 = reinterpret_cast<ngx_str_t*>(cf->args->elts)[1];
-    ngx_uint_t old_nelts = cf->args->nelts;
-
-    // Setup first arg.
-    ngx_str_t gzip_types_string = ngx_string("gzip_types");
-    reinterpret_cast<ngx_str_t*>(cf->args->elts)[0] = gzip_types_string;
-    cf->args->nelts = 2;
-
-    ngx_str_t* http_types = gzip_http_types;
-    while (http_types->data) {
-      ngx_str_t d;
-      // We allocate the http type on the configuration pool and actually
-      // leak this if we rollback. This does not seem to be a big problem,
-      // because nginx also allocates tokens in ngx_conf_file.c and does not
-      // free them. This way they can be used safely by configurations.
-      // We must use a copy of gzip_http_types array here because nginx will
-      // manipulate the values.
-      // TODO(kspoelstra): better would be to allocate once on init and not
-      // every time we enable gzip. This needs further investigation, sharing
-      // tokens might be problematic.
-      // For now I think it is not a large problem. This might add up in case
-      // of a large multi server/location config with a lot of "pagespeed on"
-      // directives.
-      // Estimates are 300-400KB for 1000 times "pagespeed on".
-      d.data = reinterpret_cast<u_char*>(
-          ngx_pnalloc(cf->pool, http_types->len + 1));
-      snprintf(reinterpret_cast<char*>(d.data), http_types->len + 1, "%s",
-               reinterpret_cast<const char*>(http_types->data));
-      d.len = http_types->len;
-      reinterpret_cast<ngx_str_t*>(cf->args->elts)[1] = d;
-      // Call the original setter.
-      ngx_http_types_slot(cf, command, gzip_conf);
-      http_types++;
-    }
-
-    // Restore args.
-    cf->args->nelts = old_nelts;
-    reinterpret_cast<ngx_str_t*>(cf->args->elts)[1] = old_elt1;
-    reinterpret_cast<ngx_str_t*>(cf->args->elts)[0] = old_elt0;
-
-    // Backup configuration location for rollback.
-    ngx_httptypes_set_.push_back(gzip_conf + command->offset);
-  }
-}
-
-void NgxGZipSetter::RollBackAndDisable(ngx_conf_t* cf) {
-  ngx_conf_log_error(NGX_LOG_INFO, cf, 0,
-                     "pagespeed: rollback gzip, explicit configuration");
-  for (std::vector<ngx_flag_t*>::iterator i = ngx_flags_set_.begin();
-       i != ngx_flags_set_.end(); ++i) {
-    *(*i)=NGX_CONF_UNSET;
-  }
-  for (std::vector<ngx_uint_t*>::iterator i = ngx_uint_set_.begin();
-       i != ngx_uint_set_.end(); ++i) {
-    *(*i)=NGX_CONF_UNSET_UINT;
-  }
-  for (std::vector<void*>::iterator i = ngx_httptypes_set_.begin();
-       i != ngx_httptypes_set_.end(); ++i) {
-    ngx_array_t** type_array = reinterpret_cast<ngx_array_t**>(*i);
-    ngx_array_destroy(*type_array);
-    *type_array = NULL;
-  }
-  enabled_ = 0;
-}
-
-}  // namespace net_instaweb

src/ngx_gzip_setter.h

@@ -1,124 +0,0 @@
-/*
- * Copyright 2014 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: kspoelstra@we-amp.com (Kees Spoelstra)
-
-/*
- * NgxGZipSetter sets up gzip for pagespeed
- * with the following configuration:
- * gzip  on;
- * gzip_vary on;
- * gzip_types application/ecmascript;
- * gzip_types application/javascript;
- * gzip_types application/json;
- * gzip_types application/pdf;
- * gzip_types application/postscript;
- * gzip_types application/x-javascript;
- * gzip_types image/svg+xml;
- * gzip_types text/css;
- * gzip_types text/csv;
- * gzip_types text/javascript;
- * gzip_types text/plain;
- * gzip_types text/xml;
- * gzip_http_version 1.0;
- *
- * If there is an explicit gzip configuration in the nginx.conf
- * pagespeed will rollback the set configuration and let the
- * user decide what the configuration will be.
- *
- * It manipulates the configuration by manipulating ngx_flag_t
- * and ngx_uint_t settings directly and using the nginx setter for
- * gzip_http_types.
- * This is probably a safe way to do it. If this mechanism
- * changes all non nginx module setup & configuration will
- * fail.
- */
-
-#ifndef NGX_GZIP_SETTER_H_
-#define NGX_GZIP_SETTER_H_
-
-extern "C" {
-#include <ngx_config.h>
-#include <ngx_core.h>
-#include <ngx_http.h>
-}
-
-#include <vector>
-
-#include "ngx_pagespeed.h"
-
-#include "net/instaweb/util/public/basictypes.h"
-
-namespace net_instaweb {
-
-// We need this class because configuration for gzip is in different modules, so
-// just saving the command will not work.
-class ngx_command_ctx {
- public:
-  ngx_command_ctx() : command_(NULL), module_(NULL) { }
-  void* GetConfPtr(ngx_conf_t* cf);
-  char* GetModuleConfPtr(ngx_conf_t* cf);
-  ngx_command_t* command_;
-  ngx_module_t* module_;
-};
-
-enum gzs_enable_result {
-  kEnableGZipOk,
-  kEnableGZipPartial,
-  kEnableGZipNotEnabled
-};
-
-class NgxGZipSetter {
-  std::vector<ngx_flag_t*> ngx_flags_set_;
-  std::vector<ngx_uint_t*> ngx_uint_set_;
-  std::vector<void*> ngx_httptypes_set_;
-  ngx_command_ctx gzip_command_;
-  ngx_command_ctx gzip_http_types_command_;
-  ngx_command_ctx gzip_proxied_command_;
-  ngx_command_ctx gzip_vary_command_;
-  ngx_command_ctx gzip_http_version_command_;
-  bool enabled_;
-
- public:
-  NgxGZipSetter();
-  ~NgxGZipSetter();
-  void Init(ngx_conf_t* cf);
-
-  void SetNgxConfFlag(ngx_conf_t* cf,
-                      ngx_command_ctx* command_ctx,
-                      ngx_flag_t value);
-  void SetNgxConfEnum(ngx_conf_t* cf,
-                      ngx_command_ctx* command_ctx,
-                      ngx_uint_t value);
-  void SetNgxConfBitmask(ngx_conf_t* cf,
-                         ngx_command_ctx* command_ctx,
-                         ngx_uint_t value);
-  void EnableGZipForLocation(ngx_conf_t* cf);
-  gzs_enable_result SetGZipForLocation(ngx_conf_t* cf, bool value);
-  void AddGZipHTTPTypes(ngx_conf_t* cf);
-  void RollBackAndDisable(ngx_conf_t* cf);
-
-  bool enabled() { return enabled_; }
-
- private:
-  DISALLOW_COPY_AND_ASSIGN(NgxGZipSetter);
-};
-
-extern NgxGZipSetter g_gzip_setter;
-
-}  // namespace net_instaweb
-
-#endif  // NGX_GZIP_SETTER_H_

src/ngx_message_handler.cc

@@ -56,9 +56,11 @@ extern "C" {
 
 namespace net_instaweb {
 
-NgxMessageHandler::NgxMessageHandler(Timer* timer, AbstractMutex* mutex)
-    : SystemMessageHandler(timer, mutex),
+NgxMessageHandler::NgxMessageHandler(AbstractMutex* mutex)
+    : mutex_(mutex),
+      buffer_(NULL),
       log_(NULL) {
+  SetPidString(static_cast<int64>(getpid()));
 }
 
 // Installs a signal handler for common crash signals, that tries to print
@@ -71,6 +73,14 @@ void NgxMessageHandler::InstallCrashHandler(ngx_log_t* log) {
   signal(SIGSEGV, signal_handler);
 }
 
+bool NgxMessageHandler::Dump(Writer* writer) {
+  // Can't dump before SharedCircularBuffer is set up.
+  if (buffer_ == NULL) {
+    return false;
+  }
+  return buffer_->Dump(writer, &handler_);
+}
+
 ngx_uint_t NgxMessageHandler::GetNgxLogLevel(MessageType type) {
   switch (type) {
     case kInfo:
@@ -88,6 +98,11 @@ ngx_uint_t NgxMessageHandler::GetNgxLogLevel(MessageType type) {
   return NGX_LOG_ALERT;
 }
 
+void NgxMessageHandler::set_buffer(SharedCircularBuffer* buff) {
+  ScopedMutex lock(mutex_.get());
+  buffer_ = buff;
+}
+
 void NgxMessageHandler::MessageVImpl(MessageType type, const char* msg,
                                      va_list args) {
   ngx_uint_t log_level = GetNgxLogLevel(type);
@@ -98,8 +113,25 @@ void NgxMessageHandler::MessageVImpl(MessageType type, const char* msg,
   } else {
     GoogleMessageHandler::MessageVImpl(type, msg, args);
   }
+
   // Prepare a log message for the SharedCircularBuffer only.
-  AddMessageToBuffer(type, formatted_message);
+  // Prepend time and severity to message.
+  // Format is [time] [severity] [pid] message.
+  GoogleString message;
+  GoogleString time;
+  PosixTimer timer;
+  if (!ConvertTimeToString(timer.NowMs(), &time)) {
+    time = "?";
+  }
+  StrAppend(&message, "[", time, "] ",
+            "[", MessageTypeToString(type), "] ");
+  StrAppend(&message, pid_string_, " ", formatted_message, "\n");
+  {
+    ScopedMutex lock(mutex_.get());
+    if (buffer_ != NULL) {
+      buffer_->Write(message);
+    }
+  }
 }
 
 void NgxMessageHandler::FileMessageVImpl(MessageType type, const char* file,

src/ngx_message_handler.h

@@ -24,30 +24,44 @@ extern "C" {
 
 #include <cstdarg>
 
-#include "net/instaweb/system/public/system_message_handler.h"
 #include "net/instaweb/util/public/basictypes.h"
+#include "net/instaweb/util/public/google_message_handler.h"
 #include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/util/public/scoped_ptr.h"
 #include "net/instaweb/util/public/string.h"
 #include "net/instaweb/util/public/string_util.h"
 
 namespace net_instaweb {
 
 class AbstractMutex;
+class SharedCircularBuffer;
 class Timer;
+class Writer;
 
 // Implementation of a message handler that uses ngx_log_error()
 // logging to emit messages, with a fallback to GoogleMessageHandler
-class NgxMessageHandler : public SystemMessageHandler {
+class NgxMessageHandler : public GoogleMessageHandler {
  public:
-  explicit NgxMessageHandler(Timer* timer, AbstractMutex* mutex);
+  explicit NgxMessageHandler(AbstractMutex* mutex);
 
   // Installs a signal handler for common crash signals that tries to print
   // out a backtrace.
   static void InstallCrashHandler(ngx_log_t* log);
 
+  // When NgxRewriteDriver instantiates the NgxMessageHandlers, the
+  // SharedCircularBuffer and ngx_log_t are not available yet. These
+  // will later be set in RootInit/Childinit
+  // Messages logged before that will be passed on to handler_;
+  void set_buffer(SharedCircularBuffer* buff);
   void set_log(ngx_log_t* log) { log_ = log; }
   ngx_log_t* log() { return log_; }
 
+  void SetPidString(const int64 pid) {
+    pid_string_ = StrCat("[", Integer64ToString(pid), "]");
+  }
+  // Dump contents of SharedCircularBuffer.
+  bool Dump(Writer* writer);
+
  protected:
   virtual void MessageVImpl(MessageType type, const char* msg, va_list args);
 
@@ -56,6 +70,13 @@ class NgxMessageHandler : public SystemMessageHandler {
 
  private:
   ngx_uint_t GetNgxLogLevel(MessageType type);
+
+  scoped_ptr<AbstractMutex> mutex_;
+  GoogleString pid_string_;
+  // handler_ is used as a fallback when we can not use ngx_log_errort
+  // It's also used when calling Dump on the internal SharedCircularBuffer
+  GoogleMessageHandler handler_;
+  SharedCircularBuffer* buffer_;
   ngx_log_t* log_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxMessageHandler);

src/ngx_pagespeed.cc

@@ -30,7 +30,6 @@
 
 #include "ngx_base_fetch.h"
 #include "ngx_caching_headers.h"
-#include "ngx_gzip_setter.h"
 #include "ngx_list_iterator.h"
 #include "ngx_message_handler.h"
 #include "ngx_rewrite_driver_factory.h"
@@ -78,10 +77,17 @@
 
 extern ngx_module_t ngx_pagespeed;
 
+// Hacks for debugging.
+#define DBG(r, args...)                                       \
+  ngx_log_error(NGX_LOG_DEBUG, (r)->connection->log, 0, args)
+#define PDBG(ctx, args...)                                       \
+  ngx_log_error(NGX_LOG_DEBUG, (ctx)->r->connection->log, 0, args)
+#define CDBG(cf, args...)                                     \
+  ngx_conf_log_error(NGX_LOG_DEBUG, cf, 0, args)
+
 // Unused flag, see
 // http://lxr.evanmiller.org/http/source/http/ngx_http_request.h#L130
 #define  NGX_HTTP_PAGESPEED_BUFFERED 0x08
-#define  POST_BUF_READ_SIZE 65536
 
 // Needed for SystemRewriteDriverFactory to use shared memory.
 #define PAGESPEED_SUPPORT_POSIX_SHARED_MEM
@@ -89,8 +95,6 @@ extern ngx_module_t ngx_pagespeed;
 namespace net_instaweb {
 
 const char* kInternalEtagName = "@psol-etag";
-bool factory_init_called = false;
-
 // The process context takes care of proactively initialising
 // a few libraries for us, some of which are not thread-safe
 // when they are initialized lazily.
@@ -101,10 +105,6 @@ StringPiece str_to_string_piece(ngx_str_t s) {
   return StringPiece(reinterpret_cast<char*>(s.data), s.len);
 }
 
-// Nginx uses memory pools, like Apache, allocating strings a request needs from
-// the pool and then throwing it all away when the request finishes.  Use this
-// when you need to pass a short string to nginx and want it to take ownership
-// of the string.
 char* string_piece_to_pool_string(ngx_pool_t* pool, StringPiece sp) {
   // Need space for the final null.
   ngx_uint_t buffer_size = sp.size() + 1;
@@ -119,10 +119,6 @@ char* string_piece_to_pool_string(ngx_pool_t* pool, StringPiece sp) {
   return s;
 }
 
-// When passing the body of http responses between filters Nginx uses a linked
-// list of buffers ("buffer chain"), again like Apache.  This constructs one of
-// those lists from a StringPiece.  This is what you use when you need to pass a
-// (potentially) longer string to nginx and want it to take ownership.
 ngx_int_t string_piece_to_buffer_chain(
     ngx_pool_t* pool, StringPiece sp, ngx_chain_t** link_ptr,
     bool send_last_buf) {
@@ -206,12 +202,9 @@ ngx_int_t string_piece_to_buffer_chain(
   return NGX_OK;
 }
 
+// modified from NgxBaseFetch::CopyHeadersFromTable()
 namespace {
 
-// Setting headers in nginx is tricky because it's not just a matter of adding
-// them to a list.  You also need to remove them if there's already one there,
-// as well as setting the shortcut pointers (both upper case and lower case).
-//
 // Based on ngx_http_add_cache_control.
 ngx_int_t ps_set_cache_control(ngx_http_request_t* r, char* cache_control) {
   // First strip existing cache-control headers.
@@ -227,7 +220,7 @@ ngx_int_t ps_set_cache_control(ngx_http_request_t* r, char* cache_control) {
   // Now add our new cache control header.
   if (r->headers_out.cache_control.elts == NULL) {
     ngx_int_t rc = ngx_array_init(&r->headers_out.cache_control, r->pool,
-                                  1, sizeof(ngx_table_elt_t*));
+                                  1, sizeof(ngx_table_elt_t *));
     if (rc != NGX_OK) {
       return NGX_ERROR;
     }
@@ -288,6 +281,11 @@ void copy_response_headers_from_ngx(const ngx_http_request_t* r,
 
   headers->set_status_code(r->headers_out.status);
 
+  if (r->headers_out.location != NULL) {
+    headers->Add(HttpAttributes::kLocation,
+                 str_to_string_piece(r->headers_out.location->value));
+  }
+
   // Manually copy over the content type because it's not included in
   // request_->headers_out.headers.
   headers->Add(HttpAttributes::kContentType,
@@ -338,7 +336,7 @@ ngx_int_t copy_response_headers_to_ngx(
       if (StringCaseEqual(name_gs, "Cache-Control")) {
         continue;
       }
-    }  // else we don't preserve any headers.
+    } // else we don't preserve any headers
 
     ngx_str_t name, value;
 
@@ -393,8 +391,8 @@ ngx_int_t copy_response_headers_to_ngx(
       // content_type_lowcase if we leave it as null.
       headers_out->content_type_lowcase = NULL;
       continue;
-      // TODO(oschaaf): are there any other headers we should not try to
-      // copy here?
+    // TODO(oschaaf): are there any other headers we should not try to
+    // copy here?
     } else if (STR_EQ_LITERAL(name, "Connection")) {
       continue;
     } else if (STR_EQ_LITERAL(name, "Keep-Alive")) {
@@ -486,7 +484,6 @@ enum Response {
   kConsole,
   kMessages,
   kAdmin,
-  kCachePurge,
   kGlobalAdmin,
   kPagespeedSubrequest,
   kNotHeadOrGet,
@@ -537,8 +534,6 @@ void ps_ignore_sigpipe() {
   sigaction(SIGPIPE, &act, NULL);
 }
 
-// Given a directory path that pagespeed needs, create it and set permissions so
-// the worker can access, but only if needed.
 char* ps_init_dir(const StringPiece& directive,
                   const StringPiece& path,
                   ngx_conf_t* cf) {
@@ -587,36 +582,11 @@ char* ps_init_dir(const StringPiece& directive,
   return NULL;
 }
 
-// We support interpretation of nginx variables in some configuration settings,
-// but we also need to support literal dollar signs in those same settings.
-// Nginx has no good solution for this, so we define $dollar to expand to '$',
-// which lets people include literal dollar signs if they need them.
-ngx_int_t ps_dollar(
-    ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) {
-  v->valid = 1;
-  v->no_cacheable = 0;
-  v->not_found = 0;
-  v->data = reinterpret_cast<u_char*>(const_cast<char*>("$"));
-  v->len = 1;
-  return NGX_OK;
-}
-
-// Parse the configuration option represented by cf and add it to options,
-// creating options if necessary.
+#define NGX_PAGESPEED_MAX_ARGS 10
 char* ps_configure(ngx_conf_t* cf,
                    NgxRewriteOptions** options,
                    MessageHandler* handler,
                    net_instaweb::RewriteOptions::OptionScope option_scope) {
-  ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
-      ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
-
-  if (!factory_init_called) {
-    // Init logging to nginx's default error_log.
-    cfg_m->driver_factory->LoggingInit(cf->cycle->log);
-    cfg_m->driver_factory->Init();
-    factory_init_called = true;
-  }
-
   // args[0] is always "pagespeed"; ignore it.
   ngx_uint_t n_args = cf->args->nelts - 1;
 
@@ -631,27 +601,6 @@ char* ps_configure(ngx_conf_t* cf,
     args[i] = str_to_string_piece(value[i+1]);
   }
 
-  if (n_args == 1) {
-    if (StringCaseEqual(args[0], "on")) {
-      // safe to call if the setter is disabled
-      g_gzip_setter.EnableGZipForLocation(cf);
-    } else if (StringCaseEqual(args[0], "off")) {
-      g_gzip_setter.SetGZipForLocation(cf, false);
-    }
-  }
-  if (n_args == 2 && args[0].compare("gzip") == 0) {
-    if (args[1].compare("on") == 0) {
-      g_gzip_setter.SetGZipForLocation(cf, true);
-    } else if (args[1].compare("off") == 0) {
-      g_gzip_setter.SetGZipForLocation(cf, false);
-    } else {
-      char* error_message = string_piece_to_pool_string(
-          cf->pool, StringPiece("Invalid pagespeed gzip setting"));
-      return error_message;
-    }
-    return NGX_CONF_OK;
-  }
-
   // Some options require the worker process to be able to read and write to
   // a specific directory.  Generally the master process is root while the
   // worker is nobody, so we need to change permissions and create the directory
@@ -667,30 +616,14 @@ char* ps_configure(ngx_conf_t* cf,
     // directive yet.  That happens below in ParseAndSetOptions().
   }
 
+  ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
+      ngx_http_cycle_get_module_main_conf(cf->cycle, ngx_pagespeed));
   if (*options == NULL) {
     *options = new NgxRewriteOptions(
         cfg_m->driver_factory->thread_system());
   }
-
-  bool process_script_variables = dynamic_cast<NgxRewriteDriverFactory*>(
-      cfg_m->driver_factory)->process_script_variables();
-
-  if (process_script_variables) {
-    // To be able to use '$', we map '$ps_dollar' to '$' via a script variable.
-    ngx_str_t name = ngx_string("ps_dollar");
-    ngx_http_variable_t* var = ngx_http_add_variable(
-        cf, &name, NGX_HTTP_VAR_CHANGEABLE);
-
-    if (var == NULL) {
-      return const_cast<char*>(
-          "Failed to add global configuration variable for '$ps_dollar'");
-    }
-    var->get_handler = ps_dollar;
-  }
-
   const char* status = (*options)->ParseAndSetOptions(
-      args, n_args, cf->pool, handler, cfg_m->driver_factory, option_scope, cf,
-      process_script_variables);
+      args, n_args, cf->pool, handler, cfg_m->driver_factory, option_scope);
 
   // nginx expects us to return a string literal but doesn't mark it const.
   return const_cast<char*>(status);
@@ -713,8 +646,10 @@ char* ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
 char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
   ps_loc_conf_t* cfg_l = static_cast<ps_loc_conf_t*>(
       ngx_http_conf_get_module_loc_conf(cf, ngx_pagespeed));
-  return ps_configure(cf, &cfg_l->options, cfg_l->handler,
-                      net_instaweb::RewriteOptions::kDirectoryScope);
+
+  return ps_configure(
+      cf, &cfg_l->options, cfg_l->handler,
+      net_instaweb::RewriteOptions::kDirectoryScope);
 }
 
 void ps_cleanup_loc_conf(void* data) {
@@ -735,16 +670,7 @@ void ps_cleanup_srv_conf(void* data) {
   // from being executed
 
   if (!factory_deleted && cfg_s->server_context != NULL) {
-    NgxRewriteDriverFactory* factory = dynamic_cast<NgxRewriteDriverFactory*>(
-      cfg_s->server_context->factory());
-
-    if (!factory_init_called) {
-      factory->LoggingInit(ngx_cycle->log);
-      factory->Init();
-      factory_init_called = true;
-    }
-
-    delete factory;
+    delete cfg_s->server_context->factory();
     factory_deleted = true;
   }
   if (cfg_s->proxy_fetch_factory != NULL) {
@@ -814,7 +740,6 @@ void* ps_create_main_conf(ngx_conf_t* cf) {
       new SystemThreadSystem(),
       "" /* hostname, not used */,
       -1 /* port, not used */);
-  factory_init_called = false;
   ps_set_conf_cleanup_handler(cf, ps_cleanup_main_conf, cfg_m);
   return cfg_m;
 }
@@ -864,16 +789,6 @@ void ps_merge_options(NgxRewriteOptions* parent_options,
     NgxRewriteOptions* child_specific_options = *child_options;
     *child_options = parent_options->Clone();
     (*child_options)->Merge(*child_specific_options);
-
-    if (child_specific_options->clear_inherited_scripts()) {
-      // We don't want to inherit any inherited script lines from the parent
-      // options here, so we just stick to the child specific ones.
-      child_specific_options->CopyScriptLinesTo(*child_options);
-    } else {
-      // We append the child specific script lines to the parent's script lines
-      // so we preserve the order in which they will be executed at request time
-      child_specific_options->AppendScriptLinesTo(*child_options);
-    }
     delete child_specific_options;
   }
 }
@@ -915,9 +830,6 @@ char* ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child) {
   // let it do that, then merge in options we got from the config file.
   // Once we do that we're done with cfg_s->options.
   cfg_s->server_context->global_options()->Merge(*cfg_s->options);
-  NgxRewriteOptions* ngx_options = dynamic_cast<NgxRewriteOptions*>(
-      cfg_s->server_context->global_options());
-  cfg_s->options->CopyScriptLinesTo(ngx_options);
   delete cfg_s->options;
   cfg_s->options = NULL;
 
@@ -1116,6 +1028,7 @@ ngx_int_t ps_decline_request(ngx_http_request_t* r) {
   ctx->fetch_done = false;
   ctx->write_pending = false;
 
+  ps_release_base_fetch(ctx);
   ps_set_buffered(r, false);
 
   r->count++;
@@ -1165,9 +1078,6 @@ ngx_int_t ps_base_fetch_filter(ngx_http_request_t* r, ngx_chain_t* in) {
   return ctx->fetch_done ? NGX_OK : NGX_AGAIN;
 }
 
-// This runs on the nginx event loop in response to seeing the byte PageSpeed
-// sent over the pipe to trigger the nginx-side code.  Copy whatever is ready
-// from PageSpeed out to the browser (headers and/or body).
 ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r) {
   ps_request_ctx_t* ctx = ps_get_request_context(r);
   ngx_int_t rc;
@@ -1192,12 +1102,6 @@ ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r) {
                 STR_CASE_EQ_LITERAL(header->key, "Last-Modified") ||
                 STR_CASE_EQ_LITERAL(header->key, "Expires"))))) {
           header->hash = 0;
-          if (STR_CASE_EQ_LITERAL(header->key, "Location")) {
-            // There's a possible issue with the location header, where setting
-            // the hash to 0 is not enough. See:
-            // https://github.com/nginx/nginx/blob/master/src/http/ngx_http_header_filter_module.c#L314
-            r->headers_out.location = NULL;
-          }
         }
       }
     } else {
@@ -1235,8 +1139,7 @@ ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r) {
   // too much memory in busy servers.
 
   rc = ctx->base_fetch->CollectAccumulatedWrites(&cl);
-  ngx_log_error(NGX_LOG_DEBUG, ctx->r->connection->log, 0,
-                "CollectAccumulatedWrites, %d", rc);
+  PDBG(ctx, "CollectAccumulatedWrites, %d", rc);
 
   if (rc == NGX_ERROR) {
     ps_set_buffered(r, false);
@@ -1264,8 +1167,6 @@ void ps_base_fetch_filter_init() {
 
 }  // namespace
 
-// Do some bookkeeping, cleanup, and error checking to keep the mess out of
-// ps_base_fetch_handler.
 void ps_connection_read_handler(ngx_event_t* ev) {
   CHECK(ev != NULL);
   ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
@@ -1300,15 +1201,6 @@ void ps_connection_read_handler(ngx_event_t* ev) {
     rc = read(c->fd, chr, 256);
   } while (rc > 0 || (rc == -1 && errno == EINTR));  // Retry on EINTR.
 
-  if (r->connection->error) {
-    ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
-                  "pagespeed [%p] request already finalized", r);
-    ctx->pagespeed_connection = NULL;
-    ngx_close_connection(c);
-    ngx_http_finalize_request(r, NGX_ERROR);
-    return;
-  }
-
   if (rc == -1 && errno != EAGAIN && errno != EWOULDBLOCK) {
     ctx->pagespeed_connection = NULL;
     ngx_close_connection(c);
@@ -1369,6 +1261,10 @@ ngx_int_t ps_create_connection(
 
 // Populate cfg_* with configuration information for this request.
 // Thin wrappers around ngx_http_get_module_*_conf and cast.
+ps_main_conf_t* ps_get_main_config(ngx_http_request_t* r) {
+  return static_cast<ps_main_conf_t*>(
+      ngx_http_get_module_main_conf(r, ngx_pagespeed));
+}
 ps_srv_conf_t* ps_get_srv_config(ngx_http_request_t* r) {
   return static_cast<ps_srv_conf_t*>(
       ngx_http_get_module_srv_conf(r, ngx_pagespeed));
@@ -1384,28 +1280,20 @@ RewriteOptions* ps_determine_request_options(
     const RewriteOptions* domain_options, /* may be null */
     RequestHeaders* request_headers,
     ResponseHeaders* response_headers,
-    RequestContextPtr request_context,
     ps_srv_conf_t* cfg_s,
-    GoogleUrl* url,
-    GoogleString* pagespeed_query_params,
-    GoogleString* pagespeed_option_cookies) {
+    GoogleUrl* url) {
   // Sets option from request headers and url.
   RewriteQuery rewrite_query;
   if (!cfg_s->server_context->GetQueryOptions(
-          request_context, domain_options, url, request_headers,
-          response_headers, &rewrite_query)) {
+          domain_options, url, request_headers, response_headers,
+          &rewrite_query)) {
     // Failed to parse query params or request headers.  Treat this as if there
     // were no query params given.
     ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
-                  "ps_route request: parsing headers or query params failed.");
+                  "ps_route rerquest: parsing headers or query params failed.");
     return NULL;
   }
 
-  *pagespeed_query_params =
-      rewrite_query.pagespeed_query_params().ToEscapedString();
-  *pagespeed_option_cookies =
-      rewrite_query.pagespeed_option_cookies().ToEscapedString();
-
   // Will be NULL if there aren't any options set with query params or in
   // headers.
   return rewrite_query.ReleaseOptions();
@@ -1460,7 +1348,7 @@ bool ps_set_experiment_state_and_cookie(ngx_http_request_t* r,
 }
 
 // There are many sources of options:
-//  - the request (query parameters, headers, and cookies)
+//  - the request (query parameters and headers)
 //  - location block
 //  - global server options
 //  - experiment framework
@@ -1471,12 +1359,9 @@ bool ps_determine_options(ngx_http_request_t* r,
                           RequestHeaders* request_headers,
                           ResponseHeaders* response_headers,
                           RewriteOptions** options,
-                          RequestContextPtr request_context,
-                          ps_srv_conf_t* cfg_s,
                           GoogleUrl* url,
-                          GoogleString* pagespeed_query_params,
-                          GoogleString* pagespeed_option_cookies,
                           bool html_rewrite) {
+  ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
   ps_loc_conf_t* cfg_l = ps_get_loc_config(r);
 
   // Global options for this server.  Never null.
@@ -1489,19 +1374,14 @@ bool ps_determine_options(ngx_http_request_t* r,
   // Request-specific options, nearly always null.  If set they need to be
   // rebased on the directory options or the global options.
   RewriteOptions* request_options = ps_determine_request_options(
-      r, directory_options, request_headers, response_headers, request_context,
-      cfg_s, url, pagespeed_query_params, pagespeed_option_cookies);
+      r, directory_options, request_headers, response_headers, cfg_s, url);
   bool have_request_options = request_options != NULL;
 
   // Because the caller takes ownership of any options we return, the only
   // situation in which we can avoid allocating a new RewriteOptions is if the
-  // global options are ok as they are and we don't have script variables we
-  // need to evaluate at this point.
-  NgxRewriteOptions* ngx_global_options =
-      dynamic_cast<NgxRewriteOptions*>(global_options);
+  // global options are ok as are.
   if (!have_request_options && directory_options == NULL &&
-      !global_options->running_experiment() &&
-      ngx_global_options->script_lines().size() == 0) {
+      !global_options->running_experiment()) {
     return true;
   }
 
@@ -1512,18 +1392,6 @@ bool ps_determine_options(ngx_http_request_t* r,
     *options = global_options->Clone();
   }
 
-  NgxRewriteDriverFactory* ngx_factory = dynamic_cast<NgxRewriteDriverFactory*>(
-    cfg_s->server_context->factory());
-  NgxRewriteOptions* ngx_options = dynamic_cast<NgxRewriteOptions*>(*options);
-
-  // ExecuteScriptVariables() sets 'pagespeed off' on ngx_options when execution
-  // fails and then returns false. When that happens we return, as we don't want
-  // to allow enabling pagespeed by request and execute without the intended
-  // configuration.
-  if (!ngx_options->ExecuteScriptVariables(r, cfg_s->handler, ngx_factory)) {
-    return false;
-  }
-
   // Modify our options in response to request options if specified.
   if (have_request_options) {
     (*options)->Merge(*request_options);
@@ -1629,8 +1497,7 @@ void ps_release_base_fetch(ps_request_ctx_t* ctx) {
 }
 
 // TODO(chaizhenhua): merge into NgxBaseFetch ctor
-ngx_int_t ps_create_base_fetch(ps_request_ctx_t* ctx,
-                               RequestContextPtr request_context) {
+ngx_int_t ps_create_base_fetch(ps_request_ctx_t* ctx) {
   ngx_http_request_t* r = ctx->r;
   ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
   int file_descriptors[2];
@@ -1672,7 +1539,8 @@ ngx_int_t ps_create_base_fetch(ps_request_ctx_t* ctx,
   // the BaseFetch ourselves.
   ctx->base_fetch = new NgxBaseFetch(
       r, file_descriptors[1], cfg_s->server_context,
-      request_context, ctx->preserve_caching_headers);
+      RequestContextPtr(cfg_s->server_context->NewRequestContext(r)),
+      ctx->preserve_caching_headers);
 
   return NGX_OK;
 }
@@ -1754,21 +1622,15 @@ RequestRouting::Response ps_route_request(ngx_http_request_t* r,
     return RequestRouting::kConsole;
   } else if (StringCaseEqual(path, global_options->messages_path())) {
     return RequestRouting::kMessages;
-  } else if (
-      // The admin handlers get everything under a path (/path/*) while all the
-      // other handlers only get exact matches (/path).  So match all paths
-      // starting with the handler path.
-      !global_options->admin_path().empty() &&
-      StringCaseStartsWith(path, global_options->admin_path())) {
+  } else if (// The admin handlers get everything under a path (/path/*) while
+             // all the other handlers only get exact matches (/path).  So match
+             // all paths starting with the handler path.
+             !global_options->admin_path().empty() &&
+             StringCaseStartsWith(path, global_options->admin_path())) {
     return RequestRouting::kAdmin;
   } else if (!global_options->global_admin_path().empty() &&
              StringCaseStartsWith(path, global_options->global_admin_path())) {
     return RequestRouting::kGlobalAdmin;
-  } else if (global_options->enable_cache_purge() &&
-             !global_options->purge_method().empty() &&
-             (global_options->purge_method() ==
-              str_to_string_piece(r->method_name))) {
-    return RequestRouting::kCachePurge;
   }
 
   const GoogleString* beacon_url;
@@ -1799,9 +1661,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
 
   if (!html_rewrite &&
       r->method != NGX_HTTP_GET &&
-      r->method != NGX_HTTP_HEAD &&
-      r->method != NGX_HTTP_POST &&
-      response_category != RequestRouting::kCachePurge) {
+      r->method != NGX_HTTP_HEAD) {
     return NGX_DECLINED;
   }
 
@@ -1816,16 +1676,10 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
   copy_request_headers_from_ngx(r, request_headers.get());
   copy_response_headers_from_ngx(r, response_headers.get());
 
-  RequestContextPtr request_context(
-      cfg_s->server_context->NewRequestContext(r));
   RewriteOptions* options = NULL;
-  GoogleString pagespeed_query_params;
-  GoogleString pagespeed_option_cookies;
 
   if (!ps_determine_options(r, request_headers.get(), response_headers.get(),
-                            &options, request_context, cfg_s, &url,
-                            &pagespeed_query_params, &pagespeed_option_cookies,
-                            html_rewrite)) {
+                            &options, &url, html_rewrite)) {
     return NGX_ERROR;
   }
 
@@ -1840,8 +1694,6 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     return NGX_DECLINED;
   }
 
-  request_context->set_options(options->ComputeHttpOptions());
-
   // ps_determine_options modified url, removing any ModPagespeedFoo=Bar query
   // parameters.  Keep url_string in sync with url.
   url.Spec().CopyToString(&url_string);
@@ -1863,8 +1715,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
       response_category == RequestRouting::kGlobalStatistics ||
       response_category == RequestRouting::kConsole ||
       response_category == RequestRouting::kAdmin ||
-      response_category == RequestRouting::kGlobalAdmin ||
-      response_category == RequestRouting::kCachePurge;
+      response_category == RequestRouting::kGlobalAdmin;
 
   if (html_rewrite) {
     ps_release_base_fetch(ctx);
@@ -1873,19 +1724,12 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     CHECK(ctx == NULL);
     ctx = new ps_request_ctx_t();
 
-    ctx->base_fetch = NULL;
-    ctx->pagespeed_connection = NULL;
     ctx->r = r;
+    ctx->write_pending = false;
     ctx->html_rewrite = false;
     ctx->in_place = false;
-    ctx->write_pending = false;
-    ctx->fetch_done = false;
+    ctx->pagespeed_connection = NULL;
     ctx->preserve_caching_headers = kDontPreserveHeaders;
-    ctx->proxy_fetch = NULL;
-    ctx->inflater_ = NULL;
-    ctx->driver = NULL;
-    ctx->recorder = NULL;
-    ctx->ipro_response_headers = NULL;
 
     // See build_context_for_request() in mod_instaweb.cc
     // TODO(jefftk): Is this the right place to be modifying caching headers for
@@ -1923,8 +1767,8 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     ngx_http_set_ctx(r, ctx, ngx_pagespeed);
   }
 
-  if (ps_create_base_fetch(ctx, request_context) != NGX_OK) {
-    // Do not need to release request context 'ctx'.
+  if (ps_create_base_fetch(ctx)!= NGX_OK) {
+    // Do not need to release request context.
     // http_pool_cleanup will call ps_release_request_context
     return NGX_ERROR;
   }
@@ -1970,7 +1814,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     } else if (response_category == RequestRouting::kConsole) {
       cfg_s->server_context->ConsoleHandler(
           *cfg_s->server_context->config(),
-          AdminSite::kStatistics,
+          SystemServerContext::kStatistics,
           query_params,
           ctx->base_fetch);
     } else if (response_category == RequestRouting::kAdmin ||
@@ -1982,11 +1826,6 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
           custom_options == NULL ? cfg_s->server_context->config()
                                  : custom_options.get(),
           ctx->base_fetch);
-    } else if (response_category == RequestRouting::kCachePurge) {
-      AdminSite* admin_site = cfg_s->server_context->admin_site();
-      admin_site->PurgeHandler(url_string,
-                               cfg_s->server_context->cache_path(),
-                               ctx->base_fetch);
     } else {
       CHECK(false);
     }
@@ -2004,11 +1843,11 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
 
     if (custom_options.get() == NULL) {
       driver = cfg_s->server_context->NewRewriteDriver(
-          ctx->base_fetch->request_context());
+         ctx->base_fetch->request_context());
     } else {
       // NewCustomRewriteDriver takes ownership of custom_options.
       driver = cfg_s->server_context->NewCustomRewriteDriver(
-          custom_options.release(), ctx->base_fetch->request_context());
+      custom_options.release(), ctx->base_fetch->request_context());
     }
 
     StringPiece user_agent = ctx->base_fetch->request_headers()->Lookup1(
@@ -2017,8 +1856,6 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
       driver->SetUserAgent(user_agent);
     }
     driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
-    driver->set_pagespeed_query_params(pagespeed_query_params);
-    driver->set_pagespeed_option_cookies(pagespeed_option_cookies);
 
     // TODO(jefftk): FlushEarlyFlow would go here.
 
@@ -2059,7 +1896,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
         url_string.c_str());
 
     ctx->in_place = true;
-    ctx->base_fetch->set_ipro_lookup(true);
+    ctx->base_fetch->set_handle_error(false);
     ctx->driver->FetchInPlaceResource(
         url, false /* proxy_mode */, ctx->base_fetch);
 
@@ -2069,9 +1906,8 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
   // NOTE: We are using the below debug message as is for some of our system
   // tests. So, be careful about test breakages caused by changing or
   // removing this line.
-  ngx_log_error(NGX_LOG_DEBUG, r->connection->log, 0,
-                "Passing on content handling for non-pagespeed resource '%s'",
-                url_string.c_str());
+  DBG(r, "Passing on content handling for non-pagespeed resource '%s'",
+      url_string.c_str());
 
   ctx->base_fetch->Done(false);
   ps_release_base_fetch(ctx);
@@ -2108,7 +1944,8 @@ void ps_send_to_pagespeed(ngx_http_request_t* r,
         int num_inflated_bytes = ctx->inflater_->InflateBytes(
             buf, kStackBufferSize);
         if (num_inflated_bytes < 0) {
-          cfg_s->handler->Message(kWarning, "Corrupted inflation");
+          cfg_s->handler->Message(kWarning,
+                                  "Corrupted inflation");
         } else if (num_inflated_bytes > 0) {
           ctx->proxy_fetch->Write(StringPiece(buf, num_inflated_bytes),
                                   cfg_s->handler);
@@ -2132,10 +1969,10 @@ void ps_send_to_pagespeed(ngx_http_request_t* r,
 #ifndef ngx_http_clear_etag
 // The ngx_http_clear_etag(r) macro was added in 1.3.3.  Backport it if it's not
 // present.
-#define ngx_http_clear_etag(r)                  \
-  if (r->headers_out.etag) {                    \
-    r->headers_out.etag->hash = 0;              \
-    r->headers_out.etag = NULL;                 \
+#define ngx_http_clear_etag(r)       \
+  if (r->headers_out.etag) {         \
+    r->headers_out.etag->hash = 0;   \
+    r->headers_out.etag = NULL;      \
   }
 #endif
 
@@ -2222,8 +2059,6 @@ namespace html_rewrite {
 ngx_http_output_header_filter_pt ngx_http_next_header_filter;
 ngx_http_output_body_filter_pt ngx_http_next_body_filter;
 
-// After pagespeed has had a chance to run, copy the headers it produced to
-// nginx so it can send them out to the browser.
 ngx_int_t ps_html_rewrite_header_filter(ngx_http_request_t* r) {
   ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
   if (cfg_s->server_context == NULL) {
@@ -2362,9 +2197,6 @@ namespace in_place {
 ngx_http_output_header_filter_pt ngx_http_next_header_filter;
 ngx_http_output_body_filter_pt ngx_http_next_body_filter;
 
-// Header filter to support IPRO.
-//
-// The control flow here is tricky, probably excessively so.
 ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
   ps_request_ctx_t* ctx = ps_get_request_context(r);
 
@@ -2436,9 +2268,6 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
         cache_url.c_str());
     const SystemRewriteOptions* options = SystemRewriteOptions::DynamicCast(
         ctx->driver->options());
-    RequestContextPtr request_context(
-        cfg_s->server_context->NewRequestContext(r));
-    request_context->set_options(options->ComputeHttpOptions());
     RequestHeaders request_headers;
     copy_request_headers_from_ngx(r, &request_headers);
     // This URL was not found in cache (neither the input resource nor
@@ -2446,12 +2275,14 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
     // (or at least a note that it cannot be cached stored there).
     // We do that using an Apache output filter.
     ctx->recorder = new InPlaceResourceRecorder(
-        request_context,
+        RequestContextPtr(cfg_s->server_context->NewRequestContext(r)),
         cache_url,
         ctx->driver->CacheFragment(),
         request_headers.GetProperties(),
+        options->respect_vary(),
         options->ipro_max_response_bytes(),
         options->ipro_max_concurrent_recordings(),
+        options->implicit_cache_ttl_ms(),
         server_context->http_cache(),
         server_context->statistics(),
         message_handler);
@@ -2462,8 +2293,8 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
     // to the backend.
   } else {
     server_context->rewrite_stats()->ipro_not_rewritable()->Add(1);
-    message_handler->Message(
-        kInfo, "Could not rewrite resource in-place: %s", url.c_str());
+    message_handler->Message(kInfo,
+           "Could not rewrite resource in-place: %s", url.c_str());
   }
 
   ctx->driver->Cleanup();
@@ -2475,8 +2306,6 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
   return ps_decline_request(r);
 }
 
-// If we've decided that we should record this response for future optimization
-// with IPRO, then log the bytes as they come through
 ngx_int_t ps_in_place_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
   ps_request_ctx_t* ctx = ps_get_request_context(r);
   if (ctx == NULL || ctx->recorder == NULL) {
@@ -2489,10 +2318,10 @@ ngx_int_t ps_in_place_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
   InPlaceResourceRecorder* recorder = ctx->recorder;
   for (ngx_chain_t* cl = in; cl; cl = cl->next) {
     if (ngx_buf_size(cl->buf)) {
-      CHECK(ngx_buf_in_memory(cl->buf));
-      StringPiece contents(reinterpret_cast<char*>(cl->buf->pos),
-                           ngx_buf_size(cl->buf));
-      recorder->Write(contents, recorder->handler());
+       CHECK(ngx_buf_in_memory(cl->buf));
+       StringPiece contents(reinterpret_cast<char*>(cl->buf->pos),
+                            ngx_buf_size(cl->buf));
+       recorder->Write(contents, recorder->handler());
     }
 
     if (cl->buf->flush) {
@@ -2552,8 +2381,6 @@ ngx_int_t send_out_headers_and_body(
   return ngx_http_output_filter(r, out);
 }
 
-// Handle responses where we have the content we need in memory and can just
-// send it right out.
 ngx_int_t ps_simple_handler(ngx_http_request_t* r,
                             NgxServerContext* server_context,
                             RequestRouting::Response response_category) {
@@ -2596,7 +2423,7 @@ ngx_int_t ps_simple_handler(ngx_http_request_t* r,
                      "Please check if it's enabled in pagespeed.conf.\n",
                      message_handler);
       } else {
-        HtmlKeywords::WritePre(log, "", &writer, message_handler);
+        HtmlKeywords::WritePre(log, &writer, message_handler);
       }
       break;
     }
@@ -2656,16 +2483,10 @@ void ps_beacon_handler_helper(ngx_http_request_t* r,
   ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
   CHECK(cfg_s != NULL);
 
-  RequestContextPtr request_context(
-      cfg_s->server_context->NewRequestContext(r));
-  // TODO(sligocki): Do we want custom options here? It probably doesn't matter
-  // for beacons.
-  request_context->set_options(
-      cfg_s->server_context->global_options()->ComputeHttpOptions());
-
-  cfg_s->server_context->HandleBeacon(beacon_data,
-                                      user_agent,
-                                      request_context);
+  cfg_s->server_context->HandleBeacon(
+      beacon_data,
+      user_agent,
+      RequestContextPtr(cfg_s->server_context->NewRequestContext(r)));
 
   ps_set_cache_control(r, const_cast<char*>("max-age=0, no-cache"));
 
@@ -2673,6 +2494,7 @@ void ps_beacon_handler_helper(ngx_http_request_t* r,
   // header so wget doesn't hang.
 }
 
+
 // Load the request body into out.  ngx_http_read_client_request_body must
 // already have been called.  Return false on failure, true on success.
 bool ps_request_body_to_string_piece(
@@ -2685,32 +2507,13 @@ bool ps_request_body_to_string_piece(
   }
 
   if (r->request_body->temp_file) {
-    u_char buf[POST_BUF_READ_SIZE];
-    int count = 0;
-    ssize_t ret;
-    GoogleString tmp;
-
-    // Note that we depend on nginx to impose sensible limits on post data.
-    while ((ret = ngx_read_file(&r->request_body->temp_file->file,
-                                buf, POST_BUF_READ_SIZE, count)) > 0) {
-      tmp.append(reinterpret_cast<char*>(buf), ret);
-      count += ret;
-    }
-
-    if (ret < 0) {
-      ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
-                    "ps_request_body_to_string_piece: "
-                    "error reading post body.");
-      return false;
-    }
-
-    // We have read the complete post body, copy it to a buffer
-    // from the request's pool.
-    u_char* data = reinterpret_cast<u_char*>(ngx_palloc(r->pool, count));
-    memcpy(data, tmp.c_str(), count);
-    *out = StringPiece(reinterpret_cast<char*>(data), count);
-
-    return true;
+    // For now raise an error instead of figuring out how to read temporary
+    // files.
+    ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
+                  "ps_request_body_to_string_piece: "
+                  "request body in temporary file unsupported."
+                  "Increase client_body_buffer_size.");
+    return false;
   } else if (r->request_body->bufs->next == NULL) {
     // There's just one buffer, so we can simply return a StringPiece pointing
     // to this buffer.
@@ -2799,10 +2602,6 @@ void ps_beacon_body_handler(ngx_http_request_t* r) {
   }
 }
 
-// We need to get the beacon data to ps_beacon_body_handler so it can pass it
-// along to SystemServerContext::HandleBeacon, but it might have been POSTed.
-// If it's posted we need to make an async call to get the data, otherwise just
-// read it out of the query params.
 ngx_int_t ps_beacon_handler(ngx_http_request_t* r) {
   if (r->method == NGX_HTTP_POST) {
     // Use post body. Handler functions are called before the request body has
@@ -2825,12 +2624,8 @@ ngx_int_t ps_beacon_handler(ngx_http_request_t* r) {
   }
 }
 
-// Some things pagespeed filters on the way past (html) and other things it
-// actually handles, like requests for resources
-// (example.css.pagespeed.ce.LyfcM6Wulf.css) and static content
-// (/ngx_pagespeed_static/js_defer.q1EBmcgYOC.js).  This is called once we know
-// we need to handle a resource, and it figures out which particular handler can
-// supply it to the user.
+// Handle requests for resources like example.css.pagespeed.ce.LyfcM6Wulf.css
+// and for static content like /ngx_pagespeed_static/js_defer.q1EBmcgYOC.js
 ngx_int_t ps_content_handler(ngx_http_request_t* r) {
   ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
   if (cfg_s->server_context == NULL) {
@@ -2866,7 +2661,6 @@ ngx_int_t ps_content_handler(ngx_http_request_t* r) {
     case RequestRouting::kConsole:
     case RequestRouting::kAdmin:
     case RequestRouting::kGlobalAdmin:
-    case RequestRouting::kCachePurge:
     case RequestRouting::kResource:
       return ps_resource_handler(
           r, false /* html rewrite */, response_category);
@@ -2877,9 +2671,9 @@ ngx_int_t ps_content_handler(ngx_http_request_t* r) {
 }
 
 ngx_int_t ps_phase_handler(ngx_http_request_t* r,
-                           ngx_http_phase_handler_t* ph) {
+      ngx_http_phase_handler_t* ph) {
   ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                 "pagespeed phase: %ui", r->phase_handler);
+                   "pagespeed phase: %ui", r->phase_handler);
 
   r->write_event_handler = ngx_http_request_empty_handler;
 
@@ -2901,12 +2695,10 @@ ngx_int_t ps_phase_handler(ngx_http_request_t* r,
 namespace fix_headers {
 ngx_http_output_header_filter_pt ngx_http_next_header_filter;
 
-// Clear a few headers from html responses.
-
 ngx_int_t ps_html_rewrite_fix_headers_filter(ngx_http_request_t* r) {
   ps_request_ctx_t* ctx = ps_get_request_context(r);
-  if (r != r->main || ctx == NULL || !ctx->html_rewrite ||
-      ctx->preserve_caching_headers == kPreserveAllCachingHeaders) {
+  if (r != r->main || ctx == NULL || !ctx->html_rewrite
+      || ctx->preserve_caching_headers == kPreserveAllCachingHeaders) {
     return ngx_http_next_header_filter(r);
   }
   if (ctx->preserve_caching_headers == kDontPreserveHeaders) {
@@ -2956,8 +2748,8 @@ ngx_int_t ps_preaccess_handler(ngx_http_request_t* r) {
 
   i = r->phase_handler;
   // move handlers before try_files && content phase
-  while (ph[i + 1].checker != ngx_http_core_try_files_phase &&
-         ph[i + 1].checker != ngx_http_core_content_phase) {
+  while (ph[i + 1].checker != ngx_http_core_try_files_phase
+      && ph[i + 1].checker != ngx_http_core_content_phase) {
     ph[i] = ph[i + 1];
     ph[i].next--;
     i++;
@@ -2983,14 +2775,6 @@ ngx_int_t ps_etag_filter_init(ngx_conf_t* cf) {
   return NGX_OK;
 }
 
-// Called before configuration.
-ngx_int_t ps_pre_init(ngx_conf_t* cf) {
-  // Setup an intervention setter for gzip configuration and check
-  // gzip configuration command signatures.
-  g_gzip_setter.Init(cf);
-  return NGX_OK;
-}
-
 ngx_int_t ps_init(ngx_conf_t* cf) {
   // Only put register pagespeed code to run if there was a "pagespeed"
   // configuration option set in the config file.  With "pagespeed off" we
@@ -3040,7 +2824,7 @@ ngx_http_module_t ps_etag_filter_module = {
 };
 
 ngx_http_module_t ps_module = {
-  ps_pre_init,  // preconfiguration
+  NULL,  // preconfiguration
   ps_init,  // postconfiguration
 
   ps_create_main_conf,
@@ -3110,15 +2894,10 @@ ngx_int_t ps_init_module(ngx_cycle_t* cycle) {
           "UseNativeFetcher is on, please configure a resolver.");
       return NGX_ERROR;
     }
-    // Update logging to the configured error_log in the http{} block.
+
     cfg_m->driver_factory->LoggingInit(cycle->log);
     cfg_m->driver_factory->RootInit();
   } else {
-    if (!factory_init_called) {
-      cfg_m->driver_factory->LoggingInit(cycle->log);
-      cfg_m->driver_factory->Init();
-      factory_init_called = true;
-    }
     delete cfg_m->driver_factory;
     cfg_m->driver_factory = NULL;
   }
@@ -3155,11 +2934,11 @@ ngx_int_t ps_init_child_process(ngx_cycle_t* cycle) {
     // Some server{} blocks may not have a ServerContext in that case we must
     // not instantiate a ProxyFetchFactory.
     if (cfg_s->server_context != NULL) {
+      cfg_s->proxy_fetch_factory = new ProxyFetchFactory(cfg_s->server_context);
       ngx_http_core_loc_conf_t* clcf = static_cast<ngx_http_core_loc_conf_t*>(
           cscfp[s]->ctx->loc_conf[ngx_http_core_module.ctx_index]);
       cfg_m->driver_factory->SetServerContextMessageHandler(
           cfg_s->server_context, clcf->error_log);
-      cfg_s->proxy_fetch_factory = new ProxyFetchFactory(cfg_s->server_context);
     }
   }
 

src/ngx_rewrite_driver_factory.cc

@@ -71,20 +71,17 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
         NULL /* default shared memory runtime */, hostname, port),
       main_conf_(NULL),
       threads_started_(false),
-      ngx_message_handler_(
-          new NgxMessageHandler(timer(), thread_system()->NewMutex())),
+      use_per_vhost_statistics_(false),
+      ngx_message_handler_(new NgxMessageHandler(thread_system()->NewMutex())),
       ngx_html_parse_message_handler_(
-          new NgxMessageHandler(timer(), thread_system()->NewMutex())),
+          new NgxMessageHandler(thread_system()->NewMutex())),
+      install_crash_handler_(false),
       log_(NULL),
       resolver_timeout_(NGX_CONF_UNSET_MSEC),
       use_native_fetcher_(false),
-      // 100 Aligns to nginx's server-side default.
-      native_fetcher_max_keepalive_requests_(100),
       ngx_shared_circular_buffer_(NULL),
       hostname_(hostname.as_string()),
-      port_(port),
-      process_script_variables_(false),
-      process_script_variables_set_(false) {
+      port_(port) {
   InitializeDefaultOptions();
   default_options()->set_beacon_url("/ngx_pagespeed_beacon");
   SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
@@ -114,7 +111,6 @@ UrlAsyncFetcher* NgxRewriteDriverFactory::AllocateFetcher(
         resolver_timeout_,
         config->blocking_fetch_timeout_ms(),
         resolver_,
-        native_fetcher_max_keepalive_requests_,
         thread_system(),
         message_handler());
     ngx_url_async_fetchers_.push_back(fetcher);
@@ -212,7 +208,7 @@ void NgxRewriteDriverFactory::StartThreads() {
 
 void NgxRewriteDriverFactory::LoggingInit(ngx_log_t* log) {
   net_instaweb::log_message_handler::Install(log);
-  if (install_crash_handler()) {
+  if (install_crash_handler_) {
     NgxMessageHandler::InstallCrashHandler(log);
   }
   ngx_message_handler_->set_log(log);
@@ -229,7 +225,7 @@ void NgxRewriteDriverFactory::SetCircularBuffer(
 void NgxRewriteDriverFactory::SetServerContextMessageHandler(
     ServerContext* server_context, ngx_log_t* log) {
   NgxMessageHandler* handler = new NgxMessageHandler(
-      timer(), thread_system()->NewMutex());
+      thread_system()->NewMutex());
   handler->set_log(log);
   // The ngx_shared_circular_buffer_ will be NULL if MessageBufferSize hasn't
   // been raised from its default of 0.

src/ngx_rewrite_driver_factory.h

@@ -92,6 +92,18 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
 
   void set_main_conf(NgxRewriteOptions* main_conf) {  main_conf_ = main_conf; }
 
+  bool use_per_vhost_statistics() const {
+    return use_per_vhost_statistics_;
+  }
+  void set_use_per_vhost_statistics(bool x) {
+    use_per_vhost_statistics_ = x;
+  }
+  bool install_crash_handler() const {
+    return install_crash_handler_;
+  }
+  void set_install_crash_handler(bool x) {
+    install_crash_handler_ = x;
+  }
   void set_resolver(ngx_resolver_t* resolver) {
     resolver_ = resolver;
   }
@@ -105,14 +117,16 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   void set_use_native_fetcher(bool x) {
     use_native_fetcher_ = x;
   }
-  int native_fetcher_max_keepalive_requests() {
-    return native_fetcher_max_keepalive_requests_;
+  void set_rate_limit_background_fetches(bool x) {
+    rate_limit_background_fetches_ = x;
   }
-  void set_native_fetcher_max_keepalive_requests(int x) {
-    native_fetcher_max_keepalive_requests_ = x;
-  }
-  bool process_script_variables() {
-    return process_script_variables_;
+
+  // We use a beacon handler to collect data for critical images,
+  // css, etc., so filters should be configured accordingly.
+  //
+  // TODO(jefftk): move to SystemRewriteDriverFactory
+  virtual bool UseBeaconResultsInFilters() const {
+    return true;
   }
 
   void LoggingInit(ngx_log_t* log);
@@ -121,15 +135,6 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
 
   virtual void SetCircularBuffer(SharedCircularBuffer* buffer);
 
-  bool SetProcessScriptVariables(bool process_script_variables) {
-    if (!process_script_variables_set_) {
-      process_script_variables_ = process_script_variables;
-      process_script_variables_set_ = true;
-      return true;
-    }
-    return false;
-  }
-
  private:
   Timer* timer_;
 
@@ -138,16 +143,20 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   NgxRewriteOptions* main_conf_;
 
   bool threads_started_;
+  // If true, we'll have a separate statistics object for each vhost
+  // (along with a global aggregate), rather than just a single object
+  // aggregating all of them.
+  bool use_per_vhost_statistics_;
   NgxMessageHandler* ngx_message_handler_;
   NgxMessageHandler* ngx_html_parse_message_handler_;
+  bool install_crash_handler_;
 
   std::vector<NgxUrlAsyncFetcher*> ngx_url_async_fetchers_;
   ngx_log_t* log_;
   ngx_msec_t resolver_timeout_;
   ngx_resolver_t* resolver_;
   bool use_native_fetcher_;
-  int native_fetcher_max_keepalive_requests_;
-
+  bool rate_limit_background_fetches_;
   typedef std::set<NgxMessageHandler*> NgxMessageHandlerSet;
   NgxMessageHandlerSet server_context_message_handlers_;
 
@@ -157,8 +166,6 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
 
   GoogleString hostname_;
   int port_;
-  bool process_script_variables_;
-  bool process_script_variables_set_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
 };

src/ngx_rewrite_options.cc

@@ -31,7 +31,6 @@ extern "C" {
 #include "net/instaweb/rewriter/public/file_load_policy.h"
 #include "net/instaweb/rewriter/public/rewrite_options.h"
 #include "net/instaweb/system/public/system_caches.h"
-#include "net/instaweb/util/public/message_handler.h"
 #include "net/instaweb/util/public/timer.h"
 
 namespace net_instaweb {
@@ -47,10 +46,9 @@ const char kGlobalAdminPath[] = "GlobalAdminPath";
 
 // These options are copied from mod_instaweb.cc, where APACHE_CONFIG_OPTIONX
 // indicates that they can not be set at the directory/location level. They set
-// options in the RewriteDriverFactory, so they're entirely global and do not
-// appear in RewriteOptions.  They are not alphabetized on purpose, but rather
-// left in the same order as in mod_instaweb.cc in case we end up needing to
-// compare.
+// options in the RewriteDriverFactory, so they do not appear in RewriteOptions.
+// They are not alphabetized on purpose, but rather left in the same order as in
+// mod_instaweb.cc in case we end up needing to compare.
 // TODO(oschaaf): this duplication is a short term solution.
 const char* const server_only_options[] = {
   "FetcherTimeoutMs",
@@ -63,7 +61,6 @@ const char* const server_only_options[] = {
   "MessageBufferSize",
   "NumRewriteThreads",
   "NumExpensiveRewriteThreads",
-  "StaticAssetPrefix",
   "TrackOriginalContentLength",
   "UsePerVHostStatistics",  // TODO(anupama): What to do about "No longer used"
   "BlockingRewriteRefererUrls",
@@ -72,14 +69,12 @@ const char* const server_only_options[] = {
   "LoadFromFileMatch",
   "LoadFromFileRule",
   "LoadFromFileRuleMatch",
-  "UseNativeFetcher",
-  "NativeFetcherMaxKeepaliveRequests"
+  "UseNativeFetcher"
 };
 
 // Options that can only be used in the main (http) option scope.
 const char* const main_only_options[] = {
-  "UseNativeFetcher",
-  "NativeFetcherMaxKeepaliveRequests"
+  "UseNativeFetcher"
 };
 
 }  // namespace
@@ -100,7 +95,6 @@ NgxRewriteOptions::NgxRewriteOptions(ThreadSystem* thread_system)
 void NgxRewriteOptions::Init() {
   DCHECK(ngx_properties_ != NULL)
       << "Call NgxRewriteOptions::Initialize() before construction";
-  clear_inherited_scripts_ = false;
   InitializeOptions(ngx_properties_);
 }
 
@@ -108,27 +102,23 @@ void NgxRewriteOptions::AddProperties() {
   // Nginx-specific options.
   add_ngx_option(
       "", &NgxRewriteOptions::statistics_path_, "nsp", kStatisticsPath,
-      kServerScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics",
-      false);
+      kProcessScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics");
   add_ngx_option(
       "", &NgxRewriteOptions::global_statistics_path_, "ngsp",
       kGlobalStatisticsPath, kProcessScope,
-      "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics",
-      false);
+      "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics");
   add_ngx_option(
-      "", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kServerScope,
-      "Set the console path. Ex: /pagespeed_console", false);
+      "", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kProcessScope,
+      "Set the console path. Ex: /pagespeed_console");
   add_ngx_option(
       "", &NgxRewriteOptions::messages_path_, "nmp", kMessagesPath,
-      kServerScope, "Set the messages path.  Ex: /ngx_pagespeed_message",
-      false);
+      kProcessScope, "Set the messages path.  Ex: /ngx_pagespeed_message");
   add_ngx_option(
       "", &NgxRewriteOptions::admin_path_, "nap", kAdminPath,
-      kServerScope, "Set the admin path.  Ex: /pagespeed_admin", false);
+      kProcessScope, "Set the admin path.  Ex: /pagespeed_admin");
   add_ngx_option(
       "", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
-      kProcessScope, "Set the global admin path.  Ex: /pagespeed_global_admin",
-      false);
+      kProcessScope, "Set the global admin path.  Ex: /pagespeed_global_admin");
 
   MergeSubclassProperties(ngx_properties_);
 
@@ -258,7 +248,7 @@ const char* ps_error_string_for_option(
 const char* NgxRewriteOptions::ParseAndSetOptions(
     StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
     NgxRewriteDriverFactory* driver_factory,
-    RewriteOptions::OptionScope scope, ngx_conf_t* cf, bool compile_scripts) {
+    RewriteOptions::OptionScope scope) {
   CHECK_GE(n_args, 1);
 
   StringPiece directive = args[0];
@@ -274,139 +264,77 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
         pool, directive, "cannot be set at this scope.");
   }
 
-  ScriptLine* script_line;
-  script_line = NULL;
-  // Only allow script variable support for LoadFromFile for now.
-  // Note that LoadFromFile should not be scriptable on wildcard hosts,
-  // as browsers might be able to manipulate its natural use-case: $http_host.
-  if (!StringCaseStartsWith(directive, "LoadFromFile")) {
-    compile_scripts = false;
-  }
-
-  if (n_args == 1 && StringCaseEqual(directive, "ClearInheritedScripts")) {
-    clear_inherited_scripts_ = true;
-    return NGX_CONF_OK;
-  }
-
-  if (compile_scripts) {
-    CHECK(cf != NULL);
-    int i;
-    // Skip the first arg which is always 'pagespeed'
-    for (i = 1; i < n_args; i++) {
-      ngx_str_t script_source;
-
-      script_source.len = args[i].as_string().length();
-      std::string tmp = args[i].as_string();
-      script_source.data = reinterpret_cast<u_char*>(
-          const_cast<char*>(tmp.c_str()));
-
-      if (ngx_http_script_variables_count(&script_source) > 0) {
-        ngx_http_script_compile_t* sc =
-            reinterpret_cast<ngx_http_script_compile_t*>(
-                ngx_pcalloc(cf->pool, sizeof(ngx_http_script_compile_t)));
-        sc->cf = cf;
-        sc->source = &script_source;
-        sc->lengths = reinterpret_cast<ngx_array_t**>(
-            ngx_pcalloc(cf->pool, sizeof(ngx_array_t*)));
-        sc->values = reinterpret_cast<ngx_array_t**>(
-            ngx_pcalloc(cf->pool, sizeof(ngx_array_t*)));
-        sc->variables = 1;
-        sc->complete_lengths = 1;
-        sc->complete_values = 1;
-        if (ngx_http_script_compile(sc) != NGX_OK) {
-          return ps_error_string_for_option(
-              pool, directive, "Failed to compile script variables");
-        } else {
-          if (script_line == NULL) {
-            script_line = new ScriptLine(args, n_args, scope);
-          }
-          script_line->AddScriptAndArgIndex(sc, i);
-        }
-      }
-    }
-
-    if (script_line != NULL) {
-      script_lines_.push_back(RefCountedPtr<ScriptLine>(script_line));
-      // We have found script variables in the current configuration line, and
-      // prepared the associated rewriteoptions for that.
-      // We will defer parsing, validation and processing of this line to
-      // request time. That means we are done handling this configuration line.
-      return NGX_CONF_OK;
-    }
-  }
-
   GoogleString msg;
   OptionSettingResult result;
   if (n_args == 1) {
     result = ParseAndSetOptions0(directive, &msg, handler);
   } else if (n_args == 2) {
     StringPiece arg = args[1];
-    if (IsDirective(directive, "UseNativeFetcher")) {
+    // TODO(morlovich): Remove these special hacks, and handle these via
+    // ParseAndSetOptionFromEnum1.
+    if (IsDirective(directive, "UsePerVHostStatistics")) {
       result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
           arg, driver_factory,
-          &NgxRewriteDriverFactory::set_use_native_fetcher);
-    } else if (IsDirective(directive, "NativeFetcherMaxKeepaliveRequests")) {
-      int max_keepalive_requests;
-      if (StringToInt(arg, &max_keepalive_requests) &&
-          max_keepalive_requests > 0) {
-        driver_factory->set_native_fetcher_max_keepalive_requests(
-            max_keepalive_requests);
+          &NgxRewriteDriverFactory::set_use_per_vhost_statistics);
+    } else if (IsDirective(directive, "InstallCrashHandler")) {
+      result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
+          arg, driver_factory,
+          &NgxRewriteDriverFactory::set_install_crash_handler);
+    } else if (IsDirective(directive, "MessageBufferSize")) {
+      int message_buffer_size;
+      bool ok = StringToInt(arg.as_string(), &message_buffer_size);
+      if (ok && message_buffer_size >= 0) {
+        driver_factory->set_message_buffer_size(message_buffer_size);
         result = RewriteOptions::kOptionOk;
       } else {
         result = RewriteOptions::kOptionValueInvalid;
       }
-    } else if (StringCaseEqual("ProcessScriptVariables", args[0])) {
-      if (scope == RewriteOptions::kProcessScopeStrict) {
-        if (StringCaseEqual(arg, "on")) {
-          if (driver_factory->SetProcessScriptVariables(true)) {
-            result = RewriteOptions::kOptionOk;
-          } else {
-            return const_cast<char*>(
-                "pagespeed ProcessScriptVariables: can only be set once");
-          }
-        } else if (StringCaseEqual(arg, "off")) {
-          if (driver_factory->SetProcessScriptVariables(false)) {
-            result = RewriteOptions::kOptionOk;
-          } else {
-            return const_cast<char*>(
-                "pagespeed ProcessScriptVariables: can only be set once");
-          }
-        } else {
-          return const_cast<char*>(
-              "pagespeed ProcessScriptVariables: invalid value");
-        }
-      } else {
-        return const_cast<char*>(
-            "ProcessScriptVariables is only allowed at the top level");
-      }
+    } else if (IsDirective(directive, "UseNativeFetcher")) {
+      result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
+          arg, driver_factory,
+          &NgxRewriteDriverFactory::set_use_native_fetcher);
+    } else if (IsDirective(directive, "RateLimitBackgroundFetches")) {
+      result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
+          arg, driver_factory,
+          &NgxRewriteDriverFactory::set_rate_limit_background_fetches);
+    } else if (IsDirective(directive, "ForceCaching")) {
+      result = ParseAndSetOptionHelper<SystemRewriteDriverFactory>(
+          arg, driver_factory,
+          &SystemRewriteDriverFactory::set_force_caching);
+    } else if (IsDirective(directive, "ListOutstandingUrlsOnError")) {
+      result = ParseAndSetOptionHelper<SystemRewriteDriverFactory>(
+          arg, driver_factory,
+          &SystemRewriteDriverFactory::list_outstanding_urls_on_error);
+    } else if (IsDirective(directive, "TrackOriginalContentLength")) {
+      result = ParseAndSetOptionHelper<SystemRewriteDriverFactory>(
+          arg, driver_factory,
+          &SystemRewriteDriverFactory::set_track_original_content_length);
     } else {
-      result = ParseAndSetOptionFromName1(directive, arg, &msg, handler);
-      if (result == RewriteOptions::kOptionNameUnknown) {
-        result = driver_factory->ParseAndSetOption1(
-            directive,
-            arg,
-            scope >= RewriteOptions::kProcessScope,
-            &msg,
-            handler);
-      }
+      result = ParseAndSetOptionFromName1(directive, args[1], &msg, handler);
     }
   } else if (n_args == 3) {
-    result = ParseAndSetOptionFromName2(directive, args[1], args[2],
-                                        &msg, handler);
-    if (result == RewriteOptions::kOptionNameUnknown) {
-      result = driver_factory->ParseAndSetOption2(
-          directive,
-          args[1],
-          args[2],
-          scope >= RewriteOptions::kProcessScope,
-          &msg,
-          handler);
+    // Short-term special handling, until this moves to common code.
+    // TODO(morlovich): Clean this up.
+    if (StringCaseEqual(directive, "CreateSharedMemoryMetadataCache")) {
+      int64 kb = 0;
+      if (!StringToInt64(args[2], &kb) || kb < 0) {
+        result = RewriteOptions::kOptionValueInvalid;
+        msg = "size_kb must be a positive 64-bit integer";
+      } else {
+        bool ok = driver_factory->caches()->CreateShmMetadataCache(
+            args[1].as_string(), kb, &msg);
+        result = ok ? kOptionOk : kOptionValueInvalid;
+      }
+    } else {
+      result = ParseAndSetOptionFromName2(directive, args[1], args[2],
+                                          &msg, handler);
     }
   } else if (n_args == 4) {
     result = ParseAndSetOptionFromName3(
         directive, args[1], args[2], args[3], &msg, handler);
   } else {
-    result = RewriteOptions::kOptionNameUnknown;
+    return ps_error_string_for_option(
+        pool, directive, "not recognized or too many arguments");
   }
 
   switch (result) {
@@ -428,92 +356,13 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
   return NULL;
 }
 
-// Execute all entries in the script_lines vector, and hand the result off to
-// ParseAndSetOptions to obtain the final option values.
-bool NgxRewriteOptions::ExecuteScriptVariables(
-    ngx_http_request_t* r, MessageHandler* handler,
-    NgxRewriteDriverFactory* driver_factory) {
-  bool script_error = false;
-
-  if (script_lines_.size() > 0) {
-    std::vector<RefCountedPtr<ScriptLine> >::iterator it;
-    for (it = script_lines_.begin() ; it != script_lines_.end(); ++it) {
-      ScriptLine* script_line = it->get();
-      StringPiece args[NGX_PAGESPEED_MAX_ARGS];
-      std::vector<ScriptArgIndex*>::iterator cs_it;
-      int i;
-
-      for (i = 0; i < script_line->n_args(); i++) {
-        args[i] = script_line->args()[i];
-      }
-
-      for (cs_it = script_line->data().begin();
-           cs_it != script_line->data().end(); cs_it++) {
-        ngx_http_script_compile_t* script;
-        ngx_array_t* values;
-        ngx_array_t* lengths;
-        ngx_str_t value;
-
-        script = (*cs_it)->script();
-        lengths = *script->lengths;
-        values = *script->values;
-
-        if (ngx_http_script_run(r, &value, lengths->elts, 0, values->elts)
-            == NULL) {
-          handler->Message(kError, "ngx_http_script_run error");
-          script_error = true;
-          break;
-        } else  {
-          args[(*cs_it)->index()] = str_to_string_piece(value);
-        }
-      }
-
-      const char* status = ParseAndSetOptions(args, script_line->n_args(),
-          r->pool, handler, driver_factory, script_line->scope(), NULL /*cf*/,
-          false /*compile scripts*/);
-
-      if (status != NULL) {
-        script_error = true;
-        handler->Message(kWarning,
-            "Error setting option value from script: '%s'", status);
-        break;
-      }
-    }
-  }
-
-  if (script_error) {
-    handler->Message(kWarning,
-        "Script error(s) in configuration, disabling optimization");
-    set_enabled(RewriteOptions::kEnabledOff);
-    return false;
-  }
-
-  return true;
-}
-
-void NgxRewriteOptions::CopyScriptLinesTo(
-    NgxRewriteOptions* destination) const {
-  destination->script_lines_ = script_lines_;
-}
-
-void NgxRewriteOptions::AppendScriptLinesTo(
-    NgxRewriteOptions* destination) const {
-  destination->script_lines_.insert(destination->script_lines_.end(),
-                                    script_lines_.begin(), script_lines_.end());
-}
-
 NgxRewriteOptions* NgxRewriteOptions::Clone() const {
   NgxRewriteOptions* options = new NgxRewriteOptions(
       StrCat("cloned from ", description()), thread_system());
-  this->CopyScriptLinesTo(options);
   options->Merge(*this);
   return options;
 }
 
-void NgxRewriteOptions::Merge(const RewriteOptions& src) {
-  SystemRewriteOptions::Merge(src);
-}
-
 const NgxRewriteOptions* NgxRewriteOptions::DynamicCast(
     const RewriteOptions* instance) {
   return dynamic_cast<const NgxRewriteOptions*>(instance);

src/ngx_rewrite_options.h

@@ -27,81 +27,13 @@ extern "C" {
   #include <ngx_http.h>
 }
 
-#include <vector>
-
-#include "net/instaweb/util/public/message_handler.h"
-#include "net/instaweb/util/public/ref_counted_ptr.h"
-#include "net/instaweb/util/public/stl_util.h"          // for STLDeleteElements
 #include "net/instaweb/rewriter/public/rewrite_options.h"
 #include "net/instaweb/system/public/system_rewrite_options.h"
 
-#define NGX_PAGESPEED_MAX_ARGS 10
-
 namespace net_instaweb {
 
 class NgxRewriteDriverFactory;
 
-class ScriptArgIndex {
- public:
-  explicit ScriptArgIndex(ngx_http_script_compile_t* script, int index)
-    : script_(script), index_(index) {
-      CHECK(script != NULL);
-      CHECK(index > 0 && index < NGX_PAGESPEED_MAX_ARGS);
-  }
-
-  virtual ~ScriptArgIndex() {}
-
-  ngx_http_script_compile_t* script() { return script_; }
-  int index() { return index_; }
-
- private:
-  // Not owned.
-  ngx_http_script_compile_t* script_;
-  int index_;
-};
-
-// Refcounted, because the ScriptArgIndexes inside data_ can be shared between
-// different rewriteoptions.
-class ScriptLine : public RefCounted<ScriptLine> {
- public:
-  explicit ScriptLine(StringPiece* args, int n_args,
-                      RewriteOptions::OptionScope scope)
-    : n_args_(n_args),
-      scope_(scope) {
-
-      for (int i = 0; i < n_args; i++) {
-        args_[i] = args[i];
-      }
-  }
-
-  virtual ~ScriptLine() {
-    STLDeleteElements(&data_);
-    data_.clear();
-  }
-
-  void AddScriptAndArgIndex(ngx_http_script_compile_t* script,
-                            int script_index) {
-    CHECK(script != NULL);
-    CHECK(script_index <  NGX_PAGESPEED_MAX_ARGS);
-    data_.push_back(new ScriptArgIndex(script, script_index));
-  }
-
-  int n_args() { return n_args_;}
-  StringPiece* args() { return args_;}
-  RewriteOptions::OptionScope scope() { return scope_; }
-  std::vector<ScriptArgIndex*>& data() {
-    return data_;
-  }
-
- private:
-  StringPiece args_[NGX_PAGESPEED_MAX_ARGS];
-  int n_args_;
-  RewriteOptions::OptionScope scope_;
-  std::vector<ScriptArgIndex*> data_;
-
-  DISALLOW_COPY_AND_ASSIGN(ScriptLine);
-};
-
 class NgxRewriteOptions : public SystemRewriteOptions {
  public:
   // See rewrite_options::Initialize and ::Terminate
@@ -124,23 +56,12 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   // on failure.
   //
   // pool is a memory pool for allocating error strings.
-  // cf is only required when compile_scripts is true
-  // when compile_scripts is true, the rewrite_options will be prepared
-  // for replacing any script $variables encountered in args. when false,
-  // script variables will be substituted using the prepared rewrite options.
   const char* ParseAndSetOptions(
       StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
-      NgxRewriteDriverFactory* driver_factory, OptionScope scope,
-      ngx_conf_t* cf, bool compile_scripts);
-  bool ExecuteScriptVariables(
-      ngx_http_request_t* r, MessageHandler* handler,
-      NgxRewriteDriverFactory* driver_factory);
-  void CopyScriptLinesTo(NgxRewriteOptions* destination) const;
-  void AppendScriptLinesTo(NgxRewriteOptions* destination) const;
+      NgxRewriteDriverFactory* driver_factory, OptionScope scope);
 
   // Make an identical copy of these options and return it.
   virtual NgxRewriteOptions* Clone() const;
-  virtual void Merge(const RewriteOptions& src);
 
   // Returns a suitably down cast version of 'instance' if it is an instance
   // of this class, NULL if not.
@@ -165,12 +86,6 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   const GoogleString& global_admin_path() const {
     return global_admin_path_.value();
   }
-  const std::vector<RefCountedPtr<ScriptLine> >& script_lines() const {
-    return script_lines_;
-  }
-  const bool& clear_inherited_scripts() const {
-    return clear_inherited_scripts_;
-  }
 
  private:
   // Helper methods for ParseAndSetOptions().  Each can:
@@ -214,10 +129,9 @@ class NgxRewriteOptions : public SystemRewriteOptions {
                              const char* id,
                              StringPiece option_name,
                              OptionScope scope,
-                             const char* help,
-                             bool safe_to_print) {
+                             const char* help) {
     AddProperty(default_value, offset, id, option_name, scope, help,
-                safe_to_print, ngx_properties_);
+                ngx_properties_);
   }
 
   Option<GoogleString> statistics_path_;
@@ -227,9 +141,6 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   Option<GoogleString> admin_path_;
   Option<GoogleString> global_admin_path_;
 
-  bool clear_inherited_scripts_;
-  std::vector<RefCountedPtr<ScriptLine> > script_lines_;
-
   // Helper for ParseAndSetOptions.  Returns whether the two directives equal,
   // ignoring case.
   bool IsDirective(StringPiece config_directive, StringPiece compare_directive);

src/ngx_server_context.cc

@@ -77,9 +77,4 @@ SystemRequestContext* NgxServerContext::NewRequestContext(
                                   str_to_string_piece(local_ip));
 }
 
-GoogleString NgxServerContext::FormatOption(StringPiece option_name,
-                                            StringPiece args) {
-  return StrCat("pagespeed ", option_name, " ", args, ";");
-}
-
 }  // namespace net_instaweb

src/ngx_server_context.h

@@ -55,8 +55,6 @@ class NgxServerContext : public SystemServerContext {
     return dynamic_cast<NgxMessageHandler*>(message_handler());
   }
 
-  virtual GoogleString FormatOption(StringPiece option_name, StringPiece args);
-
  private:
   NgxRewriteDriverFactory* ngx_factory_;
 

src/ngx_url_async_fetcher.cc

@@ -55,7 +55,6 @@ namespace net_instaweb {
                                          ngx_msec_t resolver_timeout,
                                          ngx_msec_t fetch_timeout,
                                          ngx_resolver_t* resolver,
-                                         int max_keepalive_requests,
                                          ThreadSystem* thread_system,
                                          MessageHandler* handler)
     : fetchers_count_(0),
@@ -64,8 +63,7 @@ namespace net_instaweb {
       byte_count_(0),
       thread_system_(thread_system),
       message_handler_(handler),
-      mutex_(NULL),
-      max_keepalive_requests_(max_keepalive_requests) {
+      mutex_(NULL) {
     resolver_timeout_ = resolver_timeout;
     fetch_timeout_ = fetch_timeout;
     ngx_memzero(&proxy_, sizeof(proxy_));
@@ -225,7 +223,7 @@ namespace net_instaweb {
                                  AsyncFetch* async_fetch) {
     async_fetch = EnableInflation(async_fetch);
     NgxFetch* fetch = new NgxFetch(url, async_fetch,
-          message_handler, log_);
+          message_handler, fetch_timeout_, log_);
     ScopedMutex lock(mutex_);
     pending_fetches_.Add(fetch);
     SendCmd('F');
@@ -251,7 +249,7 @@ namespace net_instaweb {
 
   // This is the read event which is called in the main thread.
   // It will do the real work. Add the work event and start the fetch.
-  void NgxUrlAsyncFetcher::CommandHandler(ngx_event_t* cmdev) {
+  void NgxUrlAsyncFetcher::CommandHandler(ngx_event_t *cmdev) {
     char command;
     int rc;
     ngx_connection_t* c = static_cast<ngx_connection_t*>(cmdev->data);

src/ngx_url_async_fetcher.h

@@ -53,8 +53,7 @@ class NgxUrlAsyncFetcher : public UrlAsyncFetcher {
   NgxUrlAsyncFetcher(
       const char* proxy, ngx_log_t* log, ngx_msec_t resolver_timeout,
       ngx_msec_t fetch_timeout, ngx_resolver_t* resolver,
-      int max_keepalive_requests, ThreadSystem* thread_system,
-      MessageHandler* handler);
+      ThreadSystem* thread_system, MessageHandler* handler);
 
   ~NgxUrlAsyncFetcher();
 
@@ -140,7 +139,6 @@ class NgxUrlAsyncFetcher : public UrlAsyncFetcher {
   ngx_connection_t* command_connection_;  // the command pipe
   int pipe_fd_;  // the write pipe end
   ngx_resolver_t* resolver_;
-  int max_keepalive_requests_;
   ngx_msec_t resolver_timeout_;
   ngx_msec_t fetch_timeout_;
 

test/nginx_system_test.sh

@@ -71,12 +71,12 @@ function keepalive_test() {
       if [ -z "$POST_DATA" ]; then
         curl -m 2 -S -s -v -H "Accept-Encoding: $accept_encoding" \
           -H "Host: $HOST_NAME" $URL $URL $URL $URL $URL > /dev/null \
-          2>>"$TEST_TMP/$CURL_LOG_FILE" || true
+          2>>"$TEST_TMP/$CURL_LOG_FILE"
       else
         curl -X POST --data "$POST_DATA" -m 2 -S -s -v \
           -H "Accept-Encoding: $accept_encoding" -H "Host: $HOST_NAME"\
           $URL $URL $URL $URL $URL > /dev/null \
-          2>>"$TEST_TMP/$CURL_LOG_FILE" || true
+          2>>"$TEST_TMP/$CURL_LOG_FILE"
       fi
     done
   done
@@ -92,16 +92,13 @@ function keepalive_test() {
     | grep -v "^\\* Connection.*left intact"\
     | grep -v "^} \\[data not shown"\
     | grep -v "^\\* upload completely sent off"\
-    | grep -v "^\\* Found bundle for host"\
     | grep -v "^\\* connected"\
     | grep -v "^\\* Found bundle for host"\
     | grep -v "^\\* Adding handle"\
     | grep -v "^\\* Curl_addHandleToPipeline"\
     | grep -v "^\\* - Conn "\
     | grep -v "^\\* Server "\
-    | grep -v "^\\*   Trying.*\\.\\.\\."\
-    | grep -v "^\\* Hostname was NOT found in DNS cache" \
-    || true)
+    | grep -v "^\\*   Trying.*\\.\\.\\.")
 
   # Nothing should remain after that.
   check [ -z "$OUT" ]
@@ -109,8 +106,7 @@ function keepalive_test() {
   # Filter the nginx log from our vhost from unimportant messages.
   OUT=$(cat "$TEST_TMP/$NGX_LOG_FILE"\
     | grep -v "closed keepalive connection$" \
-    | grep -v ".*Cache Flush.*" \
-    || true)
+    | grep -v ".*Cache Flush.*")
 
   # Nothing should remain after that.
   check [ -z "$OUT" ]
@@ -222,20 +218,24 @@ check_rewriting_status() {
   else
     check_not zgrep -q "pagespeed.ic" $OUT_CONTENTS_FILE
   fi
+  # Reset WGET_ARGS.
+  WGET_ARGS=""
 }
 
 # Helper method that obtains a gzipped response and verifies that rewriting
 # has happened. Also takes an extra parameter that identifies extra headers
 # to be added during wget.
 check_for_rewriting() {
-  WGET_ARGS="$GZIP_WGET_ARGS $1" check_rewriting_status true
+  WGET_ARGS="$GZIP_WGET_ARGS $1"
+  check_rewriting_status true
 }
 
 # Helper method that obtains a gzipped response and verifies that no rewriting
 # has happened. Also takes an extra parameter that identifies extra headers
 # to be added during wget.
 check_for_no_rewriting() {
-  WGET_ARGS="$GZIP_WGET_ARGS $1" check_rewriting_status false
+  WGET_ARGS="$GZIP_WGET_ARGS $1"
+  check_rewriting_status false
 }
 
 if $RUN_TESTS; then
@@ -250,12 +250,6 @@ else
   exit 4
 fi
 
-# check_stat in system_test_helpers.sh needs to know whether statstistics are
-# enabled, which is always the case for ngx_pagespeed.
-statistics_enabled=1
-CACHE_FLUSH_TEST="on"
-CACHE_PURGE_METHODS="PURGE GET"
-
 # run generic system tests
 SYSTEM_TEST_FILE="$MOD_PAGESPEED_DIR/src/net/instaweb/system/system_test.sh"
 
@@ -265,7 +259,7 @@ if [ ! -e "$SYSTEM_TEST_FILE" ] ; then
   exit 2
 fi
 
-PSA_JS_LIBRARY_URL_PREFIX="pagespeed_custom_static"
+PSA_JS_LIBRARY_URL_PREFIX="pagespeed_static"
 
 # An expected failure can be indicated like: "~In-place resource optimization~"
 PAGESPEED_EXPECTED_FAILURES="
@@ -410,12 +404,12 @@ check_from "$OUT" egrep -q \
   '^X-Page-Speed: [0-9]+[.][0-9]+[.][0-9]+[.][0-9]+-[0-9]+'
 
 start_test pagespeed is defaulting to more than PassThrough
-fetch_until $TEST_ROOT/bot_test.html 'fgrep -c .pagespeed.' 2
+fetch_until $TEST_ROOT/bot_test.html 'grep -c \.pagespeed\.' 2
 
 start_test 404s are served and properly recorded.
 NUM_404=$(scrape_stat resource_404_count)
 echo "Initial 404s: $NUM_404"
-WGET_ERROR=$(check_not $WGET -O /dev/null $BAD_RESOURCE_URL 2>&1)
+WGET_ERROR=$($WGET -O /dev/null $BAD_RESOURCE_URL 2>&1)
 check_from "$WGET_ERROR" fgrep -q "404 Not Found"
 
 # Check that the stat got bumped.
@@ -448,13 +442,16 @@ if [ "$HOSTNAME" = "localhost:$PRIMARY_PORT" ] ; then
   ALT_STAT_URL=$(echo $STATISTICS_URL | sed s#localhost#$NON_LOCAL_IP#)
 
   echo "wget $ALT_STAT_URL >& $TEMPDIR/alt_stat_url.$$"
-  check_error_code 8 wget $ALT_STAT_URL >& "$TEMPDIR/alt_stat_url.$$"
+  wget $ALT_STAT_URL >& "$TEMPDIR/alt_stat_url.$$"
+  check [ $? = 8 ]
   rm -f "$TEMPDIR/alt_stat_url.$$"
 
   ALT_CE_URL="$ALT_STAT_URL.pagespeed.ce.8CfGBvwDhH.css"
-  check_error_code 8 wget -O - $ALT_CE_URL  >& "$TEMPDIR/alt_ce_url.$$"
-  check_error_code 8 wget -O - --header="Host: $HOSTNAME" $ALT_CE_URL \
-    >& "$TEMPDIR/alt_ce_url.$$"
+  wget -O - $ALT_CE_URL  >& "$TEMPDIR/alt_ce_url.$$"
+  check [ $? = 8 ]
+
+  wget -O - --header="Host: $HOSTNAME" $ALT_CE_URL >& "$TEMPDIR/alt_ce_url.$$"
+  check [ $? = 8 ]
   rm -f "$TEMPDIR/alt_ce_url.$$"
 
   # Even though we don't have a cookie, we will conservatively avoid
@@ -525,59 +522,7 @@ if [ "$HOSTNAME" = "localhost:$PRIMARY_PORT" ] ; then
   AUTH=""
 fi
 
-start_test "Custom statistics paths in server block"
-
-# Served on normal paths by default.
-URL="inherit-paths.example.com/ngx_pagespeed_statistics"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
-check_from "$OUT" fgrep -q cache_time_us
-
-URL="inherit-paths.example.com/ngx_pagespeed_message"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
-check_from "$OUT" fgrep -q Info
-
-URL="inherit-paths.example.com/pagespeed_console"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
-check_from "$OUT" fgrep -q console_div
-
-URL="inherit-paths.example.com/pagespeed_admin/"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
-check_from "$OUT" fgrep -q Admin
-
-# Not served on normal paths when overriden.
-URL="custom-paths.example.com/ngx_pagespeed_statistics"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET_DUMP $URL)
-check_not_from "$OUT" fgrep -q cache_time_us
-
-URL="custom-paths.example.com/ngx_pagespeed_message"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET_DUMP $URL)
-check_not_from "$OUT" fgrep -q Info
-
-URL="custom-paths.example.com/pagespeed_console"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET_DUMP $URL)
-check_not_from "$OUT" fgrep -q console_div
-
-URL="custom-paths.example.com/pagespeed_admin/"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET_DUMP $URL)
-check_not_from "$OUT" fgrep -q Admin
-
-# Served on custom paths when overriden
-URL="custom-paths.example.com/custom_pagespeed_statistics"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
-check_from "$OUT" fgrep -q cache_time_us
-
-URL="custom-paths.example.com/custom_pagespeed_message"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
-check_from "$OUT" fgrep -q Info
-
-URL="custom-paths.example.com/custom_pagespeed_console"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
-check_from "$OUT" fgrep -q console_div
-
-URL="custom-paths.example.com/custom_pagespeed_admin/"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
-check_from "$OUT" fgrep -q Admin
-
+WGET_ARGS=""
 function gunzip_grep_0ff() {
   gunzip - | fgrep -q "color:#00f"
   echo $?
@@ -604,21 +549,21 @@ start_test Accept bad query params and headers
 # The examples page should have this EXPECTED_EXAMPLES_TEXT on it.
 EXPECTED_EXAMPLES_TEXT="PageSpeed Examples Directory"
 OUT=$(wget -O - $EXAMPLE_ROOT)
-check_from "$OUT" fgrep -q "$EXPECTED_EXAMPLES_TEXT"
+check_from "$OUT" grep "$EXPECTED_EXAMPLES_TEXT"
 
 # It should still be there with bad query params.
 OUT=$(wget -O - $EXAMPLE_ROOT?PageSpeedFilters=bogus)
-check_from "$OUT" fgrep -q "$EXPECTED_EXAMPLES_TEXT"
+check_from "$OUT" grep "$EXPECTED_EXAMPLES_TEXT"
 
 # And also with bad request headers.
 OUT=$(wget -O - --header=PageSpeedFilters:bogus $EXAMPLE_ROOT)
-check_from "$OUT" fgrep -q "$EXPECTED_EXAMPLES_TEXT"
+check_from "$OUT" grep "$EXPECTED_EXAMPLES_TEXT"
 
 # Tests that an origin header with a Vary header other than Vary:Accept-Encoding
 # loses that header when we are not respecting vary.
 start_test Vary:User-Agent on resources is held by our cache.
 URL="$TEST_ROOT/vary/no_respect/index.html"
-fetch_until -save $URL 'fgrep -c .pagespeed.cf.' 1
+fetch_until -save $URL 'grep -c \.pagespeed\.cf\.' 1
 
 # Extract out the rewritten CSS file from the HTML saved by fetch_until
 # above (see -save and definition of fetch_until).  Fetch that CSS
@@ -679,12 +624,14 @@ echo "JS_URL=\$\(egrep -o http://.*[.]pagespeed.*[.]js $FETCHED\)=\"$JS_URL\""
 JS_HEADERS=$($WGET -O /dev/null -q -S --header='Accept-Encoding: gzip' \
   $JS_URL 2>&1)
 echo JS_HEADERS=$JS_HEADERS
-check_200_http_response "$JS_HEADERS"
+check_from "$JS_HEADERS" egrep -qi 'HTTP/1[.]. 200 OK'
 check_from "$JS_HEADERS" fgrep -qi 'Content-Encoding: gzip'
 check_from "$JS_HEADERS" fgrep -qi 'Vary: Accept-Encoding'
 check_from "$JS_HEADERS" egrep -qi '(Etag: W/"0")|(Etag: W/"0-gzip")'
 check_from "$JS_HEADERS" fgrep -qi 'Last-Modified:'
 
+WGET_ARGS="" # Done with test_filter, so clear WGET_ARGS.
+
 start_test Respect X-Forwarded-Proto when told to
 FETCHED=$OUTDIR/x_forwarded_proto
 URL=$SECONDARY_HOSTNAME/mod_pagespeed_example/?PageSpeedFilters=add_base_tag
@@ -697,20 +644,21 @@ check fgrep -q '<base href="https://' $FETCHED
 test_filter remove_comments retains appropriate comments.
 URL="$SECONDARY_HOSTNAME/mod_pagespeed_example/$FILE"
 check run_wget_with_args $URL --header=Host:retaincomment.example.com
-check fgrep -q retained $FETCHED        # RetainComment directive
+check grep -q retained $FETCHED        # RetainComment directive
 
 # Make sure that when in PreserveURLs mode that we don't rewrite URLs. This is
 # non-exhaustive, the unit tests should cover the rest.
 # Note: We block with psatest here because this is a negative test.  We wouldn't
 # otherwise know how many wget attempts should be made.
-start_test PreserveURLs on prevents URL rewriting
 WGET_ARGS="--header=X-PSA-Blocking-Rewrite:psatest"
 WGET_ARGS+=" --header=Host:preserveurls.example.com"
 
+start_test PreserveURLs on prevents URL rewriting
 FILE=preserveurls/on/preserveurls.html
 URL=$SECONDARY_HOSTNAME/mod_pagespeed_test/$FILE
 FETCHED=$OUTDIR/preserveurls.html
 check run_wget_with_args $URL
+WGET_ARGS=""
 check_not fgrep -q .pagespeed. $FETCHED
 
 # When PreserveURLs is off do a quick check to make sure that normal rewriting
@@ -744,8 +692,10 @@ URL="$SECONDARY_HOSTNAME/mod_pagespeed_test/no_transform/BikeCrashIcn.png"
 wget -O - -S $URL $WGET_ARGS &> $FETCHED
 # Make sure that the no-transfrom header is still there
 check grep -q 'Cache-Control:.*no-transform' $FETCHED
+WGET_ARGS=""
 
 start_test respect vary user-agent
+WGET_ARGS=""
 URL="$SECONDARY_HOSTNAME/mod_pagespeed_test/vary/index.html"
 URL+="?PageSpeedFilters=inline_css"
 FETCH_CMD="$WGET_DUMP --header=Host:respectvary.example.com $URL"
@@ -774,8 +724,9 @@ start_test rewrite on Cache-control: no-transform
 URL=$TEST_ROOT/disable_no_transform/index.html?PageSpeedFilters=inline_css
 fetch_until -save -recursive $URL 'grep -c style' 2
 
+WGET_ARGS=""
 start_test ShardDomain directive in location block
-fetch_until -save $TEST_ROOT/shard/shard.html 'fgrep -c .pagespeed.ce' 4
+fetch_until -save $TEST_ROOT/shard/shard.html 'grep -c \.pagespeed\.' 4
 check [ $(grep -ce href=\"http://shard1 $FETCH_FILE) = 2 ];
 check [ $(grep -ce href=\"http://shard2 $FETCH_FILE) = 2 ];
 
@@ -853,6 +804,7 @@ start_test combine_javascript with long URL still works
 URL=$TEST_ROOT/combine_js_very_many.html?PageSpeedFilters=combine_javascript
 fetch_until $URL 'grep -c src=' 4
 
+WGET_ARGS=""
 start_test UseExperimentalJsMinifier
 URL="$TEST_ROOT/experimental_js_minifier/index.html"
 URL+="?PageSpeedFilters=rewrite_javascript"
@@ -919,9 +871,10 @@ PageSpeedFilters=inline_javascript,debug"
 OUTFILE=$OUTDIR/blocking_rewrite.out.html
 $WGET_DUMP --header 'X-PSA-Blocking-Rewrite: psatest' $URL > $OUTFILE
 check egrep -q 'script[[:space:]]src=' $OUTFILE
-EXPECTED_COMMENT_LINE="<!--The preceding resource was not rewritten \
-because its domain (www.gstatic.com) is not authorized-->"
-check [ $(grep -o "$EXPECTED_COMMENT_LINE" $OUTFILE | wc -l) -eq 1 ]
+EXPECTED_COMMENT_LINE="<!--InlineJs: Cannot create resource: either its \
+domain is unauthorized and InlineUnauthorizedResources is not enabled, \
+or it cannot be fetched (check the server logs)-->"
+check grep -q "$EXPECTED_COMMENT_LINE" $OUTFILE
 
 start_test inline_unauthorized_resources allows inlining
 HOST_NAME="http://unauthorizedresources.example.com"
@@ -945,9 +898,10 @@ PageSpeedFilters=inline_css,debug"
 OUTFILE=$OUTDIR/blocking_rewrite.out.html
 $WGET_DUMP --header 'X-PSA-Blocking-Rewrite: psatest' $URL > $OUTFILE
 check egrep -q 'link[[:space:]]rel=' $OUTFILE
-EXPECTED_COMMENT_LINE="<!--The preceding resource was not rewritten \
-because its domain (www.google.com) is not authorized-->"
-check [ $(grep -o "$EXPECTED_COMMENT_LINE" $OUTFILE | wc -l) -eq 1 ]
+EXPECTED_COMMENT_LINE="<!--InlineCss: Cannot create resource: either its \
+domain is unauthorized and InlineUnauthorizedResources is not enabled, \
+or it cannot be fetched (check the server logs)-->"
+check grep -q "$EXPECTED_COMMENT_LINE" $OUTFILE
 
 start_test inline_unauthorized_resources allows inlining
 HOST_NAME="http://unauthorizedresources.example.com"
@@ -1045,6 +999,7 @@ wget -O - --header 'X-PSA-Blocking-Rewrite: psatest' $URL > $TEMPDIR/flush.$$
 check [ `wget -O - $URL | grep -o 'link rel="subresource"' | wc -l` = 0 ]
 rm -f $TEMPDIR/flush.$$
 
+WGET_ARGS=""
 start_test Respect custom options on resources.
 IMG_NON_CUSTOM="$EXAMPLE_ROOT/images/xPuzzle.jpg.pagespeed.ic.fakehash.jpg"
 IMG_CUSTOM="$TEST_ROOT/custom_options/xPuzzle.jpg.pagespeed.ic.fakehash.jpg"
@@ -1097,26 +1052,18 @@ WGET_ARGS="--save-headers"
 # We should be able to fetch the original ...
 echo  http_proxy=$SECONDARY_HOSTNAME $WGET --save-headers -O - $ORIGINAL
 OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET --save-headers -O - $ORIGINAL 2>&1)
-check_200_http_response "$OUT"
+check_from "$OUT" fgrep " 200 OK"
 # ... AND the rewritten version.
 echo  http_proxy=$SECONDARY_HOSTNAME $WGET --save-headers -O - $FILTERED
 OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET --save-headers -O - $FILTERED 2>&1)
-check_200_http_response "$OUT"
+check_from "$OUT" fgrep " 200 OK"
 
 start_test MapProxyDomain
 # depends on MapProxyDomain in pagespeed_test.conf.template
-LEAF="proxy_external_resource.html?PageSpeedFilters=-inline_images"
-URL="$EXAMPLE_ROOT/$LEAF"
+URL=$EXAMPLE_ROOT/proxy_external_resource.html
 echo Rewrite HTML with reference to a proxyable image.
-fetch_until -save -recursive $URL 'grep -c 1.gif.pagespeed' 1 --save-headers
-PAGESPEED_GIF=$(grep -o '/*1.gif.pagespeed[^"]*' $WGET_DIR/$LEAF)
-check_from "$PAGESPEED_GIF" grep "gif$"
-
-echo "If the next line fails, look in $WGET_DIR/wget_output.txt and you should"
-echo "see a 404.  This represents a failed attempt to download the proxied gif."
-# TODO(jefftk): debug why this test sometimes fails with the native fetcher.
-# https://github.com/pagespeed/ngx_pagespeed/issues/774
-check test -f "$WGET_DIR$PAGESPEED_GIF"
+fetch_until -save -recursive $URL?PageSpeedFilters=-inline_images \
+    'grep -c 1.gif.pagespeed' 1
 
 start_test OptimizeForBandwidth
 # We use blocking-rewrite tests because we want to make sure we don't
@@ -1156,11 +1103,12 @@ test_optimize_for_bandwidth core_filters/rewrite_css.html \
 #
 # With the proper hash, we'll get a long cache lifetime.
 SECONDARY_HOST="http://mpd.example.com/gstatic_images"
-PROXIED_IMAGE="$SECONDARY_HOST$PAGESPEED_GIF"
+PROXIED_IMAGE="$SECONDARY_HOST/$(basename $WGET_DIR/*1.gif.pagespeed*)"
+WGET_ARGS="--save-headers"
 
 start_test $PROXIED_IMAGE expecting one year cache.
 http_proxy=$SECONDARY_HOSTNAME fetch_until $PROXIED_IMAGE \
-    "grep -c max-age=31536000" 1 --save-headers
+    "grep -c max-age=31536000" 1
 
 # With the wrong hash, we'll get a short cache lifetime (and also no output
 # cache hit.
@@ -1168,12 +1116,14 @@ WRONG_HASH="0"
 PROXIED_IMAGE="$SECONDARY_HOST/1.gif.pagespeed.ce.$WRONG_HASH.jpg"
 start_test Fetching $PROXIED_IMAGE expecting short private cache.
 http_proxy=$SECONDARY_HOSTNAME fetch_until $PROXIED_IMAGE \
-    "grep -c max-age=300,private" 1 --save-headers
+    "grep -c max-age=300,private" 1
+
+WGET_ARGS=""
 
 start_test ShowCache without URL gets a form, inputs, preloaded UA.
 ADMIN_CACHE=$PRIMARY_SERVER/pagespeed_admin/cache
 OUT=$($WGET_DUMP $ADMIN_CACHE)
-check_from "$OUT" fgrep -q "<form>"
+check_from "$OUT" fgrep -q "<form "
 check_from "$OUT" fgrep -q "<input "
 check_from "$OUT" fgrep -q "Cache-Control: max-age=0, no-cache"
 # Preloaded user_agent value field leading with "Mozilla" set in
@@ -1181,8 +1131,8 @@ check_from "$OUT" fgrep -q "Cache-Control: max-age=0, no-cache"
 check_from "$OUT" fgrep -q 'name=user_agent value="Mozilla'
 
 start_test ShowCache with bogus URL gives a 404
-check_error_code 8 \
-  wget $PRIMARY_SERVER/pagespeed_cache?url=bogus_format >& /dev/null
+wget $PRIMARY_SERVER/pagespeed_cache?url=bogus_format >& /dev/null
+check [ $? = 8 ]
 
 start_test ShowCache with valid, present URL, with unique options.
 options="PageSpeedImageInlineMaxBytes=6765"
@@ -1224,17 +1174,38 @@ check_from "$OUT" grep $'^Cache-Control: max-age=0, no-cache\r$'
 
 start_test server-side includes
 fetch_until -save $TEST_ROOT/ssi/ssi.shtml?PageSpeedFilters=combine_css \
-    'fgrep -c .pagespeed.' 1
+    'grep -c \.pagespeed\.' 1
 check [ $(grep -ce $combine_css_filename $FETCH_FILE) = 1 ];
 
 start_test Embed image configuration in rewritten image URL.
+
+# The apache test names these virtual hosts as embed_config_*, which is
+# unfortunate as underscores are not allowed in domain names.  Apache doesn't
+# care, and nginx doesn't either, except when you're proxying.  So you can do:
+#
+#    GET /embed_config.html HTTP/1.1
+#    Host: embed_config_html.example.com
+#
+# and it will work fine, but if you do:
+#
+#    GET http://embed_config_html.example.com/embed_config.html HTTP/1.1
+#
+# then nginx will close the connection before you can even give it a Host
+# header.  I've modified this test code to replace embed_config_ with
+# embed-config-, but the html file on disk has the underscore versions.  Let's
+# make a new html file that has the hyphen version:
+cat "$SERVER_ROOT/mod_pagespeed_test/embed_config.html" | \
+  sed s/embed_config_/embed-config-/g > \
+  "$SERVER_ROOT/mod_pagespeed_test/embed-config.html"
+
 # The embedded configuration is placed between the "pagespeed" and "ic", e.g.
 # *xPuzzle.jpg.pagespeed.gp+jp+pj+js+rj+rp+rw+ri+cp+md+iq=73.ic.oFXPiLYMka.jpg
 # We use a regex matching "gp+jp+pj+js+rj+rp+rw+ri+cp+md+iq=73" rather than
 # spelling it out to avoid test regolds when we add image filter IDs.
+WGET_ARGS="--save-headers"
 http_proxy=$SECONDARY_HOSTNAME fetch_until -save -recursive \
-    http://embed-config-html.example.org/embed_config.html \
-    'fgrep -c .pagespeed.' 3 --save-headers
+    http://embed-config-html.example.org/embed-config.html \
+    'grep -c \.pagespeed\.' 3
 
 # with the default rewriters in vhost embed-config-resources.example.com
 # the image will be >200k.  But by enabling resizing & compression 73
@@ -1303,6 +1274,7 @@ function embed_image_config_post_flush() {
   # places where we flush cache, which requires sleeps since the
   # cache-flush is poll driven.
   start_test Embed image/css configuration decoding with clear cache.
+  WGET_ARGS=""
   echo Looking for $EMBED_CONFIGURATION_IMAGE expecting \
       $EMBED_CONFIGURATION_IMAGE_LENGTH bytes
   http_proxy=$SECONDARY_HOSTNAME fetch_until "$EMBED_CONFIGURATION_IMAGE" \
@@ -1493,7 +1465,7 @@ else
   EXPECTED="Serf status 111"
 fi
 for i in {1..100}; do
-  ERRS=$(grep -c "$EXPECTED" $FETCHER_REFUSED_PATH || true)
+  ERRS=$(grep -c "$EXPECTED" $FETCHER_REFUSED_PATH)
   if [ $ERRS -ge 1 ]; then
     break;
   fi;
@@ -1503,7 +1475,7 @@ done;
 echo "."
 # Kill the log monitor silently.
 kill $TAIL_PID
-wait $TAIL_PID 2> /dev/null || true
+wait $TAIL_PID 2> /dev/null
 check [ $ERRS -ge 1 ]
 
 # TODO(jefftk): when we support ListOutstandingUrlsOnError uncomment the below
@@ -1522,6 +1494,7 @@ check [ $ERRS -ge 1 ]
 #
 # This rewrites the CSS, absolutifying the embedded relative image URL
 # reference based on the the main server host.
+WGET_ARGS=""
 start_test Relative images embedded in a CSS file served from a mapped domain
 DIR="mod_pagespeed_test/map_css_embedded"
 URL="http://www.example.com/$DIR/issue494.html"
@@ -1578,8 +1551,7 @@ OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET -O /dev/null $ALLOWED 2>&1)
 check_from "$OUT" fgrep -q "200 OK"
 # .cf. is forbidden
 FORBIDDEN=$FORBIDDEN_STYLES_ROOT/A.all_styles.css.pagespeed.cf.UH8L-zY4b4.css
-OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET -O /dev/null $FORBIDDEN \
-  2>&1)
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET -O /dev/null $FORBIDDEN 2>&1)
 check_from "$OUT" fgrep -q "404 Not Found"
 # The image will be optimized but NOT resized to the much smaller size,
 # so it will be >200k (optimized) rather than <20k (resized).
@@ -1657,6 +1629,7 @@ function test_forbid_all_disabled() {
   $WGET $WGET_ARGS -q -O $OUTFILE $HEADER $URL
   check     egrep -q '<style>.yellow'        $OUTFILE
   rm -f $OUTFILE
+  WGET_ARGS=""
 }
 start_test ForbidAllDisabledFilters baseline check.
 test_forbid_all_disabled "" ""
@@ -1824,11 +1797,11 @@ check_not_from "$OUT" grep 'pagespeed_beacon.*exptid'
 # order they're defined in the config file.
 start_test Resource urls are rewritten to include experiment indexes.
 http_proxy=$SECONDARY_HOSTNAME \
-  fetch_until $EXP_EXTEND_CACHE \
-    "fgrep -c .pagespeed.a.ic." 1 --header=Cookie:PageSpeedExperiment=7
+  WGET_ARGS="--header Cookie:PageSpeedExperiment=7" fetch_until $EXP_EXTEND_CACHE \
+    "fgrep -c .pagespeed.a.ic." 1
 http_proxy=$SECONDARY_HOSTNAME \
-  fetch_until $EXP_EXTEND_CACHE \
-    "fgrep -c .pagespeed.b.ic." 1 --header=Cookie:PageSpeedExperiment=2
+  WGET_ARGS="--header Cookie:PageSpeedExperiment=2" fetch_until $EXP_EXTEND_CACHE \
+    "fgrep -c .pagespeed.b.ic." 1
 OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=7' \
       $EXP_EXTEND_CACHE)
 check_from "$OUT" fgrep ".pagespeed.a.ic."
@@ -1951,51 +1924,6 @@ check [ `grep -c '<style>[.]big{[^}]*}</style>' $FETCH_UNTIL_OUTFILE` = 1 ]
 check [ `grep -c '<style>[.]blue{[^}]*}[.]bold{[^}]*}</style>' \
   $FETCH_UNTIL_OUTFILE` = 1 ]
 
-# Now repeat the critical_css_filter test on a host that processes post data via
-# temp files to test that ngx_pagespeed specific code path.
-test_filter prioritize_critical_css Able to read POST data from temp file.
-URL="http://beacon-post-temp-file.example.com/mod_pagespeed_example/prioritize_critical_css.html"
-http_proxy=$SECONDARY_HOSTNAME\
-  fetch_until -save $URL 'fgrep -c pagespeed.criticalCssBeaconInit' 1
-check [ $(fgrep -o ".very_large_class_name_" $FETCH_FILE | wc -l) -eq 36 ]
-CALL_PAT=".*criticalCssBeaconInit("
-SKIP_ARG="[^,]*,"
-CAPTURE_ARG="'\([^']*\)'.*"
-BEACON_PATH=$(sed -n "s/${CALL_PAT}${CAPTURE_ARG}/\1/p" $FETCH_FILE)
-ESCAPED_URL=$( \
-  sed -n "s/${CALL_PAT}${SKIP_ARG}${CAPTURE_ARG}/\1/p" $FETCH_FILE)
-OPTIONS_HASH=$( \
-  sed -n "s/${CALL_PAT}${SKIP_ARG}${SKIP_ARG}${CAPTURE_ARG}/\1/p" $FETCH_FILE)
-NONCE=$( \
-  sed -n "s/${CALL_PAT}${SKIP_ARG}${SKIP_ARG}${SKIP_ARG}${CAPTURE_ARG}/\1/p" \
-  $FETCH_FILE)
-BEACON_URL="http://${SECONDARY_HOSTNAME}${BEACON_PATH}?url=${ESCAPED_URL}"
-BEACON_DATA="oh=${OPTIONS_HASH}&n=${NONCE}&cs=.big,.blue,.bold,.foo"
-
-OUT=$(wget -q  --save-headers -O - --no-http-keep-alive \
-      --post-data "$BEACON_DATA" "$BEACON_URL" \
-      --header "Host:beacon-post-temp-file.example.com")
-check_from "$OUT" grep '^HTTP/1.1 204'
-
-# Now make sure we see the correct critical css rules.
-http_proxy=$SECONDARY_HOSTNAME\
-  fetch_until $URL \
-    'grep -c <style>[.]blue{[^}]*}</style>' 1
-http_proxy=$SECONDARY_HOSTNAME\
-  fetch_until $URL \
-    'grep -c <style>[.]big{[^}]*}</style>' 1
-http_proxy=$SECONDARY_HOSTNAME\
-  fetch_until $URL \
-    'grep -c <style>[.]blue{[^}]*}[.]bold{[^}]*}</style>' 1
-http_proxy=$SECONDARY_HOSTNAME\
-  fetch_until -save $URL \
-    'grep -c <style>[.]foo{[^}]*}</style>' 1
-# The last one should also have the other 3, too.
-check [ `grep -c '<style>[.]blue{[^}]*}</style>' $FETCH_UNTIL_OUTFILE` = 1 ]
-check [ `grep -c '<style>[.]big{[^}]*}</style>' $FETCH_UNTIL_OUTFILE` = 1 ]
-check [ `grep -c '<style>[.]blue{[^}]*}[.]bold{[^}]*}</style>' \
-  $FETCH_UNTIL_OUTFILE` = 1 ]
-
 # This test checks that the ClientDomainRewrite directive can turn on.
 start_test ClientDomainRewrite on directive
 HOST_NAME="http://client-domain-rewrite.example.com"
@@ -2005,6 +1933,7 @@ MATCHES=$(echo "$RESPONSE_OUT" | grep -c pagespeed\.clientDomainRewriterInit)
 check [ $MATCHES -eq 1 ]
 
 # Verify rendered image dimensions test.
+WGET_ARGS=""
 start_test resize_rendered_image_dimensions with critical images beacon
 HOST_NAME="http://renderedimagebeacon.example.com"
 URL="$HOST_NAME/mod_pagespeed_test/image_rewriting/image_resize_using_rendered_dimensions.html"
@@ -2077,13 +2006,15 @@ check grep -q "Cache-Control: max-age=0, no-cache" $FETCH_UNTIL_OUTFILE
 
 # 3. We do not get an instrumented page if the wrong key is present.
 WGET_ARGS="--header=\"PS-ShouldBeacon: wrong_rebeaconing_key\""
-OUT3=$(http_proxy=$SECONDARY_HOSTNAME check_not \
+OUT3=$(http_proxy=$SECONDARY_HOSTNAME \
           $WGET_DUMP $WGET_ARGS $URL)
 check_not_from "$OUT3" egrep -q "pagespeed\.criticalCssBeaconInit"
 check_from "$OUT3" grep -q "Cache-Control: private, max-age=3000"
 
+
 # Verify that we can send a critical image beacon and that lazyload_images
 # does not try to lazyload the critical images.
+WGET_ARGS=""
 start_test lazyload_images,rewrite_images with critical images beacon
 HOST_NAME="http://imagebeacon.example.com"
 URL="$HOST_NAME/mod_pagespeed_test/image_rewriting/rewrite_images.html"
@@ -2162,12 +2093,15 @@ check_not fgrep -q "interesting_color" $FETCH_FILE
 # c) selectors that don't depend on flattening should appear in the selector
 #    list.
 check [ $(fgrep -c "non_flattened_selector" $FETCH_FILE) -eq 1 ]
-EXPECTED_IMPORT_FAILURE_LINE="<!--Flattening failed: Cannot import \
-http://www.google.com/css/maia.css as it is on an unauthorized domain-->"
-check [ $(grep -o "$EXPECTED_IMPORT_FAILURE_LINE" $FETCH_FILE | wc -l) -eq 1 ]
-EXPECTED_COMMENT_LINE="<!--The preceding resource was not rewritten \
-because its domain (www.google.com) is not authorized-->"
-check [ $(grep -o "$EXPECTED_COMMENT_LINE" $FETCH_FILE | wc -l) -eq 1 ]
+EXPECTED_IMPORT_FAILURE_LINE="<!--Flattening failed: Cannot import "
+EXPECTED_IMPORT_FAILURE_LINE+="http://www.google.com/css/maia.css: is it on "
+EXPECTED_IMPORT_FAILURE_LINE+="an unauthorized domain?-->"
+check grep -q "$EXPECTED_IMPORT_FAILURE_LINE" $FETCH_FILE
+EXPECTED_COMMENT_LINE="<!--CriticalCssBeacon: Cannot create resource: either "
+EXPECTED_COMMENT_LINE+="its domain is unauthorized and "
+EXPECTED_COMMENT_LINE+="InlineUnauthorizedResources is not enabled, or it "
+EXPECTED_COMMENT_LINE+="cannot be fetched (check the server logs)-->"
+check grep -q "$EXPECTED_COMMENT_LINE" $FETCH_FILE
 
 start_test inline_unauthorized_resources allows unauthorized css selectors
 HOST_NAME="http://unauthorizedresources.example.com"
@@ -2216,7 +2150,7 @@ keepalive_test "keepalive-beacon-post.example.com" "/ngx_pagespeed_beacon"\
 
 start_test keepalive with static resources
 keepalive_test "keepalive-static.example.com"\
-  "/pagespeed_custom_static/js_defer.0.js" ""
+  "/pagespeed_static/js_defer.0.js" ""
 
 # Test for MaxCombinedCssBytes. The html used in the test, 'combine_css.html',
 # has 4 CSS files in the following order.
@@ -2243,6 +2177,7 @@ check [ $(grep -c 'styles/big.css\"' $FETCH_UNTIL_OUTFILE) = 1 ]
 # response-headers from the server.  The reponse-headers from Serf never seem to
 # include the Connection header.  So we have to pick a JS file that is not
 # otherwise used after cache is flushed in this block.
+WGET_ARGS=""
 start_test Sane Connection header
 URL="$TEST_ROOT/normal.js"
 fetch_until -save $URL 'grep -c W/\"PSA-aj-' 1 --save-headers
@@ -2250,7 +2185,7 @@ CONNECTION=$(extract_headers $FETCH_UNTIL_OUTFILE | fgrep "Connection:")
 check_not_from "$CONNECTION" fgrep -qi "Keep-Alive, Keep-Alive"
 check_from "$CONNECTION" fgrep -qi "Keep-Alive"
 
-start_test pagespeed_custom_static defer js served with correct headers.
+start_test pagespeed_static defer js served with correct headers.
 # First, determine which hash js_defer is served with. We need a correct hash
 # to get it served up with an Etag, which is one of the things we want to test.
 URL="$HOSTNAME/mod_pagespeed_example/defer_javascript.html?PageSpeed=on&PageSpeedFilters=defer_javascript"
@@ -2295,8 +2230,11 @@ function test_ipro_for_browser_webp() {
   IN_ACCEPT="$1"; shift
   IMAGE_TYPE="$1"; shift
   OUT_CONTENT_TYPE="$1"; shift
-  OUT_VARY="${1-}"; shift || true
-  OUT_CC="${1-}"; shift || true
+  OUT_VARY="${1-}"; shift
+  OUT_CC="${1-}"; shift
+  WGET_ARGS="--save-headers \
+             ${IN_UA:+--user-agent $IN_UA} \
+             ${IN_ACCEPT:+--header=Accept:image/$IN_ACCEPT}"
   # Remaining args are the expected headers (Name:Value), photo, or synthetic.
   if [ "$IMAGE_TYPE" = "photo" ]; then
     URL="http://ipro-for-browser.example.com/images/Puzzle.jpg"
@@ -2322,9 +2260,6 @@ function test_ipro_for_browser_webp() {
     TEST_ID+=" Cache-Control:${OUT_CC}."
   fi
   start_test $TEST_ID
-  WGET_ARGS="--save-headers \
-             ${IN_UA:+--user-agent $IN_UA} \
-             ${IN_ACCEPT:+--header=Accept:image/$IN_ACCEPT}"
   http_proxy=$SECONDARY_HOSTNAME \
     fetch_until -save $URL 'grep -c W/\"PSA-aj-' 1
   check_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" \
@@ -2405,6 +2340,36 @@ test_decent_browsers "New Opera" \
   "Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14"
 
 WGETRC=$OLD_WGETRC
+WGET_ARGS=""
+
+test_filter collapse_whitespace
+start_test Cookie options on: by default comments not removed, whitespace is
+HOST_NAME="http://options-by-cookies-enabled.example.com"
+URL="$HOST_NAME/mod_pagespeed_test/forbidden.html"
+echo wget $URL
+OUT="$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)"
+check_from     "$OUT" grep -q '<!--'
+check_not_from "$OUT" grep -q '  '
+start_test Cookie options on: set option by cookie takes effect
+echo wget --header=Cookie:PageSpeedFilters=+remove_comments $URL
+OUT="$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --no-cookies \
+       --header=Cookie:PageSpeedFilters=+remove_comments $URL)"
+check_not_from "$OUT" grep -q '<!--'
+check_not_from "$OUT" grep -q '  '
+start_test Cookie options off: by default comments nor whitespace removed
+HOST_NAME="http://options-by-cookies-disabled.example.com"
+URL="$HOST_NAME/mod_pagespeed_test/forbidden.html"
+echo wget $URL
+OUT="$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)"
+check_from "$OUT" grep -q '<!--'
+check_from "$OUT" grep -q '  '
+start_test Cookie options off: set option by cookie has no effect
+echo wget --header=Cookie:PageSpeedFilters=+remove_comments $URL
+OUT="$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --no-cookies \
+       --header=Cookie:PageSpeedFilters=+remove_comments $URL)"
+check_from "$OUT" grep -q '<!--'
+check_from "$OUT" grep -q '  '
+WGET_ARGS=""
 
 start_test Request Option Override : Correct values are passed
 HOST_NAME="http://request-option-override.example.com"
@@ -2448,10 +2413,10 @@ check_from "$OUT" grep -q '<!--'
 
 start_test JS gzip headers
 
-JS_URL="$HOSTNAME/pagespeed_custom_static/js_defer.$HASH.js"
+JS_URL="$HOSTNAME/pagespeed_static/js_defer.$HASH.js"
 JS_HEADERS=$($WGET -O /dev/null -q -S --header='Accept-Encoding: gzip' \
   $JS_URL 2>&1)
-check_200_http_response "$JS_HEADERS"
+check_from "$JS_HEADERS" egrep -qi 'HTTP/1[.]. 200 OK'
 check_from "$JS_HEADERS" fgrep -qi 'Content-Encoding: gzip'
 check_from "$JS_HEADERS" fgrep -qi 'Vary: Accept-Encoding'
 # Nginx's gzip module clears etags, which we don't want. Make sure we have it.
@@ -2639,19 +2604,15 @@ ETAG=$(extract_headers $FETCH_UNTIL_OUTFILE | awk '/ETag:/ {print $2}')
 echo $WGET_DUMP --header "If-None-Match: $ETAG" $URL
 OUTFILE=$OUTDIR/etags
 # Note: -o gets debug info which is the only place that 304 message is sent.
-check_not $WGET -o $OUTFILE -O /dev/null --header "If-None-Match: $ETAG" $URL
+$WGET -o $OUTFILE -O /dev/null --header "If-None-Match: $ETAG" $URL
 check fgrep -q "awaiting response... 304" $OUTFILE
 
-# Test if the warning messages are colored in message_history page.
-# We color the messages in message_history page to make it clearer to read.
-# Red for Error messages. Brown for Warning messages.
-# Orange for Fatal messages. Black by default.
-# Won't test Error messages and Fatal messages in this test.
-start_test Messages are colored in message_history
-INJECT=$($CURL --silent $HOSTNAME/?PageSpeed=Warning_trigger)
-OUT=$($WGET -q -O - $HOSTNAME/pagespeed_admin/message_history | \
-  grep Warning_trigger)
-check_from "$OUT" fgrep -q "color:brown;"
+start_test PageSpeed resources should have a content length.
+URL="$EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css"
+OUT=$($WGET_DUMP --save-headers $URL)
+check_from "$OUT" egrep -q $'^Content-Length: ([0-9])*\r$'
+check_not_from "$OUT" egrep -iq $'^Transfer-Encoding: chunked\r$'
+check_not_from "$OUT" egrep -iq $'^Connection: close\r$'
 
 start_test Downstream cache integration caching headers.
 URL="http://downstreamcacheresource.example.com/mod_pagespeed_example/images/"
@@ -2661,6 +2622,16 @@ check_from "$OUT" egrep -iq $'^Cache-Control: .*\r$'
 check_from "$OUT" egrep -iq $'^Expires: .*\r$'
 check_from "$OUT" egrep -iq $'^Last-Modified: .*\r$'
 
+# Verify that we can control pagespeed settings via a response
+# header passed from an origin to a reverse proxy.
+start_test Honor response header direcives from origin
+URL="http://rproxy.rmcomments.example.com/"
+URL+="mod_pagespeed_example/remove_comments.html"
+echo http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL ...
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" fgrep -q "remove_comments example"
+check_not_from "$OUT" fgrep -q "This comment will be removed"
+
 # Test handling of large HTML files. We first test with a cold cache, and verify
 # that we bail out of parsing and insert a script redirecting to
 # ?PageSpeed=off. This should also insert an entry into the property cache so
@@ -2691,8 +2662,9 @@ check_from "$OUT" grep -q "/mod_pagespeed_example"
 
 start_test Check keepalive after a 304 responses.
 # '-m 2' specifies that the whole operation is allowed to take 2 seconds max.
-check curl -vv -m 2 http://$PRIMARY_HOSTNAME/foo.css.pagespeed.ce.0.css \
+curl -vv -m 2 http://$PRIMARY_HOSTNAME/foo.css.pagespeed.ce.0.css \
     -H 'If-Modified-Since: Z' http://$PRIMARY_HOSTNAME/foo
+check [ $? = "0" ]
 
 start_test Date response header set
 OUT=$($WGET_DUMP $EXAMPLE_ROOT/combine_css.html)
@@ -2703,51 +2675,12 @@ OUT=$($WGET_DUMP --header=Host:date.example.com \
 check_from "$OUT" egrep -q '^Date: Fri, 16 Oct 2009 23:05:07 GMT'
 WGET_ARGS=
 
-#very basic tests to test gzip nesting configuration
-start_test Nested gzip gzip off
-URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/"
-HEADERS="--header=Accept-Encoding:gzip --header=Host:gzip-test1.example.com"
-OUT=$($WGET_DUMP -O /dev/null -S $HEADERS $URL 2>&1)
-check_not_from "$OUT" fgrep -qi 'Content-Encoding: gzip'
-check_not_from "$OUT" fgrep -qi 'Vary: Accept-Encoding'
-
-start_test Nested gzip gzip on
-URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/styles/big.css"
-HEADERS="--header=Accept-Encoding:gzip --header=Host:gzip-test1.example.com"
-OUT=$($WGET_DUMP -O /dev/null -S $HEADERS $URL 2>&1)
-check_from "$OUT" fgrep -qi 'Content-Encoding: gzip'
-check_from "$OUT" fgrep -qi 'Vary: Accept-Encoding'
-
-start_test Nested gzip pagespeed off
-URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/"
-HEADERS="--header=Accept-Encoding:gzip --header=Host:gzip-test2.example.com"
-OUT=$($WGET_DUMP -O /dev/null -S $HEADERS $URL 2>&1)
-check_not_from "$OUT" fgrep -qi 'Content-Encoding: gzip'
-check_not_from "$OUT" fgrep -qi 'Vary: Accept-Encoding'
-
-start_test Nested gzip pagespeed on
-URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/styles/big.css"
-HEADERS="--header=Accept-Encoding:gzip --header=Host:gzip-test2.example.com"
-OUT=$($WGET_DUMP -O /dev/null -S $HEADERS $URL 2>&1)
-check_from "$OUT" fgrep -qi 'Content-Encoding: gzip'
-check_from "$OUT" fgrep -qi 'Vary: Accept-Encoding'
-
-start_test Test that POST requests are rewritten.
-URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/rewrite_images.html"
-HEADERS="--header=Host:proxy-post.example.com --post-data=abcdefgh"
-OUT=$($WGET_DUMP -S $HEADERS $URL 2>&1)
-check_from "$OUT" fgrep -qi 'addInstrumentationInit'
-
 if [ "$NATIVE_FETCHER" != "on" ]; then
   start_test Test that we can rewrite an HTTPS resource.
   fetch_until $TEST_ROOT/https_fetch/https_fetch.html \
     'grep -c /https_gstatic_dot_com/1.gif.pagespeed.ce' 1
 fi
 
-start_test Base config has purging disabled.  Check error message syntax.
-OUT=$($WGET_DUMP "$HOSTNAME/pagespeed_admin/cache?purge=*")
-check_from "$OUT" fgrep -q "pagespeed EnableCachePurge on;"
-
 if $USE_VALGRIND; then
     # It is possible that there are still ProxyFetches outstanding
     # at this point in time. Give them a few extra seconds to allow

test/pagespeed_test.conf.template

@@ -27,7 +27,6 @@ http {
   proxy_cache_path "@@PROXY_CACHE@@"  levels=1:2   keys_zone=htmlcache:60m inactive=90m  max_size=50m;
   proxy_temp_path "@@TMP_PROXY_CACHE@@";
 
-  pagespeed ProcessScriptVariables on;
   pagespeed StatisticsPath /ngx_pagespeed_statistics;
   pagespeed GlobalStatisticsPath /ngx_pagespeed_global_statistics;
   pagespeed ConsolePath /pagespeed_console;
@@ -35,14 +34,6 @@ http {
   pagespeed AdminPath /pagespeed_admin;
   pagespeed GlobalAdminPath /pagespeed_global_admin;
 
-  pagespeed StaticAssetPrefix /pagespeed_custom_static/;
-
-  pagespeed MessageBufferSize 200000;
-  # Increase the default fetcher timeout to resolve sporadic flakeyness when
-  # the native fetcher uses 8.8.8.8 to resolve.
-  pagespeed FetcherTimeoutMs 10000;
-  pagespeed NativeFetcherMaxKeepaliveRequests 50;
-
   root "@@SERVER_ROOT@@";
 
   # Block 5a: Decide on Cache-Control header value to use for outgoing
@@ -82,7 +73,6 @@ http {
     # test ModPagespeedMaxCacheableContentLength, i.e.,
     # max_cacheable_response_content_length.
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name max-cacheable-content-length.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -102,7 +92,6 @@ http {
     # running on the PRIMARY_PORT when the response is not available in
     # the cache. It also services purge requests from the upstream server.
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name proxy_cache.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -161,9 +150,9 @@ http {
       set $bypass_cache "1";
     }
 
-    # For testing purposes, we never generate values that will result in a beacon
+    # For testing purposes, we never generate values that will result in a beacon 
     # unless a test request it via "X-Allow-Beacon: yes" in its request header.
-    # This is for testing purposes, note that in a production environment,
+    # This is for testing purposes, note that in a production environment, 
     # you want 'set_random $rand 0 100;' unconditionally!
     set $rand 5;
 
@@ -185,7 +174,6 @@ http {
     # Block 4: Location block for purge requests.
     location ~ /purge(/.*) {
       allow 127.0.0.1;
-      allow ::1;
       deny all;
       proxy_cache_purge htmlcache $ps_capability_list$1$is_args$args;
     }
@@ -224,7 +212,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name if-in-server.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
 
@@ -245,7 +232,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name if-in-location.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
 
@@ -265,7 +251,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name mpd.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
 
@@ -276,7 +261,6 @@ http {
   # These two vhosts are for testing the experiment framework (Furious).
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name experiment.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
     pagespeed InPlaceResourceOptimization off;
@@ -291,7 +275,6 @@ http {
   }
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name experiment.noga.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -305,7 +288,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name preserveurls.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -335,7 +317,6 @@ http {
     # Setup a vhost with the critical image beacon and lazyload filter enabled
     # to make sure that critical images are not lazyloaded.
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name imagebeacon.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -348,7 +329,6 @@ http {
     # Setup a vhost with the critical image beacon enabled to make sure that
     # downstream caches and rebeaconing interact correctly.
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name downstreamcacherebeacon.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -364,7 +344,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name downstreamcacheresource.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
     pagespeed RewriteLevel PassThrough;
@@ -374,7 +353,29 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
+    server_name origin.rmcomments.example.com;
+    pagespeed off;
+    add_header PageSpeedFilters remove_comments;
+  }
+  server {
+    # Set up a reverse proxy (rproxy.) and origin (origin.) as vhosts for
+    # showing that we can configure PageSpeed via response headers.
+    listen @@SECONDARY_PORT@@;
+    server_name rproxy.rmcomments.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed DisableFilters add_instrumentation,remove_comments;
+    # Note that we don't enable remove_comments here; that setting
+    # comes from the response headers from origin.rmcomments.example.com
+    location / {
+      proxy_pass http://localhost:@@SECONDARY_PORT@@;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header Host origin.rmcomments.example.com;
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
     server_name renderedimagebeacon.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -392,7 +393,6 @@ http {
   # inheretence in Nginx we need to have location blocks inside a server block.
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name optimizeforbandwidth.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -424,7 +424,6 @@ http {
     # to go back to this VirtualHost so we rely on the new third optional
     # argument to MapOriginDomain.
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name customhostheader.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@_test";
     root "@@SERVER_ROOT@@/mod_pagespeed_test";
@@ -443,7 +442,6 @@ http {
     # Sets up a virtual host where we can specify forbidden filters without
     # affecting any other hosts.
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name forbidden.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -459,7 +457,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name unauthorizedresources.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -470,7 +467,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name client-domain-rewrite.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -486,7 +482,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name url-attribute.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -520,7 +515,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name domain-hyperlinks-on.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -535,7 +529,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name domain-hyperlinks-off.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -550,7 +543,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name retaincomment.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -560,7 +552,6 @@ http {
   server {
     # Test host for shared memory cache.
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name shmcache.example.com;
     pagespeed FileCachePath "@@SHM_CACHE@@";
 
@@ -570,7 +561,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name xfp.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -579,7 +569,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name xheader.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -593,7 +582,6 @@ http {
   # Note that we test with two distinct caches.
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name embed-config-html.example.org;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -613,7 +601,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name embed-config-resources.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -634,7 +621,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name ipro-for-browser.example.com;
     root "@@SERVER_ROOT@@/mod_pagespeed_example";
     pagespeed EnableFilters rewrite_images,rewrite_css;
@@ -646,7 +632,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name respectvary.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -656,7 +641,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name cache_a.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -679,7 +663,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name cache_b.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
 
@@ -690,7 +673,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name cache_c.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -704,7 +686,6 @@ http {
   server {
     # Sets up a logical home-page server on www.example.com
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name www.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -718,7 +699,6 @@ http {
     # Sets up a logical origin for CDNs to fetch content from, on
     # origin.example.com.
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name origin.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -732,7 +712,6 @@ http {
     # Sets up a logical cdn, which is where we tell browsers to fetch resources
     # from.
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name cdn.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -744,18 +723,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name lff-ipro.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-
-    pagespeed LoadFromFile
-      "http://lff-ipro.example.com/mod_pagespeed_example/lff_ipro"
-      "@@SERVER_ROOT@@/mod_pagespeed_example/lff_ipro";
-  }
-
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name tryfiles.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -769,7 +736,6 @@ http {
   server {
     # Proxy modpagespeed.com for testing Issue 582.
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name modpagespeed.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -782,26 +748,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name custom-paths.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-
-    pagespeed StatisticsPath /custom_pagespeed_statistics;
-    pagespeed ConsolePath /custom_pagespeed_console;
-    pagespeed MessagesPath /custom_pagespeed_message;
-    pagespeed AdminPath /custom_pagespeed_admin;
-  }
-
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name inherit-paths.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-  }
-
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name notransform.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
@@ -813,7 +759,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name blocking.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
 
@@ -823,7 +768,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name keepalive-html.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
     pagespeed RewriteLevel CoreFilters;
@@ -832,7 +776,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name keepalive-resource.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
     error_log "@@TEST_TMP@@/keepalive-resource.example.com.error.log" warn;
@@ -840,7 +783,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name keepalive-beacon-get.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
     error_log "@@TEST_TMP@@/keepalive-beacon-get.example.com.error.log" warn;
@@ -848,7 +790,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name keepalive-beacon-post.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
     error_log "@@TEST_TMP@@/keepalive-beacon-post.example.com.error.log" warn;
@@ -856,7 +797,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name keepalive-static.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
     error_log "@@TEST_TMP@@/keepalive-static.example.com.error.log" warn;
@@ -864,7 +804,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name response-header-filters.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
     pagespeed RewriteLevel PassThrough;
@@ -874,7 +813,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name response-header-disable.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
     pagespeed EnableFilters add_instrumentation;
@@ -884,7 +822,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name ipro.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
 
@@ -908,7 +845,6 @@ http {
   # 1. L2_d=LRU, L2_m=LRU
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name lrud-lrum.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@_lrud_lrum";
 
@@ -923,7 +859,6 @@ http {
             "@@SECONDARY_CACHE@@_lrud_shmm" 8192;
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name lrud-shmm.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@_lrud_shmm";
 
@@ -938,7 +873,6 @@ http {
             "@@SECONDARY_CACHE@@_noned_shmm" 8192;
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name noned-shmm.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@_noned_shmm";
 
@@ -949,7 +883,6 @@ http {
   # 4. L2_d=none, L2_m=none
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name noned-nonem.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@_noned_nonem";
 
@@ -959,7 +892,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name date.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
     add_header "Date" "Date: Fri, 16 Oct 2009 23:05:07 GMT";
@@ -967,11 +899,9 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name options-by-cookies-enabled.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@_optionsbycookieson";
     pagespeed AllowOptionsToBeSetByCookies true;
-    pagespeed StickyQueryParameters sticky_secret;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters collapse_whitespace;
     pagespeed DisableFilters remove_comments,add_instrumentation;
@@ -979,7 +909,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name options-by-cookies-disabled.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@_optionsbycookiesoff";
     pagespeed AllowOptionsToBeSetByCookies false;
@@ -988,19 +917,6 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name redirect.example.com;
-    pagespeed FileCachePath "@@SECONDARY_CACHE@@_redirected";
-    pagespeed RewriteLevel PassThrough;
-    pagespeed EnableFilters add_instrumentation,collapse_whitespace;
-    location /redirect {
-      rewrite ^/redirect/(.*) http://$server_name/$1 permanent;
-    }
-  }
-
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name request-option-override.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
     pagespeed RequestOptionOverride abc;
@@ -1009,45 +925,9 @@ http {
     pagespeed DisableFilters remove_comments,add_instrumentation;
   }
 
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name signed-urls.example.com;
-    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed UrlSigningKey helloworld;
-    pagespeed RewriteLevel Passthrough;
-    pagespeed EnableFilters collapse_whitespace,rewrite_images;
-    pagespeed DisableFilters remove_comments,add_instrumentation;
-  }
-
-  # For testing signed URLs, ignoring signature validity.
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name signed-urls-transition.example.com;
-    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed UrlSigningKey helloworld;
-    pagespeed AcceptInvalidSignatures true;
-    pagespeed RewriteLevel Passthrough;
-    pagespeed EnableFilters collapse_whitespace,rewrite_images;
-    pagespeed DisableFilters remove_comments,add_instrumentation;
-  }
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name unsigned-urls-transition.example.com;
-    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    # This server will not sign URLs, but AcceptInvalidSignature is on.
-    pagespeed AcceptInvalidSignatures true;
-    pagespeed RewriteLevel Passthrough;
-    pagespeed EnableFilters collapse_whitespace,rewrite_images;
-    pagespeed DisableFilters remove_comments,add_instrumentation;
-  }
-
   # Proxy + IPRO a gzip'd file for testing Issue 896.
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name ipro-proxy.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@_ipro_proxy";
 
@@ -1063,104 +943,18 @@ http {
 
   server {
     listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
     server_name compressed-css.example.com;
-    pagespeed on;
     pagespeed FileCachePath "@@FILE_CACHE@@";
     pagespeed InPlaceResourceOptimization on;
   }
 
-  # nested gzip config: pagespeed gzip on/off
   server {
-    pagespeed on;
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name gzip-test1.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-    location /mod_pagespeed_example/ {
-      pagespeed gzip off;
-    }
-    location /mod_pagespeed_example/styles/ {
-      pagespeed gzip on;
-    }
-  }
-
-  # nested gzip config: pagespeed on/off
-  server {
-    pagespeed on;
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name gzip-test2.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-    location /mod_pagespeed_example/ {
-      pagespeed off;
-    }
-    location /mod_pagespeed_example/styles/ {
-      pagespeed on;
-    }
-  }
-
-  server {
-    pagespeed on;
-    pagespeed EnableFilters add_instrumentation;
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name proxy-post.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-
-    location / {
-      # Transform the POST to a GET request for the origin,
-      # as nginx's static handler doesn't allow POST requests.
-      proxy_method GET;
-      proxy_pass http://127.0.0.1:@@SECONDARY_PORT@@/;
-      proxy_set_header "Host" "proxy-post-origin.example.com";
-    }
-  }
-
-  server {
-    pagespeed off;
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name proxy-post-origin.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
-  }
-
-  server {
-    # Write all post data to temp files
-    client_body_in_file_only clean;
-    pagespeed CriticalImagesBeaconEnabled true;
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name beacon-post-temp-file.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-  }
-
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name purge.example.com;
-
-    # Test purging individual URLs without flushing the entire metadata cache.
-    pagespeed EnableCachePurge on;
-
-    pagespeed PurgeMethod PURGE;
-    root "@@SERVER_ROOT@@";
-    pagespeed FileCachePath "@@FILE_CACHE@@_purge";
-    pagespeed DisableFilters add_instrumentation;
-    pagespeed RewriteLevel PassThrough;
-    pagespeed EnableFilters rewrite_css;
-  }
-
-  server {
-    listen @@PRIMARY_PORT@@;
-    listen [::]:@@PRIMARY_PORT@@;
+    listen       @@PRIMARY_PORT@@;
     server_name  localhost;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
     location ~ ^/pagespeed_admin {
       allow 127.0.0.1;
-      allow ::1;
       deny all;
     }
 
@@ -1201,6 +995,7 @@ http {
     #pagespeed MemcachedThreads 1;
 
     pagespeed on;
+    pagespeed MessageBufferSize 200000;
 
     #pagespeed CacheFlushPollIntervalSec 1;
 
@@ -1221,13 +1016,11 @@ http {
 
     location /ngx_pagespeed_statistics {
       allow 127.0.0.1;
-      allow ::1;
       deny all;
     }
 
     location /ngx_pagespeed_message {
       allow 127.0.0.1;
-      allow ::1;
       deny all;
     }
 
@@ -1400,11 +1193,8 @@ http {
           https://www.gstatic.com/psa/static;
     }
 
-    # $host implicitly tests script variable support. I'd love to test it more
-    # directly, but so far this is the best I've come up with and duplicating
-    # the test doesn't seem to make sense.
     pagespeed LoadFromFile
-        "http://$host:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/instant/"
+        "http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/instant/"
         "@@SERVER_ROOT@@/mod_pagespeed_test/ipro/instant/";
 
     pagespeed EnableFilters remove_comments;
@@ -1418,8 +1208,8 @@ http {
       "@@SERVER_ROOT@@/mod_pagespeed_test/load_from_file/file_\1/";
     pagespeed LoadFromFileRule Disallow
       "@@SERVER_ROOT@@/mod_pagespeed_test/load_from_file/file_dir/httponly/";
-    pagespeed LoadFromFileRuleMatch Disallow \.ssp.css$ps_dollar;
-    pagespeed LoadFromFileRuleMatch Allow exception\.ssp\.css$ps_dollar;
+    pagespeed LoadFromFileRuleMatch Disallow \.ssp.css$;
+    pagespeed LoadFromFileRuleMatch Allow exception\.ssp\.css$;
 
     #charset koi8-r;
 
@@ -1442,6 +1232,27 @@ http {
 
   keepalive_timeout  65;
 
+  # set up gzip
+  gzip  on;
+  gzip_vary on;
+  gzip_proxied any;
+  # Turn on gzip for all content types that should benefit from it.
+  gzip_types application/ecmascript;
+  gzip_types application/javascript;
+  gzip_types application/json;
+  gzip_types application/pdf;
+  gzip_types application/postscript;
+  gzip_types application/x-javascript;
+  gzip_types image/svg+xml;
+  gzip_types text/css;
+  gzip_types text/csv;
+  # "gzip_types text/html" is assumed.
+  gzip_types text/javascript;
+  gzip_types text/plain;
+  gzip_types text/xml;
+
+  gzip_http_version 1.0;
+
   types {
     text/html                             html htm shtml;
     text/css                              css;

test/run_tests.sh

@@ -73,9 +73,6 @@ function run_test_checking_failure() {
     3)
       return  # Only expected failures.
       ;;
-    4)
-      return  # Return passing error code when running manually.
-      ;;
     *)
       exit 1  # Real failure.
   esac