update version number in the config file

Fix config file based on trunk-tracking instead of master
Fix config file
2014-09-17 11:10:56 -04:00 · 2014-09-17 11:05:55 -04:00 · 2014-09-17 10:26:11 -04:00 · 2014-09-15 10:37:42 -04:00 · 2014-09-12 10:59:16 -04:00 · 2014-09-11 13:47:00 -04:00
21 changed files with 2101 additions and 720 deletions
@@ -19,94 +19,15 @@ optimizations, see our <a href="http://ngxpagespeed.com">demonstration site</a>.

 ## How to build

-Because nginx does not support dynamic loading of modules, you need to compile
-nginx from source to add ngx_pagespeed. Alternatively, if you're using Tengine you can [install ngx_pagespeed without
-recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_pagespeed-with-Tengine).
-
-1. Install dependencies:
-
-   ```bash
-   # These are for RedHat, CentOS, and Fedora.
-   $ sudo yum install gcc-c++ pcre-dev pcre-devel zlib-devel make
-
-   # These are for Debian. Ubuntu will be similar.
-   $ sudo apt-get install build-essential zlib1g-dev libpcre3 libpcre3-dev
-   ```
-
-2. Download ngx_pagespeed:
-
-   ```bash
-   $ cd ~
-   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.7.30.4-beta.zip
-   $ unzip v1.7.30.4-beta.zip # or unzip v1.7.30.4-beta
-   $ cd ngx_pagespeed-1.7.30.4-beta/
-   $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.4.tar.gz
-   $ tar -xzvf 1.7.30.4.tar.gz # expands to psol/
-   ```
-
-3. Download and build nginx:
-
-   ```bash
-   $ cd ~
-   $ # check http://nginx.org/en/download.html for the latest version
-   $ wget http://nginx.org/download/nginx-1.4.6.tar.gz
-   $ tar -xvzf nginx-1.4.6.tar.gz
-   $ cd nginx-1.4.6/
-   $ ./configure --add-module=$HOME/ngx_pagespeed-1.7.30.4-beta
-   $ make
-   $ sudo make install
-   ```
-
-If this doesn't work see the [build
-troubleshooting](https://github.com/pagespeed/ngx_pagespeed/wiki/Build-Troubleshooting) page.
-
-This will use a binary PageSpeed Optimization Library, but it's also possible to
-[build PSOL from
-source](https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source).
-
-Note: ngx_pagespeed currently doesn't support Windows or MacOS because the
-underlying PSOL library doesn't.
+Follow the steps on <a
+href="https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source">build
+ngx_pagespeed from source</a>.

 ## How to use

-In your `nginx.conf`, add to the main or server block:
-
-```nginx
-pagespeed on;
-pagespeed FileCachePath /var/ngx_pagespeed_cache;  # Use tmpfs for best results.
-```
-
-In every server block where pagespeed is enabled add:
-
-```apache
-#  Ensure requests for pagespeed optimized resources go to the pagespeed
-#  handler and no extraneous headers get set.
-location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" { add_header "" ""; }
-location ~ "^/ngx_pagespeed_static/" { }
-location ~ "^/ngx_pagespeed_beacon$" { }
-location /ngx_pagespeed_statistics { allow 127.0.0.1; deny all; }
-location /ngx_pagespeed_global_statistics { allow 127.0.0.1; deny all; }
-location /ngx_pagespeed_message { allow 127.0.0.1; deny all; }
-location /pagespeed_console { allow 127.0.0.1; deny all; }
-```
-
-To confirm that the module is loaded, fetch a page and check that you see the
-`X-Page-Speed` header:
-
-```bash
-$ curl -I 'http://localhost:8050/some_page/' | grep X-Page-Speed
-X-Page-Speed: 1.7.30.4-...
-```
-
-Looking at the source of a few pages you should see various changes, such as
-urls being replaced with new ones like `yellow.css.pagespeed.ce.lzJ8VcVi1l.css`.
-
-For complete documentation, see [Using
-PageSpeed](https://developers.google.com/speed/pagespeed/module/using).
-
-There are extensive system tests which cover most of ngx_pagespeed's
-functionality.  Consider [testing your
-installation](https://github.com/pagespeed/ngx_pagespeed/wiki/Testing).
+Follow the steps on <a
+href="https://developers.google.com/speed/pagespeed/module/configuration">PageSpeed
+configuration</a>.

 For feedback, questions, and to follow
 the progress of the project:
@@ -115,20 +36,3 @@ the progress of the project:
  list](https://groups.google.com/forum/#!forum/ngx-pagespeed-discuss)
 - [ngx-pagespeed-announce mailing
  list](https://groups.google.com/forum/#!forum/ngx-pagespeed-announce)
-
-Note: The
-[canonicalize_javascript_libraries](https://developers.google.com/speed/pagespeed/module/filter-canonicalize-js)
-depends on `pagespeed_libraries.conf` which is distributed in Apache's format.
-To convert it to the Nginx format, run:
-
-```bash
-$ scripts/pagespeed_libraries_generator.sh > ~/pagespeed_libraries.conf
-$ sudo mv ~/pagespeed_libraries.conf /etc/nginx/
-```
-
-And then include it in your Nginx configuration by reference:
-
-```nginx
-include pagespeed_libraries.conf;
-pagespeed EnableFilters canonicalize_javascript_libraries;
-```
@@ -27,8 +27,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
    echo "   You need to separately download the pagespeed library:"
    echo ""
    echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.4.tar.gz"
-    echo "     $ tar -xzvf 1.7.30.4.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.9.32.1.tar.gz"
+    echo "     $ tar -xzvf 1.9.32.1.tar.gz # expands to psol/"
    echo ""
    echo "   Or see the installation instructions:"
    echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
@@ -39,20 +39,6 @@ else
  build_from_source=true
 fi

-echo "mod_pagespeed_dir=$mod_pagespeed_dir"
-echo "build_from_source=$build_from_source"
-
-ngx_feature="psol"
-ngx_feature_name=""
-ngx_feature_run=no
-ngx_feature_incs="
-#include \"net/instaweb/htmlparse/public/html_parse.h\"
-#include \"net/instaweb/htmlparse/public/html_writer_filter.h\"
-#include \"net/instaweb/util/public/string.h\"
-#include \"net/instaweb/util/public/string_writer.h\"
-#include \"net/instaweb/util/public/null_message_handler.h\"
-"
-
 os_name='unknown_os'
 arch_name='unknown_arch'
 uname_os=`uname`
@@ -109,6 +95,35 @@ case "$NGX_GCC_VER" in
  ;;
 esac

+if [ "$WNO_ERROR" = "YES" ]; then
+    CFLAGS="$CFLAGS -Wno-error"
+fi
+
+psol_binary="${PSOL_BINARY:-unset}"
+if [ "$psol_binary" = "unset" ] ; then
+  if $build_from_source ; then
+    psol_binary="\
+        $mod_pagespeed_dir/net/instaweb/automatic/pagespeed_automatic.a"
+  else
+    psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
+    psol_binary="$psol_library_dir/pagespeed_automatic.a"
+  fi
+fi
+
+echo "mod_pagespeed_dir=$mod_pagespeed_dir"
+echo "build_from_source=$build_from_source"
+
+ngx_feature="psol"
+ngx_feature_name=""
+ngx_feature_run=no
+ngx_feature_incs="
+#include \"pagespeed/kernel/base/string.h\"
+#include \"pagespeed/kernel/base/string_writer.h\"
+#include \"pagespeed/kernel/base/null_message_handler.h\"
+#include \"pagespeed/kernel/html/html_parse.h\"
+#include \"pagespeed/kernel/html/html_writer_filter.h\"
+"
+
 pagespeed_include="\
  $mod_pagespeed_dir \
  $mod_pagespeed_dir/third_party/chromium/src \
@@ -124,15 +139,7 @@ pagespeed_include="\
  $mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include"
 ngx_feature_path="$pagespeed_include"

-if $build_from_source ; then
-  psol_library_binaries="\
-    $mod_pagespeed_dir/net/instaweb/automatic/pagespeed_automatic.a"
-else
-  psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
-  psol_library_binaries="$psol_library_dir/pagespeed_automatic.a"
-fi
-
-pagespeed_libs="-lstdc++ $psol_library_binaries -lrt -pthread -lm"
+pagespeed_libs="-lstdc++ $psol_binary -lrt -pthread -lm"
 ngx_feature_libs="$pagespeed_libs"
 ngx_feature_test="
  GoogleString output_buffer;
@@ -163,18 +170,21 @@ if [ $ngx_found = yes ]; then
    $ps_src/ngx_base_fetch.h \
    $ps_src/ngx_caching_headers.h \
    $ps_src/ngx_fetch.h \
+    $ps_src/ngx_gzip_setter.h \
    $ps_src/ngx_list_iterator.h \
    $ps_src/ngx_message_handler.h \
    $ps_src/ngx_pagespeed.h \
    $ps_src/ngx_rewrite_driver_factory.h \
    $ps_src/ngx_rewrite_options.h \
    $ps_src/ngx_server_context.h \
-    $ps_src/ngx_url_async_fetcher.h"
+    $ps_src/ngx_url_async_fetcher.h \
+    $psol_binary"
  NGX_ADDON_SRCS="$NGX_ADDON_SRCS \
    $ps_src/log_message_handler.cc \
    $ps_src/ngx_base_fetch.cc \
    $ps_src/ngx_caching_headers.cc \
    $ps_src/ngx_fetch.cc \
+    $ps_src/ngx_gzip_setter.cc \
    $ps_src/ngx_list_iterator.cc \
    $ps_src/ngx_message_handler.cc \
    $ps_src/ngx_pagespeed.cc \
@@ -39,7 +39,7 @@ NgxBaseFetch::NgxBaseFetch(ngx_http_request_t* r, int pipe_fd,
      last_buf_sent_(false),
      pipe_fd_(pipe_fd),
      references_(2),
-      handle_error_(true),
+      ipro_lookup_(false),
      preserve_caching_headers_(preserve_caching_headers) {
  if (pthread_mutex_init(&mutex_, NULL)) CHECK(0);
 }
@@ -137,14 +137,20 @@ void NgxBaseFetch::HandleHeadersComplete() {
  int status_code = response_headers()->status_code();
  bool status_ok = (status_code != 0) && (status_code < 400);

-  if (status_ok || handle_error_) {
+  if (!ipro_lookup_ || status_ok) {
    // If this is a 404 response we need to count it in the stats.
    if (response_headers()->status_code() == HttpStatus::kNotFound) {
      server_context_->rewrite_stats()->resource_404_count()->Add(1);
    }
  }

-  RequestCollection();  // Headers available.
+  // For the IPRO lookup, supress notification of the nginx side here.
+  // If we send both this event and the one from done, nasty stuff will happen
+  // if we loose the race with with the nginx side destructing this base fetch
+  // instance (and thereby clearing the byte and its pending extraneous event.
+  if (!ipro_lookup_) {
+    RequestCollection();  // Headers available.
+  }
 }

 bool NgxBaseFetch::HandleFlush(MessageHandler* handler) {
@@ -79,7 +79,7 @@ class NgxBaseFetch : public AsyncFetch {

  // Called by nginx when it's done with us.
  void Release();
-  void set_handle_error(bool x) { handle_error_ = x; }
+  void set_ipro_lookup(bool x) { ipro_lookup_ = x; }

 private:
  virtual bool HandleWrite(const StringPiece& sp, MessageHandler* handler);
@@ -117,7 +117,7 @@ class NgxBaseFetch : public AsyncFetch {
  // decremented once when Done() is called and once when Release() is called.
  int references_;
  pthread_mutex_t mutex_;
-  bool handle_error_;
+  bool ipro_lookup_;
  PreserveCachingHeaders preserve_caching_headers_;

  DISALLOW_COPY_AND_ASSIGN(NgxBaseFetch);
@@ -71,12 +71,13 @@ namespace net_instaweb {
        fetch_end_ms_(0),
        done_(false),
        content_length_(-1),
-        content_length_known_(false) {
-            ngx_memzero(&url_, sizeof(url_));
-            log_ = log;
-            pool_ = NULL;
-            timeout_event_ = NULL;
-            connection_ = NULL;
+        content_length_known_(false),
+        resolver_ctx_(NULL) {
+    ngx_memzero(&url_, sizeof(url_));
+    log_ = log;
+    pool_ = NULL;
+    timeout_event_ = NULL;
+    connection_ = NULL;
  }

  NgxFetch::~NgxFetch() {
@@ -213,6 +214,8 @@ namespace net_instaweb {
      return;
    }

+    release_resolver();
+
    if (timeout_event_ && timeout_event_->timer_set) {
      ngx_del_timer(timeout_event_);
      timeout_event_ = NULL;
@@ -288,7 +291,6 @@ namespace net_instaweb {
          kWarning, "NgxFetch: failed to resolve host [%.*s]",
          static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data);
      fetch->CallbackDone(false);
-      ngx_resolve_name_done(resolver_ctx);
      return;
    }
    ngx_memzero(&fetch->sin_, sizeof(fetch->sin_));
@@ -297,8 +299,8 @@ namespace net_instaweb {
    fetch->sin_.sin_addr.s_addr = resolver_ctx->addrs[0];
 #else

-    struct sockaddr_in *sin;
-    sin = reinterpret_cast<struct sockaddr_in *>(
+    struct sockaddr_in* sin;
+    sin = reinterpret_cast<struct sockaddr_in*>(
        resolver_ctx->addrs[0].sockaddr);
    fetch->sin_.sin_family = sin->sin_family;
    fetch->sin_.sin_addr.s_addr = sin->sin_addr.s_addr;
@@ -319,7 +321,7 @@ namespace net_instaweb {
        static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data,
        ip_address);

-    ngx_resolve_name_done(resolver_ctx);
+    fetch->release_resolver();

    if (fetch->InitRequest() != NGX_OK) {
      fetch->message_handler()->Message(kError, "NgxFetch: InitRequest failed");
@@ -416,7 +418,7 @@ namespace net_instaweb {
  int NgxFetch::Connect() {
    ngx_peer_connection_t pc;
    ngx_memzero(&pc, sizeof(pc));
-    pc.sockaddr = (struct sockaddr *) &sin_;
+    pc.sockaddr = (struct sockaddr*)&sin_;
    pc.socklen = sizeof(struct sockaddr_in);
    pc.name = &url_.host;

@@ -16,6 +16,15 @@

 // Author: x.dinic@gmail.com (Junmin Xiong)
 //
+// PageSpeed needs some way to talk to the internet and request resources.  For
+// example, if it's optimizing www.example.com/index.html and it sees html with
+// <img src="//images.example.com/cat.jpg"> and images.example.com is authorized
+// for rewriting in the config, then it needs to fetch cat.jpg from
+// images.example.com and optimize it.  In apache (always) and nginx (by
+// default) we use a fetcher called "serf".  This works fine, but it does run
+// its own event loop.  To be more efficient, this is a "native" fetcher that
+// uses nginx's event loop.
+//
 // The fetch is started by the main thread. It will fetch the remote resource
 // from the specific url asynchronously.

@@ -83,6 +92,12 @@ class NgxFetch : public PoolElement<NgxFetch> {
  void set_timeout_event(ngx_event_t* x) {
    timeout_event_ = x;
  }
+  void release_resolver() {
+    if (resolver_ctx_ != NULL && resolver_ctx_ != NGX_NO_RESOLVER) {
+      ngx_resolve_name_done(resolver_ctx_);
+      resolver_ctx_ = NULL;
+    }
+  }

 private:
  response_handler_pt response_handler;
@@ -0,0 +1,403 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: kspoelstra@we-amp.com (Kees Spoelstra)
+
+#include "ngx_gzip_setter.h"
+
+#include <ngx_conf_file.h>
+
+namespace net_instaweb {
+
+NgxGZipSetter g_gzip_setter;
+
+extern "C" {
+  // These functions replace the setters for:
+  //   gzip
+  //   gzip_types
+  //   gzip_http_version
+  //   gzip_vary
+  //
+  // If these functions are called it means there is an explicit gzip
+  // configuration. The gzip configuration set by pagespeed is then rolled
+  // back and pagespeed will stop enabling gzip automatically.
+  char* ngx_gzip_redirect_conf_set_flag_slot(
+      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    if (g_gzip_setter.enabled()) {
+      g_gzip_setter.RollBackAndDisable(cf);
+    }
+    char* ret = ngx_conf_set_flag_slot(cf, cmd, conf);
+    return ret;
+  }
+
+  char* ngx_gzip_redirect_http_types_slot(
+      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    if (g_gzip_setter.enabled()) {
+      g_gzip_setter.RollBackAndDisable(cf);
+    }
+    char* ret = ngx_http_types_slot(cf, cmd, conf);
+    return ret;
+  }
+
+  char* ngx_gzip_redirect_conf_set_enum_slot(
+      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    if (g_gzip_setter.enabled()) {
+      g_gzip_setter.RollBackAndDisable(cf);
+    }
+    char* ret = ngx_conf_set_enum_slot(cf, cmd, conf);
+    return ret;
+  }
+  char* ngx_gzip_redirect_conf_set_bitmask_slot(
+      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    if (g_gzip_setter.enabled()) {
+      g_gzip_setter.RollBackAndDisable(cf);
+    }
+    char* ret = ngx_conf_set_bitmask_slot(cf, cmd, conf);
+    return ret;
+  }
+}
+
+NgxGZipSetter::NgxGZipSetter() : enabled_(0) { }
+NgxGZipSetter::~NgxGZipSetter() { }
+
+// Helper functions to determine signature.
+bool HasLocalConfig(ngx_command_t* command) {
+  return (!(command->type & (NGX_DIRECT_CONF|NGX_MAIN_CONF)) &&
+          command->conf == NGX_HTTP_LOC_CONF_OFFSET);
+}
+bool IsNgxFlagCommand(ngx_command_t* command) {
+  return (command->set == ngx_conf_set_flag_slot &&
+          HasLocalConfig(command));
+}
+bool IsNgxHttpTypesCommand(ngx_command_t* command) {
+  return (command->set == ngx_http_types_slot &&
+          HasLocalConfig(command));
+}
+bool IsNgxEnumCommand(ngx_command_t* command) {
+  return (command->set == ngx_conf_set_enum_slot &&
+          HasLocalConfig(command));
+}
+bool IsNgxBitmaskCommand(ngx_command_t* command) {
+  return (command->set == ngx_conf_set_bitmask_slot &&
+          HasLocalConfig(command));
+}
+
+// Initialize the NgxGzipSetter.
+// Find the gzip, gzip_vary, gzip_http_version and gzip_types commands in the
+// gzip module. Enable if the signature of the zip command matches with what we
+// trust. Also sets up redirects for the configurations. These redirect handle
+// a rollback if expicit configuration is found.
+// If commands are not found the method will inform the user by logging.
+void NgxGZipSetter::Init(ngx_conf_t* cf) {
+#if (NGX_HTTP_GZIP)
+  bool gzip_signature_mismatch = false;
+  bool other_signature_mismatch = false;
+  for (int m = 0; ngx_modules[m] != NULL; m++) {
+    if (ngx_modules[m]->commands != NULL) {
+      for (int c = 0; ngx_modules[m]->commands[c].name.len; c++) {
+        ngx_command_t* current_command =& ngx_modules[m]->commands[c];
+
+        // We look for the gzip command, and the exact signature we trust
+        // this means configured as an config location offset
+        // and a ngx_flag_t setter.
+        // Also see:
+        //   ngx_conf_handler in ngx_conf_file.c
+        //   ngx_http_gzip_filter_commands in ngx_http_gzip_filter.c
+        if (gzip_command_.command_ == NULL &&
+            STR_EQ_LITERAL(current_command->name, "gzip")) {
+          if (IsNgxFlagCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
+            gzip_command_.command_ = current_command;
+            gzip_command_.module_ = ngx_modules[m];
+            enabled_ = 1;
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip, signature mismatch");
+            gzip_signature_mismatch = true;
+          }
+        }
+
+        if (!gzip_http_version_command_.command_ &&
+            STR_EQ_LITERAL(current_command->name, "gzip_http_version")) {
+          if (IsNgxEnumCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_conf_set_enum_slot;
+            gzip_http_version_command_.command_ = current_command;
+            gzip_http_version_command_.module_ = ngx_modules[m];
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip_http_version, signature mismatch");
+            other_signature_mismatch = true;
+          }
+        }
+
+        if (!gzip_proxied_command_.command_ &&
+            STR_EQ_LITERAL(current_command->name, "gzip_proxied")) {
+          if (IsNgxBitmaskCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_conf_set_bitmask_slot;
+            gzip_proxied_command_.command_ = current_command;
+            gzip_proxied_command_.module_ = ngx_modules[m];
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip_proxied, signature mismatch");
+            other_signature_mismatch = true;
+          }
+        }
+
+        if (!gzip_http_types_command_.command_ &&
+            STR_EQ_LITERAL(current_command->name, "gzip_types")) {
+          if (IsNgxHttpTypesCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_http_types_slot;
+            gzip_http_types_command_.command_ = current_command;
+            gzip_http_types_command_.module_ = ngx_modules[m];
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip_types, signature mismatch");
+            other_signature_mismatch = true;
+          }
+        }
+
+        if (!gzip_vary_command_.command_ &&
+            STR_EQ_LITERAL(current_command->name, "gzip_vary")) {
+          if (IsNgxFlagCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
+            gzip_vary_command_.command_ = current_command;
+            gzip_vary_command_.module_ = ngx_modules[m];
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip_vary, signature mismatch");
+            other_signature_mismatch = true;
+          }
+        }
+      }
+    }
+  }
+  if (gzip_signature_mismatch) {
+    return;  // Already logged error.
+  } else if (!enabled_) {
+    // Looked through all the available commands and didn't find the "gzip" one.
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: cannot set gzip, command not found");
+    return;
+  } else if (other_signature_mismatch) {
+    return;  // Already logged error.
+  } else if (!gzip_vary_command_.command_) {
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_vary");
+    return;
+  } else if (!gzip_http_types_command_.command_) {
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_types");
+    return;
+  } else if (!gzip_http_version_command_.command_) {
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_http_version");
+    return;
+  } else if (!gzip_proxied_command_.command_) {
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_proxied");
+    return;
+  } else {
+    return;  // Success.
+  }
+#else
+  ngx_conf_log_error(
+      NGX_LOG_WARN, cf, 0, "pagespeed: gzip not compiled into nginx");
+  return;
+#endif
+}
+
+void* ngx_command_ctx::GetConfPtr(ngx_conf_t* cf) {
+  return GetModuleConfPtr(cf) + command_->offset;
+}
+
+char* ngx_command_ctx::GetModuleConfPtr(ngx_conf_t* cf) {
+  return reinterpret_cast<char*>(
+      ngx_http_conf_get_module_loc_conf(cf, (*(module_))));
+}
+
+void NgxGZipSetter::SetNgxConfFlag(ngx_conf_t* cf,
+                                   ngx_command_ctx* command_ctx,
+                                   ngx_flag_t value) {
+  ngx_flag_t* flag = reinterpret_cast<ngx_flag_t*>(command_ctx->GetConfPtr(cf));
+  *flag = value;
+  // Save the flag position for possible rollback.
+  ngx_flags_set_.push_back(flag);
+}
+
+void NgxGZipSetter::SetNgxConfEnum(ngx_conf_t* cf,
+                                   ngx_command_ctx* command_ctx,
+                                   ngx_uint_t value) {
+  ngx_uint_t* enum_to_set =
+      reinterpret_cast<ngx_uint_t*>(command_ctx->GetConfPtr(cf));
+  *enum_to_set = value;
+  ngx_uint_set_.push_back(enum_to_set);
+}
+
+void NgxGZipSetter::SetNgxConfBitmask(ngx_conf_t* cf,
+                                      ngx_command_ctx* command_ctx,
+                                      ngx_uint_t value) {
+  ngx_uint_t* enum_to_set =
+      reinterpret_cast<ngx_uint_t*>(command_ctx->GetConfPtr(cf));
+  *enum_to_set = value;
+  ngx_uint_set_.push_back(enum_to_set);
+}
+
+// These are the content types we want to compress.
+ngx_str_t gzip_http_types[] = {
+  ngx_string("application/ecmascript"),
+  ngx_string("application/javascript"),
+  ngx_string("application/json"),
+  ngx_string("application/pdf"),
+  ngx_string("application/postscript"),
+  ngx_string("application/x-javascript"),
+  ngx_string("image/svg+xml"),
+  ngx_string("text/css"),
+  ngx_string("text/csv"),
+  // ngx_string("text/html"),  // This is the default implied value.
+  ngx_string("text/javascript"),
+  ngx_string("text/plain"),
+  ngx_string("text/xml"),
+  ngx_null_string  // Indicates end of array.
+};
+
+gzs_enable_result NgxGZipSetter::SetGZipForLocation(ngx_conf_t* cf,
+                                                    bool value) {
+  if (!enabled_) {
+    return kEnableGZipNotEnabled;
+  }
+  if (gzip_command_.command_) {
+    SetNgxConfFlag(cf, &gzip_command_, value);
+  }
+  return kEnableGZipOk;
+}
+
+void NgxGZipSetter::EnableGZipForLocation(ngx_conf_t* cf) {
+  if (!enabled_) {
+    return;
+  }
+
+  // When we get called twice for the same location{}, we ignore the second call
+  // to prevent adding duplicate gzip http types and so on.
+  ngx_flag_t* flag =
+      reinterpret_cast<ngx_flag_t*>(gzip_command_.GetConfPtr(cf));
+  if (*flag == 1) {
+    return;
+  }
+  SetGZipForLocation(cf, true);
+  if (gzip_vary_command_.command_) {
+    SetNgxConfFlag(cf, &gzip_vary_command_, 1);
+  }
+  if (gzip_http_version_command_.command_) {
+    SetNgxConfEnum(cf, &gzip_http_version_command_, NGX_HTTP_VERSION_10);
+  }
+  if (gzip_proxied_command_.command_) {
+    SetNgxConfBitmask(
+        cf, &gzip_http_version_command_, NGX_HTTP_GZIP_PROXIED_ANY);
+  }
+
+  // This is actually the most prone to future API changes, because gzip_types
+  // is not a simple type like ngx_flag_t. The signature check should be enough
+  // to prevent problems.
+  AddGZipHTTPTypes(cf);
+  return;
+}
+
+void NgxGZipSetter::AddGZipHTTPTypes(ngx_conf_t* cf) {
+  if (gzip_http_types_command_.command_) {
+    // Following should not happen, but if it does return gracefully.
+    if (cf->args->nalloc < 2) {
+      ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
+                         "pagespeed: unexpected small cf->args in gzip_types");
+      return;
+    }
+
+    ngx_command_t* command = gzip_http_types_command_.command_;
+    char* gzip_conf = reinterpret_cast<char* >(
+        gzip_http_types_command_.GetModuleConfPtr(cf));
+
+    // Backup the old settings.
+    ngx_str_t old_elt0 = reinterpret_cast<ngx_str_t*>(cf->args->elts)[0];
+    ngx_str_t old_elt1 = reinterpret_cast<ngx_str_t*>(cf->args->elts)[1];
+    ngx_uint_t old_nelts = cf->args->nelts;
+
+    // Setup first arg.
+    ngx_str_t gzip_types_string = ngx_string("gzip_types");
+    reinterpret_cast<ngx_str_t*>(cf->args->elts)[0] = gzip_types_string;
+    cf->args->nelts = 2;
+
+    ngx_str_t* http_types = gzip_http_types;
+    while (http_types->data) {
+      ngx_str_t d;
+      // We allocate the http type on the configuration pool and actually
+      // leak this if we rollback. This does not seem to be a big problem,
+      // because nginx also allocates tokens in ngx_conf_file.c and does not
+      // free them. This way they can be used safely by configurations.
+      // We must use a copy of gzip_http_types array here because nginx will
+      // manipulate the values.
+      // TODO(kspoelstra): better would be to allocate once on init and not
+      // every time we enable gzip. This needs further investigation, sharing
+      // tokens might be problematic.
+      // For now I think it is not a large problem. This might add up in case
+      // of a large multi server/location config with a lot of "pagespeed on"
+      // directives.
+      // Estimates are 300-400KB for 1000 times "pagespeed on".
+      d.data = reinterpret_cast<u_char*>(
+          ngx_pnalloc(cf->pool, http_types->len + 1));
+      snprintf(reinterpret_cast<char*>(d.data), http_types->len + 1, "%s",
+               reinterpret_cast<const char*>(http_types->data));
+      d.len = http_types->len;
+      reinterpret_cast<ngx_str_t*>(cf->args->elts)[1] = d;
+      // Call the original setter.
+      ngx_http_types_slot(cf, command, gzip_conf);
+      http_types++;
+    }
+
+    // Restore args.
+    cf->args->nelts = old_nelts;
+    reinterpret_cast<ngx_str_t*>(cf->args->elts)[1] = old_elt1;
+    reinterpret_cast<ngx_str_t*>(cf->args->elts)[0] = old_elt0;
+
+    // Backup configuration location for rollback.
+    ngx_httptypes_set_.push_back(gzip_conf + command->offset);
+  }
+}
+
+void NgxGZipSetter::RollBackAndDisable(ngx_conf_t* cf) {
+  ngx_conf_log_error(NGX_LOG_INFO, cf, 0,
+                     "pagespeed: rollback gzip, explicit configuration");
+  for (std::vector<ngx_flag_t*>::iterator i = ngx_flags_set_.begin();
+       i != ngx_flags_set_.end(); ++i) {
+    *(*i)=NGX_CONF_UNSET;
+  }
+  for (std::vector<ngx_uint_t*>::iterator i = ngx_uint_set_.begin();
+       i != ngx_uint_set_.end(); ++i) {
+    *(*i)=NGX_CONF_UNSET_UINT;
+  }
+  for (std::vector<void*>::iterator i = ngx_httptypes_set_.begin();
+       i != ngx_httptypes_set_.end(); ++i) {
+    ngx_array_t** type_array = reinterpret_cast<ngx_array_t**>(*i);
+    ngx_array_destroy(*type_array);
+    *type_array = NULL;
+  }
+  enabled_ = 0;
+}
+
+}  // namespace net_instaweb
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: kspoelstra@we-amp.com (Kees Spoelstra)
+
+/*
+ * NgxGZipSetter sets up gzip for pagespeed
+ * with the following configuration:
+ * gzip  on;
+ * gzip_vary on;
+ * gzip_types application/ecmascript;
+ * gzip_types application/javascript;
+ * gzip_types application/json;
+ * gzip_types application/pdf;
+ * gzip_types application/postscript;
+ * gzip_types application/x-javascript;
+ * gzip_types image/svg+xml;
+ * gzip_types text/css;
+ * gzip_types text/csv;
+ * gzip_types text/javascript;
+ * gzip_types text/plain;
+ * gzip_types text/xml;
+ * gzip_http_version 1.0;
+ *
+ * If there is an explicit gzip configuration in the nginx.conf
+ * pagespeed will rollback the set configuration and let the
+ * user decide what the configuration will be.
+ *
+ * It manipulates the configuration by manipulating ngx_flag_t
+ * and ngx_uint_t settings directly and using the nginx setter for
+ * gzip_http_types.
+ * This is probably a safe way to do it. If this mechanism
+ * changes all non nginx module setup & configuration will
+ * fail.
+ */
+
+#ifndef NGX_GZIP_SETTER_H_
+#define NGX_GZIP_SETTER_H_
+
+extern "C" {
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_http.h>
+}
+
+#include <vector>
+
+#include "ngx_pagespeed.h"
+
+#include "net/instaweb/util/public/basictypes.h"
+
+namespace net_instaweb {
+
+// We need this class because configuration for gzip is in different modules, so
+// just saving the command will not work.
+class ngx_command_ctx {
+ public:
+  ngx_command_ctx() : command_(NULL), module_(NULL) { }
+  void* GetConfPtr(ngx_conf_t* cf);
+  char* GetModuleConfPtr(ngx_conf_t* cf);
+  ngx_command_t* command_;
+  ngx_module_t* module_;
+};
+
+enum gzs_enable_result {
+  kEnableGZipOk,
+  kEnableGZipPartial,
+  kEnableGZipNotEnabled
+};
+
+class NgxGZipSetter {
+  std::vector<ngx_flag_t*> ngx_flags_set_;
+  std::vector<ngx_uint_t*> ngx_uint_set_;
+  std::vector<void*> ngx_httptypes_set_;
+  ngx_command_ctx gzip_command_;
+  ngx_command_ctx gzip_http_types_command_;
+  ngx_command_ctx gzip_proxied_command_;
+  ngx_command_ctx gzip_vary_command_;
+  ngx_command_ctx gzip_http_version_command_;
+  bool enabled_;
+
+ public:
+  NgxGZipSetter();
+  ~NgxGZipSetter();
+  void Init(ngx_conf_t* cf);
+
+  void SetNgxConfFlag(ngx_conf_t* cf,
+                      ngx_command_ctx* command_ctx,
+                      ngx_flag_t value);
+  void SetNgxConfEnum(ngx_conf_t* cf,
+                      ngx_command_ctx* command_ctx,
+                      ngx_uint_t value);
+  void SetNgxConfBitmask(ngx_conf_t* cf,
+                         ngx_command_ctx* command_ctx,
+                         ngx_uint_t value);
+  void EnableGZipForLocation(ngx_conf_t* cf);
+  gzs_enable_result SetGZipForLocation(ngx_conf_t* cf, bool value);
+  void AddGZipHTTPTypes(ngx_conf_t* cf);
+  void RollBackAndDisable(ngx_conf_t* cf);
+
+  bool enabled() { return enabled_; }
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(NgxGZipSetter);
+};
+
+extern NgxGZipSetter g_gzip_setter;
+
+}  // namespace net_instaweb
+
+#endif  // NGX_GZIP_SETTER_H_
@@ -56,11 +56,9 @@ extern "C" {

 namespace net_instaweb {

-NgxMessageHandler::NgxMessageHandler(AbstractMutex* mutex)
-    : mutex_(mutex),
-      buffer_(NULL),
+NgxMessageHandler::NgxMessageHandler(Timer* timer, AbstractMutex* mutex)
+    : SystemMessageHandler(timer, mutex),
      log_(NULL) {
-  SetPidString(static_cast<int64>(getpid()));
 }

 // Installs a signal handler for common crash signals, that tries to print
@@ -73,14 +71,6 @@ void NgxMessageHandler::InstallCrashHandler(ngx_log_t* log) {
  signal(SIGSEGV, signal_handler);
 }

-bool NgxMessageHandler::Dump(Writer* writer) {
-  // Can't dump before SharedCircularBuffer is set up.
-  if (buffer_ == NULL) {
-    return false;
-  }
-  return buffer_->Dump(writer, &handler_);
-}
-
 ngx_uint_t NgxMessageHandler::GetNgxLogLevel(MessageType type) {
  switch (type) {
    case kInfo:
@@ -98,11 +88,6 @@ ngx_uint_t NgxMessageHandler::GetNgxLogLevel(MessageType type) {
  return NGX_LOG_ALERT;
 }

-void NgxMessageHandler::set_buffer(SharedCircularBuffer* buff) {
-  ScopedMutex lock(mutex_.get());
-  buffer_ = buff;
-}
-
 void NgxMessageHandler::MessageVImpl(MessageType type, const char* msg,
                                     va_list args) {
  ngx_uint_t log_level = GetNgxLogLevel(type);
@@ -113,25 +98,8 @@ void NgxMessageHandler::MessageVImpl(MessageType type, const char* msg,
  } else {
    GoogleMessageHandler::MessageVImpl(type, msg, args);
  }
-
  // Prepare a log message for the SharedCircularBuffer only.
-  // Prepend time and severity to message.
-  // Format is [time] [severity] [pid] message.
-  GoogleString message;
-  GoogleString time;
-  PosixTimer timer;
-  if (!ConvertTimeToString(timer.NowMs(), &time)) {
-    time = "?";
-  }
-  StrAppend(&message, "[", time, "] ",
-            "[", MessageTypeToString(type), "] ");
-  StrAppend(&message, pid_string_, " ", formatted_message, "\n");
-  {
-    ScopedMutex lock(mutex_.get());
-    if (buffer_ != NULL) {
-      buffer_->Write(message);
-    }
-  }
+  AddMessageToBuffer(type, formatted_message);
 }

 void NgxMessageHandler::FileMessageVImpl(MessageType type, const char* file,
@@ -24,44 +24,30 @@ extern "C" {

 #include <cstdarg>

+#include "net/instaweb/system/public/system_message_handler.h"
 #include "net/instaweb/util/public/basictypes.h"
-#include "net/instaweb/util/public/google_message_handler.h"
 #include "net/instaweb/util/public/message_handler.h"
-#include "net/instaweb/util/public/scoped_ptr.h"
 #include "net/instaweb/util/public/string.h"
 #include "net/instaweb/util/public/string_util.h"

 namespace net_instaweb {

 class AbstractMutex;
-class SharedCircularBuffer;
 class Timer;
-class Writer;

 // Implementation of a message handler that uses ngx_log_error()
 // logging to emit messages, with a fallback to GoogleMessageHandler
-class NgxMessageHandler : public GoogleMessageHandler {
+class NgxMessageHandler : public SystemMessageHandler {
 public:
-  explicit NgxMessageHandler(AbstractMutex* mutex);
+  explicit NgxMessageHandler(Timer* timer, AbstractMutex* mutex);

  // Installs a signal handler for common crash signals that tries to print
  // out a backtrace.
  static void InstallCrashHandler(ngx_log_t* log);

-  // When NgxRewriteDriver instantiates the NgxMessageHandlers, the
-  // SharedCircularBuffer and ngx_log_t are not available yet. These
-  // will later be set in RootInit/Childinit
-  // Messages logged before that will be passed on to handler_;
-  void set_buffer(SharedCircularBuffer* buff);
  void set_log(ngx_log_t* log) { log_ = log; }
  ngx_log_t* log() { return log_; }

-  void SetPidString(const int64 pid) {
-    pid_string_ = StrCat("[", Integer64ToString(pid), "]");
-  }
-  // Dump contents of SharedCircularBuffer.
-  bool Dump(Writer* writer);
-
 protected:
  virtual void MessageVImpl(MessageType type, const char* msg, va_list args);

@@ -70,13 +56,6 @@ class NgxMessageHandler : public GoogleMessageHandler {

 private:
  ngx_uint_t GetNgxLogLevel(MessageType type);
-
-  scoped_ptr<AbstractMutex> mutex_;
-  GoogleString pid_string_;
-  // handler_ is used as a fallback when we can not use ngx_log_errort
-  // It's also used when calling Dump on the internal SharedCircularBuffer
-  GoogleMessageHandler handler_;
-  SharedCircularBuffer* buffer_;
  ngx_log_t* log_;

  DISALLOW_COPY_AND_ASSIGN(NgxMessageHandler);
@@ -33,7 +33,6 @@
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
 #include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
 #include "net/instaweb/rewriter/public/server_context.h"
-#include "net/instaweb/rewriter/public/static_asset_manager.h"
 #include "net/instaweb/system/public/in_place_resource_recorder.h"
 #include "net/instaweb/system/public/serf_url_async_fetcher.h"
 #include "net/instaweb/system/public/system_caches.h"
@@ -63,9 +62,6 @@ class UrlAsyncFetcher;
 class UrlFetcher;
 class Writer;

-const char NgxRewriteDriverFactory::kStaticAssetPrefix[] =
-    "/ngx_pagespeed_static/";
-
 class SharedCircularBuffer;

 NgxRewriteDriverFactory::NgxRewriteDriverFactory(
@@ -75,17 +71,18 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
        NULL /* default shared memory runtime */, hostname, port),
      main_conf_(NULL),
      threads_started_(false),
-      use_per_vhost_statistics_(false),
-      ngx_message_handler_(new NgxMessageHandler(thread_system()->NewMutex())),
+      ngx_message_handler_(
+          new NgxMessageHandler(timer(), thread_system()->NewMutex())),
      ngx_html_parse_message_handler_(
-          new NgxMessageHandler(thread_system()->NewMutex())),
-      install_crash_handler_(false),
+          new NgxMessageHandler(timer(), thread_system()->NewMutex())),
      log_(NULL),
      resolver_timeout_(NGX_CONF_UNSET_MSEC),
      use_native_fetcher_(false),
      ngx_shared_circular_buffer_(NULL),
      hostname_(hostname.as_string()),
-      port_(port) {
+      port_(port),
+      process_script_variables_(false),
+      process_script_variables_set_(false) {
  InitializeDefaultOptions();
  default_options()->set_beacon_url("/ngx_pagespeed_beacon");
  SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
@@ -151,11 +148,6 @@ RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
  return options;
 }

-void NgxRewriteDriverFactory::InitStaticAssetManager(
-    StaticAssetManager* static_asset_manager) {
-  static_asset_manager->set_library_url_prefix(kStaticAssetPrefix);
-}
-
 bool NgxRewriteDriverFactory::InitNgxUrlAsyncFetchers() {
  log_ = ngx_cycle->log;
  for (size_t i = 0; i < ngx_url_async_fetchers_.size(); ++i) {
@@ -217,7 +209,7 @@ void NgxRewriteDriverFactory::StartThreads() {

 void NgxRewriteDriverFactory::LoggingInit(ngx_log_t* log) {
  net_instaweb::log_message_handler::Install(log);
-  if (install_crash_handler_) {
+  if (install_crash_handler()) {
    NgxMessageHandler::InstallCrashHandler(log);
  }
  ngx_message_handler_->set_log(log);
@@ -234,7 +226,7 @@ void NgxRewriteDriverFactory::SetCircularBuffer(
 void NgxRewriteDriverFactory::SetServerContextMessageHandler(
    ServerContext* server_context, ngx_log_t* log) {
  NgxMessageHandler* handler = new NgxMessageHandler(
-      thread_system()->NewMutex());
+      timer(), thread_system()->NewMutex());
  handler->set_log(log);
  // The ngx_shared_circular_buffer_ will be NULL if MessageBufferSize hasn't
  // been raised from its default of 0.
@@ -45,14 +45,11 @@ class NgxUrlAsyncFetcher;
 class SharedCircularBuffer;
 class SharedMemRefererStatistics;
 class SlowWorker;
-class StaticAssetManager;
 class Statistics;
 class SystemThreadSystem;

 class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
 public:
-  static const char kStaticAssetPrefix[];
-
  // We take ownership of the thread system.
  explicit NgxRewriteDriverFactory(
      const ProcessContext& process_context,
@@ -68,9 +65,6 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  // Create a new RewriteOptions.  In this implementation it will be an
  // NgxRewriteOptions.
  virtual RewriteOptions* NewRewriteOptions();
-  // Initializes the StaticAssetManager.
-  virtual void InitStaticAssetManager(
-      StaticAssetManager* static_asset_manager);
  virtual ServerContext* NewDecodingServerContext();
  bool InitNgxUrlAsyncFetchers();
  // Check resolver configured or not.
@@ -98,18 +92,6 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {

  void set_main_conf(NgxRewriteOptions* main_conf) {  main_conf_ = main_conf; }

-  bool use_per_vhost_statistics() const {
-    return use_per_vhost_statistics_;
-  }
-  void set_use_per_vhost_statistics(bool x) {
-    use_per_vhost_statistics_ = x;
-  }
-  bool install_crash_handler() const {
-    return install_crash_handler_;
-  }
-  void set_install_crash_handler(bool x) {
-    install_crash_handler_ = x;
-  }
  void set_resolver(ngx_resolver_t* resolver) {
    resolver_ = resolver;
  }
@@ -123,16 +105,8 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  void set_use_native_fetcher(bool x) {
    use_native_fetcher_ = x;
  }
-  void set_rate_limit_background_fetches(bool x) {
-    rate_limit_background_fetches_ = x;
-  }
-
-  // We use a beacon handler to collect data for critical images,
-  // css, etc., so filters should be configured accordingly.
-  //
-  // TODO(jefftk): move to SystemRewriteDriverFactory
-  virtual bool UseBeaconResultsInFilters() const {
-    return true;
+  bool process_script_variables() {
+    return process_script_variables_;
  }

  void LoggingInit(ngx_log_t* log);
@@ -141,6 +115,15 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {

  virtual void SetCircularBuffer(SharedCircularBuffer* buffer);

+  bool SetProcessScriptVariables(bool process_script_variables) {
+    if (!process_script_variables_set_) {
+      process_script_variables_ = process_script_variables;
+      process_script_variables_set_ = true;
+      return true;
+    }
+    return false;
+  }
+
 private:
  Timer* timer_;

@@ -149,20 +132,14 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  NgxRewriteOptions* main_conf_;

  bool threads_started_;
-  // If true, we'll have a separate statistics object for each vhost
-  // (along with a global aggregate), rather than just a single object
-  // aggregating all of them.
-  bool use_per_vhost_statistics_;
  NgxMessageHandler* ngx_message_handler_;
  NgxMessageHandler* ngx_html_parse_message_handler_;
-  bool install_crash_handler_;

  std::vector<NgxUrlAsyncFetcher*> ngx_url_async_fetchers_;
  ngx_log_t* log_;
  ngx_msec_t resolver_timeout_;
  ngx_resolver_t* resolver_;
  bool use_native_fetcher_;
-  bool rate_limit_background_fetches_;
  typedef std::set<NgxMessageHandler*> NgxMessageHandlerSet;
  NgxMessageHandlerSet server_context_message_handlers_;

@@ -172,6 +149,8 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {

  GoogleString hostname_;
  int port_;
+  bool process_script_variables_;
+  bool process_script_variables_set_;

  DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
 };
@@ -31,17 +31,26 @@ extern "C" {
 #include "net/instaweb/rewriter/public/file_load_policy.h"
 #include "net/instaweb/rewriter/public/rewrite_options.h"
 #include "net/instaweb/system/public/system_caches.h"
+#include "net/instaweb/util/public/message_handler.h"
 #include "net/instaweb/util/public/timer.h"

 namespace net_instaweb {

 namespace {

+const char kStatisticsPath[] = "StatisticsPath";
+const char kGlobalStatisticsPath[] = "GlobalStatisticsPath";
+const char kConsolePath[] = "ConsolePath";
+const char kMessagesPath[] = "MessagesPath";
+const char kAdminPath[] = "AdminPath";
+const char kGlobalAdminPath[] = "GlobalAdminPath";
+
 // These options are copied from mod_instaweb.cc, where APACHE_CONFIG_OPTIONX
 // indicates that they can not be set at the directory/location level. They set
-// options in the RewriteDriverFactory, so they do not appear in RewriteOptions.
-// They are not alphabetized on purpose, but rather left in the same order as in
-// mod_instaweb.cc in case we end up needing to compare.
+// options in the RewriteDriverFactory, so they're entirely global and do not
+// appear in RewriteOptions.  They are not alphabetized on purpose, but rather
+// left in the same order as in mod_instaweb.cc in case we end up needing to
+// compare.
 // TODO(oschaaf): this duplication is a short term solution.
 const char* const server_only_options[] = {
  "FetcherTimeoutMs",
@@ -54,6 +63,7 @@ const char* const server_only_options[] = {
  "MessageBufferSize",
  "NumRewriteThreads",
  "NumExpensiveRewriteThreads",
+  "StaticAssetPrefix",
  "TrackOriginalContentLength",
  "UsePerVHostStatistics",  // TODO(anupama): What to do about "No longer used"
  "BlockingRewriteRefererUrls",
@@ -88,11 +98,35 @@ NgxRewriteOptions::NgxRewriteOptions(ThreadSystem* thread_system)
 void NgxRewriteOptions::Init() {
  DCHECK(ngx_properties_ != NULL)
      << "Call NgxRewriteOptions::Initialize() before construction";
+  clear_inherited_scripts_ = false;
  InitializeOptions(ngx_properties_);
 }

 void NgxRewriteOptions::AddProperties() {
-  // Nothing ngx-specific for now.
+  // Nginx-specific options.
+  add_ngx_option(
+      "", &NgxRewriteOptions::statistics_path_, "nsp", kStatisticsPath,
+      kServerScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics",
+      false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::global_statistics_path_, "ngsp",
+      kGlobalStatisticsPath, kProcessScope,
+      "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics",
+      false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kServerScope,
+      "Set the console path. Ex: /pagespeed_console", false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::messages_path_, "nmp", kMessagesPath,
+      kServerScope, "Set the messages path.  Ex: /ngx_pagespeed_message",
+      false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::admin_path_, "nap", kAdminPath,
+      kServerScope, "Set the admin path.  Ex: /pagespeed_admin", false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
+      kProcessScope, "Set the global admin path.  Ex: /pagespeed_global_admin",
+      false);

  MergeSubclassProperties(ngx_properties_);

@@ -222,7 +256,7 @@ const char* ps_error_string_for_option(
 const char* NgxRewriteOptions::ParseAndSetOptions(
    StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
    NgxRewriteDriverFactory* driver_factory,
-    RewriteOptions::OptionScope scope) {
+    RewriteOptions::OptionScope scope, ngx_conf_t* cf, bool compile_scripts) {
  CHECK_GE(n_args, 1);

  StringPiece directive = args[0];
@@ -238,77 +272,129 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
        pool, directive, "cannot be set at this scope.");
  }

+  ScriptLine* script_line;
+  script_line = NULL;
+  // Only allow script variable support for LoadFromFile for now.
+  // Note that LoadFromFile should not be scriptable on wildcard hosts,
+  // as browsers might be able to manipulate its natural use-case: $http_host.
+  if (!StringCaseStartsWith(directive, "LoadFromFile")) {
+    compile_scripts = false;
+  }
+
+  if (n_args == 1 && StringCaseEqual(directive, "ClearInheritedScripts")) {
+    clear_inherited_scripts_ = true;
+    return NGX_CONF_OK;
+  }
+
+  if (compile_scripts) {
+    CHECK(cf != NULL);
+    int i;
+    // Skip the first arg which is always 'pagespeed'
+    for (i = 1; i < n_args; i++) {
+      ngx_str_t script_source;
+
+      script_source.len = args[i].as_string().length();
+      std::string tmp = args[i].as_string();
+      script_source.data = reinterpret_cast<u_char*>(
+          const_cast<char*>(tmp.c_str()));
+
+      if (ngx_http_script_variables_count(&script_source) > 0) {
+        ngx_http_script_compile_t* sc =
+            reinterpret_cast<ngx_http_script_compile_t*>(
+                ngx_pcalloc(cf->pool, sizeof(ngx_http_script_compile_t)));
+        sc->cf = cf;
+        sc->source = &script_source;
+        sc->lengths = reinterpret_cast<ngx_array_t**>(
+            ngx_pcalloc(cf->pool, sizeof(ngx_array_t*)));
+        sc->values = reinterpret_cast<ngx_array_t**>(
+            ngx_pcalloc(cf->pool, sizeof(ngx_array_t*)));
+        sc->variables = 1;
+        sc->complete_lengths = 1;
+        sc->complete_values = 1;
+        if (ngx_http_script_compile(sc) != NGX_OK) {
+          return ps_error_string_for_option(
+              pool, directive, "Failed to compile script variables");
+        } else {
+          if (script_line == NULL) {
+            script_line = new ScriptLine(args, n_args, scope);
+          }
+          script_line->AddScriptAndArgIndex(sc, i);
+        }
+      }
+    }
+
+    if (script_line != NULL) {
+      script_lines_.push_back(RefCountedPtr<ScriptLine>(script_line));
+      // We have found script variables in the current configuration line, and
+      // prepared the associated rewriteoptions for that.
+      // We will defer parsing, validation and processing of this line to
+      // request time. That means we are done handling this configuration line.
+      return NGX_CONF_OK;
+    }
+  }
+
  GoogleString msg;
  OptionSettingResult result;
  if (n_args == 1) {
    result = ParseAndSetOptions0(directive, &msg, handler);
  } else if (n_args == 2) {
    StringPiece arg = args[1];
-    // TODO(morlovich): Remove these special hacks, and handle these via
-    // ParseAndSetOptionFromEnum1.
-    if (IsDirective(directive, "UsePerVHostStatistics")) {
-      result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
-          arg, driver_factory,
-          &NgxRewriteDriverFactory::set_use_per_vhost_statistics);
-    } else if (IsDirective(directive, "InstallCrashHandler")) {
-      result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
-          arg, driver_factory,
-          &NgxRewriteDriverFactory::set_install_crash_handler);
-    } else if (IsDirective(directive, "MessageBufferSize")) {
-      int message_buffer_size;
-      bool ok = StringToInt(arg.as_string(), &message_buffer_size);
-      if (ok && message_buffer_size >= 0) {
-        driver_factory->set_message_buffer_size(message_buffer_size);
-        result = RewriteOptions::kOptionOk;
-      } else {
-        result = RewriteOptions::kOptionValueInvalid;
-      }
-    } else if (IsDirective(directive, "UseNativeFetcher")) {
+    if (IsDirective(directive, "UseNativeFetcher")) {
      result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
          arg, driver_factory,
          &NgxRewriteDriverFactory::set_use_native_fetcher);
-    } else if (IsDirective(directive, "RateLimitBackgroundFetches")) {
-      result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
-          arg, driver_factory,
-          &NgxRewriteDriverFactory::set_rate_limit_background_fetches);
-    } else if (IsDirective(directive, "ForceCaching")) {
-      result = ParseAndSetOptionHelper<SystemRewriteDriverFactory>(
-          arg, driver_factory,
-          &SystemRewriteDriverFactory::set_force_caching);
-    } else if (IsDirective(directive, "ListOutstandingUrlsOnError")) {
-      result = ParseAndSetOptionHelper<SystemRewriteDriverFactory>(
-          arg, driver_factory,
-          &SystemRewriteDriverFactory::list_outstanding_urls_on_error);
-    } else if (IsDirective(directive, "TrackOriginalContentLength")) {
-      result = ParseAndSetOptionHelper<SystemRewriteDriverFactory>(
-          arg, driver_factory,
-          &SystemRewriteDriverFactory::set_track_original_content_length);
-    } else {
-      result = ParseAndSetOptionFromName1(directive, args[1], &msg, handler);
-    }
-  } else if (n_args == 3) {
-    // Short-term special handling, until this moves to common code.
-    // TODO(morlovich): Clean this up.
-    if (StringCaseEqual(directive, "CreateSharedMemoryMetadataCache")) {
-      int64 kb = 0;
-      if (!StringToInt64(args[2], &kb) || kb < 0) {
-        result = RewriteOptions::kOptionValueInvalid;
-        msg = "size_kb must be a positive 64-bit integer";
+    } else if (StringCaseEqual("ProcessScriptVariables", args[0])) {
+      if (scope == RewriteOptions::kProcessScopeStrict) {
+        if (StringCaseEqual(arg, "on")) {
+          if (driver_factory->SetProcessScriptVariables(true)) {
+            result = RewriteOptions::kOptionOk;
+          } else {
+            return const_cast<char*>(
+                "pagespeed ProcessScriptVariables: can only be set once");
+          }
+        } else if (StringCaseEqual(arg, "off")) {
+          if (driver_factory->SetProcessScriptVariables(false)) {
+            result = RewriteOptions::kOptionOk;
+          } else {
+            return const_cast<char*>(
+                "pagespeed ProcessScriptVariables: can only be set once");
+          }
+        } else {
+          return const_cast<char*>(
+              "pagespeed ProcessScriptVariables: invalid value");
+        }
      } else {
-        bool ok = driver_factory->caches()->CreateShmMetadataCache(
-            args[1].as_string(), kb, &msg);
-        result = ok ? kOptionOk : kOptionValueInvalid;
+        return const_cast<char*>(
+            "ProcessScriptVariables is only allowed at the top level");
      }
    } else {
-      result = ParseAndSetOptionFromName2(directive, args[1], args[2],
-                                          &msg, handler);
+      result = ParseAndSetOptionFromName1(directive, arg, &msg, handler);
+      if (result == RewriteOptions::kOptionNameUnknown) {
+        result = driver_factory->ParseAndSetOption1(
+            directive,
+            arg,
+            scope >= RewriteOptions::kProcessScope,
+            &msg,
+            handler);
+      }
+    }
+  } else if (n_args == 3) {
+    result = ParseAndSetOptionFromName2(directive, args[1], args[2],
+                                        &msg, handler);
+    if (result == RewriteOptions::kOptionNameUnknown) {
+      result = driver_factory->ParseAndSetOption2(
+          directive,
+          args[1],
+          args[2],
+          scope >= RewriteOptions::kProcessScope,
+          &msg,
+          handler);
    }
  } else if (n_args == 4) {
    result = ParseAndSetOptionFromName3(
        directive, args[1], args[2], args[3], &msg, handler);
  } else {
-    return ps_error_string_for_option(
-        pool, directive, "not recognized or too many arguments");
+    result = RewriteOptions::kOptionNameUnknown;
  }

  switch (result) {
@@ -330,13 +416,92 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
  return NULL;
 }

+// Execute all entries in the script_lines vector, and hand the result off to
+// ParseAndSetOptions to obtain the final option values.
+bool NgxRewriteOptions::ExecuteScriptVariables(
+    ngx_http_request_t* r, MessageHandler* handler,
+    NgxRewriteDriverFactory* driver_factory) {
+  bool script_error = false;
+
+  if (script_lines_.size() > 0) {
+    std::vector<RefCountedPtr<ScriptLine> >::iterator it;
+    for (it = script_lines_.begin() ; it != script_lines_.end(); ++it) {
+      ScriptLine* script_line = it->get();
+      StringPiece args[NGX_PAGESPEED_MAX_ARGS];
+      std::vector<ScriptArgIndex*>::iterator cs_it;
+      int i;
+
+      for (i = 0; i < script_line->n_args(); i++) {
+        args[i] = script_line->args()[i];
+      }
+
+      for (cs_it = script_line->data().begin();
+           cs_it != script_line->data().end(); cs_it++) {
+        ngx_http_script_compile_t* script;
+        ngx_array_t* values;
+        ngx_array_t* lengths;
+        ngx_str_t value;
+
+        script = (*cs_it)->script();
+        lengths = *script->lengths;
+        values = *script->values;
+
+        if (ngx_http_script_run(r, &value, lengths->elts, 0, values->elts)
+            == NULL) {
+          handler->Message(kError, "ngx_http_script_run error");
+          script_error = true;
+          break;
+        } else  {
+          args[(*cs_it)->index()] = str_to_string_piece(value);
+        }
+      }
+
+      const char* status = ParseAndSetOptions(args, script_line->n_args(),
+          r->pool, handler, driver_factory, script_line->scope(), NULL /*cf*/,
+          false /*compile scripts*/);
+
+      if (status != NULL) {
+        script_error = true;
+        handler->Message(kWarning,
+            "Error setting option value from script: '%s'", status);
+        break;
+      }
+    }
+  }
+
+  if (script_error) {
+    handler->Message(kWarning,
+        "Script error(s) in configuration, disabling optimization");
+    set_enabled(RewriteOptions::kEnabledOff);
+    return false;
+  }
+
+  return true;
+}
+
+void NgxRewriteOptions::CopyScriptLinesTo(
+    NgxRewriteOptions* destination) const {
+  destination->script_lines_ = script_lines_;
+}
+
+void NgxRewriteOptions::AppendScriptLinesTo(
+    NgxRewriteOptions* destination) const {
+  destination->script_lines_.insert(destination->script_lines_.end(),
+                                    script_lines_.begin(), script_lines_.end());
+}
+
 NgxRewriteOptions* NgxRewriteOptions::Clone() const {
  NgxRewriteOptions* options = new NgxRewriteOptions(
      StrCat("cloned from ", description()), thread_system());
+  this->CopyScriptLinesTo(options);
  options->Merge(*this);
  return options;
 }

+void NgxRewriteOptions::Merge(const RewriteOptions& src) {
+  SystemRewriteOptions::Merge(src);
+}
+
 const NgxRewriteOptions* NgxRewriteOptions::DynamicCast(
    const RewriteOptions* instance) {
  return dynamic_cast<const NgxRewriteOptions*>(instance);
@@ -27,13 +27,81 @@ extern "C" {
  #include <ngx_http.h>
 }

+#include <vector>
+
+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/util/public/ref_counted_ptr.h"
+#include "net/instaweb/util/public/stl_util.h"          // for STLDeleteElements
 #include "net/instaweb/rewriter/public/rewrite_options.h"
 #include "net/instaweb/system/public/system_rewrite_options.h"

+#define NGX_PAGESPEED_MAX_ARGS 10
+
 namespace net_instaweb {

 class NgxRewriteDriverFactory;

+class ScriptArgIndex {
+ public:
+  explicit ScriptArgIndex(ngx_http_script_compile_t* script, int index)
+    : script_(script), index_(index) {
+      CHECK(script != NULL);
+      CHECK(index > 0 && index < NGX_PAGESPEED_MAX_ARGS);
+  }
+
+  virtual ~ScriptArgIndex() {}
+
+  ngx_http_script_compile_t* script() { return script_; }
+  int index() { return index_; }
+
+ private:
+  // Not owned.
+  ngx_http_script_compile_t* script_;
+  int index_;
+};
+
+// Refcounted, because the ScriptArgIndexes inside data_ can be shared between
+// different rewriteoptions.
+class ScriptLine : public RefCounted<ScriptLine> {
+ public:
+  explicit ScriptLine(StringPiece* args, int n_args,
+                      RewriteOptions::OptionScope scope)
+    : n_args_(n_args),
+      scope_(scope) {
+
+      for (int i = 0; i < n_args; i++) {
+        args_[i] = args[i];
+      }
+  }
+
+  virtual ~ScriptLine() {
+    STLDeleteElements(&data_);
+    data_.clear();
+  }
+
+  void AddScriptAndArgIndex(ngx_http_script_compile_t* script,
+                            int script_index) {
+    CHECK(script != NULL);
+    CHECK(script_index <  NGX_PAGESPEED_MAX_ARGS);
+    data_.push_back(new ScriptArgIndex(script, script_index));
+  }
+
+  int n_args() { return n_args_;}
+  StringPiece* args() { return args_;}
+  RewriteOptions::OptionScope scope() { return scope_; }
+  std::vector<ScriptArgIndex*>& data() {
+    return data_;
+  }
+
+ private:
+  StringPiece args_[NGX_PAGESPEED_MAX_ARGS];
+  int n_args_;
+  RewriteOptions::OptionScope scope_;
+  std::vector<ScriptArgIndex*> data_;
+
+  DISALLOW_COPY_AND_ASSIGN(ScriptLine);
+};
+
 class NgxRewriteOptions : public SystemRewriteOptions {
 public:
  // See rewrite_options::Initialize and ::Terminate
@@ -56,18 +124,53 @@ class NgxRewriteOptions : public SystemRewriteOptions {
  // on failure.
  //
  // pool is a memory pool for allocating error strings.
+  // cf is only required when compile_scripts is true
+  // when compile_scripts is true, the rewrite_options will be prepared
+  // for replacing any script $variables encountered in args. when false,
+  // script variables will be substituted using the prepared rewrite options.
  const char* ParseAndSetOptions(
      StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
-      NgxRewriteDriverFactory* driver_factory, OptionScope scope);
+      NgxRewriteDriverFactory* driver_factory, OptionScope scope,
+      ngx_conf_t* cf, bool compile_scripts);
+  bool ExecuteScriptVariables(
+      ngx_http_request_t* r, MessageHandler* handler,
+      NgxRewriteDriverFactory* driver_factory);
+  void CopyScriptLinesTo(NgxRewriteOptions* destination) const;
+  void AppendScriptLinesTo(NgxRewriteOptions* destination) const;

  // Make an identical copy of these options and return it.
  virtual NgxRewriteOptions* Clone() const;
+  virtual void Merge(const RewriteOptions& src);

  // Returns a suitably down cast version of 'instance' if it is an instance
  // of this class, NULL if not.
  static const NgxRewriteOptions* DynamicCast(const RewriteOptions* instance);
  static NgxRewriteOptions* DynamicCast(RewriteOptions* instance);

+  const GoogleString& statistics_path() const {
+    return statistics_path_.value();
+  }
+  const GoogleString& global_statistics_path() const {
+    return global_statistics_path_.value();
+  }
+  const GoogleString& console_path() const {
+    return console_path_.value();
+  }
+  const GoogleString& messages_path() const {
+    return messages_path_.value();
+  }
+  const GoogleString& admin_path() const {
+    return admin_path_.value();
+  }
+  const GoogleString& global_admin_path() const {
+    return global_admin_path_.value();
+  }
+  const std::vector<RefCountedPtr<ScriptLine> >& script_lines() const {
+    return script_lines_;
+  }
+  const bool& clear_inherited_scripts() const {
+    return clear_inherited_scripts_;
+  }

 private:
  // Helper methods for ParseAndSetOptions().  Each can:
@@ -109,10 +212,24 @@ class NgxRewriteOptions : public SystemRewriteOptions {
  static void add_ngx_option(typename OptionClass::ValueType default_value,
                             OptionClass NgxRewriteOptions::*offset,
                             const char* id,
-                             StringPiece option_name) {
-    AddProperty(default_value, offset, id, option_name, ngx_properties_);
+                             StringPiece option_name,
+                             OptionScope scope,
+                             const char* help,
+                             bool safe_to_print) {
+    AddProperty(default_value, offset, id, option_name, scope, help,
+                safe_to_print, ngx_properties_);
  }

+  Option<GoogleString> statistics_path_;
+  Option<GoogleString> global_statistics_path_;
+  Option<GoogleString> console_path_;
+  Option<GoogleString> messages_path_;
+  Option<GoogleString> admin_path_;
+  Option<GoogleString> global_admin_path_;
+
+  bool clear_inherited_scripts_;
+  std::vector<RefCountedPtr<ScriptLine> > script_lines_;
+
  // Helper for ParseAndSetOptions.  Returns whether the two directives equal,
  // ignoring case.
  bool IsDirective(StringPiece config_directive, StringPiece compare_directive);
@@ -77,4 +77,9 @@ SystemRequestContext* NgxServerContext::NewRequestContext(
                                  str_to_string_piece(local_ip));
 }

+GoogleString NgxServerContext::FormatOption(StringPiece option_name,
+                                            StringPiece args) {
+  return StrCat("pagespeed ", option_name, " ", args, ";");
+}
+
 }  // namespace net_instaweb
@@ -55,6 +55,8 @@ class NgxServerContext : public SystemServerContext {
    return dynamic_cast<NgxMessageHandler*>(message_handler());
  }

+  virtual GoogleString FormatOption(StringPiece option_name, StringPiece args);
+
 private:
  NgxRewriteDriverFactory* ngx_factory_;

@@ -249,7 +249,7 @@ namespace net_instaweb {

  // This is the read event which is called in the main thread.
  // It will do the real work. Add the work event and start the fetch.
-  void NgxUrlAsyncFetcher::CommandHandler(ngx_event_t *cmdev) {
+  void NgxUrlAsyncFetcher::CommandHandler(ngx_event_t* cmdev) {
    char command;
    int rc;
    ngx_connection_t* c = static_cast<ngx_connection_t*>(cmdev->data);
@@ -71,12 +71,12 @@ function keepalive_test() {
      if [ -z "$POST_DATA" ]; then
        curl -m 2 -S -s -v -H "Accept-Encoding: $accept_encoding" \
          -H "Host: $HOST_NAME" $URL $URL $URL $URL $URL > /dev/null \
-          2>>"$TEST_TMP/$CURL_LOG_FILE"
+          2>>"$TEST_TMP/$CURL_LOG_FILE" || true
      else
        curl -X POST --data "$POST_DATA" -m 2 -S -s -v \
          -H "Accept-Encoding: $accept_encoding" -H "Host: $HOST_NAME"\
          $URL $URL $URL $URL $URL > /dev/null \
-          2>>"$TEST_TMP/$CURL_LOG_FILE"
+          2>>"$TEST_TMP/$CURL_LOG_FILE" || true
      fi
    done
  done
@@ -92,8 +92,16 @@ function keepalive_test() {
    | grep -v "^\\* Connection.*left intact"\
    | grep -v "^} \\[data not shown"\
    | grep -v "^\\* upload completely sent off"\
+    | grep -v "^\\* Found bundle for host"\
    | grep -v "^\\* connected"\
-    | grep -v "^\\*   Trying.*\\.\\.\\.")
+    | grep -v "^\\* Found bundle for host"\
+    | grep -v "^\\* Adding handle"\
+    | grep -v "^\\* Curl_addHandleToPipeline"\
+    | grep -v "^\\* - Conn "\
+    | grep -v "^\\* Server "\
+    | grep -v "^\\*   Trying.*\\.\\.\\."\
+    | grep -v "^\\* Hostname was NOT found in DNS cache" \
+    || true)

  # Nothing should remain after that.
  check [ -z "$OUT" ]
@@ -101,7 +109,8 @@ function keepalive_test() {
  # Filter the nginx log from our vhost from unimportant messages.
  OUT=$(cat "$TEST_TMP/$NGX_LOG_FILE"\
    | grep -v "closed keepalive connection$" \
-    | grep -v ".*Cache Flush.*")
+    | grep -v ".*Cache Flush.*" \
+    || true)

  # Nothing should remain after that.
  check [ -z "$OUT" ]
@@ -213,24 +222,20 @@ check_rewriting_status() {
  else
    check_not zgrep -q "pagespeed.ic" $OUT_CONTENTS_FILE
  fi
-  # Reset WGET_ARGS.
-  WGET_ARGS=""
 }

 # Helper method that obtains a gzipped response and verifies that rewriting
 # has happened. Also takes an extra parameter that identifies extra headers
 # to be added during wget.
 check_for_rewriting() {
-  WGET_ARGS="$GZIP_WGET_ARGS $1"
-  check_rewriting_status true
+  WGET_ARGS="$GZIP_WGET_ARGS $1" check_rewriting_status true
 }

 # Helper method that obtains a gzipped response and verifies that no rewriting
 # has happened. Also takes an extra parameter that identifies extra headers
 # to be added during wget.
 check_for_no_rewriting() {
-  WGET_ARGS="$GZIP_WGET_ARGS $1"
-  check_rewriting_status false
+  WGET_ARGS="$GZIP_WGET_ARGS $1" check_rewriting_status false
 }

 if $RUN_TESTS; then
@@ -245,6 +250,12 @@ else
  exit 4
 fi

+# check_stat in system_test_helpers.sh needs to know whether statstistics are
+# enabled, which is always the case for ngx_pagespeed.
+statistics_enabled=1
+CACHE_FLUSH_TEST="on"
+CACHE_PURGE_METHODS="PURGE GET"
+
 # run generic system tests
 SYSTEM_TEST_FILE="$MOD_PAGESPEED_DIR/src/net/instaweb/system/system_test.sh"

@@ -254,7 +265,7 @@ if [ ! -e "$SYSTEM_TEST_FILE" ] ; then
  exit 2
 fi

-PSA_JS_LIBRARY_URL_PREFIX="ngx_pagespeed_static"
+PSA_JS_LIBRARY_URL_PREFIX="pagespeed_custom_static"

 # An expected failure can be indicated like: "~In-place resource optimization~"
 PAGESPEED_EXPECTED_FAILURES="
@@ -348,29 +359,20 @@ else

  # The 1st request results in a cache miss, non-rewritten response
  # produced by pagespeed code and a subsequent purge request.
-  # Because of the random bypassing of the cache (required for beaconing
-  # integration), this request could result in a BYPASS as well.
  start_test Check for case where rewritten cache should get purged.
  check_for_no_rewriting "--header=Host:proxy_cache.example.com"
-  check egrep -q "X-Cache: MISS|BYPASS" $OUT_HEADERS_FILE
+  check egrep -q "X-Cache: MISS" $OUT_HEADERS_FILE
  fetch_until $STATISTICS_URL \
-    'grep -c downstream_cache_purge_attempts:[[:space:]]*1' 1
-
-  while [ x"$(grep "$PURGE_REQUEST_IN_ACCESS_LOG" $ACCESS_LOG)" == x"" ] ; do
-    echo "waiting for purge request to show up in access log"
-    sleep .2
-  done
+    'grep -c successful_downstream_cache_purges:[[:space:]]*1' 1

  check [ $(grep -ce "$PURGE_REQUEST_IN_ACCESS_LOG" $ACCESS_LOG) = 1 ];

  # The 2nd request results in a cache miss (because of the previous purge),
  # rewritten response produced by pagespeed code and no new purge requests.
-  # Because of the random bypassing of the cache (required for beaconing
-  # integration), this request could result in a BYPASS as well.
  start_test Check for case where rewritten cache should not get purged.
  check_for_rewriting "--header=Host:proxy_cache.example.com \
                      --header=X-PSA-Blocking-Rewrite:psatest"
-  check egrep -q "X-Cache: MISS|BYPASS" $OUT_HEADERS_FILE
+  check egrep -q "X-Cache: MISS" $OUT_HEADERS_FILE
  CURRENT_STATS=$($WGET_DUMP $STATISTICS_URL)
  check_from "$CURRENT_STATS" egrep -q \
    "downstream_cache_purge_attempts:[[:space:]]*1"
@@ -392,7 +394,8 @@ else
  # output is also marked as a cache-miss, indicating that the instrumentation
  # was done by the backend.
  start_test Check whether beaconing is accompanied by a BYPASS always.
-  WGET_ARGS="-S --header=Host:proxy_cache.example.com"
+  WGET_ARGS="-S --header=Host:proxy_cache.example.com \
+                --header=X-Allow-Beacon:yes"
  CACHABLE_HTML_LOC+="?PageSpeedFilters=lazyload_images"
  fetch_until -gzip $CACHABLE_HTML_LOC \
      "zgrep -c \"pagespeed\.CriticalImages\.Run\"" 1
@@ -407,12 +410,12 @@ check_from "$OUT" egrep -q \
  '^X-Page-Speed: [0-9]+[.][0-9]+[.][0-9]+[.][0-9]+-[0-9]+'

 start_test pagespeed is defaulting to more than PassThrough
-fetch_until $TEST_ROOT/bot_test.html 'grep -c \.pagespeed\.' 2
+fetch_until $TEST_ROOT/bot_test.html 'fgrep -c .pagespeed.' 2

 start_test 404s are served and properly recorded.
 NUM_404=$(scrape_stat resource_404_count)
 echo "Initial 404s: $NUM_404"
-WGET_ERROR=$($WGET -O /dev/null $BAD_RESOURCE_URL 2>&1)
+WGET_ERROR=$(check_not $WGET -O /dev/null $BAD_RESOURCE_URL 2>&1)
 check_from "$WGET_ERROR" fgrep -q "404 Not Found"

 # Check that the stat got bumped.
@@ -445,16 +448,13 @@ if [ "$HOSTNAME" = "localhost:$PRIMARY_PORT" ] ; then
  ALT_STAT_URL=$(echo $STATISTICS_URL | sed s#localhost#$NON_LOCAL_IP#)

  echo "wget $ALT_STAT_URL >& $TEMPDIR/alt_stat_url.$$"
-  wget $ALT_STAT_URL >& "$TEMPDIR/alt_stat_url.$$"
-  check [ $? = 8 ]
+  check_error_code 8 wget $ALT_STAT_URL >& "$TEMPDIR/alt_stat_url.$$"
  rm -f "$TEMPDIR/alt_stat_url.$$"

  ALT_CE_URL="$ALT_STAT_URL.pagespeed.ce.8CfGBvwDhH.css"
-  wget -O - $ALT_CE_URL  >& "$TEMPDIR/alt_ce_url.$$"
-  check [ $? = 8 ]
-
-  wget -O - --header="Host: $HOSTNAME" $ALT_CE_URL >& "$TEMPDIR/alt_ce_url.$$"
-  check [ $? = 8 ]
+  check_error_code 8 wget -O - $ALT_CE_URL  >& "$TEMPDIR/alt_ce_url.$$"
+  check_error_code 8 wget -O - --header="Host: $HOSTNAME" $ALT_CE_URL \
+    >& "$TEMPDIR/alt_ce_url.$$"
  rm -f "$TEMPDIR/alt_ce_url.$$"

  # Even though we don't have a cookie, we will conservatively avoid
@@ -525,7 +525,59 @@ if [ "$HOSTNAME" = "localhost:$PRIMARY_PORT" ] ; then
  AUTH=""
 fi

-WGET_ARGS=""
+start_test "Custom statistics paths in server block"
+
+# Served on normal paths by default.
+URL="inherit-paths.example.com/ngx_pagespeed_statistics"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
+check_from "$OUT" fgrep -q cache_time_us
+
+URL="inherit-paths.example.com/ngx_pagespeed_message"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
+check_from "$OUT" fgrep -q Info
+
+URL="inherit-paths.example.com/pagespeed_console"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
+check_from "$OUT" fgrep -q console_div
+
+URL="inherit-paths.example.com/pagespeed_admin/"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
+check_from "$OUT" fgrep -q Admin
+
+# Not served on normal paths when overriden.
+URL="custom-paths.example.com/ngx_pagespeed_statistics"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET_DUMP $URL)
+check_not_from "$OUT" fgrep -q cache_time_us
+
+URL="custom-paths.example.com/ngx_pagespeed_message"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET_DUMP $URL)
+check_not_from "$OUT" fgrep -q Info
+
+URL="custom-paths.example.com/pagespeed_console"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET_DUMP $URL)
+check_not_from "$OUT" fgrep -q console_div
+
+URL="custom-paths.example.com/pagespeed_admin/"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET_DUMP $URL)
+check_not_from "$OUT" fgrep -q Admin
+
+# Served on custom paths when overriden
+URL="custom-paths.example.com/custom_pagespeed_statistics"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
+check_from "$OUT" fgrep -q cache_time_us
+
+URL="custom-paths.example.com/custom_pagespeed_message"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
+check_from "$OUT" fgrep -q Info
+
+URL="custom-paths.example.com/custom_pagespeed_console"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
+check_from "$OUT" fgrep -q console_div
+
+URL="custom-paths.example.com/custom_pagespeed_admin/"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
+check_from "$OUT" fgrep -q Admin
+
 function gunzip_grep_0ff() {
  gunzip - | fgrep -q "color:#00f"
  echo $?
@@ -552,21 +604,21 @@ start_test Accept bad query params and headers
 # The examples page should have this EXPECTED_EXAMPLES_TEXT on it.
 EXPECTED_EXAMPLES_TEXT="PageSpeed Examples Directory"
 OUT=$(wget -O - $EXAMPLE_ROOT)
-check_from "$OUT" grep "$EXPECTED_EXAMPLES_TEXT"
+check_from "$OUT" fgrep -q "$EXPECTED_EXAMPLES_TEXT"

 # It should still be there with bad query params.
 OUT=$(wget -O - $EXAMPLE_ROOT?PageSpeedFilters=bogus)
-check_from "$OUT" grep "$EXPECTED_EXAMPLES_TEXT"
+check_from "$OUT" fgrep -q "$EXPECTED_EXAMPLES_TEXT"

 # And also with bad request headers.
 OUT=$(wget -O - --header=PageSpeedFilters:bogus $EXAMPLE_ROOT)
-check_from "$OUT" grep "$EXPECTED_EXAMPLES_TEXT"
+check_from "$OUT" fgrep -q "$EXPECTED_EXAMPLES_TEXT"

 # Tests that an origin header with a Vary header other than Vary:Accept-Encoding
 # loses that header when we are not respecting vary.
 start_test Vary:User-Agent on resources is held by our cache.
 URL="$TEST_ROOT/vary/no_respect/index.html"
-fetch_until -save $URL 'grep -c \.pagespeed\.cf\.' 1
+fetch_until -save $URL 'fgrep -c .pagespeed.cf.' 1

 # Extract out the rewritten CSS file from the HTML saved by fetch_until
 # above (see -save and definition of fetch_until).  Fetch that CSS
@@ -627,14 +679,12 @@ echo "JS_URL=\$\(egrep -o http://.*[.]pagespeed.*[.]js $FETCHED\)=\"$JS_URL\""
 JS_HEADERS=$($WGET -O /dev/null -q -S --header='Accept-Encoding: gzip' \
  $JS_URL 2>&1)
 echo JS_HEADERS=$JS_HEADERS
-check_from "$JS_HEADERS" egrep -qi 'HTTP/1[.]. 200 OK'
+check_200_http_response "$JS_HEADERS"
 check_from "$JS_HEADERS" fgrep -qi 'Content-Encoding: gzip'
 check_from "$JS_HEADERS" fgrep -qi 'Vary: Accept-Encoding'
 check_from "$JS_HEADERS" egrep -qi '(Etag: W/"0")|(Etag: W/"0-gzip")'
 check_from "$JS_HEADERS" fgrep -qi 'Last-Modified:'

-WGET_ARGS="" # Done with test_filter, so clear WGET_ARGS.
-
 start_test Respect X-Forwarded-Proto when told to
 FETCHED=$OUTDIR/x_forwarded_proto
 URL=$SECONDARY_HOSTNAME/mod_pagespeed_example/?PageSpeedFilters=add_base_tag
@@ -647,21 +697,20 @@ check fgrep -q '<base href="https://' $FETCHED
 test_filter remove_comments retains appropriate comments.
 URL="$SECONDARY_HOSTNAME/mod_pagespeed_example/$FILE"
 check run_wget_with_args $URL --header=Host:retaincomment.example.com
-check grep -q retained $FETCHED        # RetainComment directive
+check fgrep -q retained $FETCHED        # RetainComment directive

 # Make sure that when in PreserveURLs mode that we don't rewrite URLs. This is
 # non-exhaustive, the unit tests should cover the rest.
 # Note: We block with psatest here because this is a negative test.  We wouldn't
 # otherwise know how many wget attempts should be made.
+start_test PreserveURLs on prevents URL rewriting
 WGET_ARGS="--header=X-PSA-Blocking-Rewrite:psatest"
 WGET_ARGS+=" --header=Host:preserveurls.example.com"

-start_test PreserveURLs on prevents URL rewriting
 FILE=preserveurls/on/preserveurls.html
 URL=$SECONDARY_HOSTNAME/mod_pagespeed_test/$FILE
 FETCHED=$OUTDIR/preserveurls.html
 check run_wget_with_args $URL
-WGET_ARGS=""
 check_not fgrep -q .pagespeed. $FETCHED

 # When PreserveURLs is off do a quick check to make sure that normal rewriting
@@ -695,10 +744,8 @@ URL="$SECONDARY_HOSTNAME/mod_pagespeed_test/no_transform/BikeCrashIcn.png"
 wget -O - -S $URL $WGET_ARGS &> $FETCHED
 # Make sure that the no-transfrom header is still there
 check grep -q 'Cache-Control:.*no-transform' $FETCHED
-WGET_ARGS=""

 start_test respect vary user-agent
-WGET_ARGS=""
 URL="$SECONDARY_HOSTNAME/mod_pagespeed_test/vary/index.html"
 URL+="?PageSpeedFilters=inline_css"
 FETCH_CMD="$WGET_DUMP --header=Host:respectvary.example.com $URL"
@@ -727,9 +774,8 @@ start_test rewrite on Cache-control: no-transform
 URL=$TEST_ROOT/disable_no_transform/index.html?PageSpeedFilters=inline_css
 fetch_until -save -recursive $URL 'grep -c style' 2

-WGET_ARGS=""
 start_test ShardDomain directive in location block
-fetch_until -save $TEST_ROOT/shard/shard.html 'grep -c \.pagespeed\.' 4
+fetch_until -save $TEST_ROOT/shard/shard.html 'fgrep -c .pagespeed.ce' 4
 check [ $(grep -ce href=\"http://shard1 $FETCH_FILE) = 2 ];
 check [ $(grep -ce href=\"http://shard2 $FETCH_FILE) = 2 ];

@@ -807,7 +853,6 @@ start_test combine_javascript with long URL still works
 URL=$TEST_ROOT/combine_js_very_many.html?PageSpeedFilters=combine_javascript
 fetch_until $URL 'grep -c src=' 4

-WGET_ARGS=""
 start_test UseExperimentalJsMinifier
 URL="$TEST_ROOT/experimental_js_minifier/index.html"
 URL+="?PageSpeedFilters=rewrite_javascript"
@@ -874,10 +919,9 @@ PageSpeedFilters=inline_javascript,debug"
 OUTFILE=$OUTDIR/blocking_rewrite.out.html
 $WGET_DUMP --header 'X-PSA-Blocking-Rewrite: psatest' $URL > $OUTFILE
 check egrep -q 'script[[:space:]]src=' $OUTFILE
-EXPECTED_COMMENT_LINE="<!--InlineJs: Cannot create resource: either its \
-domain is unauthorized and InlineUnauthorizedResources is not enabled, \
-or it cannot be fetched (check the server logs)-->"
-check grep -q "$EXPECTED_COMMENT_LINE" $OUTFILE
+EXPECTED_COMMENT_LINE="<!--The preceding resource was not rewritten \
+because its domain (www.gstatic.com) is not authorized-->"
+check [ $(grep -o "$EXPECTED_COMMENT_LINE" $OUTFILE | wc -l) -eq 1 ]

 start_test inline_unauthorized_resources allows inlining
 HOST_NAME="http://unauthorizedresources.example.com"
@@ -901,10 +945,9 @@ PageSpeedFilters=inline_css,debug"
 OUTFILE=$OUTDIR/blocking_rewrite.out.html
 $WGET_DUMP --header 'X-PSA-Blocking-Rewrite: psatest' $URL > $OUTFILE
 check egrep -q 'link[[:space:]]rel=' $OUTFILE
-EXPECTED_COMMENT_LINE="<!--InlineCss: Cannot create resource: either its \
-domain is unauthorized and InlineUnauthorizedResources is not enabled, \
-or it cannot be fetched (check the server logs)-->"
-check grep -q "$EXPECTED_COMMENT_LINE" $OUTFILE
+EXPECTED_COMMENT_LINE="<!--The preceding resource was not rewritten \
+because its domain (www.google.com) is not authorized-->"
+check [ $(grep -o "$EXPECTED_COMMENT_LINE" $OUTFILE | wc -l) -eq 1 ]

 start_test inline_unauthorized_resources allows inlining
 HOST_NAME="http://unauthorizedresources.example.com"
@@ -1002,7 +1045,6 @@ wget -O - --header 'X-PSA-Blocking-Rewrite: psatest' $URL > $TEMPDIR/flush.$$
 check [ `wget -O - $URL | grep -o 'link rel="subresource"' | wc -l` = 0 ]
 rm -f $TEMPDIR/flush.$$

-WGET_ARGS=""
 start_test Respect custom options on resources.
 IMG_NON_CUSTOM="$EXAMPLE_ROOT/images/xPuzzle.jpg.pagespeed.ic.fakehash.jpg"
 IMG_CUSTOM="$TEST_ROOT/custom_options/xPuzzle.jpg.pagespeed.ic.fakehash.jpg"
@@ -1055,18 +1097,26 @@ WGET_ARGS="--save-headers"
 # We should be able to fetch the original ...
 echo  http_proxy=$SECONDARY_HOSTNAME $WGET --save-headers -O - $ORIGINAL
 OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET --save-headers -O - $ORIGINAL 2>&1)
-check_from "$OUT" fgrep " 200 OK"
+check_200_http_response "$OUT"
 # ... AND the rewritten version.
 echo  http_proxy=$SECONDARY_HOSTNAME $WGET --save-headers -O - $FILTERED
 OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET --save-headers -O - $FILTERED 2>&1)
-check_from "$OUT" fgrep " 200 OK"
+check_200_http_response "$OUT"

 start_test MapProxyDomain
 # depends on MapProxyDomain in pagespeed_test.conf.template
-URL=$EXAMPLE_ROOT/proxy_external_resource.html
+LEAF="proxy_external_resource.html?PageSpeedFilters=-inline_images"
+URL="$EXAMPLE_ROOT/$LEAF"
 echo Rewrite HTML with reference to a proxyable image.
-fetch_until -save -recursive $URL?PageSpeedFilters=-inline_images \
-    'grep -c 1.gif.pagespeed' 1
+fetch_until -save -recursive $URL 'grep -c 1.gif.pagespeed' 1 --save-headers
+PAGESPEED_GIF=$(grep -o '/*1.gif.pagespeed[^"]*' $WGET_DIR/$LEAF)
+check_from "$PAGESPEED_GIF" grep "gif$"
+
+echo "If the next line fails, look in $WGET_DIR/wget_output.txt and you should"
+echo "see a 404.  This represents a failed attempt to download the proxied gif."
+# TODO(jefftk): debug why this test sometimes fails with the native fetcher.
+# https://github.com/pagespeed/ngx_pagespeed/issues/774
+check test -f "$WGET_DIR$PAGESPEED_GIF"

 start_test OptimizeForBandwidth
 # We use blocking-rewrite tests because we want to make sure we don't
@@ -1106,12 +1156,11 @@ test_optimize_for_bandwidth core_filters/rewrite_css.html \
 #
 # With the proper hash, we'll get a long cache lifetime.
 SECONDARY_HOST="http://mpd.example.com/gstatic_images"
-PROXIED_IMAGE="$SECONDARY_HOST/$(basename $WGET_DIR/*1.gif.pagespeed*)"
-WGET_ARGS="--save-headers"
+PROXIED_IMAGE="$SECONDARY_HOST$PAGESPEED_GIF"

 start_test $PROXIED_IMAGE expecting one year cache.
 http_proxy=$SECONDARY_HOSTNAME fetch_until $PROXIED_IMAGE \
-    "grep -c max-age=31536000" 1
+    "grep -c max-age=31536000" 1 --save-headers

 # With the wrong hash, we'll get a short cache lifetime (and also no output
 # cache hit.
@@ -1119,9 +1168,45 @@ WRONG_HASH="0"
 PROXIED_IMAGE="$SECONDARY_HOST/1.gif.pagespeed.ce.$WRONG_HASH.jpg"
 start_test Fetching $PROXIED_IMAGE expecting short private cache.
 http_proxy=$SECONDARY_HOSTNAME fetch_until $PROXIED_IMAGE \
-    "grep -c max-age=300,private" 1
+    "grep -c max-age=300,private" 1 --save-headers

-WGET_ARGS=""
+start_test ShowCache without URL gets a form, inputs, preloaded UA.
+ADMIN_CACHE=$PRIMARY_SERVER/pagespeed_admin/cache
+OUT=$($WGET_DUMP $ADMIN_CACHE)
+check_from "$OUT" fgrep -q "<form>"
+check_from "$OUT" fgrep -q "<input "
+check_from "$OUT" fgrep -q "Cache-Control: max-age=0, no-cache"
+# Preloaded user_agent value field leading with "Mozilla" set in
+# ../automatic/system_test_helpers.sh to help test a "normal" flow.
+check_from "$OUT" fgrep -q 'name=user_agent value="Mozilla'
+
+start_test ShowCache with bogus URL gives a 404
+check_error_code 8 \
+  wget $PRIMARY_SERVER/pagespeed_cache?url=bogus_format >& /dev/null
+
+start_test ShowCache with valid, present URL, with unique options.
+options="PageSpeedImageInlineMaxBytes=6765"
+fetch_until -save $EXAMPLE_ROOT/rewrite_images.html?$options \
+    'grep -c Puzzle\.jpg\.pagespeed\.ic\.' 1
+URL_TAIL=$(grep Puzzle $FETCH_UNTIL_OUTFILE | cut -d \" -f 2)
+SHOW_CACHE_URL=$EXAMPLE_ROOT/$URL_TAIL
+SHOW_CACHE_QUERY=$ADMIN_CACHE?url=$SHOW_CACHE_URL\&$options
+OUT=$($WGET_DUMP $SHOW_CACHE_QUERY)
+check_from "$OUT" fgrep -q cache_ok:true
+check_from "$OUT" fgrep -q mod_pagespeed_example/images/Puzzle.jpg
+
+function show_cache_after_flush() {
+  start_test ShowCache with same URL and matching options misses after flush
+  OUT=$($WGET_DUMP $SHOW_CACHE_QUERY)
+  check_from "$OUT" fgrep -q cache_ok:false
+}
+
+on_cache_flush show_cache_after_flush
+
+start_test ShowCache with same URL but new options misses.
+options="PageSpeedImageInlineMaxBytes=6766"
+OUT=$($WGET_DUMP $ADMIN_CACHE?url=$SHOW_CACHE_URL\&$options)
+check_from "$OUT" fgrep -q cache_ok:false

 # This is dependent upon having a /ngx_pagespeed_beacon handler.
 test_filter add_instrumentation beacons load.
@@ -1139,38 +1224,17 @@ check_from "$OUT" grep $'^Cache-Control: max-age=0, no-cache\r$'

 start_test server-side includes
 fetch_until -save $TEST_ROOT/ssi/ssi.shtml?PageSpeedFilters=combine_css \
-    'grep -c \.pagespeed\.' 1
+    'fgrep -c .pagespeed.' 1
 check [ $(grep -ce $combine_css_filename $FETCH_FILE) = 1 ];

 start_test Embed image configuration in rewritten image URL.
-
-# The apache test names these virtual hosts as embed_config_*, which is
-# unfortunate as underscores are not allowed in domain names.  Apache doesn't
-# care, and nginx doesn't either, except when you're proxying.  So you can do:
-#
-#    GET /embed_config.html HTTP/1.1
-#    Host: embed_config_html.example.com
-#
-# and it will work fine, but if you do:
-#
-#    GET http://embed_config_html.example.com/embed_config.html HTTP/1.1
-#
-# then nginx will close the connection before you can even give it a Host
-# header.  I've modified this test code to replace embed_config_ with
-# embed-config-, but the html file on disk has the underscore versions.  Let's
-# make a new html file that has the hyphen version:
-cat "$SERVER_ROOT/mod_pagespeed_test/embed_config.html" | \
-  sed s/embed_config_/embed-config-/g > \
-  "$SERVER_ROOT/mod_pagespeed_test/embed-config.html"
-
 # The embedded configuration is placed between the "pagespeed" and "ic", e.g.
 # *xPuzzle.jpg.pagespeed.gp+jp+pj+js+rj+rp+rw+ri+cp+md+iq=73.ic.oFXPiLYMka.jpg
 # We use a regex matching "gp+jp+pj+js+rj+rp+rw+ri+cp+md+iq=73" rather than
 # spelling it out to avoid test regolds when we add image filter IDs.
-WGET_ARGS="--save-headers"
 http_proxy=$SECONDARY_HOSTNAME fetch_until -save -recursive \
-    http://embed-config-html.example.org/embed-config.html \
-    'grep -c \.pagespeed\.' 3
+    http://embed-config-html.example.org/embed_config.html \
+    'fgrep -c .pagespeed.' 3 --save-headers

 # with the default rewriters in vhost embed-config-resources.example.com
 # the image will be >200k.  But by enabling resizing & compression 73
@@ -1239,7 +1303,6 @@ function embed_image_config_post_flush() {
  # places where we flush cache, which requires sleeps since the
  # cache-flush is poll driven.
  start_test Embed image/css configuration decoding with clear cache.
-  WGET_ARGS=""
  echo Looking for $EMBED_CONFIGURATION_IMAGE expecting \
      $EMBED_CONFIGURATION_IMAGE_LENGTH bytes
  http_proxy=$SECONDARY_HOSTNAME fetch_until "$EMBED_CONFIGURATION_IMAGE" \
@@ -1289,10 +1352,10 @@ check touch "$SECONDARY_CACHE/cache.flush"
 check touch "$IPRO_CACHE/cache.flush"
 sleep 1

-CSS_FILE="$SERVER_ROOT/mod_pagespeed_test/update.css"
+CSS_FILE="$SERVER_ROOT/mod_pagespeed_test/cache_flush/update.css"
 echo ".class myclass { color: $COLOR0; }" > "$CSS_FILE"

-URL_PATH="mod_pagespeed_test/cache_flush_test.html"
+URL_PATH="mod_pagespeed_test/cache_flush/cache_flush_test.html"

 URL="$SECONDARY_HOSTNAME/$URL_PATH"
 CACHE_A="--header=Host:cache_a.example.com"
@@ -1430,7 +1493,7 @@ else
  EXPECTED="Serf status 111"
 fi
 for i in {1..100}; do
-  ERRS=$(grep -c "$EXPECTED" $FETCHER_REFUSED_PATH)
+  ERRS=$(grep -c "$EXPECTED" $FETCHER_REFUSED_PATH || true)
  if [ $ERRS -ge 1 ]; then
    break;
  fi;
@@ -1440,7 +1503,7 @@ done;
 echo "."
 # Kill the log monitor silently.
 kill $TAIL_PID
-wait $TAIL_PID 2> /dev/null
+wait $TAIL_PID 2> /dev/null || true
 check [ $ERRS -ge 1 ]

 # TODO(jefftk): when we support ListOutstandingUrlsOnError uncomment the below
@@ -1459,7 +1522,6 @@ check [ $ERRS -ge 1 ]
 #
 # This rewrites the CSS, absolutifying the embedded relative image URL
 # reference based on the the main server host.
-WGET_ARGS=""
 start_test Relative images embedded in a CSS file served from a mapped domain
 DIR="mod_pagespeed_test/map_css_embedded"
 URL="http://www.example.com/$DIR/issue494.html"
@@ -1516,7 +1578,8 @@ OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET -O /dev/null $ALLOWED 2>&1)
 check_from "$OUT" fgrep -q "200 OK"
 # .cf. is forbidden
 FORBIDDEN=$FORBIDDEN_STYLES_ROOT/A.all_styles.css.pagespeed.cf.UH8L-zY4b4.css
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET -O /dev/null $FORBIDDEN 2>&1)
+OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET -O /dev/null $FORBIDDEN \
+  2>&1)
 check_from "$OUT" fgrep -q "404 Not Found"
 # The image will be optimized but NOT resized to the much smaller size,
 # so it will be >200k (optimized) rather than <20k (resized).
@@ -1594,7 +1657,6 @@ function test_forbid_all_disabled() {
  $WGET $WGET_ARGS -q -O $OUTFILE $HEADER $URL
  check     egrep -q '<style>.yellow'        $OUTFILE
  rm -f $OUTFILE
-  WGET_ARGS=""
 }
 start_test ForbidAllDisabledFilters baseline check.
 test_forbid_all_disabled "" ""
@@ -1653,16 +1715,18 @@ HOST_NAME="http://domain-hyperlinks-on.example.com"
 RESPONSE_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
    $HOST_NAME/mod_pagespeed_test/rewrite_domains.html)
 MATCHES=$(echo "$RESPONSE_OUT" | fgrep -c http://dst.example.com)
-check [ $MATCHES -eq 3 ]
+check [ $MATCHES -eq 4 ]

 # Test to make sure dynamically defined url-valued attributes are rewritten by
-# rewrite_domains.  See mod_pagespeed_test/rewrite_domains.html: in addition
-# to having one <img> URL, one <form> URL, and one <a> url it also has one
-# <span src=...> URL, one <hr imgsrc=...> URL, and one <hr src=...> URL, all
-# referencing src.example.com.  The first three should be rewritten because of
-# hardcoded rules, the span.src and hr.imgsrc should be rewritten because of
-# UrlValuedAttribute directives, and the hr.src should be left
-# unmodified.  The rewritten ones should all be rewritten to dst.example.com.
+# rewrite_domains.  See mod_pagespeed_test/rewrite_domains.html: in addition to
+# having one <img> URL, one <form> URL, and one <a> url it also has one <span
+# src=...> URL, one <hr imgsrc=...> URL, one <hr src=...> URL, and one
+# <blockquote cite=...> URL, all referencing src.example.com.  The first three
+# should be rewritten because of hardcoded rules, the span.src and hr.imgsrc
+# should be rewritten because of UrlValuedAttribute directives, the hr.src
+# should be left unmodified, and the blockquote.src should be rewritten as an
+# image because of a UrlValuedAttribute override.  The rewritten ones should all
+# be rewritten to dst.example.com.
 HOST_NAME="http://url-attribute.example.com"
 TEST="$HOST_NAME/mod_pagespeed_test"
 REWRITE_DOMAINS="$TEST/rewrite_domains.html"
@@ -1673,7 +1737,7 @@ start_test Rewrite domains in dynamically defined url-valued attributes.

 RESPONSE_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $REWRITE_DOMAINS)
 MATCHES=$(echo "$RESPONSE_OUT" | fgrep -c http://dst.example.com)
-check [ $MATCHES -eq 5 ]
+check [ $MATCHES -eq 6 ]
 MATCHES=$(echo "$RESPONSE_OUT" | \
    fgrep -c '<hr src=http://src.example.com/hr-image>')
 check [ $MATCHES -eq 1 ]
@@ -1685,9 +1749,9 @@ function count_exact_matches() {
  fgrep -o "$1" | wc -l
 }

-# There are nine resources that should be optimized
+# There are ten resources that should be optimized
 http_proxy=$SECONDARY_HOSTNAME \
-    fetch_until $UVA_EXTEND_CACHE 'count_exact_matches .pagespeed.' 9
+    fetch_until $UVA_EXTEND_CACHE 'count_exact_matches .pagespeed.' 10

 # Make sure <custom d=...> isn't modified at all, but that everything else is
 # recognized as a url and rewritten from ../foo to /foo.  This means that only
@@ -1697,9 +1761,9 @@ http_proxy=$SECONDARY_HOSTNAME \
 http_proxy=$SECONDARY_HOSTNAME \
    fetch_until $UVA_EXTEND_CACHE 'fgrep -c ../mod_pa' 1

-# There are nine images that should be optimized.
+# There are ten images that should be optimized.
 http_proxy=$SECONDARY_HOSTNAME \
-    fetch_until $UVA_EXTEND_CACHE 'count_exact_matches .pagespeed.ic' 9
+    fetch_until $UVA_EXTEND_CACHE 'count_exact_matches .pagespeed.ic' 10

 # Test the experiment framework (Furious).

@@ -1760,11 +1824,11 @@ check_not_from "$OUT" grep 'pagespeed_beacon.*exptid'
 # order they're defined in the config file.
 start_test Resource urls are rewritten to include experiment indexes.
 http_proxy=$SECONDARY_HOSTNAME \
-  WGET_ARGS="--header Cookie:PageSpeedExperiment=7" fetch_until $EXP_EXTEND_CACHE \
-    "fgrep -c .pagespeed.a.ic." 1
+  fetch_until $EXP_EXTEND_CACHE \
+    "fgrep -c .pagespeed.a.ic." 1 --header=Cookie:PageSpeedExperiment=7
 http_proxy=$SECONDARY_HOSTNAME \
-  WGET_ARGS="--header Cookie:PageSpeedExperiment=2" fetch_until $EXP_EXTEND_CACHE \
-    "fgrep -c .pagespeed.b.ic." 1
+  fetch_until $EXP_EXTEND_CACHE \
+    "fgrep -c .pagespeed.b.ic." 1 --header=Cookie:PageSpeedExperiment=2
 OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=7' \
      $EXP_EXTEND_CACHE)
 check_from "$OUT" fgrep ".pagespeed.a.ic."
@@ -1887,6 +1951,51 @@ check [ `grep -c '<style>[.]big{[^}]*}</style>' $FETCH_UNTIL_OUTFILE` = 1 ]
 check [ `grep -c '<style>[.]blue{[^}]*}[.]bold{[^}]*}</style>' \
  $FETCH_UNTIL_OUTFILE` = 1 ]

+# Now repeat the critical_css_filter test on a host that processes post data via
+# temp files to test that ngx_pagespeed specific code path.
+test_filter prioritize_critical_css Able to read POST data from temp file.
+URL="http://beacon-post-temp-file.example.com/mod_pagespeed_example/prioritize_critical_css.html"
+http_proxy=$SECONDARY_HOSTNAME\
+  fetch_until -save $URL 'fgrep -c pagespeed.criticalCssBeaconInit' 1
+check [ $(fgrep -o ".very_large_class_name_" $FETCH_FILE | wc -l) -eq 36 ]
+CALL_PAT=".*criticalCssBeaconInit("
+SKIP_ARG="[^,]*,"
+CAPTURE_ARG="'\([^']*\)'.*"
+BEACON_PATH=$(sed -n "s/${CALL_PAT}${CAPTURE_ARG}/\1/p" $FETCH_FILE)
+ESCAPED_URL=$( \
+  sed -n "s/${CALL_PAT}${SKIP_ARG}${CAPTURE_ARG}/\1/p" $FETCH_FILE)
+OPTIONS_HASH=$( \
+  sed -n "s/${CALL_PAT}${SKIP_ARG}${SKIP_ARG}${CAPTURE_ARG}/\1/p" $FETCH_FILE)
+NONCE=$( \
+  sed -n "s/${CALL_PAT}${SKIP_ARG}${SKIP_ARG}${SKIP_ARG}${CAPTURE_ARG}/\1/p" \
+  $FETCH_FILE)
+BEACON_URL="http://${SECONDARY_HOSTNAME}${BEACON_PATH}?url=${ESCAPED_URL}"
+BEACON_DATA="oh=${OPTIONS_HASH}&n=${NONCE}&cs=.big,.blue,.bold,.foo"
+
+OUT=$(wget -q  --save-headers -O - --no-http-keep-alive \
+      --post-data "$BEACON_DATA" "$BEACON_URL" \
+      --header "Host:beacon-post-temp-file.example.com")
+check_from "$OUT" grep '^HTTP/1.1 204'
+
+# Now make sure we see the correct critical css rules.
+http_proxy=$SECONDARY_HOSTNAME\
+  fetch_until $URL \
+    'grep -c <style>[.]blue{[^}]*}</style>' 1
+http_proxy=$SECONDARY_HOSTNAME\
+  fetch_until $URL \
+    'grep -c <style>[.]big{[^}]*}</style>' 1
+http_proxy=$SECONDARY_HOSTNAME\
+  fetch_until $URL \
+    'grep -c <style>[.]blue{[^}]*}[.]bold{[^}]*}</style>' 1
+http_proxy=$SECONDARY_HOSTNAME\
+  fetch_until -save $URL \
+    'grep -c <style>[.]foo{[^}]*}</style>' 1
+# The last one should also have the other 3, too.
+check [ `grep -c '<style>[.]blue{[^}]*}</style>' $FETCH_UNTIL_OUTFILE` = 1 ]
+check [ `grep -c '<style>[.]big{[^}]*}</style>' $FETCH_UNTIL_OUTFILE` = 1 ]
+check [ `grep -c '<style>[.]blue{[^}]*}[.]bold{[^}]*}</style>' \
+  $FETCH_UNTIL_OUTFILE` = 1 ]
+
 # This test checks that the ClientDomainRewrite directive can turn on.
 start_test ClientDomainRewrite on directive
 HOST_NAME="http://client-domain-rewrite.example.com"
@@ -1896,7 +2005,6 @@ MATCHES=$(echo "$RESPONSE_OUT" | grep -c pagespeed\.clientDomainRewriterInit)
 check [ $MATCHES -eq 1 ]

 # Verify rendered image dimensions test.
-WGET_ARGS=""
 start_test resize_rendered_image_dimensions with critical images beacon
 HOST_NAME="http://renderedimagebeacon.example.com"
 URL="$HOST_NAME/mod_pagespeed_test/image_rewriting/image_resize_using_rendered_dimensions.html"
@@ -1960,41 +2068,38 @@ OUT1=$(http_proxy=$SECONDARY_HOSTNAME \
          $WGET_DUMP --header 'X-PSA-Blocking-Rewrite: psatest' $URL)
 check_not_from "$OUT1" egrep -q 'pagespeed\.criticalCssBeaconInit'
 check_from "$OUT1" grep -q "Cache-Control: private, max-age=3000"
+
 # 2. We get an instrumented page if the correct key is present.
-OUT2=$(http_proxy=$SECONDARY_HOSTNAME \
-          $WGET_DUMP $WGET_ARGS \
-          --header 'X-PSA-Blocking-Rewrite: psatest'\
-          --header="PS-ShouldBeacon: random_rebeaconing_key" $URL)
-check_from "$OUT2" grep -q "Cache-Control: max-age=0, no-cache"
-check_from "$OUT2" egrep -q "pagespeed\.criticalCssBeaconInit"
+http_proxy=$SECONDARY_HOSTNAME \
+  fetch_until -save $URL 'grep -c criticalCssBeaconInit' 2 \
+  "--header=PS-ShouldBeacon:random_rebeaconing_key --save-headers"
+check grep -q "Cache-Control: max-age=0, no-cache" $FETCH_UNTIL_OUTFILE
+
 # 3. We do not get an instrumented page if the wrong key is present.
 WGET_ARGS="--header=\"PS-ShouldBeacon: wrong_rebeaconing_key\""
-OUT3=$(http_proxy=$SECONDARY_HOSTNAME \
+OUT3=$(http_proxy=$SECONDARY_HOSTNAME check_not \
          $WGET_DUMP $WGET_ARGS $URL)
 check_not_from "$OUT3" egrep -q "pagespeed\.criticalCssBeaconInit"
 check_from "$OUT3" grep -q "Cache-Control: private, max-age=3000"

-
 # Verify that we can send a critical image beacon and that lazyload_images
 # does not try to lazyload the critical images.
-WGET_ARGS=""
 start_test lazyload_images,rewrite_images with critical images beacon
 HOST_NAME="http://imagebeacon.example.com"
 URL="$HOST_NAME/mod_pagespeed_test/image_rewriting/rewrite_images.html"
-# There are 3 images on rewrite_images.html. Check that they are all
-# lazyloaded by default.
+# There are 3 images on rewrite_images.html.  Since beaconing is on but we've
+# sent no beacon data, none should be lazy loaded.
+# Run until we see beaconing on the page (should happen on first visit).
 http_proxy=$SECONDARY_HOSTNAME\
-  fetch_until -save -recursive $URL 'fgrep -c pagespeed_lazy_src=' 3
-check [ $(grep -c "^pagespeed\.CriticalImages\.Run" \
-  $WGET_DIR/rewrite_images.html) = 1 ];
+  fetch_until -save $URL \
+  'fgrep -c "pagespeed.CriticalImages.Run"' 1
+check [ $(grep -c "pagespeed_lazy_src=" $FETCH_FILE) = 0 ];
 # We need the options hash and nonce to send a critical image beacon, so extract
 # it from injected beacon JS.
-OPTIONS_HASH=$(awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' \
-               $WGET_DIR/rewrite_images.html)
-NONCE=$(awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' \
-        $WGET_DIR/rewrite_images.html)
-OPTIONS_HASH=$(grep "^pagespeed\.CriticalImages\.Run" \
-  $WGET_DIR/rewrite_images.html | awk -F\' '{print $(NF-3)}')
+OPTIONS_HASH=$(
+  awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' $FETCH_FILE)
+NONCE=$(
+  awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' $FETCH_FILE)
 # Send a beacon response using POST indicating that Puzzle.jpg is a critical
 # image.
 BEACON_URL="$HOST_NAME/ngx_pagespeed_beacon"
@@ -2006,7 +2111,7 @@ OUT=$(env http_proxy=$SECONDARY_HOSTNAME \
  wget -q --save-headers -O - --no-http-keep-alive \
  --post-data "$BEACON_DATA" "$BEACON_URL")
 check_from "$OUT" egrep -q "HTTP/1[.]. 204"
-# Now only 2 of the images should be lazyloaded, Cuppa.png should not be.
+# Now 2 of the images should be lazyloaded, Puzzle.jpg should not be.
 http_proxy=$SECONDARY_HOSTNAME \
  fetch_until -save -recursive $URL 'fgrep -c pagespeed_lazy_src=' 2

@@ -2017,13 +2122,13 @@ http_proxy=$SECONDARY_HOSTNAME \
 # page without blocking.
 URL="$URL?id=4"
 http_proxy=$SECONDARY_HOSTNAME\
-  fetch_until -save -recursive $URL 'fgrep -c pagespeed_lazy_src=' 3
-check [ $(grep -c "^pagespeed\.CriticalImages\.Run" \
-    "$WGET_DIR/rewrite_images.html?id=4") = 1 ];
-OPTIONS_HASH=$(awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' \
-               "$WGET_DIR/rewrite_images.html?id=4")
-NONCE=$(awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' \
-       "$WGET_DIR/rewrite_images.html?id=4")
+  fetch_until -save $URL \
+  'fgrep -c "pagespeed.CriticalImages.Run"' 1
+check [ $(grep -c "pagespeed_lazy_src=" $FETCH_FILE) = 0 ];
+OPTIONS_HASH=$(
+  awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' $FETCH_FILE)
+NONCE=$(
+  awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' $FETCH_FILE)
 BEACON_URL="$HOST_NAME/ngx_pagespeed_beacon"
 BEACON_URL+="?url=http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F"
 BEACON_URL+="image_rewriting%2Frewrite_images.html%3Fid%3D4"
@@ -2057,15 +2162,12 @@ check_not fgrep -q "interesting_color" $FETCH_FILE
 # c) selectors that don't depend on flattening should appear in the selector
 #    list.
 check [ $(fgrep -c "non_flattened_selector" $FETCH_FILE) -eq 1 ]
-EXPECTED_IMPORT_FAILURE_LINE="<!--Flattening failed: Cannot import "
-EXPECTED_IMPORT_FAILURE_LINE+="http://www.google.com/css/maia.css: is it on "
-EXPECTED_IMPORT_FAILURE_LINE+="an unauthorized domain?-->"
-check grep -q "$EXPECTED_IMPORT_FAILURE_LINE" $FETCH_FILE
-EXPECTED_COMMENT_LINE="<!--CriticalCssBeacon: Cannot create resource: either "
-EXPECTED_COMMENT_LINE+="its domain is unauthorized and "
-EXPECTED_COMMENT_LINE+="InlineUnauthorizedResources is not enabled, or it "
-EXPECTED_COMMENT_LINE+="cannot be fetched (check the server logs)-->"
-check grep -q "$EXPECTED_COMMENT_LINE" $FETCH_FILE
+EXPECTED_IMPORT_FAILURE_LINE="<!--Flattening failed: Cannot import \
+http://www.google.com/css/maia.css as it is on an unauthorized domain-->"
+check [ $(grep -o "$EXPECTED_IMPORT_FAILURE_LINE" $FETCH_FILE | wc -l) -eq 1 ]
+EXPECTED_COMMENT_LINE="<!--The preceding resource was not rewritten \
+because its domain (www.google.com) is not authorized-->"
+check [ $(grep -o "$EXPECTED_COMMENT_LINE" $FETCH_FILE | wc -l) -eq 1 ]

 start_test inline_unauthorized_resources allows unauthorized css selectors
 HOST_NAME="http://unauthorizedresources.example.com"
@@ -2114,7 +2216,7 @@ keepalive_test "keepalive-beacon-post.example.com" "/ngx_pagespeed_beacon"\

 start_test keepalive with static resources
 keepalive_test "keepalive-static.example.com"\
-  "/ngx_pagespeed_static/js_defer.0.js" ""
+  "/pagespeed_custom_static/js_defer.0.js" ""

 # Test for MaxCombinedCssBytes. The html used in the test, 'combine_css.html',
 # has 4 CSS files in the following order.
@@ -2141,7 +2243,6 @@ check [ $(grep -c 'styles/big.css\"' $FETCH_UNTIL_OUTFILE) = 1 ]
 # response-headers from the server.  The reponse-headers from Serf never seem to
 # include the Connection header.  So we have to pick a JS file that is not
 # otherwise used after cache is flushed in this block.
-WGET_ARGS=""
 start_test Sane Connection header
 URL="$TEST_ROOT/normal.js"
 fetch_until -save $URL 'grep -c W/\"PSA-aj-' 1 --save-headers
@@ -2149,7 +2250,7 @@ CONNECTION=$(extract_headers $FETCH_UNTIL_OUTFILE | fgrep "Connection:")
 check_not_from "$CONNECTION" fgrep -qi "Keep-Alive, Keep-Alive"
 check_from "$CONNECTION" fgrep -qi "Keep-Alive"

-start_test ngx_pagespeed_static defer js served with correct headers.
+start_test pagespeed_custom_static defer js served with correct headers.
 # First, determine which hash js_defer is served with. We need a correct hash
 # to get it served up with an Etag, which is one of the things we want to test.
 URL="$HOSTNAME/mod_pagespeed_example/defer_javascript.html?PageSpeed=on&PageSpeedFilters=defer_javascript"
@@ -2194,11 +2295,8 @@ function test_ipro_for_browser_webp() {
  IN_ACCEPT="$1"; shift
  IMAGE_TYPE="$1"; shift
  OUT_CONTENT_TYPE="$1"; shift
-  OUT_VARY="${1-}"; shift
-  OUT_CC="${1-}"; shift
-  WGET_ARGS="--save-headers \
-             ${IN_UA:+--user-agent $IN_UA} \
-             ${IN_ACCEPT:+--header=Accept:image/$IN_ACCEPT}"
+  OUT_VARY="${1-}"; shift || true
+  OUT_CC="${1-}"; shift || true
  # Remaining args are the expected headers (Name:Value), photo, or synthetic.
  if [ "$IMAGE_TYPE" = "photo" ]; then
    URL="http://ipro-for-browser.example.com/images/Puzzle.jpg"
@@ -2224,6 +2322,9 @@ function test_ipro_for_browser_webp() {
    TEST_ID+=" Cache-Control:${OUT_CC}."
  fi
  start_test $TEST_ID
+  WGET_ARGS="--save-headers \
+             ${IN_UA:+--user-agent $IN_UA} \
+             ${IN_ACCEPT:+--header=Accept:image/$IN_ACCEPT}"
  http_proxy=$SECONDARY_HOSTNAME \
    fetch_until -save $URL 'grep -c W/\"PSA-aj-' 1
  check_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" \
@@ -2304,12 +2405,53 @@ test_decent_browsers "New Opera" \
  "Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14"

 WGETRC=$OLD_WGETRC
-WGET_ARGS=""

-JS_URL="$HOSTNAME/ngx_pagespeed_static/js_defer.$HASH.js"
+start_test Request Option Override : Correct values are passed
+HOST_NAME="http://request-option-override.example.com"
+OPTS="?ModPagespeed=on"
+OPTS+="&ModPagespeedFilters=+collapse_whitespace,+remove_comments"
+OPTS+="&PageSpeedRequestOptionOverride=abc"
+URL="$HOST_NAME/mod_pagespeed_test/forbidden.html$OPTS"
+OUT="$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)"
+echo wget $URL
+check_not_from "$OUT" grep -q '<!--'
+
+start_test Request Option Override : Incorrect values are passed
+HOST_NAME="http://request-option-override.example.com"
+OPTS="?ModPagespeed=on"
+OPTS+="&ModPagespeedFilters=+collapse_whitespace,+remove_comments"
+OPTS+="&PageSpeedRequestOptionOverride=notabc"
+URL="$HOST_NAME/mod_pagespeed_test/forbidden.html$OPTS"
+OUT="$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)"
+echo wget $URL
+check_from "$OUT" grep -q '<!--'
+
+start_test Request Option Override : Correct values are passed as headers
+HOST_NAME="http://request-option-override.example.com"
+OPTS="--header=ModPagespeed:on"
+OPTS+=" --header=ModPagespeedFilters:+collapse_whitespace,+remove_comments"
+OPTS+=" --header=PageSpeedRequestOptionOverride:abc"
+URL="$HOST_NAME/mod_pagespeed_test/forbidden.html"
+echo wget $OPTS $URL
+OUT="$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $OPTS $URL)"
+check_not_from "$OUT" grep -q '<!--'
+
+start_test Request Option Override : Incorrect values are passed as headers
+HOST_NAME="http://request-option-override.example.com"
+OPTS="--header=ModPagespeed:on"
+OPTS+=" --header=ModPagespeedFilters:+collapse_whitespace,+remove_comments"
+OPTS+=" --header=PageSpeedRequestOptionOverride:notabc"
+URL="$HOST_NAME/mod_pagespeed_test/forbidden.html"
+echo wget $OPTS $URL
+OUT="$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $OPTS $URL)"
+check_from "$OUT" grep -q '<!--'
+
+start_test JS gzip headers
+
+JS_URL="$HOSTNAME/pagespeed_custom_static/js_defer.$HASH.js"
 JS_HEADERS=$($WGET -O /dev/null -q -S --header='Accept-Encoding: gzip' \
  $JS_URL 2>&1)
-check_from "$JS_HEADERS" egrep -qi 'HTTP/1[.]. 200 OK'
+check_200_http_response "$JS_HEADERS"
 check_from "$JS_HEADERS" fgrep -qi 'Content-Encoding: gzip'
 check_from "$JS_HEADERS" fgrep -qi 'Vary: Accept-Encoding'
 # Nginx's gzip module clears etags, which we don't want. Make sure we have it.
@@ -2329,7 +2471,7 @@ start_test IPRO flow uses cache as expected.
 STATS=$OUTDIR/blocking_rewrite_stats
 IPRO_ROOT=http://ipro.example.com/mod_pagespeed_test/ipro
 URL=$IPRO_ROOT/test_image_dont_reuse2.png
-IPRO_STATS_URL=http://ipro.example.com/ngx_pagespeed_statistics?PageSpeed=off
+IPRO_STATS_URL=http://ipro.example.com/ngx_pagespeed_statistics

 # Initial stats.
 http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $IPRO_STATS_URL > $STATS.0
@@ -2497,15 +2639,27 @@ ETAG=$(extract_headers $FETCH_UNTIL_OUTFILE | awk '/ETag:/ {print $2}')
 echo $WGET_DUMP --header "If-None-Match: $ETAG" $URL
 OUTFILE=$OUTDIR/etags
 # Note: -o gets debug info which is the only place that 304 message is sent.
-$WGET -o $OUTFILE -O /dev/null --header "If-None-Match: $ETAG" $URL
+check_not $WGET -o $OUTFILE -O /dev/null --header "If-None-Match: $ETAG" $URL
 check fgrep -q "awaiting response... 304" $OUTFILE

-start_test PageSpeed resources should have a content length.
-URL="$EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css"
-OUT=$($WGET_DUMP --save-headers $URL)
-check_from "$OUT" egrep -q $'^Content-Length: ([0-9])*\r$'
-check_not_from "$OUT" egrep -iq $'^Transfer-Encoding: chunked\r$'
-check_not_from "$OUT" egrep -iq $'^Connection: close\r$'
+# Test if the warning messages are colored in message_history page.
+# We color the messages in message_history page to make it clearer to read.
+# Red for Error messages. Brown for Warning messages.
+# Orange for Fatal messages. Black by default.
+# Won't test Error messages and Fatal messages in this test.
+start_test Messages are colored in message_history
+INJECT=$($CURL --silent $HOSTNAME/?PageSpeed=Warning_trigger)
+OUT=$($WGET -q -O - $HOSTNAME/pagespeed_admin/message_history | \
+  grep Warning_trigger)
+check_from "$OUT" fgrep -q "color:brown;"
+
+start_test Downstream cache integration caching headers.
+URL="http://downstreamcacheresource.example.com/mod_pagespeed_example/images/"
+URL+="xCuppa.png.pagespeed.ic.0.png"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" egrep -iq $'^Cache-Control: .*\r$'
+check_from "$OUT" egrep -iq $'^Expires: .*\r$'
+check_from "$OUT" egrep -iq $'^Last-Modified: .*\r$'

 # Test handling of large HTML files. We first test with a cold cache, and verify
 # that we bail out of parsing and insert a script redirecting to
@@ -2521,13 +2675,13 @@ WGET_ARGS="--header=PageSpeedFilters:rewrite_images"
 WGET_EC="$WGET_DUMP $WGET_ARGS"
 echo $WGET_EC $URL
 LARGE_OUT=$($WGET_EC $URL)
-check_from "$LARGE_OUT" grep -q window.location=".*&ModPagespeed=off"
+check_from "$LARGE_OUT" grep -q window.location=".*&PageSpeed=off"

 # The file should now be in the property cache so make sure that the page is no
 # longer parsed. Use fetch_until because we need to wait for a potentially
 # non-blocking write to the property cache from the previous test to finish
 # before this will succeed.
-fetch_until -save $URL 'grep -c window.location=".*&ModPagespeed=off"' 0
+fetch_until -save $URL 'grep -c window.location=".*&PageSpeed=off"' 0
 check_not fgrep -q pagespeed.ic $FETCH_FILE

 start_test messages load
@@ -2537,9 +2691,8 @@ check_from "$OUT" grep -q "/mod_pagespeed_example"

 start_test Check keepalive after a 304 responses.
 # '-m 2' specifies that the whole operation is allowed to take 2 seconds max.
-curl -vv -m 2 http://$PRIMARY_HOSTNAME/foo.css.pagespeed.ce.0.css \
+check curl -vv -m 2 http://$PRIMARY_HOSTNAME/foo.css.pagespeed.ce.0.css \
    -H 'If-Modified-Since: Z' http://$PRIMARY_HOSTNAME/foo
-check [ $? = "0" ]

 start_test Date response header set
 OUT=$($WGET_DUMP $EXAMPLE_ROOT/combine_css.html)
@@ -2548,6 +2701,52 @@ check_not_from "$OUT" egrep -q '^Date: Thu, 01 Jan 1970 00:00:00 GMT'
 OUT=$($WGET_DUMP --header=Host:date.example.com \
    http://$SECONDARY_HOSTNAME/mod_pagespeed_example/combine_css.html)
 check_from "$OUT" egrep -q '^Date: Fri, 16 Oct 2009 23:05:07 GMT'
+WGET_ARGS=
+
+#very basic tests to test gzip nesting configuration
+start_test Nested gzip gzip off
+URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/"
+HEADERS="--header=Accept-Encoding:gzip --header=Host:gzip-test1.example.com"
+OUT=$($WGET_DUMP -O /dev/null -S $HEADERS $URL 2>&1)
+check_not_from "$OUT" fgrep -qi 'Content-Encoding: gzip'
+check_not_from "$OUT" fgrep -qi 'Vary: Accept-Encoding'
+
+start_test Nested gzip gzip on
+URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/styles/big.css"
+HEADERS="--header=Accept-Encoding:gzip --header=Host:gzip-test1.example.com"
+OUT=$($WGET_DUMP -O /dev/null -S $HEADERS $URL 2>&1)
+check_from "$OUT" fgrep -qi 'Content-Encoding: gzip'
+check_from "$OUT" fgrep -qi 'Vary: Accept-Encoding'
+
+start_test Nested gzip pagespeed off
+URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/"
+HEADERS="--header=Accept-Encoding:gzip --header=Host:gzip-test2.example.com"
+OUT=$($WGET_DUMP -O /dev/null -S $HEADERS $URL 2>&1)
+check_not_from "$OUT" fgrep -qi 'Content-Encoding: gzip'
+check_not_from "$OUT" fgrep -qi 'Vary: Accept-Encoding'
+
+start_test Nested gzip pagespeed on
+URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/styles/big.css"
+HEADERS="--header=Accept-Encoding:gzip --header=Host:gzip-test2.example.com"
+OUT=$($WGET_DUMP -O /dev/null -S $HEADERS $URL 2>&1)
+check_from "$OUT" fgrep -qi 'Content-Encoding: gzip'
+check_from "$OUT" fgrep -qi 'Vary: Accept-Encoding'
+
+start_test Test that POST requests are rewritten.
+URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/rewrite_images.html"
+HEADERS="--header=Host:proxy-post.example.com --post-data=abcdefgh"
+OUT=$($WGET_DUMP -S $HEADERS $URL 2>&1)
+check_from "$OUT" fgrep -qi 'addInstrumentationInit'
+
+if [ "$NATIVE_FETCHER" != "on" ]; then
+  start_test Test that we can rewrite an HTTPS resource.
+  fetch_until $TEST_ROOT/https_fetch/https_fetch.html \
+    'grep -c /https_gstatic_dot_com/1.gif.pagespeed.ce' 1
+fi
+
+start_test Base config has purging disabled.  Check error message syntax.
+OUT=$($WGET_DUMP "$HOSTNAME/pagespeed_admin/cache?purge=*")
+check_from "$OUT" fgrep -q "pagespeed EnableCachePurge on;"

 if $USE_VALGRIND; then
    # It is possible that there are still ProxyFetches outstanding
@@ -27,6 +27,18 @@ http {
  proxy_cache_path "@@PROXY_CACHE@@"  levels=1:2   keys_zone=htmlcache:60m inactive=90m  max_size=50m;
  proxy_temp_path "@@TMP_PROXY_CACHE@@";

+  pagespeed ProcessScriptVariables on;
+  pagespeed StatisticsPath /ngx_pagespeed_statistics;
+  pagespeed GlobalStatisticsPath /ngx_pagespeed_global_statistics;
+  pagespeed ConsolePath /pagespeed_console;
+  pagespeed MessagesPath /ngx_pagespeed_message;
+  pagespeed AdminPath /pagespeed_admin;
+  pagespeed GlobalAdminPath /pagespeed_global_admin;
+
+  pagespeed StaticAssetPrefix /pagespeed_custom_static/;
+
+  pagespeed MessageBufferSize 200000;
+
  root "@@SERVER_ROOT@@";

  # Block 5a: Decide on Cache-Control header value to use for outgoing
@@ -42,7 +54,6 @@ http {
  pagespeed UsePerVHostStatistics on;
  pagespeed InPlaceResourceOptimization on;
  pagespeed CreateSharedMemoryMetadataCache "@@SHM_CACHE@@" 8192;
-  pagespeed PreserveUrlRelativity on;
  pagespeed BlockingRewriteKey psatest;

  # CriticalImagesBeaconEnabled is now on by default, but we disable in testing.
@@ -67,6 +78,7 @@ http {
    # test ModPagespeedMaxCacheableContentLength, i.e.,
    # max_cacheable_response_content_length.
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name max-cacheable-content-length.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -86,6 +98,7 @@ http {
    # running on the PRIMARY_PORT when the response is not available in
    # the cache. It also services purge requests from the upstream server.
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name proxy_cache.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -144,7 +157,15 @@ http {
      set $bypass_cache "1";
    }

-    set_random $rand 0 100;
+    # For testing purposes, we never generate values that will result in a beacon
+    # unless a test request it via "X-Allow-Beacon: yes" in its request header.
+    # This is for testing purposes, note that in a production environment,
+    # you want 'set_random $rand 0 100;' unconditionally!
+    set $rand 5;
+
+    if ($http_x_allow_beacon ~ "yes") {
+      set_random $rand 0 100;
+    }
    set $should_beacon_header_val "";
    if ($rand ~* "^[0-4]$") {
      set $should_beacon_header_val "random_rebeaconing_key";
@@ -160,6 +181,7 @@ http {
    # Block 4: Location block for purge requests.
    location ~ /purge(/.*) {
      allow 127.0.0.1;
+      allow ::1;
      deny all;
      proxy_cache_purge htmlcache $ps_capability_list$1$is_args$args;
    }
@@ -198,6 +220,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name if-in-server.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@";

@@ -218,6 +241,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name if-in-location.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@";

@@ -237,6 +261,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name mpd.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@";

@@ -247,6 +272,7 @@ http {
  # These two vhosts are for testing the experiment framework (Furious).
  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name experiment.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";
    pagespeed InPlaceResourceOptimization off;
@@ -261,6 +287,7 @@ http {
  }
  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name experiment.noga.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -274,6 +301,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name preserveurls.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -303,6 +331,7 @@ http {
    # Setup a vhost with the critical image beacon and lazyload filter enabled
    # to make sure that critical images are not lazyloaded.
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name imagebeacon.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -315,6 +344,7 @@ http {
    # Setup a vhost with the critical image beacon enabled to make sure that
    # downstream caches and rebeaconing interact correctly.
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name downstreamcacherebeacon.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -330,6 +360,17 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name downstreamcacheresource.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_images;
+    pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name renderedimagebeacon.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -347,6 +388,7 @@ http {
  # inheretence in Nginx we need to have location blocks inside a server block.
  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name optimizeforbandwidth.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -378,6 +420,7 @@ http {
    # to go back to this VirtualHost so we rely on the new third optional
    # argument to MapOriginDomain.
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name customhostheader.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@_test";
    root "@@SERVER_ROOT@@/mod_pagespeed_test";
@@ -396,6 +439,7 @@ http {
    # Sets up a virtual host where we can specify forbidden filters without
    # affecting any other hosts.
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name forbidden.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -411,6 +455,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name unauthorizedresources.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -421,6 +466,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name client-domain-rewrite.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -436,6 +482,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name url-attribute.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -456,6 +503,9 @@ http {
    pagespeed UrlValuedAttribute img alt-src Image;
    pagespeed UrlValuedAttribute video alt-a Image;
    pagespeed UrlValuedAttribute video alt-b Image;
+
+    # Also test that we can redefine spec-defined attributes.
+    pagespeed UrlValuedAttribute blockquote cite Image;
  }

  server {
@@ -466,6 +516,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name domain-hyperlinks-on.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -480,6 +531,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name domain-hyperlinks-off.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -494,6 +546,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name retaincomment.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -503,6 +556,7 @@ http {
  server {
    # Test host for shared memory cache.
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name shmcache.example.com;
    pagespeed FileCachePath "@@SHM_CACHE@@";

@@ -512,6 +566,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name xfp.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -520,6 +575,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name xheader.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -533,6 +589,7 @@ http {
  # Note that we test with two distinct caches.
  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name embed-config-html.example.org;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -552,6 +609,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name embed-config-resources.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -572,6 +630,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name ipro-for-browser.example.com;
    root "@@SERVER_ROOT@@/mod_pagespeed_example";
    pagespeed EnableFilters rewrite_images,rewrite_css;
@@ -583,6 +642,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name respectvary.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -592,16 +652,30 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name cache_a.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

    pagespeed CacheFlushPollIntervalSec 1;
    pagespeed RewriteLevel PassThrough;
    pagespeed EnableFilters inline_css;
+
+    # Make a non-empty subdirectory config to make sure that
+    # cache.flush updates get transmitted to nested configurations.
+    #
+    # TODO(jmarantz): This test currently fails on ngx_pagespeed, meaning we
+    # don't handle cache flushing correctly here.  Uncomment the config below to
+    # expose the error.
+    #location /mod_pagespeed_test/cache_flush/ {
+    #  pagespeed RewriteLevel PassThrough;
+    #  pagespeed EnableFilters inline_css;
+    #  pagespeed DisableFilters add_instrumentation;
+    #}
  }

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name cache_b.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@";

@@ -612,6 +686,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name cache_c.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -625,6 +700,7 @@ http {
  server {
    # Sets up a logical home-page server on www.example.com
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name www.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -638,6 +714,7 @@ http {
    # Sets up a logical origin for CDNs to fetch content from, on
    # origin.example.com.
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name origin.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -651,6 +728,7 @@ http {
    # Sets up a logical cdn, which is where we tell browsers to fetch resources
    # from.
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name cdn.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -662,6 +740,18 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name lff-ipro.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed LoadFromFile
+      "http://lff-ipro.example.com/mod_pagespeed_example/lff_ipro"
+      "@@SERVER_ROOT@@/mod_pagespeed_example/lff_ipro";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name tryfiles.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -675,6 +765,7 @@ http {
  server {
    # Proxy modpagespeed.com for testing Issue 582.
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name modpagespeed.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -687,6 +778,26 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name custom-paths.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed StatisticsPath /custom_pagespeed_statistics;
+    pagespeed ConsolePath /custom_pagespeed_console;
+    pagespeed MessagesPath /custom_pagespeed_message;
+    pagespeed AdminPath /custom_pagespeed_admin;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name inherit-paths.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name notransform.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

@@ -698,6 +809,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name blocking.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@";

@@ -707,6 +819,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name keepalive-html.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";
    pagespeed RewriteLevel CoreFilters;
@@ -715,6 +828,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name keepalive-resource.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";
    error_log "@@TEST_TMP@@/keepalive-resource.example.com.error.log" warn;
@@ -722,6 +836,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name keepalive-beacon-get.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";
    error_log "@@TEST_TMP@@/keepalive-beacon-get.example.com.error.log" warn;
@@ -729,6 +844,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name keepalive-beacon-post.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";
    error_log "@@TEST_TMP@@/keepalive-beacon-post.example.com.error.log" warn;
@@ -736,6 +852,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name keepalive-static.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";
    error_log "@@TEST_TMP@@/keepalive-static.example.com.error.log" warn;
@@ -743,6 +860,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name response-header-filters.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
    pagespeed RewriteLevel PassThrough;
@@ -752,6 +870,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name response-header-disable.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
    pagespeed EnableFilters add_instrumentation;
@@ -761,6 +880,7 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name ipro.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@";

@@ -784,6 +904,7 @@ http {
  # 1. L2_d=LRU, L2_m=LRU
  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name lrud-lrum.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@_lrud_lrum";

@@ -798,6 +919,7 @@ http {
            "@@SECONDARY_CACHE@@_lrud_shmm" 8192;
  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name lrud-shmm.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@_lrud_shmm";

@@ -812,6 +934,7 @@ http {
            "@@SECONDARY_CACHE@@_noned_shmm" 8192;
  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name noned-shmm.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@_noned_shmm";

@@ -822,6 +945,7 @@ http {
  # 4. L2_d=none, L2_m=none
  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name noned-nonem.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@_noned_nonem";

@@ -831,14 +955,95 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name date.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";
    add_header "Date" "Date: Fri, 16 Oct 2009 23:05:07 GMT";
  }

+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name options-by-cookies-enabled.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@_optionsbycookieson";
+    pagespeed AllowOptionsToBeSetByCookies true;
+    pagespeed StickyQueryParameters sticky_secret;
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters collapse_whitespace;
+    pagespeed DisableFilters remove_comments,add_instrumentation;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name options-by-cookies-disabled.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@_optionsbycookiesoff";
+    pagespeed AllowOptionsToBeSetByCookies false;
+    pagespeed DisableFilters add_instrumentation;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name redirect.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@_redirected";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters add_instrumentation,collapse_whitespace;
+    location /redirect {
+      rewrite ^/redirect/(.*) http://$server_name/$1 permanent;
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name request-option-override.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RequestOptionOverride abc;
+    pagespeed RewriteLevel Passthrough;
+    pagespeed EnableFilters collapse_whitespace;
+    pagespeed DisableFilters remove_comments,add_instrumentation;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name signed-urls.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed UrlSigningKey helloworld;
+    pagespeed RewriteLevel Passthrough;
+    pagespeed EnableFilters collapse_whitespace,rewrite_images;
+    pagespeed DisableFilters remove_comments,add_instrumentation;
+  }
+
+  # For testing signed URLs, ignoring signature validity.
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name signed-urls-transition.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed UrlSigningKey helloworld;
+    pagespeed AcceptInvalidSignatures true;
+    pagespeed RewriteLevel Passthrough;
+    pagespeed EnableFilters collapse_whitespace,rewrite_images;
+    pagespeed DisableFilters remove_comments,add_instrumentation;
+  }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name unsigned-urls-transition.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    # This server will not sign URLs, but AcceptInvalidSignature is on.
+    pagespeed AcceptInvalidSignatures true;
+    pagespeed RewriteLevel Passthrough;
+    pagespeed EnableFilters collapse_whitespace,rewrite_images;
+    pagespeed DisableFilters remove_comments,add_instrumentation;
+  }
+
  # Proxy + IPRO a gzip'd file for testing Issue 896.
  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name ipro-proxy.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@_ipro_proxy";

@@ -854,16 +1059,107 @@ http {

  server {
    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
    server_name compressed-css.example.com;
+    pagespeed on;
    pagespeed FileCachePath "@@FILE_CACHE@@";
    pagespeed InPlaceResourceOptimization on;
  }

+  # nested gzip config: pagespeed gzip on/off
  server {
-    listen       @@PRIMARY_PORT@@;
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name gzip-test1.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    location /mod_pagespeed_example/ {
+      pagespeed gzip off;
+    }
+    location /mod_pagespeed_example/styles/ {
+      pagespeed gzip on;
+    }
+  }
+
+  # nested gzip config: pagespeed on/off
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name gzip-test2.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    location /mod_pagespeed_example/ {
+      pagespeed off;
+    }
+    location /mod_pagespeed_example/styles/ {
+      pagespeed on;
+    }
+  }
+
+  server {
+    pagespeed on;
+    pagespeed EnableFilters add_instrumentation;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name proxy-post.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    location / {
+      # Transform the POST to a GET request for the origin,
+      # as nginx's static handler doesn't allow POST requests.
+      proxy_method GET;
+      proxy_pass http://127.0.0.1:@@SECONDARY_PORT@@/;
+      proxy_set_header "Host" "proxy-post-origin.example.com";
+    }
+  }
+
+  server {
+    pagespeed off;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name proxy-post-origin.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    root "@@SERVER_ROOT@@";
+  }
+
+  server {
+    # Write all post data to temp files
+    client_body_in_file_only clean;
+    pagespeed CriticalImagesBeaconEnabled true;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name beacon-post-temp-file.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name purge.example.com;
+
+    # Test purging individual URLs without flushing the entire metadata cache.
+    pagespeed EnableCachePurge on;
+
+    pagespeed PurgeMethod PURGE;
+    root "@@SERVER_ROOT@@";
+    pagespeed FileCachePath "@@FILE_CACHE@@_purge";
+    pagespeed DisableFilters add_instrumentation;
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_css;
+  }
+
+  server {
+    listen @@PRIMARY_PORT@@;
+    listen [::]:@@PRIMARY_PORT@@;
    server_name  localhost;
    pagespeed FileCachePath "@@FILE_CACHE@@";

+    location ~ ^/pagespeed_admin {
+      allow 127.0.0.1;
+      allow ::1;
+      deny all;
+    }
+
    location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" {
      add_header "" "";
    }
@@ -901,7 +1197,6 @@ http {
    #pagespeed MemcachedThreads 1;

    pagespeed on;
-    pagespeed MessageBufferSize 200000;

    #pagespeed CacheFlushPollIntervalSec 1;

@@ -922,11 +1217,13 @@ http {

    location /ngx_pagespeed_statistics {
      allow 127.0.0.1;
+      allow ::1;
      deny all;
    }

    location /ngx_pagespeed_message {
      allow 127.0.0.1;
+      allow ::1;
      deny all;
    }

@@ -1091,8 +1388,19 @@ http {
      pagespeed UseExperimentalJsMinifier on;
    }

+    pagespeed FetchHttps enable;
+    location /mod_pagespeed_test/https_fetch/ {
+      pagespeed DisableFilters inline_images;
+      pagespeed MapProxyDomain
+          http://localhost:@@PRIMARY_PORT@@/https_gstatic_dot_com
+          https://www.gstatic.com/psa/static;
+    }
+
+    # $host implicitly tests script variable support. I'd love to test it more
+    # directly, but so far this is the best I've come up with and duplicating
+    # the test doesn't seem to make sense.
    pagespeed LoadFromFile
-        "http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/instant/"
+        "http://$host:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/instant/"
        "@@SERVER_ROOT@@/mod_pagespeed_test/ipro/instant/";

    pagespeed EnableFilters remove_comments;
@@ -1106,8 +1414,8 @@ http {
      "@@SERVER_ROOT@@/mod_pagespeed_test/load_from_file/file_\1/";
    pagespeed LoadFromFileRule Disallow
      "@@SERVER_ROOT@@/mod_pagespeed_test/load_from_file/file_dir/httponly/";
-    pagespeed LoadFromFileRuleMatch Disallow \.ssp.css$;
-    pagespeed LoadFromFileRuleMatch Allow exception\.ssp\.css$;
+    pagespeed LoadFromFileRuleMatch Disallow \.ssp.css$ps_dollar;
+    pagespeed LoadFromFileRuleMatch Allow exception\.ssp\.css$ps_dollar;

    #charset koi8-r;

@@ -1130,27 +1438,6 @@ http {

  keepalive_timeout  65;

-  # set up gzip
-  gzip  on;
-  gzip_vary on;
-  gzip_proxied any;
-  # Turn on gzip for all content types that should benefit from it.
-  gzip_types application/ecmascript;
-  gzip_types application/javascript;
-  gzip_types application/json;
-  gzip_types application/pdf;
-  gzip_types application/postscript;
-  gzip_types application/x-javascript;
-  gzip_types image/svg+xml;
-  gzip_types text/css;
-  gzip_types text/csv;
-  # "gzip_types text/html" is assumed.
-  gzip_types text/javascript;
-  gzip_types text/plain;
-  gzip_types text/xml;
-
-  gzip_http_version 1.0;
-
  types {
    text/html                             html htm shtml;
    text/css                              css;
@@ -73,6 +73,9 @@ function run_test_checking_failure() {
    3)
      return  # Only expected failures.
      ;;
+    4)
+      return  # Return passing error code when running manually.
+      ;;
    *)
      exit 1  # Real failure.
  esac