Factor out handling for errors, stats, and logs. Add nosniff protection.

keepalive-test: prevent false positives, filter a little more

README.md

@@ -37,11 +37,11 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p
 
    ```bash
    $ cd ~
-   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/release-1.5.27.3-beta.zip
-   $ unzip release-1.5.27.3-beta.zip # or unzip release-1.5.27.3-beta
-   $ cd ngx_pagespeed-release-1.5.27.3-beta/
-   $ wget https://dl.google.com/dl/page-speed/psol/1.5.27.3.tar.gz
-   $ tar -xzvf 1.5.27.3.tar.gz # expands to psol/
+   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/release-1.6.29.7-beta.zip
+   $ unzip release-1.6.29.7-beta.zip # or unzip release-1.6.29.7-beta
+   $ cd ngx_pagespeed-release-1.6.29.7-beta/
+   $ wget https://dl.google.com/dl/page-speed/psol/1.6.29.7.tar.gz
+   $ tar -xzvf 1.6.29.7.tar.gz # expands to psol/
    ```
 
 3. Download and build nginx:
@@ -51,7 +51,7 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p
    $ wget http://nginx.org/download/nginx-1.4.1.tar.gz
    $ tar -xvzf nginx-1.4.1.tar.gz
    $ cd nginx-1.4.1/
-   $ ./configure --add-module=$HOME/ngx_pagespeed-release-1.5.27.3-beta
+   $ ./configure --add-module=$HOME/ngx_pagespeed-release-1.6.29.7-beta
    $ make
    $ sudo make install
    ```
@@ -87,6 +87,7 @@ location ~ "^/ngx_pagespeed_static/" { }
 location ~ "^/ngx_pagespeed_beacon$" { }
 location /ngx_pagespeed_statistics { allow 127.0.0.1; deny all; }
 location /ngx_pagespeed_message { allow 127.0.0.1; deny all; }
+location /pagespeed_console { allow 127.0.0.1; deny all; }
 ```
 
 To confirm that the module is loaded, fetch a page and check that you see the
@@ -94,33 +95,14 @@ To confirm that the module is loaded, fetch a page and check that you see the
 
 ```bash
 $ curl -I 'http://localhost:8050/some_page/' | grep X-Page-Speed
-X-Page-Speed: 1.5.27.3-...
+X-Page-Speed: 1.6.29.7-...
 ```
 
 Looking at the source of a few pages you should see various changes, such as
 urls being replaced with new ones like `yellow.css.pagespeed.ce.lzJ8VcVi1l.css`.
 
-When reading the [mod_pagespeed
-documentation](https://developers.google.com/speed/docs/mod_pagespeed/using_mod),
-keep in mind that you need to make a small adjustment to configuration
-directives: replace **ModPagespeed** with **pagespeed**:
-
-    mod_pagespeed.conf:
-      ModPagespeedEnableFilters collapse_whitespace,add_instrumentation
-      ModPagespeedRunExperiment on
-      ModPagespeedExperimentSpec id=3;percent=50;default
-      ModPagespeedExperimentSpec id=4;percent=50
-
-    ngx_pagespeed.conf:
-      pagespeed EnableFilters collapse_whitespace,add_instrumentation;
-      pagespeed RunExperiment on;
-      pagespeed ExperimentSpec "id=3;percent=50;default";
-      pagespeed ExperimentSpec "id=4;percent=50";
-
-For more configuration details, see the [differences from mod_pagespeed
-configuration](https://github.com/pagespeed/ngx_pagespeed/wiki/Configuration-differences-from-mod_pagespeed)
-and <a href="https://github.com/pagespeed/ngx_pagespeed/wiki/Known-Issues">known
-issues</a> wiki pages.
+For complete documentation, see [Using
+PageSpeed](https://developers.google.com/speed/pagespeed/module/using).
 
 There are extensive system tests which cover most of ngx_pagespeed's
 functionality.  Consider [testing your

config

@@ -27,8 +27,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
     echo "   You need to separately download the pagespeed library:"
     echo ""
     echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.5.27.3.tar.gz"
-    echo "     $ tar -xzvf 1.5.27.3.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.6.29.7.tar.gz"
+    echo "     $ tar -xzvf 1.6.29.7.tar.gz # expands to psol/"
     echo ""
     echo "   Or see the installation instructions:"
     echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
@@ -83,6 +83,10 @@ else
   buildtype=Release
 fi
 
+# Building with HTTPS fetching enabled pulls in a version of OpenSSL that causes
+# linker errors, so disable it here.
+CFLAGS="$CFLAGS -DSERF_HTTPS_FETCHING=0"
+
 pagespeed_include="\
   $mod_pagespeed_dir \
   $mod_pagespeed_dir/third_party/chromium/src \
@@ -139,34 +143,40 @@ if [ $ngx_found = yes ]; then
   ps_src="$ngx_addon_dir/src"
   ngx_addon_name=ngx_pagespeed
   NGX_ADDON_DEPS="$NGX_ADDON_DEPS \
-    $ps_src/ngx_pagespeed.h \
-    $ps_src/ngx_fetch.h \
-    $ps_src/ngx_url_async_fetcher.h \
+    $ps_src/log_message_handler.h \
     $ps_src/ngx_base_fetch.h \
-    $ps_src/ngx_server_context.h \
-    $ps_src/ngx_rewrite_options.h \
-    $ps_src/ngx_rewrite_driver_factory.h \
-    $ps_src/ngx_thread_system.h \
+    $ps_src/ngx_caching_headers.h \
+    $ps_src/ngx_fetch.h \
+    $ps_src/ngx_list_iterator.h \
     $ps_src/ngx_message_handler.h \
-    $ps_src/pthread_shared_mem.h \
+    $ps_src/ngx_pagespeed.h \
     $ps_src/ngx_request_context.h \
-    $ps_src/log_message_handler.h"
+    $ps_src/ngx_rewrite_driver_factory.h \
+    $ps_src/ngx_rewrite_options.h \
+    $ps_src/ngx_server_context.h \
+    $ps_src/ngx_thread_system.h \
+    $ps_src/ngx_url_async_fetcher.h \
+    $ps_src/pthread_shared_mem.h"
   NGX_ADDON_SRCS="$NGX_ADDON_SRCS \
+    $ps_src/log_message_handler.cc \
+    $ps_src/ngx_base_fetch.cc \
+    $ps_src/ngx_caching_headers.cc \
+    $ps_src/ngx_fetch.cc \
+    $ps_src/ngx_list_iterator.cc \
+    $ps_src/ngx_message_handler.cc \
     $ps_src/ngx_pagespeed.cc \
+    $ps_src/ngx_request_context.cc \
     $ps_src/ngx_rewrite_driver_factory.cc \
     $ps_src/ngx_rewrite_options.cc \
     $ps_src/ngx_server_context.cc \
-    $ps_src/ngx_fetch.cc \
-    $ps_src/ngx_url_async_fetcher.cc \
-    $ps_src/ngx_base_fetch.cc \
     $ps_src/ngx_thread_system.cc \
-    $ps_src/ngx_message_handler.cc \
+    $ps_src/ngx_url_async_fetcher.cc \
     $ps_src/pthread_shared_mem.cc \
-    $ps_src/ngx_request_context.cc \
-    $ps_src/log_message_handler.cc \
-    $mod_pagespeed_dir/net/instaweb/apache/add_headers_fetcher.cc \
-    $mod_pagespeed_dir/net/instaweb/apache/loopback_route_fetcher.cc \
-    $mod_pagespeed_dir/net/instaweb/apache/serf_url_async_fetcher.cc"
+    $mod_pagespeed_dir/out/$buildtype/obj/gen/data2c_out/instaweb/net/instaweb/system/console_out.cc \
+    $mod_pagespeed_dir/out/$buildtype/obj/gen/data2c_out/instaweb/net/instaweb/system/console_css_out.cc \
+    $mod_pagespeed_dir/net/instaweb/system/add_headers_fetcher.cc \
+    $mod_pagespeed_dir/net/instaweb/system/loopback_route_fetcher.cc \
+    $mod_pagespeed_dir/net/instaweb/system/serf_url_async_fetcher.cc"
   # Make pagespeed run immediately before gzip.
   HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
     sed "s/$HTTP_GZIP_FILTER_MODULE/$HTTP_GZIP_FILTER_MODULE $ngx_addon_name/")

scripts/prepare_psol.sh

@@ -63,6 +63,8 @@ rsync -arvz "$MOD_PAGESPEED_SRC/" "psol/include/" --prune-empty-dirs \
   --include="apr_memcache2.c" \
   --include="loopback_route_fetcher.cc" \
   --include="add_headers_fetcher.cc" \
+  --include="console_css_out.cc" \
+  --include="console_out.cc" \
   --include="dense_hash_map" \
   --include="dense_hash_set" \
   --include="sparse_hash_map" \

src/ngx_base_fetch.cc

@@ -17,6 +17,7 @@
 // Author: jefftk@google.com (Jeff Kaufman)
 
 #include "ngx_base_fetch.h"
+#include "ngx_list_iterator.h"
 
 #include "ngx_pagespeed.h"
 
@@ -82,24 +83,11 @@ void NgxBaseFetch::CopyHeadersFromTable(ngx_list_t* headers_from,
   headers_to->set_major_version(request_->http_version / 1000);
   headers_to->set_minor_version(request_->http_version % 1000);
 
-  // Standard nginx idiom for iterating over a list.  See ngx_list.h
-  ngx_uint_t i;
-  ngx_list_part_t* part = &headers_from->part;
-  ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(part->elts);
-
-  for (i = 0 ; /* void */; i++) {
-    if (i >= part->nelts) {
-      if (part->next == NULL) {
-        break;
-      }
-
-      part = part->next;
-      header = static_cast<ngx_table_elt_t*>(part->elts);
-      i = 0;
-    }
-
-    StringPiece key = ngx_psol::str_to_string_piece(header[i].key);
-    StringPiece value = ngx_psol::str_to_string_piece(header[i].value);
+  ngx_table_elt_t* header;
+  NgxListIterator it(&headers_from->part);
+  while ((header = it.Next()) != NULL) {
+    StringPiece key = ngx_psol::str_to_string_piece(header->key);
+    StringPiece value = ngx_psol::str_to_string_piece(header->value);
 
     headers_to->Add(key, value);
   }

src/ngx_caching_headers.cc

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#include "ngx_caching_headers.h"
+#include "ngx_list_iterator.h"
+
+#include "ngx_pagespeed.h"
+
+namespace net_instaweb {
+
+bool NgxCachingHeaders::Lookup(const GoogleString& key,
+                               StringPieceVector* values) {
+  ngx_table_elt_t* header;
+  NgxListIterator it(&(request_->headers_out.headers.part));
+  while ((header = it.Next()) != NULL) {
+    if (header->hash != 0 &&
+        key == ngx_psol::str_to_string_piece(header->key)) {
+      // This will be called multiple times if there are multiple headers with
+      // this name.  Each time it will append to values.
+      SplitStringPieceToVector(
+          ngx_psol::str_to_string_piece(header->value), ",", values,
+                                        true /* omit empty strings */);
+    }
+  }
+
+  if (values->size() == 0) {
+    // No header found with this name.
+    return false;
+  }
+
+  for (int i = 0, n = values->size(); i < n; ++i) {
+    TrimWhitespace(&((*values)[i]));
+  }
+
+  return true;
+}
+
+}  // namespace net_instaweb

src/ngx_caching_headers.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#ifndef NGX_CACHING_HEADERS_H_
+#define NGX_CACHING_HEADERS_H_
+
+extern "C" {
+  #include <ngx_http.h>
+}
+
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/http/caching_headers.h"
+
+namespace net_instaweb {
+
+// Based off of ApacheCachingHeaders in net/instaweb/apache/header_util.cc
+// Needed so ps_header_filter can call GenerateDisabledCacheControl().
+class NgxCachingHeaders : public CachingHeaders {
+ public:
+  explicit NgxCachingHeaders(ngx_http_request_t* request)
+      : CachingHeaders(request->headers_out.status),
+        request_(request) {
+  }
+
+  virtual bool Lookup(const GoogleString& key, StringPieceVector* values);
+
+  virtual bool IsLikelyStaticResourceType() const {
+    DCHECK(false);  // not called in our use-case.
+    return false;
+  }
+
+  virtual bool IsCacheableResourceStatusCode() const {
+    DCHECK(false);  // not called in our use-case.
+    return false;
+  }
+
+ private:
+  ngx_http_request_t* request_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxCachingHeaders);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_CACHING_HEADERS_H_

src/ngx_fetch.h

@@ -35,7 +35,7 @@ extern "C" {
 #include "net/instaweb/util/public/basictypes.h"
 #include "net/instaweb/util/public/pool.h"
 #include "net/instaweb/util/public/string.h"
-#include "net/instaweb/http/public/url_pollable_async_fetcher.h"
+#include "net/instaweb/http/public/url_async_fetcher.h"
 #include "net/instaweb/http/public/response_headers.h"
 #include "net/instaweb/http/public/response_headers_parser.h"
 

src/ngx_list_iterator.cc

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#include "ngx_list_iterator.h"
+
+namespace net_instaweb {
+
+NgxListIterator::NgxListIterator(ngx_list_part_t* part) :
+    part_(part),
+    index_within_part_(0) {}
+
+ngx_table_elt_t* NgxListIterator::Next() {
+  if (index_within_part_ >= part_->nelts) {
+    if (part_->next == NULL) {
+      return NULL;
+    }
+    part_ = part_->next;
+    index_within_part_ = 0;
+  }
+  ngx_table_elt_t* elts = static_cast<ngx_table_elt_t*>(part_->elts);
+  return &elts[index_within_part_++];  // Intentional post-increment.
+}
+
+}  // namespace net_instaweb

src/ngx_list_iterator.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+//
+// Simplifies iteration over nginx lists.
+//
+//
+
+#ifndef NGX_LIST_ITERATOR_H_
+#define NGX_LIST_ITERATOR_H_
+
+extern "C" {
+#include <ngx_http.h>
+}
+
+#include "base/basictypes.h"
+
+namespace net_instaweb {
+
+class NgxListIterator {
+ public:
+  NgxListIterator(ngx_list_part_t* part);
+
+  // Return the next element of the list if there is one, NULL otherwise.
+  ngx_table_elt_t* Next();
+
+ private:
+  ngx_list_part_t* part_;
+  ngx_uint_t index_within_part_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxListIterator);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_LIST_ITERATOR_H_

src/ngx_pagespeed.cc

@@ -29,6 +29,8 @@
 #include <set>
 
 #include "ngx_base_fetch.h"
+#include "ngx_caching_headers.h"
+#include "ngx_list_iterator.h"
 #include "ngx_message_handler.h"
 #include "ngx_request_context.h"
 #include "ngx_rewrite_driver_factory.h"
@@ -41,12 +43,13 @@
 #include "net/instaweb/automatic/public/proxy_fetch.h"
 #include "net/instaweb/http/public/content_type.h"
 #include "net/instaweb/http/public/request_context.h"
-#include "net/instaweb/rewriter/public/furious_matcher.h"
-#include "net/instaweb/rewriter/public/furious_util.h"
+#include "net/instaweb/rewriter/public/experiment_matcher.h"
+#include "net/instaweb/rewriter/public/experiment_util.h"
 #include "net/instaweb/rewriter/public/process_context.h"
 #include "net/instaweb/rewriter/public/resource_fetch.h"
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
 #include "net/instaweb/rewriter/public/static_asset_manager.h"
+#include "net/instaweb/system/public/handlers.h"
 #include "net/instaweb/public/global_constants.h"
 #include "net/instaweb/public/version.h"
 #include "net/instaweb/util/public/google_message_handler.h"
@@ -54,11 +57,13 @@
 #include "net/instaweb/util/public/gzip_inflater.h"
 #include "pthread_shared_mem.h"
 #include "net/instaweb/util/public/query_params.h"
+#include "net/instaweb/util/public/statistics_logger.h"
 #include "net/instaweb/util/public/stdio_file_system.h"
 #include "net/instaweb/util/public/string.h"
 #include "net/instaweb/util/public/string_writer.h"
 #include "net/instaweb/util/public/time_util.h"
 #include "net/instaweb/util/stack_buffer.h"
+#include "pagespeed/kernel/html/html_keywords.h"
 
 extern ngx_module_t ngx_pagespeed;
 
@@ -303,6 +308,8 @@ typedef struct {
   // there as well.
   net_instaweb::NgxServerContext* server_context;
   net_instaweb::ProxyFetchFactory* proxy_fetch_factory;
+  // Only used while parsing config.  After we merge cfg_s and cfg_m you most
+  // likely want cfg_s->server_context->config() as options here will be NULL.
   net_instaweb::NgxRewriteOptions* options;
   net_instaweb::MessageHandler* handler;
 } ps_srv_conf_t;
@@ -312,8 +319,6 @@ typedef struct {
   net_instaweb::MessageHandler* handler;
 } ps_loc_conf_t;
 
-ngx_int_t ps_body_filter(ngx_http_request_t* r, ngx_chain_t* in);
-
 void* ps_create_srv_conf(ngx_conf_t* cf);
 
 char* ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child);
@@ -346,6 +351,7 @@ enum Response {
   kPagespeedDisabled,
   kBeacon,
   kStatistics,
+  kConsole,
   kMessages,
   kPagespeedSubrequest,
   kNotHeadOrGet,
@@ -481,9 +487,6 @@ char* ps_configure(ngx_conf_t* cf,
                    net_instaweb::NgxRewriteOptions** options,
                    net_instaweb::MessageHandler* handler,
                    PsConfigure::OptionLevel option_level) {
-  if (*options == NULL) {
-    *options = new net_instaweb::NgxRewriteOptions();
-  }
   // args[0] is always "pagespeed"; ignore it.
   ngx_uint_t n_args = cf->args->nelts - 1;
 
@@ -513,6 +516,10 @@ char* ps_configure(ngx_conf_t* cf,
 
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cf->cycle, ngx_pagespeed));
+  if (*options == NULL) {
+    *options = new net_instaweb::NgxRewriteOptions(
+        cfg_m->driver_factory->thread_system());
+  }
   const char* status = (*options)->ParseAndSetOptions(
       args, n_args, cf->pool, handler, cfg_m->driver_factory);
 
@@ -1137,32 +1144,32 @@ net_instaweb::RewriteOptions* ps_determine_request_options(
 // classify them into one by setting a cookie.  Then set options appropriately
 // for their experiment.
 //
-// See InstawebContext::SetFuriousStateAndCookie()
-bool ps_set_furious_state_and_cookie(ngx_http_request_t* r,
+// See InstawebContext::SetExperimentStateAndCookie()
+bool ps_set_experiment_state_and_cookie(ngx_http_request_t* r,
                                      ps_request_ctx_t* ctx,
                                      net_instaweb::RewriteOptions* options,
                                      const StringPiece& host) {
-  CHECK(options->running_furious());
+  CHECK(options->running_experiment());
   ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
-  bool need_cookie = cfg_s->server_context->furious_matcher()->
+  bool need_cookie = cfg_s->server_context->experiment_matcher()->
       ClassifyIntoExperiment(*ctx->base_fetch->request_headers(), options);
   if (need_cookie && host.length() > 0) {
     int64 time_now_us = apr_time_now();
     int64 expiration_time_ms = (time_now_us/1000 +
-                                options->furious_cookie_duration_ms());
+                                options->experiment_cookie_duration_ms());
 
-    // TODO(jefftk): refactor SetFuriousCookie to expose the value we want to
+    // TODO(jefftk): refactor SetExperimentCookie to expose the value we want to
     // set on the cookie.
-    int state = options->furious_id();
+    int state = options->experiment_id();
     GoogleString expires;
     net_instaweb::ConvertTimeToString(expiration_time_ms, &expires);
     GoogleString value = StringPrintf(
         "%s=%s; Expires=%s; Domain=.%s; Path=/",
-        net_instaweb::furious::kFuriousCookie,
-        net_instaweb::furious::FuriousStateToCookieString(state).c_str(),
+        net_instaweb::experiment::kExperimentCookie,
+        net_instaweb::experiment::ExperimentStateToCookieString(state).c_str(),
         expires.c_str(), host.as_string().c_str());
 
-    // Set the GFURIOUS cookie.
+    // Set the PagespeedExperiment cookie.
     ngx_table_elt_t* cookie = static_cast<ngx_table_elt_t*>(
         ngx_list_push(&r->headers_out.headers));
     if (cookie == NULL) {
@@ -1212,7 +1219,7 @@ bool ps_determine_options(ngx_http_request_t* r,
   // only situation in which we can avoid allocating a new RewriteOptions is if
   // the global options are ok as are.
   if (directory_options == NULL && request_options == NULL &&
-      !global_options->running_furious()) {
+      !global_options->running_experiment()) {
     return true;
   }
 
@@ -1230,8 +1237,8 @@ bool ps_determine_options(ngx_http_request_t* r,
   if (request_options != NULL) {
     (*options)->Merge(*request_options);
     delete request_options;
-  } else if ((*options)->running_furious()) {
-    bool ok = ps_set_furious_state_and_cookie(r, ctx, *options, url->Host());
+  } else if ((*options)->running_experiment()) {
+    bool ok = ps_set_experiment_state_and_cookie(r, ctx, *options, url->Host());
     if (!ok) {
       if (*options != NULL) {
         delete *options;
@@ -1255,23 +1262,11 @@ bool ps_apply_x_forwarded_proto(ngx_http_request_t* r, GoogleString* url) {
   // First check for an X-Forwarded-Proto header.
   const ngx_str_t* x_forwarded_proto_header = NULL;
 
-  // Standard nginx idiom for iterating over a list.  See ngx_list.h
-  ngx_uint_t i;
-  ngx_list_part_t* part = &(r->headers_in.headers.part);
-  ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(part->elts);
-
-  for (i = 0 ; /* void */; i++) {
-    if (i >= part->nelts) {
-      if (part->next == NULL) {
-        break;
-      }
-
-      part = part->next;
-      header = static_cast<ngx_table_elt_t*>(part->elts);
-      i = 0;
-    }
-    if (STR_CASE_EQ_LITERAL(header[i].key, "X-Forwarded-Proto")) {
-      x_forwarded_proto_header = &header[i].value;
+  ngx_table_elt_t* header;
+  net_instaweb::NgxListIterator it(&(r->headers_in.headers.part));
+  while ((header = it.Next()) != NULL) {
+    if (STR_CASE_EQ_LITERAL(header->key, "X-Forwarded-Proto")) {
+      x_forwarded_proto_header = &header->value;
       break;
     }
   }
@@ -1280,8 +1275,8 @@ bool ps_apply_x_forwarded_proto(ngx_http_request_t* r, GoogleString* url) {
     return false;  // No X-Forwarded-Proto header found.
   }
 
-  StringPiece x_forwarded_proto
-      = str_to_string_piece(*x_forwarded_proto_header);
+  StringPiece x_forwarded_proto =
+      str_to_string_piece(*x_forwarded_proto_header);
 
   if (!STR_CASE_EQ_LITERAL(*x_forwarded_proto_header, "http") &&
       !STR_CASE_EQ_LITERAL(*x_forwarded_proto_header, "https")) {
@@ -1378,13 +1373,10 @@ ps_initiate_property_cache_lookup(
   bool added_callback = false;
   net_instaweb::PropertyPageStarVector property_callbacks;
 
-  net_instaweb::ProxyFetchPropertyCallback* client_callback = NULL;
   net_instaweb::ProxyFetchPropertyCallback* property_callback = NULL;
   net_instaweb::ProxyFetchPropertyCallback* fallback_property_callback = NULL;
   net_instaweb::PropertyCache* page_property_cache =
       server_context->page_property_cache();
-  net_instaweb::PropertyCache* client_property_cache =
-      server_context->client_property_cache();
   if (!is_resource_fetch &&
       server_context->page_property_cache()->enabled() &&
       UrlMightHavePropertyCacheEntry(request_url) &&
@@ -1411,12 +1403,18 @@ ps_initiate_property_cache_lookup(
     // Trigger property cache lookup for the requests which contains query param
     // as cache key without query params. The result of this lookup will be used
     // if actual property page does not contains property value.
+    GoogleString fallback_page_key;
     if (options != NULL &&
         options->use_fallback_property_cache_values() &&
-        request_url.has_query()) {
-      GoogleString fallback_page_key =
-          server_context->GetFallbackPagePropertyCacheKey(
-              request_url.AllExceptQuery(), options, device_type_suffix);
+        request_url.has_query() &&
+        request_url.PathAndLeaf() != "/" &&
+        !request_url.PathAndLeaf().empty()) {
+      // Don't bother looking up fallback properties for the root, "/", since
+      // there is nothing to fall back to.
+      fallback_page_key = server_context->GetFallbackPagePropertyCacheKey(
+          request_url, options, device_type_suffix);
+    }
+    if (!fallback_page_key.empty()) {
       fallback_property_callback = new net_instaweb::ProxyFetchPropertyCallback(
           net_instaweb::ProxyFetchPropertyCallback::kPropertyCacheFallbackPage,
           page_property_cache, fallback_page_key, device_type,
@@ -1426,25 +1424,6 @@ ps_initiate_property_cache_lookup(
     }
   }
 
-  // Initiate client property cache lookup.
-  if (async_fetch != NULL) {
-    const char* client_id = async_fetch->request_headers()->Lookup1(
-        net_instaweb::HttpAttributes::kXGooglePagespeedClientId);
-    if (client_id != NULL) {
-      if (client_property_cache->enabled()) {
-        net_instaweb::AbstractMutex* mutex =
-            server_context->thread_system()->NewMutex();
-        client_callback = new net_instaweb::ProxyFetchPropertyCallback(
-            net_instaweb::ProxyFetchPropertyCallback::kClientPropertyCachePage,
-            client_property_cache, client_id,
-            net_instaweb::UserAgentMatcher::kEndOfDeviceType,
-            callback_collector.get(), mutex);
-        callback_collector->AddCallback(client_callback);
-        added_callback = true;
-      }
-    }
-  }
-
   // All callbacks need to be registered before Reads to avoid race.
   if (property_callback != NULL) {
     page_property_cache->Read(property_callback);
@@ -1452,9 +1431,6 @@ ps_initiate_property_cache_lookup(
   if (fallback_property_callback != NULL) {
     page_property_cache->Read(fallback_property_callback);
   }
-  if (client_callback != NULL) {
-    client_property_cache->Read(client_callback);
-  }
 
   if (!added_callback) {
     callback_collector.reset(NULL);
@@ -1500,6 +1476,9 @@ CreateRequestContext::Response ps_create_request_context(
       || url.PathSansQuery() == "/ngx_pagespeed_global_statistics" ) {
     return CreateRequestContext::kStatistics;
   }
+  if (url.PathSansQuery() == "/pagespeed_console") {
+    return CreateRequestContext::kConsole;
+  }
   if (url.PathSansQuery() == "/ngx_pagespeed_message") {
     return CreateRequestContext::kMessages;
   }
@@ -1518,11 +1497,16 @@ CreateRequestContext::Response ps_create_request_context(
     return CreateRequestContext::kBeacon;
   }
 
-  if (r->method != NGX_HTTP_GET && r->method != NGX_HTTP_HEAD) {
+  if (is_resource_fetch &&
+      r->method != NGX_HTTP_GET &&
+      r->method != NGX_HTTP_HEAD) {
     return CreateRequestContext::kNotHeadOrGet;
   }
 
   if (is_resource_fetch && !cfg_s->server_context->IsPagespeedResource(url)) {
+    // NOTE: We are using the below debug message as is for some of our system
+    // tests. So, be careful about test breakages caused by changing or
+    // removing this line.
     DBG(r, "Passing on content handling for non-pagespeed resource '%s'",
         url_string.c_str());
     return CreateRequestContext::kNotUnderstood;
@@ -1641,6 +1625,12 @@ CreateRequestContext::Response ps_create_request_context(
           custom_options, ctx->base_fetch->request_context());
     }
 
+    StringPiece user_agent = ctx->base_fetch->request_headers()->Lookup1(
+        net_instaweb::HttpAttributes::kUserAgent);
+    if (!user_agent.empty()) {
+      ctx->driver->SetUserAgent(user_agent);
+    }
+    ctx->driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
     // TODO(jefftk): FlushEarlyFlow would go here.
 
     // Will call StartParse etc.  The rewrite driver will take care of deleting
@@ -1774,6 +1764,17 @@ ngx_int_t ps_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
 
 // Based on ngx_http_add_cache_control.
 ngx_int_t ps_set_cache_control(ngx_http_request_t* r, char* cache_control) {
+  // First strip existing cache-control headers.
+  ngx_table_elt_t* header;
+  net_instaweb::NgxListIterator it(&(r->headers_out.headers.part));
+  while ((header = it.Next()) != NULL) {
+    if (STR_CASE_EQ_LITERAL(header->key, "Cache-Control")) {
+      // Response headers with hash of 0 are excluded from the response.
+      header->hash = 0;
+    }
+  }
+
+  // Now add our new cache control header.
   if (r->headers_out.cache_control.elts == NULL) {
     ngx_int_t rc = ngx_array_init(&r->headers_out.cache_control, r->pool,
                                   1, sizeof(ngx_table_elt_t *));
@@ -1805,32 +1806,19 @@ void ps_strip_html_headers(ngx_http_request_t* r) {
   // calculate on the fly.
   ngx_http_clear_content_length(r);
 
-  // Standard nginx idiom for iterating over a list.  See ngx_list.h
-  ngx_uint_t i;
-  ngx_list_part_t* part = &(r->headers_out.headers.part);
-  ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(part->elts);
-
-  for (i = 0 ; /* void */; i++) {
-    if (i >= part->nelts) {
-      if (part->next == NULL) {
-        break;
-      }
-
-      part = part->next;
-      header = static_cast<ngx_table_elt_t*>(part->elts);
-      i = 0;
-    }
-
+  ngx_table_elt_t* header;
+  net_instaweb::NgxListIterator it(&(r->headers_out.headers.part));
+  while ((header = it.Next()) != NULL) {
     // We also need to strip:
     //   Accept-Ranges
     //    - won't work because our html changes
     //   Vary: Accept-Encoding
     //    - our gzip filter will add this later
-    if (STR_CASE_EQ_LITERAL(header[i].key, "Accept-Ranges") ||
-        (STR_CASE_EQ_LITERAL(header[i].key, "Vary") &&
-         STR_CASE_EQ_LITERAL(header[i].value, "Accept-Encoding"))) {
+    if (STR_CASE_EQ_LITERAL(header->key, "Accept-Ranges") ||
+        (STR_CASE_EQ_LITERAL(header->key, "Vary") &&
+         STR_CASE_EQ_LITERAL(header->value, "Accept-Encoding"))) {
       // Response headers with hash of 0 are excluded from the response.
-      header[i].hash = 0;
+      header->hash = 0;
     }
   }
 }
@@ -1876,28 +1864,15 @@ bool ps_has_stacked_content_encoding(ngx_http_request_t* r) {
 }
 
 ngx_int_t ps_etag_header_filter(ngx_http_request_t* r) {
-  // Standard nginx idiom for iterating over a list.  See ngx_list.h
-  ngx_uint_t i;
-  ngx_list_part_t* part = &(r->headers_out.headers.part);
-  ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(part->elts);
   u_char* etag = reinterpret_cast<u_char*>(
       const_cast<char*>(kInternalEtagName));
-
-  for (i = 0 ; /* void */; i++) {
-    if (i >= part->nelts) {
-      if (part->next == NULL) {
-        break;
-      }
-
-      part = part->next;
-      header = static_cast<ngx_table_elt_t*>(part->elts);
-      i = 0;
-    }
-
-    if (header[i].key.len == strlen(kInternalEtagName) &&
-        !ngx_strncasecmp(header[i].key.data, etag, header[i].key.len)) {
-      header[i].key.data = reinterpret_cast<u_char*>(const_cast<char*>("ETag"));
-      header[i].key.len = 4;
+  ngx_table_elt_t* header;
+  net_instaweb::NgxListIterator it(&(r->headers_out.headers.part));
+  while ((header = it.Next()) != NULL) {
+    if (header->key.len == strlen(kInternalEtagName) &&
+        !ngx_strncasecmp(header->key.data, etag, header->key.len)) {
+      header->key.data = reinterpret_cast<u_char*>(const_cast<char*>("ETag"));
+      header->key.len = 4;
       r->headers_out.etag = header;
       break;
     }
@@ -1942,6 +1917,19 @@ ngx_int_t ps_header_filter(ngx_http_request_t* r) {
     return ngx_http_next_header_filter(r);
   }
 
+  ngx_table_elt_t* header;
+  net_instaweb::NgxListIterator it(&(r->headers_out.headers.part));
+  while ((header = it.Next()) != NULL) {
+    // If there is a proxy_cache configured in front of this ngx server,
+    // we expect it to add a X-Cache header with the value of the cache
+    // status (one of HIT, MISS, EXPIRED).
+    if (STR_CASE_EQ_LITERAL(header->key, "X-Cache") &&
+        STR_CASE_EQ_LITERAL(header->value, "HIT")) {
+      // Bypass content handling by pagespeed modules if this is a cache hit.
+      return ngx_http_next_header_filter(r);
+    }
+  }
+
   switch (ps_create_request_context(
       r, false /* not a resource fetch */)) {
     case CreateRequestContext::kError:
@@ -1957,6 +1945,7 @@ ngx_int_t ps_header_filter(ngx_http_request_t* r) {
     case CreateRequestContext::kBeacon:
     case CreateRequestContext::kStaticContent:
     case CreateRequestContext::kStatistics:
+    case CreateRequestContext::kConsole:
     case CreateRequestContext::kMessages:
     case CreateRequestContext::kPagespeedSubrequest:
     case CreateRequestContext::kPagespeedDisabled:
@@ -2004,7 +1993,9 @@ ngx_int_t ps_header_filter(ngx_http_request_t* r) {
 
   if (options->modify_caching_headers()) {
     // Don't cache html.  See mod_instaweb:instaweb_fix_headers_filter.
-    ps_set_cache_control(r, const_cast<char*>("max-age=0, no-cache"));
+    net_instaweb::NgxCachingHeaders caching_headers(r);
+    ps_set_cache_control(r, string_piece_to_pool_string(
+        r->pool, caching_headers.GenerateDisabledCacheControl()));
 
     // Pagespeed html doesn't need etags: it should never be cached.
     ngx_http_clear_etag(r);
@@ -2153,6 +2144,12 @@ void ps_write_handler_response(const StringPiece& output,
 
   response_headers.Add(net_instaweb::HttpAttributes::kContentType,
                        content_type.mime_type());
+  // http://msdn.microsoft.com/en-us/library/ie/gg622941(v=vs.85).aspx
+  // Script and styleSheet elements will reject responses with
+  // incorrect MIME types if the server sends the response header
+  // "X-Content-Type-Options: nosniff". This is a security feature
+  // that helps prevent attacks based on MIME-type confusion.
+  response_headers.Add("X-Content-Type-Options", "nosniff");
 
   int64 now_ms = timer->NowMs();
   response_headers.SetDate(now_ms);
@@ -2162,14 +2159,6 @@ void ps_write_handler_response(const StringPiece& output,
   send_out_headers_and_body(r, response_headers, output.as_string());
 }
 
-// Writes text wrapped in a <pre> block
-void ps_write_pre(StringPiece str, net_instaweb::Writer* writer,
-              net_instaweb::MessageHandler* handler) {
-  writer->Write("<pre>\n", handler);
-  writer->Write(str, handler);
-  writer->Write("</pre>\n", handler);
-}
-
 void ps_write_handler_response(const StringPiece& output,
                             ngx_http_request_t* r,
                             net_instaweb::ContentType content_type,
@@ -2183,6 +2172,20 @@ void ps_write_handler_response(const StringPiece& output, ngx_http_request_t* r,
   ps_write_handler_response(output, r, net_instaweb::kContentTypeHtml, timer);
 }
 
+ngx_int_t ps_console_handler(
+    ngx_http_request_t* r,
+    net_instaweb::NgxServerContext* server_context) {
+  net_instaweb::NgxRewriteDriverFactory* factory =
+      static_cast<net_instaweb::NgxRewriteDriverFactory*>(
+          server_context->factory());
+  net_instaweb::MessageHandler* message_handler = factory->message_handler();
+  GoogleString output;
+  net_instaweb::StringWriter writer(&output);
+  ConsoleHandler(server_context->config(), &writer, message_handler);
+  ps_write_handler_response(output, r, factory->timer());
+  return NGX_OK;
+}
+
 // TODO(oschaaf): port SPDY specific functionality, shmcache stats
 // TODO(oschaaf): refactor this with the apache code to share this code
 ngx_int_t ps_statistics_handler(
@@ -2266,7 +2269,7 @@ ngx_int_t ps_statistics_handler(
   GoogleString output;
   net_instaweb::StringWriter writer(&output);
   if (json) {
-    statistics->console_logger()->DumpJSON(var_titles, hist_titles,
+    statistics->console_logger()->DumpJSON(var_titles,
                                            start_time, end_time,
                                            granularity_ms, &writer,
                                            message_handler);
@@ -2297,25 +2300,27 @@ ngx_int_t ps_statistics_handler(
             message_handler);
       }
 
-      // Write <pre></pre> for Dump to keep good format.
-      writer.Write("<pre>", message_handler);
-      statistics->Dump(&writer, message_handler);
-      writer.Write("</pre>", message_handler);
+      GoogleString stats;
+      net_instaweb::StringWriter stats_writer(&stats);
+      statistics->Dump(&stats_writer, message_handler);
+      net_instaweb::HtmlKeywords::WritePre(stats, &writer, message_handler);
       statistics->RenderHistograms(&writer, message_handler);
 
       if (params.Has("memcached")) {
         GoogleString memcached_stats;
         factory->PrintMemCacheStats(&memcached_stats);
         if (!memcached_stats.empty()) {
-          ps_write_pre(memcached_stats, &writer, message_handler);
+          net_instaweb::HtmlKeywords::WritePre(
+              memcached_stats, &writer, message_handler);
         }
       }
     }
 
     if (print_normal_config) {
       writer.Write("Configuration:<br>", message_handler);
-      ps_write_pre(server_context->config()->OptionsToString(),
-               &writer, message_handler);
+      net_instaweb::HtmlKeywords::WritePre(
+          server_context->config()->OptionsToString(),
+          &writer, message_handler);
     }
   }
 
@@ -2338,14 +2343,15 @@ ngx_int_t ps_messages_handler(
       server_context->ngx_rewrite_driver_factory();
   net_instaweb::NgxMessageHandler* message_handler =
       factory->ngx_message_handler();
-  // Write <pre></pre> for Dump to keep good format.
-  writer.Write("<pre>", message_handler);
-  if (!message_handler->Dump(&writer)) {
+  GoogleString log;
+  net_instaweb::StringWriter log_writer(&log);
+  if (!message_handler->Dump(&log_writer)) {
     writer.Write("Writing to ngx_pagespeed_message failed. \n"
                  "Please check if it's enabled in pagespeed.conf.\n",
                  message_handler);
+  } else {
+    net_instaweb::HtmlKeywords::WritePre(log, &writer, message_handler);
   }
-  writer.Write("</pre>", message_handler);
   ps_write_handler_response(output, r, factory->timer());
   return NGX_OK;
 }
@@ -2454,13 +2460,32 @@ bool ps_request_body_to_string_piece(
   }
 }
 
+// Parses out query params from the request.
+void ps_query_params_handler(ngx_http_request_t* r, StringPiece* data) {
+  StringPiece unparsed_uri = str_to_string_piece(r->unparsed_uri);
+  stringpiece_ssize_type question_mark_index = unparsed_uri.find("?");
+  if (question_mark_index == StringPiece::npos) {
+    *data = "";
+  } else {
+    *data = unparsed_uri.substr(
+        question_mark_index+1, unparsed_uri.size() - (question_mark_index+1));
+  }
+}
+
 // Called after nginx reads the request body from the client.  For another
 // example processing request buffers, see ngx_http_form_input_module.c
 void ps_beacon_body_handler(ngx_http_request_t* r) {
+  // Even if the beacon is a POST, the originating url should be in the query
+  // params, not the POST body.
+  StringPiece query_param_beacon_data;
+  ps_query_params_handler(r, &query_param_beacon_data);
+
   StringPiece request_body;
   bool ok = ps_request_body_to_string_piece(r, &request_body);
+  GoogleString beacon_data = net_instaweb::StrCat(
+      query_param_beacon_data, "&", request_body);
   if (ok) {
-    ps_beacon_handler_helper(r, request_body);
+    ps_beacon_handler_helper(r, beacon_data.c_str());
     ngx_http_finalize_request(r, NGX_HTTP_NO_CONTENT);
   } else {
     ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);
@@ -2482,16 +2507,9 @@ ngx_int_t ps_beacon_handler(ngx_http_request_t* r) {
     return NGX_DONE;
   } else {
     // Use query params.
-    StringPiece beacon_data;
-    StringPiece unparsed_uri = str_to_string_piece(r->unparsed_uri);
-    stringpiece_ssize_type question_mark_index = unparsed_uri.find("?");
-    if (question_mark_index == StringPiece::npos) {
-      beacon_data = "";
-    } else {
-      beacon_data = unparsed_uri.substr(
-          question_mark_index+1, unparsed_uri.size() - (question_mark_index+1));
-    }
-    ps_beacon_handler_helper(r, beacon_data);
+    StringPiece query_param_beacon_data;
+    ps_query_params_handler(r, &query_param_beacon_data);
+    ps_beacon_handler_helper(r, query_param_beacon_data);
     return NGX_HTTP_NO_CONTENT;
   }
 }
@@ -2528,6 +2546,8 @@ ngx_int_t ps_content_handler(ngx_http_request_t* r) {
       return ps_static_handler(r);
     case CreateRequestContext::kStatistics:
       return ps_statistics_handler(r, cfg_s->server_context);
+    case CreateRequestContext::kConsole:
+      return ps_console_handler(r, cfg_s->server_context);
     case CreateRequestContext::kMessages:
       return ps_messages_handler(r, cfg_s->server_context);
     case CreateRequestContext::kOk:
@@ -2692,10 +2712,8 @@ ngx_int_t ps_init_module(ngx_cycle_t* cycle) {
       // allows statistics to work if ngx_pagespeed gets turned on via
       // .htaccess or query param.
       if ((statistics == NULL) && config->statistics_enabled()) {
-        statistics = cfg_m->driver_factory->MakeGlobalSharedMemStatistics(
-            config->statistics_logging_enabled(),
-            config->statistics_logging_interval_ms(),
-            config->statistics_logging_file());
+        statistics = \
+            cfg_m->driver_factory->MakeGlobalSharedMemStatistics(*config);
       }
 
       // The hostname identifier is used by the shared memory statistics

src/ngx_rewrite_driver_factory.cc

@@ -28,21 +28,22 @@
 #include "ngx_url_async_fetcher.h"
 #include "pthread_shared_mem.h"
 
-#include "net/instaweb/apache/serf_url_async_fetcher.h"
 #include "net/instaweb/http/public/content_type.h"
-#include "net/instaweb/http/public/fake_url_async_fetcher.h"
+#include "net/instaweb/http/public/rate_controller.h"
+#include "net/instaweb/http/public/rate_controlling_url_async_fetcher.h"
 #include "net/instaweb/http/public/wget_url_fetcher.h"
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
 #include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
 #include "net/instaweb/rewriter/public/server_context.h"
 #include "net/instaweb/rewriter/public/static_asset_manager.h"
+#include "net/instaweb/system/public/serf_url_async_fetcher.h"
 #include "net/instaweb/system/public/system_caches.h"
 #include "net/instaweb/system/public/system_rewrite_options.h"
 #include "net/instaweb/util/public/google_message_handler.h"
 #include "net/instaweb/util/public/null_shared_mem.h"
+#include "net/instaweb/util/public/posix_timer.h"
 #include "net/instaweb/util/public/property_cache.h"
 #include "net/instaweb/util/public/scheduler_thread.h"
-#include "net/instaweb/util/public/posix_timer.h"
 #include "net/instaweb/util/public/shared_circular_buffer.h"
 #include "net/instaweb/util/public/shared_mem_statistics.h"
 #include "net/instaweb/util/public/slow_worker.h"
@@ -119,19 +120,16 @@ Hasher* NgxRewriteDriverFactory::NewHasher() {
   return new MD5Hasher;
 }
 
-UrlFetcher* NgxRewriteDriverFactory::DefaultUrlFetcher() {
-  CHECK(false) << "Nothing should still be using DefaultUrlFetcher()";
-  return NULL;
-}
-
 UrlAsyncFetcher* NgxRewriteDriverFactory::DefaultAsyncUrlFetcher() {
   const char* fetcher_proxy = "";
   if (main_conf_ != NULL) {
     fetcher_proxy = main_conf_->fetcher_proxy().c_str();
   }
 
+  UrlAsyncFetcher* fetcher = NULL;
+
   if (use_native_fetcher_) {
-    net_instaweb::NgxUrlAsyncFetcher* fetcher =
+    ngx_url_async_fetcher_ =
         new net_instaweb::NgxUrlAsyncFetcher(
             fetcher_proxy,
             log_,
@@ -140,10 +138,9 @@ UrlAsyncFetcher* NgxRewriteDriverFactory::DefaultAsyncUrlFetcher() {
             resolver_,
             thread_system(),
             message_handler());
-    ngx_url_async_fetcher_ = fetcher;
-    return fetcher;
+    fetcher = ngx_url_async_fetcher_;
   } else {
-    net_instaweb::SerfUrlAsyncFetcher* fetcher =
+    net_instaweb::SerfUrlAsyncFetcher* serf_fetcher =
         new net_instaweb::SerfUrlAsyncFetcher(
             fetcher_proxy,
             NULL,
@@ -152,10 +149,37 @@ UrlAsyncFetcher* NgxRewriteDriverFactory::DefaultAsyncUrlFetcher() {
             timer(),
             2500,
             message_handler());
-    // Make sure we don't block the nginx event loop
-    fetcher->set_force_threaded(true);
-    return fetcher;
+    fetcher = serf_fetcher;
   }
+
+  SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
+      default_options());
+  if (rate_limit_background_fetches_) {
+    // Unfortunately, we need stats for load-shedding.
+    if (system_options->statistics_enabled()) {
+      // TODO(oschaaf): mps bases this multiplier on the configured
+      // num_rewrite_threads_ which we don't have (yet).
+      int multiplier = 4;
+      fetcher = new RateControllingUrlAsyncFetcher(
+          fetcher,
+          500 * multiplier /* max queue size */,
+          multiplier /* requests/host */,
+          500 * multiplier /* queued per host */,
+          thread_system(),
+          statistics());
+      if (ngx_url_async_fetcher_ == NULL) {
+        defer_cleanup(new Deleter<SerfUrlAsyncFetcher>(
+            static_cast<net_instaweb::SerfUrlAsyncFetcher*>(fetcher)));
+      } else  {
+        defer_cleanup(new Deleter<net_instaweb::NgxUrlAsyncFetcher>(
+            ngx_url_async_fetcher_));
+      }
+    } else {
+      message_handler()->Message(
+          kError, "Can't enable fetch rate-limiting without statistics");
+    }
+  }
+  return fetcher;
 }
 
 MessageHandler* NgxRewriteDriverFactory::DefaultHtmlParseMessageHandler() {
@@ -180,20 +204,21 @@ NamedLockManager* NgxRewriteDriverFactory::DefaultLockManager() {
 }
 
 void NgxRewriteDriverFactory::SetupCaches(ServerContext* server_context) {
+  // TODO(anupama): Remove duplication wrt mod_pagespeed code.
   caches_->SetupCaches(server_context);
 
   server_context->set_enable_property_cache(true);
   PropertyCache* pcache = server_context->page_property_cache();
-  if (pcache->GetCohort(RewriteDriver::kBeaconCohort) == NULL) {
+  const PropertyCache::Cohort* cohort =
     pcache->AddCohort(RewriteDriver::kBeaconCohort);
-  }
-  if (pcache->GetCohort(RewriteDriver::kDomCohort) == NULL) {
-    pcache->AddCohort(RewriteDriver::kDomCohort);
-  }
+  server_context->set_beacon_cohort(cohort);
+
+  cohort = pcache->AddCohort(RewriteDriver::kDomCohort);
+  server_context->set_dom_cohort(cohort);
 }
 
 RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
-  NgxRewriteOptions* options = new NgxRewriteOptions();
+  NgxRewriteOptions* options = new NgxRewriteOptions(thread_system());
   options->SetRewriteLevel(RewriteOptions::kCoreFilters);
   return options;
 }
@@ -347,11 +372,10 @@ void NgxRewriteDriverFactory::ChildInit(ngx_log_t* log) {
 // help with the settings if needed.
 // Note: does not call set_statistics() on the factory.
 Statistics* NgxRewriteDriverFactory::MakeGlobalSharedMemStatistics(
-    bool logging, int64 logging_interval_ms,
-    const GoogleString& logging_file_base) {
+    const NgxRewriteOptions& options) {
   if (shared_mem_statistics_.get() == NULL) {
     shared_mem_statistics_.reset(AllocateAndInitSharedMemStatistics(
-        "global", logging, logging_interval_ms, logging_file_base));
+        "global", options));
   }
   DCHECK(!statistics_frozen_);
   statistics_frozen_ = true;
@@ -361,14 +385,22 @@ Statistics* NgxRewriteDriverFactory::MakeGlobalSharedMemStatistics(
 
 SharedMemStatistics* NgxRewriteDriverFactory::
 AllocateAndInitSharedMemStatistics(
-    const StringPiece& name, const bool logging,
-    const int64 logging_interval_ms,
-    const GoogleString& logging_file_base) {
+    const StringPiece& name,
+    const NgxRewriteOptions& options) {
+  GoogleString log_filename;
+  bool logging_enabled = false;
+  if (!options.log_dir().empty()) {
+    // Only enable statistics logging if a log_dir() is actually specified.
+    log_filename = StrCat(options.log_dir(), "/stats_log_", name);
+    logging_enabled = options.statistics_logging_enabled();
+  }
   // Note that we create the statistics object in the parent process, and
   // it stays around in the kids but gets reinitialized for them
   // inside ChildInit(), called from pagespeed_child_init.
   SharedMemStatistics* stats = new SharedMemStatistics(
-      logging_interval_ms, StrCat(logging_file_base, name), logging,
+      options.statistics_logging_interval_ms(),
+      options.statistics_logging_max_file_size_kb(),
+      log_filename, logging_enabled,
       StrCat(filename_prefix(), name), shared_mem_runtime(), message_handler(),
       file_system(), timer());
   InitStats(stats);
@@ -378,14 +410,12 @@ AllocateAndInitSharedMemStatistics(
 
 void NgxRewriteDriverFactory::InitStats(Statistics* statistics) {
   // Init standard PSOL stats.
+  SystemRewriteDriverFactory::InitStats(statistics);
   RewriteDriverFactory::InitStats(statistics);
+  RateController::InitStats(statistics);
 
   // Init Ngx-specific stats.
   NgxServerContext::InitStats(statistics);
-  SystemCaches::InitStats(statistics);
-  SerfUrlAsyncFetcher::InitStats(statistics);
-  PropertyCache::InitCohortStats(RewriteDriver::kBeaconCohort, statistics);
-  PropertyCache::InitCohortStats(RewriteDriver::kDomCohort, statistics);
 
   statistics->AddVariable(kShutdownCount);
 }

src/ngx_rewrite_driver_factory.h

@@ -61,7 +61,6 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   explicit NgxRewriteDriverFactory(NgxThreadSystem* ngx_thread_system);
   virtual ~NgxRewriteDriverFactory();
   virtual Hasher* NewHasher();
-  virtual UrlFetcher* DefaultUrlFetcher();
   virtual UrlAsyncFetcher* DefaultAsyncUrlFetcher();
   virtual MessageHandler* DefaultHtmlParseMessageHandler();
   virtual MessageHandler* DefaultMessageHandler();
@@ -126,14 +125,11 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   void SharedCircularBufferInit(bool is_root);
   // Build global shared-memory statistics.  This is invoked if at least
   // one server context (global or VirtualHost) enables statistics.
-  Statistics* MakeGlobalSharedMemStatistics(bool logging,
-                                            int64 logging_interval_ms,
-                                            const GoogleString& logging_file);
+  Statistics* MakeGlobalSharedMemStatistics(const NgxRewriteOptions& options);
 
   // Creates and ::Initializes a shared memory statistics object.
   SharedMemStatistics* AllocateAndInitSharedMemStatistics(
-      const StringPiece& name, const bool logging,
-      const int64 logging_interval_ms, const GoogleString& logging_file);
+      const StringPiece& name, const NgxRewriteOptions& options);
 
   NgxMessageHandler* ngx_message_handler() { return ngx_message_handler_; }
   void set_main_conf(NgxRewriteOptions* main_conf) {  main_conf_ = main_conf; }
@@ -169,6 +165,9 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   void set_use_native_fetcher(bool x) {
     use_native_fetcher_ = x;
   }
+  void set_rate_limit_background_fetches(bool x) {
+    rate_limit_background_fetches_ = x;
+  }
 
   // We use a beacon handler to collect data for critical images,
   // css, etc., so filters should be configured accordingly.
@@ -211,6 +210,7 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   ngx_msec_t resolver_timeout_;
   ngx_resolver_t* resolver_;
   bool use_native_fetcher_;
+  bool rate_limit_background_fetches_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
 };

src/ngx_rewrite_options.cc

@@ -35,9 +35,16 @@ extern "C" {
 
 namespace net_instaweb {
 
+namespace {
+
+const char kNgxPagespeedStatisticsHandlerPath[] = "/ngx_pagespeed_statistics";
+
+}  // namespace
+
 RewriteOptions::Properties* NgxRewriteOptions::ngx_properties_ = NULL;
 
-NgxRewriteOptions::NgxRewriteOptions() {
+NgxRewriteOptions::NgxRewriteOptions(ThreadSystem* thread_system)
+    : SystemRewriteOptions(thread_system) {
   Init();
 }
 
@@ -45,14 +52,22 @@ void NgxRewriteOptions::Init() {
   DCHECK(ngx_properties_ != NULL)
       << "Call NgxRewriteOptions::Initialize() before construction";
   InitializeOptions(ngx_properties_);
+
+  // Nginx-specific default.
+  // TODO(sligocki): Get rid of this line and let both Apache and Nginx use
+  // /pagespeed_statistics as the handler.
+  statistics_handler_path_.set_default(kNgxPagespeedStatisticsHandlerPath);
 }
 
 void NgxRewriteOptions::AddProperties() {
   // Nothing ngx-specific for now.
 
   MergeSubclassProperties(ngx_properties_);
-  NgxRewriteOptions config;
-  config.InitializeSignaturesAndDefaults();
+  // We create a dummy NgxRewriteOptions object here so that we can
+  // call InitializeSignaturesAndDefaults on it, and in turn get the
+  // global defaults (for say, X-Page-Speed header value) setup correctly.
+  NgxRewriteOptions dummy_config(NULL);
+  dummy_config.InitializeSignaturesAndDefaults();
 }
 
 void NgxRewriteOptions::InitializeSignaturesAndDefaults() {
@@ -121,15 +136,6 @@ NgxRewriteOptions::ParseAndSetOptions(
     NgxRewriteDriverFactory* driver_factory) {
   CHECK_GE(n_args, 1);
 
-  int i;
-  fprintf(stderr, "Setting option from (");
-  for (i = 0 ; i < n_args ; i++) {
-    fprintf(stderr, "%s\"%s\"",
-            i == 0 ? "" : ", ",
-            args[i].as_string().c_str());
-  }
-  fprintf(stderr, ")\n");
-
   StringPiece directive = args[0];
 
   // Remove initial "ModPagespeed" if there is one.
@@ -189,6 +195,17 @@ NgxRewriteOptions::ParseAndSetOptions(
         } else {
           result = RewriteOptions::kOptionValueInvalid;
         }
+      } else if (IsDirective(directive, "RateLimitBackgroundFetches")) {
+        // TODO(oschaaf): mod_pagespeed has a nicer way to do this.
+        if (IsDirective(arg, "on")) {
+          driver_factory->set_rate_limit_background_fetches(true);
+          result = RewriteOptions::kOptionOk;
+        } else if (IsDirective(arg, "off")) {
+          driver_factory->set_rate_limit_background_fetches(false);
+          result = RewriteOptions::kOptionOk;
+        } else {
+          result = RewriteOptions::kOptionValueInvalid;
+        }
       } else {
         result = ParseAndSetOptionFromName1(directive, args[1], &msg, handler);
       }
@@ -240,7 +257,7 @@ NgxRewriteOptions::ParseAndSetOptions(
 }
 
 NgxRewriteOptions* NgxRewriteOptions::Clone() const {
-  NgxRewriteOptions* options = new NgxRewriteOptions();
+  NgxRewriteOptions* options = new NgxRewriteOptions(thread_system());
   options->Merge(*this);
   return options;
 }

src/ngx_rewrite_options.h

@@ -40,7 +40,7 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   static void Initialize();
   static void Terminate();
 
-  NgxRewriteOptions();
+  explicit NgxRewriteOptions(ThreadSystem* thread_system);
   virtual ~NgxRewriteOptions() { }
 
   // args is an array of n_args StringPieces together representing a directive.

src/ngx_server_context.cc

@@ -19,12 +19,12 @@
 #include "ngx_server_context.h"
 
 #include "ngx_request_context.h"
-#include "ngx_rewrite_options.h"
 #include "ngx_rewrite_driver_factory.h"
-#include "net/instaweb/apache/add_headers_fetcher.h"
-#include "net/instaweb/apache/loopback_route_fetcher.h"
+#include "ngx_rewrite_options.h"
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
 #include "net/instaweb/rewriter/public/rewrite_stats.h"
+#include "net/instaweb/system/public/add_headers_fetcher.h"
+#include "net/instaweb/system/public/loopback_route_fetcher.h"
 #include "net/instaweb/system/public/system_caches.h"
 #include "net/instaweb/util/public/shared_mem_statistics.h"
 #include "net/instaweb/util/public/split_statistics.h"
@@ -83,10 +83,7 @@ void NgxServerContext::CreateLocalStatistics(
     Statistics* global_statistics) {
   local_statistics_ =
       ngx_factory_->AllocateAndInitSharedMemStatistics(
-          hostname_identifier(),
-          config()->statistics_logging_enabled(),
-          config()->statistics_logging_interval_ms(),
-          config()->statistics_logging_file());
+          hostname_identifier(), *config());
   split_statistics_.reset(new SplitStatistics(
       ngx_factory_->thread_system(), local_statistics_, global_statistics));
   // local_statistics_ was ::InitStat'd by AllocateAndInitSharedMemStatistics,

src/ngx_url_async_fetcher.h

@@ -33,11 +33,11 @@ extern "C" {
 }
 
 #include <vector>
+#include "net/instaweb/http/public/url_async_fetcher.h"
 #include "net/instaweb/util/public/basictypes.h"
 #include "net/instaweb/util/public/pool.h"
 #include "net/instaweb/util/public/string.h"
 #include "net/instaweb/util/public/thread_system.h"
-#include "net/instaweb/http/public/url_pollable_async_fetcher.h"
 
 
 namespace net_instaweb {

test/nginx_system_test.sh

@@ -54,6 +54,59 @@ function check_not_simple() {
   "$@" && handle_failure_simple
 }
 
+# Argument list:
+# host_name, path, post-data
+# Runs 5 keepalive requests both with and without gzip for a few times.
+# Curl will use keepalive when running multiple request with one command.
+# When post-data is empty, a get request will be executed.
+function keepalive_test() {
+  HOST_NAME=$1
+  URL="$SECONDARY_HOSTNAME$2"
+  CURL_LOG_FILE="$1.curl.log"
+  NGX_LOG_FILE="$1.error.log"
+  POST_DATA=$3
+
+  for ((i=0; i < 100; i++)); do
+    for accept_encoding in "" "-H \"Accept-Encoding:gzip\""; do
+      if [ -z "$POST_DATA" ]; then
+        curl -m 1 -S -s -v $accept_encoding -H "Host: $HOST_NAME" \
+          $URL $URL $URL $URL $URL > /dev/null \
+          2>>"$TEST_TMP/$CURL_LOG_FILE"
+      else
+        curl -X POST --data "$POST_DATA" -m 1 -S -s -v \
+          $accept_encoding -H "Host: $HOST_NAME"\
+          $URL $URL $URL $URL $URL > /dev/null \
+          2>>"$TEST_TMP/$CURL_LOG_FILE"
+      fi
+    done
+  done
+
+  # Filter the curl output from unimportant messages
+  OUT=$(cat "$TEST_TMP/$CURL_LOG_FILE"\
+    | grep -v "^[<>]"\
+    | grep -v "^{ \\[data not shown"\
+    | grep -v "^\\* About to connect"\
+    | grep -v "^\\* Closing"\
+    | grep -v "^\\* Connected to"\
+    | grep -v "^\\* Re-using"\
+    | grep -v "^\\* Connection.*left intact"\
+    | grep -v "^} \\[data not shown"\
+    | grep -v "^\\* upload completely sent off"\
+    | grep -v "^\\* connected"\
+    | grep -v "^\\*   Trying.*\\.\\.\\.")
+
+  # Nothing should remain after that.
+  check [ -z "$OUT" ]
+
+  # Filter the nginx log from our vhost from unimportant messages.
+  OUT=$(cat "$TEST_TMP/$NGX_LOG_FILE"\
+    | grep -v "closed keepalive connection$")
+
+  # Nothing should remain after that.
+  check [ -z "$OUT" ]
+}
+
+
 this_dir="$( cd $(dirname "$0") && pwd)"
 
 # stop nginx
@@ -62,7 +115,13 @@ killall nginx
 TEST_TMP="$this_dir/tmp"
 rm -r "$TEST_TMP"
 check_simple mkdir "$TEST_TMP"
-FILE_CACHE="$TEST_TMP/file-cache/"
+PROXY_CACHE="$TEST_TMP/proxycache"
+TMP_PROXY_CACHE="$TEST_TMP/tmpproxycache"
+ERROR_LOG="$TEST_TMP/error.log"
+ACCESS_LOG="$TEST_TMP/access.log"
+check_simple mkdir "$PROXY_CACHE"
+check_simple mkdir "$TMP_PROXY_CACHE"
+FILE_CACHE="$TEST_TMP/file-cache"
 check_simple mkdir "$FILE_CACHE"
 SECONDARY_CACHE="$TEST_TMP/file-cache/secondary/"
 check_simple mkdir "$SECONDARY_CACHE"
@@ -99,6 +158,10 @@ cat $PAGESPEED_CONF_TEMPLATE \
   | sed 's#@@DAEMON@@#'"$DAEMON"'#' \
   | sed 's#@@MASTER_PROCESS@@#'"$MASTER_PROCESS"'#' \
   | sed 's#@@TEST_TMP@@#'"$TEST_TMP/"'#' \
+  | sed 's#@@PROXY_CACHE@@#'"$PROXY_CACHE/"'#' \
+  | sed 's#@@TMP_PROXY_CACHE@@#'"$TMP_PROXY_CACHE/"'#' \
+  | sed 's#@@ERROR_LOG@@#'"$ERROR_LOG"'#' \
+  | sed 's#@@ACCESS_LOG@@#'"$ACCESS_LOG"'#' \
   | sed 's#@@FILE_CACHE@@#'"$FILE_CACHE/"'#' \
   | sed 's#@@SECONDARY_CACHE@@#'"$SECONDARY_CACHE/"'#' \
   | sed 's#@@SHM_CACHE@@#'"$SHM_CACHE/"'#' \
@@ -117,7 +180,19 @@ if $USE_VALGRIND; then
   echo "  valgrind $NGINX_EXECUTABLE -c $PAGESPEED_CONF"
   read
 else
-  check_simple "$NGINX_EXECUTABLE" -c "$PAGESPEED_CONF"
+  TRACE_FILE="$TEST_TMP/conf_loading_trace"
+  $NGINX_EXECUTABLE -c $PAGESPEED_CONF >& "$TRACE_FILE"
+  if [[ $? -ne 0 ]]; then
+    echo "FAIL"
+    cat $TRACE_FILE
+    if [[ $(grep -c "unknown directive \"proxy_cache_purge\"" $TRACE_FILE) == 1 ]]; then
+      echo "This test requires proxy_cache_purge. One way to do this:"
+      echo "Run git clone https://github.com/FRiCKLE/ngx_cache_purge.git"
+      echo "And compile nginx with the additional ngx_cache_purge module."
+    fi
+    rm $TRACE_FILE
+    exit 1
+  fi
 fi
 
 if $RUN_TESTS; then
@@ -141,6 +216,7 @@ PSA_JS_LIBRARY_URL_PREFIX="ngx_pagespeed_static"
 PAGESPEED_EXPECTED_FAILURES="
   ~convert_meta_tags~
   ~In-place resource optimization~
+  ~keepalive with html rewriting~
 "
 
 # The existing system test takes its arguments as positional parameters, and
@@ -175,6 +251,46 @@ function run_post_cache_flush() {
 
 # nginx-specific system tests
 
+# Tests related to rewritten response (downstream) caching.
+CACHABLE_HTML_LOC="${SECONDARY_HOSTNAME}/mod_pagespeed_test/cachable_rewritten_html"
+TMP_LOG_LINE="proxy_cache.example.com GET /purge/mod_pagespeed_test/cachable_rewritten_"
+PURGE_REQUEST_IN_ACCESS_LOG=$TMP_LOG_LINE"html/downstream_caching.html.*(200)"
+
+# Number of downstream cache purges should be 0 here.
+CURRENT_STATS=$($WGET_DUMP $STATISTICS_URL)
+check_from "$CURRENT_STATS" egrep -q "downstream_cache_purge_attempts:\s*0"
+
+# The 1st request results in a cache miss, non-rewritten response
+# produced by pagespeed code and a subsequent purge request.
+start_test Check for case where rewritten cache should get purged.
+WGET_ARGS="--header=Host:proxy_cache.example.com"
+OUT=$($WGET_DUMP $WGET_ARGS $CACHABLE_HTML_LOC/downstream_caching.html)
+check_not_from "$OUT" egrep -q "pagespeed.ic"
+check_from "$OUT" egrep -q "X-Cache: MISS"
+fetch_until $STATISTICS_URL 'grep -c downstream_cache_purge_attempts:\s*1' 1
+check [ $(grep -ce "$PURGE_REQUEST_IN_ACCESS_LOG" $ACCESS_LOG) = 1 ];
+
+# The 2nd request results in a cache miss (because of the previous purge),
+# rewritten response produced by pagespeed code and no new purge requests.
+start_test Check for case where rewritten cache should not get purged.
+BLOCKING_WGET_ARGS=$WGET_ARGS" --header=X-PSA-Blocking-Rewrite:psatest"
+OUT=$($WGET_DUMP $BLOCKING_WGET_ARGS $CACHABLE_HTML_LOC/downstream_caching.html)
+check_from "$OUT" egrep -q "pagespeed.ic"
+check_from "$OUT" egrep -q "X-Cache: MISS"
+CURRENT_STATS=$($WGET_DUMP $STATISTICS_URL)
+check_from "$CURRENT_STATS" egrep -q "downstream_cache_purge_attempts:\s*1"
+check [ $(grep -ce "$PURGE_REQUEST_IN_ACCESS_LOG" $ACCESS_LOG) = 1 ];
+
+# The 3rd request results in a cache hit (because the previous response is
+# now present in cache), rewritten response served out from cache and not
+# by pagespeed code and no new purge requests.
+start_test Check for case where there is a rewritten cache hit.
+OUT=$($WGET_DUMP $WGET_ARGS $CACHABLE_HTML_LOC/downstream_caching.html)
+check_from "$OUT" egrep -q "pagespeed.ic"
+check_from "$OUT" egrep -q "X-Cache: HIT"
+fetch_until $STATISTICS_URL 'grep -c downstream_cache_purge_attempts:\s*1' 1
+check [ $(grep -ce "$PURGE_REQUEST_IN_ACCESS_LOG" $ACCESS_LOG) = 1 ];
+
 start_test Check for correct default X-Page-Speed header format.
 OUT=$($WGET_DUMP $EXAMPLE_ROOT/combine_css.html)
 check_from "$OUT" egrep -q \
@@ -240,11 +356,11 @@ OUT=$(wget -O - $EXAMPLE_ROOT)
 check_from "$OUT" grep "$EXPECTED_EXAMPLES_TEXT"
 
 # It should still be there with bad query params.
-OUT=$(wget -O - $EXAMPLE_ROOT?ModPagespeedFilters=bogus)
+OUT=$(wget -O - $EXAMPLE_ROOT?PageSpeedFilters=bogus)
 check_from "$OUT" grep "$EXPECTED_EXAMPLES_TEXT"
 
 # And also with bad request headers.
-OUT=$(wget -O - --header=ModPagespeedFilters:bogus $EXAMPLE_ROOT)
+OUT=$(wget -O - --header=PageSpeedFilters:bogus $EXAMPLE_ROOT)
 check_from "$OUT" grep "$EXPECTED_EXAMPLES_TEXT"
 
 # Test that loopback route fetcher works with vhosts not listening on
@@ -267,7 +383,7 @@ start_test "Loopback fetches go to local IPs without DNS lookup"
 # we'll not rewrite any resources.
 
 URL="$HOSTNAME/mod_pagespeed_example/combine_javascript.html"
-URL+="?ModPagespeed=on&ModPagespeedFilters=combine_javascript"
+URL+="?PageSpeed=on&PageSpeedFilters=combine_javascript"
 fetch_until "$URL" "fgrep -c .pagespeed." 1 --header=Host:www.google.com
 
 # If this accepts the Host header and fetches from google.com it will fail with
@@ -306,7 +422,7 @@ WGET_ARGS="" # Done with test_filter, so clear WGET_ARGS.
 
 start_test Respect X-Forwarded-Proto when told to
 FETCHED=$OUTDIR/x_forwarded_proto
-URL=$SECONDARY_HOSTNAME/mod_pagespeed_example/?ModPagespeedFilters=add_base_tag
+URL=$SECONDARY_HOSTNAME/mod_pagespeed_example/?PageSpeedFilters=add_base_tag
 HEADERS="--header=X-Forwarded-Proto:https --header=Host:xfp.example.com"
 check $WGET_DUMP -O $FETCHED $HEADERS $URL
 # When enabled, we respect X-Forwarded-Proto and thus list base as https.
@@ -345,7 +461,7 @@ fetch_until $URL 'egrep -c big.css.pagespeed.' 1
 # Check that introspection.js was inlined.
 fetch_until $URL 'grep -c document\.write(\"External' 1
 # Check that the image was optimized.
-fetch_until $URL 'grep -c BikeCrashIcn\.png\.pagespeed\.' 2
+fetch_until $URL 'grep -c BikeCrashIcn\.png\.pagespeed\.' 1
 
 # When Cache-Control: no-transform is in the response make sure that
 # the URL is not rewritten and that the no-transform header remains
@@ -380,7 +496,7 @@ fetch_until $URL 'grep -c "pagespeed_no_transform"' 0 \
 check run_wget_with_args $URL
 check_file_size "$OUTDIR/xBikeCrashIcn*" -lt 25000     # re-encoded
 check_file_size "$OUTDIR/*256x192*Puzzle*" -lt 24126   # resized
-URL=$EXAMPLE_ROOT"/rewrite_images.html?ModPagespeedFilters=rewrite_images"
+URL=$EXAMPLE_ROOT"/rewrite_images.html?PageSpeedFilters=rewrite_images"
 IMG_URL=$(egrep -o http://.*.pagespeed.*.jpg $FETCHED | head -n1)
 check [ x"$IMG_URL" != x ]
 start_test headers for rewritten image
@@ -408,12 +524,12 @@ check_from "$IMG_HEADERS" fgrep -qi 'X-Extra-Header'
 start_test Last-modified is present.
 check_from "$IMG_HEADERS" fgrep -qi 'Last-Modified'
 
-IMAGES_QUALITY="ModPagespeedImageRecompressionQuality"
-JPEG_QUALITY="ModPagespeedJpegRecompressionQuality"
-WEBP_QUALITY="ModPagespeedImageWebpRecompressionQuality"
+IMAGES_QUALITY="PageSpeedImageRecompressionQuality"
+JPEG_QUALITY="PageSpeedJpegRecompressionQuality"
+WEBP_QUALITY="PageSpeedImageWebpRecompressionQuality"
 start_test quality of jpeg output images with generic quality flag
 IMG_REWRITE=$TEST_ROOT"/image_rewriting/rewrite_images.html"
-REWRITE_URL=$IMG_REWRITE"?ModPagespeedFilters=rewrite_images"
+REWRITE_URL=$IMG_REWRITE"?PageSpeedFilters=rewrite_images"
 URL=$REWRITE_URL"&"$IMAGES_QUALITY"=75"
 fetch_until -save -recursive $URL 'grep -c .pagespeed.ic' 2 # 2 images optimized
 # This filter produces different images on 32 vs 64 bit builds. On 32 bit, the
@@ -427,13 +543,13 @@ fetch_until -save -recursive $URL 'grep -c .pagespeed.ic' 2 # 2 images optimized
 # is tuned to avoid that.
 check_file_size "$OUTDIR/*256x192*Puzzle*" -le 8157   # resized
 
-IMAGES_QUALITY="ModPagespeedImageRecompressionQuality"
-JPEG_QUALITY="ModPagespeedJpegRecompressionQuality"
-WEBP_QUALITY="ModPagespeedImageWebpRecompressionQuality"
+IMAGES_QUALITY="PageSpeedImageRecompressionQuality"
+JPEG_QUALITY="PageSpeedJpegRecompressionQuality"
+WEBP_QUALITY="PageSpeedImageWebpRecompressionQuality"
 
 start_test quality of jpeg output images
 IMG_REWRITE=$TEST_ROOT"/jpeg_rewriting/rewrite_images.html"
-REWRITE_URL=$IMG_REWRITE"?ModPagespeedFilters=rewrite_images"
+REWRITE_URL=$IMG_REWRITE"?PageSpeedFilters=rewrite_images"
 URL=$REWRITE_URL",recompress_jpeg&"$IMAGES_QUALITY"=85&"$JPEG_QUALITY"=70"
 fetch_until -save -recursive $URL 'grep -c .pagespeed.ic' 2 # 2 images optimized
 #
@@ -447,7 +563,7 @@ start_test quality of webp output images
 rm -rf $OUTDIR
 mkdir $OUTDIR
 IMG_REWRITE=$TEST_ROOT"/webp_rewriting/rewrite_images.html"
-REWRITE_URL=$IMG_REWRITE"?ModPagespeedFilters=rewrite_images"
+REWRITE_URL=$IMG_REWRITE"?PageSpeedFilters=rewrite_images"
 URL=$REWRITE_URL",convert_jpeg_to_webp&"$IMAGES_QUALITY"=75&"$WEBP_QUALITY"=65"
 check run_wget_with_args --header 'X-PSA-Blocking-Rewrite: psatest' $URL
 check_file_size "$OUTDIR/*webp*" -le 1784   # resized, optimized to webp
@@ -455,7 +571,7 @@ check_file_size "$OUTDIR/*webp*" -le 1784   # resized, optimized to webp
 start_test respect vary user-agent
 WGET_ARGS=""
 URL="$SECONDARY_HOSTNAME/mod_pagespeed_test/vary/index.html"
-URL+="?ModPagespeedFilters=inline_css"
+URL+="?PageSpeedFilters=inline_css"
 FETCH_CMD="$WGET_DUMP --header=Host:respectvary.example.com $URL"
 OUT=$($FETCH_CMD)
 # We want to verify that css is not inlined, but if we just check once then
@@ -465,13 +581,13 @@ OUT=$($FETCH_CMD)
 check_not_from "$OUT" fgrep "<style>"
 
 WGET_ARGS=""
-start_test ModPagespeedShardDomain directive in location block
+start_test PageSpeedShardDomain directive in location block
 fetch_until -save $TEST_ROOT/shard/shard.html 'grep -c \.pagespeed\.' 4
 check [ $(grep -ce href=\"http://shard1 $FETCH_FILE) = 2 ];
 check [ $(grep -ce href=\"http://shard2 $FETCH_FILE) = 2 ];
 
-start_test ModPagespeedLoadFromFile
-URL=$TEST_ROOT/load_from_file/index.html?ModPagespeedFilters=inline_css
+start_test PageSpeedLoadFromFile
+URL=$TEST_ROOT/load_from_file/index.html?PageSpeedFilters=inline_css
 fetch_until $URL 'grep -c blue' 1
 
 # The "httponly" directory is disallowed.
@@ -495,8 +611,8 @@ check_simple "$NGINX_EXECUTABLE" -s reload -c "$PAGESPEED_CONF"
 check wget $EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css \
   -O /dev/null
 
-start_test ModPagespeedLoadFromFileMatch
-URL=$TEST_ROOT/load_from_file_match/index.html?ModPagespeedFilters=inline_css
+start_test PageSpeedLoadFromFileMatch
+URL=$TEST_ROOT/load_from_file_match/index.html?PageSpeedFilters=inline_css
 fetch_until $URL 'grep -c blue' 1
 
 start_test Custom headers remain on HTML, but cache should be disabled.
@@ -508,7 +624,7 @@ check_from "$HTML_HEADERS" egrep -q "X-Extra-Header: 1"
 check_not_from "$HTML_HEADERS" egrep -q "X-Extra-Header: 1, 1"
 check_from "$HTML_HEADERS" egrep -q 'Cache-Control: max-age=0, no-cache'
 
-start_test ModPagespeedModifyCachingHeaders
+start_test PageSpeedModifyCachingHeaders
 URL=$TEST_ROOT/retain_cache_control/index.html
 OUT=$($WGET_DUMP $URL)
 check_from "$OUT" grep -q "Cache-Control: private, max-age=3000"
@@ -516,34 +632,34 @@ check_from "$OUT" grep -q "Last-Modified:"
 
 test_filter combine_javascript combines 2 JS files into 1.
 start_test combine_javascript with long URL still works
-URL=$TEST_ROOT/combine_js_very_many.html?ModPagespeedFilters=combine_javascript
+URL=$TEST_ROOT/combine_js_very_many.html?PageSpeedFilters=combine_javascript
 fetch_until $URL 'grep -c src=' 4
 
 start_test aris disables js combining for introspective js and only i-js
 URL="$TEST_ROOT/avoid_renaming_introspective_javascript__on/"
-URL+="?ModPagespeedFilters=combine_javascript"
+URL+="?PageSpeedFilters=combine_javascript"
 fetch_until $URL 'grep -c src=' 2
 
 start_test aris disables js combining only when enabled
 URL="$TEST_ROOT/avoid_renaming_introspective_javascript__off.html?"
-URL+="ModPagespeedFilters=combine_javascript"
+URL+="PageSpeedFilters=combine_javascript"
 fetch_until $URL 'grep -c src=' 1
 
 test_filter inline_javascript inlines a small JS file
 start_test aris disables js inlining for introspective js and only i-js
 URL="$TEST_ROOT/avoid_renaming_introspective_javascript__on/"
-URL+="?ModPagespeedFilters=inline_javascript"
+URL+="?PageSpeedFilters=inline_javascript"
 fetch_until $URL 'grep -c src=' 1
 
 start_test aris disables js inlining only when enabled
 URL="$TEST_ROOT/avoid_renaming_introspective_javascript__off.html"
-URL+="?ModPagespeedFilters=inline_javascript"
+URL+="?PageSpeedFilters=inline_javascript"
 fetch_until $URL 'grep -c src=' 0
 
 test_filter rewrite_javascript minifies JavaScript and saves bytes.
 start_test aris disables js cache extention for introspective js and only i-js
 URL="$TEST_ROOT/avoid_renaming_introspective_javascript__on/"
-URL+="?ModPagespeedFilters=rewrite_javascript"
+URL+="?PageSpeedFilters=rewrite_javascript"
 # first check something that should get rewritten to know we're done with
 # rewriting
 fetch_until -save $URL 'grep -c "src=\"../normal.js\""' 0
@@ -551,7 +667,7 @@ check [ $(grep -c "src=\"../introspection.js\"" $FETCH_FILE) = 1 ]
 
 start_test aris disables js cache extension only when enabled
 URL="$TEST_ROOT/avoid_renaming_introspective_javascript__off.html"
-URL+="?ModPagespeedFilters=rewrite_javascript"
+URL+="?PageSpeedFilters=rewrite_javascript"
 fetch_until -save $URL 'grep -c src=\"normal.js\"' 0
 check [ $(grep -c src=\"introspection.js\" $FETCH_FILE) = 0 ]
 
@@ -559,7 +675,7 @@ check [ $(grep -c src=\"introspection.js\" $FETCH_FILE) = 0 ]
 # avoid_renaming_introspective_javascript is on
 start_test aris disables url modification for introspective js
 URL="$TEST_ROOT/avoid_renaming_introspective_javascript__on/"
-URL+="?ModPagespeedFilters=testing,core"
+URL+="?PageSpeedFilters=testing,core"
 # first check something that should get rewritten to know we're done with
 # rewriting
 fetch_until -save $URL 'grep -c src=\"../normal.js\"' 0
@@ -567,13 +683,13 @@ check [ $(grep -c src=\"../introspection.js\" $FETCH_FILE) = 1 ]
 
 start_test aris disables url modification only when enabled
 URL="$TEST_ROOT/avoid_renaming_introspective_javascript__off.html"
-URL+="?ModPagespeedFilters=testing,core"
+URL+="?PageSpeedFilters=testing,core"
 fetch_until -save $URL 'grep -c src=\"normal.js\"' 0
 check [ $(grep -c src=\"introspection.js\" $FETCH_FILE) = 0 ]
 
 start_test HTML add_instrumentation lacks '&amp;' and does not contain CDATA
 $WGET -O $WGET_OUTPUT $TEST_ROOT/add_instrumentation.html\
-?ModPagespeedFilters=add_instrumentation
+?PageSpeedFilters=add_instrumentation
 check [ $(grep -c "\&amp;" $WGET_OUTPUT) = 0 ]
 # In mod_pagespeed this check is that we *do* contain CDATA.  That's because
 # mod_pagespeed generally runs before response headers are finalized so it has
@@ -586,20 +702,20 @@ check [ $(grep -c '//<\!\[CDATA\[' $WGET_OUTPUT) = 0 ]
 
 start_test XHTML add_instrumentation also lacks '&amp;' but contains CDATA
 $WGET -O $WGET_OUTPUT $TEST_ROOT/add_instrumentation.xhtml\
-?ModPagespeedFilters=add_instrumentation
+?PageSpeedFilters=add_instrumentation
 check [ $(grep -c "\&amp;" $WGET_OUTPUT) = 0 ]
 check [ $(grep -c '//<\!\[CDATA\[' $WGET_OUTPUT) = 1 ]
 
 start_test cache_partial_html enabled has no effect
 $WGET -O $WGET_OUTPUT $TEST_ROOT/add_instrumentation.html\
-?ModPagespeedFilters=cache_partial_html
+?PageSpeedFilters=cache_partial_html
 check [ $(grep -c '<html>' $WGET_OUTPUT) = 1 ]
 check [ $(grep -c '<body>' $WGET_OUTPUT) = 1 ]
 check [ $(grep -c 'pagespeed.panelLoader' $WGET_OUTPUT) = 0 ]
 
 start_test flush_subresources rewriter is not applied
 URL="$TEST_ROOT/flush_subresources.html?\
-ModPagespeedFilters=flush_subresources,extend_cache_css,\
+PageSpeedFilters=flush_subresources,extend_cache_css,\
 extend_cache_scripts"
 # Fetch once with X-PSA-Blocking-Rewrite so that the resources get rewritten and
 # property cache (once it's ported to ngx_pagespeed) is updated with them.
@@ -619,8 +735,8 @@ IMG_CUSTOM="$TEST_ROOT/custom_options/xPuzzle.jpg.pagespeed.ic.fakehash.jpg"
 # Identical images, but in the location block for the custom_options directory
 # we additionally disable core-filter convert_jpeg_to_progressive which gives a
 # larger file.
-fetch_until $IMG_NON_CUSTOM 'wc -c' 216942
-fetch_until $IMG_CUSTOM 'wc -c' 231192
+fetch_until $IMG_NON_CUSTOM 'wc -c' 98276 "" -le
+fetch_until $IMG_CUSTOM 'wc -c' 102902 "" -le
 
 # Test our handling of headers when a FLUSH event occurs.
 start_test PHP is enabled.
@@ -674,8 +790,7 @@ start_test MapProxyDomain
 # depends on MapProxyDomain in pagespeed_test.conf.template
 URL=$EXAMPLE_ROOT/proxy_external_resource.html
 echo Rewrite HTML with reference to a proxyable image.
-fetch_until -save -recursive $URL?ModPagespeedFilters=-inline_images \
-    'grep -c 1.gif.pagespeed' 1
+fetch_until -save -recursive $URL 'grep -c pss-architecture.png.pagespeed.ic' 1
 
 # To make sure that we can reconstruct the proxied content by going back
 # to the origin, we must avoid hitting the output cache.
@@ -685,8 +800,9 @@ fetch_until -save -recursive $URL?ModPagespeedFilters=-inline_images \
 # virtual host attached to a different cache.
 #
 # With the proper hash, we'll get a long cache lifetime.
-SECONDARY_HOST="http://mpd.example.com/gstatic_images"
-PROXIED_IMAGE="$SECONDARY_HOST/$(basename $OUTDIR/*1.gif.pagespeed*)"
+SECONDARY_HOST="http://mpd.example.com/gstatic_images/devconsole"
+PROXIED_IMAGE="$SECONDARY_HOST/$(basename \
+    $OUTDIR/*pss-architecture.png.pagespeed.ic*)"
 WGET_ARGS="--save-headers"
 
 start_test $PROXIED_IMAGE expecting one year cache.
@@ -696,7 +812,8 @@ http_proxy=$SECONDARY_HOSTNAME fetch_until $PROXIED_IMAGE \
 # With the wrong hash, we'll get a short cache lifetime (and also no output
 # cache hit.
 WRONG_HASH="0"
-PROXIED_IMAGE="$SECONDARY_HOST/1.gif.pagespeed.ce.$WRONG_HASH.jpg"
+PROXIED_IMAGE="$SECONDARY_HOST/\
+xpss-architecture.png.pagespeed.ic.$WRONG_HASH.jpg"
 start_test Fetching $PROXIED_IMAGE expecting short private cache.
 http_proxy=$SECONDARY_HOSTNAME fetch_until $PROXIED_IMAGE \
     "grep -c max-age=300,private" 1
@@ -717,7 +834,7 @@ check_from "$OUT" grep '^HTTP/1.1 204'
 check_from "$OUT" grep $'^Cache-Control: max-age=0, no-cache\r$'
 
 start_test server-side includes
-fetch_until -save $TEST_ROOT/ssi/ssi.shtml?ModPagespeedFilters=combine_css \
+fetch_until -save $TEST_ROOT/ssi/ssi.shtml?PageSpeedFilters=combine_css \
     'grep -c \.pagespeed\.' 1
 check [ $(grep -ce $combine_css_filename $FETCH_FILE) = 1 ];
 
@@ -912,7 +1029,7 @@ NUM_INITIAL_FLUSHES_C=$(cache_flush_count_scraper c)
 # Now change the file to $COLOR1.
 echo ".class myclass { color: $COLOR1; }" > "$CSS_FILE"
 
-# We expect to have a stale cache for 5 minutes, so the result should stay
+# We expect to have a stale cache for 5 seconds, so the result should stay
 # $COLOR0.  This only works because we have only one worker process.  If we had
 # more than one then the worker process handling this request might be different
 # than the one that got the previous one, and it wouldn't be in cache.
@@ -1050,7 +1167,7 @@ WGET_ARGS=""
 start_test Relative images embedded in a CSS file served from a mapped domain
 DIR="mod_pagespeed_test/map_css_embedded"
 URL="http://www.example.com/$DIR/issue494.html"
-MAPPED_CSS="$DIR/A.styles.css.pagespeed.cf.w9O-kBfMWw.css"
+MAPPED_CSS="$DIR/A.styles.css.pagespeed.cf.RTch9OLvuX.css"
 http_proxy=$SECONDARY_HOSTNAME fetch_until $URL \
     "grep -c cdn.example.com/$MAPPED_CSS" 1
 
@@ -1083,11 +1200,11 @@ function test_forbid_filters() {
 start_test ForbidFilters baseline check.
 test_forbid_filters "" ""
 start_test ForbidFilters query parameters check.
-QUERYP="?ModPagespeedFilters="
+QUERYP="?PageSpeedFilters="
 QUERYP="${QUERYP}+remove_quotes,+remove_comments,+collapse_whitespace"
 test_forbid_filters $QUERYP ""
 start_test "ForbidFilters request headers check."
-HEADER="--header=ModPagespeedFilters:"
+HEADER="--header=PageSpeedFilters:"
 HEADER="${HEADER}+remove_quotes,+remove_comments,+collapse_whitespace"
 test_forbid_filters "" $HEADER
 
@@ -1123,7 +1240,7 @@ start_test Blocking rewrite enabled.
 # rewrite than the rewrite deadline and it is not already accessed by
 # another request earlier.
 BLOCKING_REWRITE_URL="$TEST_ROOT/blocking_rewrite.html"
-BLOCKING_REWRITE_URL+="?ModPagespeedFilters=rewrite_images"
+BLOCKING_REWRITE_URL+="?PageSpeedFilters=rewrite_images"
 OUTFILE=$OUTDIR/blocking_rewrite.out.html
 OLDSTATS=$OUTDIR/blocking_rewrite_stats.old
 NEWSTATS=$OUTDIR/blocking_rewrite_stats.new
@@ -1150,7 +1267,7 @@ http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP \
   --header 'X-PSA-Blocking-Rewrite: junk' \
   $URL > $OUTFILE
 check [ $(grep -c "[.]pagespeed[.]" $OUTFILE) -lt 1 ]
-  
+
 http_proxy=$SECONDARY_HOSTNAME fetch_until $URL \
   'grep -c [.]pagespeed[.]' 1
 
@@ -1168,7 +1285,7 @@ function test_forbid_all_disabled() {
   if [ -n "$QUERYP" ]; then
     INLINE_CSS=",-inline_css"
   else
-    INLINE_CSS="?ModPagespeedFilters=-inline_css"
+    INLINE_CSS="?PageSpeedFilters=-inline_css"
   fi
   WGET_ARGS="--header=X-PSA-Blocking-Rewrite:psatest"
   URL=$TEST_ROOT/forbid_all_disabled/disabled/forbidden.html
@@ -1189,11 +1306,11 @@ function test_forbid_all_disabled() {
 start_test ForbidAllDisabledFilters baseline check.
 test_forbid_all_disabled "" ""
 start_test ForbidAllDisabledFilters query parameters check.
-QUERYP="?ModPagespeedFilters="
+QUERYP="?PageSpeedFilters="
 QUERYP="${QUERYP}+remove_quotes,+remove_comments,+collapse_whitespace"
 test_forbid_all_disabled $QUERYP ""
 start_test ForbidAllDisabledFilters request headers check.
-HEADER="--header=ModPagespeedFilters:"
+HEADER="--header=PageSpeedFilters:"
 HEADER="${HEADER}+remove_quotes,+remove_comments,+collapse_whitespace"
 test_forbid_all_disabled "" $HEADER
 
@@ -1216,8 +1333,8 @@ RESPONSE_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header \
 check_from     "$RESPONSE_OUT" fgrep -qi small.js.pagespeed.
 check_not_from "$RESPONSE_OUT" fgrep -qi large.js.pagespeed.
 
-# This test checks that the ModPagespeedXHeaderValue directive works.
-start_test ModPagespeedXHeaderValue directive
+# This test checks that the PageSpeedXHeaderValue directive works.
+start_test PageSpeedXHeaderValue directive
 
 RESPONSE_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
   http://xheader.example.com/mod_pagespeed_example)
@@ -1257,7 +1374,7 @@ HOST_NAME="http://url-attribute.example.com"
 TEST="$HOST_NAME/mod_pagespeed_test"
 REWRITE_DOMAINS="$TEST/rewrite_domains.html"
 UVA_EXTEND_CACHE="$TEST/url_valued_attribute_extend_cache.html"
-UVA_EXTEND_CACHE+="?ModPagespeedFilters=core,+left_trim_urls"
+UVA_EXTEND_CACHE+="?PageSpeedFilters=core,+left_trim_urls"
 
 start_test Rewrite domains in dynamically defined url-valued attributes.
 
@@ -1288,38 +1405,38 @@ http_proxy=$SECONDARY_HOSTNAME \
 
 # Test the experiment framework (Furious).
 
-start_test _GFURIOUS cookie is set.
+start_test PageSpeedExperiment cookie is set.
 EXP_EXAMPLE="http://experiment.example.com/mod_pagespeed_example"
 EXP_EXTEND_CACHE="$EXP_EXAMPLE/extend_cache.html"
 OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $EXP_EXTEND_CACHE)
-check_from "$OUT" fgrep "_GFURIOUS="
+check_from "$OUT" fgrep "PageSpeedExperiment="
 
-start_test ModPagespeedFilters query param should disable experiments.
-URL="$EXP_EXTEND_CACHE?ModPagespeed=on&ModPagespeedFilters=rewrite_css"
+start_test PageSpeedFilters query param should disable experiments.
+URL="$EXP_EXTEND_CACHE?PageSpeed=on&PageSpeedFilters=rewrite_css"
 OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
-check_not_from "$OUT" fgrep '_GFURIOUS='
+check_not_from "$OUT" fgrep 'PageSpeedExperiment='
 
 start_test If the user is already assigned, no need to assign them again.
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: _GFURIOUS=2' \
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=2' \
       $EXP_EXTEND_CACHE)
-check_not_from "$OUT" fgrep '_GFURIOUS='
+check_not_from "$OUT" fgrep 'PageSpeedExperiment='
 
 start_test The beacon should include the experiment id.
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: _GFURIOUS=2' \
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=2' \
       $EXP_EXTEND_CACHE)
 BEACON_CODE="pagespeed.addInstrumentationInit('/ngx_pagespeed_beacon', 'load',"
-BEACON_CODE+=" '', '', '', '2', 'http://experiment.example.com/"
+BEACON_CODE+=" '&exptid=2', 'http://experiment.example.com/"
 BEACON_CODE+="mod_pagespeed_example/extend_cache.html');"
 check_from "$OUT" grep "$BEACON_CODE"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: _GFURIOUS=7' \
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=7' \
       $EXP_EXTEND_CACHE)
 BEACON_CODE="pagespeed.addInstrumentationInit('/ngx_pagespeed_beacon', 'load',"
-BEACON_CODE+=" '', '', '', '7', 'http://experiment.example.com/"
+BEACON_CODE+=" '&exptid=7', 'http://experiment.example.com/"
 BEACON_CODE+="mod_pagespeed_example/extend_cache.html');"
 check_from "$OUT" grep "$BEACON_CODE"
 
 start_test The no-experiment group beacon should not include an experiment id.
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: _GFURIOUS=0' \
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=0' \
      $EXP_EXTEND_CACHE)
 check_not_from "$OUT" grep 'mod_pagespeed_beacon.*exptid'
 
@@ -1327,15 +1444,15 @@ check_not_from "$OUT" grep 'mod_pagespeed_beacon.*exptid'
 # order they're defined in the config file.
 start_test Resource urls are rewritten to include experiment indexes.
 http_proxy=$SECONDARY_HOSTNAME \
-  WGET_ARGS="--header 'Cookie:_GFURIOUS=7'" fetch_until $EXP_EXTEND_CACHE \
+  WGET_ARGS="--header 'Cookie:PageSpeedExperiment=7'" fetch_until $EXP_EXTEND_CACHE \
     "fgrep -c .pagespeed.a.ic." 1
 http_proxy=$SECONDARY_HOSTNAME \
-  WGET_ARGS="--header 'Cookie:_GFURIOUS=2'" fetch_until $EXP_EXTEND_CACHE \
+  WGET_ARGS="--header 'Cookie:PageSpeedExperiment=2'" fetch_until $EXP_EXTEND_CACHE \
     "fgrep -c .pagespeed.b.ic." 1
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: _GFURIOUS=7' \
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=7' \
       $EXP_EXTEND_CACHE)
 check_from "$OUT" fgrep ".pagespeed.a.ic."
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: _GFURIOUS=2' \
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=2' \
       $EXP_EXTEND_CACHE)
 check_from "$OUT" fgrep ".pagespeed.b.ic."
 
@@ -1350,32 +1467,32 @@ start_test Images are different when the url specifies different experiments.
 
 IMG_A="$EXP_EXAMPLE/images/xPuzzle.jpg.pagespeed.a.ic.fakehash.jpg"
 IMG_B="$EXP_EXAMPLE/images/xPuzzle.jpg.pagespeed.b.ic.fakehash.jpg"
-http_proxy=$SECONDARY_HOSTNAME fetch_until $IMG_A 'wc -c' 231192
-http_proxy=$SECONDARY_HOSTNAME fetch_until $IMG_B 'wc -c' 216942
+http_proxy=$SECONDARY_HOSTNAME fetch_until $IMG_A 'wc -c' 102902 "" -le
+http_proxy=$SECONDARY_HOSTNAME fetch_until $IMG_B 'wc -c'  98276 "" -le
 
 start_test Analytics javascript is added for the experimental group.
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: _GFURIOUS=2' \
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=2' \
       $EXP_EXTEND_CACHE)
 check_from "$OUT" fgrep -q 'Experiment: 2'
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: _GFURIOUS=7' \
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=7' \
       $EXP_EXTEND_CACHE)
 check_from "$OUT" fgrep -q 'Experiment: 7'
 
 start_test Analytics javascript is not added for the no-experiment group.
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: _GFURIOUS=0' \
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=0' \
       $EXP_EXTEND_CACHE)
 check_not_from "$OUT" fgrep -q 'Experiment:'
 
 start_test Analytics javascript is not added for any group with Analytics off.
 EXP_NO_GA_EXTEND_CACHE="http://experiment.noga.example.com"
 EXP_NO_GA_EXTEND_CACHE+="/mod_pagespeed_example/extend_cache.html"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: _GFURIOUS=2' \
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=2' \
       $EXP_NO_GA_EXTEND_CACHE)
 check_not_from "$OUT" fgrep -q 'Experiment:'
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: _GFURIOUS=7' \
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=7' \
       $EXP_NO_GA_EXTEND_CACHE)
 check_not_from "$OUT" fgrep -q 'Experiment:'
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: _GFURIOUS=0' \
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=0' \
       $EXP_NO_GA_EXTEND_CACHE)
 check_not_from "$OUT" fgrep -q 'Experiment:'
 
@@ -1397,6 +1514,12 @@ function check_failures_and_exit() {
   exit 0
 }
 
+start_test Make sure nostore on a subdirectory is retained
+URL=$TEST_ROOT/nostore/nostore.html
+HTML_HEADERS=$($WGET_DUMP $URL)
+check_from "$HTML_HEADERS" egrep -q \
+  'Cache-Control: max-age=0, no-cache, no-store'
+
 start_test Custom headers remain on resources, but cache should be 1 year.
 URL="$TEST_ROOT/compressed/hello_js.custom_ext.pagespeed.ce.HdziXmtLIV.txt"
 echo $WGET_DUMP $URL
@@ -1422,8 +1545,11 @@ BEACON_PATH=$(sed -n "s/${CALL_PAT}${CAPTURE_ARG}/\1/p" $FETCH_FILE)
 ESCAPED_URL=$(sed -n "s/${CALL_PAT}${SKIP_ARG}${CAPTURE_ARG}/\1/p" $FETCH_FILE)
 OPTIONS_HASH=$( \
   sed -n "s/${CALL_PAT}${SKIP_ARG}${SKIP_ARG}${CAPTURE_ARG}/\1/p" $FETCH_FILE)
-BEACON_URL="http://${HOSTNAME}${BEACON_PATH}"
-BEACON_DATA="url=${ESCAPED_URL}&oh=${OPTIONS_HASH}&cs=.big,.blue,.bold,.foo"
+NONCE=$( \
+  sed -n "s/${CALL_PAT}${SKIP_ARG}${SKIP_ARG}${SKIP_ARG}${CAPTURE_ARG}/\1/p" \
+  $FETCH_FILE)
+BEACON_URL="http://${HOSTNAME}${BEACON_PATH}?url=${ESCAPED_URL}"
+BEACON_DATA="oh=${OPTIONS_HASH}&n=${NONCE}&cs=.big,.blue,.bold,.foo"
 
 # See the comments about 204 responses and --no-http-keep-alive above.
 OUT=$(wget -q  --save-headers -O - --no-http-keep-alive \
@@ -1493,4 +1619,29 @@ check_from "$OUT" egrep -q "HTTP/1[.]. 204"
 http_proxy=$SECONDARY_HOSTNAME \
   fetch_until -save -recursive $URL 'fgrep -c pagespeed_lazy_src=' 1
 
+start_test keepalive with html rewriting
+keepalive_test "keepalive-html.example.com"\
+  "/mod_pagespeed_example/rewrite_images.html" ""
+
+start_test keepalive with serving resources
+keepalive_test "keepalive-resource.example.com"\
+  "/mod_pagespeed_example/combine_javascript2.js+combine_javascript1.js+combine_javascript2.js.pagespeed.jc.0.js"\
+  ""
+
+start_test keepalive with beacon get requests
+keepalive_test "keepalive-beacon-get.example.com"\
+  "/ngx_pagespeed_beacon?ets=load:13" ""
+
+BEACON_DATA="url=http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F"
+BEACON_DATA+="image_rewriting%2Frewrite_images.html"
+BEACON_DATA+="&oh=$OPTIONS_HASH&ci=2932493096"
+
+start_test keepalive with beacon post requests
+keepalive_test "keepalive-beacon-post.example.com" "/ngx_pagespeed_beacon"\
+  "$BEACON_DATA"
+
+start_test keepalive with static resources
+keepalive_test "keepalive-static.example.com"\
+  "/ngx_pagespeed_static/js_defer.0.js" ""
+
 check_failures_and_exit

test/pagespeed_test.conf.template

@@ -7,7 +7,7 @@ worker_processes  1;
 daemon @@DAEMON@@;
 master_process @@MASTER_PROCESS@@;
 
-error_log "@@TEST_TMP@@/error.log" debug;
+error_log "@@ERROR_LOG@@" debug;
 pid "@@TEST_TMP@@/nginx.pid";
 
 events {
@@ -16,7 +16,16 @@ events {
 
 
 http {
-  access_log "@@TEST_TMP@@/access.log";
+  server_names_hash_bucket_size 128;
+
+  log_format cache '$time_local '
+                   '$upstream_cache_status '
+                   '$http_host $request ($status) '
+                   '"$http_user_agent"';
+  access_log "@@ACCESS_LOG@@" cache;
+
+  proxy_cache_path "@@PROXY_CACHE@@"  levels=1:2   keys_zone=htmlcache:60m inactive=90m  max_size=50m;
+  proxy_temp_path "@@TMP_PROXY_CACHE@@";
 
   root "@@SERVER_ROOT@@";
   pagespeed UsePerVHostStatistics on;
@@ -30,6 +39,10 @@ http {
   # critical images to be inlined, so we just disable the option here.
   pagespeed CriticalImagesBeaconEnabled false;
 
+  pagespeed Statistics on;
+  pagespeed StatisticsLogging on;
+  pagespeed LogDir "@@TEST_TMP@@/logdir";
+
   server {
     # Sets up a logical home-page server on
     # max-cacheable-content-length.example.com.  This server is only used to
@@ -49,6 +62,70 @@ http {
   pagespeed UseNativeFetcher "@@NATIVE_FETCHER@@";
   @@RESOLVER@@
 
+  server {
+    # This server represents the external caching layer server which
+    # receives user requests and proxies them to the upstream server
+    # running on the PRIMARY_PORT when the response is not available in
+    # the cache. It also services purge requests from the upstream server.
+    listen @@SECONDARY_PORT@@;
+    server_name proxy_cache.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed off;
+
+    set $ua_dependent_ps_capability_list "";
+    set $bypass_cache 1;
+    if ($http_user_agent ~* "Chrome/|Firefox/|MSIE |Safari|Wget") {
+      # User Agents that support lazyload-images (ll), inline-images (ii) and
+      # defer-javascript (dj).
+      set $ua_dependent_ps_capability_list "ll,ii,dj:";
+      set $bypass_cache 0;
+    }
+    if ($http_user_agent ~*
+        "Chrome/[2][3-9]+\.|Chrome/[[3-9][0-9]+\.|Chrome/[0-9]{3,}\.") {
+      # User Agents that support lazyload-images (ll), inline-images (ii),
+      # defer-javascript (dj), webp (jw) and webp-lossless (ws).
+      set $ua_dependent_ps_capability_list "ll,ii,dj,jw,ws:";
+      set $bypass_cache 0;
+    }
+
+    # All User Agents that represent
+    # 1) mobiles
+    # 2) tablets
+    # 3) desktop browsers that do not have defer-javascript capability at a minimum
+    # are made to go to the pagespeed server directly bypassing the proxy_cache.
+    if ($http_user_agent ~* "Firefox/[1-2]\.|MSIE [5-8]\.") {
+      set $ua_dependent_ps_capability_list "";
+      set $bypass_cache 1;
+    }
+    if ($http_user_agent ~* "Mozilla.*Android.*Mobile*|iPhone|BlackBerry|Opera Mobi|Opera Mini|SymbianOS|UP.Browser|J-PHONE|Profile/MIDP|portalmmm|DoCoMo|Obigo") {
+      # These are Mobile User Agents. We don't cache responses for these.
+      set $ua_dependent_ps_capability_list "";
+      set $bypass_cache 1;
+    }
+    if ($http_user_agent ~* "Android|iPad|TouchPad|Silk-Accelerated|Kindle Fire") {
+      # These are Tablet User Agents. We don't cache responses for these.
+      set $ua_dependent_ps_capability_list "";
+      set $bypass_cache 1;
+    }
+
+    location ~ /purge(/.*) {
+      allow all;
+      proxy_cache_purge htmlcache $ua_dependent_ps_capability_list$1$is_args$args;
+    }
+
+    location /mod_pagespeed_test/cachable_rewritten_html/ {
+      proxy_pass http://localhost:@@PRIMARY_PORT@@;
+      proxy_set_header Host $host;
+      proxy_cache_valid 200 30s;
+      proxy_cache htmlcache;
+      proxy_ignore_headers Cache-Control;
+      add_header X-Cache $upstream_cache_status;
+      proxy_cache_key $ua_dependent_ps_capability_list$uri$is_args$args;
+      proxy_cache_bypass $bypass_cache;
+    }
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     server_name mpd.example.com;
@@ -390,6 +467,42 @@ http {
     pagespeed EnableFilters rewrite_images;
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name keepalive-html.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel CoreFilters;
+    error_log "@@TEST_TMP@@/keepalive-html.example.com.error.log" info;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name keepalive-resource.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    error_log "@@TEST_TMP@@/keepalive-resource.example.com.error.log" info;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name keepalive-beacon-get.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    error_log "@@TEST_TMP@@/keepalive-beacon-get.example.com.error.log" info;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name keepalive-beacon-post.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    error_log "@@TEST_TMP@@/keepalive-beacon-post.example.com.error.log" info;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name keepalive-static.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    error_log "@@TEST_TMP@@/keepalive-static.example.com.error.log" info;
+  }
+
   server {
     listen       @@PRIMARY_PORT@@;
     server_name  localhost;
@@ -399,6 +512,17 @@ http {
       add_header "" "";
     }
 
+    location /mod_pagespeed_test/cachable_rewritten_html/ {
+      # This location has the html files that will be configured to be stored
+      # in the proxy_cache layer.
+      pagespeed DownstreamCachePurgeMethod "GET";
+      pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:8051/purge";
+      # We use a very small deadline here to force the rewriting to not complete
+      # in the very first attempt.
+      pagespeed RewriteDeadlinePerFlushMs 1;
+      pagespeed RewriteLevel PassThrough;
+      pagespeed EnableFilters collapse_whitespace,extend_cache,recompress_images,convert_jpeg_to_webp,defer_javascript;
+    }
     # uncomment the following two lines if you're testing memcached
     #pagespeed MemcachedServers "localhost:11211";
     #pagespeed MemcachedThreads 1;
@@ -444,6 +568,12 @@ http {
     # As opposed to mod_pagespeed, our default for aris is 'on'
     pagespeed AvoidRenamingIntrospectiveJavascript off;
 
+    location /mod_pagespeed_test/nostore {
+      add_header "Cache-Control" "max-age=12345";
+      add_header "Cache-Control" "public, no-store";
+      add_header "Cache-Control" "max-age=14";
+    }
+
     location /mod_pagespeed_test/forbid_all_disabled/disabled {
       # Prevent the enabling of these filters for files in this directory
       # -and- all subdirectories.