Update for 1.8.31.4

Pulling in upstream merge to local master.

README.md

@@ -37,21 +37,22 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p
 
    ```bash
    $ cd ~
-   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/release-1.7.30.1-beta.zip
-   $ unzip release-1.7.30.1-beta.zip # or unzip release-1.7.30.1-beta
-   $ cd ngx_pagespeed-release-1.7.30.1-beta/
-   $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.1.tar.gz
-   $ tar -xzvf 1.7.30.1.tar.gz # expands to psol/
+   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.8.31.4-beta.zip
+   $ unzip v1.8.31.4-beta.zip # or unzip v1.8.31.4-beta
+   $ cd ngx_pagespeed-1.8.31.4-beta/
+   $ wget https://dl.google.com/dl/page-speed/psol/1.8.31.4.tar.gz
+   $ tar -xzvf 1.8.31.4.tar.gz # expands to psol/
    ```
 
 3. Download and build nginx:
 
    ```bash
+   $ cd ~
    $ # check http://nginx.org/en/download.html for the latest version
-   $ wget http://nginx.org/download/nginx-1.4.3.tar.gz
-   $ tar -xvzf nginx-1.4.3.tar.gz
-   $ cd nginx-1.4.3/
-   $ ./configure --add-module=$HOME/ngx_pagespeed-release-1.7.30.1-beta
+   $ wget http://nginx.org/download/nginx-1.6.0.tar.gz
+   $ tar -xvzf nginx-1.6.0.tar.gz
+   $ cd nginx-1.6.0/
+   $ ./configure --add-module=$HOME/ngx_pagespeed-1.8.31.4-beta
    $ make
    $ sudo make install
    ```
@@ -81,12 +82,8 @@ In every server block where pagespeed is enabled add:
 #  Ensure requests for pagespeed optimized resources go to the pagespeed
 #  handler and no extraneous headers get set.
 location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" { add_header "" ""; }
-location ~ "^/ngx_pagespeed_static/" { }
+location ~ "^/pagespeed_static/" { }
 location ~ "^/ngx_pagespeed_beacon$" { }
-location /ngx_pagespeed_statistics { allow 127.0.0.1; deny all; }
-location /ngx_pagespeed_global_statistics { allow 127.0.0.1; deny all; }
-location /ngx_pagespeed_message { allow 127.0.0.1; deny all; }
-location /pagespeed_console { allow 127.0.0.1; deny all; }
 ```
 
 To confirm that the module is loaded, fetch a page and check that you see the
@@ -94,7 +91,7 @@ To confirm that the module is loaded, fetch a page and check that you see the
 
 ```bash
 $ curl -I 'http://localhost:8050/some_page/' | grep X-Page-Speed
-X-Page-Speed: 1.7.30.1-...
+X-Page-Speed: 1.8.31.4-...
 ```
 
 Looking at the source of a few pages you should see various changes, such as

config

@@ -27,8 +27,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
     echo "   You need to separately download the pagespeed library:"
     echo ""
     echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.1.tar.gz"
-    echo "     $ tar -xzvf 1.7.30.1.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.8.31.4.tar.gz"
+    echo "     $ tar -xzvf 1.8.31.4.tar.gz # expands to psol/"
     echo ""
     echo "   Or see the installation instructions:"
     echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
@@ -89,9 +89,7 @@ if [ "$uname_arch" = "i686" ]; then
   FLAG_MARCH='-march=i686'
 fi
 
-# Building with HTTPS fetching enabled pulls in a version of OpenSSL that causes
-# linker errors, so disable it here.
-CFLAGS="$CFLAGS -DSERF_HTTPS_FETCHING=0 $FLAG_MARCH"
+CFLAGS="$CFLAGS $FLAG_MARCH"
 
 case "$NGX_GCC_VER" in
   4.8*)
@@ -111,6 +109,10 @@ case "$NGX_GCC_VER" in
   ;;
 esac
 
+if [ "$WNO_ERROR" = "YES" ]; then
+    CFLAGS="$CFLAGS -Wno-error"
+fi
+
 pagespeed_include="\
   $mod_pagespeed_dir \
   $mod_pagespeed_dir/third_party/chromium/src \
@@ -128,17 +130,10 @@ ngx_feature_path="$pagespeed_include"
 
 if $build_from_source ; then
   psol_library_binaries="\
-    $mod_pagespeed_dir/net/instaweb/automatic/pagespeed_automatic.a \
-    $mod_pagespeed_dir/out/$buildtype/obj.target/third_party/serf/libserf.a \
-    $mod_pagespeed_dir/out/$buildtype/obj.target/third_party/aprutil/libaprutil.a \
-    $mod_pagespeed_dir/out/$buildtype/obj.target/third_party/apr/libapr.a"
+    $mod_pagespeed_dir/net/instaweb/automatic/pagespeed_automatic.a"
 else
   psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
-  psol_library_binaries="\
-    $psol_library_dir/pagespeed_automatic.a \
-    $psol_library_dir/libserf.a \
-    $psol_library_dir/libaprutil.a \
-    $psol_library_dir/libapr.a"
+  psol_library_binaries="$psol_library_dir/pagespeed_automatic.a"
 fi
 
 pagespeed_libs="-lstdc++ $psol_library_binaries -lrt -pthread -lm"
@@ -190,13 +185,7 @@ if [ $ngx_found = yes ]; then
     $ps_src/ngx_rewrite_driver_factory.cc \
     $ps_src/ngx_rewrite_options.cc \
     $ps_src/ngx_server_context.cc \
-    $ps_src/ngx_url_async_fetcher.cc \
-    $mod_pagespeed_dir/out/$buildtype/obj/gen/data2c_out/instaweb/net/instaweb/apache/install/mod_pagespeed_example/mod_pagespeed_console_out.cc \
-    $mod_pagespeed_dir/out/$buildtype/obj/gen/data2c_out/instaweb/net/instaweb/apache/install/mod_pagespeed_example/mod_pagespeed_console_css_out.cc \
-    $mod_pagespeed_dir/out/$buildtype/obj/gen/data2c_out/instaweb/net/instaweb/apache/install/mod_pagespeed_example/mod_pagespeed_console_html_out.cc \
-    $mod_pagespeed_dir/net/instaweb/system/add_headers_fetcher.cc \
-    $mod_pagespeed_dir/net/instaweb/system/loopback_route_fetcher.cc \
-    $mod_pagespeed_dir/net/instaweb/system/serf_url_async_fetcher.cc"
+    $ps_src/ngx_url_async_fetcher.cc"
 
   # Make pagespeed run immediately before gzip.
   HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
@@ -206,9 +195,11 @@ if [ $ngx_found = yes ]; then
     sed "s/$HTTP_GZIP_FILTER_MODULE/ngx_pagespeed_etag_filter $HTTP_GZIP_FILTER_MODULE/")
   CORE_LIBS="$CORE_LIBS $pagespeed_libs"
   CORE_INCS="$CORE_INCS $pagespeed_include"
+  echo "List of modules (in reverse order of applicability): "$HTTP_FILTER_MODULES
 else
   cat << END
-$0: error: module ngx_pagespeed requires the pagespeed optimization library
+$0: error: module ngx_pagespeed requires the pagespeed optimization library.
+Look in obj/autoconf.err for more details.
 END
   exit 1
 fi

src/ngx_base_fetch.cc

@@ -106,11 +106,10 @@ ngx_int_t NgxBaseFetch::CollectAccumulatedWrites(ngx_chain_t** link_ptr) {
 ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
   const ResponseHeaders* pagespeed_headers = response_headers();
 
-  // TODO(chaizhenhua): Add and check.
-  // if (content_length_known()) {
-  //   headers_out->content_length = NULL;
-  //   headers_out->content_length_n = content_length();
-  // }
+  if (content_length_known()) {
+     headers_out->content_length = NULL;
+     headers_out->content_length_n = content_length();
+  }
 
   return copy_response_headers_to_ngx(request_, *pagespeed_headers,
                                       preserve_caching_headers_);

src/ngx_fetch.cc

@@ -24,6 +24,10 @@
 //  - The read handler parses the response. Add the response to the buffer at
 //    last.
 
+extern "C" {
+#include <nginx.h>
+}
+
 #include "ngx_fetch.h"
 #include "net/instaweb/util/public/basictypes.h"
 #include "base/logging.h"
@@ -66,12 +70,14 @@ namespace net_instaweb {
         fetch_start_ms_(0),
         fetch_end_ms_(0),
         done_(false),
-        content_length_(0) {
-            ngx_memzero(&url_, sizeof(url_));
-            log_ = log;
-            pool_ = NULL;
-            timeout_event_ = NULL;
-            connection_ = NULL;
+        content_length_(-1),
+        content_length_known_(false),
+        resolver_ctx_(NULL) {
+    ngx_memzero(&url_, sizeof(url_));
+    log_ = log;
+    pool_ = NULL;
+    timeout_event_ = NULL;
+    connection_ = NULL;
   }
 
   NgxFetch::~NgxFetch() {
@@ -138,19 +144,26 @@ namespace net_instaweb {
     // The host is either a domain name or an IP address.  First check
     // if it's a valid IP address and only if that fails fall back to
     // using the DNS resolver.
-    GoogleString s_ipaddress(reinterpret_cast<char*>(url_.host.data),
-                             url_.host.len);
+
+    // Maybe we have a Proxy.
+    ngx_url_t* tmp_url = &url_;
+    if (0 != fetcher_->proxy_.url.len) {
+      tmp_url = &fetcher_->proxy_;
+    }
+
+    GoogleString s_ipaddress(reinterpret_cast<char*>(tmp_url->host.data),
+                             tmp_url->host.len);
     ngx_memzero(&sin_, sizeof(sin_));
     sin_.sin_family = AF_INET;
-    sin_.sin_port = htons(url_.port);
+    sin_.sin_port = htons(tmp_url->port);
     sin_.sin_addr.s_addr = inet_addr(s_ipaddress.c_str());
 
     if (sin_.sin_addr.s_addr == INADDR_NONE) {
       // inet_addr returned INADDR_NONE, which means the hostname
       // isn't a valid IP address.  Check DNS.
       ngx_resolver_ctx_t temp;
-      temp.name.data = url_.host.data;
-      temp.name.len = url_.host.len;
+      temp.name.data = tmp_url->host.data;
+      temp.name.len = tmp_url->host.len;
       resolver_ctx_ = ngx_resolve_start(fetcher_->resolver_, &temp);
       if (resolver_ctx_ == NULL || resolver_ctx_ == NGX_NO_RESOLVER) {
         // TODO(oschaaf): this spams the log, but is useful in the fetcher's
@@ -162,9 +175,13 @@ namespace net_instaweb {
       }
 
       resolver_ctx_->data = this;
-      resolver_ctx_->name.data = url_.host.data;
-      resolver_ctx_->name.len = url_.host.len;
+      resolver_ctx_->name.data = tmp_url->host.data;
+      resolver_ctx_->name.len = tmp_url->host.len;
+
+#if (nginx_version < 1005008)
       resolver_ctx_->type = NGX_RESOLVE_A;
+#endif
+
       resolver_ctx_->handler = NgxFetchResolveDone;
       resolver_ctx_->timeout = fetcher_->resolver_timeout_;
 
@@ -197,6 +214,8 @@ namespace net_instaweb {
       return;
     }
 
+    release_resolver();
+
     if (timeout_event_ && timeout_event_->timer_set) {
       ngx_del_timer(timeout_event_);
       timeout_event_ = NULL;
@@ -255,37 +274,14 @@ namespace net_instaweb {
       return false;
     }
     str_url_.copy(reinterpret_cast<char*>(url_.url.data), str_url_.length(), 0);
-    size_t scheme_offset;
-    u_short port;
-    if (ngx_strncasecmp(url_.url.data, reinterpret_cast<u_char*>(
-                                     const_cast<char*>("http://")), 7) == 0) {
-      scheme_offset = 7;
-      port = 80;
-    } else if (ngx_strncasecmp(url_.url.data, reinterpret_cast<u_char*>(
-                                    const_cast<char*>("https://")), 8) == 0) {
-      scheme_offset = 8;
-      port = 443;
-    } else {
-      scheme_offset = 0;
-      port = 80;
-    }
 
-    url_.url.data += scheme_offset;
-    url_.url.len -= scheme_offset;
-    url_.default_port = port;
-    // See: http://lxr.evanmiller.org/http/source/core/ngx_inet.c#L875
-    url_.no_resolve = 0;
-    url_.uri_part = 1;
-
-    if (ngx_parse_url(pool_, &url_) == NGX_OK) {
-      return true;
-    }
-    return false;
+    return NgxUrlAsyncFetcher::ParseUrl(&url_, pool_);
   }
 
   // Issue a request after the resolver is done
   void NgxFetch::NgxFetchResolveDone(ngx_resolver_ctx_t* resolver_ctx) {
     NgxFetch* fetch = static_cast<NgxFetch*>(resolver_ctx->data);
+    NgxUrlAsyncFetcher* fetcher = fetch->fetcher_;
     if (resolver_ctx->state != NGX_OK) {
       if (fetch->timeout_event() != NULL && fetch->timeout_event()->timer_set) {
         ngx_del_timer(fetch->timeout_event());
@@ -295,13 +291,28 @@ namespace net_instaweb {
           kWarning, "NgxFetch: failed to resolve host [%.*s]",
           static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data);
       fetch->CallbackDone(false);
-      ngx_resolve_name_done(resolver_ctx);
       return;
     }
     ngx_memzero(&fetch->sin_, sizeof(fetch->sin_));
+
+#if (nginx_version < 1005008)
+    fetch->sin_.sin_addr.s_addr = resolver_ctx->addrs[0];
+#else
+
+    struct sockaddr_in *sin;
+    sin = reinterpret_cast<struct sockaddr_in *>(
+        resolver_ctx->addrs[0].sockaddr);
+    fetch->sin_.sin_family = sin->sin_family;
+    fetch->sin_.sin_addr.s_addr = sin->sin_addr.s_addr;
+#endif
+
     fetch->sin_.sin_family = AF_INET;
     fetch->sin_.sin_port = htons(fetch->url_.port);
-    fetch->sin_.sin_addr.s_addr = resolver_ctx->addrs[0];
+
+    // Maybe we have Proxy
+    if (0 != fetcher->proxy_.url.len) {
+      fetch->sin_.sin_port = htons(fetcher->proxy_.port);
+    }
 
     char* ip_address = inet_ntoa(fetch->sin_.sin_addr);
 
@@ -310,7 +321,7 @@ namespace net_instaweb {
         static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data,
         ip_address);
 
-    ngx_resolve_name_done(resolver_ctx);
+    fetch->release_resolver();
 
     if (fetch->InitRequest() != NGX_OK) {
       fetch->message_handler()->Message(kError, "NgxFetch: InitRequest failed");
@@ -333,7 +344,13 @@ namespace net_instaweb {
     bool have_host = false;
     GoogleString port;
 
-    size = sizeof("GET ") - 1 + url_.uri.len + sizeof(" HTTP/1.0\r\n") - 1;
+    const char* method = request_headers->method_string();
+    size_t method_len = strlen(method);
+
+    size = (method_len +
+            1 /* for the space */ +
+            url_.uri.len +
+            sizeof(" HTTP/1.0\r\n") - 1);
 
     for (int i = 0; i < request_headers->NumAttributes(); i++) {
       // if no explicit host header is given in the request headers,
@@ -361,7 +378,8 @@ namespace net_instaweb {
       return NGX_ERROR;
     }
 
-    out_->last = ngx_cpymem(out_->last, "GET ", 4);
+    out_->last = ngx_cpymem(out_->last, method, method_len);
+    out_->last = ngx_cpymem(out_->last, " ", 1);
     out_->last = ngx_cpymem(out_->last, url_.uri.data, url_.uri.len);
     out_->last = ngx_cpymem(out_->last, " HTTP/1.0\r\n", 11);
 
@@ -469,9 +487,16 @@ namespace net_instaweb {
       }
 
       if (n == 0) {
-        // connection is closed prematurely by remote server,
-        // or the content-length was 0
-        fetch->CallbackDone(fetch->content_length_ == 0);
+        // If the content length was not known, we assume that we have read
+        // all if we at least parsed the headers.
+        // If we do know the content length, having a mismatch on the bytes read
+        // will be interpreted as an error.
+        if (fetch->content_length_known_) {
+          fetch->CallbackDone(fetch->content_length_ == fetch->bytes_received_);
+        } else {
+          fetch->CallbackDone(fetch->parser_.headers_complete());
+        }
+
         return;
       } else if (n > 0) {
         fetch->in_->pos = fetch->in_->start;
@@ -538,13 +563,21 @@ namespace net_instaweb {
     if (n > size) {
       return false;
     } else if (fetch->parser_.headers_complete()) {
-      int64 content_length = -1;
-      fetch->async_fetch_->response_headers()->FindContentLength(
-          &content_length);
-      fetch->content_length_ = content_length;
-      if (fetch->fetcher_->track_original_content_length()) {
+      if (fetch->async_fetch_->response_headers()->FindContentLength(
+              &fetch->content_length_)) {
+        if (fetch->content_length_ < 0) {
+          fetch->message_handler_->Message(
+              kError, "Negative content-length in response header");
+          return false;
+        } else {
+          fetch->content_length_known_ = true;
+        }
+      }
+
+      if (fetch->fetcher_->track_original_content_length()
+         && fetch->content_length_known_) {
         fetch->async_fetch_->response_headers()->SetOriginalContentLength(
-            content_length);
+            fetch->content_length_);
       }
 
       fetch->in_->pos += n;
@@ -563,11 +596,12 @@ namespace net_instaweb {
       return true;
     }
 
-    fetch->bytes_received_add(static_cast<int64>(size));
+    fetch->bytes_received_add(size);
+
     if (fetch->async_fetch_->Write(StringPiece(data, size),
         fetch->message_handler())) {
-      fetch->content_length_ -= size;
-      if (fetch->content_length_ <= 0) {
+      if (fetch->content_length_known_ &&
+          fetch->bytes_received_ == fetch->content_length_) {
         fetch->done_ = true;
       }
       return true;

src/ngx_fetch.h

@@ -83,6 +83,12 @@ class NgxFetch : public PoolElement<NgxFetch> {
   void set_timeout_event(ngx_event_t* x) {
     timeout_event_ = x;
   }
+  void release_resolver() {
+    if (resolver_ctx_ != NULL && resolver_ctx_ != NGX_NO_RESOLVER) {
+      ngx_resolve_name_done(resolver_ctx_);
+      resolver_ctx_ = NULL;
+    }
+  }
 
  private:
   response_handler_pt response_handler;
@@ -121,12 +127,13 @@ class NgxFetch : public PoolElement<NgxFetch> {
   AsyncFetch* async_fetch_;
   ResponseHeadersParser parser_;
   MessageHandler* message_handler_;
-  size_t bytes_received_;
+  int64 bytes_received_;
   int64 fetch_start_ms_;
   int64 fetch_end_ms_;
   int64 timeout_ms_;
   bool done_;
   int64 content_length_;
+  bool content_length_known_;
 
   struct sockaddr_in sin_;
   ngx_log_t* log_;

src/ngx_message_handler.cc

@@ -18,13 +18,13 @@
 
 #include <signal.h>
 
-#include "apr_time.h"
-
 #include "net/instaweb/util/public/abstract_mutex.h"
 #include "net/instaweb/util/public/debug.h"
 #include "net/instaweb/util/public/shared_circular_buffer.h"
 #include "net/instaweb/util/public/string_util.h"
 #include "net/instaweb/public/version.h"
+#include "pagespeed/kernel/base/posix_timer.h"
+#include "pagespeed/kernel/base/time_util.h"
 
 namespace {
 
@@ -118,10 +118,9 @@ void NgxMessageHandler::MessageVImpl(MessageType type, const char* msg,
   // Prepend time and severity to message.
   // Format is [time] [severity] [pid] message.
   GoogleString message;
-  char time_buffer[APR_CTIME_LEN + 1];
-  const char* time = time_buffer;
-  apr_status_t status = apr_ctime(time_buffer, apr_time_now());
-  if (status != APR_SUCCESS) {
+  GoogleString time;
+  PosixTimer timer;
+  if (!ConvertTimeToString(timer.NowMs(), &time)) {
     time = "?";
   }
   StrAppend(&message, "[", time, "] ",
@@ -149,14 +148,4 @@ void NgxMessageHandler::FileMessageVImpl(MessageType type, const char* file,
   }
 }
 
-// TODO(sligocki): It'd be nice not to do so much string copying.
-GoogleString NgxMessageHandler::Format(const char* msg, va_list args) {
-  GoogleString buffer;
-
-  // Ignore the name of this routine: it formats with vsnprintf.
-  // See base/stringprintf.cc.
-  StringAppendV(&buffer, msg, args);
-  return buffer;
-}
-
 }  // namespace net_instaweb

src/ngx_message_handler.h

@@ -70,7 +70,6 @@ class NgxMessageHandler : public GoogleMessageHandler {
 
  private:
   ngx_uint_t GetNgxLogLevel(MessageType type);
-  GoogleString Format(const char* msg, va_list args);
 
   scoped_ptr<AbstractMutex> mutex_;
   GoogleString pid_string_;

src/ngx_pagespeed.h

@@ -103,6 +103,10 @@ typedef struct {
   RewriteDriver* driver;
   InPlaceResourceRecorder* recorder;
   ResponseHeaders* ipro_response_headers;
+
+  // We need to remember the URL here as well since we may modify what NGX
+  // gets by stripping our special query params and honoring X-Forwarded-Proto.
+  GoogleString url_string;
 } ps_request_ctx_t;
 
 
@@ -117,6 +121,8 @@ ngx_int_t copy_response_headers_to_ngx(
     const ResponseHeaders& pagespeed_headers,
     PreserveCachingHeaders preserve_caching_headers);
 
+StringPiece ps_determine_host(ngx_http_request_t* r);
+
 }  // namespace net_instaweb
 
 #endif  // NGX_PAGESPEED_H_

src/ngx_rewrite_driver_factory.cc

@@ -33,7 +33,6 @@
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
 #include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
 #include "net/instaweb/rewriter/public/server_context.h"
-#include "net/instaweb/rewriter/public/static_asset_manager.h"
 #include "net/instaweb/system/public/in_place_resource_recorder.h"
 #include "net/instaweb/system/public/serf_url_async_fetcher.h"
 #include "net/instaweb/system/public/system_caches.h"
@@ -63,14 +62,12 @@ class UrlAsyncFetcher;
 class UrlFetcher;
 class Writer;
 
-const char NgxRewriteDriverFactory::kStaticAssetPrefix[] =
-    "/ngx_pagespeed_static/";
-
 class SharedCircularBuffer;
 
 NgxRewriteDriverFactory::NgxRewriteDriverFactory(
+    const ProcessContext& process_context,
     SystemThreadSystem* system_thread_system, StringPiece hostname, int port)
-    : SystemRewriteDriverFactory(system_thread_system,
+    : SystemRewriteDriverFactory(process_context, system_thread_system,
         NULL /* default shared memory runtime */, hostname, port),
       main_conf_(NULL),
       threads_started_(false),
@@ -82,7 +79,9 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
       log_(NULL),
       resolver_timeout_(NGX_CONF_UNSET_MSEC),
       use_native_fetcher_(false),
-      ngx_shared_circular_buffer_(NULL) {
+      ngx_shared_circular_buffer_(NULL),
+      hostname_(hostname.as_string()),
+      port_(port) {
   InitializeDefaultOptions();
   default_options()->set_beacon_url("/ngx_pagespeed_beacon");
   SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
@@ -148,11 +147,6 @@ RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
   return options;
 }
 
-void NgxRewriteDriverFactory::InitStaticAssetManager(
-    StaticAssetManager* static_asset_manager) {
-  static_asset_manager->set_library_url_prefix(kStaticAssetPrefix);
-}
-
 bool NgxRewriteDriverFactory::InitNgxUrlAsyncFetchers() {
   log_ = ngx_cycle->log;
   for (size_t i = 0; i < ngx_url_async_fetchers_.size(); ++i) {
@@ -177,6 +171,12 @@ NgxServerContext* NgxRewriteDriverFactory::MakeNgxServerContext(
   return server_context;
 }
 
+ServerContext* NgxRewriteDriverFactory::NewDecodingServerContext() {
+  ServerContext* sc = new NgxServerContext(this, hostname_, port_);
+  InitStubDecodingServerContext(sc);
+  return sc;
+}
+
 ServerContext* NgxRewriteDriverFactory::NewServerContext() {
   LOG(DFATAL) << "MakeNgxServerContext should be used instead";
   return NULL;
@@ -218,6 +218,8 @@ void NgxRewriteDriverFactory::LoggingInit(ngx_log_t* log) {
 void NgxRewriteDriverFactory::SetCircularBuffer(
     SharedCircularBuffer* buffer) {
   ngx_shared_circular_buffer_ = buffer;
+  ngx_message_handler_->set_buffer(buffer);
+  ngx_html_parse_message_handler_->set_buffer(buffer);
 }
 
 void NgxRewriteDriverFactory::SetServerContextMessageHandler(

src/ngx_rewrite_driver_factory.h

@@ -28,7 +28,6 @@ extern "C" {
 
 #include <set>
 
-#include "apr_pools.h"
 #include "net/instaweb/system/public/system_rewrite_driver_factory.h"
 #include "net/instaweb/util/public/md5_hasher.h"
 #include "net/instaweb/util/public/scoped_ptr.h"
@@ -46,16 +45,14 @@ class NgxUrlAsyncFetcher;
 class SharedCircularBuffer;
 class SharedMemRefererStatistics;
 class SlowWorker;
-class StaticAssetManager;
 class Statistics;
 class SystemThreadSystem;
 
 class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  public:
-  static const char kStaticAssetPrefix[];
-
   // We take ownership of the thread system.
   explicit NgxRewriteDriverFactory(
+      const ProcessContext& process_context,
       SystemThreadSystem* system_thread_system, StringPiece hostname, int port);
   virtual ~NgxRewriteDriverFactory();
   virtual Hasher* NewHasher();
@@ -68,9 +65,7 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   // Create a new RewriteOptions.  In this implementation it will be an
   // NgxRewriteOptions.
   virtual RewriteOptions* NewRewriteOptions();
-  // Initializes the StaticAssetManager.
-  virtual void InitStaticAssetManager(
-      StaticAssetManager* static_asset_manager);
+  virtual ServerContext* NewDecodingServerContext();
   bool InitNgxUrlAsyncFetchers();
   // Check resolver configured or not.
   bool CheckResolver();
@@ -80,7 +75,7 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   // platform-independent statistics.
   static void InitStats(Statistics* statistics);
   NgxServerContext* MakeNgxServerContext(StringPiece hostname, int port);
-  ServerContext* NewServerContext();
+  virtual ServerContext* NewServerContext();
 
   // Starts pagespeed threads if they've not been started already.  Must be
   // called after the caller has finished any forking it intends to do.
@@ -169,6 +164,9 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   // TODO(jefftk): merge the nginx and apache ways of doing this.
   SharedCircularBuffer* ngx_shared_circular_buffer_;
 
+  GoogleString hostname_;
+  int port_;
+
   DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
 };
 

src/ngx_rewrite_options.cc

@@ -37,7 +37,47 @@ namespace net_instaweb {
 
 namespace {
 
-const char kNgxPagespeedStatisticsHandlerPath[] = "/ngx_pagespeed_statistics";
+const char kStatisticsPath[] = "StatisticsPath";
+const char kGlobalStatisticsPath[] = "GlobalStatisticsPath";
+const char kConsolePath[] = "ConsolePath";
+const char kMessagesPath[] = "MessagesPath";
+const char kAdminPath[] = "AdminPath";
+const char kGlobalAdminPath[] = "GlobalAdminPath";
+
+// These options are copied from mod_instaweb.cc, where APACHE_CONFIG_OPTIONX
+// indicates that they can not be set at the directory/location level. They set
+// options in the RewriteDriverFactory, so they're entirely global and do not
+// appear in RewriteOptions.  They are not alphabetized on purpose, but rather
+// left in the same order as in mod_instaweb.cc in case we end up needing to
+// compare.
+// TODO(oschaaf): this duplication is a short term solution.
+const char* const server_only_options[] = {
+  "FetcherTimeoutMs",
+  "FetchProxy",
+  "ForceCaching",
+  "GeneratedFilePrefix",
+  "ImgMaxRewritesAtOnce",
+  "InheritVHostConfig",
+  "InstallCrashHandler",
+  "MessageBufferSize",
+  "NumRewriteThreads",
+  "NumExpensiveRewriteThreads",
+  "StaticAssetPrefix",
+  "TrackOriginalContentLength",
+  "UsePerVHostStatistics",  // TODO(anupama): What to do about "No longer used"
+  "BlockingRewriteRefererUrls",
+  "CreateSharedMemoryMetadataCache",
+  "LoadFromFile",
+  "LoadFromFileMatch",
+  "LoadFromFileRule",
+  "LoadFromFileRuleMatch",
+  "UseNativeFetcher"
+};
+
+// Options that can only be used in the main (http) option scope.
+const char* const main_only_options[] = {
+  "UseNativeFetcher"
+};
 
 }  // namespace
 
@@ -58,15 +98,29 @@ void NgxRewriteOptions::Init() {
   DCHECK(ngx_properties_ != NULL)
       << "Call NgxRewriteOptions::Initialize() before construction";
   InitializeOptions(ngx_properties_);
-
-  // Nginx-specific default.
-  // TODO(sligocki): Get rid of this line and let both Apache and Nginx use
-  // /pagespeed_statistics as the handler.
-  statistics_handler_path_.set_default(kNgxPagespeedStatisticsHandlerPath);
 }
 
 void NgxRewriteOptions::AddProperties() {
-  // Nothing ngx-specific for now.
+  // Nginx-specific options.
+  add_ngx_option(
+      "", &NgxRewriteOptions::statistics_path_, "nsp", kStatisticsPath,
+      kServerScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics");
+  add_ngx_option(
+      "", &NgxRewriteOptions::global_statistics_path_, "ngsp",
+      kGlobalStatisticsPath, kProcessScope,
+      "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics");
+  add_ngx_option(
+      "", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kServerScope,
+      "Set the console path. Ex: /pagespeed_console");
+  add_ngx_option(
+      "", &NgxRewriteOptions::messages_path_, "nmp", kMessagesPath,
+      kServerScope, "Set the messages path.  Ex: /ngx_pagespeed_message");
+  add_ngx_option(
+      "", &NgxRewriteOptions::admin_path_, "nap", kAdminPath,
+      kServerScope, "Set the admin path.  Ex: /pagespeed_admin");
+  add_ngx_option(
+      "", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
+      kProcessScope, "Set the global admin path.  Ex: /pagespeed_global_admin");
 
   MergeSubclassProperties(ngx_properties_);
 
@@ -94,6 +148,39 @@ bool NgxRewriteOptions::IsDirective(StringPiece config_directive,
   return StringCaseEqual(config_directive, compare_directive);
 }
 
+RewriteOptions::OptionScope NgxRewriteOptions::GetOptionScope(
+    StringPiece option_name) {
+  ngx_uint_t i;
+  ngx_uint_t size = sizeof(main_only_options) / sizeof(char*);
+  for (i = 0; i < size; i++) {
+    if (StringCaseEqual(main_only_options[i], option_name)) {
+      return kProcessScopeStrict;
+    }
+  }
+
+  size = sizeof(server_only_options) / sizeof(char*);
+  for (i = 0; i < size; i++) {
+    if (StringCaseEqual(server_only_options[i], option_name)) {
+      return kServerScope;
+    }
+  }
+
+  // This could be made more efficient if RewriteOptions provided a map allowing
+  // access of options by their name. It's not too much of a worry at present
+  // since this is just during initialization.
+  for (OptionBaseVector::const_iterator it = all_options().begin();
+       it != all_options().end(); ++it) {
+    RewriteOptions::OptionBase* option = *it;
+    if (option->option_name() == option_name) {
+      // We treat kProcessScope as kProcessScopeStrict, failing to start if an
+      // option is out of place.
+      return option->scope() == kProcessScope ? kProcessScopeStrict
+                                              : option->scope();
+    }
+  }
+  return kDirectoryScope;
+}
+
 RewriteOptions::OptionSettingResult NgxRewriteOptions::ParseAndSetOptions0(
     StringPiece directive, GoogleString* msg, MessageHandler* handler) {
   if (IsDirective(directive, "on")) {
@@ -144,10 +231,26 @@ RewriteOptions::OptionSettingResult ParseAndSetOptionHelper(
   return RewriteOptions::kOptionOk;
 }
 
+namespace {
+
+const char* ps_error_string_for_option(
+    ngx_pool_t* pool, StringPiece directive, StringPiece warning) {
+  GoogleString msg =
+      StrCat("\"", directive, "\" ", warning);
+  char* s = string_piece_to_pool_string(pool, msg);
+  if (s == NULL) {
+    return "failed to allocate memory";
+  }
+  return s;
+}
+
+}  // namespace
+
 // Very similar to apache/mod_instaweb::ParseDirective.
 const char* NgxRewriteOptions::ParseAndSetOptions(
     StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
-    NgxRewriteDriverFactory* driver_factory) {
+    NgxRewriteDriverFactory* driver_factory,
+    RewriteOptions::OptionScope scope) {
   CHECK_GE(n_args, 1);
 
   StringPiece directive = args[0];
@@ -158,6 +261,11 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
     directive.remove_prefix(mod_pagespeed.size());
   }
 
+  if (GetOptionScope(directive) > scope) {
+    return ps_error_string_for_option(
+        pool, directive, "cannot be set at this scope.");
+  }
+
   GoogleString msg;
   OptionSettingResult result;
   if (n_args == 1) {
@@ -203,8 +311,11 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
       result = ParseAndSetOptionHelper<SystemRewriteDriverFactory>(
           arg, driver_factory,
           &SystemRewriteDriverFactory::set_track_original_content_length);
+    } else if (IsDirective(directive, "StaticAssetPrefix")) {
+      driver_factory->set_static_asset_prefix(arg);
+      result = RewriteOptions::kOptionOk;
     } else {
-      result = ParseAndSetOptionFromName1(directive, args[1], &msg, handler);
+      result = ParseAndSetOptionFromName1(directive, arg, &msg, handler);
     }
   } else if (n_args == 3) {
     // Short-term special handling, until this moves to common code.
@@ -227,25 +338,22 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
     result = ParseAndSetOptionFromName3(
         directive, args[1], args[2], args[3], &msg, handler);
   } else {
-    return "unknown option";
+    return ps_error_string_for_option(
+        pool, directive, "not recognized or too many arguments");
   }
 
   switch (result) {
     case RewriteOptions::kOptionOk:
       return NGX_CONF_OK;
     case RewriteOptions::kOptionNameUnknown:
-      return "unknown option";
+      return ps_error_string_for_option(
+          pool, directive, "not recognized or too many arguments");
     case RewriteOptions::kOptionValueInvalid: {
-      GoogleString full_directive = "\"";
+      GoogleString full_directive;
       for (int i = 0 ; i < n_args ; i++) {
         StrAppend(&full_directive, i == 0 ? "" : " ", args[i]);
       }
-      StrAppend(&full_directive, "\": ", msg);
-      char* s = string_piece_to_pool_string(pool, full_directive);
-      if (s == NULL) {
-        return "failed to allocate memory";
-      }
-      return s;
+      return ps_error_string_for_option(pool, full_directive, msg);
     }
   }
 

src/ngx_rewrite_options.h

@@ -40,7 +40,8 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   static void Initialize();
   static void Terminate();
 
-  NgxRewriteOptions(const StringPiece& description, ThreadSystem* thread_system);
+  NgxRewriteOptions(const StringPiece& description,
+                    ThreadSystem* thread_system);
   explicit NgxRewriteOptions(ThreadSystem* thread_system);
   virtual ~NgxRewriteOptions() { }
 
@@ -57,7 +58,7 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   // pool is a memory pool for allocating error strings.
   const char* ParseAndSetOptions(
       StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
-      NgxRewriteDriverFactory* driver_factory);
+      NgxRewriteDriverFactory* driver_factory, OptionScope scope);
 
   // Make an identical copy of these options and return it.
   virtual NgxRewriteOptions* Clone() const;
@@ -67,6 +68,24 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   static const NgxRewriteOptions* DynamicCast(const RewriteOptions* instance);
   static NgxRewriteOptions* DynamicCast(RewriteOptions* instance);
 
+  const GoogleString& statistics_path() const {
+    return statistics_path_.value();
+  }
+  const GoogleString& global_statistics_path() const {
+    return global_statistics_path_.value();
+  }
+  const GoogleString& console_path() const {
+    return console_path_.value();
+  }
+  const GoogleString& messages_path() const {
+    return messages_path_.value();
+  }
+  const GoogleString& admin_path() const {
+    return admin_path_.value();
+  }
+  const GoogleString& global_admin_path() const {
+    return global_admin_path_.value();
+  }
 
  private:
   // Helper methods for ParseAndSetOptions().  Each can:
@@ -108,14 +127,27 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   static void add_ngx_option(typename OptionClass::ValueType default_value,
                              OptionClass NgxRewriteOptions::*offset,
                              const char* id,
-                             StringPiece option_name) {
-    AddProperty(default_value, offset, id, option_name, ngx_properties_);
+                             StringPiece option_name,
+                             OptionScope scope,
+                             const char* help) {
+    AddProperty(default_value, offset, id, option_name, scope, help,
+                ngx_properties_);
   }
 
+  Option<GoogleString> statistics_path_;
+  Option<GoogleString> global_statistics_path_;
+  Option<GoogleString> console_path_;
+  Option<GoogleString> messages_path_;
+  Option<GoogleString> admin_path_;
+  Option<GoogleString> global_admin_path_;
+
   // Helper for ParseAndSetOptions.  Returns whether the two directives equal,
   // ignoring case.
   bool IsDirective(StringPiece config_directive, StringPiece compare_directive);
 
+  // Returns a given option's scope.
+  RewriteOptions::OptionScope GetOptionScope(StringPiece option_name);
+
   // TODO(jefftk): support fetch proxy in server and location blocks.
 
   DISALLOW_COPY_AND_ASSIGN(NgxRewriteOptions);

src/ngx_server_context.cc

@@ -72,6 +72,7 @@ SystemRequestContext* NgxServerContext::NewRequestContext(
 
   return new SystemRequestContext(thread_system()->NewMutex(),
                                   timer(),
+                                  ps_determine_host(r),
                                   local_port,
                                   str_to_string_piece(local_ip));
 }

src/ngx_url_async_fetcher.cc

@@ -66,10 +66,10 @@ namespace net_instaweb {
       mutex_(NULL) {
     resolver_timeout_ = resolver_timeout;
     fetch_timeout_ = fetch_timeout;
-    ngx_memzero(&url_, sizeof(url_));
+    ngx_memzero(&proxy_, sizeof(proxy_));
     if (proxy != NULL && *proxy != '\0') {
-      url_.url.data = reinterpret_cast<u_char*>(const_cast<char*>(proxy));
-      url_.url.len = ngx_strlen(proxy);
+      proxy_.url.data = reinterpret_cast<u_char*>(const_cast<char*>(proxy));
+      proxy_.url.len = ngx_strlen(proxy);
     }
     mutex_ = thread_system_->NewMutex();
     log_ = log;
@@ -106,6 +106,36 @@ namespace net_instaweb {
     }
   }
 
+
+  bool NgxUrlAsyncFetcher::ParseUrl(ngx_url_t* url, ngx_pool_t* pool) {
+    size_t scheme_offset;
+    u_short port;
+    if (ngx_strncasecmp(url->url.data, reinterpret_cast<u_char*>(
+                                     const_cast<char*>("http://")), 7) == 0) {
+      scheme_offset = 7;
+      port = 80;
+    } else if (ngx_strncasecmp(url->url.data, reinterpret_cast<u_char*>(
+                                    const_cast<char*>("https://")), 8) == 0) {
+      scheme_offset = 8;
+      port = 443;
+    } else {
+      scheme_offset = 0;
+      port = 80;
+    }
+
+    url->url.data += scheme_offset;
+    url->url.len -= scheme_offset;
+    url->default_port = port;
+    // See: http://lxr.evanmiller.org/http/source/core/ngx_inet.c#L875
+    url->no_resolve = 0;
+    url->uri_part = 1;
+
+    if (ngx_parse_url(pool, url) == NGX_OK) {
+      return true;
+    }
+    return false;
+  }
+
   // If there are still active requests, cancel them.
   void NgxUrlAsyncFetcher::CancelActiveFetches() {
     // TODO(oschaaf): this seems tricky, this may end up calling
@@ -167,15 +197,15 @@ namespace net_instaweb {
     command_connection_->read->handler = CommandHandler;
     ngx_add_event(command_connection_->read, NGX_READ_EVENT, 0);
 
-    if (url_.url.len == 0) {
+    if (proxy_.url.len == 0) {
       return true;
     }
 
     // TODO(oschaaf): shouldn't we do this earlier? Do we need to clean
     // up when parsing the url failed?
-    if (ngx_parse_url(pool_, &url_) != NGX_OK) {
+    if (!ParseUrl(&proxy_, pool_)) {
       ngx_log_error(NGX_LOG_ERR, log_, 0,
-          "NgxUrlAsyncFetcher::Init parse proxy[%V] failed", &url_.url);
+          "NgxUrlAsyncFetcher::Init parse proxy[%V] failed", &proxy_.url);
       return false;
     }
     return true;
@@ -191,7 +221,7 @@ namespace net_instaweb {
   void NgxUrlAsyncFetcher::Fetch(const GoogleString& url,
                                  MessageHandler* message_handler,
                                  AsyncFetch* async_fetch) {
-    async_fetch = EnableInflation(async_fetch, NULL);
+    async_fetch = EnableInflation(async_fetch);
     NgxFetch* fetch = new NgxFetch(url, async_fetch,
           message_handler, fetch_timeout_, log_);
     ScopedMutex lock(mutex_);

src/ngx_url_async_fetcher.h

@@ -115,13 +115,14 @@ class NgxUrlAsyncFetcher : public UrlAsyncFetcher {
 
  private:
   static void TimeoutHandler(ngx_event_t* tev);
+  static bool ParseUrl(ngx_url_t* url, ngx_pool_t* pool);
   friend class NgxFetch;
 
   NgxFetchPool active_fetches_;
   // Add the pending task to this list
   NgxFetchPool pending_fetches_;
   NgxFetchPool completed_fetches_;
-  ngx_url_t url_;
+  ngx_url_t proxy_;
 
   int fetchers_count_;
   bool shutdown_;

test/pagespeed_test.conf.template

@@ -5,7 +5,7 @@
 worker_processes  1;
 
 daemon @@DAEMON@@;
-master_process @@MASTER_PROCESS@@;
+master_process on;
 
 error_log "@@ERROR_LOG@@" debug;
 pid "@@TEST_TMP@@/nginx.pid";
@@ -27,11 +27,31 @@ http {
   proxy_cache_path "@@PROXY_CACHE@@"  levels=1:2   keys_zone=htmlcache:60m inactive=90m  max_size=50m;
   proxy_temp_path "@@TMP_PROXY_CACHE@@";
 
+  pagespeed StatisticsPath /ngx_pagespeed_statistics;
+  pagespeed GlobalStatisticsPath /ngx_pagespeed_global_statistics;
+  pagespeed ConsolePath /pagespeed_console;
+  pagespeed MessagesPath /ngx_pagespeed_message;
+  pagespeed AdminPath /pagespeed_admin;
+  pagespeed GlobalAdminPath /pagespeed_global_admin;
+
+  pagespeed StaticAssetPrefix /pagespeed_custom_static/;
+
   root "@@SERVER_ROOT@@";
+
+  # Block 5a: Decide on Cache-Control header value to use for outgoing
+  # response.
+  # Map new_cache_control_header_val to "no-cache, max-age=0" if the
+  # content is html and use the original Cache-Control header value
+  # in all other cases.
+  map $upstream_http_content_type $new_cache_control_header_val {
+    default $upstream_http_cache_control;
+    "~*text/html" "no-cache, max-age=0";
+  }
+
   pagespeed UsePerVHostStatistics on;
   pagespeed InPlaceResourceOptimization on;
   pagespeed CreateSharedMemoryMetadataCache "@@SHM_CACHE@@" 8192;
-  pagespeed PreserveUrlRelativity on;
+  pagespeed BlockingRewriteKey psatest;
 
   # CriticalImagesBeaconEnabled is now on by default, but we disable in testing.
   # With this option enabled, the inline image system test will currently fail.
@@ -40,6 +60,11 @@ http {
   # critical images to be inlined, so we just disable the option here.
   pagespeed CriticalImagesBeaconEnabled false;
 
+  # By default, resources will not be used for inlining without explicit
+  # authorization. Supported values are off or a comma-separated list of strings
+  # from {Script,Stylesheet}.
+  pagespeed InlineResourcesWithoutExplicitAuthorization off;
+
   pagespeed Statistics on;
   pagespeed StatisticsLogging on;
   pagespeed LogDir "@@TEST_TMP@@/logdir";
@@ -53,7 +78,6 @@ http {
     server_name max-cacheable-content-length.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed BlockingRewriteKey psatest;
 
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_javascript;
@@ -64,6 +88,7 @@ http {
   @@RESOLVER@@
 
   server {
+    # Block 1:  Basic port, server_name definitions.
     # This server represents the external caching layer server which
     # receives user requests and proxies them to the upstream server
     # running on the PRIMARY_PORT when the response is not available in
@@ -72,58 +97,157 @@ http {
     server_name proxy_cache.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
+    # Disable PageSpeed on this server.
     pagespeed off;
 
-    set $ua_dependent_ps_capability_list "";
-    set $bypass_cache 1;
+    # Block 2: Define prefix for proxy_cache_key based on the UserAgent.
+
+    # Define placeholder PS-CapabilityList header values for large and small
+    # screens with no UA dependent optimizations. Note that these placeholder
+    # values should not contain any of ll, ii, dj, jw or ws, since these
+    # codes will end up representing optimizations to be supported for the
+    # request.
+    set $default_ps_capability_list_for_large_screens "LargeScreen.SkipUADependentOptimizations";
+    set $default_ps_capability_list_for_small_screens "TinyScreen.SkipUADependentOptimizations";
+
+    # As a fallback, the PS-CapabilityList header that is sent to the upstream
+    # PageSpeed server should be for a large screen device with no browser
+    # specific optimizations.
+    set $ps_capability_list $default_ps_capability_list_for_large_screens;
+
+    # Cache-fragment 1: Desktop User-Agents that support lazyload_images (ll),
+    # inline_images (ii) and defer_javascript (dj).
+    # Note: Wget is added for testing purposes only.
     if ($http_user_agent ~* "Chrome/|Firefox/|MSIE |Safari|Wget") {
-      # User Agents that support lazyload-images (ll), inline-images (ii) and
-      # defer-javascript (dj).
-      set $ua_dependent_ps_capability_list "ll,ii,dj:";
-      set $bypass_cache 0;
+      set $ps_capability_list "ll,ii,dj:";
     }
+    # Cache-fragment 2: Desktop User-Agents that support lazyload_images (ll),
+    # inline_images (ii), defer_javascript (dj), webp (jw) and lossless_webp
+    # (ws).
     if ($http_user_agent ~*
         "Chrome/[2][3-9]+\.|Chrome/[[3-9][0-9]+\.|Chrome/[0-9]{3,}\.") {
-      # User Agents that support lazyload-images (ll), inline-images (ii),
-      # defer-javascript (dj), webp (jw) and webp-lossless (ws).
-      set $ua_dependent_ps_capability_list "ll,ii,dj,jw,ws:";
-      set $bypass_cache 0;
+      set $ps_capability_list "ll,ii,dj,jw,ws:";
+    }
+    # Cache-fragment 3: This fragment contains (a) Desktop User-Agents that
+    # match fragments 1 or 2 but should not because they represent older
+    # versions of certain browsers or bots and (b) Tablet User-Agents that
+    # correspond to large screens. These will only get optimizations that work
+    # on all browsers and use image compression qualities applicable to large
+    # screens. Note that even Tablets that are capable of supporting inline or
+    # webp images, e.g. Android 4.1.2, will not get these advanced
+    # optimizations.
+    if ($http_user_agent ~* "Firefox/[1-2]\.|MSIE [5-8]\.|bot|Yahoo!|Ruby|RPT-HTTPClient|(Google \(\+https\:\/\/developers\.google\.com\/\+\/web\/snippet\/\))|Android|iPad|TouchPad|Silk-Accelerated|Kindle Fire") {
+      set $ps_capability_list $default_ps_capability_list_for_large_screens;
+    }
+    # Cache-fragment 4: Mobiles and small screen Tablets will use image compression
+    # qualities applicable to small screens, but all other optimizations will be
+    # those that work on all browsers.
+    if ($http_user_agent ~* "Mozilla.*Android.*Mobile*|iPhone|BlackBerry|Opera Mobi|Opera Mini|SymbianOS|UP.Browser|J-PHONE|Profile/MIDP|portalmmm|DoCoMo|Obigo|Galaxy Nexus|GT-I9300|GT-N7100|HTC One|Nexus [4|7|S]|Xoom|XT907") {
+      set $ps_capability_list $default_ps_capability_list_for_small_screens;
     }
 
-    # All User Agents that represent
-    # 1) mobiles
-    # 2) tablets
-    # 3) desktop browsers that do not have defer-javascript capability at a minimum
-    # are made to go to the pagespeed server directly bypassing the proxy_cache.
-    if ($http_user_agent ~* "Firefox/[1-2]\.|MSIE [5-8]\.") {
-      set $ua_dependent_ps_capability_list "";
-      set $bypass_cache 1;
+    # Block 3a: Bypass the cache for .pagespeed. resource. PageSpeed has its own
+    # cache for these, and these could bloat up the caching layer.
+    if ($uri ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+") {
+      set $bypass_cache "1";
     }
-    if ($http_user_agent ~* "Mozilla.*Android.*Mobile*|iPhone|BlackBerry|Opera Mobi|Opera Mini|SymbianOS|UP.Browser|J-PHONE|Profile/MIDP|portalmmm|DoCoMo|Obigo") {
-      # These are Mobile User Agents. We don't cache responses for these.
-      set $ua_dependent_ps_capability_list "";
-      set $bypass_cache 1;
+
+    # For testing purposes, we never generate values that will result in a beacon 
+    # unless a test request it via "X-Allow-Beacon: yes" in its request header.
+    # This is for testing purposes, note that in a production environment, 
+    # you want 'set_random $rand 0 100;' unconditionally!
+    set $rand 5;
+
+    if ($http_x_allow_beacon ~ "yes") {
+      set_random $rand 0 100;
     }
-    if ($http_user_agent ~* "Android|iPad|TouchPad|Silk-Accelerated|Kindle Fire") {
-      # These are Tablet User Agents. We don't cache responses for these.
-      set $ua_dependent_ps_capability_list "";
+    set $should_beacon_header_val "";
+    if ($rand ~* "^[0-4]$") {
+      set $should_beacon_header_val "random_rebeaconing_key";
       set $bypass_cache 1;
     }
 
+    # Block 3b: Only cache responses to clients that support gzip.  Most clients
+    # do, and the cache holds much more if it stores gzipped responses.
+    if ($http_accept_encoding !~* gzip) {
+      set $bypass_cache "1";
+    }
+
+    # Block 4: Location block for purge requests.
     location ~ /purge(/.*) {
-      allow all;
-      proxy_cache_purge htmlcache $ua_dependent_ps_capability_list$1$is_args$args;
+      allow 127.0.0.1;
+      deny all;
+      proxy_cache_purge htmlcache $ps_capability_list$1$is_args$args;
     }
 
+    # Block 6: Location block with proxy_cache directives.
     location /mod_pagespeed_test/cachable_rewritten_html/ {
+      # 1: Upstream PageSpeed server is running at localhost:8050.
       proxy_pass http://localhost:@@PRIMARY_PORT@@;
-      proxy_set_header Host $host;
-      proxy_cache_valid 200 30s;
+      # 2: Use htmlcache as the zone for caching.
       proxy_cache htmlcache;
-      proxy_ignore_headers Cache-Control;
-      add_header X-Cache $upstream_cache_status;
-      proxy_cache_key $ua_dependent_ps_capability_list$uri$is_args$args;
+      # 3: Bypass requests that correspond to .pagespeed. resources
+      # or clients that do not support gzip etc.
       proxy_cache_bypass $bypass_cache;
+      # 4: Use the redefined proxy_cache_key and make sure the /purge/
+      # block uses the same key.
+      proxy_cache_key $ps_capability_list$uri$is_args$args;
+      # 5: Forward Host header to upstream server.
+      proxy_set_header Host $host;
+      # 6: Set the PS-CapabilityList header for PageSpeed server to respect.
+      proxy_set_header PS-CapabilityList $ps_capability_list;
+      add_header PS-CapabilityList $ps_capability_list;
+      # 7: Add a header for identifying cache hits/misses/expires. This is
+      # for debugging purposes only and can be commented out in production.
+      add_header X-Cache $upstream_cache_status;
+
+      # Block 5b: Override Cache-Control headers as needed.
+      # Hide the upstream cache control header.
+      proxy_hide_header Cache-Control;
+      # Add the inferred Cache-Control header.
+      add_header Cache-Control $new_cache_control_header_val;
+
+      proxy_set_header PS-ShouldBeacon $should_beacon_header_val;
+      proxy_hide_header PS-ShouldBeacon;
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name if-in-server.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+
+    pagespeed RewriteLevel PassThrough;
+    set $inline_javascript "No";
+
+    if ($http_x_custom_header_inline_js) {
+      # TODO(jefftk): Turn on NGX_HTTP_SIF_CONF and figure out how to get
+      # pagespeed directives inside of a server location block to be respected,
+      # then uncomment the following line and duplicate the if-in-location test
+      # for if-in-server.
+      #pagespeed EnableFilters inline_javascript;
+      set $inline_javascript "Yes";
+    }
+
+    add_header "X-Inline-Javascript" $inline_javascript;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name if-in-location.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+
+
+    location / {
+      set $inline_javascript "No";
+      pagespeed RewriteLevel PassThrough;
+
+      if ($http_x_custom_header_inline_js) {
+        pagespeed EnableFilters inline_javascript;
+        set $inline_javascript "Yes";
+      }
+
+      add_header "X-Inline-Javascript" $inline_javascript;
     }
   }
 
@@ -141,6 +265,7 @@ http {
     listen @@SECONDARY_PORT@@;
     server_name experiment.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed InPlaceResourceOptimization off;
 
     pagespeed RunExperiment on;
     pagespeed AnalyticsID "123-45-6734";
@@ -148,6 +273,7 @@ http {
     pagespeed ExperimentSpec
       "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
     pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
+    pagespeed ExperimentSpec "id=3;default;percent=0";
   }
   server {
     listen @@SECONDARY_PORT@@;
@@ -159,6 +285,7 @@ http {
     pagespeed ExperimentSpec
       "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
     pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
+    pagespeed ExperimentSpec "id=3;default;percent=0";
   }
 
   server {
@@ -200,6 +327,32 @@ http {
     pagespeed CriticalImagesBeaconEnabled true;
   }
 
+  server {
+    # Setup a vhost with the critical image beacon enabled to make sure that
+    # downstream caches and rebeaconing interact correctly.
+    listen @@SECONDARY_PORT@@;
+    server_name downstreamcacherebeacon.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed RewriteLevel PassThrough;
+    pagespeed CriticalImagesBeaconEnabled true;
+    # Enable the downstream caching feature and specify a rebeaconing key.
+    pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
+    pagespeed DownstreamCacheRebeaconingKey random_rebeaconing_key;
+    location ~ .*[.]html {
+      add_header Cache-Control "private, max-age=3000";
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name downstreamcacheresource.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_images;
+    pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     server_name renderedimagebeacon.example.com;
@@ -210,6 +363,60 @@ http {
     pagespeed CriticalImagesBeaconEnabled true;
   }
 
+
+  # Build a configuration hierarchy where at the root we have turned on
+  # OptimizeForBandwidth, and in various subdirectories we override settings
+  # to make them more aggressive.
+  #
+  # In Apache we can do this all with Directory blocks, but to get the same
+  # inheretence in Nginx we need to have location blocks inside a server block.
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name optimizeforbandwidth.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed RewriteLevel OptimizeForBandwidth;
+    pagespeed DisableFilters add_instrumentation;
+
+    location /mod_pagespeed_test/optimize_for_bandwidth/inline_css {
+      pagespeed EnableFilters inline_css;
+    }
+    location /mod_pagespeed_test/optimize_for_bandwidth/css_urls {
+      pagespeed CssPreserveURLs off;
+    }
+    location /mod_pagespeed_test/optimize_for_bandwidth/image_urls {
+      pagespeed ImagePreserveURLs off;
+    }
+    location /mod_pagespeed_test/optimize_for_bandwidth/core_filters {
+      pagespeed RewriteLevel CoreFilters;
+    }
+  }
+
+  server {
+    # For testing with a custom origin header.  In this VirtualHost,
+    # /mod_pagespeed_test is included in our DocumentRoot and thus does
+    # not need to be in any resource URL paths.  This helps us verify that
+    # we are looping back to the corect VirtualHost -- if we hit the wrong
+    # one it will not work.  Also we don't have a VirtualHost for
+    # sharedcdn.example.com, so the default Host header used for
+    # origin-mapping won't work either.  Instead, we want origin-fetches
+    # to go back to this VirtualHost so we rely on the new third optional
+    # argument to MapOriginDomain.
+    listen @@SECONDARY_PORT@@;
+    server_name customhostheader.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@_test";
+    root "@@SERVER_ROOT@@/mod_pagespeed_test";
+
+    pagespeed on;
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_images;
+    # Don't use localhost, as ngx_pagespeed's native fetcher cannot resolve it
+    pagespeed MapOriginDomain 127.0.0.1:@@SECONDARY_PORT@@/customhostheader
+              sharedcdn.example.com/test customhostheader.example.com;
+    pagespeed JpegRecompressionQuality 50;
+    pagespeed CriticalImagesBeaconEnabled false;
+  }
+
   server {
     # Sets up a virtual host where we can specify forbidden filters without
     # affecting any other hosts.
@@ -217,7 +424,6 @@ http {
     server_name forbidden.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed BlockingRewriteKey psatest;
 
     # Start with all core filters enabled ...
     pagespeed RewriteLevel CoreFilters;
@@ -228,6 +434,16 @@ http {
     pagespeed DisableFilters inline_css;
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name unauthorizedresources.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed RewriteLevel PassThrough;
+    pagespeed InlineResourcesWithoutExplicitAuthorization Script,Stylesheet;
+    pagespeed CssInlineMaxBytes 1000000;
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     server_name client-domain-rewrite.example.com;
@@ -265,6 +481,9 @@ http {
     pagespeed UrlValuedAttribute img alt-src Image;
     pagespeed UrlValuedAttribute video alt-a Image;
     pagespeed UrlValuedAttribute video alt-b Image;
+
+    # Also test that we can redefine spec-defined attributes.
+    pagespeed UrlValuedAttribute blockquote cite Image;
   }
 
   server {
@@ -342,7 +561,7 @@ http {
   # Note that we test with two distinct caches.
   server {
     listen @@SECONDARY_PORT@@;
-    server_name embed-config-html.example.com;
+    server_name embed-config-html.example.org;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
     root "@@SERVER_ROOT@@/mod_pagespeed_test";
@@ -352,6 +571,9 @@ http {
     pagespeed DisableFilters inline_css,extend_cache,inline_javascript;
     pagespeed Domain embed-config-resources.example.com;
 
+    # Share a cache keyspace with embed-config-resources.example.com.
+    pagespeed CacheFragment "embed-config";
+
     pagespeed LoadFromFile "http://embed-config-resources.example.com/"
       "@@SERVER_ROOT@@/mod_pagespeed_example/";
   }
@@ -367,12 +589,26 @@ http {
 
     # Note that we do not set the jpeg quality here, but take
     # it from image URL query parameters that we synthesize in
-    # from embed-config-html.example.com.
+    # from embed-config-html.example.org.
+
+    # Share a cache keyspace with embed-config-html.example.org.
+    pagespeed CacheFragment "embed-config";
 
     pagespeed LoadFromFile "http://embed-config-resources.example.com/"
       "@@SERVER_ROOT@@/mod_pagespeed_example/";
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name ipro-for-browser.example.com;
+    root "@@SERVER_ROOT@@/mod_pagespeed_example";
+    pagespeed EnableFilters rewrite_images,rewrite_css;
+    pagespeed EnableFilters convert_to_webp_lossless;
+    pagespeed EnableFilters in_place_optimize_for_browser;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed FileCachePath "@@IPRO_CACHE@@";
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     server_name respectvary.example.com;
@@ -424,6 +660,7 @@ http {
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
+    pagespeed CacheFragment "example";
   }
   server {
     # Sets up a logical origin for CDNs to fetch content from, on
@@ -436,6 +673,7 @@ http {
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
+    pagespeed CacheFragment "example";
   }
   server {
     # Sets up a logical cdn, which is where we tell browsers to fetch resources
@@ -475,6 +713,23 @@ http {
     }
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name custom-paths.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed StatisticsPath /custom_pagespeed_statistics;
+    pagespeed ConsolePath /custom_pagespeed_console;
+    pagespeed MessagesPath /custom_pagespeed_message;
+    pagespeed AdminPath /custom_pagespeed_admin;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name inherit-paths.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     server_name notransform.example.com;
@@ -491,7 +746,6 @@ http {
     server_name blocking.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
 
-    pagespeed BlockingRewriteKey psatest;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_images;
   }
@@ -561,6 +815,11 @@ http {
     location /mod_pagespeed_test/ipro/nocache/test_image_dont_reuse.png {
       add_header Cache-Control no-cache;
     }
+
+    location /mod_pagespeed_test/ipro/no-cache-control-header {
+      add_header Cache-Control "";
+      pagespeed ImplicitCacheTtlMs 333000;
+    }
   }
 
   # Test hosts to cover all possible cache configurations.  L1 will be filecache
@@ -615,25 +874,68 @@ http {
     pagespeed CriticalImagesBeaconEnabled false;
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name date.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    add_header "Date" "Date: Fri, 16 Oct 2009 23:05:07 GMT";
+  }
+
+  # Proxy + IPRO a gzip'd file for testing Issue 896.
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name ipro-proxy.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@_ipro_proxy";
+
+    pagespeed on;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed EnableFilters rewrite_domains;
+
+    location / {
+      proxy_pass
+       "http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/mod_deflate/";
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name compressed-css.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed InPlaceResourceOptimization on;
+  }
+
   server {
     listen       @@PRIMARY_PORT@@;
     server_name  localhost;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
+    location ~ ^/pagespeed_admin {
+      allow 127.0.0.1;
+      deny all;
+    }
+
     location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" {
       add_header "" "";
     }
 
+    # Backend for ipro-proxy.example.com
+    location /mod_pagespeed_test/ipro/mod_deflate/ {
+      pagespeed off;
+    }
+
     location /mod_pagespeed_test/cachable_rewritten_html/ {
       # This location has the html files that will be configured to be stored
       # in the proxy_cache layer.
       pagespeed DownstreamCachePurgeMethod "GET";
       pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
+      pagespeed DownstreamCacheRebeaconingKey "random_rebeaconing_key";
       # We use a very small deadline here to force the rewriting to not complete
       # in the very first attempt.
       pagespeed RewriteDeadlinePerFlushMs 1;
       pagespeed RewriteLevel PassThrough;
-      pagespeed EnableFilters collapse_whitespace,extend_cache,recompress_images,convert_jpeg_to_webp,defer_javascript;
+      pagespeed EnableFilters collapse_whitespace,extend_cache,recompress_images;
+      pagespeed CriticalImagesBeaconEnabled true;
+      add_header Cache-Control "public, max-age=100";
     }
 
     location /mod_pagespeed_test/disable_no_transform/index.html {
@@ -649,6 +951,7 @@ http {
     #pagespeed MemcachedThreads 1;
 
     pagespeed on;
+    pagespeed MessageBufferSize 200000;
 
     #pagespeed CacheFlushPollIntervalSec 1;
 
@@ -660,14 +963,6 @@ http {
     pagespeed Library 43 1o978_K0_LNE5_ystNklf
     http://www.modpagespeed.com/rewrite_javascript.js;
 
-    # If X-PSA-Blocking-Rewrite request header is present and its value matches
-    # the value of BlockingRewriteKey below, the response will be fully
-    # rewritten before being flushed to the client.
-    pagespeed BlockingRewriteKey psatest;
-
-    # Disable parsing if the size of the HTML exceeds 50kB.
-    pagespeed MaxHtmlParseBytes 50000;
-
     add_header X-Extra-Header 1;
 
     # Establish a proxy mapping where the current server proxies an image
@@ -706,6 +1001,10 @@ http {
       pagespeed DisableFilters collapse_whitespace;
     }
 
+    location /mod_pagespeed_test/max_html_parse_size {
+      pagespeed MaxHtmlParseBytes 5000;
+    }
+
     location ~ \.php$ {
       fastcgi_param SCRIPT_FILENAME $request_filename;
       fastcgi_param QUERY_STRING $query_string;
@@ -781,6 +1080,79 @@ http {
       expires 5m;
     }
 
+    location /mod_pagespeed_test/ipro/instant/wait/ {
+      pagespeed InPlaceWaitForOptimized on;
+
+      # TODO(jefftk): When running with valgrind we need to raise the rewrite
+      # deadline or else we get "Deadline exceeded for rewrite of resource".
+      # I had thought InPlaceWaitForOptimized was supposed to disable the
+      # rewrite deadline, but it seems not to.
+      #
+      # InPlaceWaitForOptimized is definitely doing something, though: when I
+      # remove the InPlaceRewriteDeadlineMs line below and run without valgrind,
+      # purple.css is optimized via ipro on the first request but if I set
+      # InPlaceWaitForOptimized to off then it is not.
+      pagespeed InPlaceRewriteDeadlineMs 1000;
+    }
+
+    location /mod_pagespeed_test/ipro/instant/deadline/ {
+      pagespeed InPlaceRewriteDeadlineMs -1;
+    }
+
+    # Test to make sure that user-authenticated resources do not get cached and
+    # optimized.
+    location /mod_pagespeed_test/auth/ {
+       auth_basic "Restricted";
+       auth_basic_user_file "@@SERVER_ROOT@@mod_pagespeed_test/auth/passwd.conf";
+    }
+
+    location /mod_pagespeed_test/ipro/cookie/ {
+      # Add Vary:Cookie.  This should prevent us from optimizing the
+      # vary_cookie.css even though PagespeedRespectVary is off.
+      # test/nginx_system_test.sh does the fetches test with and w/o cookies.
+      add_header Vary Cookie;
+      pagespeed RespectVary off;
+      pagespeed InPlaceWaitForOptimized on;
+    }
+
+    location /mod_pagespeed_test/ipro/cookie2/ {
+      # Add Vary:Cookie2.  This should prevent us from optimizing the
+      # vary_cookie2.css even though PagespeedRespectVary is off.
+      # test/nginx_system_test.sh does the fetches test with and w/o cookie2.
+      add_header Vary Cookie2;
+      pagespeed RespectVary off;
+      pagespeed InPlaceWaitForOptimized on;
+    }
+
+    location /mod_pagespeed_test/vary/ {
+      pagespeed RespectVary on;
+    }
+
+    location ~ /mod_pagespeed_test/vary/\.(js|css)$ {
+        add_header Vary User-Agent;
+    }
+
+    location /mod_pagespeed_test/vary/no_respect/ {
+      pagespeed DisableFilters add_instrumentation,inline_css;
+      pagespeed RespectVary off;
+    }
+
+    location /mod_pagespeed_test/experimental_js_minifier/ {
+      pagespeed UseExperimentalJsMinifier on;
+    }
+
+    pagespeed FetchHttps enable;
+    location /mod_pagespeed_test/https_fetch/ {
+      pagespeed DisableFilters inline_images;
+      pagespeed MapProxyDomain
+          http://localhost:@@PRIMARY_PORT@@/https_gstatic_dot_com
+          https://www.gstatic.com/psa/static;
+    }
+
+    pagespeed LoadFromFile
+        "http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/instant/"
+        "@@SERVER_ROOT@@/mod_pagespeed_test/ipro/instant/";
+
     pagespeed EnableFilters remove_comments;
 
     # Test LoadFromFile mapping by mapping one dir to another.
@@ -819,6 +1191,7 @@ http {
   # set up gzip
   gzip  on;
   gzip_vary on;
+  gzip_proxied any;
   # Turn on gzip for all content types that should benefit from it.
   gzip_types application/ecmascript;
   gzip_types application/javascript;

test/valgrind.sup

@@ -0,0 +1,165 @@
+# Copyright 2013 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Author: oschaaf@we-amp.com (Otto van der Schaaf)
+
+
+# The first few suppressions can be found in other modules
+# and easily found when searched for, and seem false positives.
+{
+   <nginx false positive>
+   Memcheck:Param
+   socketcall.sendmsg(msg.msg_iov[i])
+   fun:__sendmsg_nocancel
+   fun:ngx_write_channel
+   fun:ngx_signal_worker_processes
+   fun:ngx_master_process_cycle
+   fun:main
+}
+{
+   <nginx false positive>
+   Memcheck:Param
+   socketcall.sendmsg(msg.msg_iov[i])
+   fun:__sendmsg_nocancel
+   fun:ngx_write_channel
+   fun:ngx_master_process_cycle
+   fun:main
+}
+{
+   <nginx false positive>
+   Memcheck:Param
+   socketcall.sendmsg(msg.msg_iov[i])
+   fun:__sendmsg_nocancel
+   fun:ngx_write_channel
+   fun:ngx_pass_open_channel
+   fun:ngx_start_cache_manager_processes
+   fun:ngx_master_process_cycle
+   fun:main
+}
+{
+   <nginx false positive>
+   Memcheck:Param
+   socketcall.sendmsg(msg.msg_iov[i])
+   fun:__sendmsg_nocancel
+   fun:ngx_write_channel
+   fun:ngx_pass_open_channel
+   fun:ngx_start_cache_manager_processes
+   fun:ngx_master_process_cycle
+   fun:main
+}
+{
+   <nginx false positive>
+   Memcheck:Leak
+   fun:malloc
+   fun:ngx_alloc
+   fun:ngx_event_process_init
+   fun:ngx_worker_process_init
+   fun:ngx_worker_process_cycle
+   fun:ngx_spawn_process
+   fun:ngx_start_worker_processes
+   fun:ngx_master_process_cycle
+   fun:main
+}
+{
+   <nginx false positive>
+   Memcheck:Param
+   socketcall.sendmsg(msg.msg_iov[i])
+   fun:__sendmsg_nocancel
+   fun:ngx_write_channel
+   fun:ngx_pass_open_channel
+   fun:ngx_start_worker_processes
+   fun:ngx_master_process_cycle
+   fun:main
+}
+
+# similar to http://trac.nginx.org/nginx/ticket/369
+{
+   <nginx false positive>
+   Memcheck:Param
+   pwrite64(buf)
+   obj:/lib/x86_64-linux-gnu/libpthread-2.15.so
+   fun:ngx_write_file
+   fun:ngx_write_chain_to_file
+   fun:ngx_write_chain_to_temp_file
+   fun:ngx_event_pipe_write_chain_to_temp_file
+   fun:ngx_event_pipe
+   fun:ngx_http_upstream_process_upstream
+   fun:ngx_http_upstream_process_header
+   fun:ngx_http_upstream_handler
+   fun:ngx_epoll_process_events
+   fun:ngx_process_events_and_timers
+   fun:ngx_worker_process_cycle
+}
+# Mentioned in https://github.com/pagespeed/ngx_pagespeed/issues/103
+# Assuming a false postives as the issue is closed.
+{
+   <nginx false positive>
+   Memcheck:Param
+   write(buf)
+   obj:/lib/x86_64-linux-gnu/libpthread-2.15.so
+   fun:ngx_log_error_core
+   fun:ngx_http_parse_complex_uri
+   fun:ngx_http_process_request_uri
+   fun:ngx_http_process_request_line
+   fun:ngx_http_wait_request_handler
+   fun:ngx_epoll_process_events
+   fun:ngx_process_events_and_timers
+   fun:ngx_worker_process_cycle
+   fun:ngx_spawn_process
+   fun:ngx_start_worker_processes
+   fun:ngx_master_process_cycle
+}
+
+# Extra suppresions for testing in release mode:
+
+{
+   <re2 uninitialised value in optimized code>
+   Memcheck:Cond
+   fun:_ZN3re24Prog8OptimizeEv
+   ...
+}
+{
+   <re2 uninitialised value in optimized code>
+   Memcheck:Value8
+   fun:_ZN3re24Prog8OptimizeEv
+   ...
+}
+
+{
+   <re2 uninitialised value in optimized code>
+   Memcheck:Cond
+   fun:_ZN3re2L4AddQEPNS_9SparseSetEi
+   ...
+}
+
+{
+   <re2 uninitialised value in optimized code>
+   Memcheck:Value8
+   fun:_ZN3re2L4AddQEPNS_9SparseSetEi
+   ...
+}
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Value8
+   fun:_ZN3re23DFA10AddToQueueEPNS0_5WorkqEij
+   ...
+}
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Cond
+   fun:_ZN3re23DFA10AddToQueueEPNS0_5WorkqEij
+   ...
+}