trunk-tracking: update from r3924 to r3934

There aren't actually any changes required from r3924 to r3934 but the admin
work from previous versions wasn't completely ported before.

This change:
 * adds support for /pagespeed_admin and /pagespeed_global_admin
 * removes NgxPagespeedConsoleAsyncFetch using NgxBaseFetch instead
 * adds options StatisticsPath, GlobalStatisticsPath, ConsolePath,
   MessagesPath, AdminPath, and GlobalAdminPath which allow site owners to
   control what path these are served under
 * changes /ngx_pagespeed_statistics etc from default-on to default-off,
   requiring explicit path configuration to function
 * does not fix the problem where trunk-tracking fails tests on 1.5 post #653

README.md

@@ -37,11 +37,11 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p
 
    ```bash
    $ cd ~
-   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.7.30.3-beta.zip
-   $ unzip v1.7.30.3-beta.zip # or unzip v1.7.30.3-beta
-   $ cd ngx_pagespeed-1.7.30.3-beta/
-   $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.3.tar.gz
-   $ tar -xzvf 1.7.30.3.tar.gz # expands to psol/
+   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.7.30.4-beta.zip
+   $ unzip v1.7.30.4-beta.zip # or unzip v1.7.30.4-beta
+   $ cd ngx_pagespeed-1.7.30.4-beta/
+   $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.4.tar.gz
+   $ tar -xzvf 1.7.30.4.tar.gz # expands to psol/
    ```
 
 3. Download and build nginx:
@@ -49,10 +49,10 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p
    ```bash
    $ cd ~
    $ # check http://nginx.org/en/download.html for the latest version
-   $ wget http://nginx.org/download/nginx-1.4.4.tar.gz
-   $ tar -xvzf nginx-1.4.4.tar.gz
-   $ cd nginx-1.4.4/
-   $ ./configure --add-module=$HOME/ngx_pagespeed-1.7.30.3-beta
+   $ wget http://nginx.org/download/nginx-1.4.6.tar.gz
+   $ tar -xvzf nginx-1.4.6.tar.gz
+   $ cd nginx-1.4.6/
+   $ ./configure --add-module=$HOME/ngx_pagespeed-1.7.30.4-beta
    $ make
    $ sudo make install
    ```
@@ -95,7 +95,7 @@ To confirm that the module is loaded, fetch a page and check that you see the
 
 ```bash
 $ curl -I 'http://localhost:8050/some_page/' | grep X-Page-Speed
-X-Page-Speed: 1.7.30.3-...
+X-Page-Speed: 1.7.30.4-...
 ```
 
 Looking at the source of a few pages you should see various changes, such as

config

@@ -27,8 +27,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
     echo "   You need to separately download the pagespeed library:"
     echo ""
     echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.2.tar.gz"
-    echo "     $ tar -xzvf 1.7.30.2.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.4.tar.gz"
+    echo "     $ tar -xzvf 1.7.30.4.tar.gz # expands to psol/"
     echo ""
     echo "   Or see the installation instructions:"
     echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
@@ -109,6 +109,10 @@ case "$NGX_GCC_VER" in
   ;;
 esac
 
+if [ "$WNO_ERROR" = "YES" ]; then
+    CFLAGS="$CFLAGS -Wno-error"
+fi
+
 pagespeed_include="\
   $mod_pagespeed_dir \
   $mod_pagespeed_dir/third_party/chromium/src \

src/ngx_base_fetch.cc

@@ -106,11 +106,10 @@ ngx_int_t NgxBaseFetch::CollectAccumulatedWrites(ngx_chain_t** link_ptr) {
 ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
   const ResponseHeaders* pagespeed_headers = response_headers();
 
-  // TODO(chaizhenhua): Add and check.
-  // if (content_length_known()) {
-  //   headers_out->content_length = NULL;
-  //   headers_out->content_length_n = content_length();
-  // }
+  if (content_length_known()) {
+     headers_out->content_length = NULL;
+     headers_out->content_length_n = content_length();
+  }
 
   return copy_response_headers_to_ngx(request_, *pagespeed_headers,
                                       preserve_caching_headers_);

src/ngx_fetch.cc

@@ -72,11 +72,11 @@ namespace net_instaweb {
         done_(false),
         content_length_(-1),
         content_length_known_(false) {
-            ngx_memzero(&url_, sizeof(url_));
-            log_ = log;
-            pool_ = NULL;
-            timeout_event_ = NULL;
-            connection_ = NULL;
+    ngx_memzero(&url_, sizeof(url_));
+    log_ = log;
+    pool_ = NULL;
+    timeout_event_ = NULL;
+    connection_ = NULL;
   }
 
   NgxFetch::~NgxFetch() {

src/ngx_pagespeed.cc

@@ -37,6 +37,7 @@
 #include "ngx_server_context.h"
 
 #include "net/instaweb/automatic/public/proxy_fetch.h"
+#include "net/instaweb/http/public/async_fetch.h"
 #include "net/instaweb/http/public/cache_url_async_fetcher.h"
 #include "net/instaweb/http/public/content_type.h"
 #include "net/instaweb/http/public/request_context.h"
@@ -48,13 +49,14 @@
 #include "net/instaweb/rewriter/public/resource_fetch.h"
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
 #include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "net/instaweb/rewriter/public/rewrite_query.h"
 #include "net/instaweb/rewriter/public/rewrite_stats.h"
 #include "net/instaweb/rewriter/public/static_asset_manager.h"
-#include "net/instaweb/system/public/handlers.h"
 #include "net/instaweb/system/public/in_place_resource_recorder.h"
 #include "net/instaweb/system/public/system_caches.h"
 #include "net/instaweb/system/public/system_request_context.h"
 #include "net/instaweb/system/public/system_rewrite_options.h"
+#include "net/instaweb/system/public/system_server_context.h"
 #include "net/instaweb/system/public/system_thread_system.h"
 #include "net/instaweb/util/public/fallback_property_page.h"
 #include "net/instaweb/util/public/google_message_handler.h"
@@ -69,6 +71,7 @@
 #include "net/instaweb/util/public/time_util.h"
 #include "net/instaweb/util/stack_buffer.h"
 #include "pagespeed/kernel/base/posix_timer.h"
+#include "pagespeed/kernel/http/query_params.h"
 #include "pagespeed/kernel/html/html_keywords.h"
 #include "pagespeed/kernel/thread/pthread_shared_mem.h"
 
@@ -239,15 +242,18 @@ void copy_response_headers_from_ngx(const ngx_http_request_t* r,
 
   headers->set_status_code(r->headers_out.status);
 
+  if (r->headers_out.location != NULL) {
+    headers->Add(HttpAttributes::kLocation,
+                 str_to_string_piece(r->headers_out.location->value));
+  }
+
   // Manually copy over the content type because it's not included in
   // request_->headers_out.headers.
   headers->Add(HttpAttributes::kContentType,
                str_to_string_piece(r->headers_out.content_type));
 
-  // When we don't have a date header, invent one.
-  const char* date = headers->Lookup1(HttpAttributes::kDate);
-
-  if (date == NULL) {
+  // When we don't have a date header, set one with the current time.
+  if (headers->Lookup1(HttpAttributes::kDate) == NULL) {
     headers->SetDate(ngx_current_msec);
   }
 
@@ -277,7 +283,7 @@ ngx_int_t copy_response_headers_to_ngx(
     const GoogleString& name_gs = pagespeed_headers.Name(i);
     const GoogleString& value_gs = pagespeed_headers.Value(i);
 
-    if (preserve_caching_headers != kDontPreserveHeaders) {
+    if (preserve_caching_headers == kPreserveAllCachingHeaders) {
       if (StringCaseEqual(name_gs, "ETag") ||
           StringCaseEqual(name_gs, "Expires") ||
           StringCaseEqual(name_gs, "Date") ||
@@ -285,7 +291,16 @@ ngx_int_t copy_response_headers_to_ngx(
           StringCaseEqual(name_gs, "Cache-Control")) {
         continue;
       }
-    }
+    } else if (preserve_caching_headers == kPreserveOnlyCacheControl) {
+      // Retain the original Cache-Control header, but send the recomputed
+      // values for all other cache-related headers.
+      if (StringCaseEqual(name_gs, "ETag") ||
+          StringCaseEqual(name_gs, "Expires") ||
+          StringCaseEqual(name_gs, "Date") ||
+          StringCaseEqual(name_gs, "Last-Modified")) {
+        continue;
+      }
+    } // else we don't preserve any headers
 
     ngx_str_t name, value;
 
@@ -426,8 +441,11 @@ enum Response {
   kPagespeedDisabled,
   kBeacon,
   kStatistics,
+  kGlobalStatistics,
   kConsole,
   kMessages,
+  kAdmin,
+  kGlobalAdmin,
   kPagespeedSubrequest,
   kNotHeadOrGet,
   kErrorResponse,
@@ -458,7 +476,7 @@ ngx_command_t ps_commands[] = {
     NULL },
 
   { ngx_string("pagespeed"),
-    NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1|
+    NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF|NGX_CONF_TAKE1|
     NGX_CONF_TAKE2|NGX_CONF_TAKE3|NGX_CONF_TAKE4|NGX_CONF_TAKE5,
     ps_loc_configure,
     NGX_HTTP_SRV_CONF_OFFSET,
@@ -504,15 +522,24 @@ char* ps_init_dir(const StringPiece& directive,
     return NULL;  // We're not root, so we're staying whoever we are.
   }
 
-  ngx_core_conf_t* ccf =
-      (ngx_core_conf_t*)(ngx_get_conf(cf->cycle->conf_ctx, ngx_core_module));
+  // chown if owner differs from nginx worker user.
+  ngx_core_conf_t* ccf = reinterpret_cast<ngx_core_conf_t*>(
+      ngx_get_conf(cf->cycle->conf_ctx, ngx_core_module));
   CHECK(ccf != NULL);
-
-  if (chown(gs_path.c_str(), ccf->user, ccf->group) != 0) {
+  struct stat gs_stat;
+  if (stat(gs_path.c_str(), &gs_stat) != 0) {
     return string_piece_to_pool_string(
         cf->pool, net_instaweb::StrCat(
-            directive, " ", path, " unable to set permissions"));
+            directive, " ", path, " stat() failed"));
   }
+  if (gs_stat.st_uid != ccf->user) {
+    if (chown(gs_path.c_str(), ccf->user, ccf->group) != 0) {
+      return string_piece_to_pool_string(
+          cf->pool, net_instaweb::StrCat(
+              directive, " ", path, " unable to set permissions"));
+    }
+  }
+
   return NULL;
 }
 
@@ -785,19 +812,28 @@ char* ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child) {
 }
 
 char* ps_merge_loc_conf(ngx_conf_t* cf, void* parent, void* child) {
-  ps_loc_conf_t* parent_cfg_l = static_cast<ps_loc_conf_t*>(parent);
-
-  // The variant of the pagespeed directive that is acceptable in location
-  // blocks is only acceptable in location blocks, so we should never be merging
-  // in options from a server or main block.
-  CHECK(parent_cfg_l->options == NULL);
-
   ps_loc_conf_t* cfg_l = static_cast<ps_loc_conf_t*>(child);
   if (cfg_l->options == NULL) {
     // No directory specific options.
     return NGX_CONF_OK;
   }
 
+  // While you can't put a "location" block inside a "location" block you can
+  // put an "if" block inside a "location" block, which is implemented by making
+  // a pretend "location" block.  In this case we may have pagespeed options
+  // from the parent "location" block as well as from the current locationish
+  // "if" block.
+  ps_loc_conf_t* parent_cfg_l = static_cast<ps_loc_conf_t*>(parent);
+  if (parent_cfg_l->options != NULL) {
+    // Rebase our options off of the ones defined in the parent location block.
+    ps_merge_options(parent_cfg_l->options, &cfg_l->options);
+    return NGX_CONF_OK;
+  }
+
+  // Pagespeed options are defined in this location block, and it either has no
+  // parent (typical case) or is an if block whose parent location block defines
+  // no pagespeed options.  Base our options off of those in the server block.
+
   ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
       ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
 
@@ -896,18 +932,9 @@ int ps_determine_port(ngx_http_request_t* r) {
 
   return port;
 }
+}  // namespace
 
-GoogleString ps_determine_url(ngx_http_request_t* r) {
-  int port = ps_determine_port(r);
-  GoogleString port_string;
-  if ((ps_is_https(r) && (port == 443 || port == -1)) ||
-      (!ps_is_https(r) && (port == 80 || port == -1))) {
-    // No port specifier needed for requests on default ports.
-    port_string = "";
-  } else {
-    port_string = StrCat(":", IntegerToString(port));
-  }
-
+StringPiece ps_determine_host(ngx_http_request_t* r) {
   StringPiece host = str_to_string_piece(r->headers_in.server);
   if (host.size() == 0) {
     // If host is unspecified, perhaps because of a pure HTTP 1.0 "GET /path",
@@ -922,6 +949,23 @@ GoogleString ps_determine_url(ngx_http_request_t* r) {
     }
     host = str_to_string_piece(s);
   }
+  return host;
+}
+
+namespace {
+
+GoogleString ps_determine_url(ngx_http_request_t* r) {
+  int port = ps_determine_port(r);
+  GoogleString port_string;
+  if ((ps_is_https(r) && (port == 443 || port == -1)) ||
+      (!ps_is_https(r) && (port == 80 || port == -1))) {
+    // No port specifier needed for requests on default ports.
+    port_string = "";
+  } else {
+    port_string = StrCat(":", IntegerToString(port));
+  }
+
+  StringPiece host = ps_determine_host(r);
 
   return StrCat(ps_is_https(r) ? "https://" : "http://",
                 host, port_string, str_to_string_piece(r->unparsed_uri));
@@ -1202,11 +1246,9 @@ RewriteOptions* ps_determine_request_options(
   // make cache key consistent for both lookup and storing in cache.
   //
   // Sets option from request headers and url.
-  ServerContext::OptionsBoolPair query_options_success =
-      cfg_s->server_context->GetQueryOptions(url, request_headers,
-                                             response_headers);
-  bool get_query_options_success = query_options_success.second;
-  if (!get_query_options_success) {
+  RewriteQuery rewrite_query;
+  if (!cfg_s->server_context->GetQueryOptions(
+          url, request_headers, response_headers, &rewrite_query)) {
     // Failed to parse query params or request headers.  Treat this as if there
     // were no query params given.
     ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
@@ -1216,7 +1258,7 @@ RewriteOptions* ps_determine_request_options(
 
   // Will be NULL if there aren't any options set with query params or in
   // headers.
-  return query_options_success.first;
+  return rewrite_query.ReleaseOptions();
 }
 
 // Check whether this visitor is already in an experiment.  If they're not,
@@ -1295,11 +1337,12 @@ bool ps_determine_options(ngx_http_request_t* r,
   // rebased on the directory options or the global options.
   RewriteOptions* request_options = ps_determine_request_options(
       r, request_headers, response_headers, cfg_s, url);
+  bool have_request_options = request_options != NULL;
 
   // Because the caller takes ownership of any options we return, the only
   // situation in which we can avoid allocating a new RewriteOptions is if the
   // global options are ok as are.
-  if (directory_options == NULL && request_options == NULL &&
+  if (!have_request_options && directory_options == NULL &&
       !global_options->running_experiment()) {
     return true;
   }
@@ -1311,14 +1354,24 @@ bool ps_determine_options(ngx_http_request_t* r,
     *options = global_options->Clone();
   }
 
-  // Modify our options in response to request options or experiment settings,
-  // if we need to.  If there are request options then ignore the experiment
-  // because we don't want experiments to be contaminated with unexpected
-  // settings.
-  if (request_options != NULL) {
+  // Modify our options in response to request options if specified.
+  if (have_request_options) {
     (*options)->Merge(*request_options);
     delete request_options;
-  } else if ((*options)->running_experiment() && html_rewrite) {
+    request_options = NULL;
+  }
+
+  // If we're running an experiment and processing html then modify our options
+  // in response to the experiment.  Except we generally don't want experiments
+  // to be contaminated with unexpected settings, so ignore experiments if we
+  // have request-specific options.  Unless EnrollExperiment is on, probably set
+  // by a query parameter, in which case we want to go ahead and apply the
+  // experimental settings even if it means bad data, because we're just seeing
+  // what it looks like.
+  if ((*options)->running_experiment() &&
+      html_rewrite &&
+      (!have_request_options ||
+       (*options)->enroll_experiment())) {
     bool ok = ps_set_experiment_state_and_cookie(
         r, request_headers, *options, url->Host());
     if (!ok) {
@@ -1355,7 +1408,8 @@ bool ps_apply_x_forwarded_proto(ngx_http_request_t* r, GoogleString* url) {
     return false;  // No X-Forwarded-Proto header found.
   }
 
-  StringPiece x_forwarded_proto = str_to_string_piece(*x_forwarded_proto_header);
+  StringPiece x_forwarded_proto =
+      str_to_string_piece(*x_forwarded_proto_header);
   if (!STR_CASE_EQ_LITERAL(*x_forwarded_proto_header, "http") &&
       !STR_CASE_EQ_LITERAL(*x_forwarded_proto_header, "https")) {
     LOG(WARNING) << "Unsupported X-Forwarded-Proto: " << x_forwarded_proto
@@ -1481,11 +1535,6 @@ void ps_release_request_context(void* data) {
     ctx->recorder = NULL;
   }
 
-  if (ctx->ipro_response_headers != NULL) {
-    delete ctx->ipro_response_headers;
-    ctx->ipro_response_headers = NULL;
-  }
-
   ps_release_base_fetch(ctx);
   delete ctx;
 }
@@ -1518,18 +1567,29 @@ RequestRouting::Response ps_route_request(ngx_http_request_t* r,
 
   if (is_pagespeed_subrequest(r)) {
     return RequestRouting::kPagespeedSubrequest;
-  } else if (url.PathSansLeaf() == NgxRewriteDriverFactory::kStaticAssetPrefix) {
+  } else if (url.PathSansLeaf() ==
+             NgxRewriteDriverFactory::kStaticAssetPrefix) {
     return RequestRouting::kStaticContent;
-  } else if (url.PathSansQuery() == "/ngx_pagespeed_statistics" ||
-             url.PathSansQuery() == "/ngx_pagespeed_global_statistics" ) {
-    return RequestRouting::kStatistics;
-  } else if (url.PathSansQuery() == "/pagespeed_console") {
-    return RequestRouting::kConsole;
-  } else if (url.PathSansQuery() == "/ngx_pagespeed_message") {
-    return RequestRouting::kMessages;
   }
 
-  RewriteOptions* global_options = cfg_s->server_context->global_options();
+  NgxRewriteOptions* global_options = cfg_s->server_context->config();
+
+  StringPiece path = url.PathSansQuery();
+  if (path == global_options->statistics_path()) {
+    return RequestRouting::kStatistics;
+  } else if (path == global_options->global_statistics_path()) {
+    return RequestRouting::kGlobalStatistics;
+  } else if (path == global_options->console_path()) {
+    return RequestRouting::kConsole;
+  } else if (path == global_options->messages_path()) {
+    return RequestRouting::kMessages;
+  } else if (!global_options->admin_path().empty() &&
+             path.starts_with(global_options->admin_path())) {
+    return RequestRouting::kAdmin;
+  } else if (!global_options->global_admin_path().empty() &&
+             path.starts_with(global_options->global_admin_path())) {
+    return RequestRouting::kGlobalAdmin;
+  }
 
   const GoogleString* beacon_url;
   if (ps_is_https(r)) {
@@ -1545,7 +1605,9 @@ RequestRouting::Response ps_route_request(ngx_http_request_t* r,
   return RequestRouting::kResource;
 }
 
-ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
+ngx_int_t ps_resource_handler(ngx_http_request_t* r,
+                              bool html_rewrite,
+                              RequestRouting::Response response_category) {
   if (r != r->main) {
     return NGX_DECLINED;
   }
@@ -1555,7 +1617,9 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
 
   CHECK(!(html_rewrite && (ctx == NULL || ctx->html_rewrite == false)));
 
-  if (!html_rewrite && r->method != NGX_HTTP_GET && r->method != NGX_HTTP_HEAD) {
+  if (!html_rewrite &&
+      r->method != NGX_HTTP_GET &&
+      r->method != NGX_HTTP_HEAD) {
     return NGX_DECLINED;
   }
 
@@ -1602,6 +1666,15 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
     }
   }
 
+  bool pagespeed_resource =
+      !html_rewrite && cfg_s->server_context->IsPagespeedResource(url);
+  bool is_an_admin_handler =
+      response_category == RequestRouting::kStatistics ||
+      response_category == RequestRouting::kGlobalStatistics ||
+      response_category == RequestRouting::kConsole ||
+      response_category == RequestRouting::kAdmin ||
+      response_category == RequestRouting::kGlobalAdmin;
+
   if (html_rewrite) {
     ps_release_base_fetch(ctx);
   } else {
@@ -1610,11 +1683,12 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
     ctx = new ps_request_ctx_t();
 
     ctx->r = r;
-    ctx->ipro_response_headers = NULL;
     ctx->write_pending = false;
     ctx->html_rewrite = false;
     ctx->in_place = false;
     ctx->pagespeed_connection = NULL;
+    ctx->preserve_caching_headers = kDontPreserveHeaders;
+
     // See build_context_for_request() in mod_instaweb.cc
     // TODO(jefftk): Is this the right place to be modifying caching headers for
     // html fetches?  Or should that be done later, in the headers flow for
@@ -1625,7 +1699,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
       // Downstream cache integration is not enabled. Disable original
       // Cache-Control headers.
       ctx->preserve_caching_headers = kDontPreserveHeaders;
-    } else {
+    } else if (!pagespeed_resource && !is_an_admin_handler) {
       ctx->preserve_caching_headers = kPreserveOnlyCacheControl;
       // Downstream cache integration is enabled. If a rebeaconing key has been
       // configured and there is a ShouldBeacon header with the correct key,
@@ -1636,7 +1710,9 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
         ctx->preserve_caching_headers = kDontPreserveHeaders;
       }
     }
+
     ctx->recorder = NULL;
+    ctx->url_string = url_string;
 
     // Set up a cleanup handler on the request.
     ngx_http_cleanup_t* cleanup = ngx_http_cleanup_add(r, 0);
@@ -1669,7 +1745,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
               false /* requires_blink_cohort (no longer unused) */,
               &page_callback_added));
 
-  if (!html_rewrite && cfg_s->server_context->IsPagespeedResource(url)) {
+  if (pagespeed_resource) {
     // TODO(jefftk): Set using_spdy appropriately.  See
     // ProxyInterface::ProxyRequestCallback
     ResourceFetch::Start(
@@ -1677,6 +1753,42 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
         custom_options.release() /* null if there aren't custom options */,
         false /* using_spdy */, cfg_s->server_context, ctx->base_fetch);
     return ps_async_wait_response(r);
+  } else if (is_an_admin_handler) {
+    QueryParams query_params;
+    query_params.Parse(url.Query());
+
+    PosixTimer timer;
+    int64 now_ms = timer.NowMs();
+    ctx->base_fetch->response_headers()->SetDateAndCaching(
+        now_ms, 0 /* max-age */, ", no-cache");
+
+    if (response_category == RequestRouting::kStatistics ||
+        response_category == RequestRouting::kGlobalStatistics) {
+      cfg_s->server_context->StatisticsPage(
+          response_category == RequestRouting::kGlobalStatistics,
+          query_params,
+          cfg_s->server_context->config(),
+          ctx->base_fetch);
+    } else if (response_category == RequestRouting::kConsole) {
+      cfg_s->server_context->ConsoleHandler(
+          *cfg_s->server_context->config(),
+          SystemServerContext::kStatistics,
+          query_params,
+          ctx->base_fetch);
+    } else if (response_category == RequestRouting::kAdmin ||
+               response_category == RequestRouting::kGlobalAdmin) {
+      cfg_s->server_context->AdminPage(
+          response_category == RequestRouting::kGlobalAdmin,
+          url,
+          query_params,
+          custom_options == NULL ? cfg_s->server_context->config()
+                                 : custom_options.get(),
+          ctx->base_fetch);
+    } else {
+      CHECK(false);
+    }
+
+    return ps_async_wait_response(r);
   }
 
   if (html_rewrite) {
@@ -1983,7 +2095,8 @@ ngx_int_t ps_html_rewrite_header_filter(ngx_http_request_t* r) {
     return ngx_http_next_header_filter(r);
   }
 
-  ngx_int_t rc = ps_resource_handler(r, true /* html rewrite */);
+  ngx_int_t rc = ps_resource_handler(r, true /* html rewrite */,
+                                     RequestRouting::kResource);
   if (rc != NGX_OK) {
     ctx->html_rewrite = false;
     return ngx_http_next_header_filter(r);
@@ -2084,12 +2197,40 @@ ngx_http_output_body_filter_pt ngx_http_next_body_filter;
 ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
   ps_request_ctx_t* ctx = ps_get_request_context(r);
 
-  if (ctx == NULL || !ctx->in_place) {
+  if (ctx == NULL) {
+    return ngx_http_next_header_filter(r);
+  }
+
+  if (ctx->recorder != NULL) {
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "ps in place check header filter recording: %V", &r->uri);
+
+    CHECK(!ctx->in_place);
+
+    // We didn't find this resource in cache originally, so we're recording it
+    // as it passes us by.  At this point the headers from things that run
+    // before us are set but not things that run after us, which means here is
+    // where we need to check whether there's a "Content-Encoding: gzip".  If we
+    // waited to do this in ps_in_place_body_filter we wouldn't be able to tell
+    // the difference between response headers that have "C-E: gz" because we're
+    // proxying for an upstream that gzipped the content and response headers
+    // that have it because the gzip filter (which runs after us) is going to
+    // produce gzipped output.
+    //
+    // The recorder will do this checking, so pass it the headers.
+    ResponseHeaders response_headers;
+    copy_response_headers_from_ngx(r, &response_headers);
+    ctx->recorder->ConsiderResponseHeaders(
+        InPlaceResourceRecorder::kPreliminaryHeaders, &response_headers);
+    return ngx_http_next_header_filter(r);
+  }
+
+  if (!ctx->in_place) {
     return ngx_http_next_header_filter(r);
   }
 
   ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                 "ps in place check header filter: %V", &r->uri);
+                 "ps in place check header filter initial: %V", &r->uri);
 
   int status_code = r->headers_out.status;
   bool status_ok = (status_code != 0) && (status_code < 400);
@@ -2098,6 +2239,9 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
   NgxServerContext* server_context = cfg_s->server_context;
   MessageHandler* message_handler = cfg_s->handler;
   GoogleString url = ps_determine_url(r);
+  // The URL we use for cache key is a bit different since it may
+  // have PageSpeed query params removed.
+  GoogleString cache_url = ctx->url_string;
 
   // continue process
   if (status_ok) {
@@ -2118,7 +2262,7 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
         kInfo,
         "Could not rewrite resource in-place "
         "because URL is not in cache: %s",
-        url.c_str());
+        cache_url.c_str());
     const SystemRewriteOptions* options = SystemRewriteOptions::DynamicCast(
         ctx->driver->options());
     RequestHeaders request_headers;
@@ -2128,8 +2272,10 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
     // (or at least a note that it cannot be cached stored there).
     // We do that using an Apache output filter.
     ctx->recorder = new InPlaceResourceRecorder(
-        url,
-        request_headers,
+        RequestContextPtr(cfg_s->server_context->NewRequestContext(r)),
+        cache_url,
+        ctx->driver->CacheFragment(),
+        request_headers.GetProperties(),
         options->respect_vary(),
         options->ipro_max_response_bytes(),
         options->ipro_max_concurrent_recordings(),
@@ -2139,6 +2285,9 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
         message_handler);
     // set in memory flag for in place_body_filter
     r->filter_need_in_memory = 1;
+
+    // We don't have the response headers at all yet because we haven't yet gone
+    // to the backend.
   } else {
     server_context->rewrite_stats()->ipro_not_rewritable()->Add(1);
     message_handler->Message(kInfo,
@@ -2164,42 +2313,11 @@ ngx_int_t ps_in_place_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
                  "ps in place body filter: %V", &r->uri);
 
   InPlaceResourceRecorder* recorder = ctx->recorder;
-
-  if (ctx->ipro_response_headers == NULL) {
-    // Prepare response headers.
-    ctx->ipro_response_headers = new ResponseHeaders();
-
-    // TODO(oschaaf): We don't get a Date response header here.
-    // Currently, we invent one and set it to the current date/time.
-    // We need to investigate why we don't receive it.
-    ctx->ipro_response_headers->set_major_version(r->http_version / 1000);
-    ctx->ipro_response_headers->set_minor_version(r->http_version % 1000);
-    copy_headers_from_table(r->headers_out.headers, ctx->ipro_response_headers);
-    ctx->ipro_response_headers->set_status_code(r->headers_out.status);
-    ctx->ipro_response_headers->Add(HttpAttributes::kContentType,
-                str_to_string_piece(r->headers_out.content_type));
-    if (r->headers_out.location != NULL) {
-      ctx->ipro_response_headers->Add(HttpAttributes::kLocation,
-                  str_to_string_piece(r->headers_out.location->value));
-    }
-    StringPiece date =
-        ctx->ipro_response_headers->Lookup1(HttpAttributes::kDate);
-    if (date.empty()) {
-      ctx->ipro_response_headers->SetDate(ngx_current_msec);
-    }
-    ctx->ipro_response_headers->ComputeCaching();
-
-    // Unlike in Apache we get the final response headers before we get the
-    // content.  This means we can consider them earlier and abort the
-    // request if need be without buffering everything.
-    recorder->ConsiderResponseHeaders(ctx->ipro_response_headers);
-  }
-
   for (ngx_chain_t* cl = in; cl; cl = cl->next) {
     if (ngx_buf_size(cl->buf)) {
        CHECK(ngx_buf_in_memory(cl->buf));
-       StringPiece contents(reinterpret_cast<char *>(cl->buf->pos),
-                                 ngx_buf_size(cl->buf));
+       StringPiece contents(reinterpret_cast<char*>(cl->buf->pos),
+                            ngx_buf_size(cl->buf));
        recorder->Write(contents, recorder->handler());
     }
 
@@ -2208,7 +2326,9 @@ ngx_int_t ps_in_place_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
     }
 
     if (cl->buf->last_buf || recorder->failed()) {
-      ctx->recorder->DoneAndSetHeaders(ctx->ipro_response_headers);
+      ResponseHeaders response_headers;
+      copy_response_headers_from_ngx(r, &response_headers);
+      ctx->recorder->DoneAndSetHeaders(&response_headers);
       ctx->recorder = NULL;
       break;
     }
@@ -2267,6 +2387,13 @@ ngx_int_t ps_simple_handler(ngx_http_request_t* r,
   NgxMessageHandler* message_handler = factory->ngx_message_handler();
   StringPiece request_uri_path = str_to_string_piece(r->uri);
 
+  GoogleString url_string = ps_determine_url(r);
+  GoogleUrl url(url_string);
+  QueryParams query_params;
+  if (url.IsWebValid()) {
+    query_params.Parse(url.Query());
+  }
+
   GoogleString output;
   StringWriter writer(&output);
   HttpStatus::Code status = HttpStatus::kOK;
@@ -2286,22 +2413,6 @@ ngx_int_t ps_simple_handler(ngx_http_request_t* r,
       file_contents.CopyToString(&output);
       break;
     }
-    case RequestRouting::kStatistics:
-      error_message = StatisticsHandler(
-          factory,
-          server_context,
-          NULL,  // No SPDY-specific config in ngx_pagespeed.
-          !factory->use_per_vhost_statistics() || StringCaseStartsWith(
-              request_uri_path, "/ngx_pagespeed_global_statistics"),
-          StringPiece(reinterpret_cast<char*>(r->args.data), r->args.len),
-          &content_type,
-          &writer,
-          message_handler);
-      break;
-    case RequestRouting::kConsole:
-      ConsoleHandler(
-          server_context, server_context->config(), &writer, message_handler);
-      break;
     case RequestRouting::kMessages: {
       GoogleString log;
       StringWriter log_writer(&log);
@@ -2541,12 +2652,16 @@ ngx_int_t ps_content_handler(ngx_http_request_t* r) {
     case RequestRouting::kBeacon:
       return ps_beacon_handler(r);
     case RequestRouting::kStaticContent:
-    case RequestRouting::kStatistics:
-    case RequestRouting::kConsole:
     case RequestRouting::kMessages:
       return ps_simple_handler(r, cfg_s->server_context, response_category);
+    case RequestRouting::kStatistics:
+    case RequestRouting::kGlobalStatistics:
+    case RequestRouting::kConsole:
+    case RequestRouting::kAdmin:
+    case RequestRouting::kGlobalAdmin:
     case RequestRouting::kResource:
-      return ps_resource_handler(r, false /* html rewrite */);
+      return ps_resource_handler(
+          r, false /* html rewrite */, response_category);
   }
 
   CHECK(0);

src/ngx_pagespeed.h

@@ -103,6 +103,10 @@ typedef struct {
   RewriteDriver* driver;
   InPlaceResourceRecorder* recorder;
   ResponseHeaders* ipro_response_headers;
+
+  // We need to remember the URL here as well since we may modify what NGX
+  // gets by stripping our special query params and honoring X-Forwarded-Proto.
+  GoogleString url_string;
 } ps_request_ctx_t;
 
 
@@ -117,6 +121,8 @@ ngx_int_t copy_response_headers_to_ngx(
     const ResponseHeaders& pagespeed_headers,
     PreserveCachingHeaders preserve_caching_headers);
 
+StringPiece ps_determine_host(ngx_http_request_t* r);
+
 }  // namespace net_instaweb
 
 #endif  // NGX_PAGESPEED_H_

src/ngx_rewrite_options.cc

@@ -37,6 +37,13 @@ namespace net_instaweb {
 
 namespace {
 
+const char kStatisticsPath[] = "StatisticsPath";
+const char kGlobalStatisticsPath[] = "GlobalStatisticsPath";
+const char kConsolePath[] = "ConsolePath";
+const char kMessagesPath[] = "MessagesPath";
+const char kAdminPath[] = "AdminPath";
+const char kGlobalAdminPath[] = "GlobalAdminPath";
+
 // These options are copied from mod_instaweb.cc, where APACHE_CONFIG_OPTIONX
 // indicates that they can not be set at the directory/location level. They set
 // options in the RewriteDriverFactory, so they do not appear in RewriteOptions.
@@ -70,8 +77,6 @@ const char* const main_only_options[] = {
   "UseNativeFetcher"
 };
 
-const char kNgxPagespeedStatisticsHandlerPath[] = "/ngx_pagespeed_statistics";
-
 }  // namespace
 
 RewriteOptions::Properties* NgxRewriteOptions::ngx_properties_ = NULL;
@@ -91,15 +96,29 @@ void NgxRewriteOptions::Init() {
   DCHECK(ngx_properties_ != NULL)
       << "Call NgxRewriteOptions::Initialize() before construction";
   InitializeOptions(ngx_properties_);
-
-  // Nginx-specific default.
-  // TODO(sligocki): Get rid of this line and let both Apache and Nginx use
-  // /pagespeed_statistics as the handler.
-  statistics_handler_path_.set_default(kNgxPagespeedStatisticsHandlerPath);
 }
 
 void NgxRewriteOptions::AddProperties() {
-  // Nothing ngx-specific for now.
+  // Nginx-specific options.
+  add_ngx_option(
+      "", &NgxRewriteOptions::statistics_path_, "nsp", kStatisticsPath,
+      kProcessScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics");
+  add_ngx_option(
+      "", &NgxRewriteOptions::global_statistics_path_, "ngsp",
+      kGlobalStatisticsPath, kProcessScope,
+      "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics");
+  add_ngx_option(
+      "", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kProcessScope,
+      "Set the console path. Ex: /pagespeed_console");
+  add_ngx_option(
+      "", &NgxRewriteOptions::messages_path_, "nmp", kMessagesPath,
+      kProcessScope, "Set the messages path.  Ex: /ngx_pagespeed_message");
+  add_ngx_option(
+      "", &NgxRewriteOptions::admin_path_, "nap", kAdminPath,
+      kProcessScope, "Set the admin path.  Ex: /pagespeed_admin");
+  add_ngx_option(
+      "", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
+      kProcessScope, "Set the global admin path.  Ex: /pagespeed_global_admin");
 
   MergeSubclassProperties(ngx_properties_);
 

src/ngx_rewrite_options.h

@@ -40,7 +40,8 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   static void Initialize();
   static void Terminate();
 
-  NgxRewriteOptions(const StringPiece& description, ThreadSystem* thread_system);
+  NgxRewriteOptions(const StringPiece& description,
+                    ThreadSystem* thread_system);
   explicit NgxRewriteOptions(ThreadSystem* thread_system);
   virtual ~NgxRewriteOptions() { }
 
@@ -67,6 +68,24 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   static const NgxRewriteOptions* DynamicCast(const RewriteOptions* instance);
   static NgxRewriteOptions* DynamicCast(RewriteOptions* instance);
 
+  const GoogleString& statistics_path() const {
+    return statistics_path_.value();
+  }
+  const GoogleString& global_statistics_path() const {
+    return global_statistics_path_.value();
+  }
+  const GoogleString& console_path() const {
+    return console_path_.value();
+  }
+  const GoogleString& messages_path() const {
+    return messages_path_.value();
+  }
+  const GoogleString& admin_path() const {
+    return admin_path_.value();
+  }
+  const GoogleString& global_admin_path() const {
+    return global_admin_path_.value();
+  }
 
  private:
   // Helper methods for ParseAndSetOptions().  Each can:
@@ -108,10 +127,20 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   static void add_ngx_option(typename OptionClass::ValueType default_value,
                              OptionClass NgxRewriteOptions::*offset,
                              const char* id,
-                             StringPiece option_name) {
-    AddProperty(default_value, offset, id, option_name, ngx_properties_);
+                             StringPiece option_name,
+                             OptionScope scope,
+                             const char* help) {
+    AddProperty(default_value, offset, id, option_name, scope, help,
+                ngx_properties_);
   }
 
+  Option<GoogleString> statistics_path_;
+  Option<GoogleString> global_statistics_path_;
+  Option<GoogleString> console_path_;
+  Option<GoogleString> messages_path_;
+  Option<GoogleString> admin_path_;
+  Option<GoogleString> global_admin_path_;
+
   // Helper for ParseAndSetOptions.  Returns whether the two directives equal,
   // ignoring case.
   bool IsDirective(StringPiece config_directive, StringPiece compare_directive);

src/ngx_server_context.cc

@@ -72,6 +72,7 @@ SystemRequestContext* NgxServerContext::NewRequestContext(
 
   return new SystemRequestContext(thread_system()->NewMutex(),
                                   timer(),
+                                  ps_determine_host(r),
                                   local_port,
                                   str_to_string_piece(local_ip));
 }

src/ngx_url_async_fetcher.cc

@@ -221,7 +221,7 @@ namespace net_instaweb {
   void NgxUrlAsyncFetcher::Fetch(const GoogleString& url,
                                  MessageHandler* message_handler,
                                  AsyncFetch* async_fetch) {
-    async_fetch = EnableInflation(async_fetch, NULL);
+    async_fetch = EnableInflation(async_fetch);
     NgxFetch* fetch = new NgxFetch(url, async_fetch,
           message_handler, fetch_timeout_, log_);
     ScopedMutex lock(mutex_);

test/nginx_system_test.sh

@@ -127,6 +127,7 @@ check_simple mkdir "$FILE_CACHE"
 
 # And directories that don't.
 SECONDARY_CACHE="$TEST_TMP/file-cache/secondary/"
+IPRO_CACHE="$TEST_TMP/file-cache/ipro/"
 SHM_CACHE="$TEST_TMP/file-cache/intermediate/directories/with_shm/"
 
 VALGRIND_OPTIONS=""
@@ -162,6 +163,7 @@ cat $PAGESPEED_CONF_TEMPLATE \
   | sed 's#@@ACCESS_LOG@@#'"$ACCESS_LOG"'#' \
   | sed 's#@@FILE_CACHE@@#'"$FILE_CACHE/"'#' \
   | sed 's#@@SECONDARY_CACHE@@#'"$SECONDARY_CACHE/"'#' \
+  | sed 's#@@IPRO_CACHE@@#'"$IPRO_CACHE/"'#' \
   | sed 's#@@SHM_CACHE@@#'"$SHM_CACHE/"'#' \
   | sed 's#@@SERVER_ROOT@@#'"$SERVER_ROOT"'#' \
   | sed 's#@@PRIMARY_PORT@@#'"$PRIMARY_PORT"'#' \
@@ -256,7 +258,6 @@ PSA_JS_LIBRARY_URL_PREFIX="ngx_pagespeed_static"
 
 # An expected failure can be indicated like: "~In-place resource optimization~"
 PAGESPEED_EXPECTED_FAILURES="
-~IPRO-optimized resources should have fixed size, not chunked.~
 "
 
 # Some tests are flakey under valgrind. For now, add them to the expected failures
@@ -276,7 +277,7 @@ fi
 set -- "$PRIMARY_HOSTNAME"
 source $SYSTEM_TEST_FILE
 
-STATISTICS_URL=http://$HOSTNAME/ngx_pagespeed_statistics
+STATISTICS_URL=$PRIMARY_SERVER/ngx_pagespeed_statistics
 
 # Define a mechanism to start a test before the cache-flush and finish it
 # after the cache-flush.  This mechanism is preferable to flushing cache
@@ -303,6 +304,28 @@ function run_post_cache_flush() {
 
 # nginx-specific system tests
 
+start_test Test pagespeed directive inside if block inside location block.
+
+URL="http://if-in-location.example.com/"
+URL+="mod_pagespeed_example/inline_javascript.html"
+
+# When we specify the X-Custom-Header-Inline-Js that triggers an if block in the
+# config which turns on inline_javascript.
+WGET_ARGS="--header=X-Custom-Header-Inline-Js:Yes"
+http_proxy=$SECONDARY_HOSTNAME \
+  fetch_until $URL 'grep -c document.write' 1
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $WGET_ARGS $URL)
+check_from "$OUT" fgrep "X-Inline-Javascript: Yes"
+check_not_from "$OUT" fgrep "inline_javascript.js"
+
+# Without that custom header we don't trigger the if block, and shouldn't get
+# any inline javascript.
+WGET_ARGS=""
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $WGET_ARGS $URL)
+check_from "$OUT" fgrep "X-Inline-Javascript: No"
+check_from "$OUT" fgrep "inline_javascript.js"
+check_not_from "$OUT" fgrep "document.write"
+
 # Tests related to rewritten response (downstream) caching.
 
 if [ "$NATIVE_FETCHER" = "on" ]; then
@@ -398,7 +421,7 @@ echo "Final 404s: $NUM_404_FINAL"
 check [ $(expr $NUM_404_FINAL - $NUM_404) -eq 1 ]
 
 # Check that the stat doesn't get bumped on non-404s.
-URL="http://$HOSTNAME/mod_pagespeed_example/styles/"
+URL="$PRIMARY_SERVER/mod_pagespeed_example/styles/"
 URL+="W.rewrite_css_images.css.pagespeed.cf.Hash.css"
 OUT=$(wget -O - -q $URL)
 check_from "$OUT" grep background-image
@@ -433,8 +456,97 @@ if [ "$HOSTNAME" = "localhost:$PRIMARY_PORT" ] ; then
   wget -O - --header="Host: $HOSTNAME" $ALT_CE_URL >& "$TEMPDIR/alt_ce_url.$$"
   check [ $? = 8 ]
   rm -f "$TEMPDIR/alt_ce_url.$$"
+
+  # Even though we don't have a cookie, we will conservatively avoid
+  # optimizing resources with Vary:Cookie set on the response, so we
+  # will not get the instant response, of "body{background:#9370db}":
+  # 24 bytes, but will get the full original text:
+  #     "body {\n    background: MediumPurple;\n}\n"
+  # This will happen whether or not we send a cookie.
+  #
+  # Testing this requires proving we'll never optimize something, which
+  # can't be distinguished from the not-yet-optimized case, except by the
+  # ipro_not_rewritable stat, so we loop by scraping that stat and seeing
+  # when it changes.
+
+  # Executes commands until ipro_no_rewrite_count changes.  The
+  # command-line options are all passed to WGET_DUMP.  Leaves command
+  # wget output in $IPRO_OUTPUT.
+  function ipro_expect_no_rewrite() {
+    ipro_no_rewrite_count_start=$(scrape_stat ipro_not_rewritable)
+    ipro_no_rewrite_count=$ipro_no_rewrite_count_start
+    iters=0
+    while [ $ipro_no_rewrite_count -eq $ipro_no_rewrite_count_start ]; do
+      if [ $iters -ne 0 ]; then
+        sleep 0.1
+        if [ $iters -gt 100 ]; then
+          echo TIMEOUT
+          exit 1
+        fi
+      fi
+      IPRO_OUTPUT=$($WGET_DUMP "$@")
+      ipro_no_rewrite_count=$(scrape_stat ipro_not_rewritable)
+      iters=$((iters + 1))
+    done
+  }
+
+  start_test ipro with vary:cookie with no cookie set
+  ipro_expect_no_rewrite $TEST_ROOT/ipro/cookie/vary_cookie.css
+  check_from "$IPRO_OUTPUT" fgrep -q '    background: MediumPurple;'
+  check_from "$IPRO_OUTPUT" fgrep -q 'Vary: Cookie'
+
+  start_test ipro with vary:cookie with cookie set
+  ipro_expect_no_rewrite $TEST_ROOT/ipro/cookie/vary_cookie.css \
+    --header=Cookie:cookie-data
+  check_from "$IPRO_OUTPUT" fgrep -q '    background: MediumPurple;'
+  check_from "$IPRO_OUTPUT" fgrep -q 'Vary: Cookie'
+
+  start_test ipro with vary:cookie2 with no cookie2 set
+  ipro_expect_no_rewrite $TEST_ROOT/ipro/cookie2/vary_cookie2.css
+  check_from "$IPRO_OUTPUT" fgrep -q '    background: MediumPurple;'
+  check_from "$IPRO_OUTPUT" fgrep -q 'Vary: Cookie2'
+
+  start_test ipro with vary:cookie2 with cookie2 set
+  ipro_expect_no_rewrite $TEST_ROOT/ipro/cookie2/vary_cookie2.css \
+    --header=Cookie2:cookie2-data
+  check_from "$IPRO_OUTPUT" fgrep -q '    background: MediumPurple;'
+  check_from "$IPRO_OUTPUT" fgrep -q 'Vary: Cookie2'
+
+  start_test authorized resources do not get cached and optimized.
+  URL="$TEST_ROOT/auth/medium_purple.css"
+  AUTH="Authorization:Basic dXNlcjE6cGFzc3dvcmQ="
+  not_cacheable_start=$(scrape_stat ipro_recorder_not_cacheable)
+  echo $WGET_DUMP --header="$AUTH" "$URL"
+  OUT=$($WGET_DUMP --header="$AUTH" "$URL")
+  check_from "$OUT" fgrep -q 'background: MediumPurple;'
+  not_cacheable=$(scrape_stat ipro_recorder_not_cacheable)
+  check [ $not_cacheable = $((not_cacheable_start + 1)) ]
+  URL=""
+  AUTH=""
 fi
 
+WGET_ARGS=""
+function gunzip_grep_0ff() {
+  gunzip - | fgrep -q "color:#00f"
+  echo $?
+}
+
+start_test ipro with mod_deflate
+CSS_FILE="http://compressed-css.example.com/"
+CSS_FILE+="mod_pagespeed_test/ipro/mod_deflate/big.css"
+http_proxy=$SECONDARY_HOSTNAME fetch_until -gzip $CSS_FILE gunzip_grep_0ff 0
+
+start_test ipro with reverse proxy of compressed content
+http_proxy=$SECONDARY_HOSTNAME \
+  fetch_until -gzip http://ipro-proxy.example.com/big.css \
+    gunzip_grep_0ff 0
+
+# Also test the .pagespeed. version, to make sure we didn't accidentally gunzip
+# stuff above when we shouldn't have.
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET -q -O - \
+      http://ipro-proxy.example.com/A.big.css.pagespeed.cf.0.css)
+check_from "$OUT" fgrep -q "big{color:#00f}"
+
 start_test Accept bad query params and headers
 
 # The examples page should have this EXPECTED_EXAMPLES_TEXT on it.
@@ -450,6 +562,22 @@ check_from "$OUT" grep "$EXPECTED_EXAMPLES_TEXT"
 OUT=$(wget -O - --header=PageSpeedFilters:bogus $EXAMPLE_ROOT)
 check_from "$OUT" grep "$EXPECTED_EXAMPLES_TEXT"
 
+# Tests that an origin header with a Vary header other than Vary:Accept-Encoding
+# loses that header when we are not respecting vary.
+start_test Vary:User-Agent on resources is held by our cache.
+URL="$TEST_ROOT/vary/no_respect/index.html"
+fetch_until -save $URL 'grep -c \.pagespeed\.cf\.' 1
+
+# Extract out the rewritten CSS file from the HTML saved by fetch_until
+# above (see -save and definition of fetch_until).  Fetch that CSS
+# file with headers and make sure the Vary is stripped.
+CSS_URL=$(grep stylesheet $FETCH_UNTIL_OUTFILE | cut -d\" -f 4)
+CSS_URL="$TEST_ROOT/vary/no_respect/$(basename $CSS_URL)"
+echo CSS_URL=$CSS_URL
+CSS_OUT=$($WGET_DUMP $CSS_URL)
+check_from "$CSS_OUT" fgrep -q "Vary: Accept-Encoding"
+check_not_from "$CSS_OUT" fgrep -q "User-Agent"
+
 # Test that loopback route fetcher works with vhosts not listening on
 # 127.0.0.1
 start_test IP choice for loopback fetches.
@@ -622,7 +750,15 @@ fetch_until $URL 'fgrep -c file.exception.ssp.css' 1
 start_test statistics load
 
 OUT=$($WGET_DUMP $STATISTICS_URL)
-check_from "$OUT" grep 'VHost-Specific Statistics'
+check_from "$OUT" grep 'PageSpeed Statistics'
+
+start_test statistics handler full-featured
+OUT=$($WGET_DUMP $STATISTICS_URL?config)
+check_from "$OUT" grep "InPlaceResourceOptimization (ipro)"
+
+start_test statistics handler properly sets JSON content-type
+OUT=$($WGET_DUMP $STATISTICS_URL?json)
+check_from "$OUT" grep "Content-Type: application/javascript"
 
 start_test scrape stats works
 
@@ -676,15 +812,51 @@ start_test UseExperimentalJsMinifier
 URL="$TEST_ROOT/experimental_js_minifier/index.html"
 URL+="?PageSpeedFilters=rewrite_javascript"
 # External scripts rewritten.
-fetch_until -save -recursive \
-  $URL 'grep -c src=.*rewrite_javascript\.js\.pagespeed\.jm\.' 2
-check_not grep removed $WGET_DIR/*.pagespeed.jm.*  # No comments should remain.
-check_file_size $FETCH_FILE -lt 1560               # Net savings
-check grep -q preserved $FETCH_FILE                # Preserves certain comments.
+fetch_until -save -recursive $URL 'grep -c src=.*\.pagespeed\.jm\.' 1
+check_not grep "removed" $WGET_DIR/*   # No comments should remain.
+check grep -q "preserved" $WGET_DIR/*  # Contents of <script src=> element kept.
+ORIGINAL_HTML_SIZE=1484
+check_file_size $FETCH_FILE -lt $ORIGINAL_HTML_SIZE  # Net savings
 # Rewritten JS is cache-extended.
 check grep -qi "Cache-control: max-age=31536000" $WGET_OUTPUT
 check grep -qi "Expires:" $WGET_OUTPUT
 
+start_test Source map tests
+URL="$TEST_ROOT/experimental_js_minifier/index.html"
+URL+="?PageSpeedFilters=rewrite_javascript,include_js_source_maps"
+# All rewriting still happening as expected.
+fetch_until -save -recursive $URL 'grep -c src=.*\.pagespeed\.jm\.' 1
+check_not grep "removed" $WGET_DIR/*  # No comments should remain.
+check_file_size $FETCH_FILE -lt $ORIGINAL_HTML_SIZE  # Net savings
+check grep -qi "Cache-control: max-age=31536000" $WGET_OUTPUT
+check grep -qi "Expires:" $WGET_OUTPUT
+
+# No source map for inline JS
+check_not grep sourceMappingURL $FETCH_FILE
+# Yes source_map for external JS
+check grep -q sourceMappingURL $WGET_DIR/script.js.pagespeed.*
+SOURCE_MAP_URL=$(grep sourceMappingURL $WGET_DIR/script.js.pagespeed.* |
+                 grep -o 'http://.*')
+OUTFILE=$OUTDIR/source_map
+check $WGET_DUMP -O $OUTFILE $SOURCE_MAP_URL
+check grep -qi "Cache-control: max-age=31536000" $OUTFILE  # Long cache
+check grep -q "script.js?PageSpeed=off" $OUTFILE  # Has source URL.
+check grep -q '"mappings":' $OUTFILE  # Has mappings.
+
+start_test IPRO source map tests
+URL="$TEST_ROOT/experimental_js_minifier/script.js"
+URL+="?PageSpeedFilters=rewrite_javascript,include_js_source_maps"
+# Fetch until IPRO removes comments.
+fetch_until -save $URL 'grep -c removed' 0
+# Yes source_map for external JS
+check grep -q sourceMappingURL $FETCH_FILE
+SOURCE_MAP_URL=$(grep sourceMappingURL $FETCH_FILE | grep -o 'http://.*')
+OUTFILE=$OUTDIR/source_map
+check $WGET_DUMP -O $OUTFILE $SOURCE_MAP_URL
+check grep -qi "Cache-control: max-age=31536000" $OUTFILE  # Long cache
+check grep -q "script.js?PageSpeed=off" $OUTFILE  # Has source URL.
+check grep -q '"mappings":' $OUTFILE  # Has mappings.
+
 start_test aris disables js combining for introspective js and only i-js
 URL="$TEST_ROOT/avoid_renaming_introspective_javascript__on/"
 URL+="?PageSpeedFilters=combine_javascript"
@@ -951,6 +1123,44 @@ http_proxy=$SECONDARY_HOSTNAME fetch_until $PROXIED_IMAGE \
 
 WGET_ARGS=""
 
+start_test ShowCache without URL gets a form, inputs, preloaded UA.
+ADMIN_CACHE=$PRIMARY_SERVER/pagespeed_admin/cache
+OUT=$($WGET_DUMP $ADMIN_CACHE)
+check_from "$OUT" fgrep -q "<form "
+check_from "$OUT" fgrep -q "<input "
+check_from "$OUT" fgrep -q "Cache-Control: max-age=0, no-cache"
+# Preloaded user_agent value field leading with "Mozilla" set in
+# ../automatic/system_test_helpers.sh to help test a "normal" flow.
+check_from "$OUT" fgrep -q 'name=user_agent value="Mozilla'
+
+start_test ShowCache with bogus URL gives a 404
+wget $PRIMARY_SERVER/pagespeed_cache?url=bogus_format >& /dev/null
+check [ $? = 8 ]
+
+start_test ShowCache with valid, present URL, with unique options.
+options="PageSpeedImageInlineMaxBytes=6765"
+fetch_until -save $EXAMPLE_ROOT/rewrite_images.html?$options \
+    'grep -c Puzzle\.jpg\.pagespeed\.ic\.' 1
+URL_TAIL=$(grep Puzzle $FETCH_UNTIL_OUTFILE | cut -d \" -f 2)
+SHOW_CACHE_URL=$EXAMPLE_ROOT/$URL_TAIL
+SHOW_CACHE_QUERY=$ADMIN_CACHE?url=$SHOW_CACHE_URL\&$options
+OUT=$($WGET_DUMP $SHOW_CACHE_QUERY)
+check_from "$OUT" fgrep -q cache_ok:true
+check_from "$OUT" fgrep -q mod_pagespeed_example/images/Puzzle.jpg
+
+function show_cache_after_flush() {
+  start_test ShowCache with same URL and matching options misses after flush
+  OUT=$($WGET_DUMP $SHOW_CACHE_QUERY)
+  check_from "$OUT" fgrep -q cache_ok:false
+}
+
+on_cache_flush show_cache_after_flush
+
+start_test ShowCache with same URL but new options misses.
+options="PageSpeedImageInlineMaxBytes=6766"
+OUT=$($WGET_DUMP $ADMIN_CACHE?url=$SHOW_CACHE_URL\&$options)
+check_from "$OUT" fgrep -q cache_ok:false
+
 # This is dependent upon having a /ngx_pagespeed_beacon handler.
 test_filter add_instrumentation beacons load.
 
@@ -960,7 +1170,7 @@ test_filter add_instrumentation beacons load.
 # respond with that as well. Check that we got a 204.
 BEACON_URL="http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F"
 OUT=$(wget -q  --save-headers -O - --no-http-keep-alive \
-      "http://$HOSTNAME/ngx_pagespeed_beacon?ets=load:13&url=$BEACON_URL")
+      "$PRIMARY_SERVER/ngx_pagespeed_beacon?ets=load:13&url=$BEACON_URL")
 check_from "$OUT" grep '^HTTP/1.1 204'
 # The $'...' tells bash to interpret c-style escapes, \r in this case.
 check_from "$OUT" grep $'^Cache-Control: max-age=0, no-cache\r$'
@@ -997,8 +1207,8 @@ cat "$SERVER_ROOT/mod_pagespeed_test/embed_config.html" | \
 # spelling it out to avoid test regolds when we add image filter IDs.
 WGET_ARGS="--save-headers"
 http_proxy=$SECONDARY_HOSTNAME fetch_until -save -recursive \
-    http://embed-config-html.example.com/embed-config.html \
-    'fgrep -c .pagespeed.' 3
+    http://embed-config-html.example.org/embed-config.html \
+    'grep -c \.pagespeed\.' 3
 
 # with the default rewriters in vhost embed-config-resources.example.com
 # the image will be >200k.  But by enabling resizing & compression 73
@@ -1114,6 +1324,7 @@ echo "Clear out our existing state before we begin the test."
 check touch "$FILE_CACHE/cache.flush"
 check touch "$FILE_CACHE/othercache.flush"
 check touch "$SECONDARY_CACHE/cache.flush"
+check touch "$IPRO_CACHE/cache.flush"
 sleep 1
 
 CSS_FILE="$SERVER_ROOT/mod_pagespeed_test/update.css"
@@ -1543,14 +1754,30 @@ URL="$EXP_EXTEND_CACHE?PageSpeed=on&PageSpeedFilters=rewrite_css"
 OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
 check_not_from "$OUT" fgrep 'PageSpeedExperiment='
 
+start_test experiment assignment can be forced
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
+      "$EXP_EXTEND_CACHE?PageSpeedEnrollExperiment=2")
+check_from "$OUT" fgrep 'PageSpeedExperiment=2'
+
+start_test experiment assignment can be forced to a 0% experiment
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
+      "$EXP_EXTEND_CACHE?PageSpeedEnrollExperiment=3")
+check_from "$OUT" fgrep 'PageSpeedExperiment=3'
+
+start_test experiment assignment can be forced even if already assigned
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
+      --header Cookie:PageSpeedExperiment=7 \
+      "$EXP_EXTEND_CACHE?PageSpeedEnrollExperiment=2")
+check_from "$OUT" fgrep 'PageSpeedExperiment=2'
+
 start_test If the user is already assigned, no need to assign them again.
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=2' \
-      $EXP_EXTEND_CACHE)
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
+      --header='Cookie: PageSpeedExperiment=2' $EXP_EXTEND_CACHE)
 check_not_from "$OUT" fgrep 'PageSpeedExperiment='
 
 start_test The beacon should include the experiment id.
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=2' \
-      $EXP_EXTEND_CACHE)
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
+      --header='Cookie: PageSpeedExperiment=2' $EXP_EXTEND_CACHE)
 BEACON_CODE="pagespeed.addInstrumentationInit('/ngx_pagespeed_beacon', 'load',"
 BEACON_CODE+=" '&exptid=2', 'http://experiment.example.com/"
 BEACON_CODE+="mod_pagespeed_example/extend_cache.html');"
@@ -1563,8 +1790,8 @@ BEACON_CODE+="mod_pagespeed_example/extend_cache.html');"
 check_from "$OUT" grep "$BEACON_CODE"
 
 start_test The no-experiment group beacon should not include an experiment id.
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=0' \
-     $EXP_EXTEND_CACHE)
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
+      --header='Cookie: PageSpeedExperiment=0' $EXP_EXTEND_CACHE)
 check_not_from "$OUT" grep 'pagespeed_beacon.*exptid'
 
 # We expect id=7 to be index=a and id=2 to be index=b because that's the
@@ -1984,29 +2211,28 @@ http_proxy=$SECONDARY_HOSTNAME fetch_until -save "$URL" \
 # User-Agent:  Accept:  Image type   Result
 # -----------  -------  ----------   ----------------------------------
 #    IE         N/A     photo        image/jpeg, Cache-Control: private *
-#     :         N/A     synthetic    image/png,  Cache-Control: private *
+#     :         N/A     synthetic    image/png,  no vary
 #  Old Opera     no     photo        image/jpeg, Vary: Accept
-#     :          no     synthetic    image/png,  Vary: Accept +
+#     :          no     synthetic    image/png,  no vary
 #     :         webp    photo        image/webp, Vary: Accept, Lossy
-#     :         webp    synthetic    image/png,  Cache-Control: private +
+#     :         webp    synthetic    image/png,  no vary
 #  Chrome or     no     photo        image/jpeg, Vary: Accept
-# Firefox or     no     synthetic    image/png,  Vary: Accept
+# Firefox or     no     synthetic    image/png,  no vary
 #  New Opera    webp    photo        image/webp, Vary: Accept, Lossy
-#     :         webp    synthetic    image/webp, Cache-Control: private +
-#                                    Lossless webp image returned
+#     :         webp    synthetic    image/webp, no vary
 # TODO(jmaessen): * cases currently send Vary: Accept.  Fix (in progress).
 # + has been rejected for now in favor of image/png, Vary: Accept.
-# TODO(jmaessen): eliminate Vary: Accept headers from synthetic images that
-# are never going to be considered for webp conversion.  This may prove
-# irrelevant if we instead decide to support Lossless via Vary: Accept and
-# abandon old Opera versions.
+# TODO(jmaessen): Send image/webp lossless for synthetic and alpha-channel
+# images.  Will require reverting to Vary: Accept for these.  Stuff like
+# animated webp will have to remain unconverted still in IPRO mode, or switch
+# to cc: private, but right now animated webp support is still pending anyway.
 function test_ipro_for_browser_webp() {
   IN_UA_PRETTY="$1"; shift
   IN_UA="$1"; shift
   IN_ACCEPT="$1"; shift
   IMAGE_TYPE="$1"; shift
   OUT_CONTENT_TYPE="$1"; shift
-  OUT_VARY="$1"; shift
+  OUT_VARY="${1-}"; shift
   OUT_CC="${1-}"; shift
   WGET_ARGS="--save-headers \
              ${IN_UA:+--user-agent $IN_UA} \
@@ -2024,7 +2250,7 @@ function test_ipro_for_browser_webp() {
   else
     TEST_ID+=" Accept:$IN_ACCEPT, "
   fi
-  TEST_ID+=" $IMAGE_TYPE.  Expect image/${IMAGE_TYPE}, "
+  TEST_ID+=" $IMAGE_TYPE.  Expect image/${OUT_CONTENT_TYPE}, "
   if [ -z "$OUT_VARY" ]; then
     TEST_ID+=" no vary, "
   else
@@ -2048,7 +2274,7 @@ function test_ipro_for_browser_webp() {
       fgrep -q "Vary: $OUT_VARY"
   fi
   check_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" \
-    fgrep -q "Cache-Control: ${OUT_CC:-max-age=}"
+    grep -q "Cache-Control: ${OUT_CC:-max-age=[0-9]*}$"
   # TODO: check file type of webp.  Irrelevant for now.
 }
 
@@ -2061,12 +2287,12 @@ test_ipro_for_browser_webp "" "webp-la" ""     photo jpeg "Accept"
 test_ipro_for_browser_webp "None" ""    "webp" photo webp "Accept"
 test_ipro_for_browser_webp "" "webp"    "webp" photo webp "Accept"
 test_ipro_for_browser_webp "" "webp-la" "webp" photo webp "Accept"
-test_ipro_for_browser_webp "None" ""    ""     synth png  "Accept"
-test_ipro_for_browser_webp "" "webp"    ""     synth png  "Accept"
-test_ipro_for_browser_webp "" "webp-la" ""     synth png  "Accept"
-test_ipro_for_browser_webp "None" ""    "webp" synth png  "Accept"
-test_ipro_for_browser_webp "" "webp"    "webp" synth png  "Accept"
-test_ipro_for_browser_webp "" "webp-la" "webp" synth png  "Accept"
+test_ipro_for_browser_webp "None" ""    ""     synth png
+test_ipro_for_browser_webp "" "webp"    ""     synth png
+test_ipro_for_browser_webp "" "webp-la" ""     synth png
+test_ipro_for_browser_webp "None" ""    "webp" synth png
+test_ipro_for_browser_webp "" "webp"    "webp" synth png
+test_ipro_for_browser_webp "" "webp-la" "webp" synth png
 ##############################################################################
 
 # Wordy UAs need to be stored in the WGETRC file to avoid death by quoting.
@@ -2074,34 +2300,37 @@ OLD_WGETRC=$WGETRC
 WGETRC=$TEMPDIR/wgetrc-ua
 export WGETRC
 
-# IE 11 does not cache Vary: Accept.  For now we send it nonetheless.  See
-# TODO above.
-IE11_UA="Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko"
+# IE 9 and later must re-validate Vary: Accept.  We should send CC: private.
+IE9_UA="Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))"
+IE11_UA="Mozilla/5.0 (Windows NT 6.1; WOW64; ***********; rv:11.0) like Gecko"
+echo "user_agent = $IE9_UA" > $WGETRC
+#                           (no accept)  Type  Out  Vary CC
+test_ipro_for_browser_webp "IE 9"  "" "" photo jpeg ""   "max-age=[0-9]*,private"
+test_ipro_for_browser_webp "IE 9"  "" "" synth png
 echo "user_agent = $IE11_UA" > $WGETRC
-#                            (no accept) Type  Out  Vary
-test_ipro_for_browser_webp "IE 11" "" "" photo jpeg "Accept"
-test_ipro_for_browser_webp "IE 11" "" "" synth png  "Accept"
+test_ipro_for_browser_webp "IE 11" "" "" photo jpeg ""   "max-age=[0-9]*,private"
+test_ipro_for_browser_webp "IE 11" "" "" synth png
 
 # Older Opera did not support webp.
 OPERA_UA="Opera/9.80 (Windows NT 5.2; U; en) Presto/2.7.62 Version/11.01"
 echo "user_agent = $OPERA_UA" > $WGETRC
 #                                (no accept) Type  Out  Vary
 test_ipro_for_browser_webp "Old Opera" "" "" photo jpeg "Accept"
-test_ipro_for_browser_webp "Old Opera" "" "" synth png  "Accept"
+test_ipro_for_browser_webp "Old Opera" "" "" synth png
 # Slightly newer opera supports only lossy webp, sends header.
 OPERA_UA="Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10"
 echo "user_agent = $OPERA_UA" > $WGETRC
 #                                           Accept Type  Out  Vary
 test_ipro_for_browser_webp "Newer Opera" "" "webp" photo webp "Accept"
-test_ipro_for_browser_webp "Newer Opera" "" "webp" synth png  "Accept"
+test_ipro_for_browser_webp "Newer Opera" "" "webp" synth png
 
 function test_decent_browsers() {
   echo "user_agent = $2" > $WGETRC
   #                          UA      Accept Type      Out  Vary
   test_ipro_for_browser_webp "$1" "" ""     photo     jpeg "Accept"
-  test_ipro_for_browser_webp "$1" "" ""     synthetic  png "Accept"
+  test_ipro_for_browser_webp "$1" "" ""     synthetic  png
   test_ipro_for_browser_webp "$1" "" "webp" photo     webp "Accept"
-  test_ipro_for_browser_webp "$1" "" "webp" synthetic  png "Accept"
+  test_ipro_for_browser_webp "$1" "" "webp" synthetic  png
 }
 CHROME_UA="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 "
 CHROME_UA+="(KHTML, like Gecko) Chrome/32.0.1700.102 Safari/537.36"
@@ -2138,7 +2367,7 @@ start_test IPRO flow uses cache as expected.
 STATS=$OUTDIR/blocking_rewrite_stats
 IPRO_ROOT=http://ipro.example.com/mod_pagespeed_test/ipro
 URL=$IPRO_ROOT/test_image_dont_reuse2.png
-IPRO_STATS_URL=http://ipro.example.com/ngx_pagespeed_statistics?PageSpeed=off
+IPRO_STATS_URL=http://ipro.example.com/ngx_pagespeed_statistics
 
 # Initial stats.
 http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $IPRO_STATS_URL > $STATS.0
@@ -2256,9 +2485,12 @@ OUTFILE=$OUTDIR/ipro_resource_output
 # Fetch the HTML to initiate rewriting and caching of the image.
 http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $HTML_URL -O $OUTFILE
 
-# First IPRO resource request after a short wait: it will have the full TTL.
+# First IPRO resource request after a short wait: never be optimized
+# because our non-load-from-file flow doesn't support that, but it will have
+# the full TTL.
 sleep 2
 http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $RESOURCE_URL -O $OUTFILE
+check_file_size "$OUTFILE" -gt 15000 # not optimized
 RESOURCE_MAX_AGE=$( \
   extract_headers $OUTFILE | \
   grep 'Cache-Control:' | tr -d '\r' | \
@@ -2266,17 +2498,25 @@ RESOURCE_MAX_AGE=$( \
 check test -n "$RESOURCE_MAX_AGE"
 check test $RESOURCE_MAX_AGE -eq 333
 
-# Second IPRO resource request after a short wait: the TTL will be reduced.
-sleep 2
+# Second IPRO resource request after a short wait: it will still be optimized
+# and the TTL will be reduced.
 http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $RESOURCE_URL -O $OUTFILE
+check_file_size "$OUTFILE" -lt 15000 # optimized
 RESOURCE_MAX_AGE=$( \
   extract_headers $OUTFILE | \
   grep 'Cache-Control:' | tr -d '\r' | \
   sed -e 's/^ *Cache-Control: *//' | sed -e 's/^.*max-age=\([0-9]*\).*$/\1/')
 check test -n "$RESOURCE_MAX_AGE"
+
 check test $RESOURCE_MAX_AGE -lt 333
 check test $RESOURCE_MAX_AGE -gt 300
 
+# TODO(jmaessen, jefftk): Port proxying tests, which rely on pointing a
+# MapProxyDomain construct at a static server.  Perhaps localhost:8050 will
+# serve, but the tests need to use different urls then.  For mod_pagespeed these
+# tests immediately precede "IPRO-optimized resources should have fixed size,
+# not chunked." in system_test.sh.
+
 start_test IPRO-optimized resources should have fixed size, not chunked.
 URL="$EXAMPLE_ROOT/images/Puzzle.jpg"
 URL+="?PageSpeedJpegRecompressionQuality=75"
@@ -2288,6 +2528,31 @@ check [ "$CONTENT_LENGTH" -lt 90000 ];
 check_not_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" \
     fgrep -q 'Transfer-Encoding: chunked'
 
+start_test IPRO 304 with etags
+# Reuses $URL and $FETCH_UNTIL_OUTFILE from previous test.
+check_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" fgrep -q 'ETag:'
+ETAG=$(extract_headers $FETCH_UNTIL_OUTFILE | awk '/ETag:/ {print $2}')
+echo $WGET_DUMP --header "If-None-Match: $ETAG" $URL
+OUTFILE=$OUTDIR/etags
+# Note: -o gets debug info which is the only place that 304 message is sent.
+$WGET -o $OUTFILE -O /dev/null --header "If-None-Match: $ETAG" $URL
+check fgrep -q "awaiting response... 304" $OUTFILE
+
+start_test PageSpeed resources should have a content length.
+URL="$EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css"
+OUT=$($WGET_DUMP --save-headers $URL)
+check_from "$OUT" egrep -q $'^Content-Length: ([0-9])*\r$'
+check_not_from "$OUT" egrep -iq $'^Transfer-Encoding: chunked\r$'
+check_not_from "$OUT" egrep -iq $'^Connection: close\r$'
+
+start_test Downstream cache integration caching headers.
+URL="http://downstreamcacheresource.example.com/mod_pagespeed_example/images/"
+URL+="xCuppa.png.pagespeed.ic.0.png"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" egrep -iq $'^Cache-Control: .*\r$'
+check_from "$OUT" egrep -iq $'^Expires: .*\r$'
+check_from "$OUT" egrep -iq $'^Last-Modified: .*\r$'
+
 # Test handling of large HTML files. We first test with a cold cache, and verify
 # that we bail out of parsing and insert a script redirecting to
 # ?PageSpeed=off. This should also insert an entry into the property cache so

test/pagespeed_test.conf.template

@@ -27,6 +27,13 @@ http {
   proxy_cache_path "@@PROXY_CACHE@@"  levels=1:2   keys_zone=htmlcache:60m inactive=90m  max_size=50m;
   proxy_temp_path "@@TMP_PROXY_CACHE@@";
 
+  pagespeed StatisticsPath /ngx_pagespeed_statistics;
+  pagespeed GlobalStatisticsPath /ngx_pagespeed_global_statistics;
+  pagespeed ConsolePath /pagespeed_console;
+  pagespeed MessagesPath /ngx_pagespeed_message;
+  pagespeed AdminPath /pagespeed_admin;
+  pagespeed GlobalAdminPath /pagespeed_global_admin;
+
   root "@@SERVER_ROOT@@";
 
   # Block 5a: Decide on Cache-Control header value to use for outgoing
@@ -42,7 +49,6 @@ http {
   pagespeed UsePerVHostStatistics on;
   pagespeed InPlaceResourceOptimization on;
   pagespeed CreateSharedMemoryMetadataCache "@@SHM_CACHE@@" 8192;
-  pagespeed PreserveUrlRelativity on;
   pagespeed BlockingRewriteKey psatest;
 
   # CriticalImagesBeaconEnabled is now on by default, but we disable in testing.
@@ -196,6 +202,84 @@ http {
     }
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name if-in-server.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+
+    pagespeed RewriteLevel PassThrough;
+    set $inline_javascript "No";
+
+    if ($http_x_custom_header_inline_js) {
+      # TODO(jefftk): Turn on NGX_HTTP_SIF_CONF and figure out how to get
+      # pagespeed directives inside of a server location block to be respected,
+      # then uncomment the following line and duplicate the if-in-location test
+      # for if-in-server.
+      #pagespeed EnableFilters inline_javascript;
+      set $inline_javascript "Yes";
+    }
+
+    add_header "X-Inline-Javascript" $inline_javascript;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name if-in-location.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+
+
+    location / {
+      set $inline_javascript "No";
+      pagespeed RewriteLevel PassThrough;
+
+      if ($http_x_custom_header_inline_js) {
+        pagespeed EnableFilters inline_javascript;
+        set $inline_javascript "Yes";
+      }
+
+      add_header "X-Inline-Javascript" $inline_javascript;
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name if-in-server.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+
+    pagespeed RewriteLevel PassThrough;
+    set $inline_javascript "No";
+
+    if ($http_x_custom_header_inline_js) {
+      # TODO(jefftk): Turn on NGX_HTTP_SIF_CONF and figure out how to get
+      # pagespeed directives inside of a server location block to be respected,
+      # then uncomment the following line and duplicate the if-in-location test
+      # for if-in-server.
+      #pagespeed EnableFilters inline_javascript;
+      set $inline_javascript "Yes";
+    }
+
+    add_header "X-Inline-Javascript" $inline_javascript;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name if-in-location.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+
+
+    location / {
+      set $inline_javascript "No";
+      pagespeed RewriteLevel PassThrough;
+
+      if ($http_x_custom_header_inline_js) {
+        pagespeed EnableFilters inline_javascript;
+        set $inline_javascript "Yes";
+      }
+
+      add_header "X-Inline-Javascript" $inline_javascript;
+    }
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     server_name mpd.example.com;
@@ -218,6 +302,7 @@ http {
     pagespeed ExperimentSpec
       "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
     pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
+    pagespeed ExperimentSpec "id=3;default;percent=0";
   }
   server {
     listen @@SECONDARY_PORT@@;
@@ -229,6 +314,7 @@ http {
     pagespeed ExperimentSpec
       "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
     pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
+    pagespeed ExperimentSpec "id=3;default;percent=0";
   }
 
   server {
@@ -287,6 +373,15 @@ http {
     }
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name downstreamcacheresource.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_images;
+    pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     server_name renderedimagebeacon.example.com;
@@ -340,7 +435,7 @@ http {
     server_name customhostheader.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@_test";
     root "@@SERVER_ROOT@@/mod_pagespeed_test";
-    
+
     pagespeed on;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_images;
@@ -492,7 +587,7 @@ http {
   # Note that we test with two distinct caches.
   server {
     listen @@SECONDARY_PORT@@;
-    server_name embed-config-html.example.com;
+    server_name embed-config-html.example.org;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
     root "@@SERVER_ROOT@@/mod_pagespeed_test";
@@ -502,6 +597,9 @@ http {
     pagespeed DisableFilters inline_css,extend_cache,inline_javascript;
     pagespeed Domain embed-config-resources.example.com;
 
+    # Share a cache keyspace with embed-config-resources.example.com.
+    pagespeed CacheFragment "embed-config";
+
     pagespeed LoadFromFile "http://embed-config-resources.example.com/"
       "@@SERVER_ROOT@@/mod_pagespeed_example/";
   }
@@ -517,7 +615,10 @@ http {
 
     # Note that we do not set the jpeg quality here, but take
     # it from image URL query parameters that we synthesize in
-    # from embed-config-html.example.com.
+    # from embed-config-html.example.org.
+
+    # Share a cache keyspace with embed-config-html.example.org.
+    pagespeed CacheFragment "embed-config";
 
     pagespeed LoadFromFile "http://embed-config-resources.example.com/"
       "@@SERVER_ROOT@@/mod_pagespeed_example/";
@@ -531,7 +632,7 @@ http {
     pagespeed EnableFilters convert_to_webp_lossless;
     pagespeed EnableFilters in_place_optimize_for_browser;
     pagespeed InPlaceResourceOptimization on;
-    pagespeed FileCachePath "@@FILE_CACHE@@_ipro_for_browser";
+    pagespeed FileCachePath "@@IPRO_CACHE@@";
   }
 
   server {
@@ -585,6 +686,7 @@ http {
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
+    pagespeed CacheFragment "example";
   }
   server {
     # Sets up a logical origin for CDNs to fetch content from, on
@@ -597,6 +699,7 @@ http {
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
+    pagespeed CacheFragment "example";
   }
   server {
     # Sets up a logical cdn, which is where we tell browsers to fetch resources
@@ -787,15 +890,48 @@ http {
     add_header "Date" "Date: Fri, 16 Oct 2009 23:05:07 GMT";
   }
 
+  # Proxy + IPRO a gzip'd file for testing Issue 896.
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name ipro-proxy.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@_ipro_proxy";
+
+    pagespeed on;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed EnableFilters rewrite_domains;
+
+    location / {
+      proxy_pass
+       "http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/mod_deflate/";
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name compressed-css.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed InPlaceResourceOptimization on;
+  }
+
   server {
     listen       @@PRIMARY_PORT@@;
     server_name  localhost;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
+    location ~ ^/pagespeed_admin {
+      allow 127.0.0.1;
+      deny all;
+    }
+
     location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" {
       add_header "" "";
     }
 
+    # Backend for ipro-proxy.example.com
+    location /mod_pagespeed_test/ipro/mod_deflate/ {
+      pagespeed off;
+    }
+
     location /mod_pagespeed_test/cachable_rewritten_html/ {
       # This location has the html files that will be configured to be stored
       # in the proxy_cache layer.
@@ -955,7 +1091,16 @@ http {
 
     location /mod_pagespeed_test/ipro/instant/wait/ {
       pagespeed InPlaceWaitForOptimized on;
-      # TODO: Valgrind runs pass only if the below line is uncommented.
+
+      # TODO(jefftk): When running with valgrind we need to raise the rewrite
+      # deadline or else we get "Deadline exceeded for rewrite of resource".
+      # I had thought InPlaceWaitForOptimized was supposed to disable the
+      # rewrite deadline, but it seems not to.
+      #
+      # InPlaceWaitForOptimized is definitely doing something, though: when I
+      # remove the InPlaceRewriteDeadlineMs line below and run without valgrind,
+      # purple.css is optimized via ipro on the first request but if I set
+      # InPlaceWaitForOptimized to off then it is not.
       pagespeed InPlaceRewriteDeadlineMs 1000;
     }
 
@@ -963,6 +1108,44 @@ http {
       pagespeed InPlaceRewriteDeadlineMs -1;
     }
 
+    # Test to make sure that user-authenticated resources do not get cached and
+    # optimized.
+    location /mod_pagespeed_test/auth/ {
+       auth_basic "Restricted";
+       auth_basic_user_file "@@SERVER_ROOT@@mod_pagespeed_test/auth/passwd.conf";
+    }
+
+    location /mod_pagespeed_test/ipro/cookie/ {
+      # Add Vary:Cookie.  This should prevent us from optimizing the
+      # vary_cookie.css even though PagespeedRespectVary is off.
+      # test/nginx_system_test.sh does the fetches test with and w/o cookies.
+      add_header Vary Cookie;
+      pagespeed RespectVary off;
+      pagespeed InPlaceWaitForOptimized on;
+    }
+
+    location /mod_pagespeed_test/ipro/cookie2/ {
+      # Add Vary:Cookie2.  This should prevent us from optimizing the
+      # vary_cookie2.css even though PagespeedRespectVary is off.
+      # test/nginx_system_test.sh does the fetches test with and w/o cookie2.
+      add_header Vary Cookie2;
+      pagespeed RespectVary off;
+      pagespeed InPlaceWaitForOptimized on;
+    }
+
+    location /mod_pagespeed_test/vary/ {
+      pagespeed RespectVary on;
+    }
+
+    location ~ /mod_pagespeed_test/vary/\.(js|css)$ {
+        add_header Vary User-Agent;
+    }
+
+    location /mod_pagespeed_test/vary/no_respect/ {
+      pagespeed DisableFilters add_instrumentation,inline_css;
+      pagespeed RespectVary off;
+    }
+
     location /mod_pagespeed_test/experimental_js_minifier/ {
       pagespeed UseExperimentalJsMinifier on;
     }
@@ -1009,6 +1192,7 @@ http {
   # set up gzip
   gzip  on;
   gzip_vary on;
+  gzip_proxied any;
   # Turn on gzip for all content types that should benefit from it.
   gzip_types application/ecmascript;
   gzip_types application/javascript;