Merge remote-tracking branch 'origin/release-1.8.31.3-beta'

Pulling in upstream merge to local master.

README.md

@@ -37,11 +37,11 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p
 
    ```bash
    $ cd ~
-   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.7.30.4-beta.zip
-   $ unzip v1.7.30.4-beta.zip # or unzip v1.7.30.4-beta
-   $ cd ngx_pagespeed-1.7.30.4-beta/
-   $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.4.tar.gz
-   $ tar -xzvf 1.7.30.4.tar.gz # expands to psol/
+   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.8.31.3-beta.zip
+   $ unzip v1.8.31.3-beta.zip # or unzip v1.8.31.3-beta
+   $ cd ngx_pagespeed-1.8.31.3-beta/
+   $ wget https://dl.google.com/dl/page-speed/psol/1.8.31.3.tar.gz
+   $ tar -xzvf 1.8.31.3.tar.gz # expands to psol/
    ```
 
 3. Download and build nginx:
@@ -49,10 +49,10 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p
    ```bash
    $ cd ~
    $ # check http://nginx.org/en/download.html for the latest version
-   $ wget http://nginx.org/download/nginx-1.4.6.tar.gz
-   $ tar -xvzf nginx-1.4.6.tar.gz
-   $ cd nginx-1.4.6/
-   $ ./configure --add-module=$HOME/ngx_pagespeed-1.7.30.4-beta
+   $ wget http://nginx.org/download/nginx-1.6.0.tar.gz
+   $ tar -xvzf nginx-1.6.0.tar.gz
+   $ cd nginx-1.6.0/
+   $ ./configure --add-module=$HOME/ngx_pagespeed-1.8.31.3-beta
    $ make
    $ sudo make install
    ```
@@ -82,12 +82,8 @@ In every server block where pagespeed is enabled add:
 #  Ensure requests for pagespeed optimized resources go to the pagespeed
 #  handler and no extraneous headers get set.
 location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" { add_header "" ""; }
-location ~ "^/ngx_pagespeed_static/" { }
+location ~ "^/pagespeed_static/" { }
 location ~ "^/ngx_pagespeed_beacon$" { }
-location /ngx_pagespeed_statistics { allow 127.0.0.1; deny all; }
-location /ngx_pagespeed_global_statistics { allow 127.0.0.1; deny all; }
-location /ngx_pagespeed_message { allow 127.0.0.1; deny all; }
-location /pagespeed_console { allow 127.0.0.1; deny all; }
 ```
 
 To confirm that the module is loaded, fetch a page and check that you see the
@@ -95,7 +91,7 @@ To confirm that the module is loaded, fetch a page and check that you see the
 
 ```bash
 $ curl -I 'http://localhost:8050/some_page/' | grep X-Page-Speed
-X-Page-Speed: 1.7.30.4-...
+X-Page-Speed: 1.8.31.3-...
 ```
 
 Looking at the source of a few pages you should see various changes, such as

config

@@ -27,8 +27,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
     echo "   You need to separately download the pagespeed library:"
     echo ""
     echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.4.tar.gz"
-    echo "     $ tar -xzvf 1.7.30.4.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.8.31.3.tar.gz"
+    echo "     $ tar -xzvf 1.8.31.3.tar.gz # expands to psol/"
     echo ""
     echo "   Or see the installation instructions:"
     echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
@@ -109,6 +109,10 @@ case "$NGX_GCC_VER" in
   ;;
 esac
 
+if [ "$WNO_ERROR" = "YES" ]; then
+    CFLAGS="$CFLAGS -Wno-error"
+fi
+
 pagespeed_include="\
   $mod_pagespeed_dir \
   $mod_pagespeed_dir/third_party/chromium/src \

src/ngx_fetch.cc

@@ -71,7 +71,8 @@ namespace net_instaweb {
         fetch_end_ms_(0),
         done_(false),
         content_length_(-1),
-        content_length_known_(false) {
+        content_length_known_(false),
+        resolver_ctx_(NULL) {
     ngx_memzero(&url_, sizeof(url_));
     log_ = log;
     pool_ = NULL;
@@ -213,6 +214,8 @@ namespace net_instaweb {
       return;
     }
 
+    release_resolver();
+
     if (timeout_event_ && timeout_event_->timer_set) {
       ngx_del_timer(timeout_event_);
       timeout_event_ = NULL;
@@ -288,7 +291,6 @@ namespace net_instaweb {
           kWarning, "NgxFetch: failed to resolve host [%.*s]",
           static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data);
       fetch->CallbackDone(false);
-      ngx_resolve_name_done(resolver_ctx);
       return;
     }
     ngx_memzero(&fetch->sin_, sizeof(fetch->sin_));
@@ -319,7 +321,7 @@ namespace net_instaweb {
         static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data,
         ip_address);
 
-    ngx_resolve_name_done(resolver_ctx);
+    fetch->release_resolver();
 
     if (fetch->InitRequest() != NGX_OK) {
       fetch->message_handler()->Message(kError, "NgxFetch: InitRequest failed");

src/ngx_fetch.h

@@ -83,6 +83,12 @@ class NgxFetch : public PoolElement<NgxFetch> {
   void set_timeout_event(ngx_event_t* x) {
     timeout_event_ = x;
   }
+  void release_resolver() {
+    if (resolver_ctx_ != NULL && resolver_ctx_ != NGX_NO_RESOLVER) {
+      ngx_resolve_name_done(resolver_ctx_);
+      resolver_ctx_ = NULL;
+    }
+  }
 
  private:
   response_handler_pt response_handler;

src/ngx_pagespeed.cc

@@ -205,6 +205,45 @@ ngx_int_t string_piece_to_buffer_chain(
 // modified from NgxBaseFetch::CopyHeadersFromTable()
 namespace {
 
+// Based on ngx_http_add_cache_control.
+ngx_int_t ps_set_cache_control(ngx_http_request_t* r, char* cache_control) {
+  // First strip existing cache-control headers.
+  ngx_table_elt_t* header;
+  NgxListIterator it(&(r->headers_out.headers.part));
+  while ((header = it.Next()) != NULL) {
+    if (STR_CASE_EQ_LITERAL(header->key, "Cache-Control")) {
+      // Response headers with hash of 0 are excluded from the response.
+      header->hash = 0;
+    }
+  }
+
+  // Now add our new cache control header.
+  if (r->headers_out.cache_control.elts == NULL) {
+    ngx_int_t rc = ngx_array_init(&r->headers_out.cache_control, r->pool,
+                                  1, sizeof(ngx_table_elt_t *));
+    if (rc != NGX_OK) {
+      return NGX_ERROR;
+    }
+  }
+  ngx_table_elt_t** cache_control_headers = static_cast<ngx_table_elt_t**>(
+      ngx_array_push(&r->headers_out.cache_control));
+  if (cache_control_headers == NULL) {
+    return NGX_ERROR;
+  }
+  cache_control_headers[0] = static_cast<ngx_table_elt_t*>(
+      ngx_list_push(&r->headers_out.headers));
+  if (cache_control_headers[0] == NULL) {
+    return NGX_ERROR;
+  }
+  cache_control_headers[0]->hash = 1;
+  ngx_str_set(&cache_control_headers[0]->key, "Cache-Control");
+  cache_control_headers[0]->value.len = strlen(cache_control);
+  cache_control_headers[0]->value.data =
+      reinterpret_cast<u_char*>(cache_control);
+
+  return NGX_OK;
+}
+
 template<class Headers>
 void copy_headers_from_table(const ngx_list_t &from, Headers* to) {
   // Standard nginx idiom for iterating over a list.  See ngx_list.h
@@ -283,7 +322,7 @@ ngx_int_t copy_response_headers_to_ngx(
     const GoogleString& name_gs = pagespeed_headers.Name(i);
     const GoogleString& value_gs = pagespeed_headers.Value(i);
 
-    if (preserve_caching_headers != kDontPreserveHeaders) {
+    if (preserve_caching_headers == kPreserveAllCachingHeaders) {
       if (StringCaseEqual(name_gs, "ETag") ||
           StringCaseEqual(name_gs, "Expires") ||
           StringCaseEqual(name_gs, "Date") ||
@@ -291,7 +330,13 @@ ngx_int_t copy_response_headers_to_ngx(
           StringCaseEqual(name_gs, "Cache-Control")) {
         continue;
       }
+    } else if (preserve_caching_headers == kPreserveOnlyCacheControl) {
+      // Retain the original Cache-Control header, but send the recomputed
+      // values for all other cache-related headers.
+      if (StringCaseEqual(name_gs, "Cache-Control")) {
+        continue;
       }
+    } // else we don't preserve any headers
 
     ngx_str_t name, value;
 
@@ -325,7 +370,10 @@ ngx_int_t copy_response_headers_to_ngx(
       return NGX_ERROR;
     }
 
-    if (STR_EQ_LITERAL(name, "Content-Type")) {
+    if (STR_EQ_LITERAL(name, "Cache-Control")) {
+      ps_set_cache_control(r, const_cast<char*>(value_gs.c_str()));
+      continue;
+    } else if (STR_EQ_LITERAL(name, "Content-Type")) {
       // Unlike all the other headers, content_type is just a string.
       headers_out->content_type.data = value_s;
       headers_out->content_type.len = value.len;
@@ -432,8 +480,11 @@ enum Response {
   kPagespeedDisabled,
   kBeacon,
   kStatistics,
+  kGlobalStatistics,
   kConsole,
   kMessages,
+  kAdmin,
+  kGlobalAdmin,
   kPagespeedSubrequest,
   kNotHeadOrGet,
   kErrorResponse,
@@ -1555,19 +1606,33 @@ RequestRouting::Response ps_route_request(ngx_http_request_t* r,
 
   if (is_pagespeed_subrequest(r)) {
     return RequestRouting::kPagespeedSubrequest;
-  } else if (url.PathSansLeaf() ==
-             NgxRewriteDriverFactory::kStaticAssetPrefix) {
+  } else if (
+      url.PathSansLeaf() == dynamic_cast<NgxRewriteDriverFactory*>(
+          cfg_s->server_context->factory())->static_asset_prefix()) {
     return RequestRouting::kStaticContent;
-  } else if (url.PathSansQuery() == "/ngx_pagespeed_statistics" ||
-             url.PathSansQuery() == "/ngx_pagespeed_global_statistics" ) {
-    return RequestRouting::kStatistics;
-  } else if (url.PathSansQuery() == "/pagespeed_console") {
-    return RequestRouting::kConsole;
-  } else if (url.PathSansQuery() == "/ngx_pagespeed_message") {
-    return RequestRouting::kMessages;
   }
 
-  RewriteOptions* global_options = cfg_s->server_context->global_options();
+  const NgxRewriteOptions* global_options = cfg_s->server_context->config();
+
+  StringPiece path = url.PathSansQuery();
+  if (StringCaseEqual(path, global_options->statistics_path())) {
+    return RequestRouting::kStatistics;
+  } else if (StringCaseEqual(path, global_options->global_statistics_path())) {
+    return RequestRouting::kGlobalStatistics;
+  } else if (StringCaseEqual(path, global_options->console_path())) {
+    return RequestRouting::kConsole;
+  } else if (StringCaseEqual(path, global_options->messages_path())) {
+    return RequestRouting::kMessages;
+  } else if (// The admin handlers get everything under a path (/path/*) while
+             // all the other handlers only get exact matches (/path).  So match
+             // all paths starting with the handler path.
+             !global_options->admin_path().empty() &&
+             StringCaseStartsWith(path, global_options->admin_path())) {
+    return RequestRouting::kAdmin;
+  } else if (!global_options->global_admin_path().empty() &&
+             StringCaseStartsWith(path, global_options->global_admin_path())) {
+    return RequestRouting::kGlobalAdmin;
+  }
 
   const GoogleString* beacon_url;
   if (ps_is_https(r)) {
@@ -1583,7 +1648,9 @@ RequestRouting::Response ps_route_request(ngx_http_request_t* r,
   return RequestRouting::kResource;
 }
 
-ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
+ngx_int_t ps_resource_handler(ngx_http_request_t* r,
+                              bool html_rewrite,
+                              RequestRouting::Response response_category) {
   if (r != r->main) {
     return NGX_DECLINED;
   }
@@ -1642,6 +1709,15 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
     }
   }
 
+  bool pagespeed_resource =
+      !html_rewrite && cfg_s->server_context->IsPagespeedResource(url);
+  bool is_an_admin_handler =
+      response_category == RequestRouting::kStatistics ||
+      response_category == RequestRouting::kGlobalStatistics ||
+      response_category == RequestRouting::kConsole ||
+      response_category == RequestRouting::kAdmin ||
+      response_category == RequestRouting::kGlobalAdmin;
+
   if (html_rewrite) {
     ps_release_base_fetch(ctx);
   } else {
@@ -1654,6 +1730,8 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
     ctx->html_rewrite = false;
     ctx->in_place = false;
     ctx->pagespeed_connection = NULL;
+    ctx->preserve_caching_headers = kDontPreserveHeaders;
+
     // See build_context_for_request() in mod_instaweb.cc
     // TODO(jefftk): Is this the right place to be modifying caching headers for
     // html fetches?  Or should that be done later, in the headers flow for
@@ -1664,7 +1742,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
       // Downstream cache integration is not enabled. Disable original
       // Cache-Control headers.
       ctx->preserve_caching_headers = kDontPreserveHeaders;
-    } else {
+    } else if (!pagespeed_resource && !is_an_admin_handler) {
       ctx->preserve_caching_headers = kPreserveOnlyCacheControl;
       // Downstream cache integration is enabled. If a rebeaconing key has been
       // configured and there is a ShouldBeacon header with the correct key,
@@ -1675,6 +1753,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
         ctx->preserve_caching_headers = kDontPreserveHeaders;
       }
     }
+
     ctx->recorder = NULL;
     ctx->url_string = url_string;
 
@@ -1709,7 +1788,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
               false /* requires_blink_cohort (no longer unused) */,
               &page_callback_added));
 
-  if (!html_rewrite && cfg_s->server_context->IsPagespeedResource(url)) {
+  if (pagespeed_resource) {
     // TODO(jefftk): Set using_spdy appropriately.  See
     // ProxyInterface::ProxyRequestCallback
     ResourceFetch::Start(
@@ -1717,6 +1796,42 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
         custom_options.release() /* null if there aren't custom options */,
         false /* using_spdy */, cfg_s->server_context, ctx->base_fetch);
     return ps_async_wait_response(r);
+  } else if (is_an_admin_handler) {
+    QueryParams query_params;
+    query_params.Parse(url.Query());
+
+    PosixTimer timer;
+    int64 now_ms = timer.NowMs();
+    ctx->base_fetch->response_headers()->SetDateAndCaching(
+        now_ms, 0 /* max-age */, ", no-cache");
+
+    if (response_category == RequestRouting::kStatistics ||
+        response_category == RequestRouting::kGlobalStatistics) {
+      cfg_s->server_context->StatisticsPage(
+          response_category == RequestRouting::kGlobalStatistics,
+          query_params,
+          cfg_s->server_context->config(),
+          ctx->base_fetch);
+    } else if (response_category == RequestRouting::kConsole) {
+      cfg_s->server_context->ConsoleHandler(
+          *cfg_s->server_context->config(),
+          SystemServerContext::kStatistics,
+          query_params,
+          ctx->base_fetch);
+    } else if (response_category == RequestRouting::kAdmin ||
+               response_category == RequestRouting::kGlobalAdmin) {
+      cfg_s->server_context->AdminPage(
+          response_category == RequestRouting::kGlobalAdmin,
+          url,
+          query_params,
+          custom_options == NULL ? cfg_s->server_context->config()
+                                 : custom_options.get(),
+          ctx->base_fetch);
+    } else {
+      CHECK(false);
+    }
+
+    return ps_async_wait_response(r);
   }
 
   if (html_rewrite) {
@@ -1862,45 +1977,6 @@ void ps_send_to_pagespeed(ngx_http_request_t* r,
   }
 #endif
 
-// Based on ngx_http_add_cache_control.
-ngx_int_t ps_set_cache_control(ngx_http_request_t* r, char* cache_control) {
-  // First strip existing cache-control headers.
-  ngx_table_elt_t* header;
-  NgxListIterator it(&(r->headers_out.headers.part));
-  while ((header = it.Next()) != NULL) {
-    if (STR_CASE_EQ_LITERAL(header->key, "Cache-Control")) {
-      // Response headers with hash of 0 are excluded from the response.
-      header->hash = 0;
-    }
-  }
-
-  // Now add our new cache control header.
-  if (r->headers_out.cache_control.elts == NULL) {
-    ngx_int_t rc = ngx_array_init(&r->headers_out.cache_control, r->pool,
-                                  1, sizeof(ngx_table_elt_t *));
-    if (rc != NGX_OK) {
-      return NGX_ERROR;
-    }
-  }
-  ngx_table_elt_t** cache_control_headers = static_cast<ngx_table_elt_t**>(
-      ngx_array_push(&r->headers_out.cache_control));
-  if (cache_control_headers == NULL) {
-    return NGX_ERROR;
-  }
-  cache_control_headers[0] = static_cast<ngx_table_elt_t*>(
-      ngx_list_push(&r->headers_out.headers));
-  if (cache_control_headers[0] == NULL) {
-    return NGX_ERROR;
-  }
-  cache_control_headers[0]->hash = 1;
-  ngx_str_set(&cache_control_headers[0]->key, "Cache-Control");
-  cache_control_headers[0]->value.len = strlen(cache_control);
-  cache_control_headers[0]->value.data =
-      reinterpret_cast<u_char*>(cache_control);
-
-  return NGX_OK;
-}
-
 void ps_strip_html_headers(ngx_http_request_t* r) {
   // We're modifying content, so switch to 'Transfer-Encoding: chunked' and
   // calculate on the fly.
@@ -2023,7 +2099,8 @@ ngx_int_t ps_html_rewrite_header_filter(ngx_http_request_t* r) {
     return ngx_http_next_header_filter(r);
   }
 
-  ngx_int_t rc = ps_resource_handler(r, true /* html rewrite */);
+  ngx_int_t rc = ps_resource_handler(r, true /* html rewrite */,
+                                     RequestRouting::kResource);
   if (rc != NGX_OK) {
     ctx->html_rewrite = false;
     return ngx_http_next_header_filter(r);
@@ -2305,27 +2382,6 @@ ngx_int_t send_out_headers_and_body(
   return ngx_http_output_filter(r, out);
 }
 
-namespace {
-
-// TODO(jefftk): This class is a temporary shim to just support console fetch,
-// but we eventually want to convert everything in ps_simple_handler to use
-// something akin to NgxBaseFetch (which sends results direct to nginx).  This
-// is work in progress (but well along) in the mod_pagespeed world.
-class NgxPagespeedConsoleAsyncFetch : public AsyncFetchUsingWriter {
- public:
-  NgxPagespeedConsoleAsyncFetch(const RequestContextPtr& request_context,
-                                Writer* writer)
-      : AsyncFetchUsingWriter(request_context, writer) { }
-  virtual void HandleDone(bool status) { }
-  virtual void HandleHeadersComplete() { }
-
-  void FlushToNgx(const GoogleString& output, ngx_http_request_t* r) {
-    send_out_headers_and_body(r, *response_headers(), output);
-  }
-};
-
-}  // namespace
-
 ngx_int_t ps_simple_handler(ngx_http_request_t* r,
                             NgxServerContext* server_context,
                             RequestRouting::Response response_category) {
@@ -2353,41 +2409,13 @@ ngx_int_t ps_simple_handler(ngx_http_request_t* r,
     case RequestRouting::kStaticContent: {
       StringPiece file_contents;
       if (!server_context->static_asset_manager()->GetAsset(
-              request_uri_path.substr(
-                  strlen(NgxRewriteDriverFactory::kStaticAssetPrefix)),
+              request_uri_path.substr(factory->static_asset_prefix().length()),
               &file_contents, &content_type, &cache_control)) {
         return NGX_DECLINED;
       }
       file_contents.CopyToString(&output);
       break;
     }
-    case RequestRouting::kStatistics: {
-      bool is_global_request =
-          StringCaseStartsWith(
-              request_uri_path, "/ngx_pagespeed_global_statistics");
-      ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
-      RequestContextPtr request_context(
-          cfg_s->server_context->NewRequestContext(r));
-      NgxPagespeedConsoleAsyncFetch fetch(request_context, &writer);
-      server_context->StatisticsPage(
-          is_global_request,
-          query_params,
-          &fetch);
-      fetch.FlushToNgx(output, r);
-      return NGX_OK;
-    }
-    case RequestRouting::kConsole: {
-      ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
-      RequestContextPtr request_context(
-          cfg_s->server_context->NewRequestContext(r));
-      NgxPagespeedConsoleAsyncFetch fetch(request_context, &writer);
-      server_context->ConsoleHandler(*server_context->config(),
-                                     SystemServerContext::kStatistics,
-                                     query_params,
-                                     &fetch);
-      fetch.FlushToNgx(output, r);
-      return NGX_OK;
-    }
     case RequestRouting::kMessages: {
       GoogleString log;
       StringWriter log_writer(&log);
@@ -2627,12 +2655,16 @@ ngx_int_t ps_content_handler(ngx_http_request_t* r) {
     case RequestRouting::kBeacon:
       return ps_beacon_handler(r);
     case RequestRouting::kStaticContent:
-    case RequestRouting::kStatistics:
-    case RequestRouting::kConsole:
     case RequestRouting::kMessages:
       return ps_simple_handler(r, cfg_s->server_context, response_category);
+    case RequestRouting::kStatistics:
+    case RequestRouting::kGlobalStatistics:
+    case RequestRouting::kConsole:
+    case RequestRouting::kAdmin:
+    case RequestRouting::kGlobalAdmin:
     case RequestRouting::kResource:
-      return ps_resource_handler(r, false /* html rewrite */);
+      return ps_resource_handler(
+          r, false /* html rewrite */, response_category);
   }
 
   CHECK(0);

src/ngx_rewrite_driver_factory.cc

@@ -33,7 +33,6 @@
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
 #include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
 #include "net/instaweb/rewriter/public/server_context.h"
-#include "net/instaweb/rewriter/public/static_asset_manager.h"
 #include "net/instaweb/system/public/in_place_resource_recorder.h"
 #include "net/instaweb/system/public/serf_url_async_fetcher.h"
 #include "net/instaweb/system/public/system_caches.h"
@@ -63,9 +62,6 @@ class UrlAsyncFetcher;
 class UrlFetcher;
 class Writer;
 
-const char NgxRewriteDriverFactory::kStaticAssetPrefix[] =
-    "/ngx_pagespeed_static/";
-
 class SharedCircularBuffer;
 
 NgxRewriteDriverFactory::NgxRewriteDriverFactory(
@@ -151,11 +147,6 @@ RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
   return options;
 }
 
-void NgxRewriteDriverFactory::InitStaticAssetManager(
-    StaticAssetManager* static_asset_manager) {
-  static_asset_manager->set_library_url_prefix(kStaticAssetPrefix);
-}
-
 bool NgxRewriteDriverFactory::InitNgxUrlAsyncFetchers() {
   log_ = ngx_cycle->log;
   for (size_t i = 0; i < ngx_url_async_fetchers_.size(); ++i) {

src/ngx_rewrite_driver_factory.h

@@ -45,14 +45,11 @@ class NgxUrlAsyncFetcher;
 class SharedCircularBuffer;
 class SharedMemRefererStatistics;
 class SlowWorker;
-class StaticAssetManager;
 class Statistics;
 class SystemThreadSystem;
 
 class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  public:
-  static const char kStaticAssetPrefix[];
-
   // We take ownership of the thread system.
   explicit NgxRewriteDriverFactory(
       const ProcessContext& process_context,
@@ -68,9 +65,6 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   // Create a new RewriteOptions.  In this implementation it will be an
   // NgxRewriteOptions.
   virtual RewriteOptions* NewRewriteOptions();
-  // Initializes the StaticAssetManager.
-  virtual void InitStaticAssetManager(
-      StaticAssetManager* static_asset_manager);
   virtual ServerContext* NewDecodingServerContext();
   bool InitNgxUrlAsyncFetchers();
   // Check resolver configured or not.

src/ngx_rewrite_options.cc

@@ -37,11 +37,19 @@ namespace net_instaweb {
 
 namespace {
 
+const char kStatisticsPath[] = "StatisticsPath";
+const char kGlobalStatisticsPath[] = "GlobalStatisticsPath";
+const char kConsolePath[] = "ConsolePath";
+const char kMessagesPath[] = "MessagesPath";
+const char kAdminPath[] = "AdminPath";
+const char kGlobalAdminPath[] = "GlobalAdminPath";
+
 // These options are copied from mod_instaweb.cc, where APACHE_CONFIG_OPTIONX
 // indicates that they can not be set at the directory/location level. They set
-// options in the RewriteDriverFactory, so they do not appear in RewriteOptions.
-// They are not alphabetized on purpose, but rather left in the same order as in
-// mod_instaweb.cc in case we end up needing to compare.
+// options in the RewriteDriverFactory, so they're entirely global and do not
+// appear in RewriteOptions.  They are not alphabetized on purpose, but rather
+// left in the same order as in mod_instaweb.cc in case we end up needing to
+// compare.
 // TODO(oschaaf): this duplication is a short term solution.
 const char* const server_only_options[] = {
   "FetcherTimeoutMs",
@@ -54,6 +62,7 @@ const char* const server_only_options[] = {
   "MessageBufferSize",
   "NumRewriteThreads",
   "NumExpensiveRewriteThreads",
+  "StaticAssetPrefix",
   "TrackOriginalContentLength",
   "UsePerVHostStatistics",  // TODO(anupama): What to do about "No longer used"
   "BlockingRewriteRefererUrls",
@@ -92,7 +101,26 @@ void NgxRewriteOptions::Init() {
 }
 
 void NgxRewriteOptions::AddProperties() {
-  // Nothing ngx-specific for now.
+  // Nginx-specific options.
+  add_ngx_option(
+      "", &NgxRewriteOptions::statistics_path_, "nsp", kStatisticsPath,
+      kServerScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics");
+  add_ngx_option(
+      "", &NgxRewriteOptions::global_statistics_path_, "ngsp",
+      kGlobalStatisticsPath, kProcessScope,
+      "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics");
+  add_ngx_option(
+      "", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kServerScope,
+      "Set the console path. Ex: /pagespeed_console");
+  add_ngx_option(
+      "", &NgxRewriteOptions::messages_path_, "nmp", kMessagesPath,
+      kServerScope, "Set the messages path.  Ex: /ngx_pagespeed_message");
+  add_ngx_option(
+      "", &NgxRewriteOptions::admin_path_, "nap", kAdminPath,
+      kServerScope, "Set the admin path.  Ex: /pagespeed_admin");
+  add_ngx_option(
+      "", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
+      kProcessScope, "Set the global admin path.  Ex: /pagespeed_global_admin");
 
   MergeSubclassProperties(ngx_properties_);
 
@@ -283,8 +311,11 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
       result = ParseAndSetOptionHelper<SystemRewriteDriverFactory>(
           arg, driver_factory,
           &SystemRewriteDriverFactory::set_track_original_content_length);
+    } else if (IsDirective(directive, "StaticAssetPrefix")) {
+      driver_factory->set_static_asset_prefix(arg);
+      result = RewriteOptions::kOptionOk;
     } else {
-      result = ParseAndSetOptionFromName1(directive, args[1], &msg, handler);
+      result = ParseAndSetOptionFromName1(directive, arg, &msg, handler);
     }
   } else if (n_args == 3) {
     // Short-term special handling, until this moves to common code.

src/ngx_rewrite_options.h

@@ -68,6 +68,24 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   static const NgxRewriteOptions* DynamicCast(const RewriteOptions* instance);
   static NgxRewriteOptions* DynamicCast(RewriteOptions* instance);
 
+  const GoogleString& statistics_path() const {
+    return statistics_path_.value();
+  }
+  const GoogleString& global_statistics_path() const {
+    return global_statistics_path_.value();
+  }
+  const GoogleString& console_path() const {
+    return console_path_.value();
+  }
+  const GoogleString& messages_path() const {
+    return messages_path_.value();
+  }
+  const GoogleString& admin_path() const {
+    return admin_path_.value();
+  }
+  const GoogleString& global_admin_path() const {
+    return global_admin_path_.value();
+  }
 
  private:
   // Helper methods for ParseAndSetOptions().  Each can:
@@ -109,10 +127,20 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   static void add_ngx_option(typename OptionClass::ValueType default_value,
                              OptionClass NgxRewriteOptions::*offset,
                              const char* id,
-                             StringPiece option_name) {
-    AddProperty(default_value, offset, id, option_name, ngx_properties_);
+                             StringPiece option_name,
+                             OptionScope scope,
+                             const char* help) {
+    AddProperty(default_value, offset, id, option_name, scope, help,
+                ngx_properties_);
   }
 
+  Option<GoogleString> statistics_path_;
+  Option<GoogleString> global_statistics_path_;
+  Option<GoogleString> console_path_;
+  Option<GoogleString> messages_path_;
+  Option<GoogleString> admin_path_;
+  Option<GoogleString> global_admin_path_;
+
   // Helper for ParseAndSetOptions.  Returns whether the two directives equal,
   // ignoring case.
   bool IsDirective(StringPiece config_directive, StringPiece compare_directive);

test/nginx_system_test.sh

@@ -93,6 +93,11 @@ function keepalive_test() {
     | grep -v "^} \\[data not shown"\
     | grep -v "^\\* upload completely sent off"\
     | grep -v "^\\* connected"\
+    | grep -v "^\\* Found bundle for host"\
+    | grep -v "^\\* Adding handle"\
+    | grep -v "^\\* Curl_addHandleToPipeline"\
+    | grep -v "^\\* - Conn "\
+    | grep -v "^\\* Server "\
     | grep -v "^\\*   Trying.*\\.\\.\\.")
 
   # Nothing should remain after that.
@@ -254,7 +259,7 @@ if [ ! -e "$SYSTEM_TEST_FILE" ] ; then
   exit 2
 fi
 
-PSA_JS_LIBRARY_URL_PREFIX="ngx_pagespeed_static"
+PSA_JS_LIBRARY_URL_PREFIX="pagespeed_custom_static"
 
 # An expected failure can be indicated like: "~In-place resource optimization~"
 PAGESPEED_EXPECTED_FAILURES="
@@ -348,29 +353,20 @@ else
 
   # The 1st request results in a cache miss, non-rewritten response
   # produced by pagespeed code and a subsequent purge request.
-  # Because of the random bypassing of the cache (required for beaconing
-  # integration), this request could result in a BYPASS as well.
   start_test Check for case where rewritten cache should get purged.
   check_for_no_rewriting "--header=Host:proxy_cache.example.com"
-  check egrep -q "X-Cache: MISS|BYPASS" $OUT_HEADERS_FILE
+  check egrep -q "X-Cache: MISS" $OUT_HEADERS_FILE
   fetch_until $STATISTICS_URL \
-    'grep -c downstream_cache_purge_attempts:[[:space:]]*1' 1
-
-  while [ x"$(grep "$PURGE_REQUEST_IN_ACCESS_LOG" $ACCESS_LOG)" == x"" ] ; do
-    echo "waiting for purge request to show up in access log"
-    sleep .2
-  done
+    'grep -c successful_downstream_cache_purges:[[:space:]]*1' 1
 
   check [ $(grep -ce "$PURGE_REQUEST_IN_ACCESS_LOG" $ACCESS_LOG) = 1 ];
 
   # The 2nd request results in a cache miss (because of the previous purge),
   # rewritten response produced by pagespeed code and no new purge requests.
-  # Because of the random bypassing of the cache (required for beaconing
-  # integration), this request could result in a BYPASS as well.
   start_test Check for case where rewritten cache should not get purged.
   check_for_rewriting "--header=Host:proxy_cache.example.com \
                       --header=X-PSA-Blocking-Rewrite:psatest"
-  check egrep -q "X-Cache: MISS|BYPASS" $OUT_HEADERS_FILE
+  check egrep -q "X-Cache: MISS" $OUT_HEADERS_FILE
   CURRENT_STATS=$($WGET_DUMP $STATISTICS_URL)
   check_from "$CURRENT_STATS" egrep -q \
     "downstream_cache_purge_attempts:[[:space:]]*1"
@@ -392,7 +388,8 @@ else
   # output is also marked as a cache-miss, indicating that the instrumentation
   # was done by the backend.
   start_test Check whether beaconing is accompanied by a BYPASS always.
-  WGET_ARGS="-S --header=Host:proxy_cache.example.com"
+  WGET_ARGS="-S --header=Host:proxy_cache.example.com \
+                --header=X-Allow-Beacon:yes"
   CACHABLE_HTML_LOC+="?PageSpeedFilters=lazyload_images"
   fetch_until -gzip $CACHABLE_HTML_LOC \
       "zgrep -c \"pagespeed\.CriticalImages\.Run\"" 1
@@ -525,6 +522,60 @@ if [ "$HOSTNAME" = "localhost:$PRIMARY_PORT" ] ; then
   AUTH=""
 fi
 
+start_test "Custom statistics paths in server block"
+
+# Served on normal paths by default.
+URL="inherit-paths.example.com/ngx_pagespeed_statistics"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep cache_time_us
+
+URL="inherit-paths.example.com/ngx_pagespeed_message"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep Info
+
+URL="inherit-paths.example.com/pagespeed_console"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep console_div
+
+URL="inherit-paths.example.com/pagespeed_admin/"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep Admin
+
+# Not served on normal paths when overriden.
+URL="custom-paths.example.com/ngx_pagespeed_statistics"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_not_from "$OUT" grep cache_time_us
+
+URL="custom-paths.example.com/ngx_pagespeed_message"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_not_from "$OUT" grep Info
+
+URL="custom-paths.example.com/pagespeed_console"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_not_from "$OUT" grep console_div
+
+URL="custom-paths.example.com/pagespeed_admin/"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_not_from "$OUT" grep Admin
+
+# Served on custom paths when overriden
+URL="custom-paths.example.com/custom_pagespeed_statistics"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep cache_time_us
+
+URL="custom-paths.example.com/custom_pagespeed_message"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep Info
+
+URL="custom-paths.example.com/custom_pagespeed_console"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep console_div
+
+URL="custom-paths.example.com/custom_pagespeed_admin/"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep Admin
+
+
 WGET_ARGS=""
 function gunzip_grep_0ff() {
   gunzip - | fgrep -q "color:#00f"
@@ -1123,6 +1174,44 @@ http_proxy=$SECONDARY_HOSTNAME fetch_until $PROXIED_IMAGE \
 
 WGET_ARGS=""
 
+start_test ShowCache without URL gets a form, inputs, preloaded UA.
+ADMIN_CACHE=$PRIMARY_SERVER/pagespeed_admin/cache
+OUT=$($WGET_DUMP $ADMIN_CACHE)
+check_from "$OUT" fgrep -q "<form "
+check_from "$OUT" fgrep -q "<input "
+check_from "$OUT" fgrep -q "Cache-Control: max-age=0, no-cache"
+# Preloaded user_agent value field leading with "Mozilla" set in
+# ../automatic/system_test_helpers.sh to help test a "normal" flow.
+check_from "$OUT" fgrep -q 'name=user_agent value="Mozilla'
+
+start_test ShowCache with bogus URL gives a 404
+wget $PRIMARY_SERVER/pagespeed_cache?url=bogus_format >& /dev/null
+check [ $? = 8 ]
+
+start_test ShowCache with valid, present URL, with unique options.
+options="PageSpeedImageInlineMaxBytes=6765"
+fetch_until -save $EXAMPLE_ROOT/rewrite_images.html?$options \
+    'grep -c Puzzle\.jpg\.pagespeed\.ic\.' 1
+URL_TAIL=$(grep Puzzle $FETCH_UNTIL_OUTFILE | cut -d \" -f 2)
+SHOW_CACHE_URL=$EXAMPLE_ROOT/$URL_TAIL
+SHOW_CACHE_QUERY=$ADMIN_CACHE?url=$SHOW_CACHE_URL\&$options
+OUT=$($WGET_DUMP $SHOW_CACHE_QUERY)
+check_from "$OUT" fgrep -q cache_ok:true
+check_from "$OUT" fgrep -q mod_pagespeed_example/images/Puzzle.jpg
+
+function show_cache_after_flush() {
+  start_test ShowCache with same URL and matching options misses after flush
+  OUT=$($WGET_DUMP $SHOW_CACHE_QUERY)
+  check_from "$OUT" fgrep -q cache_ok:false
+}
+
+on_cache_flush show_cache_after_flush
+
+start_test ShowCache with same URL but new options misses.
+options="PageSpeedImageInlineMaxBytes=6766"
+OUT=$($WGET_DUMP $ADMIN_CACHE?url=$SHOW_CACHE_URL\&$options)
+check_from "$OUT" fgrep -q cache_ok:false
+
 # This is dependent upon having a /ngx_pagespeed_beacon handler.
 test_filter add_instrumentation beacons load.
 
@@ -1653,16 +1742,18 @@ HOST_NAME="http://domain-hyperlinks-on.example.com"
 RESPONSE_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
     $HOST_NAME/mod_pagespeed_test/rewrite_domains.html)
 MATCHES=$(echo "$RESPONSE_OUT" | fgrep -c http://dst.example.com)
-check [ $MATCHES -eq 3 ]
+check [ $MATCHES -eq 4 ]
 
 # Test to make sure dynamically defined url-valued attributes are rewritten by
-# rewrite_domains.  See mod_pagespeed_test/rewrite_domains.html: in addition
-# to having one <img> URL, one <form> URL, and one <a> url it also has one
-# <span src=...> URL, one <hr imgsrc=...> URL, and one <hr src=...> URL, all
-# referencing src.example.com.  The first three should be rewritten because of
-# hardcoded rules, the span.src and hr.imgsrc should be rewritten because of
-# UrlValuedAttribute directives, and the hr.src should be left
-# unmodified.  The rewritten ones should all be rewritten to dst.example.com.
+# rewrite_domains.  See mod_pagespeed_test/rewrite_domains.html: in addition to
+# having one <img> URL, one <form> URL, and one <a> url it also has one <span
+# src=...> URL, one <hr imgsrc=...> URL, one <hr src=...> URL, and one
+# <blockquote cite=...> URL, all referencing src.example.com.  The first three
+# should be rewritten because of hardcoded rules, the span.src and hr.imgsrc
+# should be rewritten because of UrlValuedAttribute directives, the hr.src
+# should be left unmodified, and the blockquote.src should be rewritten as an
+# image because of a UrlValuedAttribute override.  The rewritten ones should all
+# be rewritten to dst.example.com.
 HOST_NAME="http://url-attribute.example.com"
 TEST="$HOST_NAME/mod_pagespeed_test"
 REWRITE_DOMAINS="$TEST/rewrite_domains.html"
@@ -1673,7 +1764,7 @@ start_test Rewrite domains in dynamically defined url-valued attributes.
 
 RESPONSE_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $REWRITE_DOMAINS)
 MATCHES=$(echo "$RESPONSE_OUT" | fgrep -c http://dst.example.com)
-check [ $MATCHES -eq 5 ]
+check [ $MATCHES -eq 6 ]
 MATCHES=$(echo "$RESPONSE_OUT" | \
     fgrep -c '<hr src=http://src.example.com/hr-image>')
 check [ $MATCHES -eq 1 ]
@@ -1685,9 +1776,9 @@ function count_exact_matches() {
   fgrep -o "$1" | wc -l
 }
 
-# There are nine resources that should be optimized
+# There are ten resources that should be optimized
 http_proxy=$SECONDARY_HOSTNAME \
-    fetch_until $UVA_EXTEND_CACHE 'count_exact_matches .pagespeed.' 9
+    fetch_until $UVA_EXTEND_CACHE 'count_exact_matches .pagespeed.' 10
 
 # Make sure <custom d=...> isn't modified at all, but that everything else is
 # recognized as a url and rewritten from ../foo to /foo.  This means that only
@@ -1697,9 +1788,9 @@ http_proxy=$SECONDARY_HOSTNAME \
 http_proxy=$SECONDARY_HOSTNAME \
     fetch_until $UVA_EXTEND_CACHE 'fgrep -c ../mod_pa' 1
 
-# There are nine images that should be optimized.
+# There are ten images that should be optimized.
 http_proxy=$SECONDARY_HOSTNAME \
-    fetch_until $UVA_EXTEND_CACHE 'count_exact_matches .pagespeed.ic' 9
+    fetch_until $UVA_EXTEND_CACHE 'count_exact_matches .pagespeed.ic' 10
 
 # Test the experiment framework (Furious).
 
@@ -1981,20 +2072,19 @@ WGET_ARGS=""
 start_test lazyload_images,rewrite_images with critical images beacon
 HOST_NAME="http://imagebeacon.example.com"
 URL="$HOST_NAME/mod_pagespeed_test/image_rewriting/rewrite_images.html"
-# There are 3 images on rewrite_images.html. Check that they are all
-# lazyloaded by default.
+# There are 3 images on rewrite_images.html.  Since beaconing is on but we've
+# sent no beacon data, none should be lazy loaded.
+# Run until we see beaconing on the page (should happen on first visit).
 http_proxy=$SECONDARY_HOSTNAME\
-  fetch_until -save -recursive $URL 'fgrep -c pagespeed_lazy_src=' 3
-check [ $(grep -c "^pagespeed\.CriticalImages\.Run" \
-  $WGET_DIR/rewrite_images.html) = 1 ];
+  fetch_until -save $URL \
+  'fgrep -c "pagespeed.CriticalImages.Run"' 1
+check [ $(grep -c "pagespeed_lazy_src=" $FETCH_FILE) = 0 ];
 # We need the options hash and nonce to send a critical image beacon, so extract
 # it from injected beacon JS.
-OPTIONS_HASH=$(awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' \
-               $WGET_DIR/rewrite_images.html)
-NONCE=$(awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' \
-        $WGET_DIR/rewrite_images.html)
-OPTIONS_HASH=$(grep "^pagespeed\.CriticalImages\.Run" \
-  $WGET_DIR/rewrite_images.html | awk -F\' '{print $(NF-3)}')
+OPTIONS_HASH=$(
+  awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' $FETCH_FILE)
+NONCE=$(
+  awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' $FETCH_FILE)
 # Send a beacon response using POST indicating that Puzzle.jpg is a critical
 # image.
 BEACON_URL="$HOST_NAME/ngx_pagespeed_beacon"
@@ -2006,7 +2096,7 @@ OUT=$(env http_proxy=$SECONDARY_HOSTNAME \
   wget -q --save-headers -O - --no-http-keep-alive \
   --post-data "$BEACON_DATA" "$BEACON_URL")
 check_from "$OUT" egrep -q "HTTP/1[.]. 204"
-# Now only 2 of the images should be lazyloaded, Cuppa.png should not be.
+# Now 2 of the images should be lazyloaded, Puzzle.jpg should not be.
 http_proxy=$SECONDARY_HOSTNAME \
   fetch_until -save -recursive $URL 'fgrep -c pagespeed_lazy_src=' 2
 
@@ -2017,13 +2107,13 @@ http_proxy=$SECONDARY_HOSTNAME \
 # page without blocking.
 URL="$URL?id=4"
 http_proxy=$SECONDARY_HOSTNAME\
-  fetch_until -save -recursive $URL 'fgrep -c pagespeed_lazy_src=' 3
-check [ $(grep -c "^pagespeed\.CriticalImages\.Run" \
-    "$WGET_DIR/rewrite_images.html?id=4") = 1 ];
-OPTIONS_HASH=$(awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' \
-               "$WGET_DIR/rewrite_images.html?id=4")
-NONCE=$(awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' \
-       "$WGET_DIR/rewrite_images.html?id=4")
+  fetch_until -save $URL \
+  'fgrep -c "pagespeed.CriticalImages.Run"' 1
+check [ $(grep -c "pagespeed_lazy_src=" $FETCH_FILE) = 0 ];
+OPTIONS_HASH=$(
+  awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' $FETCH_FILE)
+NONCE=$(
+  awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' $FETCH_FILE)
 BEACON_URL="$HOST_NAME/ngx_pagespeed_beacon"
 BEACON_URL+="?url=http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F"
 BEACON_URL+="image_rewriting%2Frewrite_images.html%3Fid%3D4"
@@ -2114,7 +2204,7 @@ keepalive_test "keepalive-beacon-post.example.com" "/ngx_pagespeed_beacon"\
 
 start_test keepalive with static resources
 keepalive_test "keepalive-static.example.com"\
-  "/ngx_pagespeed_static/js_defer.0.js" ""
+  "/pagespeed_custom_static/js_defer.0.js" ""
 
 # Test for MaxCombinedCssBytes. The html used in the test, 'combine_css.html',
 # has 4 CSS files in the following order.
@@ -2149,7 +2239,7 @@ CONNECTION=$(extract_headers $FETCH_UNTIL_OUTFILE | fgrep "Connection:")
 check_not_from "$CONNECTION" fgrep -qi "Keep-Alive, Keep-Alive"
 check_from "$CONNECTION" fgrep -qi "Keep-Alive"
 
-start_test ngx_pagespeed_static defer js served with correct headers.
+start_test pagespeed_custom_static defer js served with correct headers.
 # First, determine which hash js_defer is served with. We need a correct hash
 # to get it served up with an Etag, which is one of the things we want to test.
 URL="$HOSTNAME/mod_pagespeed_example/defer_javascript.html?PageSpeed=on&PageSpeedFilters=defer_javascript"
@@ -2306,7 +2396,9 @@ test_decent_browsers "New Opera" \
 WGETRC=$OLD_WGETRC
 WGET_ARGS=""
 
-JS_URL="$HOSTNAME/ngx_pagespeed_static/js_defer.$HASH.js"
+start_test JS gzip headers
+
+JS_URL="$HOSTNAME/pagespeed_custom_static/js_defer.$HASH.js"
 JS_HEADERS=$($WGET -O /dev/null -q -S --header='Accept-Encoding: gzip' \
   $JS_URL 2>&1)
 check_from "$JS_HEADERS" egrep -qi 'HTTP/1[.]. 200 OK'
@@ -2329,7 +2421,7 @@ start_test IPRO flow uses cache as expected.
 STATS=$OUTDIR/blocking_rewrite_stats
 IPRO_ROOT=http://ipro.example.com/mod_pagespeed_test/ipro
 URL=$IPRO_ROOT/test_image_dont_reuse2.png
-IPRO_STATS_URL=http://ipro.example.com/ngx_pagespeed_statistics?PageSpeed=off
+IPRO_STATS_URL=http://ipro.example.com/ngx_pagespeed_statistics
 
 # Initial stats.
 http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $IPRO_STATS_URL > $STATS.0
@@ -2500,12 +2592,23 @@ OUTFILE=$OUTDIR/etags
 $WGET -o $OUTFILE -O /dev/null --header "If-None-Match: $ETAG" $URL
 check fgrep -q "awaiting response... 304" $OUTFILE
 
-start_test PageSpeed resources should have a content length.
+start_test PageSpeed CSS loaded in fallback mode is always chunked
+# This is because the domains in urls may be rewritten, changing content
+# length from what's in the cache.
 URL="$EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css"
-OUT=$($WGET_DUMP --save-headers $URL)
-check_from "$OUT" egrep -q $'^Content-Length: ([0-9])*\r$'
-check_not_from "$OUT" egrep -iq $'^Transfer-Encoding: chunked\r$'
-check_not_from "$OUT" egrep -iq $'^Connection: close\r$'
+fetch_until -save $URL "egrep -c Transfer-Encoding:" 1 "--save-headers"
+OUT=$(cat $FETCH_UNTIL_OUTFILE)
+check_from "$OUT" egrep -iq $'^Transfer-Encoding: chunked\r$'
+check_not_from "$OUT" egrep -iq '^Content-Length:'
+check_not_from "$OUT" egrep -iq '^Connection: close'
+
+start_test Downstream cache integration caching headers.
+URL="http://downstreamcacheresource.example.com/mod_pagespeed_example/images/"
+URL+="xCuppa.png.pagespeed.ic.0.png"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" egrep -iq $'^Cache-Control: .*\r$'
+check_from "$OUT" egrep -iq $'^Expires: .*\r$'
+check_from "$OUT" egrep -iq $'^Last-Modified: .*\r$'
 
 # Test handling of large HTML files. We first test with a cold cache, and verify
 # that we bail out of parsing and insert a script redirecting to
@@ -2521,13 +2624,13 @@ WGET_ARGS="--header=PageSpeedFilters:rewrite_images"
 WGET_EC="$WGET_DUMP $WGET_ARGS"
 echo $WGET_EC $URL
 LARGE_OUT=$($WGET_EC $URL)
-check_from "$LARGE_OUT" grep -q window.location=".*&ModPagespeed=off"
+check_from "$LARGE_OUT" grep -q window.location=".*&PageSpeed=off"
 
 # The file should now be in the property cache so make sure that the page is no
 # longer parsed. Use fetch_until because we need to wait for a potentially
 # non-blocking write to the property cache from the previous test to finish
 # before this will succeed.
-fetch_until -save $URL 'grep -c window.location=".*&ModPagespeed=off"' 0
+fetch_until -save $URL 'grep -c window.location=".*&PageSpeed=off"' 0
 check_not fgrep -q pagespeed.ic $FETCH_FILE
 
 start_test messages load
@@ -2548,6 +2651,13 @@ check_not_from "$OUT" egrep -q '^Date: Thu, 01 Jan 1970 00:00:00 GMT'
 OUT=$($WGET_DUMP --header=Host:date.example.com \
     http://$SECONDARY_HOSTNAME/mod_pagespeed_example/combine_css.html)
 check_from "$OUT" egrep -q '^Date: Fri, 16 Oct 2009 23:05:07 GMT'
+WGET_ARGS=
+
+if [ "$NATIVE_FETCHER" != "on" ]; then
+  start_test Test that we can rewrite an HTTPS resource.
+  fetch_until $TEST_ROOT/https_fetch/https_fetch.html \
+    'grep -c /https_gstatic_dot_com/1.gif.pagespeed.ce' 1
+fi
 
 if $USE_VALGRIND; then
     # It is possible that there are still ProxyFetches outstanding

test/pagespeed_test.conf.template

@@ -27,6 +27,15 @@ http {
   proxy_cache_path "@@PROXY_CACHE@@"  levels=1:2   keys_zone=htmlcache:60m inactive=90m  max_size=50m;
   proxy_temp_path "@@TMP_PROXY_CACHE@@";
 
+  pagespeed StatisticsPath /ngx_pagespeed_statistics;
+  pagespeed GlobalStatisticsPath /ngx_pagespeed_global_statistics;
+  pagespeed ConsolePath /pagespeed_console;
+  pagespeed MessagesPath /ngx_pagespeed_message;
+  pagespeed AdminPath /pagespeed_admin;
+  pagespeed GlobalAdminPath /pagespeed_global_admin;
+
+  pagespeed StaticAssetPrefix /pagespeed_custom_static/;
+
   root "@@SERVER_ROOT@@";
 
   # Block 5a: Decide on Cache-Control header value to use for outgoing
@@ -42,7 +51,6 @@ http {
   pagespeed UsePerVHostStatistics on;
   pagespeed InPlaceResourceOptimization on;
   pagespeed CreateSharedMemoryMetadataCache "@@SHM_CACHE@@" 8192;
-  pagespeed PreserveUrlRelativity on;
   pagespeed BlockingRewriteKey psatest;
 
   # CriticalImagesBeaconEnabled is now on by default, but we disable in testing.
@@ -144,7 +152,15 @@ http {
       set $bypass_cache "1";
     }
 
+    # For testing purposes, we never generate values that will result in a beacon 
+    # unless a test request it via "X-Allow-Beacon: yes" in its request header.
+    # This is for testing purposes, note that in a production environment, 
+    # you want 'set_random $rand 0 100;' unconditionally!
+    set $rand 5;
+
+    if ($http_x_allow_beacon ~ "yes") {
       set_random $rand 0 100;
+    }
     set $should_beacon_header_val "";
     if ($rand ~* "^[0-4]$") {
       set $should_beacon_header_val "random_rebeaconing_key";
@@ -328,6 +344,15 @@ http {
     }
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name downstreamcacheresource.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_images;
+    pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     server_name renderedimagebeacon.example.com;
@@ -456,6 +481,9 @@ http {
     pagespeed UrlValuedAttribute img alt-src Image;
     pagespeed UrlValuedAttribute video alt-a Image;
     pagespeed UrlValuedAttribute video alt-b Image;
+
+    # Also test that we can redefine spec-defined attributes.
+    pagespeed UrlValuedAttribute blockquote cite Image;
   }
 
   server {
@@ -685,6 +713,23 @@ http {
     }
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name custom-paths.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed StatisticsPath /custom_pagespeed_statistics;
+    pagespeed ConsolePath /custom_pagespeed_console;
+    pagespeed MessagesPath /custom_pagespeed_message;
+    pagespeed AdminPath /custom_pagespeed_admin;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name inherit-paths.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     server_name notransform.example.com;
@@ -864,6 +909,11 @@ http {
     server_name  localhost;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
+    location ~ ^/pagespeed_admin {
+      allow 127.0.0.1;
+      deny all;
+    }
+
     location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" {
       add_header "" "";
     }
@@ -1091,6 +1141,14 @@ http {
       pagespeed UseExperimentalJsMinifier on;
     }
 
+    pagespeed FetchHttps enable;
+    location /mod_pagespeed_test/https_fetch/ {
+      pagespeed DisableFilters inline_images;
+      pagespeed MapProxyDomain
+          http://localhost:@@PRIMARY_PORT@@/https_gstatic_dot_com
+          https://www.gstatic.com/psa/static;
+    }
+
     pagespeed LoadFromFile
         "http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/instant/"
         "@@SERVER_ROOT@@/mod_pagespeed_test/ipro/instant/";