Port to handlers.h removal (r3816)

README.md

@@ -37,11 +37,11 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p
 
    ```bash
    $ cd ~
-   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.8.31.4-beta.zip
-   $ unzip v1.8.31.4-beta.zip # or unzip v1.8.31.4-beta
-   $ cd ngx_pagespeed-1.8.31.4-beta/
-   $ wget https://dl.google.com/dl/page-speed/psol/1.8.31.4.tar.gz
-   $ tar -xzvf 1.8.31.4.tar.gz # expands to psol/
+   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.7.30.3-beta.zip
+   $ unzip v1.7.30.3-beta.zip # or unzip v1.7.30.3-beta
+   $ cd ngx_pagespeed-1.7.30.3-beta/
+   $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.3.tar.gz
+   $ tar -xzvf 1.7.30.3.tar.gz # expands to psol/
    ```
 
 3. Download and build nginx:
@@ -49,10 +49,10 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p
    ```bash
    $ cd ~
    $ # check http://nginx.org/en/download.html for the latest version
-   $ wget http://nginx.org/download/nginx-1.6.0.tar.gz
-   $ tar -xvzf nginx-1.6.0.tar.gz
-   $ cd nginx-1.6.0/
-   $ ./configure --add-module=$HOME/ngx_pagespeed-1.8.31.4-beta
+   $ wget http://nginx.org/download/nginx-1.4.4.tar.gz
+   $ tar -xvzf nginx-1.4.4.tar.gz
+   $ cd nginx-1.4.4/
+   $ ./configure --add-module=$HOME/ngx_pagespeed-1.7.30.3-beta
    $ make
    $ sudo make install
    ```
@@ -82,8 +82,12 @@ In every server block where pagespeed is enabled add:
 #  Ensure requests for pagespeed optimized resources go to the pagespeed
 #  handler and no extraneous headers get set.
 location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" { add_header "" ""; }
-location ~ "^/pagespeed_static/" { }
+location ~ "^/ngx_pagespeed_static/" { }
 location ~ "^/ngx_pagespeed_beacon$" { }
+location /ngx_pagespeed_statistics { allow 127.0.0.1; deny all; }
+location /ngx_pagespeed_global_statistics { allow 127.0.0.1; deny all; }
+location /ngx_pagespeed_message { allow 127.0.0.1; deny all; }
+location /pagespeed_console { allow 127.0.0.1; deny all; }
 ```
 
 To confirm that the module is loaded, fetch a page and check that you see the
@@ -91,7 +95,7 @@ To confirm that the module is loaded, fetch a page and check that you see the
 
 ```bash
 $ curl -I 'http://localhost:8050/some_page/' | grep X-Page-Speed
-X-Page-Speed: 1.8.31.4-...
+X-Page-Speed: 1.7.30.3-...
 ```
 
 Looking at the source of a few pages you should see various changes, such as

config

@@ -27,8 +27,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
     echo "   You need to separately download the pagespeed library:"
     echo ""
     echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.8.31.4.tar.gz"
-    echo "     $ tar -xzvf 1.8.31.4.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.2.tar.gz"
+    echo "     $ tar -xzvf 1.7.30.2.tar.gz # expands to psol/"
     echo ""
     echo "   Or see the installation instructions:"
     echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
@@ -109,10 +109,6 @@ case "$NGX_GCC_VER" in
   ;;
 esac
 
-if [ "$WNO_ERROR" = "YES" ]; then
-    CFLAGS="$CFLAGS -Wno-error"
-fi
-
 pagespeed_include="\
   $mod_pagespeed_dir \
   $mod_pagespeed_dir/third_party/chromium/src \

src/ngx_base_fetch.cc

@@ -106,10 +106,11 @@ ngx_int_t NgxBaseFetch::CollectAccumulatedWrites(ngx_chain_t** link_ptr) {
 ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
   const ResponseHeaders* pagespeed_headers = response_headers();
 
-  if (content_length_known()) {
-     headers_out->content_length = NULL;
-     headers_out->content_length_n = content_length();
-  }
+  // TODO(chaizhenhua): Add and check.
+  // if (content_length_known()) {
+  //   headers_out->content_length = NULL;
+  //   headers_out->content_length_n = content_length();
+  // }
 
   return copy_response_headers_to_ngx(request_, *pagespeed_headers,
                                       preserve_caching_headers_);

src/ngx_fetch.cc

@@ -71,13 +71,12 @@ namespace net_instaweb {
         fetch_end_ms_(0),
         done_(false),
         content_length_(-1),
-        content_length_known_(false),
-        resolver_ctx_(NULL) {
-    ngx_memzero(&url_, sizeof(url_));
-    log_ = log;
-    pool_ = NULL;
-    timeout_event_ = NULL;
-    connection_ = NULL;
+        content_length_known_(false) {
+            ngx_memzero(&url_, sizeof(url_));
+            log_ = log;
+            pool_ = NULL;
+            timeout_event_ = NULL;
+            connection_ = NULL;
   }
 
   NgxFetch::~NgxFetch() {
@@ -214,8 +213,6 @@ namespace net_instaweb {
       return;
     }
 
-    release_resolver();
-
     if (timeout_event_ && timeout_event_->timer_set) {
       ngx_del_timer(timeout_event_);
       timeout_event_ = NULL;
@@ -291,6 +288,7 @@ namespace net_instaweb {
           kWarning, "NgxFetch: failed to resolve host [%.*s]",
           static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data);
       fetch->CallbackDone(false);
+      ngx_resolve_name_done(resolver_ctx);
       return;
     }
     ngx_memzero(&fetch->sin_, sizeof(fetch->sin_));
@@ -321,7 +319,7 @@ namespace net_instaweb {
         static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data,
         ip_address);
 
-    fetch->release_resolver();
+    ngx_resolve_name_done(resolver_ctx);
 
     if (fetch->InitRequest() != NGX_OK) {
       fetch->message_handler()->Message(kError, "NgxFetch: InitRequest failed");

src/ngx_fetch.h

@@ -83,12 +83,6 @@ class NgxFetch : public PoolElement<NgxFetch> {
   void set_timeout_event(ngx_event_t* x) {
     timeout_event_ = x;
   }
-  void release_resolver() {
-    if (resolver_ctx_ != NULL && resolver_ctx_ != NGX_NO_RESOLVER) {
-      ngx_resolve_name_done(resolver_ctx_);
-      resolver_ctx_ = NULL;
-    }
-  }
 
  private:
   response_handler_pt response_handler;

src/ngx_pagespeed.cc

@@ -37,7 +37,6 @@
 #include "ngx_server_context.h"
 
 #include "net/instaweb/automatic/public/proxy_fetch.h"
-#include "net/instaweb/http/public/async_fetch.h"
 #include "net/instaweb/http/public/cache_url_async_fetcher.h"
 #include "net/instaweb/http/public/content_type.h"
 #include "net/instaweb/http/public/request_context.h"
@@ -49,7 +48,6 @@
 #include "net/instaweb/rewriter/public/resource_fetch.h"
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
 #include "net/instaweb/rewriter/public/rewrite_options.h"
-#include "net/instaweb/rewriter/public/rewrite_query.h"
 #include "net/instaweb/rewriter/public/rewrite_stats.h"
 #include "net/instaweb/rewriter/public/static_asset_manager.h"
 #include "net/instaweb/system/public/in_place_resource_recorder.h"
@@ -71,7 +69,6 @@
 #include "net/instaweb/util/public/time_util.h"
 #include "net/instaweb/util/stack_buffer.h"
 #include "pagespeed/kernel/base/posix_timer.h"
-#include "pagespeed/kernel/http/query_params.h"
 #include "pagespeed/kernel/html/html_keywords.h"
 #include "pagespeed/kernel/thread/pthread_shared_mem.h"
 
@@ -205,45 +202,6 @@ ngx_int_t string_piece_to_buffer_chain(
 // modified from NgxBaseFetch::CopyHeadersFromTable()
 namespace {
 
-// Based on ngx_http_add_cache_control.
-ngx_int_t ps_set_cache_control(ngx_http_request_t* r, char* cache_control) {
-  // First strip existing cache-control headers.
-  ngx_table_elt_t* header;
-  NgxListIterator it(&(r->headers_out.headers.part));
-  while ((header = it.Next()) != NULL) {
-    if (STR_CASE_EQ_LITERAL(header->key, "Cache-Control")) {
-      // Response headers with hash of 0 are excluded from the response.
-      header->hash = 0;
-    }
-  }
-
-  // Now add our new cache control header.
-  if (r->headers_out.cache_control.elts == NULL) {
-    ngx_int_t rc = ngx_array_init(&r->headers_out.cache_control, r->pool,
-                                  1, sizeof(ngx_table_elt_t *));
-    if (rc != NGX_OK) {
-      return NGX_ERROR;
-    }
-  }
-  ngx_table_elt_t** cache_control_headers = static_cast<ngx_table_elt_t**>(
-      ngx_array_push(&r->headers_out.cache_control));
-  if (cache_control_headers == NULL) {
-    return NGX_ERROR;
-  }
-  cache_control_headers[0] = static_cast<ngx_table_elt_t*>(
-      ngx_list_push(&r->headers_out.headers));
-  if (cache_control_headers[0] == NULL) {
-    return NGX_ERROR;
-  }
-  cache_control_headers[0]->hash = 1;
-  ngx_str_set(&cache_control_headers[0]->key, "Cache-Control");
-  cache_control_headers[0]->value.len = strlen(cache_control);
-  cache_control_headers[0]->value.data =
-      reinterpret_cast<u_char*>(cache_control);
-
-  return NGX_OK;
-}
-
 template<class Headers>
 void copy_headers_from_table(const ngx_list_t &from, Headers* to) {
   // Standard nginx idiom for iterating over a list.  See ngx_list.h
@@ -281,18 +239,15 @@ void copy_response_headers_from_ngx(const ngx_http_request_t* r,
 
   headers->set_status_code(r->headers_out.status);
 
-  if (r->headers_out.location != NULL) {
-    headers->Add(HttpAttributes::kLocation,
-                 str_to_string_piece(r->headers_out.location->value));
-  }
-
   // Manually copy over the content type because it's not included in
   // request_->headers_out.headers.
   headers->Add(HttpAttributes::kContentType,
                str_to_string_piece(r->headers_out.content_type));
 
-  // When we don't have a date header, set one with the current time.
-  if (headers->Lookup1(HttpAttributes::kDate) == NULL) {
+  // When we don't have a date header, invent one.
+  const char* date = headers->Lookup1(HttpAttributes::kDate);
+
+  if (date == NULL) {
     headers->SetDate(ngx_current_msec);
   }
 
@@ -322,7 +277,7 @@ ngx_int_t copy_response_headers_to_ngx(
     const GoogleString& name_gs = pagespeed_headers.Name(i);
     const GoogleString& value_gs = pagespeed_headers.Value(i);
 
-    if (preserve_caching_headers == kPreserveAllCachingHeaders) {
+    if (preserve_caching_headers != kDontPreserveHeaders) {
       if (StringCaseEqual(name_gs, "ETag") ||
           StringCaseEqual(name_gs, "Expires") ||
           StringCaseEqual(name_gs, "Date") ||
@@ -330,13 +285,7 @@ ngx_int_t copy_response_headers_to_ngx(
           StringCaseEqual(name_gs, "Cache-Control")) {
         continue;
       }
-    } else if (preserve_caching_headers == kPreserveOnlyCacheControl) {
-      // Retain the original Cache-Control header, but send the recomputed
-      // values for all other cache-related headers.
-      if (StringCaseEqual(name_gs, "Cache-Control")) {
-        continue;
-      }
-    } // else we don't preserve any headers
+    }
 
     ngx_str_t name, value;
 
@@ -370,10 +319,7 @@ ngx_int_t copy_response_headers_to_ngx(
       return NGX_ERROR;
     }
 
-    if (STR_EQ_LITERAL(name, "Cache-Control")) {
-      ps_set_cache_control(r, const_cast<char*>(value_gs.c_str()));
-      continue;
-    } else if (STR_EQ_LITERAL(name, "Content-Type")) {
+    if (STR_EQ_LITERAL(name, "Content-Type")) {
       // Unlike all the other headers, content_type is just a string.
       headers_out->content_type.data = value_s;
       headers_out->content_type.len = value.len;
@@ -480,11 +426,8 @@ enum Response {
   kPagespeedDisabled,
   kBeacon,
   kStatistics,
-  kGlobalStatistics,
   kConsole,
   kMessages,
-  kAdmin,
-  kGlobalAdmin,
   kPagespeedSubrequest,
   kNotHeadOrGet,
   kErrorResponse,
@@ -515,7 +458,7 @@ ngx_command_t ps_commands[] = {
     NULL },
 
   { ngx_string("pagespeed"),
-    NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF|NGX_CONF_TAKE1|
+    NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1|
     NGX_CONF_TAKE2|NGX_CONF_TAKE3|NGX_CONF_TAKE4|NGX_CONF_TAKE5,
     ps_loc_configure,
     NGX_HTTP_SRV_CONF_OFFSET,
@@ -561,24 +504,15 @@ char* ps_init_dir(const StringPiece& directive,
     return NULL;  // We're not root, so we're staying whoever we are.
   }
 
-  // chown if owner differs from nginx worker user.
-  ngx_core_conf_t* ccf = reinterpret_cast<ngx_core_conf_t*>(
-      ngx_get_conf(cf->cycle->conf_ctx, ngx_core_module));
+  ngx_core_conf_t* ccf =
+      (ngx_core_conf_t*)(ngx_get_conf(cf->cycle->conf_ctx, ngx_core_module));
   CHECK(ccf != NULL);
-  struct stat gs_stat;
-  if (stat(gs_path.c_str(), &gs_stat) != 0) {
+
+  if (chown(gs_path.c_str(), ccf->user, ccf->group) != 0) {
     return string_piece_to_pool_string(
         cf->pool, net_instaweb::StrCat(
-            directive, " ", path, " stat() failed"));
+            directive, " ", path, " unable to set permissions"));
   }
-  if (gs_stat.st_uid != ccf->user) {
-    if (chown(gs_path.c_str(), ccf->user, ccf->group) != 0) {
-      return string_piece_to_pool_string(
-          cf->pool, net_instaweb::StrCat(
-              directive, " ", path, " unable to set permissions"));
-    }
-  }
-
   return NULL;
 }
 
@@ -851,28 +785,19 @@ char* ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child) {
 }
 
 char* ps_merge_loc_conf(ngx_conf_t* cf, void* parent, void* child) {
+  ps_loc_conf_t* parent_cfg_l = static_cast<ps_loc_conf_t*>(parent);
+
+  // The variant of the pagespeed directive that is acceptable in location
+  // blocks is only acceptable in location blocks, so we should never be merging
+  // in options from a server or main block.
+  CHECK(parent_cfg_l->options == NULL);
+
   ps_loc_conf_t* cfg_l = static_cast<ps_loc_conf_t*>(child);
   if (cfg_l->options == NULL) {
     // No directory specific options.
     return NGX_CONF_OK;
   }
 
-  // While you can't put a "location" block inside a "location" block you can
-  // put an "if" block inside a "location" block, which is implemented by making
-  // a pretend "location" block.  In this case we may have pagespeed options
-  // from the parent "location" block as well as from the current locationish
-  // "if" block.
-  ps_loc_conf_t* parent_cfg_l = static_cast<ps_loc_conf_t*>(parent);
-  if (parent_cfg_l->options != NULL) {
-    // Rebase our options off of the ones defined in the parent location block.
-    ps_merge_options(parent_cfg_l->options, &cfg_l->options);
-    return NGX_CONF_OK;
-  }
-
-  // Pagespeed options are defined in this location block, and it either has no
-  // parent (typical case) or is an if block whose parent location block defines
-  // no pagespeed options.  Base our options off of those in the server block.
-
   ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
       ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
 
@@ -971,9 +896,18 @@ int ps_determine_port(ngx_http_request_t* r) {
 
   return port;
 }
-}  // namespace
 
-StringPiece ps_determine_host(ngx_http_request_t* r) {
+GoogleString ps_determine_url(ngx_http_request_t* r) {
+  int port = ps_determine_port(r);
+  GoogleString port_string;
+  if ((ps_is_https(r) && (port == 443 || port == -1)) ||
+      (!ps_is_https(r) && (port == 80 || port == -1))) {
+    // No port specifier needed for requests on default ports.
+    port_string = "";
+  } else {
+    port_string = StrCat(":", IntegerToString(port));
+  }
+
   StringPiece host = str_to_string_piece(r->headers_in.server);
   if (host.size() == 0) {
     // If host is unspecified, perhaps because of a pure HTTP 1.0 "GET /path",
@@ -988,23 +922,6 @@ StringPiece ps_determine_host(ngx_http_request_t* r) {
     }
     host = str_to_string_piece(s);
   }
-  return host;
-}
-
-namespace {
-
-GoogleString ps_determine_url(ngx_http_request_t* r) {
-  int port = ps_determine_port(r);
-  GoogleString port_string;
-  if ((ps_is_https(r) && (port == 443 || port == -1)) ||
-      (!ps_is_https(r) && (port == 80 || port == -1))) {
-    // No port specifier needed for requests on default ports.
-    port_string = "";
-  } else {
-    port_string = StrCat(":", IntegerToString(port));
-  }
-
-  StringPiece host = ps_determine_host(r);
 
   return StrCat(ps_is_https(r) ? "https://" : "http://",
                 host, port_string, str_to_string_piece(r->unparsed_uri));
@@ -1285,9 +1202,11 @@ RewriteOptions* ps_determine_request_options(
   // make cache key consistent for both lookup and storing in cache.
   //
   // Sets option from request headers and url.
-  RewriteQuery rewrite_query;
-  if (!cfg_s->server_context->GetQueryOptions(
-          url, request_headers, response_headers, &rewrite_query)) {
+  ServerContext::OptionsBoolPair query_options_success =
+      cfg_s->server_context->GetQueryOptions(url, request_headers,
+                                             response_headers);
+  bool get_query_options_success = query_options_success.second;
+  if (!get_query_options_success) {
     // Failed to parse query params or request headers.  Treat this as if there
     // were no query params given.
     ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
@@ -1297,7 +1216,7 @@ RewriteOptions* ps_determine_request_options(
 
   // Will be NULL if there aren't any options set with query params or in
   // headers.
-  return rewrite_query.ReleaseOptions();
+  return query_options_success.first;
 }
 
 // Check whether this visitor is already in an experiment.  If they're not,
@@ -1376,12 +1295,11 @@ bool ps_determine_options(ngx_http_request_t* r,
   // rebased on the directory options or the global options.
   RewriteOptions* request_options = ps_determine_request_options(
       r, request_headers, response_headers, cfg_s, url);
-  bool have_request_options = request_options != NULL;
 
   // Because the caller takes ownership of any options we return, the only
   // situation in which we can avoid allocating a new RewriteOptions is if the
   // global options are ok as are.
-  if (!have_request_options && directory_options == NULL &&
+  if (directory_options == NULL && request_options == NULL &&
       !global_options->running_experiment()) {
     return true;
   }
@@ -1393,24 +1311,14 @@ bool ps_determine_options(ngx_http_request_t* r,
     *options = global_options->Clone();
   }
 
-  // Modify our options in response to request options if specified.
-  if (have_request_options) {
+  // Modify our options in response to request options or experiment settings,
+  // if we need to.  If there are request options then ignore the experiment
+  // because we don't want experiments to be contaminated with unexpected
+  // settings.
+  if (request_options != NULL) {
     (*options)->Merge(*request_options);
     delete request_options;
-    request_options = NULL;
-  }
-
-  // If we're running an experiment and processing html then modify our options
-  // in response to the experiment.  Except we generally don't want experiments
-  // to be contaminated with unexpected settings, so ignore experiments if we
-  // have request-specific options.  Unless EnrollExperiment is on, probably set
-  // by a query parameter, in which case we want to go ahead and apply the
-  // experimental settings even if it means bad data, because we're just seeing
-  // what it looks like.
-  if ((*options)->running_experiment() &&
-      html_rewrite &&
-      (!have_request_options ||
-       (*options)->enroll_experiment())) {
+  } else if ((*options)->running_experiment() && html_rewrite) {
     bool ok = ps_set_experiment_state_and_cookie(
         r, request_headers, *options, url->Host());
     if (!ok) {
@@ -1447,8 +1355,7 @@ bool ps_apply_x_forwarded_proto(ngx_http_request_t* r, GoogleString* url) {
     return false;  // No X-Forwarded-Proto header found.
   }
 
-  StringPiece x_forwarded_proto =
-      str_to_string_piece(*x_forwarded_proto_header);
+  StringPiece x_forwarded_proto = str_to_string_piece(*x_forwarded_proto_header);
   if (!STR_CASE_EQ_LITERAL(*x_forwarded_proto_header, "http") &&
       !STR_CASE_EQ_LITERAL(*x_forwarded_proto_header, "https")) {
     LOG(WARNING) << "Unsupported X-Forwarded-Proto: " << x_forwarded_proto
@@ -1574,6 +1481,11 @@ void ps_release_request_context(void* data) {
     ctx->recorder = NULL;
   }
 
+  if (ctx->ipro_response_headers != NULL) {
+    delete ctx->ipro_response_headers;
+    ctx->ipro_response_headers = NULL;
+  }
+
   ps_release_base_fetch(ctx);
   delete ctx;
 }
@@ -1606,34 +1518,19 @@ RequestRouting::Response ps_route_request(ngx_http_request_t* r,
 
   if (is_pagespeed_subrequest(r)) {
     return RequestRouting::kPagespeedSubrequest;
-  } else if (
-      url.PathSansLeaf() == dynamic_cast<NgxRewriteDriverFactory*>(
-          cfg_s->server_context->factory())->static_asset_prefix()) {
+  } else if (url.PathSansLeaf() == NgxRewriteDriverFactory::kStaticAssetPrefix) {
     return RequestRouting::kStaticContent;
-  }
-
-  const NgxRewriteOptions* global_options = cfg_s->server_context->config();
-
-  StringPiece path = url.PathSansQuery();
-  if (StringCaseEqual(path, global_options->statistics_path())) {
+  } else if (url.PathSansQuery() == "/ngx_pagespeed_statistics" ||
+             url.PathSansQuery() == "/ngx_pagespeed_global_statistics" ) {
     return RequestRouting::kStatistics;
-  } else if (StringCaseEqual(path, global_options->global_statistics_path())) {
-    return RequestRouting::kGlobalStatistics;
-  } else if (StringCaseEqual(path, global_options->console_path())) {
+  } else if (url.PathSansQuery() == "/pagespeed_console") {
     return RequestRouting::kConsole;
-  } else if (StringCaseEqual(path, global_options->messages_path())) {
+  } else if (url.PathSansQuery() == "/ngx_pagespeed_message") {
     return RequestRouting::kMessages;
-  } else if (// The admin handlers get everything under a path (/path/*) while
-             // all the other handlers only get exact matches (/path).  So match
-             // all paths starting with the handler path.
-             !global_options->admin_path().empty() &&
-             StringCaseStartsWith(path, global_options->admin_path())) {
-    return RequestRouting::kAdmin;
-  } else if (!global_options->global_admin_path().empty() &&
-             StringCaseStartsWith(path, global_options->global_admin_path())) {
-    return RequestRouting::kGlobalAdmin;
   }
 
+  RewriteOptions* global_options = cfg_s->server_context->global_options();
+
   const GoogleString* beacon_url;
   if (ps_is_https(r)) {
     beacon_url = &(global_options->beacon_url().https);
@@ -1648,9 +1545,7 @@ RequestRouting::Response ps_route_request(ngx_http_request_t* r,
   return RequestRouting::kResource;
 }
 
-ngx_int_t ps_resource_handler(ngx_http_request_t* r,
-                              bool html_rewrite,
-                              RequestRouting::Response response_category) {
+ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
   if (r != r->main) {
     return NGX_DECLINED;
   }
@@ -1660,9 +1555,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
 
   CHECK(!(html_rewrite && (ctx == NULL || ctx->html_rewrite == false)));
 
-  if (!html_rewrite &&
-      r->method != NGX_HTTP_GET &&
-      r->method != NGX_HTTP_HEAD) {
+  if (!html_rewrite && r->method != NGX_HTTP_GET && r->method != NGX_HTTP_HEAD) {
     return NGX_DECLINED;
   }
 
@@ -1709,15 +1602,6 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     }
   }
 
-  bool pagespeed_resource =
-      !html_rewrite && cfg_s->server_context->IsPagespeedResource(url);
-  bool is_an_admin_handler =
-      response_category == RequestRouting::kStatistics ||
-      response_category == RequestRouting::kGlobalStatistics ||
-      response_category == RequestRouting::kConsole ||
-      response_category == RequestRouting::kAdmin ||
-      response_category == RequestRouting::kGlobalAdmin;
-
   if (html_rewrite) {
     ps_release_base_fetch(ctx);
   } else {
@@ -1726,12 +1610,11 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     ctx = new ps_request_ctx_t();
 
     ctx->r = r;
+    ctx->ipro_response_headers = NULL;
     ctx->write_pending = false;
     ctx->html_rewrite = false;
     ctx->in_place = false;
     ctx->pagespeed_connection = NULL;
-    ctx->preserve_caching_headers = kDontPreserveHeaders;
-
     // See build_context_for_request() in mod_instaweb.cc
     // TODO(jefftk): Is this the right place to be modifying caching headers for
     // html fetches?  Or should that be done later, in the headers flow for
@@ -1742,7 +1625,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
       // Downstream cache integration is not enabled. Disable original
       // Cache-Control headers.
       ctx->preserve_caching_headers = kDontPreserveHeaders;
-    } else if (!pagespeed_resource && !is_an_admin_handler) {
+    } else {
       ctx->preserve_caching_headers = kPreserveOnlyCacheControl;
       // Downstream cache integration is enabled. If a rebeaconing key has been
       // configured and there is a ShouldBeacon header with the correct key,
@@ -1753,9 +1636,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
         ctx->preserve_caching_headers = kDontPreserveHeaders;
       }
     }
-
     ctx->recorder = NULL;
-    ctx->url_string = url_string;
 
     // Set up a cleanup handler on the request.
     ngx_http_cleanup_t* cleanup = ngx_http_cleanup_add(r, 0);
@@ -1788,7 +1669,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
               false /* requires_blink_cohort (no longer unused) */,
               &page_callback_added));
 
-  if (pagespeed_resource) {
+  if (!html_rewrite && cfg_s->server_context->IsPagespeedResource(url)) {
     // TODO(jefftk): Set using_spdy appropriately.  See
     // ProxyInterface::ProxyRequestCallback
     ResourceFetch::Start(
@@ -1796,42 +1677,6 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
         custom_options.release() /* null if there aren't custom options */,
         false /* using_spdy */, cfg_s->server_context, ctx->base_fetch);
     return ps_async_wait_response(r);
-  } else if (is_an_admin_handler) {
-    QueryParams query_params;
-    query_params.Parse(url.Query());
-
-    PosixTimer timer;
-    int64 now_ms = timer.NowMs();
-    ctx->base_fetch->response_headers()->SetDateAndCaching(
-        now_ms, 0 /* max-age */, ", no-cache");
-
-    if (response_category == RequestRouting::kStatistics ||
-        response_category == RequestRouting::kGlobalStatistics) {
-      cfg_s->server_context->StatisticsPage(
-          response_category == RequestRouting::kGlobalStatistics,
-          query_params,
-          cfg_s->server_context->config(),
-          ctx->base_fetch);
-    } else if (response_category == RequestRouting::kConsole) {
-      cfg_s->server_context->ConsoleHandler(
-          *cfg_s->server_context->config(),
-          SystemServerContext::kStatistics,
-          query_params,
-          ctx->base_fetch);
-    } else if (response_category == RequestRouting::kAdmin ||
-               response_category == RequestRouting::kGlobalAdmin) {
-      cfg_s->server_context->AdminPage(
-          response_category == RequestRouting::kGlobalAdmin,
-          url,
-          query_params,
-          custom_options == NULL ? cfg_s->server_context->config()
-                                 : custom_options.get(),
-          ctx->base_fetch);
-    } else {
-      CHECK(false);
-    }
-
-    return ps_async_wait_response(r);
   }
 
   if (html_rewrite) {
@@ -1977,6 +1822,45 @@ void ps_send_to_pagespeed(ngx_http_request_t* r,
   }
 #endif
 
+// Based on ngx_http_add_cache_control.
+ngx_int_t ps_set_cache_control(ngx_http_request_t* r, char* cache_control) {
+  // First strip existing cache-control headers.
+  ngx_table_elt_t* header;
+  NgxListIterator it(&(r->headers_out.headers.part));
+  while ((header = it.Next()) != NULL) {
+    if (STR_CASE_EQ_LITERAL(header->key, "Cache-Control")) {
+      // Response headers with hash of 0 are excluded from the response.
+      header->hash = 0;
+    }
+  }
+
+  // Now add our new cache control header.
+  if (r->headers_out.cache_control.elts == NULL) {
+    ngx_int_t rc = ngx_array_init(&r->headers_out.cache_control, r->pool,
+                                  1, sizeof(ngx_table_elt_t *));
+    if (rc != NGX_OK) {
+      return NGX_ERROR;
+    }
+  }
+  ngx_table_elt_t** cache_control_headers = static_cast<ngx_table_elt_t**>(
+      ngx_array_push(&r->headers_out.cache_control));
+  if (cache_control_headers == NULL) {
+    return NGX_ERROR;
+  }
+  cache_control_headers[0] = static_cast<ngx_table_elt_t*>(
+      ngx_list_push(&r->headers_out.headers));
+  if (cache_control_headers[0] == NULL) {
+    return NGX_ERROR;
+  }
+  cache_control_headers[0]->hash = 1;
+  ngx_str_set(&cache_control_headers[0]->key, "Cache-Control");
+  cache_control_headers[0]->value.len = strlen(cache_control);
+  cache_control_headers[0]->value.data =
+      reinterpret_cast<u_char*>(cache_control);
+
+  return NGX_OK;
+}
+
 void ps_strip_html_headers(ngx_http_request_t* r) {
   // We're modifying content, so switch to 'Transfer-Encoding: chunked' and
   // calculate on the fly.
@@ -2099,8 +1983,7 @@ ngx_int_t ps_html_rewrite_header_filter(ngx_http_request_t* r) {
     return ngx_http_next_header_filter(r);
   }
 
-  ngx_int_t rc = ps_resource_handler(r, true /* html rewrite */,
-                                     RequestRouting::kResource);
+  ngx_int_t rc = ps_resource_handler(r, true /* html rewrite */);
   if (rc != NGX_OK) {
     ctx->html_rewrite = false;
     return ngx_http_next_header_filter(r);
@@ -2201,40 +2084,12 @@ ngx_http_output_body_filter_pt ngx_http_next_body_filter;
 ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
   ps_request_ctx_t* ctx = ps_get_request_context(r);
 
-  if (ctx == NULL) {
-    return ngx_http_next_header_filter(r);
-  }
-
-  if (ctx->recorder != NULL) {
-    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                   "ps in place check header filter recording: %V", &r->uri);
-
-    CHECK(!ctx->in_place);
-
-    // We didn't find this resource in cache originally, so we're recording it
-    // as it passes us by.  At this point the headers from things that run
-    // before us are set but not things that run after us, which means here is
-    // where we need to check whether there's a "Content-Encoding: gzip".  If we
-    // waited to do this in ps_in_place_body_filter we wouldn't be able to tell
-    // the difference between response headers that have "C-E: gz" because we're
-    // proxying for an upstream that gzipped the content and response headers
-    // that have it because the gzip filter (which runs after us) is going to
-    // produce gzipped output.
-    //
-    // The recorder will do this checking, so pass it the headers.
-    ResponseHeaders response_headers;
-    copy_response_headers_from_ngx(r, &response_headers);
-    ctx->recorder->ConsiderResponseHeaders(
-        InPlaceResourceRecorder::kPreliminaryHeaders, &response_headers);
-    return ngx_http_next_header_filter(r);
-  }
-
-  if (!ctx->in_place) {
+  if (ctx == NULL || !ctx->in_place) {
     return ngx_http_next_header_filter(r);
   }
 
   ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                 "ps in place check header filter initial: %V", &r->uri);
+                 "ps in place check header filter: %V", &r->uri);
 
   int status_code = r->headers_out.status;
   bool status_ok = (status_code != 0) && (status_code < 400);
@@ -2243,9 +2098,6 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
   NgxServerContext* server_context = cfg_s->server_context;
   MessageHandler* message_handler = cfg_s->handler;
   GoogleString url = ps_determine_url(r);
-  // The URL we use for cache key is a bit different since it may
-  // have PageSpeed query params removed.
-  GoogleString cache_url = ctx->url_string;
 
   // continue process
   if (status_ok) {
@@ -2266,7 +2118,7 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
         kInfo,
         "Could not rewrite resource in-place "
         "because URL is not in cache: %s",
-        cache_url.c_str());
+        url.c_str());
     const SystemRewriteOptions* options = SystemRewriteOptions::DynamicCast(
         ctx->driver->options());
     RequestHeaders request_headers;
@@ -2276,10 +2128,8 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
     // (or at least a note that it cannot be cached stored there).
     // We do that using an Apache output filter.
     ctx->recorder = new InPlaceResourceRecorder(
-        RequestContextPtr(cfg_s->server_context->NewRequestContext(r)),
-        cache_url,
-        ctx->driver->CacheFragment(),
-        request_headers.GetProperties(),
+        url,
+        request_headers,
         options->respect_vary(),
         options->ipro_max_response_bytes(),
         options->ipro_max_concurrent_recordings(),
@@ -2289,9 +2139,6 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
         message_handler);
     // set in memory flag for in place_body_filter
     r->filter_need_in_memory = 1;
-
-    // We don't have the response headers at all yet because we haven't yet gone
-    // to the backend.
   } else {
     server_context->rewrite_stats()->ipro_not_rewritable()->Add(1);
     message_handler->Message(kInfo,
@@ -2317,11 +2164,42 @@ ngx_int_t ps_in_place_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
                  "ps in place body filter: %V", &r->uri);
 
   InPlaceResourceRecorder* recorder = ctx->recorder;
+
+  if (ctx->ipro_response_headers == NULL) {
+    // Prepare response headers.
+    ctx->ipro_response_headers = new ResponseHeaders();
+
+    // TODO(oschaaf): We don't get a Date response header here.
+    // Currently, we invent one and set it to the current date/time.
+    // We need to investigate why we don't receive it.
+    ctx->ipro_response_headers->set_major_version(r->http_version / 1000);
+    ctx->ipro_response_headers->set_minor_version(r->http_version % 1000);
+    copy_headers_from_table(r->headers_out.headers, ctx->ipro_response_headers);
+    ctx->ipro_response_headers->set_status_code(r->headers_out.status);
+    ctx->ipro_response_headers->Add(HttpAttributes::kContentType,
+                str_to_string_piece(r->headers_out.content_type));
+    if (r->headers_out.location != NULL) {
+      ctx->ipro_response_headers->Add(HttpAttributes::kLocation,
+                  str_to_string_piece(r->headers_out.location->value));
+    }
+    StringPiece date =
+        ctx->ipro_response_headers->Lookup1(HttpAttributes::kDate);
+    if (date.empty()) {
+      ctx->ipro_response_headers->SetDate(ngx_current_msec);
+    }
+    ctx->ipro_response_headers->ComputeCaching();
+
+    // Unlike in Apache we get the final response headers before we get the
+    // content.  This means we can consider them earlier and abort the
+    // request if need be without buffering everything.
+    recorder->ConsiderResponseHeaders(ctx->ipro_response_headers);
+  }
+
   for (ngx_chain_t* cl = in; cl; cl = cl->next) {
     if (ngx_buf_size(cl->buf)) {
        CHECK(ngx_buf_in_memory(cl->buf));
-       StringPiece contents(reinterpret_cast<char*>(cl->buf->pos),
-                            ngx_buf_size(cl->buf));
+       StringPiece contents(reinterpret_cast<char *>(cl->buf->pos),
+                                 ngx_buf_size(cl->buf));
        recorder->Write(contents, recorder->handler());
     }
 
@@ -2330,9 +2208,7 @@ ngx_int_t ps_in_place_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
     }
 
     if (cl->buf->last_buf || recorder->failed()) {
-      ResponseHeaders response_headers;
-      copy_response_headers_from_ngx(r, &response_headers);
-      ctx->recorder->DoneAndSetHeaders(&response_headers);
+      ctx->recorder->DoneAndSetHeaders(ctx->ipro_response_headers);
       ctx->recorder = NULL;
       break;
     }
@@ -2391,13 +2267,6 @@ ngx_int_t ps_simple_handler(ngx_http_request_t* r,
   NgxMessageHandler* message_handler = factory->ngx_message_handler();
   StringPiece request_uri_path = str_to_string_piece(r->uri);
 
-  GoogleString url_string = ps_determine_url(r);
-  GoogleUrl url(url_string);
-  QueryParams query_params;
-  if (url.IsWebValid()) {
-    query_params.Parse(url.Query());
-  }
-
   GoogleString output;
   StringWriter writer(&output);
   HttpStatus::Code status = HttpStatus::kOK;
@@ -2409,13 +2278,32 @@ ngx_int_t ps_simple_handler(ngx_http_request_t* r,
     case RequestRouting::kStaticContent: {
       StringPiece file_contents;
       if (!server_context->static_asset_manager()->GetAsset(
-              request_uri_path.substr(factory->static_asset_prefix().length()),
+              request_uri_path.substr(
+                  strlen(NgxRewriteDriverFactory::kStaticAssetPrefix)),
               &file_contents, &content_type, &cache_control)) {
         return NGX_DECLINED;
       }
       file_contents.CopyToString(&output);
       break;
     }
+    case RequestRouting::kStatistics: {
+      bool is_global_request =
+          !factory->use_per_vhost_statistics() ||
+          StringCaseStartsWith(
+              request_uri_path, "/ngx_pagespeed_global_statistics");
+      error_message = server_context->StatisticsHandler(
+          factory->caches(),
+          is_global_request ?
+              factory->statistics() : server_context->statistics(),
+          NULL,  // No SPDY-specific config in ngx_pagespeed.
+          StringPiece(reinterpret_cast<char*>(r->args.data), r->args.len),
+          &content_type,
+          &writer);
+      break;
+    }
+    case RequestRouting::kConsole:
+      server_context->ConsoleHandler(server_context->config(), &writer);
+      break;
     case RequestRouting::kMessages: {
       GoogleString log;
       StringWriter log_writer(&log);
@@ -2655,16 +2543,12 @@ ngx_int_t ps_content_handler(ngx_http_request_t* r) {
     case RequestRouting::kBeacon:
       return ps_beacon_handler(r);
     case RequestRouting::kStaticContent:
+    case RequestRouting::kStatistics:
+    case RequestRouting::kConsole:
     case RequestRouting::kMessages:
       return ps_simple_handler(r, cfg_s->server_context, response_category);
-    case RequestRouting::kStatistics:
-    case RequestRouting::kGlobalStatistics:
-    case RequestRouting::kConsole:
-    case RequestRouting::kAdmin:
-    case RequestRouting::kGlobalAdmin:
     case RequestRouting::kResource:
-      return ps_resource_handler(
-          r, false /* html rewrite */, response_category);
+      return ps_resource_handler(r, false /* html rewrite */);
   }
 
   CHECK(0);

src/ngx_pagespeed.h

@@ -103,10 +103,6 @@ typedef struct {
   RewriteDriver* driver;
   InPlaceResourceRecorder* recorder;
   ResponseHeaders* ipro_response_headers;
-
-  // We need to remember the URL here as well since we may modify what NGX
-  // gets by stripping our special query params and honoring X-Forwarded-Proto.
-  GoogleString url_string;
 } ps_request_ctx_t;
 
 
@@ -121,8 +117,6 @@ ngx_int_t copy_response_headers_to_ngx(
     const ResponseHeaders& pagespeed_headers,
     PreserveCachingHeaders preserve_caching_headers);
 
-StringPiece ps_determine_host(ngx_http_request_t* r);
-
 }  // namespace net_instaweb
 
 #endif  // NGX_PAGESPEED_H_

src/ngx_rewrite_driver_factory.cc

@@ -33,6 +33,7 @@
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
 #include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
 #include "net/instaweb/rewriter/public/server_context.h"
+#include "net/instaweb/rewriter/public/static_asset_manager.h"
 #include "net/instaweb/system/public/in_place_resource_recorder.h"
 #include "net/instaweb/system/public/serf_url_async_fetcher.h"
 #include "net/instaweb/system/public/system_caches.h"
@@ -62,6 +63,9 @@ class UrlAsyncFetcher;
 class UrlFetcher;
 class Writer;
 
+const char NgxRewriteDriverFactory::kStaticAssetPrefix[] =
+    "/ngx_pagespeed_static/";
+
 class SharedCircularBuffer;
 
 NgxRewriteDriverFactory::NgxRewriteDriverFactory(
@@ -147,6 +151,11 @@ RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
   return options;
 }
 
+void NgxRewriteDriverFactory::InitStaticAssetManager(
+    StaticAssetManager* static_asset_manager) {
+  static_asset_manager->set_library_url_prefix(kStaticAssetPrefix);
+}
+
 bool NgxRewriteDriverFactory::InitNgxUrlAsyncFetchers() {
   log_ = ngx_cycle->log;
   for (size_t i = 0; i < ngx_url_async_fetchers_.size(); ++i) {

src/ngx_rewrite_driver_factory.h

@@ -45,11 +45,14 @@ class NgxUrlAsyncFetcher;
 class SharedCircularBuffer;
 class SharedMemRefererStatistics;
 class SlowWorker;
+class StaticAssetManager;
 class Statistics;
 class SystemThreadSystem;
 
 class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  public:
+  static const char kStaticAssetPrefix[];
+
   // We take ownership of the thread system.
   explicit NgxRewriteDriverFactory(
       const ProcessContext& process_context,
@@ -65,6 +68,9 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   // Create a new RewriteOptions.  In this implementation it will be an
   // NgxRewriteOptions.
   virtual RewriteOptions* NewRewriteOptions();
+  // Initializes the StaticAssetManager.
+  virtual void InitStaticAssetManager(
+      StaticAssetManager* static_asset_manager);
   virtual ServerContext* NewDecodingServerContext();
   bool InitNgxUrlAsyncFetchers();
   // Check resolver configured or not.

src/ngx_rewrite_options.cc

@@ -37,19 +37,11 @@ namespace net_instaweb {
 
 namespace {
 
-const char kStatisticsPath[] = "StatisticsPath";
-const char kGlobalStatisticsPath[] = "GlobalStatisticsPath";
-const char kConsolePath[] = "ConsolePath";
-const char kMessagesPath[] = "MessagesPath";
-const char kAdminPath[] = "AdminPath";
-const char kGlobalAdminPath[] = "GlobalAdminPath";
-
 // These options are copied from mod_instaweb.cc, where APACHE_CONFIG_OPTIONX
 // indicates that they can not be set at the directory/location level. They set
-// options in the RewriteDriverFactory, so they're entirely global and do not
-// appear in RewriteOptions.  They are not alphabetized on purpose, but rather
-// left in the same order as in mod_instaweb.cc in case we end up needing to
-// compare.
+// options in the RewriteDriverFactory, so they do not appear in RewriteOptions.
+// They are not alphabetized on purpose, but rather left in the same order as in
+// mod_instaweb.cc in case we end up needing to compare.
 // TODO(oschaaf): this duplication is a short term solution.
 const char* const server_only_options[] = {
   "FetcherTimeoutMs",
@@ -62,7 +54,6 @@ const char* const server_only_options[] = {
   "MessageBufferSize",
   "NumRewriteThreads",
   "NumExpensiveRewriteThreads",
-  "StaticAssetPrefix",
   "TrackOriginalContentLength",
   "UsePerVHostStatistics",  // TODO(anupama): What to do about "No longer used"
   "BlockingRewriteRefererUrls",
@@ -79,6 +70,8 @@ const char* const main_only_options[] = {
   "UseNativeFetcher"
 };
 
+const char kNgxPagespeedStatisticsHandlerPath[] = "/ngx_pagespeed_statistics";
+
 }  // namespace
 
 RewriteOptions::Properties* NgxRewriteOptions::ngx_properties_ = NULL;
@@ -98,29 +91,15 @@ void NgxRewriteOptions::Init() {
   DCHECK(ngx_properties_ != NULL)
       << "Call NgxRewriteOptions::Initialize() before construction";
   InitializeOptions(ngx_properties_);
+
+  // Nginx-specific default.
+  // TODO(sligocki): Get rid of this line and let both Apache and Nginx use
+  // /pagespeed_statistics as the handler.
+  statistics_handler_path_.set_default(kNgxPagespeedStatisticsHandlerPath);
 }
 
 void NgxRewriteOptions::AddProperties() {
-  // Nginx-specific options.
-  add_ngx_option(
-      "", &NgxRewriteOptions::statistics_path_, "nsp", kStatisticsPath,
-      kServerScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics");
-  add_ngx_option(
-      "", &NgxRewriteOptions::global_statistics_path_, "ngsp",
-      kGlobalStatisticsPath, kProcessScope,
-      "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics");
-  add_ngx_option(
-      "", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kServerScope,
-      "Set the console path. Ex: /pagespeed_console");
-  add_ngx_option(
-      "", &NgxRewriteOptions::messages_path_, "nmp", kMessagesPath,
-      kServerScope, "Set the messages path.  Ex: /ngx_pagespeed_message");
-  add_ngx_option(
-      "", &NgxRewriteOptions::admin_path_, "nap", kAdminPath,
-      kServerScope, "Set the admin path.  Ex: /pagespeed_admin");
-  add_ngx_option(
-      "", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
-      kProcessScope, "Set the global admin path.  Ex: /pagespeed_global_admin");
+  // Nothing ngx-specific for now.
 
   MergeSubclassProperties(ngx_properties_);
 
@@ -311,11 +290,8 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
       result = ParseAndSetOptionHelper<SystemRewriteDriverFactory>(
           arg, driver_factory,
           &SystemRewriteDriverFactory::set_track_original_content_length);
-    } else if (IsDirective(directive, "StaticAssetPrefix")) {
-      driver_factory->set_static_asset_prefix(arg);
-      result = RewriteOptions::kOptionOk;
     } else {
-      result = ParseAndSetOptionFromName1(directive, arg, &msg, handler);
+      result = ParseAndSetOptionFromName1(directive, args[1], &msg, handler);
     }
   } else if (n_args == 3) {
     // Short-term special handling, until this moves to common code.

src/ngx_rewrite_options.h

@@ -40,8 +40,7 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   static void Initialize();
   static void Terminate();
 
-  NgxRewriteOptions(const StringPiece& description,
-                    ThreadSystem* thread_system);
+  NgxRewriteOptions(const StringPiece& description, ThreadSystem* thread_system);
   explicit NgxRewriteOptions(ThreadSystem* thread_system);
   virtual ~NgxRewriteOptions() { }
 
@@ -68,24 +67,6 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   static const NgxRewriteOptions* DynamicCast(const RewriteOptions* instance);
   static NgxRewriteOptions* DynamicCast(RewriteOptions* instance);
 
-  const GoogleString& statistics_path() const {
-    return statistics_path_.value();
-  }
-  const GoogleString& global_statistics_path() const {
-    return global_statistics_path_.value();
-  }
-  const GoogleString& console_path() const {
-    return console_path_.value();
-  }
-  const GoogleString& messages_path() const {
-    return messages_path_.value();
-  }
-  const GoogleString& admin_path() const {
-    return admin_path_.value();
-  }
-  const GoogleString& global_admin_path() const {
-    return global_admin_path_.value();
-  }
 
  private:
   // Helper methods for ParseAndSetOptions().  Each can:
@@ -127,20 +108,10 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   static void add_ngx_option(typename OptionClass::ValueType default_value,
                              OptionClass NgxRewriteOptions::*offset,
                              const char* id,
-                             StringPiece option_name,
-                             OptionScope scope,
-                             const char* help) {
-    AddProperty(default_value, offset, id, option_name, scope, help,
-                ngx_properties_);
+                             StringPiece option_name) {
+    AddProperty(default_value, offset, id, option_name, ngx_properties_);
   }
 
-  Option<GoogleString> statistics_path_;
-  Option<GoogleString> global_statistics_path_;
-  Option<GoogleString> console_path_;
-  Option<GoogleString> messages_path_;
-  Option<GoogleString> admin_path_;
-  Option<GoogleString> global_admin_path_;
-
   // Helper for ParseAndSetOptions.  Returns whether the two directives equal,
   // ignoring case.
   bool IsDirective(StringPiece config_directive, StringPiece compare_directive);

src/ngx_server_context.cc

@@ -72,7 +72,6 @@ SystemRequestContext* NgxServerContext::NewRequestContext(
 
   return new SystemRequestContext(thread_system()->NewMutex(),
                                   timer(),
-                                  ps_determine_host(r),
                                   local_port,
                                   str_to_string_piece(local_ip));
 }

src/ngx_url_async_fetcher.cc

@@ -221,7 +221,7 @@ namespace net_instaweb {
   void NgxUrlAsyncFetcher::Fetch(const GoogleString& url,
                                  MessageHandler* message_handler,
                                  AsyncFetch* async_fetch) {
-    async_fetch = EnableInflation(async_fetch);
+    async_fetch = EnableInflation(async_fetch, NULL);
     NgxFetch* fetch = new NgxFetch(url, async_fetch,
           message_handler, fetch_timeout_, log_);
     ScopedMutex lock(mutex_);

test/nginx_system_test.sh

@@ -93,11 +93,6 @@ function keepalive_test() {
     | grep -v "^} \\[data not shown"\
     | grep -v "^\\* upload completely sent off"\
     | grep -v "^\\* connected"\
-    | grep -v "^\\* Found bundle for host"\
-    | grep -v "^\\* Adding handle"\
-    | grep -v "^\\* Curl_addHandleToPipeline"\
-    | grep -v "^\\* - Conn "\
-    | grep -v "^\\* Server "\
     | grep -v "^\\*   Trying.*\\.\\.\\.")
 
   # Nothing should remain after that.
@@ -132,7 +127,6 @@ check_simple mkdir "$FILE_CACHE"
 
 # And directories that don't.
 SECONDARY_CACHE="$TEST_TMP/file-cache/secondary/"
-IPRO_CACHE="$TEST_TMP/file-cache/ipro/"
 SHM_CACHE="$TEST_TMP/file-cache/intermediate/directories/with_shm/"
 
 VALGRIND_OPTIONS=""
@@ -168,7 +162,6 @@ cat $PAGESPEED_CONF_TEMPLATE \
   | sed 's#@@ACCESS_LOG@@#'"$ACCESS_LOG"'#' \
   | sed 's#@@FILE_CACHE@@#'"$FILE_CACHE/"'#' \
   | sed 's#@@SECONDARY_CACHE@@#'"$SECONDARY_CACHE/"'#' \
-  | sed 's#@@IPRO_CACHE@@#'"$IPRO_CACHE/"'#' \
   | sed 's#@@SHM_CACHE@@#'"$SHM_CACHE/"'#' \
   | sed 's#@@SERVER_ROOT@@#'"$SERVER_ROOT"'#' \
   | sed 's#@@PRIMARY_PORT@@#'"$PRIMARY_PORT"'#' \
@@ -259,10 +252,11 @@ if [ ! -e "$SYSTEM_TEST_FILE" ] ; then
   exit 2
 fi
 
-PSA_JS_LIBRARY_URL_PREFIX="pagespeed_custom_static"
+PSA_JS_LIBRARY_URL_PREFIX="ngx_pagespeed_static"
 
 # An expected failure can be indicated like: "~In-place resource optimization~"
 PAGESPEED_EXPECTED_FAILURES="
+~IPRO-optimized resources should have fixed size, not chunked.~
 "
 
 # Some tests are flakey under valgrind. For now, add them to the expected failures
@@ -282,7 +276,7 @@ fi
 set -- "$PRIMARY_HOSTNAME"
 source $SYSTEM_TEST_FILE
 
-STATISTICS_URL=$PRIMARY_SERVER/ngx_pagespeed_statistics
+STATISTICS_URL=http://$HOSTNAME/ngx_pagespeed_statistics
 
 # Define a mechanism to start a test before the cache-flush and finish it
 # after the cache-flush.  This mechanism is preferable to flushing cache
@@ -309,28 +303,6 @@ function run_post_cache_flush() {
 
 # nginx-specific system tests
 
-start_test Test pagespeed directive inside if block inside location block.
-
-URL="http://if-in-location.example.com/"
-URL+="mod_pagespeed_example/inline_javascript.html"
-
-# When we specify the X-Custom-Header-Inline-Js that triggers an if block in the
-# config which turns on inline_javascript.
-WGET_ARGS="--header=X-Custom-Header-Inline-Js:Yes"
-http_proxy=$SECONDARY_HOSTNAME \
-  fetch_until $URL 'grep -c document.write' 1
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $WGET_ARGS $URL)
-check_from "$OUT" fgrep "X-Inline-Javascript: Yes"
-check_not_from "$OUT" fgrep "inline_javascript.js"
-
-# Without that custom header we don't trigger the if block, and shouldn't get
-# any inline javascript.
-WGET_ARGS=""
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $WGET_ARGS $URL)
-check_from "$OUT" fgrep "X-Inline-Javascript: No"
-check_from "$OUT" fgrep "inline_javascript.js"
-check_not_from "$OUT" fgrep "document.write"
-
 # Tests related to rewritten response (downstream) caching.
 
 if [ "$NATIVE_FETCHER" = "on" ]; then
@@ -353,20 +325,29 @@ else
 
   # The 1st request results in a cache miss, non-rewritten response
   # produced by pagespeed code and a subsequent purge request.
+  # Because of the random bypassing of the cache (required for beaconing
+  # integration), this request could result in a BYPASS as well.
   start_test Check for case where rewritten cache should get purged.
   check_for_no_rewriting "--header=Host:proxy_cache.example.com"
-  check egrep -q "X-Cache: MISS" $OUT_HEADERS_FILE
+  check egrep -q "X-Cache: MISS|BYPASS" $OUT_HEADERS_FILE
   fetch_until $STATISTICS_URL \
-    'grep -c successful_downstream_cache_purges:[[:space:]]*1' 1
+    'grep -c downstream_cache_purge_attempts:[[:space:]]*1' 1
+
+  while [ x"$(grep "$PURGE_REQUEST_IN_ACCESS_LOG" $ACCESS_LOG)" == x"" ] ; do
+    echo "waiting for purge request to show up in access log"
+    sleep .2
+  done
 
   check [ $(grep -ce "$PURGE_REQUEST_IN_ACCESS_LOG" $ACCESS_LOG) = 1 ];
 
   # The 2nd request results in a cache miss (because of the previous purge),
   # rewritten response produced by pagespeed code and no new purge requests.
+  # Because of the random bypassing of the cache (required for beaconing
+  # integration), this request could result in a BYPASS as well.
   start_test Check for case where rewritten cache should not get purged.
   check_for_rewriting "--header=Host:proxy_cache.example.com \
                       --header=X-PSA-Blocking-Rewrite:psatest"
-  check egrep -q "X-Cache: MISS" $OUT_HEADERS_FILE
+  check egrep -q "X-Cache: MISS|BYPASS" $OUT_HEADERS_FILE
   CURRENT_STATS=$($WGET_DUMP $STATISTICS_URL)
   check_from "$CURRENT_STATS" egrep -q \
     "downstream_cache_purge_attempts:[[:space:]]*1"
@@ -388,8 +369,7 @@ else
   # output is also marked as a cache-miss, indicating that the instrumentation
   # was done by the backend.
   start_test Check whether beaconing is accompanied by a BYPASS always.
-  WGET_ARGS="-S --header=Host:proxy_cache.example.com \
-                --header=X-Allow-Beacon:yes"
+  WGET_ARGS="-S --header=Host:proxy_cache.example.com"
   CACHABLE_HTML_LOC+="?PageSpeedFilters=lazyload_images"
   fetch_until -gzip $CACHABLE_HTML_LOC \
       "zgrep -c \"pagespeed\.CriticalImages\.Run\"" 1
@@ -418,7 +398,7 @@ echo "Final 404s: $NUM_404_FINAL"
 check [ $(expr $NUM_404_FINAL - $NUM_404) -eq 1 ]
 
 # Check that the stat doesn't get bumped on non-404s.
-URL="$PRIMARY_SERVER/mod_pagespeed_example/styles/"
+URL="http://$HOSTNAME/mod_pagespeed_example/styles/"
 URL+="W.rewrite_css_images.css.pagespeed.cf.Hash.css"
 OUT=$(wget -O - -q $URL)
 check_from "$OUT" grep background-image
@@ -508,96 +488,8 @@ if [ "$HOSTNAME" = "localhost:$PRIMARY_PORT" ] ; then
     --header=Cookie2:cookie2-data
   check_from "$IPRO_OUTPUT" fgrep -q '    background: MediumPurple;'
   check_from "$IPRO_OUTPUT" fgrep -q 'Vary: Cookie2'
-
-  start_test authorized resources do not get cached and optimized.
-  URL="$TEST_ROOT/auth/medium_purple.css"
-  AUTH="Authorization:Basic dXNlcjE6cGFzc3dvcmQ="
-  not_cacheable_start=$(scrape_stat ipro_recorder_not_cacheable)
-  echo $WGET_DUMP --header="$AUTH" "$URL"
-  OUT=$($WGET_DUMP --header="$AUTH" "$URL")
-  check_from "$OUT" fgrep -q 'background: MediumPurple;'
-  not_cacheable=$(scrape_stat ipro_recorder_not_cacheable)
-  check [ $not_cacheable = $((not_cacheable_start + 1)) ]
-  URL=""
-  AUTH=""
 fi
 
-start_test "Custom statistics paths in server block"
-
-# Served on normal paths by default.
-URL="inherit-paths.example.com/ngx_pagespeed_statistics"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
-check_from "$OUT" grep cache_time_us
-
-URL="inherit-paths.example.com/ngx_pagespeed_message"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
-check_from "$OUT" grep Info
-
-URL="inherit-paths.example.com/pagespeed_console"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
-check_from "$OUT" grep console_div
-
-URL="inherit-paths.example.com/pagespeed_admin/"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
-check_from "$OUT" grep Admin
-
-# Not served on normal paths when overriden.
-URL="custom-paths.example.com/ngx_pagespeed_statistics"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
-check_not_from "$OUT" grep cache_time_us
-
-URL="custom-paths.example.com/ngx_pagespeed_message"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
-check_not_from "$OUT" grep Info
-
-URL="custom-paths.example.com/pagespeed_console"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
-check_not_from "$OUT" grep console_div
-
-URL="custom-paths.example.com/pagespeed_admin/"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
-check_not_from "$OUT" grep Admin
-
-# Served on custom paths when overriden
-URL="custom-paths.example.com/custom_pagespeed_statistics"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
-check_from "$OUT" grep cache_time_us
-
-URL="custom-paths.example.com/custom_pagespeed_message"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
-check_from "$OUT" grep Info
-
-URL="custom-paths.example.com/custom_pagespeed_console"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
-check_from "$OUT" grep console_div
-
-URL="custom-paths.example.com/custom_pagespeed_admin/"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
-check_from "$OUT" grep Admin
-
-
-WGET_ARGS=""
-function gunzip_grep_0ff() {
-  gunzip - | fgrep -q "color:#00f"
-  echo $?
-}
-
-start_test ipro with mod_deflate
-CSS_FILE="http://compressed-css.example.com/"
-CSS_FILE+="mod_pagespeed_test/ipro/mod_deflate/big.css"
-http_proxy=$SECONDARY_HOSTNAME fetch_until -gzip $CSS_FILE gunzip_grep_0ff 0
-
-start_test ipro with reverse proxy of compressed content
-http_proxy=$SECONDARY_HOSTNAME \
-  fetch_until -gzip http://ipro-proxy.example.com/big.css \
-    gunzip_grep_0ff 0
-
-# Also test the .pagespeed. version, to make sure we didn't accidentally gunzip
-# stuff above when we shouldn't have.
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET -q -O - \
-      http://ipro-proxy.example.com/A.big.css.pagespeed.cf.0.css)
-check_from "$OUT" fgrep -q "big{color:#00f}"
-
 start_test Accept bad query params and headers
 
 # The examples page should have this EXPECTED_EXAMPLES_TEXT on it.
@@ -801,15 +693,7 @@ fetch_until $URL 'fgrep -c file.exception.ssp.css' 1
 start_test statistics load
 
 OUT=$($WGET_DUMP $STATISTICS_URL)
-check_from "$OUT" grep 'PageSpeed Statistics'
-
-start_test statistics handler full-featured
-OUT=$($WGET_DUMP $STATISTICS_URL?config)
-check_from "$OUT" grep "InPlaceResourceOptimization (ipro)"
-
-start_test statistics handler properly sets JSON content-type
-OUT=$($WGET_DUMP $STATISTICS_URL?json)
-check_from "$OUT" grep "Content-Type: application/javascript"
+check_from "$OUT" grep 'VHost-Specific Statistics'
 
 start_test scrape stats works
 
@@ -863,51 +747,15 @@ start_test UseExperimentalJsMinifier
 URL="$TEST_ROOT/experimental_js_minifier/index.html"
 URL+="?PageSpeedFilters=rewrite_javascript"
 # External scripts rewritten.
-fetch_until -save -recursive $URL 'grep -c src=.*\.pagespeed\.jm\.' 1
-check_not grep "removed" $WGET_DIR/*   # No comments should remain.
-check grep -q "preserved" $WGET_DIR/*  # Contents of <script src=> element kept.
-ORIGINAL_HTML_SIZE=1484
-check_file_size $FETCH_FILE -lt $ORIGINAL_HTML_SIZE  # Net savings
+fetch_until -save -recursive \
+  $URL 'grep -c src=.*rewrite_javascript\.js\.pagespeed\.jm\.' 2
+check_not grep removed $WGET_DIR/*.pagespeed.jm.*  # No comments should remain.
+check_file_size $FETCH_FILE -lt 1560               # Net savings
+check grep -q preserved $FETCH_FILE                # Preserves certain comments.
 # Rewritten JS is cache-extended.
 check grep -qi "Cache-control: max-age=31536000" $WGET_OUTPUT
 check grep -qi "Expires:" $WGET_OUTPUT
 
-start_test Source map tests
-URL="$TEST_ROOT/experimental_js_minifier/index.html"
-URL+="?PageSpeedFilters=rewrite_javascript,include_js_source_maps"
-# All rewriting still happening as expected.
-fetch_until -save -recursive $URL 'grep -c src=.*\.pagespeed\.jm\.' 1
-check_not grep "removed" $WGET_DIR/*  # No comments should remain.
-check_file_size $FETCH_FILE -lt $ORIGINAL_HTML_SIZE  # Net savings
-check grep -qi "Cache-control: max-age=31536000" $WGET_OUTPUT
-check grep -qi "Expires:" $WGET_OUTPUT
-
-# No source map for inline JS
-check_not grep sourceMappingURL $FETCH_FILE
-# Yes source_map for external JS
-check grep -q sourceMappingURL $WGET_DIR/script.js.pagespeed.*
-SOURCE_MAP_URL=$(grep sourceMappingURL $WGET_DIR/script.js.pagespeed.* |
-                 grep -o 'http://.*')
-OUTFILE=$OUTDIR/source_map
-check $WGET_DUMP -O $OUTFILE $SOURCE_MAP_URL
-check grep -qi "Cache-control: max-age=31536000" $OUTFILE  # Long cache
-check grep -q "script.js?PageSpeed=off" $OUTFILE  # Has source URL.
-check grep -q '"mappings":' $OUTFILE  # Has mappings.
-
-start_test IPRO source map tests
-URL="$TEST_ROOT/experimental_js_minifier/script.js"
-URL+="?PageSpeedFilters=rewrite_javascript,include_js_source_maps"
-# Fetch until IPRO removes comments.
-fetch_until -save $URL 'grep -c removed' 0
-# Yes source_map for external JS
-check grep -q sourceMappingURL $FETCH_FILE
-SOURCE_MAP_URL=$(grep sourceMappingURL $FETCH_FILE | grep -o 'http://.*')
-OUTFILE=$OUTDIR/source_map
-check $WGET_DUMP -O $OUTFILE $SOURCE_MAP_URL
-check grep -qi "Cache-control: max-age=31536000" $OUTFILE  # Long cache
-check grep -q "script.js?PageSpeed=off" $OUTFILE  # Has source URL.
-check grep -q '"mappings":' $OUTFILE  # Has mappings.
-
 start_test aris disables js combining for introspective js and only i-js
 URL="$TEST_ROOT/avoid_renaming_introspective_javascript__on/"
 URL+="?PageSpeedFilters=combine_javascript"
@@ -1174,44 +1022,6 @@ http_proxy=$SECONDARY_HOSTNAME fetch_until $PROXIED_IMAGE \
 
 WGET_ARGS=""
 
-start_test ShowCache without URL gets a form, inputs, preloaded UA.
-ADMIN_CACHE=$PRIMARY_SERVER/pagespeed_admin/cache
-OUT=$($WGET_DUMP $ADMIN_CACHE)
-check_from "$OUT" fgrep -q "<form "
-check_from "$OUT" fgrep -q "<input "
-check_from "$OUT" fgrep -q "Cache-Control: max-age=0, no-cache"
-# Preloaded user_agent value field leading with "Mozilla" set in
-# ../automatic/system_test_helpers.sh to help test a "normal" flow.
-check_from "$OUT" fgrep -q 'name=user_agent value="Mozilla'
-
-start_test ShowCache with bogus URL gives a 404
-wget $PRIMARY_SERVER/pagespeed_cache?url=bogus_format >& /dev/null
-check [ $? = 8 ]
-
-start_test ShowCache with valid, present URL, with unique options.
-options="PageSpeedImageInlineMaxBytes=6765"
-fetch_until -save $EXAMPLE_ROOT/rewrite_images.html?$options \
-    'grep -c Puzzle\.jpg\.pagespeed\.ic\.' 1
-URL_TAIL=$(grep Puzzle $FETCH_UNTIL_OUTFILE | cut -d \" -f 2)
-SHOW_CACHE_URL=$EXAMPLE_ROOT/$URL_TAIL
-SHOW_CACHE_QUERY=$ADMIN_CACHE?url=$SHOW_CACHE_URL\&$options
-OUT=$($WGET_DUMP $SHOW_CACHE_QUERY)
-check_from "$OUT" fgrep -q cache_ok:true
-check_from "$OUT" fgrep -q mod_pagespeed_example/images/Puzzle.jpg
-
-function show_cache_after_flush() {
-  start_test ShowCache with same URL and matching options misses after flush
-  OUT=$($WGET_DUMP $SHOW_CACHE_QUERY)
-  check_from "$OUT" fgrep -q cache_ok:false
-}
-
-on_cache_flush show_cache_after_flush
-
-start_test ShowCache with same URL but new options misses.
-options="PageSpeedImageInlineMaxBytes=6766"
-OUT=$($WGET_DUMP $ADMIN_CACHE?url=$SHOW_CACHE_URL\&$options)
-check_from "$OUT" fgrep -q cache_ok:false
-
 # This is dependent upon having a /ngx_pagespeed_beacon handler.
 test_filter add_instrumentation beacons load.
 
@@ -1221,7 +1031,7 @@ test_filter add_instrumentation beacons load.
 # respond with that as well. Check that we got a 204.
 BEACON_URL="http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F"
 OUT=$(wget -q  --save-headers -O - --no-http-keep-alive \
-      "$PRIMARY_SERVER/ngx_pagespeed_beacon?ets=load:13&url=$BEACON_URL")
+      "http://$HOSTNAME/ngx_pagespeed_beacon?ets=load:13&url=$BEACON_URL")
 check_from "$OUT" grep '^HTTP/1.1 204'
 # The $'...' tells bash to interpret c-style escapes, \r in this case.
 check_from "$OUT" grep $'^Cache-Control: max-age=0, no-cache\r$'
@@ -1258,8 +1068,8 @@ cat "$SERVER_ROOT/mod_pagespeed_test/embed_config.html" | \
 # spelling it out to avoid test regolds when we add image filter IDs.
 WGET_ARGS="--save-headers"
 http_proxy=$SECONDARY_HOSTNAME fetch_until -save -recursive \
-    http://embed-config-html.example.org/embed-config.html \
-    'grep -c \.pagespeed\.' 3
+    http://embed-config-html.example.com/embed-config.html \
+    'fgrep -c .pagespeed.' 3
 
 # with the default rewriters in vhost embed-config-resources.example.com
 # the image will be >200k.  But by enabling resizing & compression 73
@@ -1375,7 +1185,6 @@ echo "Clear out our existing state before we begin the test."
 check touch "$FILE_CACHE/cache.flush"
 check touch "$FILE_CACHE/othercache.flush"
 check touch "$SECONDARY_CACHE/cache.flush"
-check touch "$IPRO_CACHE/cache.flush"
 sleep 1
 
 CSS_FILE="$SERVER_ROOT/mod_pagespeed_test/update.css"
@@ -1742,18 +1551,16 @@ HOST_NAME="http://domain-hyperlinks-on.example.com"
 RESPONSE_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
     $HOST_NAME/mod_pagespeed_test/rewrite_domains.html)
 MATCHES=$(echo "$RESPONSE_OUT" | fgrep -c http://dst.example.com)
-check [ $MATCHES -eq 4 ]
+check [ $MATCHES -eq 3 ]
 
 # Test to make sure dynamically defined url-valued attributes are rewritten by
-# rewrite_domains.  See mod_pagespeed_test/rewrite_domains.html: in addition to
-# having one <img> URL, one <form> URL, and one <a> url it also has one <span
-# src=...> URL, one <hr imgsrc=...> URL, one <hr src=...> URL, and one
-# <blockquote cite=...> URL, all referencing src.example.com.  The first three
-# should be rewritten because of hardcoded rules, the span.src and hr.imgsrc
-# should be rewritten because of UrlValuedAttribute directives, the hr.src
-# should be left unmodified, and the blockquote.src should be rewritten as an
-# image because of a UrlValuedAttribute override.  The rewritten ones should all
-# be rewritten to dst.example.com.
+# rewrite_domains.  See mod_pagespeed_test/rewrite_domains.html: in addition
+# to having one <img> URL, one <form> URL, and one <a> url it also has one
+# <span src=...> URL, one <hr imgsrc=...> URL, and one <hr src=...> URL, all
+# referencing src.example.com.  The first three should be rewritten because of
+# hardcoded rules, the span.src and hr.imgsrc should be rewritten because of
+# UrlValuedAttribute directives, and the hr.src should be left
+# unmodified.  The rewritten ones should all be rewritten to dst.example.com.
 HOST_NAME="http://url-attribute.example.com"
 TEST="$HOST_NAME/mod_pagespeed_test"
 REWRITE_DOMAINS="$TEST/rewrite_domains.html"
@@ -1764,7 +1571,7 @@ start_test Rewrite domains in dynamically defined url-valued attributes.
 
 RESPONSE_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $REWRITE_DOMAINS)
 MATCHES=$(echo "$RESPONSE_OUT" | fgrep -c http://dst.example.com)
-check [ $MATCHES -eq 6 ]
+check [ $MATCHES -eq 5 ]
 MATCHES=$(echo "$RESPONSE_OUT" | \
     fgrep -c '<hr src=http://src.example.com/hr-image>')
 check [ $MATCHES -eq 1 ]
@@ -1776,9 +1583,9 @@ function count_exact_matches() {
   fgrep -o "$1" | wc -l
 }
 
-# There are ten resources that should be optimized
+# There are nine resources that should be optimized
 http_proxy=$SECONDARY_HOSTNAME \
-    fetch_until $UVA_EXTEND_CACHE 'count_exact_matches .pagespeed.' 10
+    fetch_until $UVA_EXTEND_CACHE 'count_exact_matches .pagespeed.' 9
 
 # Make sure <custom d=...> isn't modified at all, but that everything else is
 # recognized as a url and rewritten from ../foo to /foo.  This means that only
@@ -1788,9 +1595,9 @@ http_proxy=$SECONDARY_HOSTNAME \
 http_proxy=$SECONDARY_HOSTNAME \
     fetch_until $UVA_EXTEND_CACHE 'fgrep -c ../mod_pa' 1
 
-# There are ten images that should be optimized.
+# There are nine images that should be optimized.
 http_proxy=$SECONDARY_HOSTNAME \
-    fetch_until $UVA_EXTEND_CACHE 'count_exact_matches .pagespeed.ic' 10
+    fetch_until $UVA_EXTEND_CACHE 'count_exact_matches .pagespeed.ic' 9
 
 # Test the experiment framework (Furious).
 
@@ -1807,30 +1614,14 @@ URL="$EXP_EXTEND_CACHE?PageSpeed=on&PageSpeedFilters=rewrite_css"
 OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
 check_not_from "$OUT" fgrep 'PageSpeedExperiment='
 
-start_test experiment assignment can be forced
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
-      "$EXP_EXTEND_CACHE?PageSpeedEnrollExperiment=2")
-check_from "$OUT" fgrep 'PageSpeedExperiment=2'
-
-start_test experiment assignment can be forced to a 0% experiment
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
-      "$EXP_EXTEND_CACHE?PageSpeedEnrollExperiment=3")
-check_from "$OUT" fgrep 'PageSpeedExperiment=3'
-
-start_test experiment assignment can be forced even if already assigned
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
-      --header Cookie:PageSpeedExperiment=7 \
-      "$EXP_EXTEND_CACHE?PageSpeedEnrollExperiment=2")
-check_from "$OUT" fgrep 'PageSpeedExperiment=2'
-
 start_test If the user is already assigned, no need to assign them again.
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
-      --header='Cookie: PageSpeedExperiment=2' $EXP_EXTEND_CACHE)
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=2' \
+      $EXP_EXTEND_CACHE)
 check_not_from "$OUT" fgrep 'PageSpeedExperiment='
 
 start_test The beacon should include the experiment id.
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
-      --header='Cookie: PageSpeedExperiment=2' $EXP_EXTEND_CACHE)
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=2' \
+      $EXP_EXTEND_CACHE)
 BEACON_CODE="pagespeed.addInstrumentationInit('/ngx_pagespeed_beacon', 'load',"
 BEACON_CODE+=" '&exptid=2', 'http://experiment.example.com/"
 BEACON_CODE+="mod_pagespeed_example/extend_cache.html');"
@@ -1843,8 +1634,8 @@ BEACON_CODE+="mod_pagespeed_example/extend_cache.html');"
 check_from "$OUT" grep "$BEACON_CODE"
 
 start_test The no-experiment group beacon should not include an experiment id.
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
-      --header='Cookie: PageSpeedExperiment=0' $EXP_EXTEND_CACHE)
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=0' \
+     $EXP_EXTEND_CACHE)
 check_not_from "$OUT" grep 'pagespeed_beacon.*exptid'
 
 # We expect id=7 to be index=a and id=2 to be index=b because that's the
@@ -2072,19 +1863,20 @@ WGET_ARGS=""
 start_test lazyload_images,rewrite_images with critical images beacon
 HOST_NAME="http://imagebeacon.example.com"
 URL="$HOST_NAME/mod_pagespeed_test/image_rewriting/rewrite_images.html"
-# There are 3 images on rewrite_images.html.  Since beaconing is on but we've
-# sent no beacon data, none should be lazy loaded.
-# Run until we see beaconing on the page (should happen on first visit).
+# There are 3 images on rewrite_images.html. Check that they are all
+# lazyloaded by default.
 http_proxy=$SECONDARY_HOSTNAME\
-  fetch_until -save $URL \
-  'fgrep -c "pagespeed.CriticalImages.Run"' 1
-check [ $(grep -c "pagespeed_lazy_src=" $FETCH_FILE) = 0 ];
+  fetch_until -save -recursive $URL 'fgrep -c pagespeed_lazy_src=' 3
+check [ $(grep -c "^pagespeed\.CriticalImages\.Run" \
+  $WGET_DIR/rewrite_images.html) = 1 ];
 # We need the options hash and nonce to send a critical image beacon, so extract
 # it from injected beacon JS.
-OPTIONS_HASH=$(
-  awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' $FETCH_FILE)
-NONCE=$(
-  awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' $FETCH_FILE)
+OPTIONS_HASH=$(awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' \
+               $WGET_DIR/rewrite_images.html)
+NONCE=$(awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' \
+        $WGET_DIR/rewrite_images.html)
+OPTIONS_HASH=$(grep "^pagespeed\.CriticalImages\.Run" \
+  $WGET_DIR/rewrite_images.html | awk -F\' '{print $(NF-3)}')
 # Send a beacon response using POST indicating that Puzzle.jpg is a critical
 # image.
 BEACON_URL="$HOST_NAME/ngx_pagespeed_beacon"
@@ -2096,7 +1888,7 @@ OUT=$(env http_proxy=$SECONDARY_HOSTNAME \
   wget -q --save-headers -O - --no-http-keep-alive \
   --post-data "$BEACON_DATA" "$BEACON_URL")
 check_from "$OUT" egrep -q "HTTP/1[.]. 204"
-# Now 2 of the images should be lazyloaded, Puzzle.jpg should not be.
+# Now only 2 of the images should be lazyloaded, Cuppa.png should not be.
 http_proxy=$SECONDARY_HOSTNAME \
   fetch_until -save -recursive $URL 'fgrep -c pagespeed_lazy_src=' 2
 
@@ -2107,13 +1899,13 @@ http_proxy=$SECONDARY_HOSTNAME \
 # page without blocking.
 URL="$URL?id=4"
 http_proxy=$SECONDARY_HOSTNAME\
-  fetch_until -save $URL \
-  'fgrep -c "pagespeed.CriticalImages.Run"' 1
-check [ $(grep -c "pagespeed_lazy_src=" $FETCH_FILE) = 0 ];
-OPTIONS_HASH=$(
-  awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' $FETCH_FILE)
-NONCE=$(
-  awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' $FETCH_FILE)
+  fetch_until -save -recursive $URL 'fgrep -c pagespeed_lazy_src=' 3
+check [ $(grep -c "^pagespeed\.CriticalImages\.Run" \
+    "$WGET_DIR/rewrite_images.html?id=4") = 1 ];
+OPTIONS_HASH=$(awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' \
+               "$WGET_DIR/rewrite_images.html?id=4")
+NONCE=$(awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' \
+       "$WGET_DIR/rewrite_images.html?id=4")
 BEACON_URL="$HOST_NAME/ngx_pagespeed_beacon"
 BEACON_URL+="?url=http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F"
 BEACON_URL+="image_rewriting%2Frewrite_images.html%3Fid%3D4"
@@ -2204,7 +1996,7 @@ keepalive_test "keepalive-beacon-post.example.com" "/ngx_pagespeed_beacon"\
 
 start_test keepalive with static resources
 keepalive_test "keepalive-static.example.com"\
-  "/pagespeed_custom_static/js_defer.0.js" ""
+  "/ngx_pagespeed_static/js_defer.0.js" ""
 
 # Test for MaxCombinedCssBytes. The html used in the test, 'combine_css.html',
 # has 4 CSS files in the following order.
@@ -2239,7 +2031,7 @@ CONNECTION=$(extract_headers $FETCH_UNTIL_OUTFILE | fgrep "Connection:")
 check_not_from "$CONNECTION" fgrep -qi "Keep-Alive, Keep-Alive"
 check_from "$CONNECTION" fgrep -qi "Keep-Alive"
 
-start_test pagespeed_custom_static defer js served with correct headers.
+start_test ngx_pagespeed_static defer js served with correct headers.
 # First, determine which hash js_defer is served with. We need a correct hash
 # to get it served up with an Etag, which is one of the things we want to test.
 URL="$HOSTNAME/mod_pagespeed_example/defer_javascript.html?PageSpeed=on&PageSpeedFilters=defer_javascript"
@@ -2263,28 +2055,29 @@ http_proxy=$SECONDARY_HOSTNAME fetch_until -save "$URL" \
 # User-Agent:  Accept:  Image type   Result
 # -----------  -------  ----------   ----------------------------------
 #    IE         N/A     photo        image/jpeg, Cache-Control: private *
-#     :         N/A     synthetic    image/png,  no vary
+#     :         N/A     synthetic    image/png,  Cache-Control: private *
 #  Old Opera     no     photo        image/jpeg, Vary: Accept
-#     :          no     synthetic    image/png,  no vary
+#     :          no     synthetic    image/png,  Vary: Accept +
 #     :         webp    photo        image/webp, Vary: Accept, Lossy
-#     :         webp    synthetic    image/png,  no vary
+#     :         webp    synthetic    image/png,  Cache-Control: private +
 #  Chrome or     no     photo        image/jpeg, Vary: Accept
-# Firefox or     no     synthetic    image/png,  no vary
+# Firefox or     no     synthetic    image/png,  Vary: Accept
 #  New Opera    webp    photo        image/webp, Vary: Accept, Lossy
-#     :         webp    synthetic    image/webp, no vary
+#     :         webp    synthetic    image/webp, Cache-Control: private +
+#                                    Lossless webp image returned
 # TODO(jmaessen): * cases currently send Vary: Accept.  Fix (in progress).
 # + has been rejected for now in favor of image/png, Vary: Accept.
-# TODO(jmaessen): Send image/webp lossless for synthetic and alpha-channel
-# images.  Will require reverting to Vary: Accept for these.  Stuff like
-# animated webp will have to remain unconverted still in IPRO mode, or switch
-# to cc: private, but right now animated webp support is still pending anyway.
+# TODO(jmaessen): eliminate Vary: Accept headers from synthetic images that
+# are never going to be considered for webp conversion.  This may prove
+# irrelevant if we instead decide to support Lossless via Vary: Accept and
+# abandon old Opera versions.
 function test_ipro_for_browser_webp() {
   IN_UA_PRETTY="$1"; shift
   IN_UA="$1"; shift
   IN_ACCEPT="$1"; shift
   IMAGE_TYPE="$1"; shift
   OUT_CONTENT_TYPE="$1"; shift
-  OUT_VARY="${1-}"; shift
+  OUT_VARY="$1"; shift
   OUT_CC="${1-}"; shift
   WGET_ARGS="--save-headers \
              ${IN_UA:+--user-agent $IN_UA} \
@@ -2302,7 +2095,7 @@ function test_ipro_for_browser_webp() {
   else
     TEST_ID+=" Accept:$IN_ACCEPT, "
   fi
-  TEST_ID+=" $IMAGE_TYPE.  Expect image/${OUT_CONTENT_TYPE}, "
+  TEST_ID+=" $IMAGE_TYPE.  Expect image/${IMAGE_TYPE}, "
   if [ -z "$OUT_VARY" ]; then
     TEST_ID+=" no vary, "
   else
@@ -2326,7 +2119,7 @@ function test_ipro_for_browser_webp() {
       fgrep -q "Vary: $OUT_VARY"
   fi
   check_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" \
-    grep -q "Cache-Control: ${OUT_CC:-max-age=[0-9]*}$"
+    fgrep -q "Cache-Control: ${OUT_CC:-max-age=}"
   # TODO: check file type of webp.  Irrelevant for now.
 }
 
@@ -2339,12 +2132,12 @@ test_ipro_for_browser_webp "" "webp-la" ""     photo jpeg "Accept"
 test_ipro_for_browser_webp "None" ""    "webp" photo webp "Accept"
 test_ipro_for_browser_webp "" "webp"    "webp" photo webp "Accept"
 test_ipro_for_browser_webp "" "webp-la" "webp" photo webp "Accept"
-test_ipro_for_browser_webp "None" ""    ""     synth png
-test_ipro_for_browser_webp "" "webp"    ""     synth png
-test_ipro_for_browser_webp "" "webp-la" ""     synth png
-test_ipro_for_browser_webp "None" ""    "webp" synth png
-test_ipro_for_browser_webp "" "webp"    "webp" synth png
-test_ipro_for_browser_webp "" "webp-la" "webp" synth png
+test_ipro_for_browser_webp "None" ""    ""     synth png  "Accept"
+test_ipro_for_browser_webp "" "webp"    ""     synth png  "Accept"
+test_ipro_for_browser_webp "" "webp-la" ""     synth png  "Accept"
+test_ipro_for_browser_webp "None" ""    "webp" synth png  "Accept"
+test_ipro_for_browser_webp "" "webp"    "webp" synth png  "Accept"
+test_ipro_for_browser_webp "" "webp-la" "webp" synth png  "Accept"
 ##############################################################################
 
 # Wordy UAs need to be stored in the WGETRC file to avoid death by quoting.
@@ -2352,37 +2145,34 @@ OLD_WGETRC=$WGETRC
 WGETRC=$TEMPDIR/wgetrc-ua
 export WGETRC
 
-# IE 9 and later must re-validate Vary: Accept.  We should send CC: private.
-IE9_UA="Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))"
-IE11_UA="Mozilla/5.0 (Windows NT 6.1; WOW64; ***********; rv:11.0) like Gecko"
-echo "user_agent = $IE9_UA" > $WGETRC
-#                           (no accept)  Type  Out  Vary CC
-test_ipro_for_browser_webp "IE 9"  "" "" photo jpeg ""   "max-age=[0-9]*,private"
-test_ipro_for_browser_webp "IE 9"  "" "" synth png
+# IE 11 does not cache Vary: Accept.  For now we send it nonetheless.  See
+# TODO above.
+IE11_UA="Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko"
 echo "user_agent = $IE11_UA" > $WGETRC
-test_ipro_for_browser_webp "IE 11" "" "" photo jpeg ""   "max-age=[0-9]*,private"
-test_ipro_for_browser_webp "IE 11" "" "" synth png
+#                            (no accept) Type  Out  Vary
+test_ipro_for_browser_webp "IE 11" "" "" photo jpeg "Accept"
+test_ipro_for_browser_webp "IE 11" "" "" synth png  "Accept"
 
 # Older Opera did not support webp.
 OPERA_UA="Opera/9.80 (Windows NT 5.2; U; en) Presto/2.7.62 Version/11.01"
 echo "user_agent = $OPERA_UA" > $WGETRC
 #                                (no accept) Type  Out  Vary
 test_ipro_for_browser_webp "Old Opera" "" "" photo jpeg "Accept"
-test_ipro_for_browser_webp "Old Opera" "" "" synth png
+test_ipro_for_browser_webp "Old Opera" "" "" synth png  "Accept"
 # Slightly newer opera supports only lossy webp, sends header.
 OPERA_UA="Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10"
 echo "user_agent = $OPERA_UA" > $WGETRC
 #                                           Accept Type  Out  Vary
 test_ipro_for_browser_webp "Newer Opera" "" "webp" photo webp "Accept"
-test_ipro_for_browser_webp "Newer Opera" "" "webp" synth png
+test_ipro_for_browser_webp "Newer Opera" "" "webp" synth png  "Accept"
 
 function test_decent_browsers() {
   echo "user_agent = $2" > $WGETRC
   #                          UA      Accept Type      Out  Vary
   test_ipro_for_browser_webp "$1" "" ""     photo     jpeg "Accept"
-  test_ipro_for_browser_webp "$1" "" ""     synthetic  png
+  test_ipro_for_browser_webp "$1" "" ""     synthetic  png "Accept"
   test_ipro_for_browser_webp "$1" "" "webp" photo     webp "Accept"
-  test_ipro_for_browser_webp "$1" "" "webp" synthetic  png
+  test_ipro_for_browser_webp "$1" "" "webp" synthetic  png "Accept"
 }
 CHROME_UA="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 "
 CHROME_UA+="(KHTML, like Gecko) Chrome/32.0.1700.102 Safari/537.36"
@@ -2396,9 +2186,7 @@ test_decent_browsers "New Opera" \
 WGETRC=$OLD_WGETRC
 WGET_ARGS=""
 
-start_test JS gzip headers
-
-JS_URL="$HOSTNAME/pagespeed_custom_static/js_defer.$HASH.js"
+JS_URL="$HOSTNAME/ngx_pagespeed_static/js_defer.$HASH.js"
 JS_HEADERS=$($WGET -O /dev/null -q -S --header='Accept-Encoding: gzip' \
   $JS_URL 2>&1)
 check_from "$JS_HEADERS" egrep -qi 'HTTP/1[.]. 200 OK'
@@ -2421,7 +2209,7 @@ start_test IPRO flow uses cache as expected.
 STATS=$OUTDIR/blocking_rewrite_stats
 IPRO_ROOT=http://ipro.example.com/mod_pagespeed_test/ipro
 URL=$IPRO_ROOT/test_image_dont_reuse2.png
-IPRO_STATS_URL=http://ipro.example.com/ngx_pagespeed_statistics
+IPRO_STATS_URL=http://ipro.example.com/ngx_pagespeed_statistics?PageSpeed=off
 
 # Initial stats.
 http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $IPRO_STATS_URL > $STATS.0
@@ -2539,12 +2327,9 @@ OUTFILE=$OUTDIR/ipro_resource_output
 # Fetch the HTML to initiate rewriting and caching of the image.
 http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $HTML_URL -O $OUTFILE
 
-# First IPRO resource request after a short wait: never be optimized
-# because our non-load-from-file flow doesn't support that, but it will have
-# the full TTL.
+# First IPRO resource request after a short wait: it will have the full TTL.
 sleep 2
 http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $RESOURCE_URL -O $OUTFILE
-check_file_size "$OUTFILE" -gt 15000 # not optimized
 RESOURCE_MAX_AGE=$( \
   extract_headers $OUTFILE | \
   grep 'Cache-Control:' | tr -d '\r' | \
@@ -2552,25 +2337,17 @@ RESOURCE_MAX_AGE=$( \
 check test -n "$RESOURCE_MAX_AGE"
 check test $RESOURCE_MAX_AGE -eq 333
 
-# Second IPRO resource request after a short wait: it will still be optimized
-# and the TTL will be reduced.
+# Second IPRO resource request after a short wait: the TTL will be reduced.
+sleep 2
 http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $RESOURCE_URL -O $OUTFILE
-check_file_size "$OUTFILE" -lt 15000 # optimized
 RESOURCE_MAX_AGE=$( \
   extract_headers $OUTFILE | \
   grep 'Cache-Control:' | tr -d '\r' | \
   sed -e 's/^ *Cache-Control: *//' | sed -e 's/^.*max-age=\([0-9]*\).*$/\1/')
 check test -n "$RESOURCE_MAX_AGE"
-
 check test $RESOURCE_MAX_AGE -lt 333
 check test $RESOURCE_MAX_AGE -gt 300
 
-# TODO(jmaessen, jefftk): Port proxying tests, which rely on pointing a
-# MapProxyDomain construct at a static server.  Perhaps localhost:8050 will
-# serve, but the tests need to use different urls then.  For mod_pagespeed these
-# tests immediately precede "IPRO-optimized resources should have fixed size,
-# not chunked." in system_test.sh.
-
 start_test IPRO-optimized resources should have fixed size, not chunked.
 URL="$EXAMPLE_ROOT/images/Puzzle.jpg"
 URL+="?PageSpeedJpegRecompressionQuality=75"
@@ -2582,34 +2359,6 @@ check [ "$CONTENT_LENGTH" -lt 90000 ];
 check_not_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" \
     fgrep -q 'Transfer-Encoding: chunked'
 
-start_test IPRO 304 with etags
-# Reuses $URL and $FETCH_UNTIL_OUTFILE from previous test.
-check_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" fgrep -q 'ETag:'
-ETAG=$(extract_headers $FETCH_UNTIL_OUTFILE | awk '/ETag:/ {print $2}')
-echo $WGET_DUMP --header "If-None-Match: $ETAG" $URL
-OUTFILE=$OUTDIR/etags
-# Note: -o gets debug info which is the only place that 304 message is sent.
-$WGET -o $OUTFILE -O /dev/null --header "If-None-Match: $ETAG" $URL
-check fgrep -q "awaiting response... 304" $OUTFILE
-
-start_test PageSpeed CSS loaded in fallback mode is always chunked
-# This is because the domains in urls may be rewritten, changing content
-# length from what's in the cache.
-URL="$EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css"
-fetch_until -save $URL "egrep -c Transfer-Encoding:" 1 "--save-headers"
-OUT=$(cat $FETCH_UNTIL_OUTFILE)
-check_from "$OUT" egrep -iq $'^Transfer-Encoding: chunked\r$'
-check_not_from "$OUT" egrep -iq '^Content-Length:'
-check_not_from "$OUT" egrep -iq '^Connection: close'
-
-start_test Downstream cache integration caching headers.
-URL="http://downstreamcacheresource.example.com/mod_pagespeed_example/images/"
-URL+="xCuppa.png.pagespeed.ic.0.png"
-OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
-check_from "$OUT" egrep -iq $'^Cache-Control: .*\r$'
-check_from "$OUT" egrep -iq $'^Expires: .*\r$'
-check_from "$OUT" egrep -iq $'^Last-Modified: .*\r$'
-
 # Test handling of large HTML files. We first test with a cold cache, and verify
 # that we bail out of parsing and insert a script redirecting to
 # ?PageSpeed=off. This should also insert an entry into the property cache so
@@ -2624,13 +2373,13 @@ WGET_ARGS="--header=PageSpeedFilters:rewrite_images"
 WGET_EC="$WGET_DUMP $WGET_ARGS"
 echo $WGET_EC $URL
 LARGE_OUT=$($WGET_EC $URL)
-check_from "$LARGE_OUT" grep -q window.location=".*&PageSpeed=off"
+check_from "$LARGE_OUT" grep -q window.location=".*&ModPagespeed=off"
 
 # The file should now be in the property cache so make sure that the page is no
 # longer parsed. Use fetch_until because we need to wait for a potentially
 # non-blocking write to the property cache from the previous test to finish
 # before this will succeed.
-fetch_until -save $URL 'grep -c window.location=".*&PageSpeed=off"' 0
+fetch_until -save $URL 'grep -c window.location=".*&ModPagespeed=off"' 0
 check_not fgrep -q pagespeed.ic $FETCH_FILE
 
 start_test messages load
@@ -2651,13 +2400,6 @@ check_not_from "$OUT" egrep -q '^Date: Thu, 01 Jan 1970 00:00:00 GMT'
 OUT=$($WGET_DUMP --header=Host:date.example.com \
     http://$SECONDARY_HOSTNAME/mod_pagespeed_example/combine_css.html)
 check_from "$OUT" egrep -q '^Date: Fri, 16 Oct 2009 23:05:07 GMT'
-WGET_ARGS=
-
-if [ "$NATIVE_FETCHER" != "on" ]; then
-  start_test Test that we can rewrite an HTTPS resource.
-  fetch_until $TEST_ROOT/https_fetch/https_fetch.html \
-    'grep -c /https_gstatic_dot_com/1.gif.pagespeed.ce' 1
-fi
 
 if $USE_VALGRIND; then
     # It is possible that there are still ProxyFetches outstanding

test/pagespeed_test.conf.template

@@ -27,15 +27,6 @@ http {
   proxy_cache_path "@@PROXY_CACHE@@"  levels=1:2   keys_zone=htmlcache:60m inactive=90m  max_size=50m;
   proxy_temp_path "@@TMP_PROXY_CACHE@@";
 
-  pagespeed StatisticsPath /ngx_pagespeed_statistics;
-  pagespeed GlobalStatisticsPath /ngx_pagespeed_global_statistics;
-  pagespeed ConsolePath /pagespeed_console;
-  pagespeed MessagesPath /ngx_pagespeed_message;
-  pagespeed AdminPath /pagespeed_admin;
-  pagespeed GlobalAdminPath /pagespeed_global_admin;
-
-  pagespeed StaticAssetPrefix /pagespeed_custom_static/;
-
   root "@@SERVER_ROOT@@";
 
   # Block 5a: Decide on Cache-Control header value to use for outgoing
@@ -51,6 +42,7 @@ http {
   pagespeed UsePerVHostStatistics on;
   pagespeed InPlaceResourceOptimization on;
   pagespeed CreateSharedMemoryMetadataCache "@@SHM_CACHE@@" 8192;
+  pagespeed PreserveUrlRelativity on;
   pagespeed BlockingRewriteKey psatest;
 
   # CriticalImagesBeaconEnabled is now on by default, but we disable in testing.
@@ -152,15 +144,7 @@ http {
       set $bypass_cache "1";
     }
 
-    # For testing purposes, we never generate values that will result in a beacon 
-    # unless a test request it via "X-Allow-Beacon: yes" in its request header.
-    # This is for testing purposes, note that in a production environment, 
-    # you want 'set_random $rand 0 100;' unconditionally!
-    set $rand 5;
-
-    if ($http_x_allow_beacon ~ "yes") {
-      set_random $rand 0 100;
-    }
+    set_random $rand 0 100;
     set $should_beacon_header_val "";
     if ($rand ~* "^[0-4]$") {
       set $should_beacon_header_val "random_rebeaconing_key";
@@ -212,45 +196,6 @@ http {
     }
   }
 
-  server {
-    listen @@SECONDARY_PORT@@;
-    server_name if-in-server.example.com;
-    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-
-    pagespeed RewriteLevel PassThrough;
-    set $inline_javascript "No";
-
-    if ($http_x_custom_header_inline_js) {
-      # TODO(jefftk): Turn on NGX_HTTP_SIF_CONF and figure out how to get
-      # pagespeed directives inside of a server location block to be respected,
-      # then uncomment the following line and duplicate the if-in-location test
-      # for if-in-server.
-      #pagespeed EnableFilters inline_javascript;
-      set $inline_javascript "Yes";
-    }
-
-    add_header "X-Inline-Javascript" $inline_javascript;
-  }
-
-  server {
-    listen @@SECONDARY_PORT@@;
-    server_name if-in-location.example.com;
-    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-
-
-    location / {
-      set $inline_javascript "No";
-      pagespeed RewriteLevel PassThrough;
-
-      if ($http_x_custom_header_inline_js) {
-        pagespeed EnableFilters inline_javascript;
-        set $inline_javascript "Yes";
-      }
-
-      add_header "X-Inline-Javascript" $inline_javascript;
-    }
-  }
-
   server {
     listen @@SECONDARY_PORT@@;
     server_name mpd.example.com;
@@ -273,7 +218,6 @@ http {
     pagespeed ExperimentSpec
       "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
     pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
-    pagespeed ExperimentSpec "id=3;default;percent=0";
   }
   server {
     listen @@SECONDARY_PORT@@;
@@ -285,7 +229,6 @@ http {
     pagespeed ExperimentSpec
       "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
     pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
-    pagespeed ExperimentSpec "id=3;default;percent=0";
   }
 
   server {
@@ -344,15 +287,6 @@ http {
     }
   }
 
-  server {
-    listen @@SECONDARY_PORT@@;
-    server_name downstreamcacheresource.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-    pagespeed RewriteLevel PassThrough;
-    pagespeed EnableFilters rewrite_images;
-    pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
-  }
-
   server {
     listen @@SECONDARY_PORT@@;
     server_name renderedimagebeacon.example.com;
@@ -481,9 +415,6 @@ http {
     pagespeed UrlValuedAttribute img alt-src Image;
     pagespeed UrlValuedAttribute video alt-a Image;
     pagespeed UrlValuedAttribute video alt-b Image;
-
-    # Also test that we can redefine spec-defined attributes.
-    pagespeed UrlValuedAttribute blockquote cite Image;
   }
 
   server {
@@ -561,7 +492,7 @@ http {
   # Note that we test with two distinct caches.
   server {
     listen @@SECONDARY_PORT@@;
-    server_name embed-config-html.example.org;
+    server_name embed-config-html.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
     root "@@SERVER_ROOT@@/mod_pagespeed_test";
@@ -571,9 +502,6 @@ http {
     pagespeed DisableFilters inline_css,extend_cache,inline_javascript;
     pagespeed Domain embed-config-resources.example.com;
 
-    # Share a cache keyspace with embed-config-resources.example.com.
-    pagespeed CacheFragment "embed-config";
-
     pagespeed LoadFromFile "http://embed-config-resources.example.com/"
       "@@SERVER_ROOT@@/mod_pagespeed_example/";
   }
@@ -589,10 +517,7 @@ http {
 
     # Note that we do not set the jpeg quality here, but take
     # it from image URL query parameters that we synthesize in
-    # from embed-config-html.example.org.
-
-    # Share a cache keyspace with embed-config-html.example.org.
-    pagespeed CacheFragment "embed-config";
+    # from embed-config-html.example.com.
 
     pagespeed LoadFromFile "http://embed-config-resources.example.com/"
       "@@SERVER_ROOT@@/mod_pagespeed_example/";
@@ -606,7 +531,7 @@ http {
     pagespeed EnableFilters convert_to_webp_lossless;
     pagespeed EnableFilters in_place_optimize_for_browser;
     pagespeed InPlaceResourceOptimization on;
-    pagespeed FileCachePath "@@IPRO_CACHE@@";
+    pagespeed FileCachePath "@@FILE_CACHE@@_ipro_for_browser";
   }
 
   server {
@@ -660,7 +585,6 @@ http {
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
-    pagespeed CacheFragment "example";
   }
   server {
     # Sets up a logical origin for CDNs to fetch content from, on
@@ -673,7 +597,6 @@ http {
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
-    pagespeed CacheFragment "example";
   }
   server {
     # Sets up a logical cdn, which is where we tell browsers to fetch resources
@@ -713,23 +636,6 @@ http {
     }
   }
 
-  server {
-    listen @@SECONDARY_PORT@@;
-    server_name custom-paths.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-
-    pagespeed StatisticsPath /custom_pagespeed_statistics;
-    pagespeed ConsolePath /custom_pagespeed_console;
-    pagespeed MessagesPath /custom_pagespeed_message;
-    pagespeed AdminPath /custom_pagespeed_admin;
-  }
-
-  server {
-    listen @@SECONDARY_PORT@@;
-    server_name inherit-paths.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-  }
-
   server {
     listen @@SECONDARY_PORT@@;
     server_name notransform.example.com;
@@ -881,48 +787,15 @@ http {
     add_header "Date" "Date: Fri, 16 Oct 2009 23:05:07 GMT";
   }
 
-  # Proxy + IPRO a gzip'd file for testing Issue 896.
-  server {
-    listen @@SECONDARY_PORT@@;
-    server_name ipro-proxy.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@_ipro_proxy";
-
-    pagespeed on;
-    pagespeed InPlaceResourceOptimization on;
-    pagespeed EnableFilters rewrite_domains;
-
-    location / {
-      proxy_pass
-       "http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/mod_deflate/";
-    }
-  }
-
-  server {
-    listen @@SECONDARY_PORT@@;
-    server_name compressed-css.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-    pagespeed InPlaceResourceOptimization on;
-  }
-
   server {
     listen       @@PRIMARY_PORT@@;
     server_name  localhost;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    location ~ ^/pagespeed_admin {
-      allow 127.0.0.1;
-      deny all;
-    }
-
     location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" {
       add_header "" "";
     }
 
-    # Backend for ipro-proxy.example.com
-    location /mod_pagespeed_test/ipro/mod_deflate/ {
-      pagespeed off;
-    }
-
     location /mod_pagespeed_test/cachable_rewritten_html/ {
       # This location has the html files that will be configured to be stored
       # in the proxy_cache layer.
@@ -1082,16 +955,7 @@ http {
 
     location /mod_pagespeed_test/ipro/instant/wait/ {
       pagespeed InPlaceWaitForOptimized on;
-
-      # TODO(jefftk): When running with valgrind we need to raise the rewrite
-      # deadline or else we get "Deadline exceeded for rewrite of resource".
-      # I had thought InPlaceWaitForOptimized was supposed to disable the
-      # rewrite deadline, but it seems not to.
-      #
-      # InPlaceWaitForOptimized is definitely doing something, though: when I
-      # remove the InPlaceRewriteDeadlineMs line below and run without valgrind,
-      # purple.css is optimized via ipro on the first request but if I set
-      # InPlaceWaitForOptimized to off then it is not.
+      # TODO: Valgrind runs pass only if the below line is uncommented.
       pagespeed InPlaceRewriteDeadlineMs 1000;
     }
 
@@ -1099,13 +963,6 @@ http {
       pagespeed InPlaceRewriteDeadlineMs -1;
     }
 
-    # Test to make sure that user-authenticated resources do not get cached and
-    # optimized.
-    location /mod_pagespeed_test/auth/ {
-       auth_basic "Restricted";
-       auth_basic_user_file "@@SERVER_ROOT@@mod_pagespeed_test/auth/passwd.conf";
-    }
-
     location /mod_pagespeed_test/ipro/cookie/ {
       # Add Vary:Cookie.  This should prevent us from optimizing the
       # vary_cookie.css even though PagespeedRespectVary is off.
@@ -1141,14 +998,6 @@ http {
       pagespeed UseExperimentalJsMinifier on;
     }
 
-    pagespeed FetchHttps enable;
-    location /mod_pagespeed_test/https_fetch/ {
-      pagespeed DisableFilters inline_images;
-      pagespeed MapProxyDomain
-          http://localhost:@@PRIMARY_PORT@@/https_gstatic_dot_com
-          https://www.gstatic.com/psa/static;
-    }
-
     pagespeed LoadFromFile
         "http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/instant/"
         "@@SERVER_ROOT@@/mod_pagespeed_test/ipro/instant/";
@@ -1191,7 +1040,6 @@ http {
   # set up gzip
   gzip  on;
   gzip_vary on;
-  gzip_proxied any;
   # Turn on gzip for all content types that should benefit from it.
   gzip_types application/ecmascript;
   gzip_types application/javascript;