Update for 1.8.31.4

Merge remote-tracking branch 'origin/release-1.8.31.3-beta'
Pulling in upstream merge to local master.
2014-06-17 13:32:51 -04:00 · 2014-05-29 15:06:03 -04:00 · 2014-05-29 15:01:45 -04:00 · 2014-05-29 14:51:36 -04:00 · 2014-05-29 14:32:18 -04:00 · 2014-05-29 14:30:03 -04:00
17 changed files with 1535 additions and 705 deletions
@@ -37,11 +37,11 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p

   ```bash
   $ cd ~
-   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.7.30.3-beta.zip
-   $ unzip v1.7.30.3-beta.zip # or unzip v1.7.30.3-beta
-   $ cd ngx_pagespeed-1.7.30.3-beta/
-   $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.3.tar.gz
-   $ tar -xzvf 1.7.30.3.tar.gz # expands to psol/
+   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.8.31.4-beta.zip
+   $ unzip v1.8.31.4-beta.zip # or unzip v1.8.31.4-beta
+   $ cd ngx_pagespeed-1.8.31.4-beta/
+   $ wget https://dl.google.com/dl/page-speed/psol/1.8.31.4.tar.gz
+   $ tar -xzvf 1.8.31.4.tar.gz # expands to psol/
   ```

 3. Download and build nginx:
@@ -49,10 +49,10 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p
   ```bash
   $ cd ~
   $ # check http://nginx.org/en/download.html for the latest version
-   $ wget http://nginx.org/download/nginx-1.4.4.tar.gz
-   $ tar -xvzf nginx-1.4.4.tar.gz
-   $ cd nginx-1.4.4/
-   $ ./configure --add-module=$HOME/ngx_pagespeed-1.7.30.3-beta
+   $ wget http://nginx.org/download/nginx-1.6.0.tar.gz
+   $ tar -xvzf nginx-1.6.0.tar.gz
+   $ cd nginx-1.6.0/
+   $ ./configure --add-module=$HOME/ngx_pagespeed-1.8.31.4-beta
   $ make
   $ sudo make install
   ```
@@ -82,12 +82,8 @@ In every server block where pagespeed is enabled add:
 #  Ensure requests for pagespeed optimized resources go to the pagespeed
 #  handler and no extraneous headers get set.
 location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" { add_header "" ""; }
-location ~ "^/ngx_pagespeed_static/" { }
+location ~ "^/pagespeed_static/" { }
 location ~ "^/ngx_pagespeed_beacon$" { }
-location /ngx_pagespeed_statistics { allow 127.0.0.1; deny all; }
-location /ngx_pagespeed_global_statistics { allow 127.0.0.1; deny all; }
-location /ngx_pagespeed_message { allow 127.0.0.1; deny all; }
-location /pagespeed_console { allow 127.0.0.1; deny all; }
 ```

 To confirm that the module is loaded, fetch a page and check that you see the
@@ -95,7 +91,7 @@ To confirm that the module is loaded, fetch a page and check that you see the

 ```bash
 $ curl -I 'http://localhost:8050/some_page/' | grep X-Page-Speed
-X-Page-Speed: 1.7.30.3-...
+X-Page-Speed: 1.8.31.4-...
 ```

 Looking at the source of a few pages you should see various changes, such as
@@ -27,8 +27,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
    echo "   You need to separately download the pagespeed library:"
    echo ""
    echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.2.tar.gz"
-    echo "     $ tar -xzvf 1.7.30.2.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.8.31.4.tar.gz"
+    echo "     $ tar -xzvf 1.8.31.4.tar.gz # expands to psol/"
    echo ""
    echo "   Or see the installation instructions:"
    echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
@@ -109,6 +109,10 @@ case "$NGX_GCC_VER" in
  ;;
 esac

+if [ "$WNO_ERROR" = "YES" ]; then
+    CFLAGS="$CFLAGS -Wno-error"
+fi
+
 pagespeed_include="\
  $mod_pagespeed_dir \
  $mod_pagespeed_dir/third_party/chromium/src \
@@ -106,11 +106,10 @@ ngx_int_t NgxBaseFetch::CollectAccumulatedWrites(ngx_chain_t** link_ptr) {
 ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
  const ResponseHeaders* pagespeed_headers = response_headers();

-  // TODO(chaizhenhua): Add and check.
-  // if (content_length_known()) {
-  //   headers_out->content_length = NULL;
-  //   headers_out->content_length_n = content_length();
-  // }
+  if (content_length_known()) {
+     headers_out->content_length = NULL;
+     headers_out->content_length_n = content_length();
+  }

  return copy_response_headers_to_ngx(request_, *pagespeed_headers,
                                      preserve_caching_headers_);
@@ -71,12 +71,13 @@ namespace net_instaweb {
        fetch_end_ms_(0),
        done_(false),
        content_length_(-1),
-        content_length_known_(false) {
-            ngx_memzero(&url_, sizeof(url_));
-            log_ = log;
-            pool_ = NULL;
-            timeout_event_ = NULL;
-            connection_ = NULL;
+        content_length_known_(false),
+        resolver_ctx_(NULL) {
+    ngx_memzero(&url_, sizeof(url_));
+    log_ = log;
+    pool_ = NULL;
+    timeout_event_ = NULL;
+    connection_ = NULL;
  }

  NgxFetch::~NgxFetch() {
@@ -213,6 +214,8 @@ namespace net_instaweb {
      return;
    }

+    release_resolver();
+
    if (timeout_event_ && timeout_event_->timer_set) {
      ngx_del_timer(timeout_event_);
      timeout_event_ = NULL;
@@ -288,7 +291,6 @@ namespace net_instaweb {
          kWarning, "NgxFetch: failed to resolve host [%.*s]",
          static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data);
      fetch->CallbackDone(false);
-      ngx_resolve_name_done(resolver_ctx);
      return;
    }
    ngx_memzero(&fetch->sin_, sizeof(fetch->sin_));
@@ -319,7 +321,7 @@ namespace net_instaweb {
        static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data,
        ip_address);

-    ngx_resolve_name_done(resolver_ctx);
+    fetch->release_resolver();

    if (fetch->InitRequest() != NGX_OK) {
      fetch->message_handler()->Message(kError, "NgxFetch: InitRequest failed");
@@ -83,6 +83,12 @@ class NgxFetch : public PoolElement<NgxFetch> {
  void set_timeout_event(ngx_event_t* x) {
    timeout_event_ = x;
  }
+  void release_resolver() {
+    if (resolver_ctx_ != NULL && resolver_ctx_ != NGX_NO_RESOLVER) {
+      ngx_resolve_name_done(resolver_ctx_);
+      resolver_ctx_ = NULL;
+    }
+  }

 private:
  response_handler_pt response_handler;
@@ -148,14 +148,4 @@ void NgxMessageHandler::FileMessageVImpl(MessageType type, const char* file,
  }
 }

-// TODO(sligocki): It'd be nice not to do so much string copying.
-GoogleString NgxMessageHandler::Format(const char* msg, va_list args) {
-  GoogleString buffer;
-
-  // Ignore the name of this routine: it formats with vsnprintf.
-  // See base/stringprintf.cc.
-  StringAppendV(&buffer, msg, args);
-  return buffer;
-}
-
 }  // namespace net_instaweb
@@ -70,7 +70,6 @@ class NgxMessageHandler : public GoogleMessageHandler {

 private:
  ngx_uint_t GetNgxLogLevel(MessageType type);
-  GoogleString Format(const char* msg, va_list args);

  scoped_ptr<AbstractMutex> mutex_;
  GoogleString pid_string_;
@@ -36,27 +36,28 @@
 #include "ngx_rewrite_options.h"
 #include "ngx_server_context.h"

-#include "apr_time.h"
-
 #include "net/instaweb/automatic/public/proxy_fetch.h"
+#include "net/instaweb/http/public/async_fetch.h"
 #include "net/instaweb/http/public/cache_url_async_fetcher.h"
 #include "net/instaweb/http/public/content_type.h"
 #include "net/instaweb/http/public/request_context.h"
+#include "net/instaweb/public/global_constants.h"
+#include "net/instaweb/public/version.h"
 #include "net/instaweb/rewriter/public/experiment_matcher.h"
 #include "net/instaweb/rewriter/public/experiment_util.h"
 #include "net/instaweb/rewriter/public/process_context.h"
 #include "net/instaweb/rewriter/public/resource_fetch.h"
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "net/instaweb/rewriter/public/rewrite_query.h"
 #include "net/instaweb/rewriter/public/rewrite_stats.h"
 #include "net/instaweb/rewriter/public/static_asset_manager.h"
-#include "net/instaweb/system/public/handlers.h"
 #include "net/instaweb/system/public/in_place_resource_recorder.h"
 #include "net/instaweb/system/public/system_caches.h"
 #include "net/instaweb/system/public/system_request_context.h"
 #include "net/instaweb/system/public/system_rewrite_options.h"
+#include "net/instaweb/system/public/system_server_context.h"
 #include "net/instaweb/system/public/system_thread_system.h"
-#include "net/instaweb/public/global_constants.h"
-#include "net/instaweb/public/version.h"
 #include "net/instaweb/util/public/fallback_property_page.h"
 #include "net/instaweb/util/public/google_message_handler.h"
 #include "net/instaweb/util/public/google_url.h"
@@ -70,8 +71,9 @@
 #include "net/instaweb/util/public/time_util.h"
 #include "net/instaweb/util/stack_buffer.h"
 #include "pagespeed/kernel/base/posix_timer.h"
-#include "pagespeed/kernel/thread/pthread_shared_mem.h"
+#include "pagespeed/kernel/http/query_params.h"
 #include "pagespeed/kernel/html/html_keywords.h"
+#include "pagespeed/kernel/thread/pthread_shared_mem.h"

 extern ngx_module_t ngx_pagespeed;

@@ -203,6 +205,45 @@ ngx_int_t string_piece_to_buffer_chain(
 // modified from NgxBaseFetch::CopyHeadersFromTable()
 namespace {

+// Based on ngx_http_add_cache_control.
+ngx_int_t ps_set_cache_control(ngx_http_request_t* r, char* cache_control) {
+  // First strip existing cache-control headers.
+  ngx_table_elt_t* header;
+  NgxListIterator it(&(r->headers_out.headers.part));
+  while ((header = it.Next()) != NULL) {
+    if (STR_CASE_EQ_LITERAL(header->key, "Cache-Control")) {
+      // Response headers with hash of 0 are excluded from the response.
+      header->hash = 0;
+    }
+  }
+
+  // Now add our new cache control header.
+  if (r->headers_out.cache_control.elts == NULL) {
+    ngx_int_t rc = ngx_array_init(&r->headers_out.cache_control, r->pool,
+                                  1, sizeof(ngx_table_elt_t *));
+    if (rc != NGX_OK) {
+      return NGX_ERROR;
+    }
+  }
+  ngx_table_elt_t** cache_control_headers = static_cast<ngx_table_elt_t**>(
+      ngx_array_push(&r->headers_out.cache_control));
+  if (cache_control_headers == NULL) {
+    return NGX_ERROR;
+  }
+  cache_control_headers[0] = static_cast<ngx_table_elt_t*>(
+      ngx_list_push(&r->headers_out.headers));
+  if (cache_control_headers[0] == NULL) {
+    return NGX_ERROR;
+  }
+  cache_control_headers[0]->hash = 1;
+  ngx_str_set(&cache_control_headers[0]->key, "Cache-Control");
+  cache_control_headers[0]->value.len = strlen(cache_control);
+  cache_control_headers[0]->value.data =
+      reinterpret_cast<u_char*>(cache_control);
+
+  return NGX_OK;
+}
+
 template<class Headers>
 void copy_headers_from_table(const ngx_list_t &from, Headers* to) {
  // Standard nginx idiom for iterating over a list.  See ngx_list.h
@@ -240,18 +281,21 @@ void copy_response_headers_from_ngx(const ngx_http_request_t* r,

  headers->set_status_code(r->headers_out.status);

+  if (r->headers_out.location != NULL) {
+    headers->Add(HttpAttributes::kLocation,
+                 str_to_string_piece(r->headers_out.location->value));
+  }
+
  // Manually copy over the content type because it's not included in
  // request_->headers_out.headers.
  headers->Add(HttpAttributes::kContentType,
               str_to_string_piece(r->headers_out.content_type));

-  // When we don't have a date header, invent one.
-  const char* date = headers->Lookup1(HttpAttributes::kDate);
-  
-  if (date == NULL) {
+  // When we don't have a date header, set one with the current time.
+  if (headers->Lookup1(HttpAttributes::kDate) == NULL) {
    headers->SetDate(ngx_current_msec);
  }
-  
+
  // TODO(oschaaf): ComputeCaching should be called in setupforhtml()?
  headers->ComputeCaching();
 }
@@ -278,7 +322,7 @@ ngx_int_t copy_response_headers_to_ngx(
    const GoogleString& name_gs = pagespeed_headers.Name(i);
    const GoogleString& value_gs = pagespeed_headers.Value(i);

-    if (preserve_caching_headers != kDontPreserveHeaders) {
+    if (preserve_caching_headers == kPreserveAllCachingHeaders) {
      if (StringCaseEqual(name_gs, "ETag") ||
          StringCaseEqual(name_gs, "Expires") ||
          StringCaseEqual(name_gs, "Date") ||
@@ -286,7 +330,13 @@ ngx_int_t copy_response_headers_to_ngx(
          StringCaseEqual(name_gs, "Cache-Control")) {
        continue;
      }
-    }
+    } else if (preserve_caching_headers == kPreserveOnlyCacheControl) {
+      // Retain the original Cache-Control header, but send the recomputed
+      // values for all other cache-related headers.
+      if (StringCaseEqual(name_gs, "Cache-Control")) {
+        continue;
+      }
+    } // else we don't preserve any headers

    ngx_str_t name, value;

@@ -320,7 +370,10 @@ ngx_int_t copy_response_headers_to_ngx(
      return NGX_ERROR;
    }

-    if (STR_EQ_LITERAL(name, "Content-Type")) {
+    if (STR_EQ_LITERAL(name, "Cache-Control")) {
+      ps_set_cache_control(r, const_cast<char*>(value_gs.c_str()));
+      continue;
+    } else if (STR_EQ_LITERAL(name, "Content-Type")) {
      // Unlike all the other headers, content_type is just a string.
      headers_out->content_type.data = value_s;
      headers_out->content_type.len = value.len;
@@ -342,8 +395,6 @@ ngx_int_t copy_response_headers_to_ngx(
    // copy here?
    } else if (STR_EQ_LITERAL(name, "Connection")) {
      continue;
-    } else if (STR_EQ_LITERAL(name, "Vary")) {
-      continue;
    } else if (STR_EQ_LITERAL(name, "Keep-Alive")) {
      continue;
    } else if (STR_EQ_LITERAL(name, "Transfer-Encoding")) {
@@ -429,8 +480,11 @@ enum Response {
  kPagespeedDisabled,
  kBeacon,
  kStatistics,
+  kGlobalStatistics,
  kConsole,
  kMessages,
+  kAdmin,
+  kGlobalAdmin,
  kPagespeedSubrequest,
  kNotHeadOrGet,
  kErrorResponse,
@@ -438,12 +492,22 @@ enum Response {
 };
 }  // namespace RequestRouting

+char* ps_main_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 char* ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);

+// TODO(jud): Verify that all the offsets should be NGX_HTTP_SRV_CONF_OFFSET and
+// not NGX_HTTP_LOC_CONF_OFFSET or NGX_HTTP_MAIN_CONF_OFFSET.
 ngx_command_t ps_commands[] = {
  { ngx_string("pagespeed"),
-    NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1|
+    NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1|
+    NGX_CONF_TAKE2|NGX_CONF_TAKE3|NGX_CONF_TAKE4|NGX_CONF_TAKE5,
+    ps_main_configure,
+    NGX_HTTP_SRV_CONF_OFFSET,
+    0,
+    NULL },
+  { ngx_string("pagespeed"),
+    NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1|
    NGX_CONF_TAKE2|NGX_CONF_TAKE3|NGX_CONF_TAKE4|NGX_CONF_TAKE5,
    ps_srv_configure,
    NGX_HTTP_SRV_CONF_OFFSET,
@@ -451,7 +515,7 @@ ngx_command_t ps_commands[] = {
    NULL },

  { ngx_string("pagespeed"),
-    NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1|
+    NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF|NGX_CONF_TAKE1|
    NGX_CONF_TAKE2|NGX_CONF_TAKE3|NGX_CONF_TAKE4|NGX_CONF_TAKE5,
    ps_loc_configure,
    NGX_HTTP_SRV_CONF_OFFSET,
@@ -470,80 +534,6 @@ void ps_ignore_sigpipe() {
  sigaction(SIGPIPE, &act, NULL);
 }

-namespace PsConfigure {
-enum OptionLevel {
-  kServer,
-  kLocation,
-};
-}  // namespace PsConfigure
-
-// These options are copied from mod_instaweb.cc, where
-// APACHE_CONFIG_OPTIONX indicates that they can not be set at the
-// directory/location level. They are not alphabetized on purpose,
-// but rather left in the same order as in mod_instaweb.cc in case
-// we end up needing te compare.
-// TODO(oschaaf): this duplication is a short term solution.
-const char* const global_only_options[] = {
-  "BlockingRewriteKey",
-  "CacheFlushFilename",
-  "CacheFlushPollIntervalSec",
-  "DangerPermitFetchFromUnknownHosts",
-  "CriticalImagesBeaconEnabled",
-  "ExperimentalFetchFromModSpdy",
-  "FetcherTimeoutMs",
-  "FetchHttps",
-  "FetchWithGzip",
-  "FileCacheCleanIntervalMs",
-  "FileCacheInodeLimit",
-  "FileCachePath",
-  "FileCacheSizeKb",
-  "ForceCaching",
-  "ImageMaxRewritesAtOnce",
-  "ImgMaxRewritesAtOnce",
-  "InheritVHostConfig",
-  "InstallCrashHandler",
-  "LRUCacheByteLimit",
-  "LRUCacheKbPerProcess",
-  "MaxCacheableContentLength",
-  "MemcachedServers",
-  "MemcachedThreads",
-  "MemcachedTimeoutUs",
-  "MessageBufferSize",
-  "NumRewriteThreads",
-  "NumExpensiveRewriteThreads",
-  "RateLimitBackgroundFetches",
-  "ReportUnloadTime",
-  "RespectXForwardedProto",
-  "SharedMemoryLocks",
-  "SlurpDirectory",
-  "SlurpFlushLimit",
-  "SlurpReadOnly",
-  "SupportNoScriptEnabled",
-  "StatisticsLoggingChartsCSS",
-  "StatisticsLoggingChartsJS",
-  "TestProxy",
-  "TestProxySlurp",
-  "TrackOriginalContentLength",
-  "UsePerVHostStatistics",
-  "XHeaderValue",
-  "LoadFromFile",
-  "LoadFromFileMatch",
-  "LoadFromFileRule",
-  "LoadFromFileRuleMatch",
-  "UseNativeFetcher"
-};
-
-bool ps_is_global_only_option(const StringPiece& option_name) {
-  ngx_uint_t i;
-  ngx_uint_t size = sizeof(global_only_options) / sizeof(char*);
-  for (i = 0; i < size; i++) {
-    if (StringCaseEqual(global_only_options[i], option_name)) {
-      return true;
-    }
-  }
-  return false;
-}
-
 char* ps_init_dir(const StringPiece& directive,
                  const StringPiece& path,
                  ngx_conf_t* cf) {
@@ -571,15 +561,24 @@ char* ps_init_dir(const StringPiece& directive,
    return NULL;  // We're not root, so we're staying whoever we are.
  }

-  ngx_core_conf_t* ccf =
-      (ngx_core_conf_t*)(ngx_get_conf(cf->cycle->conf_ctx, ngx_core_module));
+  // chown if owner differs from nginx worker user.
+  ngx_core_conf_t* ccf = reinterpret_cast<ngx_core_conf_t*>(
+      ngx_get_conf(cf->cycle->conf_ctx, ngx_core_module));
  CHECK(ccf != NULL);
-
-  if (chown(gs_path.c_str(), ccf->user, ccf->group) != 0) {
+  struct stat gs_stat;
+  if (stat(gs_path.c_str(), &gs_stat) != 0) {
    return string_piece_to_pool_string(
        cf->pool, net_instaweb::StrCat(
-            directive, " ", path, " unable to set permissions"));
+            directive, " ", path, " stat() failed"));
  }
+  if (gs_stat.st_uid != ccf->user) {
+    if (chown(gs_path.c_str(), ccf->user, ccf->group) != 0) {
+      return string_piece_to_pool_string(
+          cf->pool, net_instaweb::StrCat(
+              directive, " ", path, " unable to set permissions"));
+    }
+  }
+
  return NULL;
 }

@@ -587,7 +586,7 @@ char* ps_init_dir(const StringPiece& directive,
 char* ps_configure(ngx_conf_t* cf,
                   NgxRewriteOptions** options,
                   MessageHandler* handler,
-                   PsConfigure::OptionLevel option_level) {
+                   net_instaweb::RewriteOptions::OptionScope option_scope) {
  // args[0] is always "pagespeed"; ignore it.
  ngx_uint_t n_args = cf->args->nelts - 1;

@@ -602,19 +601,6 @@ char* ps_configure(ngx_conf_t* cf,
    args[i] = str_to_string_piece(value[i+1]);
  }

-  if (StringCaseEqual("UseNativeFetcher", args[0])) {
-    if (option_level != PsConfigure::kServer) {
-      return const_cast<char*>(
-          "UseNativeFetcher can only be set in the http{} block.");
-    }
-  }
-  if (option_level == PsConfigure::kLocation && n_args > 1) {
-    if (ps_is_global_only_option(args[0])) {
-      return string_piece_to_pool_string(cf->pool, StrCat(
-          "\"", args[0], "\" cannot be set at location scope"));
-    }
-  }
-
  // Some options require the worker process to be able to read and write to
  // a specific directory.  Generally the master process is root while the
  // worker is nobody, so we need to change permissions and create the directory
@@ -637,25 +623,33 @@ char* ps_configure(ngx_conf_t* cf,
        cfg_m->driver_factory->thread_system());
  }
  const char* status = (*options)->ParseAndSetOptions(
-      args, n_args, cf->pool, handler, cfg_m->driver_factory);
+      args, n_args, cf->pool, handler, cfg_m->driver_factory, option_scope);

  // nginx expects us to return a string literal but doesn't mark it const.
  return const_cast<char*>(status);
 }

+char* ps_main_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+  ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
+      ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
+  return ps_configure(cf, &cfg_s->options, cfg_s->handler,
+                      net_instaweb::RewriteOptions::kProcessScopeStrict);
+}
+
 char* ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
  ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
      ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
  return ps_configure(cf, &cfg_s->options, cfg_s->handler,
-                      PsConfigure::kServer);
+                      net_instaweb::RewriteOptions::kServerScope);
 }

 char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
  ps_loc_conf_t* cfg_l = static_cast<ps_loc_conf_t*>(
      ngx_http_conf_get_module_loc_conf(cf, ngx_pagespeed));

-  return ps_configure(cf, &cfg_l->options, cfg_l->handler,
-                      PsConfigure::kLocation);
+  return ps_configure(
+      cf, &cfg_l->options, cfg_l->handler,
+      net_instaweb::RewriteOptions::kDirectoryScope);
 }

 void ps_cleanup_loc_conf(void* data) {
@@ -742,6 +736,7 @@ void* ps_create_main_conf(ngx_conf_t* cf) {
  NgxRewriteDriverFactory::Initialize();

  cfg_m->driver_factory = new NgxRewriteDriverFactory(
+      *process_context,
      new SystemThreadSystem(),
      "" /* hostname, not used */,
      -1 /* port, not used */);
@@ -856,19 +851,28 @@ char* ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child) {
 }

 char* ps_merge_loc_conf(ngx_conf_t* cf, void* parent, void* child) {
-  ps_loc_conf_t* parent_cfg_l = static_cast<ps_loc_conf_t*>(parent);
-
-  // The variant of the pagespeed directive that is acceptable in location
-  // blocks is only acceptable in location blocks, so we should never be merging
-  // in options from a server or main block.
-  CHECK(parent_cfg_l->options == NULL);
-
  ps_loc_conf_t* cfg_l = static_cast<ps_loc_conf_t*>(child);
  if (cfg_l->options == NULL) {
    // No directory specific options.
    return NGX_CONF_OK;
  }

+  // While you can't put a "location" block inside a "location" block you can
+  // put an "if" block inside a "location" block, which is implemented by making
+  // a pretend "location" block.  In this case we may have pagespeed options
+  // from the parent "location" block as well as from the current locationish
+  // "if" block.
+  ps_loc_conf_t* parent_cfg_l = static_cast<ps_loc_conf_t*>(parent);
+  if (parent_cfg_l->options != NULL) {
+    // Rebase our options off of the ones defined in the parent location block.
+    ps_merge_options(parent_cfg_l->options, &cfg_l->options);
+    return NGX_CONF_OK;
+  }
+
+  // Pagespeed options are defined in this location block, and it either has no
+  // parent (typical case) or is an if block whose parent location block defines
+  // no pagespeed options.  Base our options off of those in the server block.
+
  ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
      ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));

@@ -967,18 +971,9 @@ int ps_determine_port(ngx_http_request_t* r) {

  return port;
 }
+}  // namespace

-GoogleString ps_determine_url(ngx_http_request_t* r) {
-  int port = ps_determine_port(r);
-  GoogleString port_string;
-  if ((ps_is_https(r) && (port == 443 || port == -1)) ||
-      (!ps_is_https(r) && (port == 80 || port == -1))) {
-    // No port specifier needed for requests on default ports.
-    port_string = "";
-  } else {
-    port_string = StrCat(":", IntegerToString(port));
-  }
-
+StringPiece ps_determine_host(ngx_http_request_t* r) {
  StringPiece host = str_to_string_piece(r->headers_in.server);
  if (host.size() == 0) {
    // If host is unspecified, perhaps because of a pure HTTP 1.0 "GET /path",
@@ -993,6 +988,23 @@ GoogleString ps_determine_url(ngx_http_request_t* r) {
    }
    host = str_to_string_piece(s);
  }
+  return host;
+}
+
+namespace {
+
+GoogleString ps_determine_url(ngx_http_request_t* r) {
+  int port = ps_determine_port(r);
+  GoogleString port_string;
+  if ((ps_is_https(r) && (port == 443 || port == -1)) ||
+      (!ps_is_https(r) && (port == 80 || port == -1))) {
+    // No port specifier needed for requests on default ports.
+    port_string = "";
+  } else {
+    port_string = StrCat(":", IntegerToString(port));
+  }
+
+  StringPiece host = ps_determine_host(r);

  return StrCat(ps_is_https(r) ? "https://" : "http://",
                host, port_string, str_to_string_piece(r->unparsed_uri));
@@ -1273,11 +1285,9 @@ RewriteOptions* ps_determine_request_options(
  // make cache key consistent for both lookup and storing in cache.
  //
  // Sets option from request headers and url.
-  ServerContext::OptionsBoolPair query_options_success =
-      cfg_s->server_context->GetQueryOptions(url, request_headers,
-                                             response_headers);
-  bool get_query_options_success = query_options_success.second;
-  if (!get_query_options_success) {
+  RewriteQuery rewrite_query;
+  if (!cfg_s->server_context->GetQueryOptions(
+          url, request_headers, response_headers, &rewrite_query)) {
    // Failed to parse query params or request headers.  Treat this as if there
    // were no query params given.
    ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
@@ -1287,7 +1297,7 @@ RewriteOptions* ps_determine_request_options(

  // Will be NULL if there aren't any options set with query params or in
  // headers.
-  return query_options_success.first;
+  return rewrite_query.ReleaseOptions();
 }

 // Check whether this visitor is already in an experiment.  If they're not,
@@ -1366,11 +1376,12 @@ bool ps_determine_options(ngx_http_request_t* r,
  // rebased on the directory options or the global options.
  RewriteOptions* request_options = ps_determine_request_options(
      r, request_headers, response_headers, cfg_s, url);
+  bool have_request_options = request_options != NULL;

  // Because the caller takes ownership of any options we return, the only
  // situation in which we can avoid allocating a new RewriteOptions is if the
  // global options are ok as are.
-  if (directory_options == NULL && request_options == NULL &&
+  if (!have_request_options && directory_options == NULL &&
      !global_options->running_experiment()) {
    return true;
  }
@@ -1382,14 +1393,24 @@ bool ps_determine_options(ngx_http_request_t* r,
    *options = global_options->Clone();
  }

-  // Modify our options in response to request options or experiment settings,
-  // if we need to.  If there are request options then ignore the experiment
-  // because we don't want experiments to be contaminated with unexpected
-  // settings.
-  if (request_options != NULL) {
+  // Modify our options in response to request options if specified.
+  if (have_request_options) {
    (*options)->Merge(*request_options);
    delete request_options;
-  } else if ((*options)->running_experiment() && html_rewrite) {
+    request_options = NULL;
+  }
+
+  // If we're running an experiment and processing html then modify our options
+  // in response to the experiment.  Except we generally don't want experiments
+  // to be contaminated with unexpected settings, so ignore experiments if we
+  // have request-specific options.  Unless EnrollExperiment is on, probably set
+  // by a query parameter, in which case we want to go ahead and apply the
+  // experimental settings even if it means bad data, because we're just seeing
+  // what it looks like.
+  if ((*options)->running_experiment() &&
+      html_rewrite &&
+      (!have_request_options ||
+       (*options)->enroll_experiment())) {
    bool ok = ps_set_experiment_state_and_cookie(
        r, request_headers, *options, url->Host());
    if (!ok) {
@@ -1426,7 +1447,8 @@ bool ps_apply_x_forwarded_proto(ngx_http_request_t* r, GoogleString* url) {
    return false;  // No X-Forwarded-Proto header found.
  }

-  StringPiece x_forwarded_proto = str_to_string_piece(*x_forwarded_proto_header);
+  StringPiece x_forwarded_proto =
+      str_to_string_piece(*x_forwarded_proto_header);
  if (!STR_CASE_EQ_LITERAL(*x_forwarded_proto_header, "http") &&
      !STR_CASE_EQ_LITERAL(*x_forwarded_proto_header, "https")) {
    LOG(WARNING) << "Unsupported X-Forwarded-Proto: " << x_forwarded_proto
@@ -1552,11 +1574,6 @@ void ps_release_request_context(void* data) {
    ctx->recorder = NULL;
  }

-  if (ctx->ipro_response_headers != NULL) {
-    delete ctx->ipro_response_headers;
-    ctx->ipro_response_headers = NULL;
-  }
-
  ps_release_base_fetch(ctx);
  delete ctx;
 }
@@ -1589,18 +1606,33 @@ RequestRouting::Response ps_route_request(ngx_http_request_t* r,

  if (is_pagespeed_subrequest(r)) {
    return RequestRouting::kPagespeedSubrequest;
-  } else if (url.PathSansLeaf() == NgxRewriteDriverFactory::kStaticAssetPrefix) {
+  } else if (
+      url.PathSansLeaf() == dynamic_cast<NgxRewriteDriverFactory*>(
+          cfg_s->server_context->factory())->static_asset_prefix()) {
    return RequestRouting::kStaticContent;
-  } else if (url.PathSansQuery() == "/ngx_pagespeed_statistics" ||
-             url.PathSansQuery() == "/ngx_pagespeed_global_statistics" ) {
-    return RequestRouting::kStatistics;
-  } else if (url.PathSansQuery() == "/pagespeed_console") {
-    return RequestRouting::kConsole;
-  } else if (url.PathSansQuery() == "/ngx_pagespeed_message") {
-    return RequestRouting::kMessages;
  }

-  RewriteOptions* global_options = cfg_s->server_context->global_options();
+  const NgxRewriteOptions* global_options = cfg_s->server_context->config();
+
+  StringPiece path = url.PathSansQuery();
+  if (StringCaseEqual(path, global_options->statistics_path())) {
+    return RequestRouting::kStatistics;
+  } else if (StringCaseEqual(path, global_options->global_statistics_path())) {
+    return RequestRouting::kGlobalStatistics;
+  } else if (StringCaseEqual(path, global_options->console_path())) {
+    return RequestRouting::kConsole;
+  } else if (StringCaseEqual(path, global_options->messages_path())) {
+    return RequestRouting::kMessages;
+  } else if (// The admin handlers get everything under a path (/path/*) while
+             // all the other handlers only get exact matches (/path).  So match
+             // all paths starting with the handler path.
+             !global_options->admin_path().empty() &&
+             StringCaseStartsWith(path, global_options->admin_path())) {
+    return RequestRouting::kAdmin;
+  } else if (!global_options->global_admin_path().empty() &&
+             StringCaseStartsWith(path, global_options->global_admin_path())) {
+    return RequestRouting::kGlobalAdmin;
+  }

  const GoogleString* beacon_url;
  if (ps_is_https(r)) {
@@ -1616,7 +1648,9 @@ RequestRouting::Response ps_route_request(ngx_http_request_t* r,
  return RequestRouting::kResource;
 }

-ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
+ngx_int_t ps_resource_handler(ngx_http_request_t* r,
+                              bool html_rewrite,
+                              RequestRouting::Response response_category) {
  if (r != r->main) {
    return NGX_DECLINED;
  }
@@ -1626,7 +1660,9 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {

  CHECK(!(html_rewrite && (ctx == NULL || ctx->html_rewrite == false)));

-  if (!html_rewrite && r->method != NGX_HTTP_GET && r->method != NGX_HTTP_HEAD) {
+  if (!html_rewrite &&
+      r->method != NGX_HTTP_GET &&
+      r->method != NGX_HTTP_HEAD) {
    return NGX_DECLINED;
  }

@@ -1673,6 +1709,15 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
    }
  }

+  bool pagespeed_resource =
+      !html_rewrite && cfg_s->server_context->IsPagespeedResource(url);
+  bool is_an_admin_handler =
+      response_category == RequestRouting::kStatistics ||
+      response_category == RequestRouting::kGlobalStatistics ||
+      response_category == RequestRouting::kConsole ||
+      response_category == RequestRouting::kAdmin ||
+      response_category == RequestRouting::kGlobalAdmin;
+
  if (html_rewrite) {
    ps_release_base_fetch(ctx);
  } else {
@@ -1681,11 +1726,12 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
    ctx = new ps_request_ctx_t();

    ctx->r = r;
-    ctx->ipro_response_headers = NULL;
    ctx->write_pending = false;
    ctx->html_rewrite = false;
    ctx->in_place = false;
    ctx->pagespeed_connection = NULL;
+    ctx->preserve_caching_headers = kDontPreserveHeaders;
+
    // See build_context_for_request() in mod_instaweb.cc
    // TODO(jefftk): Is this the right place to be modifying caching headers for
    // html fetches?  Or should that be done later, in the headers flow for
@@ -1696,7 +1742,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
      // Downstream cache integration is not enabled. Disable original
      // Cache-Control headers.
      ctx->preserve_caching_headers = kDontPreserveHeaders;
-    } else {
+    } else if (!pagespeed_resource && !is_an_admin_handler) {
      ctx->preserve_caching_headers = kPreserveOnlyCacheControl;
      // Downstream cache integration is enabled. If a rebeaconing key has been
      // configured and there is a ShouldBeacon header with the correct key,
@@ -1707,7 +1753,9 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
        ctx->preserve_caching_headers = kDontPreserveHeaders;
      }
    }
+
    ctx->recorder = NULL;
+    ctx->url_string = url_string;

    // Set up a cleanup handler on the request.
    ngx_http_cleanup_t* cleanup = ngx_http_cleanup_add(r, 0);
@@ -1740,7 +1788,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
              false /* requires_blink_cohort (no longer unused) */,
              &page_callback_added));

-  if (!html_rewrite && cfg_s->server_context->IsPagespeedResource(url)) {
+  if (pagespeed_resource) {
    // TODO(jefftk): Set using_spdy appropriately.  See
    // ProxyInterface::ProxyRequestCallback
    ResourceFetch::Start(
@@ -1748,6 +1796,42 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r, bool html_rewrite) {
        custom_options.release() /* null if there aren't custom options */,
        false /* using_spdy */, cfg_s->server_context, ctx->base_fetch);
    return ps_async_wait_response(r);
+  } else if (is_an_admin_handler) {
+    QueryParams query_params;
+    query_params.Parse(url.Query());
+
+    PosixTimer timer;
+    int64 now_ms = timer.NowMs();
+    ctx->base_fetch->response_headers()->SetDateAndCaching(
+        now_ms, 0 /* max-age */, ", no-cache");
+
+    if (response_category == RequestRouting::kStatistics ||
+        response_category == RequestRouting::kGlobalStatistics) {
+      cfg_s->server_context->StatisticsPage(
+          response_category == RequestRouting::kGlobalStatistics,
+          query_params,
+          cfg_s->server_context->config(),
+          ctx->base_fetch);
+    } else if (response_category == RequestRouting::kConsole) {
+      cfg_s->server_context->ConsoleHandler(
+          *cfg_s->server_context->config(),
+          SystemServerContext::kStatistics,
+          query_params,
+          ctx->base_fetch);
+    } else if (response_category == RequestRouting::kAdmin ||
+               response_category == RequestRouting::kGlobalAdmin) {
+      cfg_s->server_context->AdminPage(
+          response_category == RequestRouting::kGlobalAdmin,
+          url,
+          query_params,
+          custom_options == NULL ? cfg_s->server_context->config()
+                                 : custom_options.get(),
+          ctx->base_fetch);
+    } else {
+      CHECK(false);
+    }
+
+    return ps_async_wait_response(r);
  }

  if (html_rewrite) {
@@ -1893,45 +1977,6 @@ void ps_send_to_pagespeed(ngx_http_request_t* r,
  }
 #endif

-// Based on ngx_http_add_cache_control.
-ngx_int_t ps_set_cache_control(ngx_http_request_t* r, char* cache_control) {
-  // First strip existing cache-control headers.
-  ngx_table_elt_t* header;
-  NgxListIterator it(&(r->headers_out.headers.part));
-  while ((header = it.Next()) != NULL) {
-    if (STR_CASE_EQ_LITERAL(header->key, "Cache-Control")) {
-      // Response headers with hash of 0 are excluded from the response.
-      header->hash = 0;
-    }
-  }
-
-  // Now add our new cache control header.
-  if (r->headers_out.cache_control.elts == NULL) {
-    ngx_int_t rc = ngx_array_init(&r->headers_out.cache_control, r->pool,
-                                  1, sizeof(ngx_table_elt_t *));
-    if (rc != NGX_OK) {
-      return NGX_ERROR;
-    }
-  }
-  ngx_table_elt_t** cache_control_headers = static_cast<ngx_table_elt_t**>(
-      ngx_array_push(&r->headers_out.cache_control));
-  if (cache_control_headers == NULL) {
-    return NGX_ERROR;
-  }
-  cache_control_headers[0] = static_cast<ngx_table_elt_t*>(
-      ngx_list_push(&r->headers_out.headers));
-  if (cache_control_headers[0] == NULL) {
-    return NGX_ERROR;
-  }
-  cache_control_headers[0]->hash = 1;
-  ngx_str_set(&cache_control_headers[0]->key, "Cache-Control");
-  cache_control_headers[0]->value.len = strlen(cache_control);
-  cache_control_headers[0]->value.data =
-      reinterpret_cast<u_char*>(cache_control);
-
-  return NGX_OK;
-}
-
 void ps_strip_html_headers(ngx_http_request_t* r) {
  // We're modifying content, so switch to 'Transfer-Encoding: chunked' and
  // calculate on the fly.
@@ -2054,7 +2099,8 @@ ngx_int_t ps_html_rewrite_header_filter(ngx_http_request_t* r) {
    return ngx_http_next_header_filter(r);
  }

-  ngx_int_t rc = ps_resource_handler(r, true /* html rewrite */);
+  ngx_int_t rc = ps_resource_handler(r, true /* html rewrite */,
+                                     RequestRouting::kResource);
  if (rc != NGX_OK) {
    ctx->html_rewrite = false;
    return ngx_http_next_header_filter(r);
@@ -2155,12 +2201,40 @@ ngx_http_output_body_filter_pt ngx_http_next_body_filter;
 ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
  ps_request_ctx_t* ctx = ps_get_request_context(r);

-  if (ctx == NULL || !ctx->in_place) {
+  if (ctx == NULL) {
+    return ngx_http_next_header_filter(r);
+  }
+
+  if (ctx->recorder != NULL) {
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "ps in place check header filter recording: %V", &r->uri);
+
+    CHECK(!ctx->in_place);
+
+    // We didn't find this resource in cache originally, so we're recording it
+    // as it passes us by.  At this point the headers from things that run
+    // before us are set but not things that run after us, which means here is
+    // where we need to check whether there's a "Content-Encoding: gzip".  If we
+    // waited to do this in ps_in_place_body_filter we wouldn't be able to tell
+    // the difference between response headers that have "C-E: gz" because we're
+    // proxying for an upstream that gzipped the content and response headers
+    // that have it because the gzip filter (which runs after us) is going to
+    // produce gzipped output.
+    //
+    // The recorder will do this checking, so pass it the headers.
+    ResponseHeaders response_headers;
+    copy_response_headers_from_ngx(r, &response_headers);
+    ctx->recorder->ConsiderResponseHeaders(
+        InPlaceResourceRecorder::kPreliminaryHeaders, &response_headers);
+    return ngx_http_next_header_filter(r);
+  }
+
+  if (!ctx->in_place) {
    return ngx_http_next_header_filter(r);
  }

  ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                 "ps in place check header filter: %V", &r->uri);
+                 "ps in place check header filter initial: %V", &r->uri);

  int status_code = r->headers_out.status;
  bool status_ok = (status_code != 0) && (status_code < 400);
@@ -2169,6 +2243,9 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
  NgxServerContext* server_context = cfg_s->server_context;
  MessageHandler* message_handler = cfg_s->handler;
  GoogleString url = ps_determine_url(r);
+  // The URL we use for cache key is a bit different since it may
+  // have PageSpeed query params removed.
+  GoogleString cache_url = ctx->url_string;

  // continue process
  if (status_ok) {
@@ -2189,26 +2266,32 @@ ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
        kInfo,
        "Could not rewrite resource in-place "
        "because URL is not in cache: %s",
-        url.c_str());
+        cache_url.c_str());
    const SystemRewriteOptions* options = SystemRewriteOptions::DynamicCast(
        ctx->driver->options());
-    scoped_ptr<RequestHeaders> request_headers(new RequestHeaders);
-    copy_request_headers_from_ngx(r, request_headers.get());
+    RequestHeaders request_headers;
+    copy_request_headers_from_ngx(r, &request_headers);
    // This URL was not found in cache (neither the input resource nor
    // a ResourceNotCacheable entry) so we need to get it into cache
    // (or at least a note that it cannot be cached stored there).
    // We do that using an Apache output filter.
    ctx->recorder = new InPlaceResourceRecorder(
-        url,
-        request_headers.release(),
+        RequestContextPtr(cfg_s->server_context->NewRequestContext(r)),
+        cache_url,
+        ctx->driver->CacheFragment(),
+        request_headers.GetProperties(),
        options->respect_vary(),
        options->ipro_max_response_bytes(),
        options->ipro_max_concurrent_recordings(),
+        options->implicit_cache_ttl_ms(),
        server_context->http_cache(),
        server_context->statistics(),
        message_handler);
    // set in memory flag for in place_body_filter
    r->filter_need_in_memory = 1;
+
+    // We don't have the response headers at all yet because we haven't yet gone
+    // to the backend.
  } else {
    server_context->rewrite_stats()->ipro_not_rewritable()->Add(1);
    message_handler->Message(kInfo,
@@ -2234,42 +2317,11 @@ ngx_int_t ps_in_place_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
                 "ps in place body filter: %V", &r->uri);

  InPlaceResourceRecorder* recorder = ctx->recorder;
-
-  if (ctx->ipro_response_headers == NULL) {
-    // Prepare response headers.
-    ctx->ipro_response_headers = new ResponseHeaders();
-
-    // TODO(oschaaf): We don't get a Date response header here.
-    // Currently, we invent one and set it to the current date/time.
-    // We need to investigate why we don't receive it.
-    ctx->ipro_response_headers->set_major_version(r->http_version / 1000);
-    ctx->ipro_response_headers->set_minor_version(r->http_version % 1000);
-    copy_headers_from_table(r->headers_out.headers, ctx->ipro_response_headers);
-    ctx->ipro_response_headers->set_status_code(r->headers_out.status);
-    ctx->ipro_response_headers->Add(HttpAttributes::kContentType,
-                str_to_string_piece(r->headers_out.content_type));
-    if (r->headers_out.location != NULL) {
-      ctx->ipro_response_headers->Add(HttpAttributes::kLocation,
-                  str_to_string_piece(r->headers_out.location->value));
-    }
-    StringPiece date =
-        ctx->ipro_response_headers->Lookup1(HttpAttributes::kDate);
-    if (date.empty()) {
-      ctx->ipro_response_headers->SetDate(ngx_current_msec);
-    }
-    ctx->ipro_response_headers->ComputeCaching();
-
-    // Unlike in Apache we get the final response headers before we get the
-    // content.  This means we can consider them earlier and abort the
-    // request if need be without buffering everything.
-    recorder->ConsiderResponseHeaders(ctx->ipro_response_headers);
-  }
-
  for (ngx_chain_t* cl = in; cl; cl = cl->next) {
    if (ngx_buf_size(cl->buf)) {
       CHECK(ngx_buf_in_memory(cl->buf));
-       StringPiece contents(reinterpret_cast<char *>(cl->buf->pos),
-                                 ngx_buf_size(cl->buf));
+       StringPiece contents(reinterpret_cast<char*>(cl->buf->pos),
+                            ngx_buf_size(cl->buf));
       recorder->Write(contents, recorder->handler());
    }

@@ -2278,7 +2330,9 @@ ngx_int_t ps_in_place_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
    }

    if (cl->buf->last_buf || recorder->failed()) {
-      ctx->recorder->DoneAndSetHeaders(ctx->ipro_response_headers);
+      ResponseHeaders response_headers;
+      copy_response_headers_from_ngx(r, &response_headers);
+      ctx->recorder->DoneAndSetHeaders(&response_headers);
      ctx->recorder = NULL;
      break;
    }
@@ -2337,6 +2391,13 @@ ngx_int_t ps_simple_handler(ngx_http_request_t* r,
  NgxMessageHandler* message_handler = factory->ngx_message_handler();
  StringPiece request_uri_path = str_to_string_piece(r->uri);

+  GoogleString url_string = ps_determine_url(r);
+  GoogleUrl url(url_string);
+  QueryParams query_params;
+  if (url.IsWebValid()) {
+    query_params.Parse(url.Query());
+  }
+
  GoogleString output;
  StringWriter writer(&output);
  HttpStatus::Code status = HttpStatus::kOK;
@@ -2348,30 +2409,13 @@ ngx_int_t ps_simple_handler(ngx_http_request_t* r,
    case RequestRouting::kStaticContent: {
      StringPiece file_contents;
      if (!server_context->static_asset_manager()->GetAsset(
-              request_uri_path.substr(
-                  strlen(NgxRewriteDriverFactory::kStaticAssetPrefix)),
+              request_uri_path.substr(factory->static_asset_prefix().length()),
              &file_contents, &content_type, &cache_control)) {
        return NGX_DECLINED;
      }
      file_contents.CopyToString(&output);
      break;
    }
-    case RequestRouting::kStatistics:
-      error_message = StatisticsHandler(
-          factory,
-          server_context,
-          NULL,  // No SPDY-specific config in ngx_pagespeed.
-          !factory->use_per_vhost_statistics() || StringCaseStartsWith(
-              request_uri_path, "/ngx_pagespeed_global_statistics"),
-          StringPiece(reinterpret_cast<char*>(r->args.data), r->args.len),
-          &content_type,
-          &writer,
-          message_handler);
-      break;
-    case RequestRouting::kConsole:
-      ConsoleHandler(
-          server_context, server_context->config(), &writer, message_handler);
-      break;
    case RequestRouting::kMessages: {
      GoogleString log;
      StringWriter log_writer(&log);
@@ -2611,12 +2655,16 @@ ngx_int_t ps_content_handler(ngx_http_request_t* r) {
    case RequestRouting::kBeacon:
      return ps_beacon_handler(r);
    case RequestRouting::kStaticContent:
-    case RequestRouting::kStatistics:
-    case RequestRouting::kConsole:
    case RequestRouting::kMessages:
      return ps_simple_handler(r, cfg_s->server_context, response_category);
+    case RequestRouting::kStatistics:
+    case RequestRouting::kGlobalStatistics:
+    case RequestRouting::kConsole:
+    case RequestRouting::kAdmin:
+    case RequestRouting::kGlobalAdmin:
    case RequestRouting::kResource:
-      return ps_resource_handler(r, false /* html rewrite */);
+      return ps_resource_handler(
+          r, false /* html rewrite */, response_category);
  }

  CHECK(0);
@@ -2866,6 +2914,8 @@ ngx_int_t ps_init_child_process(ngx_cycle_t* cycle) {
    return NGX_OK;
  }

+  SystemRewriteDriverFactory::InitApr();
+
  // ChildInit() will initialise all ServerContexts, which we need to
  // create ProxyFetchFactories below
  cfg_m->driver_factory->LoggingInit(cycle->log);
@@ -103,6 +103,10 @@ typedef struct {
  RewriteDriver* driver;
  InPlaceResourceRecorder* recorder;
  ResponseHeaders* ipro_response_headers;
+
+  // We need to remember the URL here as well since we may modify what NGX
+  // gets by stripping our special query params and honoring X-Forwarded-Proto.
+  GoogleString url_string;
 } ps_request_ctx_t;


@@ -117,6 +121,8 @@ ngx_int_t copy_response_headers_to_ngx(
    const ResponseHeaders& pagespeed_headers,
    PreserveCachingHeaders preserve_caching_headers);

+StringPiece ps_determine_host(ngx_http_request_t* r);
+
 }  // namespace net_instaweb

 #endif  // NGX_PAGESPEED_H_
@@ -33,7 +33,6 @@
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
 #include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
 #include "net/instaweb/rewriter/public/server_context.h"
-#include "net/instaweb/rewriter/public/static_asset_manager.h"
 #include "net/instaweb/system/public/in_place_resource_recorder.h"
 #include "net/instaweb/system/public/serf_url_async_fetcher.h"
 #include "net/instaweb/system/public/system_caches.h"
@@ -63,14 +62,12 @@ class UrlAsyncFetcher;
 class UrlFetcher;
 class Writer;

-const char NgxRewriteDriverFactory::kStaticAssetPrefix[] =
-    "/ngx_pagespeed_static/";
-
 class SharedCircularBuffer;

 NgxRewriteDriverFactory::NgxRewriteDriverFactory(
+    const ProcessContext& process_context,
    SystemThreadSystem* system_thread_system, StringPiece hostname, int port)
-    : SystemRewriteDriverFactory(system_thread_system,
+    : SystemRewriteDriverFactory(process_context, system_thread_system,
        NULL /* default shared memory runtime */, hostname, port),
      main_conf_(NULL),
      threads_started_(false),
@@ -82,7 +79,9 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
      log_(NULL),
      resolver_timeout_(NGX_CONF_UNSET_MSEC),
      use_native_fetcher_(false),
-      ngx_shared_circular_buffer_(NULL) {
+      ngx_shared_circular_buffer_(NULL),
+      hostname_(hostname.as_string()),
+      port_(port) {
  InitializeDefaultOptions();
  default_options()->set_beacon_url("/ngx_pagespeed_beacon");
  SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
@@ -148,11 +147,6 @@ RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
  return options;
 }

-void NgxRewriteDriverFactory::InitStaticAssetManager(
-    StaticAssetManager* static_asset_manager) {
-  static_asset_manager->set_library_url_prefix(kStaticAssetPrefix);
-}
-
 bool NgxRewriteDriverFactory::InitNgxUrlAsyncFetchers() {
  log_ = ngx_cycle->log;
  for (size_t i = 0; i < ngx_url_async_fetchers_.size(); ++i) {
@@ -177,6 +171,12 @@ NgxServerContext* NgxRewriteDriverFactory::MakeNgxServerContext(
  return server_context;
 }

+ServerContext* NgxRewriteDriverFactory::NewDecodingServerContext() {
+  ServerContext* sc = new NgxServerContext(this, hostname_, port_);
+  InitStubDecodingServerContext(sc);
+  return sc;
+}
+
 ServerContext* NgxRewriteDriverFactory::NewServerContext() {
  LOG(DFATAL) << "MakeNgxServerContext should be used instead";
  return NULL;
@@ -28,7 +28,6 @@ extern "C" {

 #include <set>

-#include "apr_pools.h"
 #include "net/instaweb/system/public/system_rewrite_driver_factory.h"
 #include "net/instaweb/util/public/md5_hasher.h"
 #include "net/instaweb/util/public/scoped_ptr.h"
@@ -46,16 +45,14 @@ class NgxUrlAsyncFetcher;
 class SharedCircularBuffer;
 class SharedMemRefererStatistics;
 class SlowWorker;
-class StaticAssetManager;
 class Statistics;
 class SystemThreadSystem;

 class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
 public:
-  static const char kStaticAssetPrefix[];
-
  // We take ownership of the thread system.
  explicit NgxRewriteDriverFactory(
+      const ProcessContext& process_context,
      SystemThreadSystem* system_thread_system, StringPiece hostname, int port);
  virtual ~NgxRewriteDriverFactory();
  virtual Hasher* NewHasher();
@@ -68,9 +65,7 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  // Create a new RewriteOptions.  In this implementation it will be an
  // NgxRewriteOptions.
  virtual RewriteOptions* NewRewriteOptions();
-  // Initializes the StaticAssetManager.
-  virtual void InitStaticAssetManager(
-      StaticAssetManager* static_asset_manager);
+  virtual ServerContext* NewDecodingServerContext();
  bool InitNgxUrlAsyncFetchers();
  // Check resolver configured or not.
  bool CheckResolver();
@@ -80,7 +75,7 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  // platform-independent statistics.
  static void InitStats(Statistics* statistics);
  NgxServerContext* MakeNgxServerContext(StringPiece hostname, int port);
-  ServerContext* NewServerContext();
+  virtual ServerContext* NewServerContext();

  // Starts pagespeed threads if they've not been started already.  Must be
  // called after the caller has finished any forking it intends to do.
@@ -169,6 +164,9 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  // TODO(jefftk): merge the nginx and apache ways of doing this.
  SharedCircularBuffer* ngx_shared_circular_buffer_;

+  GoogleString hostname_;
+  int port_;
+
  DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
 };

@@ -37,7 +37,47 @@ namespace net_instaweb {

 namespace {

-const char kNgxPagespeedStatisticsHandlerPath[] = "/ngx_pagespeed_statistics";
+const char kStatisticsPath[] = "StatisticsPath";
+const char kGlobalStatisticsPath[] = "GlobalStatisticsPath";
+const char kConsolePath[] = "ConsolePath";
+const char kMessagesPath[] = "MessagesPath";
+const char kAdminPath[] = "AdminPath";
+const char kGlobalAdminPath[] = "GlobalAdminPath";
+
+// These options are copied from mod_instaweb.cc, where APACHE_CONFIG_OPTIONX
+// indicates that they can not be set at the directory/location level. They set
+// options in the RewriteDriverFactory, so they're entirely global and do not
+// appear in RewriteOptions.  They are not alphabetized on purpose, but rather
+// left in the same order as in mod_instaweb.cc in case we end up needing to
+// compare.
+// TODO(oschaaf): this duplication is a short term solution.
+const char* const server_only_options[] = {
+  "FetcherTimeoutMs",
+  "FetchProxy",
+  "ForceCaching",
+  "GeneratedFilePrefix",
+  "ImgMaxRewritesAtOnce",
+  "InheritVHostConfig",
+  "InstallCrashHandler",
+  "MessageBufferSize",
+  "NumRewriteThreads",
+  "NumExpensiveRewriteThreads",
+  "StaticAssetPrefix",
+  "TrackOriginalContentLength",
+  "UsePerVHostStatistics",  // TODO(anupama): What to do about "No longer used"
+  "BlockingRewriteRefererUrls",
+  "CreateSharedMemoryMetadataCache",
+  "LoadFromFile",
+  "LoadFromFileMatch",
+  "LoadFromFileRule",
+  "LoadFromFileRuleMatch",
+  "UseNativeFetcher"
+};
+
+// Options that can only be used in the main (http) option scope.
+const char* const main_only_options[] = {
+  "UseNativeFetcher"
+};

 }  // namespace

@@ -58,15 +98,29 @@ void NgxRewriteOptions::Init() {
  DCHECK(ngx_properties_ != NULL)
      << "Call NgxRewriteOptions::Initialize() before construction";
  InitializeOptions(ngx_properties_);
-
-  // Nginx-specific default.
-  // TODO(sligocki): Get rid of this line and let both Apache and Nginx use
-  // /pagespeed_statistics as the handler.
-  statistics_handler_path_.set_default(kNgxPagespeedStatisticsHandlerPath);
 }

 void NgxRewriteOptions::AddProperties() {
-  // Nothing ngx-specific for now.
+  // Nginx-specific options.
+  add_ngx_option(
+      "", &NgxRewriteOptions::statistics_path_, "nsp", kStatisticsPath,
+      kServerScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics");
+  add_ngx_option(
+      "", &NgxRewriteOptions::global_statistics_path_, "ngsp",
+      kGlobalStatisticsPath, kProcessScope,
+      "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics");
+  add_ngx_option(
+      "", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kServerScope,
+      "Set the console path. Ex: /pagespeed_console");
+  add_ngx_option(
+      "", &NgxRewriteOptions::messages_path_, "nmp", kMessagesPath,
+      kServerScope, "Set the messages path.  Ex: /ngx_pagespeed_message");
+  add_ngx_option(
+      "", &NgxRewriteOptions::admin_path_, "nap", kAdminPath,
+      kServerScope, "Set the admin path.  Ex: /pagespeed_admin");
+  add_ngx_option(
+      "", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
+      kProcessScope, "Set the global admin path.  Ex: /pagespeed_global_admin");

  MergeSubclassProperties(ngx_properties_);

@@ -94,6 +148,39 @@ bool NgxRewriteOptions::IsDirective(StringPiece config_directive,
  return StringCaseEqual(config_directive, compare_directive);
 }

+RewriteOptions::OptionScope NgxRewriteOptions::GetOptionScope(
+    StringPiece option_name) {
+  ngx_uint_t i;
+  ngx_uint_t size = sizeof(main_only_options) / sizeof(char*);
+  for (i = 0; i < size; i++) {
+    if (StringCaseEqual(main_only_options[i], option_name)) {
+      return kProcessScopeStrict;
+    }
+  }
+
+  size = sizeof(server_only_options) / sizeof(char*);
+  for (i = 0; i < size; i++) {
+    if (StringCaseEqual(server_only_options[i], option_name)) {
+      return kServerScope;
+    }
+  }
+
+  // This could be made more efficient if RewriteOptions provided a map allowing
+  // access of options by their name. It's not too much of a worry at present
+  // since this is just during initialization.
+  for (OptionBaseVector::const_iterator it = all_options().begin();
+       it != all_options().end(); ++it) {
+    RewriteOptions::OptionBase* option = *it;
+    if (option->option_name() == option_name) {
+      // We treat kProcessScope as kProcessScopeStrict, failing to start if an
+      // option is out of place.
+      return option->scope() == kProcessScope ? kProcessScopeStrict
+                                              : option->scope();
+    }
+  }
+  return kDirectoryScope;
+}
+
 RewriteOptions::OptionSettingResult NgxRewriteOptions::ParseAndSetOptions0(
    StringPiece directive, GoogleString* msg, MessageHandler* handler) {
  if (IsDirective(directive, "on")) {
@@ -144,10 +231,26 @@ RewriteOptions::OptionSettingResult ParseAndSetOptionHelper(
  return RewriteOptions::kOptionOk;
 }

+namespace {
+
+const char* ps_error_string_for_option(
+    ngx_pool_t* pool, StringPiece directive, StringPiece warning) {
+  GoogleString msg =
+      StrCat("\"", directive, "\" ", warning);
+  char* s = string_piece_to_pool_string(pool, msg);
+  if (s == NULL) {
+    return "failed to allocate memory";
+  }
+  return s;
+}
+
+}  // namespace
+
 // Very similar to apache/mod_instaweb::ParseDirective.
 const char* NgxRewriteOptions::ParseAndSetOptions(
    StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
-    NgxRewriteDriverFactory* driver_factory) {
+    NgxRewriteDriverFactory* driver_factory,
+    RewriteOptions::OptionScope scope) {
  CHECK_GE(n_args, 1);

  StringPiece directive = args[0];
@@ -158,6 +261,11 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
    directive.remove_prefix(mod_pagespeed.size());
  }

+  if (GetOptionScope(directive) > scope) {
+    return ps_error_string_for_option(
+        pool, directive, "cannot be set at this scope.");
+  }
+
  GoogleString msg;
  OptionSettingResult result;
  if (n_args == 1) {
@@ -203,8 +311,11 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
      result = ParseAndSetOptionHelper<SystemRewriteDriverFactory>(
          arg, driver_factory,
          &SystemRewriteDriverFactory::set_track_original_content_length);
+    } else if (IsDirective(directive, "StaticAssetPrefix")) {
+      driver_factory->set_static_asset_prefix(arg);
+      result = RewriteOptions::kOptionOk;
    } else {
-      result = ParseAndSetOptionFromName1(directive, args[1], &msg, handler);
+      result = ParseAndSetOptionFromName1(directive, arg, &msg, handler);
    }
  } else if (n_args == 3) {
    // Short-term special handling, until this moves to common code.
@@ -227,25 +338,22 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
    result = ParseAndSetOptionFromName3(
        directive, args[1], args[2], args[3], &msg, handler);
  } else {
-    return "unknown option";
+    return ps_error_string_for_option(
+        pool, directive, "not recognized or too many arguments");
  }

  switch (result) {
    case RewriteOptions::kOptionOk:
      return NGX_CONF_OK;
    case RewriteOptions::kOptionNameUnknown:
-      return "unknown option";
+      return ps_error_string_for_option(
+          pool, directive, "not recognized or too many arguments");
    case RewriteOptions::kOptionValueInvalid: {
-      GoogleString full_directive = "\"";
+      GoogleString full_directive;
      for (int i = 0 ; i < n_args ; i++) {
        StrAppend(&full_directive, i == 0 ? "" : " ", args[i]);
      }
-      StrAppend(&full_directive, "\": ", msg);
-      char* s = string_piece_to_pool_string(pool, full_directive);
-      if (s == NULL) {
-        return "failed to allocate memory";
-      }
-      return s;
+      return ps_error_string_for_option(pool, full_directive, msg);
    }
  }

@@ -40,7 +40,8 @@ class NgxRewriteOptions : public SystemRewriteOptions {
  static void Initialize();
  static void Terminate();

-  NgxRewriteOptions(const StringPiece& description, ThreadSystem* thread_system);
+  NgxRewriteOptions(const StringPiece& description,
+                    ThreadSystem* thread_system);
  explicit NgxRewriteOptions(ThreadSystem* thread_system);
  virtual ~NgxRewriteOptions() { }

@@ -57,7 +58,7 @@ class NgxRewriteOptions : public SystemRewriteOptions {
  // pool is a memory pool for allocating error strings.
  const char* ParseAndSetOptions(
      StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
-      NgxRewriteDriverFactory* driver_factory);
+      NgxRewriteDriverFactory* driver_factory, OptionScope scope);

  // Make an identical copy of these options and return it.
  virtual NgxRewriteOptions* Clone() const;
@@ -67,6 +68,24 @@ class NgxRewriteOptions : public SystemRewriteOptions {
  static const NgxRewriteOptions* DynamicCast(const RewriteOptions* instance);
  static NgxRewriteOptions* DynamicCast(RewriteOptions* instance);

+  const GoogleString& statistics_path() const {
+    return statistics_path_.value();
+  }
+  const GoogleString& global_statistics_path() const {
+    return global_statistics_path_.value();
+  }
+  const GoogleString& console_path() const {
+    return console_path_.value();
+  }
+  const GoogleString& messages_path() const {
+    return messages_path_.value();
+  }
+  const GoogleString& admin_path() const {
+    return admin_path_.value();
+  }
+  const GoogleString& global_admin_path() const {
+    return global_admin_path_.value();
+  }

 private:
  // Helper methods for ParseAndSetOptions().  Each can:
@@ -108,14 +127,27 @@ class NgxRewriteOptions : public SystemRewriteOptions {
  static void add_ngx_option(typename OptionClass::ValueType default_value,
                             OptionClass NgxRewriteOptions::*offset,
                             const char* id,
-                             StringPiece option_name) {
-    AddProperty(default_value, offset, id, option_name, ngx_properties_);
+                             StringPiece option_name,
+                             OptionScope scope,
+                             const char* help) {
+    AddProperty(default_value, offset, id, option_name, scope, help,
+                ngx_properties_);
  }

+  Option<GoogleString> statistics_path_;
+  Option<GoogleString> global_statistics_path_;
+  Option<GoogleString> console_path_;
+  Option<GoogleString> messages_path_;
+  Option<GoogleString> admin_path_;
+  Option<GoogleString> global_admin_path_;
+
  // Helper for ParseAndSetOptions.  Returns whether the two directives equal,
  // ignoring case.
  bool IsDirective(StringPiece config_directive, StringPiece compare_directive);

+  // Returns a given option's scope.
+  RewriteOptions::OptionScope GetOptionScope(StringPiece option_name);
+
  // TODO(jefftk): support fetch proxy in server and location blocks.

  DISALLOW_COPY_AND_ASSIGN(NgxRewriteOptions);
@@ -72,6 +72,7 @@ SystemRequestContext* NgxServerContext::NewRequestContext(

  return new SystemRequestContext(thread_system()->NewMutex(),
                                  timer(),
+                                  ps_determine_host(r),
                                  local_port,
                                  str_to_string_piece(local_ip));
 }
@@ -221,7 +221,7 @@ namespace net_instaweb {
  void NgxUrlAsyncFetcher::Fetch(const GoogleString& url,
                                 MessageHandler* message_handler,
                                 AsyncFetch* async_fetch) {
-    async_fetch = EnableInflation(async_fetch, NULL);
+    async_fetch = EnableInflation(async_fetch);
    NgxFetch* fetch = new NgxFetch(url, async_fetch,
          message_handler, fetch_timeout_, log_);
    ScopedMutex lock(mutex_);
@@ -27,11 +27,30 @@ http {
  proxy_cache_path "@@PROXY_CACHE@@"  levels=1:2   keys_zone=htmlcache:60m inactive=90m  max_size=50m;
  proxy_temp_path "@@TMP_PROXY_CACHE@@";

+  pagespeed StatisticsPath /ngx_pagespeed_statistics;
+  pagespeed GlobalStatisticsPath /ngx_pagespeed_global_statistics;
+  pagespeed ConsolePath /pagespeed_console;
+  pagespeed MessagesPath /ngx_pagespeed_message;
+  pagespeed AdminPath /pagespeed_admin;
+  pagespeed GlobalAdminPath /pagespeed_global_admin;
+
+  pagespeed StaticAssetPrefix /pagespeed_custom_static/;
+
  root "@@SERVER_ROOT@@";
+
+  # Block 5a: Decide on Cache-Control header value to use for outgoing
+  # response.
+  # Map new_cache_control_header_val to "no-cache, max-age=0" if the
+  # content is html and use the original Cache-Control header value
+  # in all other cases.
+  map $upstream_http_content_type $new_cache_control_header_val {
+    default $upstream_http_cache_control;
+    "~*text/html" "no-cache, max-age=0";
+  }
+
  pagespeed UsePerVHostStatistics on;
  pagespeed InPlaceResourceOptimization on;
  pagespeed CreateSharedMemoryMetadataCache "@@SHM_CACHE@@" 8192;
-  pagespeed PreserveUrlRelativity on;
  pagespeed BlockingRewriteKey psatest;

  # CriticalImagesBeaconEnabled is now on by default, but we disable in testing.
@@ -41,6 +60,11 @@ http {
  # critical images to be inlined, so we just disable the option here.
  pagespeed CriticalImagesBeaconEnabled false;

+  # By default, resources will not be used for inlining without explicit
+  # authorization. Supported values are off or a comma-separated list of strings
+  # from {Script,Stylesheet}.
+  pagespeed InlineResourcesWithoutExplicitAuthorization off;
+
  pagespeed Statistics on;
  pagespeed StatisticsLogging on;
  pagespeed LogDir "@@TEST_TMP@@/logdir";
@@ -64,6 +88,7 @@ http {
  @@RESOLVER@@

  server {
+    # Block 1:  Basic port, server_name definitions.
    # This server represents the external caching layer server which
    # receives user requests and proxies them to the upstream server
    # running on the PRIMARY_PORT when the response is not available in
@@ -72,58 +97,157 @@ http {
    server_name proxy_cache.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";

+    # Disable PageSpeed on this server.
    pagespeed off;

-    set $ua_dependent_ps_capability_list "";
-    set $bypass_cache 1;
+    # Block 2: Define prefix for proxy_cache_key based on the UserAgent.
+
+    # Define placeholder PS-CapabilityList header values for large and small
+    # screens with no UA dependent optimizations. Note that these placeholder
+    # values should not contain any of ll, ii, dj, jw or ws, since these
+    # codes will end up representing optimizations to be supported for the
+    # request.
+    set $default_ps_capability_list_for_large_screens "LargeScreen.SkipUADependentOptimizations";
+    set $default_ps_capability_list_for_small_screens "TinyScreen.SkipUADependentOptimizations";
+
+    # As a fallback, the PS-CapabilityList header that is sent to the upstream
+    # PageSpeed server should be for a large screen device with no browser
+    # specific optimizations.
+    set $ps_capability_list $default_ps_capability_list_for_large_screens;
+
+    # Cache-fragment 1: Desktop User-Agents that support lazyload_images (ll),
+    # inline_images (ii) and defer_javascript (dj).
+    # Note: Wget is added for testing purposes only.
    if ($http_user_agent ~* "Chrome/|Firefox/|MSIE |Safari|Wget") {
-      # User Agents that support lazyload-images (ll), inline-images (ii) and
-      # defer-javascript (dj).
-      set $ua_dependent_ps_capability_list "ll,ii,dj:";
-      set $bypass_cache 0;
+      set $ps_capability_list "ll,ii,dj:";
    }
+    # Cache-fragment 2: Desktop User-Agents that support lazyload_images (ll),
+    # inline_images (ii), defer_javascript (dj), webp (jw) and lossless_webp
+    # (ws).
    if ($http_user_agent ~*
        "Chrome/[2][3-9]+\.|Chrome/[[3-9][0-9]+\.|Chrome/[0-9]{3,}\.") {
-      # User Agents that support lazyload-images (ll), inline-images (ii),
-      # defer-javascript (dj), webp (jw) and webp-lossless (ws).
-      set $ua_dependent_ps_capability_list "ll,ii,dj,jw,ws:";
-      set $bypass_cache 0;
+      set $ps_capability_list "ll,ii,dj,jw,ws:";
+    }
+    # Cache-fragment 3: This fragment contains (a) Desktop User-Agents that
+    # match fragments 1 or 2 but should not because they represent older
+    # versions of certain browsers or bots and (b) Tablet User-Agents that
+    # correspond to large screens. These will only get optimizations that work
+    # on all browsers and use image compression qualities applicable to large
+    # screens. Note that even Tablets that are capable of supporting inline or
+    # webp images, e.g. Android 4.1.2, will not get these advanced
+    # optimizations.
+    if ($http_user_agent ~* "Firefox/[1-2]\.|MSIE [5-8]\.|bot|Yahoo!|Ruby|RPT-HTTPClient|(Google \(\+https\:\/\/developers\.google\.com\/\+\/web\/snippet\/\))|Android|iPad|TouchPad|Silk-Accelerated|Kindle Fire") {
+      set $ps_capability_list $default_ps_capability_list_for_large_screens;
+    }
+    # Cache-fragment 4: Mobiles and small screen Tablets will use image compression
+    # qualities applicable to small screens, but all other optimizations will be
+    # those that work on all browsers.
+    if ($http_user_agent ~* "Mozilla.*Android.*Mobile*|iPhone|BlackBerry|Opera Mobi|Opera Mini|SymbianOS|UP.Browser|J-PHONE|Profile/MIDP|portalmmm|DoCoMo|Obigo|Galaxy Nexus|GT-I9300|GT-N7100|HTC One|Nexus [4|7|S]|Xoom|XT907") {
+      set $ps_capability_list $default_ps_capability_list_for_small_screens;
    }

-    # All User Agents that represent
-    # 1) mobiles
-    # 2) tablets
-    # 3) desktop browsers that do not have defer-javascript capability at a minimum
-    # are made to go to the pagespeed server directly bypassing the proxy_cache.
-    if ($http_user_agent ~* "Firefox/[1-2]\.|MSIE [5-8]\.") {
-      set $ua_dependent_ps_capability_list "";
-      set $bypass_cache 1;
+    # Block 3a: Bypass the cache for .pagespeed. resource. PageSpeed has its own
+    # cache for these, and these could bloat up the caching layer.
+    if ($uri ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+") {
+      set $bypass_cache "1";
    }
-    if ($http_user_agent ~* "Mozilla.*Android.*Mobile*|iPhone|BlackBerry|Opera Mobi|Opera Mini|SymbianOS|UP.Browser|J-PHONE|Profile/MIDP|portalmmm|DoCoMo|Obigo") {
-      # These are Mobile User Agents. We don't cache responses for these.
-      set $ua_dependent_ps_capability_list "";
-      set $bypass_cache 1;
+
+    # For testing purposes, we never generate values that will result in a beacon 
+    # unless a test request it via "X-Allow-Beacon: yes" in its request header.
+    # This is for testing purposes, note that in a production environment, 
+    # you want 'set_random $rand 0 100;' unconditionally!
+    set $rand 5;
+
+    if ($http_x_allow_beacon ~ "yes") {
+      set_random $rand 0 100;
    }
-    if ($http_user_agent ~* "Android|iPad|TouchPad|Silk-Accelerated|Kindle Fire") {
-      # These are Tablet User Agents. We don't cache responses for these.
-      set $ua_dependent_ps_capability_list "";
+    set $should_beacon_header_val "";
+    if ($rand ~* "^[0-4]$") {
+      set $should_beacon_header_val "random_rebeaconing_key";
      set $bypass_cache 1;
    }

+    # Block 3b: Only cache responses to clients that support gzip.  Most clients
+    # do, and the cache holds much more if it stores gzipped responses.
+    if ($http_accept_encoding !~* gzip) {
+      set $bypass_cache "1";
+    }
+
+    # Block 4: Location block for purge requests.
    location ~ /purge(/.*) {
-      allow all;
-      proxy_cache_purge htmlcache $ua_dependent_ps_capability_list$1$is_args$args;
+      allow 127.0.0.1;
+      deny all;
+      proxy_cache_purge htmlcache $ps_capability_list$1$is_args$args;
    }

+    # Block 6: Location block with proxy_cache directives.
    location /mod_pagespeed_test/cachable_rewritten_html/ {
+      # 1: Upstream PageSpeed server is running at localhost:8050.
      proxy_pass http://localhost:@@PRIMARY_PORT@@;
-      proxy_set_header Host $host;
-      proxy_cache_valid 200 30s;
+      # 2: Use htmlcache as the zone for caching.
      proxy_cache htmlcache;
-      proxy_ignore_headers Cache-Control;
-      add_header X-Cache $upstream_cache_status;
-      proxy_cache_key $ua_dependent_ps_capability_list$uri$is_args$args;
+      # 3: Bypass requests that correspond to .pagespeed. resources
+      # or clients that do not support gzip etc.
      proxy_cache_bypass $bypass_cache;
+      # 4: Use the redefined proxy_cache_key and make sure the /purge/
+      # block uses the same key.
+      proxy_cache_key $ps_capability_list$uri$is_args$args;
+      # 5: Forward Host header to upstream server.
+      proxy_set_header Host $host;
+      # 6: Set the PS-CapabilityList header for PageSpeed server to respect.
+      proxy_set_header PS-CapabilityList $ps_capability_list;
+      add_header PS-CapabilityList $ps_capability_list;
+      # 7: Add a header for identifying cache hits/misses/expires. This is
+      # for debugging purposes only and can be commented out in production.
+      add_header X-Cache $upstream_cache_status;
+
+      # Block 5b: Override Cache-Control headers as needed.
+      # Hide the upstream cache control header.
+      proxy_hide_header Cache-Control;
+      # Add the inferred Cache-Control header.
+      add_header Cache-Control $new_cache_control_header_val;
+
+      proxy_set_header PS-ShouldBeacon $should_beacon_header_val;
+      proxy_hide_header PS-ShouldBeacon;
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name if-in-server.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+
+    pagespeed RewriteLevel PassThrough;
+    set $inline_javascript "No";
+
+    if ($http_x_custom_header_inline_js) {
+      # TODO(jefftk): Turn on NGX_HTTP_SIF_CONF and figure out how to get
+      # pagespeed directives inside of a server location block to be respected,
+      # then uncomment the following line and duplicate the if-in-location test
+      # for if-in-server.
+      #pagespeed EnableFilters inline_javascript;
+      set $inline_javascript "Yes";
+    }
+
+    add_header "X-Inline-Javascript" $inline_javascript;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name if-in-location.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+
+
+    location / {
+      set $inline_javascript "No";
+      pagespeed RewriteLevel PassThrough;
+
+      if ($http_x_custom_header_inline_js) {
+        pagespeed EnableFilters inline_javascript;
+        set $inline_javascript "Yes";
+      }
+
+      add_header "X-Inline-Javascript" $inline_javascript;
    }
  }

@@ -149,6 +273,7 @@ http {
    pagespeed ExperimentSpec
      "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
    pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
+    pagespeed ExperimentSpec "id=3;default;percent=0";
  }
  server {
    listen @@SECONDARY_PORT@@;
@@ -160,6 +285,7 @@ http {
    pagespeed ExperimentSpec
      "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
    pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
+    pagespeed ExperimentSpec "id=3;default;percent=0";
  }

  server {
@@ -218,6 +344,15 @@ http {
    }
  }

+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name downstreamcacheresource.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_images;
+    pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
+  }
+
  server {
    listen @@SECONDARY_PORT@@;
    server_name renderedimagebeacon.example.com;
@@ -271,7 +406,7 @@ http {
    server_name customhostheader.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@_test";
    root "@@SERVER_ROOT@@/mod_pagespeed_test";
-    
+
    pagespeed on;
    pagespeed RewriteLevel PassThrough;
    pagespeed EnableFilters rewrite_images;
@@ -305,7 +440,7 @@ http {
    pagespeed FileCachePath "@@FILE_CACHE@@";

    pagespeed RewriteLevel PassThrough;
-    pagespeed InlineUnauthorizedResourcesExperimental true;
+    pagespeed InlineResourcesWithoutExplicitAuthorization Script,Stylesheet;
    pagespeed CssInlineMaxBytes 1000000;
  }

@@ -346,6 +481,9 @@ http {
    pagespeed UrlValuedAttribute img alt-src Image;
    pagespeed UrlValuedAttribute video alt-a Image;
    pagespeed UrlValuedAttribute video alt-b Image;
+
+    # Also test that we can redefine spec-defined attributes.
+    pagespeed UrlValuedAttribute blockquote cite Image;
  }

  server {
@@ -423,7 +561,7 @@ http {
  # Note that we test with two distinct caches.
  server {
    listen @@SECONDARY_PORT@@;
-    server_name embed-config-html.example.com;
+    server_name embed-config-html.example.org;
    pagespeed FileCachePath "@@FILE_CACHE@@";

    root "@@SERVER_ROOT@@/mod_pagespeed_test";
@@ -433,6 +571,9 @@ http {
    pagespeed DisableFilters inline_css,extend_cache,inline_javascript;
    pagespeed Domain embed-config-resources.example.com;

+    # Share a cache keyspace with embed-config-resources.example.com.
+    pagespeed CacheFragment "embed-config";
+
    pagespeed LoadFromFile "http://embed-config-resources.example.com/"
      "@@SERVER_ROOT@@/mod_pagespeed_example/";
  }
@@ -448,12 +589,26 @@ http {

    # Note that we do not set the jpeg quality here, but take
    # it from image URL query parameters that we synthesize in
-    # from embed-config-html.example.com.
+    # from embed-config-html.example.org.
+
+    # Share a cache keyspace with embed-config-html.example.org.
+    pagespeed CacheFragment "embed-config";

    pagespeed LoadFromFile "http://embed-config-resources.example.com/"
      "@@SERVER_ROOT@@/mod_pagespeed_example/";
  }

+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name ipro-for-browser.example.com;
+    root "@@SERVER_ROOT@@/mod_pagespeed_example";
+    pagespeed EnableFilters rewrite_images,rewrite_css;
+    pagespeed EnableFilters convert_to_webp_lossless;
+    pagespeed EnableFilters in_place_optimize_for_browser;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed FileCachePath "@@IPRO_CACHE@@";
+  }
+
  server {
    listen @@SECONDARY_PORT@@;
    server_name respectvary.example.com;
@@ -505,6 +660,7 @@ http {
    pagespeed MapRewriteDomain cdn.example.com origin.example.com;
    pagespeed RewriteLevel PassThrough;
    pagespeed EnableFilters rewrite_css,rewrite_images;
+    pagespeed CacheFragment "example";
  }
  server {
    # Sets up a logical origin for CDNs to fetch content from, on
@@ -517,6 +673,7 @@ http {
    pagespeed MapRewriteDomain cdn.example.com origin.example.com;
    pagespeed RewriteLevel PassThrough;
    pagespeed EnableFilters rewrite_css,rewrite_images;
+    pagespeed CacheFragment "example";
  }
  server {
    # Sets up a logical cdn, which is where we tell browsers to fetch resources
@@ -556,6 +713,23 @@ http {
    }
  }

+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name custom-paths.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed StatisticsPath /custom_pagespeed_statistics;
+    pagespeed ConsolePath /custom_pagespeed_console;
+    pagespeed MessagesPath /custom_pagespeed_message;
+    pagespeed AdminPath /custom_pagespeed_admin;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name inherit-paths.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+
  server {
    listen @@SECONDARY_PORT@@;
    server_name notransform.example.com;
@@ -641,6 +815,11 @@ http {
    location /mod_pagespeed_test/ipro/nocache/test_image_dont_reuse.png {
      add_header Cache-Control no-cache;
    }
+
+    location /mod_pagespeed_test/ipro/no-cache-control-header {
+      add_header Cache-Control "";
+      pagespeed ImplicitCacheTtlMs 333000;
+    }
  }

  # Test hosts to cover all possible cache configurations.  L1 will be filecache
@@ -702,25 +881,61 @@ http {
    add_header "Date" "Date: Fri, 16 Oct 2009 23:05:07 GMT";
  }

+  # Proxy + IPRO a gzip'd file for testing Issue 896.
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name ipro-proxy.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@_ipro_proxy";
+
+    pagespeed on;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed EnableFilters rewrite_domains;
+
+    location / {
+      proxy_pass
+       "http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/mod_deflate/";
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name compressed-css.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed InPlaceResourceOptimization on;
+  }
+
  server {
    listen       @@PRIMARY_PORT@@;
    server_name  localhost;
    pagespeed FileCachePath "@@FILE_CACHE@@";

+    location ~ ^/pagespeed_admin {
+      allow 127.0.0.1;
+      deny all;
+    }
+
    location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" {
      add_header "" "";
    }

+    # Backend for ipro-proxy.example.com
+    location /mod_pagespeed_test/ipro/mod_deflate/ {
+      pagespeed off;
+    }
+
    location /mod_pagespeed_test/cachable_rewritten_html/ {
      # This location has the html files that will be configured to be stored
      # in the proxy_cache layer.
      pagespeed DownstreamCachePurgeMethod "GET";
      pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
+      pagespeed DownstreamCacheRebeaconingKey "random_rebeaconing_key";
      # We use a very small deadline here to force the rewriting to not complete
      # in the very first attempt.
      pagespeed RewriteDeadlinePerFlushMs 1;
      pagespeed RewriteLevel PassThrough;
-      pagespeed EnableFilters collapse_whitespace,extend_cache,recompress_images,convert_jpeg_to_webp,defer_javascript;
+      pagespeed EnableFilters collapse_whitespace,extend_cache,recompress_images;
+      pagespeed CriticalImagesBeaconEnabled true;
+      add_header Cache-Control "public, max-age=100";
    }

    location /mod_pagespeed_test/disable_no_transform/index.html {
@@ -865,6 +1080,79 @@ http {
      expires 5m;
    }

+    location /mod_pagespeed_test/ipro/instant/wait/ {
+      pagespeed InPlaceWaitForOptimized on;
+
+      # TODO(jefftk): When running with valgrind we need to raise the rewrite
+      # deadline or else we get "Deadline exceeded for rewrite of resource".
+      # I had thought InPlaceWaitForOptimized was supposed to disable the
+      # rewrite deadline, but it seems not to.
+      #
+      # InPlaceWaitForOptimized is definitely doing something, though: when I
+      # remove the InPlaceRewriteDeadlineMs line below and run without valgrind,
+      # purple.css is optimized via ipro on the first request but if I set
+      # InPlaceWaitForOptimized to off then it is not.
+      pagespeed InPlaceRewriteDeadlineMs 1000;
+    }
+
+    location /mod_pagespeed_test/ipro/instant/deadline/ {
+      pagespeed InPlaceRewriteDeadlineMs -1;
+    }
+
+    # Test to make sure that user-authenticated resources do not get cached and
+    # optimized.
+    location /mod_pagespeed_test/auth/ {
+       auth_basic "Restricted";
+       auth_basic_user_file "@@SERVER_ROOT@@mod_pagespeed_test/auth/passwd.conf";
+    }
+
+    location /mod_pagespeed_test/ipro/cookie/ {
+      # Add Vary:Cookie.  This should prevent us from optimizing the
+      # vary_cookie.css even though PagespeedRespectVary is off.
+      # test/nginx_system_test.sh does the fetches test with and w/o cookies.
+      add_header Vary Cookie;
+      pagespeed RespectVary off;
+      pagespeed InPlaceWaitForOptimized on;
+    }
+
+    location /mod_pagespeed_test/ipro/cookie2/ {
+      # Add Vary:Cookie2.  This should prevent us from optimizing the
+      # vary_cookie2.css even though PagespeedRespectVary is off.
+      # test/nginx_system_test.sh does the fetches test with and w/o cookie2.
+      add_header Vary Cookie2;
+      pagespeed RespectVary off;
+      pagespeed InPlaceWaitForOptimized on;
+    }
+
+    location /mod_pagespeed_test/vary/ {
+      pagespeed RespectVary on;
+    }
+
+    location ~ /mod_pagespeed_test/vary/\.(js|css)$ {
+        add_header Vary User-Agent;
+    }
+
+    location /mod_pagespeed_test/vary/no_respect/ {
+      pagespeed DisableFilters add_instrumentation,inline_css;
+      pagespeed RespectVary off;
+    }
+
+    location /mod_pagespeed_test/experimental_js_minifier/ {
+      pagespeed UseExperimentalJsMinifier on;
+    }
+
+    pagespeed FetchHttps enable;
+    location /mod_pagespeed_test/https_fetch/ {
+      pagespeed DisableFilters inline_images;
+      pagespeed MapProxyDomain
+          http://localhost:@@PRIMARY_PORT@@/https_gstatic_dot_com
+          https://www.gstatic.com/psa/static;
+    }
+
+    pagespeed LoadFromFile
+        "http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/instant/"
+        "@@SERVER_ROOT@@/mod_pagespeed_test/ipro/instant/";
+
    pagespeed EnableFilters remove_comments;

    # Test LoadFromFile mapping by mapping one dir to another.
@@ -903,6 +1191,7 @@ http {
  # set up gzip
  gzip  on;
  gzip_vary on;
+  gzip_proxied any;
  # Turn on gzip for all content types that should benefit from it.
  gzip_types application/ecmascript;
  gzip_types application/javascript;
Author	SHA1	Message	Date
Jan-Willem Maessen	4d940fb84e	Update for 1.8.31.4	2014-06-17 13:32:51 -04:00
Jan-Willem Maessen	a6a2da765b	Merge remote-tracking branch 'origin/release-1.8.31.3-beta' Pulling in upstream merge to local master.	2014-05-29 15:06:03 -04:00
Jan-Willem Maessen	70038bc64a	Merge pull request #714 from pagespeed/jmaessen-prepare-release-1.8.31.3-beta Jmaessen prepare release 1.8.31.3 beta	2014-05-29 15:01:45 -04:00
Jan-Willem Maessen	815fa29985	Clarify test name and test comment.	2014-05-29 14:51:36 -04:00
Jan-Willem Maessen	c82bb2c7a4	Version 1.8.31.2 -> 1.8.31.3	2014-05-29 14:32:18 -04:00
Jan-Willem Maessen	e9eaa23356	Test fixups because fallback css now always uses chunked encoding in 1.8.31.3	2014-05-29 14:30:03 -04:00
Jeff Kaufman	168df6ae68	readme: with 1.8 we need different default config	2014-05-28 16:34:12 -04:00
Jan-Willem Maessen	a7969a6382	Merge pull request #710 from pagespeed/jefftk-unbackport-roo-test fixup: unbackport the RequestOptionOverride tests from `8246c03f`	2014-05-28 11:17:06 -04:00
Jeff Kaufman	243d4e1931	fixup: unbackport the RequestOptionOverride tests from `8246c03f`	2014-05-28 11:14:14 -04:00
Jeff Kaufman	8246c03fda	options: support StaticAssetPrefix Backporting this to master for 1.8.31.3. Fixes #705.	2014-05-27 11:47:36 -04:00
Jeff Kaufman	99ac026f00	readme: 1.4.6 -> 1.6.0	2014-05-12 10:02:19 -04:00
Jeff Kaufman	90e4c40d86	Allow admin paths to be set at server scope.	2014-05-12 09:35:54 -04:00
Jan-Willem Maessen	0de4e20be1	release: version 1.7.30.4 -> 1.8.31.2	2014-05-09 11:28:04 -04:00
Jeff Kaufman	185632583b	Merge pull request #675 from pagespeed/jefftk-update-trunk-tracking trunk tracking: update from r3934 to r3949	2014-04-23 09:15:45 -04:00
Jeff Kaufman	c5dbfa16cd	trunk-tracking: fix comment	2014-04-23 09:14:58 -04:00
Jeff Kaufman	33b2e82ec6	trunk-tracking: port tests for r3948, PageSpeed=... not ModPagespeed=...	2014-04-22 17:56:09 -04:00
Jeff Kaufman	b87ffce5ba	trunk-tracking: port r3945 which made the location of static assets configurable	2014-04-22 17:55:57 -04:00
Jeff Kaufman	7d5fc11afd	trunk-tracking: port tests for r3937, url-valued attributes overriding spec	2014-04-22 17:38:51 -04:00
Jeff Kaufman	7cba7ba4f4	trunk-tracking: port tests for r3935 which changes image lazyloading	2014-04-22 17:38:51 -04:00
Jeff Kaufman	ba8c149a88	Merge pull request #670 from pagespeed/oschaaf-downstream-cache-headers downstream-caching-headers: make nginx 1.5.9+ pass tests as well	2014-04-21 17:07:23 -04:00
Jeff Kaufman	f5bac604a1	Merge pull request #671 from pagespeed/oschaaf-downstream-cache-test downstream-cache-test: deflake the test	2014-04-21 10:27:53 -04:00
Otto van der Schaaf	f18faa3ae1	downstream-cache-test: deflake the test As suggested by Anupama, deflake the test by only allowing beaconing when explitly requested by http request from tests. Html with beacons in it will bypass the proxy cache.	2014-04-20 00:24:32 +02:00
Otto van der Schaaf	070ae2aad2	downstream-caching-headers: make nginx 1.5.9+ pass tests as well - Fix a mistake I made in https://github.com/pagespeed/ngx_pagespeed/pull/653 This is the primary reason the test starts failing, the base fetch handled `kPreserveOnlyCacheControl` in the wrong way. - Use `ps_set_cache_control` when copying a Cache-Control header to nginx, as `r->headers_out.cache_control` is where modules running after us might expect it - Exclude a few more lines from curl's verbose output in the keep- alive tests that I came accross when testing on ubuntu 13.10 Desktop. Note that this does not deflake the downstream cache test yet, I will look at that in a follow-up.	2014-04-19 22:58:09 +02:00
Jeff Kaufman	674c70e32c	Merge pull request #669 from pagespeed/morlovich-trunk-tracking-ssl-tests Port over the basic HTTPS fetching test.	2014-04-17 16:36:01 -04:00
Maks Orlovich	b4ef143a88	Port over the basic HTTPS fetching test.	2014-04-17 16:20:10 -04:00
Jeff Kaufman	ffdc3c8fd6	Merge pull request #667 from pagespeed/jefftk-merge-master-to-tt trunk-tracking: merge in master	2014-04-17 15:21:19 -04:00
Jeff Kaufman	af10fc3ddb	trunk-tracking: merge in master	2014-04-17 15:16:43 -04:00
Jeff Kaufman	e9a099f9c5	Merge pull request #666 from pagespeed/jefftk-update-trunk-tracking trunk-tracking: update from r3924 to r3934	2014-04-17 10:43:46 -04:00
Jeff Kaufman	e4a2933c58	trunk-tracking: update from r3924 to r3934 There aren't actually any changes required from r3924 to r3934 but the admin work from previous versions wasn't completely ported before. This change: * adds support for /pagespeed_admin and /pagespeed_global_admin * removes NgxPagespeedConsoleAsyncFetch using NgxBaseFetch instead * adds options StatisticsPath, GlobalStatisticsPath, ConsolePath, MessagesPath, AdminPath, and GlobalAdminPath which allow site owners to control what path these are served under * changes /ngx_pagespeed_statistics etc from default-on to default-off, requiring explicit path configuration to function * does not fix the problem where trunk-tracking fails tests on 1.5 post #653	2014-04-17 10:10:20 -04:00
Jeff Kaufman	2601232457	Merge pull request #655 from pagespeed/oschaaf-native-fetcher-resolve-timeout native-fetcher: fix CHECK failure on DNS timeout	2014-04-14 07:28:42 -04:00
Otto van der Schaaf	a3e1810a6a	Merge pull request #653 from pagespeed/oschaaf-downstream-caching-headers downstream-caching: Fix pagespeed resource response headers	2014-04-10 21:56:01 +02:00
Otto van der Schaaf	72f3c79fec	downstream-caching: Fix pagespeed resource response headers - Make kPreserveOnlyCacheControl do what it says on the tin - Serve pagespeed resources with the response headers computed by PSOL regardless of whether downstream caching is configured. Fixes https://github.com/pagespeed/ngx_pagespeed/issues/652	2014-04-10 21:42:48 +02:00
Otto van der Schaaf	323e820fde	Merge pull request #658 from pagespeed/oschaaf-configure-wnoerror nginx-gridfs: Add configure option to build with wno-error	2014-04-09 20:38:57 +02:00
Otto van der Schaaf	a9a7bf56ac	Merge pull request #659 from pagespeed/jud-trunk-tracking trunk-tracking: update from r3895 to r3924	2014-04-08 21:58:16 +02:00
Jud Porter	2358f957cb	trunk-tracking: update from r3895 to r3924	2014-04-08 14:51:04 -04:00
Otto van der Schaaf	7d72a7c89a	nginx-gridfs: Add configure option to build with wno-error Some modules add things to CFLAGS that will make ngx_pagespeed emit warnings at compile time. For example, nginx-gridfs will add `--std=c99` - which is no good for ngx_pagespeed. @peterbowey mentioned that -wno-error fixes the build -- so to work around, make configure add `-wno-error` when used like this: `WNO_ERROR=YES ./configure` On my system, that results in a succesfull build when nginx-gridfs is added to the module mix. Fixes https://github.com/pagespeed/ngx_pagespeed/issues/626	2014-04-08 13:10:58 +02:00
Otto van der Schaaf	8830724580	native-fetcher: fix CHECK failure on DNS timeout Make sure we clear the timeout event associated with resolving a hostname in `CallbackDone()`, which would otherwise end up calling `CallbackDone()` a second time upon firing. Fixes https://github.com/pagespeed/ngx_pagespeed/issues/625	2014-04-06 09:17:45 +02:00
Otto van der Schaaf	c2d07a8d5a	Merge pull request #651 from pagespeed/morlovich-trunk-tracking-upto-r3895 trunk tracking upto r3895	2014-04-02 18:53:04 +02:00
Maks Orlovich	c3bff67637	Fix improper cache key on write for IPRO of resources with PageSpeed params We need to be writing with the base name, with params stripped. This was causing a test to "flake" under valgrind, since with it we got to the test showing the bug slowly enough for the entry written out by an earlier test under proper URL to expire.	2014-04-02 18:41:08 +02:00
Maks Orlovich	5a1a29ee8b	trunk-tracking: update from r3872 to r3895 Squash-merge of @morlovich's work on this. Note that this does not yet incorprorate the new pagespeed_admin handler from r3891	2014-04-02 18:39:08 +02:00
Jeff Kaufman	61025c5e4f	if: support pagespeed directives in location if blocks	2014-03-31 10:35:05 -04:00
Otto van der Schaaf	d3e7900704	Merge pull request #648 from pagespeed/jmaessen-trunk-tracking trunk-tracking: update from r3852 to r3872 note that it will only compile against mod_pagespeed r3872 if you first patch in r3889	2014-03-28 22:37:31 +01:00
Jan-Willem Maessen	85e19c962b	trunk-tracking: update from r3852 to r3872	2014-03-27 10:59:13 -04:00
Otto van der Schaaf	20fd7d18dd	Merge pull request #646 from pagespeed/oschaaf-chunked-transfer content-length: fix pagespeed/ipro resource content-lengths	2014-03-26 08:53:50 +01:00
Otto van der Schaaf	4667aa1fc3	content-length: fix pagespeed/ipro resource content-lengths Makes life downstream easier and more efficient	2014-03-25 21:29:28 +01:00
Jeff Kaufman	6ced8c0f65	Merge pull request #641 from pagespeed/jefftk-if-block if: support pagespeed directives in location if blocks	2014-03-25 13:16:27 -04:00
Jeff Kaufman	bfaf4268cf	Merge pull request #639 from pagespeed/jefftk-trunk-tracking trunk-tracking: update from r3827 to r3852	2014-03-19 16:57:28 -04:00
Jeff Kaufman	78cf39f9b3	if: support pagespeed directives in location if blocks	2014-03-19 16:46:48 -04:00
Jeff Kaufman	215071a383	trunk-tracking: update from r3827 to r3852 * port EnrollExperiment tests. * tricky changes to get gzip+ipro working properly. * reduce code duplication by calling copy_response_headers_from_ngx instead of reimplementing it. * port Vary: Accept test changes.	2014-03-18 16:58:50 -04:00
Jeff Kaufman	630f7d848b	lint: fix lint errors	2014-03-18 10:38:58 -04:00
Huibao Lin	d90245af69	trunk-tracking: merge changes in from master: * release: prepare 1.7.30.3 * Don't call chown() when initializing config dirs unless owner != worker user. * Update README.md * release: build against backported change * release: udpate version number 1.7.30.3 to 1.7.30.4	2014-03-18 10:38:28 -04:00
Jeff Kaufman	f25569690a	merge commit: merge 1.7.30.4 into master	2014-03-14 10:15:42 -04:00
huibaolin	707d671826	Change version 1.7.30.3 to 1.7.30.4	2014-03-14 08:16:38 -04:00
Jeff Kaufman	f8b87ea436	Merge pull request #636 from pagespeed/huibao-prepare-1.7.30.4-release release: update version number 1.7.30.3 to 1.7.30.4	2014-03-13 10:37:21 -04:00
Huibao Lin	86d840f76e	release: udpate version number 1.7.30.3 to 1.7.30.4	2014-03-13 10:27:21 -04:00
Jeff Kaufman	0110d33fa7	Merge pull request #635 from pagespeed/huibao-prepare-1.7.30.4-release release: build against backported change	2014-03-13 10:26:16 -04:00
Huibao Lin	497594ba7f	release: build against backported change	2014-03-13 10:18:57 -04:00
Jeff Kaufman	336352df38	Merge pull request #633 from pnommensen/patch-2 readme: use nginx 1.4.6	2014-03-07 13:17:42 -05:00
Patrick Nommensen	0edd405eb8	Update README.md Don't know how I missed that.	2014-03-07 10:16:43 -08:00
Jeff Kaufman	d004c4d916	Merge pull request #632 from pnommensen/patch-2 1.4.5 to 1.4.6 update	2014-03-07 11:32:01 -05:00
Jeff Kaufman	c754cf1be0	trunk-tracking: update from r3811 to r3927	2014-03-07 11:18:35 -05:00
Patrick Nommensen	091ef6399b	version update 1.4.5 to 1.4.6 http://nginx.org/en/CHANGES-1.4	2014-03-06 23:25:13 -08:00
Jeff Kaufman	9e4d26da5e	Merge pull request #629 from pagespeed/jmaessen-trunk-tracking trunk-tracking: update from r3784 to r3811	2014-03-03 16:56:15 -05:00
Jan-Willem Maessen	027d481817	trunk-tracking: update from r3784 to r3811 The big change here is Vary: headers, which are no longer being stripped from the pagespeed response. Take a look; it might be we want to be smart about merging these headers with the original request's Vary: headers.	2014-02-27 19:41:23 -05:00
Jeff Kaufman	67106e920e	apr: call SystemRewriteDriverFactory::InitApr() squash-merge of morlovich's #601.	2014-02-20 09:13:56 -05:00
Jeff Kaufman	9699caeab5	Merge pull request #623 from pnommensen/patch-1 Update README.md	2014-02-20 09:12:04 -05:00
Patrick Nommensen	c371d516a8	Update README.md nginx version update	2014-02-20 00:04:21 -08:00
Jeff Kaufman	207eb2d131	Merge pull request #622 from pagespeed/jefftk-trunk-tracking trunk-tracking: update from r3770 to r3784	2014-02-19 12:58:40 -05:00
Jeff Kaufman	d9e2142f31	trunk-tracking: update from r3770 to r3784 * r3772 changed the format of unauthorized resource inlining config * r3776 changed query param handling, but due to Jud's nice work in #616 needed no changes. * r3780 huibao's refactoring lets us remove some duplicate code * r3783 changes header passing with ipro * Also cleaned up error messages for unknown options.	2014-02-19 10:53:48 -05:00
Jeff Kaufman	bf6c6c0e9b	Merge pull request #621 from jart/dont-chown Security Fix: Don't call chown() unless necessary.	2014-02-18 14:32:32 -05:00
Justine Tunney	e8dd9fd3c3	Don't call chown() when initializing config dirs unless owner != worker user.	2014-02-15 22:32:55 -05:00
Jud Porter	33d006625e	Merge pull request #616 from pagespeed/jud-trunk-tracking trunk-tracking: update from r3736 to r3770	2014-02-11 16:43:33 -05:00
Anupama Dutta	2dcaa95278	trunk-tracking: update from r3736 to r3770 Updated global_only options to include the correct APACHE_CONFIG_OPTIONX directives. Removed repeated tests for prioritize_critical_css basic functionality. Added new tests, mostly downstream caching tests and related pagespeed.conf updates. Also added missing pagespeed.conf updates for downstream caching. Added support for checking option scope.	2014-02-11 16:40:07 -05:00
Jeff Kaufman	64eaa2a659	Merge pull request #602 from tcpper/fix_ngx_fetch_content_length fix bug in NgxFetch#content_length_	2014-01-30 11:25:52 -08:00
huibaolin	6db4d02a91	Merge pull request #600 from pagespeed/huibao-prepare-release-1.7.30.3-beta release: prepare 1.7.30.3	2014-01-16 12:12:53 -08:00
Huibao Lin	1354cee4ed	release: prepare 1.7.30.3	2014-01-16 15:06:49 -05:00
tcpper	6ccb815df3	fix bug in NgxFetch#content_length_	2014-01-16 21:06:12 +08:00
Jeff Kaufman	72ddb34a1c	readme: release 1.7.30.2	2014-01-06 16:54:56 -05:00
Jeff Kaufman	ae2d4bac7f	native-fetcher: fix to work with nginx 1.5.8+ nginx 1.5.8 changed the resolver api, which the native fetcher uses. Fixes #578. Squash-merge of @dinic's #581.	2014-01-06 16:46:48 -05:00